mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-05-16 21:00:14 +00:00
d834ec895a
### Description `lintrunner` is a linter runner successfully used by pytorch, onnx and onnx-script. It provides a uniform experience running linters locally and in CI. It supports all major dev systems: Windows, Linux and MacOs. The checks are enforced by the `Python format` workflow. This PR adopts `lintrunner` to onnxruntime and fixed ~2000 flake8 errors in Python code. `lintrunner` now runs all required python lints including `ruff`(replacing `flake8`), `black` and `isort`. Future lints like `clang-format` can be added. Most errors are auto-fixed by `ruff` and the fixes should be considered robust. Lints that are more complicated to fix are applied `# noqa` for now and should be fixed in follow up PRs. ### Notable changes 1. This PR **removed some suboptimal patterns**: - `not xxx in` -> `xxx not in` membership checks - bare excepts (`except:` -> `except Exception`) - unused imports The follow up PR will remove: - `import *` - mutable values as default in function definitions (`def func(a=[])`) - more unused imports - unused local variables 2. Use `ruff` to replace `flake8`. `ruff` is much (40x) faster than flake8 and is more robust. We are using it successfully in onnx and onnx-script. It also supports auto-fixing many flake8 errors. 3. Removed the legacy flake8 ci flow and updated docs. 4. The added workflow supports SARIF code scanning reports on github, example snapshot:  5. Removed `onnxruntime-python-checks-ci-pipeline` as redundant ### Motivation and Context <!-- - Why is this change required? What problem does it solve? - If it fixes an open issue, please link to the issue here. --> Unified linting experience in CI and local. Replacing https://github.com/microsoft/onnxruntime/pull/14306 --------- Signed-off-by: Justin Chu <justinchu@microsoft.com>
62 lines
2.5 KiB
Python
62 lines
2.5 KiB
Python
# Copyright (c) Microsoft Corporation. All rights reserved.
|
|
# Licensed under the MIT License.
|
|
|
|
import pathlib
|
|
import typing
|
|
|
|
from ..logger import get_logger
|
|
from .operator_type_usage_processors import OperatorTypeUsageManager
|
|
from .ort_model_processor import OrtFormatModelProcessor
|
|
|
|
log = get_logger("ort_format_model.utils")
|
|
|
|
|
|
def _extract_ops_and_types_from_ort_models(model_files: typing.Iterable[pathlib.Path], enable_type_reduction: bool):
|
|
required_ops = {}
|
|
op_type_usage_manager = OperatorTypeUsageManager() if enable_type_reduction else None
|
|
|
|
for model_file in model_files:
|
|
if not model_file.is_file():
|
|
raise ValueError(f"Path is not a file: '{model_file}'")
|
|
model_processor = OrtFormatModelProcessor(str(model_file), required_ops, op_type_usage_manager)
|
|
model_processor.process() # this updates required_ops and op_type_processors
|
|
|
|
return required_ops, op_type_usage_manager
|
|
|
|
|
|
def create_config_from_models(
|
|
model_files: typing.Iterable[pathlib.Path], output_file: pathlib.Path, enable_type_reduction: bool
|
|
):
|
|
"""
|
|
Create a configuration file with required operators and optionally required types.
|
|
:param model_files: Model files to use to generate the configuration file.
|
|
:param output_file: File to write configuration to.
|
|
:param enable_type_reduction: Include required type information for individual operators in the configuration.
|
|
"""
|
|
|
|
required_ops, op_type_processors = _extract_ops_and_types_from_ort_models(model_files, enable_type_reduction)
|
|
|
|
output_file.parent.mkdir(parents=True, exist_ok=True)
|
|
|
|
with open(output_file, "w") as out:
|
|
out.write("# Generated from model/s:\n")
|
|
for model_file in sorted(model_files):
|
|
out.write(f"# - {model_file}\n")
|
|
|
|
for domain in sorted(required_ops.keys()):
|
|
for opset in sorted(required_ops[domain].keys()):
|
|
ops = required_ops[domain][opset]
|
|
if ops:
|
|
out.write(f"{domain};{opset};")
|
|
if enable_type_reduction:
|
|
# type string is empty if op hasn't been seen
|
|
entries = [
|
|
"{}{}".format(op, op_type_processors.get_config_entry(domain, op) or "")
|
|
for op in sorted(ops)
|
|
]
|
|
else:
|
|
entries = sorted(ops)
|
|
|
|
out.write("{}\n".format(",".join(entries)))
|
|
|
|
log.info("Created config in %s", output_file)
|