mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-16 18:31:27 +00:00
d834ec895a
### Description `lintrunner` is a linter runner successfully used by pytorch, onnx and onnx-script. It provides a uniform experience running linters locally and in CI. It supports all major dev systems: Windows, Linux and MacOs. The checks are enforced by the `Python format` workflow. This PR adopts `lintrunner` to onnxruntime and fixed ~2000 flake8 errors in Python code. `lintrunner` now runs all required python lints including `ruff`(replacing `flake8`), `black` and `isort`. Future lints like `clang-format` can be added. Most errors are auto-fixed by `ruff` and the fixes should be considered robust. Lints that are more complicated to fix are applied `# noqa` for now and should be fixed in follow up PRs. ### Notable changes 1. This PR **removed some suboptimal patterns**: - `not xxx in` -> `xxx not in` membership checks - bare excepts (`except:` -> `except Exception`) - unused imports The follow up PR will remove: - `import *` - mutable values as default in function definitions (`def func(a=[])`) - more unused imports - unused local variables 2. Use `ruff` to replace `flake8`. `ruff` is much (40x) faster than flake8 and is more robust. We are using it successfully in onnx and onnx-script. It also supports auto-fixing many flake8 errors. 3. Removed the legacy flake8 ci flow and updated docs. 4. The added workflow supports SARIF code scanning reports on github, example snapshot:  5. Removed `onnxruntime-python-checks-ci-pipeline` as redundant ### Motivation and Context <!-- - Why is this change required? What problem does it solve? - If it fixes an open issue, please link to the issue here. --> Unified linting experience in CI and local. Replacing https://github.com/microsoft/onnxruntime/pull/14306 --------- Signed-off-by: Justin Chu <justinchu@microsoft.com>
75 lines
2.5 KiB
Python
Executable file
75 lines
2.5 KiB
Python
Executable file
#!/usr/bin/env python3
|
|
# Copyright (c) Microsoft Corporation. All rights reserved.
|
|
# Licensed under the MIT License.
|
|
|
|
import argparse
|
|
import hashlib
|
|
import os
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
import tempfile
|
|
import urllib.request # noqa: F401
|
|
import zipfile # noqa: F401
|
|
|
|
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
|
|
REPO_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, "..", "..", ".."))
|
|
|
|
sys.path.append(os.path.join(REPO_DIR, "tools", "python"))
|
|
|
|
from util import get_azcopy # noqa: E402
|
|
|
|
|
|
def _download(azcopy_path, url, local_path):
|
|
subprocess.run([azcopy_path, "cp", "--log-level", "NONE", url, local_path], check=True)
|
|
|
|
|
|
def _get_sha256_digest(file_path):
|
|
alg = hashlib.sha256()
|
|
read_bytes_length = 8192
|
|
|
|
with open(file_path, mode="rb") as archive:
|
|
while True:
|
|
read_bytes = archive.read(read_bytes_length)
|
|
if len(read_bytes) == 0:
|
|
break
|
|
alg.update(read_bytes)
|
|
|
|
return alg.hexdigest()
|
|
|
|
|
|
def _check_file_sha256_digest(path, expected_digest):
|
|
actual_digest = _get_sha256_digest(path)
|
|
match = actual_digest.lower() == expected_digest.lower()
|
|
if not match:
|
|
raise RuntimeError(
|
|
f"SHA256 digest mismatch, expected: {expected_digest.lower()}, actual: {actual_digest.lower()}"
|
|
)
|
|
|
|
|
|
def main():
|
|
parser = argparse.ArgumentParser(description="Downloads an Azure blob archive.")
|
|
parser.add_argument("--azure_blob_url", required=True, help="The Azure blob URL.")
|
|
parser.add_argument("--target_dir", required=True, help="The destination directory.")
|
|
parser.add_argument("--archive_sha256_digest", help="The SHA256 digest of the archive. Verified if provided.")
|
|
args = parser.parse_args()
|
|
|
|
with tempfile.TemporaryDirectory() as temp_dir, get_azcopy() as azcopy_path:
|
|
archive_path = os.path.join(temp_dir, "archive.zip")
|
|
print(f"Downloading archive from '{args.azure_blob_url}'...")
|
|
|
|
azure_blob_url = args.azure_blob_url
|
|
azure_blob_sas_token = os.getenv("AZURE_BLOB_SAS_TOKEN", None)
|
|
if azure_blob_sas_token and azure_blob_sas_token != "":
|
|
azure_blob_url = azure_blob_url + "?" + azure_blob_sas_token
|
|
|
|
_download(azcopy_path, azure_blob_url, archive_path)
|
|
if args.archive_sha256_digest:
|
|
_check_file_sha256_digest(archive_path, args.archive_sha256_digest)
|
|
print(f"Extracting to '{args.target_dir}'...")
|
|
shutil.unpack_archive(archive_path, args.target_dir)
|
|
print("Done.")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|