mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-05-14 20:48:00 +00:00
d834ec895a
### Description `lintrunner` is a linter runner successfully used by pytorch, onnx and onnx-script. It provides a uniform experience running linters locally and in CI. It supports all major dev systems: Windows, Linux and MacOs. The checks are enforced by the `Python format` workflow. This PR adopts `lintrunner` to onnxruntime and fixed ~2000 flake8 errors in Python code. `lintrunner` now runs all required python lints including `ruff`(replacing `flake8`), `black` and `isort`. Future lints like `clang-format` can be added. Most errors are auto-fixed by `ruff` and the fixes should be considered robust. Lints that are more complicated to fix are applied `# noqa` for now and should be fixed in follow up PRs. ### Notable changes 1. This PR **removed some suboptimal patterns**: - `not xxx in` -> `xxx not in` membership checks - bare excepts (`except:` -> `except Exception`) - unused imports The follow up PR will remove: - `import *` - mutable values as default in function definitions (`def func(a=[])`) - more unused imports - unused local variables 2. Use `ruff` to replace `flake8`. `ruff` is much (40x) faster than flake8 and is more robust. We are using it successfully in onnx and onnx-script. It also supports auto-fixing many flake8 errors. 3. Removed the legacy flake8 ci flow and updated docs. 4. The added workflow supports SARIF code scanning reports on github, example snapshot:  5. Removed `onnxruntime-python-checks-ci-pipeline` as redundant ### Motivation and Context <!-- - Why is this change required? What problem does it solve? - If it fixes an open issue, please link to the issue here. --> Unified linting experience in CI and local. Replacing https://github.com/microsoft/onnxruntime/pull/14306 --------- Signed-off-by: Justin Chu <justinchu@microsoft.com>
80 lines
2.7 KiB
Python
80 lines
2.7 KiB
Python
# Copyright (c) Microsoft Corporation. All rights reserved.
|
|
# Licensed under the MIT License.
|
|
|
|
import contextlib
|
|
import logging
|
|
import os
|
|
import platform
|
|
import re
|
|
import shutil
|
|
import stat
|
|
import subprocess
|
|
import tempfile
|
|
import urllib.parse
|
|
import urllib.request
|
|
|
|
AZCOPY_VERSION = "10.4.3"
|
|
|
|
# See here for instructions on getting stable download links:
|
|
# https://docs.microsoft.com/en-us/azure/storage/common/storage-use-azcopy-v10#obtain-a-static-download-link
|
|
_AZCOPY_DOWNLOAD_URLS = {
|
|
"Linux": "https://azcopyvnext.azureedge.net/release20200501/azcopy_linux_amd64_10.4.3.tar.gz",
|
|
"Darwin": "https://azcopyvnext.azureedge.net/release20200501/azcopy_darwin_amd64_10.4.3.zip",
|
|
"Windows": "https://azcopyvnext.azureedge.net/release20200501/azcopy_windows_amd64_10.4.3.zip",
|
|
}
|
|
|
|
_log = logging.getLogger("util.get_azcopy")
|
|
|
|
|
|
def _check_version(azcopy_path):
|
|
proc = subprocess.run([azcopy_path, "--version"], stdout=subprocess.PIPE, text=True)
|
|
match = re.search(r"\d+(?:\.\d+)+", proc.stdout)
|
|
|
|
if not match:
|
|
raise RuntimeError("Failed to determine azcopy version.")
|
|
|
|
return match.group(0) == AZCOPY_VERSION
|
|
|
|
|
|
def _find_azcopy(start_dir):
|
|
for root, _, file_names in os.walk(start_dir):
|
|
for file_name in file_names:
|
|
if file_name == "azcopy" or file_name == "azcopy.exe":
|
|
return os.path.join(root, file_name)
|
|
raise RuntimeError(f"Failed to azcopy in '{start_dir}'.")
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def get_azcopy(local_azcopy_path="azcopy"):
|
|
"""
|
|
Creates a context manager that returns a path to a particular version of
|
|
azcopy (specified in AZCOPY_VERSION). Downloads a temporary copy if needed.
|
|
|
|
:param local_azcopy_path: Path to a local azcopy to try first.
|
|
|
|
Example usage:
|
|
with get_azcopy() as azcopy_path:
|
|
subprocess.run([azcopy_path, "--version"])
|
|
"""
|
|
with contextlib.ExitStack() as context_stack:
|
|
azcopy_path = shutil.which(local_azcopy_path)
|
|
|
|
if azcopy_path is None or not _check_version(azcopy_path):
|
|
temp_dir = context_stack.enter_context(tempfile.TemporaryDirectory())
|
|
|
|
download_url = _AZCOPY_DOWNLOAD_URLS[platform.system()]
|
|
download_basename = urllib.parse.urlsplit(download_url).path.rsplit("/", 1)[-1]
|
|
assert len(download_basename) > 0
|
|
downloaded_path = os.path.join(temp_dir, download_basename)
|
|
|
|
_log.info(f"Downloading azcopy from '{download_url}'...")
|
|
urllib.request.urlretrieve(download_url, downloaded_path)
|
|
|
|
extracted_path = os.path.join(temp_dir, "azcopy")
|
|
shutil.unpack_archive(downloaded_path, extracted_path)
|
|
|
|
azcopy_path = _find_azcopy(extracted_path)
|
|
|
|
os.chmod(azcopy_path, stat.S_IXUSR)
|
|
|
|
yield azcopy_path
|