From d2dc173664da5fec74acee9122694baddc2d7639 Mon Sep 17 00:00:00 2001 From: Xuehai Pan Date: Sun, 4 Aug 2024 10:41:36 +0800 Subject: [PATCH] Remove lint dependency `ufmt` (#132573) `ufmt` is a combination of `black + usort`. This PR removes `ufmt` and run `black` and `usort` separately. Pull Request resolved: https://github.com/pytorch/pytorch/pull/132573 Approved by: https://github.com/ezyang ghstack dependencies: #129769, #132572 --- .lintrunner.toml | 7 +- .vscode/extensions.json | 1 - .vscode/settings_recommended.json | 4 +- .../{ufmt_linter.py => pyfmt_linter.py} | 83 +++++++++++-------- torch/nested/_internal/ops.py | 4 +- 5 files changed, 54 insertions(+), 45 deletions(-) rename tools/linter/adapters/{ufmt_linter.py => pyfmt_linter.py} (62%) diff --git a/.lintrunner.toml b/.lintrunner.toml index 820782c9ecc..4fe9332c930 100644 --- a/.lintrunner.toml +++ b/.lintrunner.toml @@ -1008,16 +1008,16 @@ init_command = [ 'PyYAML==6.0.1', ] -# Black + usort +# usort + black [[linter]] -code = 'UFMT' +code = 'PYFMT' include_patterns = [ '**/*.py', '**/*.pyi', ] command = [ 'python3', - 'tools/linter/adapters/ufmt_linter.py', + 'tools/linter/adapters/pyfmt_linter.py', '--', '@{{PATHSFILE}}' ] @@ -1475,7 +1475,6 @@ init_command = [ '--dry-run={{DRYRUN}}', '--no-black-binary', 'black==23.12.1', - 'ufmt==2.7.0', 'usort==1.0.8.post1', 'isort==5.13.2', ] diff --git a/.vscode/extensions.json b/.vscode/extensions.json index 0125b886536..9b22ad8d65e 100644 --- a/.vscode/extensions.json +++ b/.vscode/extensions.json @@ -1,6 +1,5 @@ { "recommendations": [ "ms-python.python", - "omnilib.ufmt" ] } diff --git a/.vscode/settings_recommended.json b/.vscode/settings_recommended.json index 2d12dd44c89..551a3ec2a5a 100644 --- a/.vscode/settings_recommended.json +++ b/.vscode/settings_recommended.json @@ -4,14 +4,12 @@ }, "files.associations": { "*.py.in": "python", - "*.pyi.in": "python", - "editor.defaultFormatter": "omnilib.ufmt" + "*.pyi.in": "python" }, "files.eol": "\n", "files.insertFinalNewline": true, "files.trimFinalNewlines": true, "files.trimTrailingWhitespace": true, - "python.formatting.provider": "none", "python.linting.enabled": true, "python.linting.flake8Enabled": true } diff --git a/tools/linter/adapters/ufmt_linter.py b/tools/linter/adapters/pyfmt_linter.py similarity index 62% rename from tools/linter/adapters/ufmt_linter.py rename to tools/linter/adapters/pyfmt_linter.py index a8e19861a1a..72af0228efd 100644 --- a/tools/linter/adapters/ufmt_linter.py +++ b/tools/linter/adapters/pyfmt_linter.py @@ -11,10 +11,9 @@ from enum import Enum from pathlib import Path from typing import Any, NamedTuple +import black import isort -from ufmt.core import ufmt_string -from ufmt.util import make_black_config -from usort import Config as UsortConfig +import usort IS_WINDOWS: bool = os.name == "nt" @@ -53,7 +52,7 @@ def format_error_message(filename: str, err: Exception) -> LintMessage: path=filename, line=None, char=None, - code="UFMT", + code="PYFMT", severity=LintSeverity.ADVICE, name="command-failed", original=None, @@ -62,39 +61,53 @@ def format_error_message(filename: str, err: Exception) -> LintMessage: ) +def run_isort(content: str, path: Path) -> str: + isort_config = isort.Config(settings_path=str(REPO_ROOT)) + + is_this_file = path.samefile(__file__) + if not is_this_file: + content = re.sub(r"(#.*\b)usort:\s*skip\b", r"\g<1>isort: split", content) + + content = isort.code(content, config=isort_config, file_path=path) + + if not is_this_file: + content = re.sub(r"(#.*\b)isort: split\b", r"\g<1>usort: skip", content) + + return content + + +def run_usort(content: str, path: Path) -> str: + usort_config = usort.Config.find(path) + + return usort.usort_string(content, path=path, config=usort_config) + + +def run_black(content: str, path: Path) -> str: + black_config = black.parse_pyproject_toml(black.find_pyproject_toml((str(path),))) # type: ignore[attr-defined,arg-type] + # manually patch options that do not have a 1-to-1 match in Mode arguments + black_config["target_versions"] = { + black.TargetVersion[ver.upper()] # type: ignore[attr-defined] + for ver in black_config.pop("target_version", []) + } + black_config["string_normalization"] = not black_config.pop( + "skip_string_normalization", False + ) + black_mode = black.Mode(**black_config) + black_mode.is_pyi = path.suffix.lower() == ".pyi" + black_mode.is_ipynb = path.suffix.lower() == ".ipynb" + + return black.format_str(content, mode=black_mode) + + def check_file(filename: str) -> list[LintMessage]: path = Path(filename).absolute() - original = path.read_text(encoding="utf-8") + original = replacement = path.read_text(encoding="utf-8") try: - isort_config = isort.Config(settings_path=str(REPO_ROOT)) - usort_config = UsortConfig.find(path) - black_config = make_black_config(path) - - if not path.samefile(__file__): - isorted_replacement = re.sub( - r"(#.*\b)isort: split\b", - r"\g<1>usort: skip", - isort.code( - re.sub(r"(#.*\b)usort:\s*skip\b", r"\g<1>isort: split", original), - config=isort_config, - file_path=path, - ), - ) - else: - isorted_replacement = isort.code( - original, - config=isort_config, - file_path=path, - ) - - # Use UFMT API to call both usort and black - replacement = ufmt_string( - path=path, - content=isorted_replacement, - usort_config=usort_config, - black_config=black_config, - ) + # NB: run isort first to enforce style for blank lines + replacement = run_isort(replacement, path=path) + replacement = run_usort(replacement, path=path) + replacement = run_black(replacement, path=path) if original == replacement: return [] @@ -104,7 +117,7 @@ def check_file(filename: str) -> list[LintMessage]: path=filename, line=None, char=None, - code="UFMT", + code="PYFMT", severity=LintSeverity.WARNING, name="format", original=original, @@ -118,7 +131,7 @@ def check_file(filename: str) -> list[LintMessage]: def main() -> None: parser = argparse.ArgumentParser( - description="Format files with ufmt (black + usort).", + description="Format files with usort + black.", fromfile_prefix_chars="@", ) parser.add_argument( diff --git a/torch/nested/_internal/ops.py b/torch/nested/_internal/ops.py index 577d7b54c85..4f1acd70b0f 100644 --- a/torch/nested/_internal/ops.py +++ b/torch/nested/_internal/ops.py @@ -963,7 +963,7 @@ def sum_dim_IntList(func, *args, **kwargs): new_kwargs["dim"], reduce_on_batch, reduce_on_ragged, - reduce_on_non_batch, # noqa: UFMT + reduce_on_non_batch, ) = _wrap_jagged_dims( inp.dim(), new_kwargs["dim"], @@ -1325,7 +1325,7 @@ def mean_dim(func, *args, **kwargs): new_kwargs["dim"], reduce_on_batch, reduce_on_ragged, - reduce_on_non_batch, # noqa: UFMT + reduce_on_non_batch, ) = _wrap_jagged_dims( inp.dim(), new_kwargs["dim"],