[BE] enforce style for empty lines in import segments (#129751)

This PR follows https://github.com/pytorch/pytorch/pull/129374#pullrequestreview-2136555775 cc @malfet:

> Lots of formatting changes unrelated to PR goal, please keep them as part of separate PR (and please add lint rule if you want to enforce those, or at least cite one)

`usort` allows empty lines within import segments. For example, `usort` do not change the following code:

```python
import torch.aaa
import torch.bbb
import torch.ccc

x = ...  # some code
```

```python
import torch.aaa

import torch.bbb
import torch.ccc

x = ...  # some code
```

```python
import torch.aaa

import torch.bbb

import torch.ccc

x = ...  # some code
```

This PR first sort imports via `isort`, then re-sort the file using `ufmt` (`usort` + `black`). This enforces the following import style:

1. no empty lines within segments.
2. single empty line between segments.
3. two spaces after import statements.

All the code snippets above will be formatted to:

```python
import torch.aaa
import torch.bbb
import torch.ccc

x = ...  # some code
```

which produces a consistent code style.

Pull Request resolved: https://github.com/pytorch/pytorch/pull/129751
Approved by: https://github.com/malfet

.lintrunner.toml

@@ -1695,8 +1695,9 @@ init_command = [
     '--dry-run={{DRYRUN}}',
     '--no-black-binary',
     'black==23.12.1',
-    'ufmt==2.1.0',
-    'usort==1.0.6',
+    'ufmt==2.7.0',
+    'usort==1.0.8.post1',
+    'isort==5.13.2',
 ]
 is_formatter = true
 

pyproject.toml

@@ -31,6 +31,7 @@ line_length = 88
 lines_after_imports = 2
 multi_line_output = 3
 include_trailing_comma = true
+combine_as_imports = true
 
 
 [tool.usort.known]

tools/linter/adapters/ufmt_linter.py

@@ -2,20 +2,89 @@ from __future__ import annotations
 
 import argparse
 import concurrent.futures
+import fnmatch
 import json
 import logging
 import os
+import re
 import sys
 from enum import Enum
 from pathlib import Path
 from typing import Any, NamedTuple
 
+import isort
+from isort import Config as IsortConfig
 from ufmt.core import ufmt_string
 from ufmt.util import make_black_config
 from usort import Config as UsortConfig
 
 
 IS_WINDOWS: bool = os.name == "nt"
+REPO_ROOT = Path(__file__).absolute().parents[3]
+ISORT_WHITELIST = re.compile(
+    "|".join(
+        (
+            r"\A\Z",  # empty string
+            *map(
+                fnmatch.translate,
+                [
+                    # **
+                    "**",
+                    # .ci/**
+                    ".ci/**",
+                    # .github/**
+                    ".github/**",
+                    # benchmarks/**
+                    "benchmarks/**",
+                    # functorch/**
+                    "functorch/**",
+                    # tools/**
+                    "tools/**",
+                    # torchgen/**
+                    "torchgen/**",
+                    # test/**
+                    "test/**",
+                    # test/[a-c]*/**
+                    "test/[a-c]*/**",
+                    # test/d*/**
+                    "test/d*/**",
+                    # test/dy*/**
+                    "test/dy*/**",
+                    # test/[e-h]*/**
+                    "test/[e-h]*/**",
+                    # test/i*/**
+                    "test/i*/**",
+                    # test/j*/**
+                    "test/j*/**",
+                    # test/[k-p]*/**
+                    "test/[k-p]*/**",
+                    # test/[q-z]*/**
+                    "test/[q-z]*/**",
+                    # torch/**
+                    "torch/**",
+                    # torch/_[a-c]*/**
+                    "torch/_[a-c]*/**",
+                    # torch/_d*/**
+                    "torch/_d*/**",
+                    # torch/_[e-h]*/**
+                    "torch/_[e-h]*/**",
+                    # torch/_i*/**
+                    "torch/_i*/**",
+                    # torch/_[j-z]*/**
+                    "torch/_[j-z]*/**",
+                    # torch/[a-c]*/**
+                    "torch/[a-c]*/**",
+                    # torch/d*/**
+                    "torch/d*/**",
+                    # torch/[e-n]*/**
+                    "torch/[e-n]*/**",
+                    # torch/[o-z]*/**
+                    "torch/[o-z]*/**",
+                ],
+            ),
+        )
+    )
+)
 
 
 def eprint(*args: Any, **kwargs: Any) -> None:
@@ -59,22 +128,33 @@ def format_error_message(filename: str, err: Exception) -> LintMessage:
     )
 
 
-def check_file(
-    filename: str,
-) -> list[LintMessage]:
-    with open(filename, "rb") as f:
-        original = f.read().decode("utf-8")
+def check_file(filename: str) -> list[LintMessage]:
+    path = Path(filename).absolute()
+    original = path.read_text(encoding="utf-8")
 
     try:
-        path = Path(filename)
-
         usort_config = UsortConfig.find(path)
         black_config = make_black_config(path)
 
+        if not path.samefile(__file__) and not ISORT_WHITELIST.match(
+            path.absolute().relative_to(REPO_ROOT).as_posix()
+        ):
+            isorted_replacement = re.sub(
+                r"(#.*\b)isort: split\b",
+                r"\g<1>usort: skip",
+                isort.code(
+                    re.sub(r"(#.*\b)usort:\s*skip\b", r"\g<1>isort: split", original),
+                    config=IsortConfig(settings_path=str(REPO_ROOT)),
+                    file_path=path,
+                ),
+            )
+        else:
+            isorted_replacement = original
+
         # Use UFMT API to call both usort and black
         replacement = ufmt_string(
             path=path,
-            content=original,
+            content=isorted_replacement,
             usort_config=usort_config,
             black_config=black_config,
         )

torch/_numpy/__init__.py

@@ -17,14 +17,15 @@ from ._ndarray import (
 from ._ufuncs import *  # noqa: F403
 from ._util import AxisError, UFuncTypeError
 
-# from . import testing
+
+from math import pi, e  # usort: skip
+
 
 alltrue = all
 sometrue = any
 
 inf = float("inf")
 nan = float("nan")
-from math import pi, e  # isort: skip
 
 False_ = False
 True_ = True

torch/package/_package_pickler.py

@@ -1,5 +1,4 @@
 # mypy: allow-untyped-defs
-"""isort:skip_file"""
 from pickle import (  # type: ignore[attr-defined]
     _compat_pickle,
     _extension_registry,