From f625bb4bc9f601727809bfee46f00d0948875227 Mon Sep 17 00:00:00 2001 From: John Reese Date: Mon, 6 Jun 2022 23:44:26 +0000 Subject: [PATCH] [codemod][usort] apply import merging for fbcode (1 of 11) (#78973) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Summary: Applies new import merging and sorting from µsort v1.0. When merging imports, µsort will make a best-effort to move associated comments to match merged elements, but there are known limitations due to the diynamic nature of Python and developer tooling. These changes should not produce any dangerous runtime changes, but may require touch-ups to satisfy linters and other tooling. Note that µsort uses case-insensitive, lexicographical sorting, which results in a different ordering compared to isort. This provides a more consistent sorting order, matching the case-insensitive order used when sorting import statements by module name, and ensures that "frog", "FROG", and "Frog" always sort next to each other. For details on µsort's sorting and merging semantics, see the user guide: https://usort.readthedocs.io/en/stable/guide.html#sorting Test Plan: S271899 Reviewed By: lisroach Differential Revision: D36402110 Pull Request resolved: https://github.com/pytorch/pytorch/pull/78973 Approved by: https://github.com/osalpekar --- test/package/test_dependency_hooks.py | 4 +--- test/package/test_directory_reader.py | 2 +- test/package/test_load_bc_packages.py | 2 +- test/package/test_mangling.py | 2 +- test/package/test_misc.py | 2 +- test/package/test_package_fx.py | 2 +- test/package/test_package_script.py | 16 +++------------- test/package/test_repackage.py | 6 +----- test/package/test_save_load.py | 2 +- tools/code_coverage/package/oss/init.py | 4 ++-- tools/code_coverage/package/oss/utils.py | 2 +- tools/code_coverage/package/tool/print_report.py | 2 +- .../package/tool/summarize_jsons.py | 2 +- tools/code_coverage/package/tool/utils.py | 4 +++- tools/code_coverage/package/util/utils.py | 2 +- tools/code_coverage/package/util/utils_init.py | 2 +- torch/package/_digraph.py | 2 +- torch/package/_package_pickler.py | 15 +++++++++++++-- torch/package/analyze/__init__.py | 8 ++------ torch/package/file_structure_representation.py | 2 +- torch/package/importer.py | 9 ++++++--- torch/package/package_exporter.py | 6 +++--- torch/package/package_importer.py | 6 +++--- 23 files changed, 50 insertions(+), 54 deletions(-) diff --git a/test/package/test_dependency_hooks.py b/test/package/test_dependency_hooks.py index 786f6f8a9cd..df155ab1dea 100644 --- a/test/package/test_dependency_hooks.py +++ b/test/package/test_dependency_hooks.py @@ -2,9 +2,7 @@ from io import BytesIO -from torch.package import ( - PackageExporter, -) +from torch.package import PackageExporter from torch.testing._internal.common_utils import run_tests try: diff --git a/test/package/test_directory_reader.py b/test/package/test_directory_reader.py index f6dde64ac80..16d1b73d288 100644 --- a/test/package/test_directory_reader.py +++ b/test/package/test_directory_reader.py @@ -11,10 +11,10 @@ from unittest import skipIf import torch from torch.package import PackageExporter, PackageImporter from torch.testing._internal.common_utils import ( - run_tests, IS_FBCODE, IS_SANDCASTLE, IS_WINDOWS, + run_tests, ) try: diff --git a/test/package/test_load_bc_packages.py b/test/package/test_load_bc_packages.py index 432e8dde9f8..af1c26cae90 100644 --- a/test/package/test_load_bc_packages.py +++ b/test/package/test_load_bc_packages.py @@ -4,7 +4,7 @@ from pathlib import Path from unittest import skipIf from torch.package import PackageImporter -from torch.testing._internal.common_utils import run_tests, IS_FBCODE, IS_SANDCASTLE +from torch.testing._internal.common_utils import IS_FBCODE, IS_SANDCASTLE, run_tests try: from .common import PackageTestCase diff --git a/test/package/test_mangling.py b/test/package/test_mangling.py index 4a36ba620d1..058a8ba2147 100644 --- a/test/package/test_mangling.py +++ b/test/package/test_mangling.py @@ -4,10 +4,10 @@ from io import BytesIO from torch.package import PackageExporter, PackageImporter from torch.package._mangling import ( - PackageMangler, demangle, get_mangle_prefix, is_mangled, + PackageMangler, ) from torch.testing._internal.common_utils import run_tests diff --git a/test/package/test_misc.py b/test/package/test_misc.py index 480217b8feb..20e9b0ebbb1 100644 --- a/test/package/test_misc.py +++ b/test/package/test_misc.py @@ -8,7 +8,7 @@ from pathlib import Path from textwrap import dedent from unittest import skipIf -from torch.package import PackageExporter, PackageImporter, is_from_package +from torch.package import is_from_package, PackageExporter, PackageImporter from torch.package.package_exporter import PackagingError from torch.testing._internal.common_utils import IS_FBCODE, IS_SANDCASTLE, run_tests diff --git a/test/package/test_package_fx.py b/test/package/test_package_fx.py index fe1059c976a..ee248d6830b 100644 --- a/test/package/test_package_fx.py +++ b/test/package/test_package_fx.py @@ -125,7 +125,7 @@ class TestPackageFX(PackageTestCase): def test_package_fx_custom_tracer(self): from package_a.test_all_leaf_modules_tracer import TestAllLeafModulesTracer - from package_a.test_module import SimpleTest, ModWithTwoSubmodsAndTensor + from package_a.test_module import ModWithTwoSubmodsAndTensor, SimpleTest class SpecialGraphModule(torch.fx.GraphModule): def __init__(self, root, graph, info): diff --git a/test/package/test_package_script.py b/test/package/test_package_script.py index abe1645d94f..6dcaa2678c4 100644 --- a/test/package/test_package_script.py +++ b/test/package/test_package_script.py @@ -6,11 +6,7 @@ from unittest import skipIf import torch from torch.package import PackageExporter, PackageImporter -from torch.testing._internal.common_utils import ( - IS_FBCODE, - IS_SANDCASTLE, - run_tests, -) +from torch.testing._internal.common_utils import IS_FBCODE, IS_SANDCASTLE, run_tests try: from .common import PackageTestCase @@ -538,10 +534,7 @@ class TestPackageScript(PackageTestCase): Test tensors shared across eager and ScriptModules on load are the same. """ - from package_a.test_module import ( - ModWithTensor, - ModWithTwoSubmodsAndTensor, - ) + from package_a.test_module import ModWithTensor, ModWithTwoSubmodsAndTensor shared_tensor = torch.ones(3, 3) @@ -595,10 +588,7 @@ class TestPackageScript(PackageTestCase): the backing cpp TensorImpl is. We load/save storages based off of this cpp TensorImpl and not the python identity. """ - from package_a.test_module import ( - ModWithTensor, - ModWithTwoSubmodsAndTensor, - ) + from package_a.test_module import ModWithTensor, ModWithTwoSubmodsAndTensor shared_tensor = torch.ones(3, 3) diff --git a/test/package/test_repackage.py b/test/package/test_repackage.py index cb743afd840..fe62871dc94 100644 --- a/test/package/test_repackage.py +++ b/test/package/test_repackage.py @@ -2,11 +2,7 @@ from io import BytesIO -from torch.package import ( - PackageExporter, - PackageImporter, - sys_importer, -) +from torch.package import PackageExporter, PackageImporter, sys_importer from torch.testing._internal.common_utils import run_tests try: diff --git a/test/package/test_save_load.py b/test/package/test_save_load.py index 0f6e8f9bfd7..2477d85699d 100644 --- a/test/package/test_save_load.py +++ b/test/package/test_save_load.py @@ -6,7 +6,7 @@ from textwrap import dedent from unittest import skipIf from torch.package import PackageExporter, PackageImporter, sys_importer -from torch.testing._internal.common_utils import run_tests, IS_FBCODE, IS_SANDCASTLE +from torch.testing._internal.common_utils import IS_FBCODE, IS_SANDCASTLE, run_tests try: from .common import PackageTestCase diff --git a/tools/code_coverage/package/oss/init.py b/tools/code_coverage/package/oss/init.py index beef653cb6f..57e37b576b9 100644 --- a/tools/code_coverage/package/oss/init.py +++ b/tools/code_coverage/package/oss/init.py @@ -1,11 +1,11 @@ import argparse import os -from typing import List, Optional, Tuple, cast +from typing import cast, List, Optional, Tuple from ..util.setting import ( + CompilerType, JSON_FOLDER_BASE_DIR, LOG_DIR, - CompilerType, Option, Test, TestList, diff --git a/tools/code_coverage/package/oss/utils.py b/tools/code_coverage/package/oss/utils.py index 739aa6e3910..1cb67dc7143 100644 --- a/tools/code_coverage/package/oss/utils.py +++ b/tools/code_coverage/package/oss/utils.py @@ -2,7 +2,7 @@ import os import subprocess from typing import List, Optional -from ..util.setting import TOOLS_FOLDER, CompilerType, TestType +from ..util.setting import CompilerType, TestType, TOOLS_FOLDER from ..util.utils import print_error, remove_file diff --git a/tools/code_coverage/package/tool/print_report.py b/tools/code_coverage/package/tool/print_report.py index 749773f8e42..0222f32cfca 100644 --- a/tools/code_coverage/package/tool/print_report.py +++ b/tools/code_coverage/package/tool/print_report.py @@ -1,6 +1,6 @@ import os import subprocess -from typing import IO, Dict, List, Set, Tuple +from typing import Dict, IO, List, Set, Tuple from ..oss.utils import get_pytorch_folder from ..util.setting import SUMMARY_FOLDER_DIR, TestList, TestStatusType diff --git a/tools/code_coverage/package/tool/summarize_jsons.py b/tools/code_coverage/package/tool/summarize_jsons.py index 0a9dbd1d72f..0ab113816c0 100644 --- a/tools/code_coverage/package/tool/summarize_jsons.py +++ b/tools/code_coverage/package/tool/summarize_jsons.py @@ -4,8 +4,8 @@ import time from typing import Any, Dict, List, Set, Tuple from ..util.setting import ( - JSON_FOLDER_BASE_DIR, CompilerType, + JSON_FOLDER_BASE_DIR, TestList, TestPlatform, TestStatusType, diff --git a/tools/code_coverage/package/tool/utils.py b/tools/code_coverage/package/tool/utils.py index 7cad7c063f8..582aa148696 100644 --- a/tools/code_coverage/package/tool/utils.py +++ b/tools/code_coverage/package/tool/utils.py @@ -14,7 +14,9 @@ def run_cpp_test(binary_file: str) -> None: def get_tool_path_by_platform(platform: TestPlatform) -> str: if platform == TestPlatform.FBCODE: - from caffe2.fb.code_coverage.tool.package.fbcode.utils import get_llvm_tool_path # type: ignore[import] + from caffe2.fb.code_coverage.tool.package.fbcode.utils import ( # type: ignore[import] + get_llvm_tool_path, + ) return get_llvm_tool_path() # type: ignore[no-any-return] else: diff --git a/tools/code_coverage/package/util/utils.py b/tools/code_coverage/package/util/utils.py index 6133c35dd83..dee9cbc1a1b 100644 --- a/tools/code_coverage/package/util/utils.py +++ b/tools/code_coverage/package/util/utils.py @@ -5,9 +5,9 @@ import time from typing import Any, NoReturn, Optional from .setting import ( + CompilerType, LOG_DIR, PROFILE_DIR, - CompilerType, TestList, TestPlatform, TestType, diff --git a/tools/code_coverage/package/util/utils_init.py b/tools/code_coverage/package/util/utils_init.py index aa79e74e886..6b453df54d3 100644 --- a/tools/code_coverage/package/util/utils_init.py +++ b/tools/code_coverage/package/util/utils_init.py @@ -6,9 +6,9 @@ from .setting import ( JSON_FOLDER_BASE_DIR, LOG_DIR, MERGED_FOLDER_BASE_DIR, + Option, PROFILE_DIR, SUMMARY_FOLDER_DIR, - Option, ) from .utils import create_folder, get_raw_profiles_folder, remove_file diff --git a/torch/package/_digraph.py b/torch/package/_digraph.py index c918b3d56fc..f84a51398f0 100644 --- a/torch/package/_digraph.py +++ b/torch/package/_digraph.py @@ -1,5 +1,5 @@ from collections import deque -from typing import Set, List +from typing import List, Set class DiGraph: diff --git a/torch/package/_package_pickler.py b/torch/package/_package_pickler.py index 9454efe94fc..cabc6a82164 100644 --- a/torch/package/_package_pickler.py +++ b/torch/package/_package_pickler.py @@ -1,6 +1,17 @@ """isort:skip_file""" -from pickle import EXT1, EXT2, EXT4, GLOBAL, STACK_GLOBAL, Pickler, PicklingError -from pickle import _compat_pickle, _extension_registry, _getattribute, _Pickler # type: ignore[attr-defined] +from pickle import ( # type: ignore[attr-defined] + _compat_pickle, + _extension_registry, + _getattribute, + _Pickler, + EXT1, + EXT2, + EXT4, + GLOBAL, + Pickler, + PicklingError, + STACK_GLOBAL, +) from struct import pack from types import FunctionType diff --git a/torch/package/analyze/__init__.py b/torch/package/analyze/__init__.py index 6146eaf937e..8ef7a1716af 100644 --- a/torch/package/analyze/__init__.py +++ b/torch/package/analyze/__init__.py @@ -1,6 +1,2 @@ -from .find_first_use_of_broken_modules import ( - find_first_use_of_broken_modules, -) -from .trace_dependencies import ( - trace_dependencies, -) +from .find_first_use_of_broken_modules import find_first_use_of_broken_modules +from .trace_dependencies import trace_dependencies diff --git a/torch/package/file_structure_representation.py b/torch/package/file_structure_representation.py index ad2e721b7ca..526b782f250 100644 --- a/torch/package/file_structure_representation.py +++ b/torch/package/file_structure_representation.py @@ -1,7 +1,7 @@ # -*- coding: utf-8 -*- from typing import Dict, List -from .glob_group import GlobPattern, GlobGroup +from .glob_group import GlobGroup, GlobPattern class Directory: diff --git a/torch/package/importer.py b/torch/package/importer.py index d7347653d47..4893730d412 100644 --- a/torch/package/importer.py +++ b/torch/package/importer.py @@ -1,9 +1,12 @@ import importlib from abc import ABC, abstractmethod -from pickle import _getattribute, _Pickler # type: ignore[attr-defined] -from pickle import whichmodule as _pickle_whichmodule # type: ignore[attr-defined] +from pickle import ( # type: ignore[attr-defined] # type: ignore[attr-defined] + _getattribute, + _Pickler, + whichmodule as _pickle_whichmodule, +) from types import ModuleType -from typing import Any, List, Optional, Tuple, Dict +from typing import Any, Dict, List, Optional, Tuple from ._mangling import demangle, get_mangle_prefix, is_mangled diff --git a/torch/package/package_exporter.py b/torch/package/package_exporter.py index 1baa865eefb..25dfe3974a7 100644 --- a/torch/package/package_exporter.py +++ b/torch/package/package_exporter.py @@ -5,7 +5,7 @@ import linecache import pickletools import platform import types -from collections import OrderedDict, defaultdict +from collections import defaultdict, OrderedDict from dataclasses import dataclass from enum import Enum from pathlib import Path @@ -13,14 +13,14 @@ from typing import ( Any, BinaryIO, Callable, + cast, + DefaultDict, Dict, List, Optional, Sequence, Set, Union, - cast, - DefaultDict, ) import torch diff --git a/torch/package/package_importer.py b/torch/package/package_importer.py index 09a147bbde8..03196f548b5 100644 --- a/torch/package/package_importer.py +++ b/torch/package/package_importer.py @@ -7,7 +7,7 @@ import os.path import types from contextlib import contextmanager from pathlib import Path -from typing import cast, Any, BinaryIO, Callable, Dict, List, Optional, Union +from typing import Any, BinaryIO, Callable, cast, Dict, List, Optional, Union from weakref import WeakValueDictionary import torch @@ -21,9 +21,9 @@ from ._importlib import ( _resolve_name, _sanity_check, ) -from ._mangling import PackageMangler, demangle +from ._mangling import demangle, PackageMangler from ._package_unpickler import PackageUnpickler -from .file_structure_representation import Directory, _create_directory_from_file_list +from .file_structure_representation import _create_directory_from_file_list, Directory from .glob_group import GlobPattern from .importer import Importer