From 2eefbc02a01efa2b4a4ec37c74bde6cf5e7f0ec2 Mon Sep 17 00:00:00 2001 From: Catherine Lee Date: Tue, 30 Jan 2024 03:02:24 +0000 Subject: [PATCH] [ez] Discover tests without importing torch (#118574) Moves test discovery into a file that doesn't have import torch so test listing can be done without having torch installed. Helpful when you don't have torch installed (aka me when I'm feeling lazy) I want to move TD into it's own job that doesn't need to wait for build to finish, so this is part of that. The first commit is a nothing more than a copy paste of the selected functions/vars into a new file, the second commit has various changes that should be checked. Pull Request resolved: https://github.com/pytorch/pytorch/pull/118574 Approved by: https://github.com/huydhn --- test/run_test.py | 139 ++------------------------------ tools/testing/discover_tests.py | 139 ++++++++++++++++++++++++++++++++ 2 files changed, 146 insertions(+), 132 deletions(-) create mode 100644 tools/testing/discover_tests.py diff --git a/test/run_test.py b/test/run_test.py index 4e5e89da1c4..635a6e67e44 100755 --- a/test/run_test.py +++ b/test/run_test.py @@ -2,7 +2,6 @@ import argparse import copy -import glob import json import os import pathlib @@ -46,6 +45,13 @@ from tools.stats.import_test_stats import ( TEST_TIMES_FILE, ) from tools.stats.upload_metrics import add_global_metric, emit_metric +from tools.testing.discover_tests import ( + CPP_TEST_PATH, + CPP_TEST_PREFIX, + CPP_TESTS_DIR, + parse_test_module, + TESTS, +) from tools.testing.target_determination.determinator import ( AggregatedHeuristics, get_prediction_confidences, @@ -67,8 +73,6 @@ sys.path.remove(str(REPO_ROOT)) RERUN_DISABLED_TESTS = os.getenv("PYTORCH_TEST_RERUN_DISABLED_TESTS", "0") == "1" -CPP_TEST_PREFIX = "cpp" -CPP_TEST_PATH = "build/bin" DISTRIBUTED_TEST_PREFIX = "distributed" INDUCTOR_TEST_PREFIX = "inductor" @@ -99,10 +103,6 @@ def strtobool(s): return True -def parse_test_module(test): - return test.split(".")[0] - - class TestChoices(list): def __init__(self, *args, **kwargs): super().__init__(args[0]) @@ -111,131 +111,6 @@ class TestChoices(list): return list.__contains__(self, parse_test_module(item)) -def discover_tests( - base_dir: Optional[pathlib.Path] = None, - cpp_tests_dir: Optional[pathlib.Path] = None, - blocklisted_patterns: Optional[List[str]] = None, - blocklisted_tests: Optional[List[str]] = None, - extra_tests: Optional[List[str]] = None, -) -> List[str]: - """ - Searches for all python files starting with test_ excluding one specified by patterns. - If cpp_tests_dir is provided, also scan for all C++ tests under that directory. They - are usually found in build/bin - """ - - def skip_test_p(name: str) -> bool: - rc = False - if blocklisted_patterns is not None: - rc |= any(name.startswith(pattern) for pattern in blocklisted_patterns) - if blocklisted_tests is not None: - rc |= name in blocklisted_tests - return rc - - cwd = pathlib.Path(__file__).resolve().parent if base_dir is None else base_dir - # This supports symlinks, so we can link domain library tests to PyTorch test directory - all_py_files = [ - pathlib.Path(p) for p in glob.glob(f"{cwd}/**/test_*.py", recursive=True) - ] - - cpp_tests_dir = ( - f"{cwd.parent}/{CPP_TEST_PATH}" if cpp_tests_dir is None else cpp_tests_dir - ) - # CPP test files are located under pytorch/build/bin. Unlike Python test, C++ tests - # are just binaries and could have any name, i.e. basic or atest - all_cpp_files = [ - pathlib.Path(p) for p in glob.glob(f"{cpp_tests_dir}/**/*", recursive=True) - ] - - rc = [str(fname.relative_to(cwd))[:-3] for fname in all_py_files] - # Add the cpp prefix for C++ tests so that we can tell them apart - rc.extend( - [ - parse_test_module(f"{CPP_TEST_PREFIX}/{fname.relative_to(cpp_tests_dir)}") - for fname in all_cpp_files - ] - ) - - # Invert slashes on Windows - if sys.platform == "win32": - rc = [name.replace("\\", "/") for name in rc] - rc = [test for test in rc if not skip_test_p(test)] - if extra_tests is not None: - rc += extra_tests - return sorted(rc) - - -CPP_TESTS_DIR = os.path.abspath(os.getenv("CPP_TESTS_DIR", default=CPP_TEST_PATH)) -TESTS = discover_tests( - cpp_tests_dir=CPP_TESTS_DIR, - blocklisted_patterns=[ - "ao", - "bottleneck_test", - "custom_backend", - "custom_operator", - "fx", # executed by test_fx.py - "jit", # executed by test_jit.py - "mobile", - "onnx_caffe2", - "package", # executed by test_package.py - "quantization", # executed by test_quantization.py - "autograd", # executed by test_autograd.py - ], - blocklisted_tests=[ - "test_bundled_images", - "test_cpp_extensions_aot", - "test_determination", - "test_jit_fuser", - "test_jit_simple", - "test_jit_string", - "test_kernel_launch_checks", - "test_nnapi", - "test_static_runtime", - "test_throughput_benchmark", - "distributed/bin/test_script", - "distributed/elastic/multiprocessing/bin/test_script", - "distributed/launcher/bin/test_script", - "distributed/launcher/bin/test_script_init_method", - "distributed/launcher/bin/test_script_is_torchelastic_launched", - "distributed/launcher/bin/test_script_local_rank", - "distributed/test_c10d_spawn", - "distributions/test_transforms", - "distributions/test_utils", - "test/inductor/test_aot_inductor_utils", - "onnx/test_pytorch_onnx_onnxruntime_cuda", - "onnx/test_models", - # These are not C++ tests - f"{CPP_TEST_PREFIX}/CMakeFiles", - f"{CPP_TEST_PREFIX}/CTestTestfile.cmake", - f"{CPP_TEST_PREFIX}/Makefile", - f"{CPP_TEST_PREFIX}/cmake_install.cmake", - f"{CPP_TEST_PREFIX}/c10_intrusive_ptr_benchmark", - f"{CPP_TEST_PREFIX}/example_allreduce", - f"{CPP_TEST_PREFIX}/parallel_benchmark", - f"{CPP_TEST_PREFIX}/protoc", - f"{CPP_TEST_PREFIX}/protoc-3.13.0.0", - f"{CPP_TEST_PREFIX}/torch_shm_manager", - f"{CPP_TEST_PREFIX}/tutorial_tensorexpr", - ], - extra_tests=[ - "test_cpp_extensions_aot_ninja", - "test_cpp_extensions_aot_no_ninja", - "distributed/elastic/timer/api_test", - "distributed/elastic/timer/local_timer_example", - "distributed/elastic/timer/local_timer_test", - "distributed/elastic/events/lib_test", - "distributed/elastic/metrics/api_test", - "distributed/elastic/utils/logging_test", - "distributed/elastic/utils/util_test", - "distributed/elastic/utils/distributed_test", - "distributed/elastic/multiprocessing/api_test", - ], -) - -# The doctests are a special case that don't correspond to a file that discover -# tests can enable. -TESTS = TESTS + ["doctests"] - FSDP_TEST = [test for test in TESTS if test.startswith("distributed/fsdp")] WINDOWS_BLOCKLIST = [ diff --git a/tools/testing/discover_tests.py b/tools/testing/discover_tests.py new file mode 100644 index 00000000000..5c37fdd4fe9 --- /dev/null +++ b/tools/testing/discover_tests.py @@ -0,0 +1,139 @@ +import glob +import os +import sys +from pathlib import Path +from typing import List, Optional, Union + +CPP_TEST_PREFIX = "cpp" +CPP_TEST_PATH = "build/bin" +CPP_TESTS_DIR = os.path.abspath(os.getenv("CPP_TESTS_DIR", default=CPP_TEST_PATH)) +REPO_ROOT = Path(__file__).resolve().parent.parent.parent + + +def parse_test_module(test: str) -> str: + return test.split(".")[0] + + +def discover_tests( + base_dir: Path = REPO_ROOT / "test", + cpp_tests_dir: Optional[Union[str, Path]] = None, + blocklisted_patterns: Optional[List[str]] = None, + blocklisted_tests: Optional[List[str]] = None, + extra_tests: Optional[List[str]] = None, +) -> List[str]: + """ + Searches for all python files starting with test_ excluding one specified by patterns. + If cpp_tests_dir is provided, also scan for all C++ tests under that directory. They + are usually found in build/bin + """ + + def skip_test_p(name: str) -> bool: + rc = False + if blocklisted_patterns is not None: + rc |= any(name.startswith(pattern) for pattern in blocklisted_patterns) + if blocklisted_tests is not None: + rc |= name in blocklisted_tests + return rc + + # This supports symlinks, so we can link domain library tests to PyTorch test directory + all_py_files = [ + Path(p) for p in glob.glob(f"{base_dir}/**/test_*.py", recursive=True) + ] + + cpp_tests_dir = ( + f"{base_dir.parent}/{CPP_TEST_PATH}" if cpp_tests_dir is None else cpp_tests_dir + ) + # CPP test files are located under pytorch/build/bin. Unlike Python test, C++ tests + # are just binaries and could have any name, i.e. basic or atest + all_cpp_files = [ + Path(p) for p in glob.glob(f"{cpp_tests_dir}/**/*", recursive=True) + ] + + rc = [str(fname.relative_to(base_dir))[:-3] for fname in all_py_files] + # Add the cpp prefix for C++ tests so that we can tell them apart + rc.extend( + [ + parse_test_module(f"{CPP_TEST_PREFIX}/{fname.relative_to(cpp_tests_dir)}") + for fname in all_cpp_files + ] + ) + + # Invert slashes on Windows + if sys.platform == "win32": + rc = [name.replace("\\", "/") for name in rc] + rc = [test for test in rc if not skip_test_p(test)] + if extra_tests is not None: + rc += extra_tests + return sorted(rc) + + +TESTS = discover_tests( + cpp_tests_dir=CPP_TESTS_DIR, + blocklisted_patterns=[ + "ao", + "bottleneck_test", + "custom_backend", + "custom_operator", + "fx", # executed by test_fx.py + "jit", # executed by test_jit.py + "mobile", + "onnx_caffe2", + "package", # executed by test_package.py + "quantization", # executed by test_quantization.py + "autograd", # executed by test_autograd.py + ], + blocklisted_tests=[ + "test_bundled_images", + "test_cpp_extensions_aot", + "test_determination", + "test_jit_fuser", + "test_jit_simple", + "test_jit_string", + "test_kernel_launch_checks", + "test_nnapi", + "test_static_runtime", + "test_throughput_benchmark", + "distributed/bin/test_script", + "distributed/elastic/multiprocessing/bin/test_script", + "distributed/launcher/bin/test_script", + "distributed/launcher/bin/test_script_init_method", + "distributed/launcher/bin/test_script_is_torchelastic_launched", + "distributed/launcher/bin/test_script_local_rank", + "distributed/test_c10d_spawn", + "distributions/test_transforms", + "distributions/test_utils", + "test/inductor/test_aot_inductor_utils", + "onnx/test_pytorch_onnx_onnxruntime_cuda", + "onnx/test_models", + # These are not C++ tests + f"{CPP_TEST_PREFIX}/CMakeFiles", + f"{CPP_TEST_PREFIX}/CTestTestfile.cmake", + f"{CPP_TEST_PREFIX}/Makefile", + f"{CPP_TEST_PREFIX}/cmake_install.cmake", + f"{CPP_TEST_PREFIX}/c10_intrusive_ptr_benchmark", + f"{CPP_TEST_PREFIX}/example_allreduce", + f"{CPP_TEST_PREFIX}/parallel_benchmark", + f"{CPP_TEST_PREFIX}/protoc", + f"{CPP_TEST_PREFIX}/protoc-3.13.0.0", + f"{CPP_TEST_PREFIX}/torch_shm_manager", + f"{CPP_TEST_PREFIX}/tutorial_tensorexpr", + ], + extra_tests=[ + "test_cpp_extensions_aot_ninja", + "test_cpp_extensions_aot_no_ninja", + "distributed/elastic/timer/api_test", + "distributed/elastic/timer/local_timer_example", + "distributed/elastic/timer/local_timer_test", + "distributed/elastic/events/lib_test", + "distributed/elastic/metrics/api_test", + "distributed/elastic/utils/logging_test", + "distributed/elastic/utils/util_test", + "distributed/elastic/utils/distributed_test", + "distributed/elastic/multiprocessing/api_test", + "doctests", + ], +) + + +if __name__ == "__main__": + print(TESTS)