From 096ff0ecca3f76dcdeebe4e515c4fc3843078753 Mon Sep 17 00:00:00 2001 From: mikey dagitses Date: Wed, 4 May 2022 06:48:20 -0700 Subject: [PATCH] introduce new --gen-dir flag to generate_code and use it in fbcode (#75800) Summary: Pull Request resolved: https://github.com/pytorch/pytorch/pull/75800 This leads to more similarities between OSS CMake and eventually OSS Bazel. We will be able to generate files with the same names and not have different file lists between the builds. ghstack-source-id: 155300043 Test Plan: Verified locally and in CI. Reviewed By: dreiss Differential Revision: D35648586 fbshipit-source-id: 9f1638b5665ebcc64466883f65ef24a2bfd05228 (cherry picked from commit 7f2acff1baa8dfafddefdc720714f8d39feda436) --- build.bzl | 78 +++++++++++++--------------- tools/build_variables.bzl | 56 ++++++++++---------- tools/setup_helpers/generate_code.py | 25 ++++++--- 3 files changed, 83 insertions(+), 76 deletions(-) diff --git a/build.bzl b/build.bzl index fac3d081ff4..247e4097d16 100644 --- a/build.bzl +++ b/build.bzl @@ -38,7 +38,7 @@ def define_targets(rules): tools = ["//tools/setup_helpers:generate_code"], outs = _GENERATED_CPP + GENERATED_AUTOGRAD_H + GENERATED_LAZY_H + GENERATED_TESTING_PY, cmd = "$(location //tools/setup_helpers:generate_code) " + - "--install_dir $(RULEDIR) " + + "--gen-dir=$(RULEDIR) " + "--native-functions-path $(location :native_functions.yaml) " + "--tags-path=$(location :tags.yaml) " + "--gen_lazy_ts_backend", @@ -72,55 +72,49 @@ def define_targets(rules): # generate-code that use these lists are moved into the shared # structure as well. -# In the open-source build, these are generated into -# torch/csrc/autograd/generated GENERATED_AUTOGRAD_H = [ - "autograd/generated/Functions.h", - "autograd/generated/VariableType.h", - "autograd/generated/python_functions.h", - "autograd/generated/variable_factories.h", + "torch/csrc/autograd/generated/Functions.h", + "torch/csrc/autograd/generated/VariableType.h", + "torch/csrc/autograd/generated/python_functions.h", + "torch/csrc/autograd/generated/variable_factories.h", ] -# In the open-source build, these are generated into -# torch/testing/_internal/generated GENERATED_TESTING_PY = [ - "annotated_fn_args.py", + "torch/testing/_internal/generated/annotated_fn_args.py", ] GENERATED_LAZY_H = [ - "lazy/generated/LazyIr.h", - "lazy/generated/LazyNativeFunctions.h", + "torch/csrc/lazy/generated/LazyIr.h", + "torch/csrc/lazy/generated/LazyNativeFunctions.h", ] -# In both open-source and fbcode builds, these are generated into -# torch/csrc/{autograd,jit}/generated.i _GENERATED_CPP = [ - "autograd/generated/Functions.cpp", - "autograd/generated/VariableType_0.cpp", - "autograd/generated/VariableType_1.cpp", - "autograd/generated/VariableType_2.cpp", - "autograd/generated/VariableType_3.cpp", - "autograd/generated/VariableType_4.cpp", - "autograd/generated/TraceType_0.cpp", - "autograd/generated/TraceType_1.cpp", - "autograd/generated/TraceType_2.cpp", - "autograd/generated/TraceType_3.cpp", - "autograd/generated/TraceType_4.cpp", - "autograd/generated/ADInplaceOrViewType_0.cpp", - "autograd/generated/ADInplaceOrViewType_1.cpp", - "autograd/generated/python_functions_0.cpp", - "autograd/generated/python_functions_1.cpp", - "autograd/generated/python_functions_2.cpp", - "autograd/generated/python_functions_3.cpp", - "autograd/generated/python_functions_4.cpp", - "autograd/generated/python_nn_functions.cpp", - "autograd/generated/python_fft_functions.cpp", - "autograd/generated/python_linalg_functions.cpp", - "autograd/generated/python_return_types.cpp", - "autograd/generated/python_sparse_functions.cpp", - "autograd/generated/python_special_functions.cpp", - "autograd/generated/python_torch_functions_0.cpp", - "autograd/generated/python_torch_functions_1.cpp", - "autograd/generated/python_torch_functions_2.cpp", - "autograd/generated/python_variable_methods.cpp", + "torch/csrc/autograd/generated/Functions.cpp", + "torch/csrc/autograd/generated/VariableType_0.cpp", + "torch/csrc/autograd/generated/VariableType_1.cpp", + "torch/csrc/autograd/generated/VariableType_2.cpp", + "torch/csrc/autograd/generated/VariableType_3.cpp", + "torch/csrc/autograd/generated/VariableType_4.cpp", + "torch/csrc/autograd/generated/TraceType_0.cpp", + "torch/csrc/autograd/generated/TraceType_1.cpp", + "torch/csrc/autograd/generated/TraceType_2.cpp", + "torch/csrc/autograd/generated/TraceType_3.cpp", + "torch/csrc/autograd/generated/TraceType_4.cpp", + "torch/csrc/autograd/generated/ADInplaceOrViewType_0.cpp", + "torch/csrc/autograd/generated/ADInplaceOrViewType_1.cpp", + "torch/csrc/autograd/generated/python_functions_0.cpp", + "torch/csrc/autograd/generated/python_functions_1.cpp", + "torch/csrc/autograd/generated/python_functions_2.cpp", + "torch/csrc/autograd/generated/python_functions_3.cpp", + "torch/csrc/autograd/generated/python_functions_4.cpp", + "torch/csrc/autograd/generated/python_nn_functions.cpp", + "torch/csrc/autograd/generated/python_fft_functions.cpp", + "torch/csrc/autograd/generated/python_linalg_functions.cpp", + "torch/csrc/autograd/generated/python_return_types.cpp", + "torch/csrc/autograd/generated/python_sparse_functions.cpp", + "torch/csrc/autograd/generated/python_special_functions.cpp", + "torch/csrc/autograd/generated/python_torch_functions_0.cpp", + "torch/csrc/autograd/generated/python_torch_functions_1.cpp", + "torch/csrc/autograd/generated/python_torch_functions_2.cpp", + "torch/csrc/autograd/generated/python_variable_methods.cpp", ] diff --git a/tools/build_variables.bzl b/tools/build_variables.bzl index 09cf9ae5d18..3e8d5f5ef7d 100644 --- a/tools/build_variables.bzl +++ b/tools/build_variables.bzl @@ -46,19 +46,19 @@ libtorch_nvfuser_generated_headers = ["{}.h".format(name.split("/")[-1].split(". def libtorch_generated_sources(gencode_pattern): return [gencode_pattern.format(name) for name in [ - "autograd/generated/Functions.cpp", - "autograd/generated/VariableType_0.cpp", - "autograd/generated/VariableType_1.cpp", - "autograd/generated/VariableType_2.cpp", - "autograd/generated/VariableType_3.cpp", - "autograd/generated/VariableType_4.cpp", - "autograd/generated/TraceType_0.cpp", - "autograd/generated/TraceType_1.cpp", - "autograd/generated/TraceType_2.cpp", - "autograd/generated/TraceType_3.cpp", - "autograd/generated/TraceType_4.cpp", - "autograd/generated/ADInplaceOrViewType_0.cpp", - "autograd/generated/ADInplaceOrViewType_1.cpp", + "torch/csrc/autograd/generated/Functions.cpp", + "torch/csrc/autograd/generated/VariableType_0.cpp", + "torch/csrc/autograd/generated/VariableType_1.cpp", + "torch/csrc/autograd/generated/VariableType_2.cpp", + "torch/csrc/autograd/generated/VariableType_3.cpp", + "torch/csrc/autograd/generated/VariableType_4.cpp", + "torch/csrc/autograd/generated/TraceType_0.cpp", + "torch/csrc/autograd/generated/TraceType_1.cpp", + "torch/csrc/autograd/generated/TraceType_2.cpp", + "torch/csrc/autograd/generated/TraceType_3.cpp", + "torch/csrc/autograd/generated/TraceType_4.cpp", + "torch/csrc/autograd/generated/ADInplaceOrViewType_0.cpp", + "torch/csrc/autograd/generated/ADInplaceOrViewType_1.cpp", ]] # copied from https://github.com/pytorch/pytorch/blob/f99a693cd9ff7a9b5fdc71357dac66b8192786d3/aten/src/ATen/core/CMakeLists.txt @@ -960,21 +960,21 @@ libtorch_python_distributed_sources = libtorch_python_distributed_core_sources + def glob_libtorch_python_sources(gencode_pattern = ":generate-code[{}]"): _libtorch_python_sources = [gencode_pattern.format(name) for name in [ - "autograd/generated/python_functions_0.cpp", - "autograd/generated/python_functions_1.cpp", - "autograd/generated/python_functions_2.cpp", - "autograd/generated/python_functions_3.cpp", - "autograd/generated/python_functions_4.cpp", - "autograd/generated/python_nn_functions.cpp", - "autograd/generated/python_fft_functions.cpp", - "autograd/generated/python_linalg_functions.cpp", - "autograd/generated/python_return_types.cpp", - "autograd/generated/python_sparse_functions.cpp", - "autograd/generated/python_special_functions.cpp", - "autograd/generated/python_torch_functions_0.cpp", - "autograd/generated/python_torch_functions_1.cpp", - "autograd/generated/python_torch_functions_2.cpp", - "autograd/generated/python_variable_methods.cpp", + "torch/csrc/autograd/generated/python_functions_0.cpp", + "torch/csrc/autograd/generated/python_functions_1.cpp", + "torch/csrc/autograd/generated/python_functions_2.cpp", + "torch/csrc/autograd/generated/python_functions_3.cpp", + "torch/csrc/autograd/generated/python_functions_4.cpp", + "torch/csrc/autograd/generated/python_nn_functions.cpp", + "torch/csrc/autograd/generated/python_fft_functions.cpp", + "torch/csrc/autograd/generated/python_linalg_functions.cpp", + "torch/csrc/autograd/generated/python_return_types.cpp", + "torch/csrc/autograd/generated/python_sparse_functions.cpp", + "torch/csrc/autograd/generated/python_special_functions.cpp", + "torch/csrc/autograd/generated/python_torch_functions_0.cpp", + "torch/csrc/autograd/generated/python_torch_functions_1.cpp", + "torch/csrc/autograd/generated/python_torch_functions_2.cpp", + "torch/csrc/autograd/generated/python_variable_methods.cpp", ]] _libtorch_python_sources.extend(libtorch_python_core_sources) diff --git a/tools/setup_helpers/generate_code.py b/tools/setup_helpers/generate_code.py index 28ced421276..4440e6c2e0a 100644 --- a/tools/setup_helpers/generate_code.py +++ b/tools/setup_helpers/generate_code.py @@ -16,6 +16,7 @@ TAGS_PATH = "aten/src/ATen/native/tags.yaml" def generate_code( + gen_dir: pathlib.Path, native_functions_path: Optional[str] = None, tags_path: Optional[str] = None, install_dir: Optional[str] = None, @@ -30,8 +31,8 @@ def generate_code( # Build ATen based Variable classes if install_dir is None: - install_dir = "torch/csrc" - python_install_dir = "torch/testing/_internal/generated" + install_dir = os.fspath(gen_dir / "torch/csrc") + python_install_dir = os.fspath(gen_dir / "torch/testing/_internal/generated") else: python_install_dir = install_dir autograd_gen_dir = os.path.join(install_dir, "autograd", "generated") @@ -128,7 +129,19 @@ def main() -> None: parser = argparse.ArgumentParser(description="Autogenerate code") parser.add_argument("--native-functions-path") parser.add_argument("--tags-path") - parser.add_argument("--install_dir") + parser.add_argument( + "--gen-dir", + type=pathlib.Path, + default=pathlib.Path("."), + help="Root directory where to install files. Defaults to the current working directory.", + ) + parser.add_argument( + "--install_dir", + help=( + "Deprecated. Use --gen-dir instead. The semantics are different, do not change " + "blindly." + ), + ) parser.add_argument( "--subset", help='Subset of source files to generate. Can be "libtorch" or "pybindings". Generates both when omitted.', @@ -166,6 +179,7 @@ def main() -> None: options = parser.parse_args() generate_code( + options.gen_dir, options.native_functions_path, options.tags_path, options.install_dir, @@ -183,9 +197,8 @@ def main() -> None: ts_backend_yaml = os.path.join(aten_path, "native/ts_native_functions.yaml") ts_native_functions = "torch/csrc/lazy/ts_backend/ts_native_functions.cpp" ts_node_base = "torch/csrc/lazy/ts_backend/ts_node.h" - if options.install_dir is None: - options.install_dir = "torch/csrc" - lazy_install_dir = os.path.join(options.install_dir, "lazy/generated") + install_dir = options.install_dir or os.fspath(options.gen_dir / "torch/csrc") + lazy_install_dir = os.path.join(install_dir, "lazy/generated") os.makedirs(lazy_install_dir, exist_ok=True) assert os.path.isfile(