diff --git a/build.bzl b/build.bzl index fac3d081ff4..247e4097d16 100644 --- a/build.bzl +++ b/build.bzl @@ -38,7 +38,7 @@ def define_targets(rules): tools = ["//tools/setup_helpers:generate_code"], outs = _GENERATED_CPP + GENERATED_AUTOGRAD_H + GENERATED_LAZY_H + GENERATED_TESTING_PY, cmd = "$(location //tools/setup_helpers:generate_code) " + - "--install_dir $(RULEDIR) " + + "--gen-dir=$(RULEDIR) " + "--native-functions-path $(location :native_functions.yaml) " + "--tags-path=$(location :tags.yaml) " + "--gen_lazy_ts_backend", @@ -72,55 +72,49 @@ def define_targets(rules): # generate-code that use these lists are moved into the shared # structure as well. -# In the open-source build, these are generated into -# torch/csrc/autograd/generated GENERATED_AUTOGRAD_H = [ - "autograd/generated/Functions.h", - "autograd/generated/VariableType.h", - "autograd/generated/python_functions.h", - "autograd/generated/variable_factories.h", + "torch/csrc/autograd/generated/Functions.h", + "torch/csrc/autograd/generated/VariableType.h", + "torch/csrc/autograd/generated/python_functions.h", + "torch/csrc/autograd/generated/variable_factories.h", ] -# In the open-source build, these are generated into -# torch/testing/_internal/generated GENERATED_TESTING_PY = [ - "annotated_fn_args.py", + "torch/testing/_internal/generated/annotated_fn_args.py", ] GENERATED_LAZY_H = [ - "lazy/generated/LazyIr.h", - "lazy/generated/LazyNativeFunctions.h", + "torch/csrc/lazy/generated/LazyIr.h", + "torch/csrc/lazy/generated/LazyNativeFunctions.h", ] -# In both open-source and fbcode builds, these are generated into -# torch/csrc/{autograd,jit}/generated.i _GENERATED_CPP = [ - "autograd/generated/Functions.cpp", - "autograd/generated/VariableType_0.cpp", - "autograd/generated/VariableType_1.cpp", - "autograd/generated/VariableType_2.cpp", - "autograd/generated/VariableType_3.cpp", - "autograd/generated/VariableType_4.cpp", - "autograd/generated/TraceType_0.cpp", - "autograd/generated/TraceType_1.cpp", - "autograd/generated/TraceType_2.cpp", - "autograd/generated/TraceType_3.cpp", - "autograd/generated/TraceType_4.cpp", - "autograd/generated/ADInplaceOrViewType_0.cpp", - "autograd/generated/ADInplaceOrViewType_1.cpp", - "autograd/generated/python_functions_0.cpp", - "autograd/generated/python_functions_1.cpp", - "autograd/generated/python_functions_2.cpp", - "autograd/generated/python_functions_3.cpp", - "autograd/generated/python_functions_4.cpp", - "autograd/generated/python_nn_functions.cpp", - "autograd/generated/python_fft_functions.cpp", - "autograd/generated/python_linalg_functions.cpp", - "autograd/generated/python_return_types.cpp", - "autograd/generated/python_sparse_functions.cpp", - "autograd/generated/python_special_functions.cpp", - "autograd/generated/python_torch_functions_0.cpp", - "autograd/generated/python_torch_functions_1.cpp", - "autograd/generated/python_torch_functions_2.cpp", - "autograd/generated/python_variable_methods.cpp", + "torch/csrc/autograd/generated/Functions.cpp", + "torch/csrc/autograd/generated/VariableType_0.cpp", + "torch/csrc/autograd/generated/VariableType_1.cpp", + "torch/csrc/autograd/generated/VariableType_2.cpp", + "torch/csrc/autograd/generated/VariableType_3.cpp", + "torch/csrc/autograd/generated/VariableType_4.cpp", + "torch/csrc/autograd/generated/TraceType_0.cpp", + "torch/csrc/autograd/generated/TraceType_1.cpp", + "torch/csrc/autograd/generated/TraceType_2.cpp", + "torch/csrc/autograd/generated/TraceType_3.cpp", + "torch/csrc/autograd/generated/TraceType_4.cpp", + "torch/csrc/autograd/generated/ADInplaceOrViewType_0.cpp", + "torch/csrc/autograd/generated/ADInplaceOrViewType_1.cpp", + "torch/csrc/autograd/generated/python_functions_0.cpp", + "torch/csrc/autograd/generated/python_functions_1.cpp", + "torch/csrc/autograd/generated/python_functions_2.cpp", + "torch/csrc/autograd/generated/python_functions_3.cpp", + "torch/csrc/autograd/generated/python_functions_4.cpp", + "torch/csrc/autograd/generated/python_nn_functions.cpp", + "torch/csrc/autograd/generated/python_fft_functions.cpp", + "torch/csrc/autograd/generated/python_linalg_functions.cpp", + "torch/csrc/autograd/generated/python_return_types.cpp", + "torch/csrc/autograd/generated/python_sparse_functions.cpp", + "torch/csrc/autograd/generated/python_special_functions.cpp", + "torch/csrc/autograd/generated/python_torch_functions_0.cpp", + "torch/csrc/autograd/generated/python_torch_functions_1.cpp", + "torch/csrc/autograd/generated/python_torch_functions_2.cpp", + "torch/csrc/autograd/generated/python_variable_methods.cpp", ] diff --git a/tools/build_variables.bzl b/tools/build_variables.bzl index 09cf9ae5d18..3e8d5f5ef7d 100644 --- a/tools/build_variables.bzl +++ b/tools/build_variables.bzl @@ -46,19 +46,19 @@ libtorch_nvfuser_generated_headers = ["{}.h".format(name.split("/")[-1].split(". def libtorch_generated_sources(gencode_pattern): return [gencode_pattern.format(name) for name in [ - "autograd/generated/Functions.cpp", - "autograd/generated/VariableType_0.cpp", - "autograd/generated/VariableType_1.cpp", - "autograd/generated/VariableType_2.cpp", - "autograd/generated/VariableType_3.cpp", - "autograd/generated/VariableType_4.cpp", - "autograd/generated/TraceType_0.cpp", - "autograd/generated/TraceType_1.cpp", - "autograd/generated/TraceType_2.cpp", - "autograd/generated/TraceType_3.cpp", - "autograd/generated/TraceType_4.cpp", - "autograd/generated/ADInplaceOrViewType_0.cpp", - "autograd/generated/ADInplaceOrViewType_1.cpp", + "torch/csrc/autograd/generated/Functions.cpp", + "torch/csrc/autograd/generated/VariableType_0.cpp", + "torch/csrc/autograd/generated/VariableType_1.cpp", + "torch/csrc/autograd/generated/VariableType_2.cpp", + "torch/csrc/autograd/generated/VariableType_3.cpp", + "torch/csrc/autograd/generated/VariableType_4.cpp", + "torch/csrc/autograd/generated/TraceType_0.cpp", + "torch/csrc/autograd/generated/TraceType_1.cpp", + "torch/csrc/autograd/generated/TraceType_2.cpp", + "torch/csrc/autograd/generated/TraceType_3.cpp", + "torch/csrc/autograd/generated/TraceType_4.cpp", + "torch/csrc/autograd/generated/ADInplaceOrViewType_0.cpp", + "torch/csrc/autograd/generated/ADInplaceOrViewType_1.cpp", ]] # copied from https://github.com/pytorch/pytorch/blob/f99a693cd9ff7a9b5fdc71357dac66b8192786d3/aten/src/ATen/core/CMakeLists.txt @@ -960,21 +960,21 @@ libtorch_python_distributed_sources = libtorch_python_distributed_core_sources + def glob_libtorch_python_sources(gencode_pattern = ":generate-code[{}]"): _libtorch_python_sources = [gencode_pattern.format(name) for name in [ - "autograd/generated/python_functions_0.cpp", - "autograd/generated/python_functions_1.cpp", - "autograd/generated/python_functions_2.cpp", - "autograd/generated/python_functions_3.cpp", - "autograd/generated/python_functions_4.cpp", - "autograd/generated/python_nn_functions.cpp", - "autograd/generated/python_fft_functions.cpp", - "autograd/generated/python_linalg_functions.cpp", - "autograd/generated/python_return_types.cpp", - "autograd/generated/python_sparse_functions.cpp", - "autograd/generated/python_special_functions.cpp", - "autograd/generated/python_torch_functions_0.cpp", - "autograd/generated/python_torch_functions_1.cpp", - "autograd/generated/python_torch_functions_2.cpp", - "autograd/generated/python_variable_methods.cpp", + "torch/csrc/autograd/generated/python_functions_0.cpp", + "torch/csrc/autograd/generated/python_functions_1.cpp", + "torch/csrc/autograd/generated/python_functions_2.cpp", + "torch/csrc/autograd/generated/python_functions_3.cpp", + "torch/csrc/autograd/generated/python_functions_4.cpp", + "torch/csrc/autograd/generated/python_nn_functions.cpp", + "torch/csrc/autograd/generated/python_fft_functions.cpp", + "torch/csrc/autograd/generated/python_linalg_functions.cpp", + "torch/csrc/autograd/generated/python_return_types.cpp", + "torch/csrc/autograd/generated/python_sparse_functions.cpp", + "torch/csrc/autograd/generated/python_special_functions.cpp", + "torch/csrc/autograd/generated/python_torch_functions_0.cpp", + "torch/csrc/autograd/generated/python_torch_functions_1.cpp", + "torch/csrc/autograd/generated/python_torch_functions_2.cpp", + "torch/csrc/autograd/generated/python_variable_methods.cpp", ]] _libtorch_python_sources.extend(libtorch_python_core_sources) diff --git a/tools/setup_helpers/generate_code.py b/tools/setup_helpers/generate_code.py index 28ced421276..4440e6c2e0a 100644 --- a/tools/setup_helpers/generate_code.py +++ b/tools/setup_helpers/generate_code.py @@ -16,6 +16,7 @@ TAGS_PATH = "aten/src/ATen/native/tags.yaml" def generate_code( + gen_dir: pathlib.Path, native_functions_path: Optional[str] = None, tags_path: Optional[str] = None, install_dir: Optional[str] = None, @@ -30,8 +31,8 @@ def generate_code( # Build ATen based Variable classes if install_dir is None: - install_dir = "torch/csrc" - python_install_dir = "torch/testing/_internal/generated" + install_dir = os.fspath(gen_dir / "torch/csrc") + python_install_dir = os.fspath(gen_dir / "torch/testing/_internal/generated") else: python_install_dir = install_dir autograd_gen_dir = os.path.join(install_dir, "autograd", "generated") @@ -128,7 +129,19 @@ def main() -> None: parser = argparse.ArgumentParser(description="Autogenerate code") parser.add_argument("--native-functions-path") parser.add_argument("--tags-path") - parser.add_argument("--install_dir") + parser.add_argument( + "--gen-dir", + type=pathlib.Path, + default=pathlib.Path("."), + help="Root directory where to install files. Defaults to the current working directory.", + ) + parser.add_argument( + "--install_dir", + help=( + "Deprecated. Use --gen-dir instead. The semantics are different, do not change " + "blindly." + ), + ) parser.add_argument( "--subset", help='Subset of source files to generate. Can be "libtorch" or "pybindings". Generates both when omitted.', @@ -166,6 +179,7 @@ def main() -> None: options = parser.parse_args() generate_code( + options.gen_dir, options.native_functions_path, options.tags_path, options.install_dir, @@ -183,9 +197,8 @@ def main() -> None: ts_backend_yaml = os.path.join(aten_path, "native/ts_native_functions.yaml") ts_native_functions = "torch/csrc/lazy/ts_backend/ts_native_functions.cpp" ts_node_base = "torch/csrc/lazy/ts_backend/ts_node.h" - if options.install_dir is None: - options.install_dir = "torch/csrc" - lazy_install_dir = os.path.join(options.install_dir, "lazy/generated") + install_dir = options.install_dir or os.fspath(options.gen_dir / "torch/csrc") + lazy_install_dir = os.path.join(install_dir, "lazy/generated") os.makedirs(lazy_install_dir, exist_ok=True) assert os.path.isfile(