saymrwulf/pytorch
1
0
Fork 0
mirror of https://github.com/saymrwulf/pytorch.git synced 2026-05-14 20:57:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

introduce new --gen-dir flag to generate_code and use it in fbcode (#75800)

Browse source 
Summary:
Pull Request resolved: https://github.com/pytorch/pytorch/pull/75800

This leads to more similarities between OSS CMake and eventually OSS
Bazel. We will be able to generate files with the same names and not
have different file lists between the builds.
ghstack-source-id: 155300043

Test Plan: Verified locally and in CI.

Reviewed By: dreiss

Differential Revision: D35648586

fbshipit-source-id: 9f1638b5665ebcc64466883f65ef24a2bfd05228
(cherry picked from commit 7f2acff1baa8dfafddefdc720714f8d39feda436)
This commit is contained in:
mikey dagitses 2022-05-04 06:48:20 -07:00 committed by PyTorch MergeBot
parent dfde877c0b
commit 096ff0ecca
3 changed files with 83 additions and 76 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

78
build.bzl
View file

@ -38,7 +38,7 @@ def define_targets(rules):
tools = ["//tools/setup_helpers:generate_code"],
outs = _GENERATED_CPP + GENERATED_AUTOGRAD_H + GENERATED_LAZY_H + GENERATED_TESTING_PY,
cmd = "$(location //tools/setup_helpers:generate_code) " +
"--install_dir $(RULEDIR) " +
"--gen-dir=$(RULEDIR) " +
"--native-functions-path $(location :native_functions.yaml) " +
"--tags-path=$(location :tags.yaml) " +
"--gen_lazy_ts_backend",
@ -72,55 +72,49 @@ def define_targets(rules):
# generate-code that use these lists are moved into the shared
# structure as well.
# In the open-source build, these are generated into
# torch/csrc/autograd/generated
GENERATED_AUTOGRAD_H = [
"autograd/generated/Functions.h",
"autograd/generated/VariableType.h",
"autograd/generated/python_functions.h",
"autograd/generated/variable_factories.h",
"torch/csrc/autograd/generated/Functions.h",
"torch/csrc/autograd/generated/VariableType.h",
"torch/csrc/autograd/generated/python_functions.h",
"torch/csrc/autograd/generated/variable_factories.h",
]
# In the open-source build, these are generated into
# torch/testing/_internal/generated
GENERATED_TESTING_PY = [
"annotated_fn_args.py",
"torch/testing/_internal/generated/annotated_fn_args.py",
]
GENERATED_LAZY_H = [
"lazy/generated/LazyIr.h",
"lazy/generated/LazyNativeFunctions.h",
"torch/csrc/lazy/generated/LazyIr.h",
"torch/csrc/lazy/generated/LazyNativeFunctions.h",
]
# In both open-source and fbcode builds, these are generated into
# torch/csrc/{autograd,jit}/generated.i
_GENERATED_CPP = [
"autograd/generated/Functions.cpp",
"autograd/generated/VariableType_0.cpp",
"autograd/generated/VariableType_1.cpp",
"autograd/generated/VariableType_2.cpp",
"autograd/generated/VariableType_3.cpp",
"autograd/generated/VariableType_4.cpp",
"autograd/generated/TraceType_0.cpp",
"autograd/generated/TraceType_1.cpp",
"autograd/generated/TraceType_2.cpp",
"autograd/generated/TraceType_3.cpp",
"autograd/generated/TraceType_4.cpp",
"autograd/generated/ADInplaceOrViewType_0.cpp",
"autograd/generated/ADInplaceOrViewType_1.cpp",
"autograd/generated/python_functions_0.cpp",
"autograd/generated/python_functions_1.cpp",
"autograd/generated/python_functions_2.cpp",
"autograd/generated/python_functions_3.cpp",
"autograd/generated/python_functions_4.cpp",
"autograd/generated/python_nn_functions.cpp",
"autograd/generated/python_fft_functions.cpp",
"autograd/generated/python_linalg_functions.cpp",
"autograd/generated/python_return_types.cpp",
"autograd/generated/python_sparse_functions.cpp",
"autograd/generated/python_special_functions.cpp",
"autograd/generated/python_torch_functions_0.cpp",
"autograd/generated/python_torch_functions_1.cpp",
"autograd/generated/python_torch_functions_2.cpp",
"autograd/generated/python_variable_methods.cpp",
"torch/csrc/autograd/generated/Functions.cpp",
"torch/csrc/autograd/generated/VariableType_0.cpp",
"torch/csrc/autograd/generated/VariableType_1.cpp",
"torch/csrc/autograd/generated/VariableType_2.cpp",
"torch/csrc/autograd/generated/VariableType_3.cpp",
"torch/csrc/autograd/generated/VariableType_4.cpp",
"torch/csrc/autograd/generated/TraceType_0.cpp",
"torch/csrc/autograd/generated/TraceType_1.cpp",
"torch/csrc/autograd/generated/TraceType_2.cpp",
"torch/csrc/autograd/generated/TraceType_3.cpp",
"torch/csrc/autograd/generated/TraceType_4.cpp",
"torch/csrc/autograd/generated/ADInplaceOrViewType_0.cpp",
"torch/csrc/autograd/generated/ADInplaceOrViewType_1.cpp",
"torch/csrc/autograd/generated/python_functions_0.cpp",
"torch/csrc/autograd/generated/python_functions_1.cpp",
"torch/csrc/autograd/generated/python_functions_2.cpp",
"torch/csrc/autograd/generated/python_functions_3.cpp",
"torch/csrc/autograd/generated/python_functions_4.cpp",
"torch/csrc/autograd/generated/python_nn_functions.cpp",
"torch/csrc/autograd/generated/python_fft_functions.cpp",
"torch/csrc/autograd/generated/python_linalg_functions.cpp",
"torch/csrc/autograd/generated/python_return_types.cpp",
"torch/csrc/autograd/generated/python_sparse_functions.cpp",
"torch/csrc/autograd/generated/python_special_functions.cpp",
"torch/csrc/autograd/generated/python_torch_functions_0.cpp",
"torch/csrc/autograd/generated/python_torch_functions_1.cpp",
"torch/csrc/autograd/generated/python_torch_functions_2.cpp",
"torch/csrc/autograd/generated/python_variable_methods.cpp",
]

56
tools/build_variables.bzl
View file

@ -46,19 +46,19 @@ libtorch_nvfuser_generated_headers = ["{}.h".format(name.split("/")[-1].split(".
def libtorch_generated_sources(gencode_pattern):
return [gencode_pattern.format(name) for name in [
"autograd/generated/Functions.cpp",
"autograd/generated/VariableType_0.cpp",
"autograd/generated/VariableType_1.cpp",
"autograd/generated/VariableType_2.cpp",
"autograd/generated/VariableType_3.cpp",
"autograd/generated/VariableType_4.cpp",
"autograd/generated/TraceType_0.cpp",
"autograd/generated/TraceType_1.cpp",
"autograd/generated/TraceType_2.cpp",
"autograd/generated/TraceType_3.cpp",
"autograd/generated/TraceType_4.cpp",
"autograd/generated/ADInplaceOrViewType_0.cpp",
"autograd/generated/ADInplaceOrViewType_1.cpp",
"torch/csrc/autograd/generated/Functions.cpp",
"torch/csrc/autograd/generated/VariableType_0.cpp",
"torch/csrc/autograd/generated/VariableType_1.cpp",
"torch/csrc/autograd/generated/VariableType_2.cpp",
"torch/csrc/autograd/generated/VariableType_3.cpp",
"torch/csrc/autograd/generated/VariableType_4.cpp",
"torch/csrc/autograd/generated/TraceType_0.cpp",
"torch/csrc/autograd/generated/TraceType_1.cpp",
"torch/csrc/autograd/generated/TraceType_2.cpp",
"torch/csrc/autograd/generated/TraceType_3.cpp",
"torch/csrc/autograd/generated/TraceType_4.cpp",
"torch/csrc/autograd/generated/ADInplaceOrViewType_0.cpp",
"torch/csrc/autograd/generated/ADInplaceOrViewType_1.cpp",
]]
# copied from https://github.com/pytorch/pytorch/blob/f99a693cd9ff7a9b5fdc71357dac66b8192786d3/aten/src/ATen/core/CMakeLists.txt
@ -960,21 +960,21 @@ libtorch_python_distributed_sources = libtorch_python_distributed_core_sources +
def glob_libtorch_python_sources(gencode_pattern = ":generate-code[{}]"):
_libtorch_python_sources = [gencode_pattern.format(name) for name in [
"autograd/generated/python_functions_0.cpp",
"autograd/generated/python_functions_1.cpp",
"autograd/generated/python_functions_2.cpp",
"autograd/generated/python_functions_3.cpp",
"autograd/generated/python_functions_4.cpp",
"autograd/generated/python_nn_functions.cpp",
"autograd/generated/python_fft_functions.cpp",
"autograd/generated/python_linalg_functions.cpp",
"autograd/generated/python_return_types.cpp",
"autograd/generated/python_sparse_functions.cpp",
"autograd/generated/python_special_functions.cpp",
"autograd/generated/python_torch_functions_0.cpp",
"autograd/generated/python_torch_functions_1.cpp",
"autograd/generated/python_torch_functions_2.cpp",
"autograd/generated/python_variable_methods.cpp",
"torch/csrc/autograd/generated/python_functions_0.cpp",
"torch/csrc/autograd/generated/python_functions_1.cpp",
"torch/csrc/autograd/generated/python_functions_2.cpp",
"torch/csrc/autograd/generated/python_functions_3.cpp",
"torch/csrc/autograd/generated/python_functions_4.cpp",
"torch/csrc/autograd/generated/python_nn_functions.cpp",
"torch/csrc/autograd/generated/python_fft_functions.cpp",
"torch/csrc/autograd/generated/python_linalg_functions.cpp",
"torch/csrc/autograd/generated/python_return_types.cpp",
"torch/csrc/autograd/generated/python_sparse_functions.cpp",
"torch/csrc/autograd/generated/python_special_functions.cpp",
"torch/csrc/autograd/generated/python_torch_functions_0.cpp",
"torch/csrc/autograd/generated/python_torch_functions_1.cpp",
"torch/csrc/autograd/generated/python_torch_functions_2.cpp",
"torch/csrc/autograd/generated/python_variable_methods.cpp",
]]
_libtorch_python_sources.extend(libtorch_python_core_sources)

25
tools/setup_helpers/generate_code.py
View file

@ -16,6 +16,7 @@ TAGS_PATH = "aten/src/ATen/native/tags.yaml"
def generate_code(
gen_dir: pathlib.Path,
native_functions_path: Optional[str] = None,
tags_path: Optional[str] = None,
install_dir: Optional[str] = None,
@ -30,8 +31,8 @@ def generate_code(
# Build ATen based Variable classes
if install_dir is None:
install_dir = "torch/csrc"
python_install_dir = "torch/testing/_internal/generated"
install_dir = os.fspath(gen_dir / "torch/csrc")
python_install_dir = os.fspath(gen_dir / "torch/testing/_internal/generated")
else:
python_install_dir = install_dir
autograd_gen_dir = os.path.join(install_dir, "autograd", "generated")
@ -128,7 +129,19 @@ def main() -> None:
parser = argparse.ArgumentParser(description="Autogenerate code")
parser.add_argument("--native-functions-path")
parser.add_argument("--tags-path")
parser.add_argument("--install_dir")
parser.add_argument(
"--gen-dir",
type=pathlib.Path,
default=pathlib.Path("."),
help="Root directory where to install files. Defaults to the current working directory.",
)
parser.add_argument(
"--install_dir",
help=(
"Deprecated. Use --gen-dir instead. The semantics are different, do not change "
"blindly."
),
)
parser.add_argument(
"--subset",
help='Subset of source files to generate. Can be "libtorch" or "pybindings". Generates both when omitted.',
@ -166,6 +179,7 @@ def main() -> None:
options = parser.parse_args()
generate_code(
options.gen_dir,
options.native_functions_path,
options.tags_path,
options.install_dir,
@ -183,9 +197,8 @@ def main() -> None:
ts_backend_yaml = os.path.join(aten_path, "native/ts_native_functions.yaml")
ts_native_functions = "torch/csrc/lazy/ts_backend/ts_native_functions.cpp"
ts_node_base = "torch/csrc/lazy/ts_backend/ts_node.h"
if options.install_dir is None:
options.install_dir = "torch/csrc"
lazy_install_dir = os.path.join(options.install_dir, "lazy/generated")
install_dir = options.install_dir or os.fspath(options.gen_dir / "torch/csrc")
lazy_install_dir = os.path.join(install_dir, "lazy/generated")
os.makedirs(lazy_install_dir, exist_ok=True)
assert os.path.isfile(