From eb18a2f26c7c7d8cd9826ab0cbd40bd541633805 Mon Sep 17 00:00:00 2001 From: "Edward Z. Yang" Date: Tue, 27 Mar 2018 22:09:20 -0400 Subject: [PATCH] Reorganize third-party libraries into top-level third_party directory (#6025) - gloo, pybind11, nanopb and nccl now live in third_party. - ATen builds in aten/build rather than torch/lib/build/aten - A bit of faffing about in the scripts was necessary, because they used to assume that everything lived in the same directory. Now you are expected to cd into the correct directory before calling one of the build functions. The actual builder script lives in tools - Lint now just unconditionally ignores third_party, rather than enumerating folders explicitly --- .gitmodules | 6 +- {torch/lib => aten/src}/README.md | 6 +- aten/tools/subtree_dir_add.sh | 17 ---- aten/tools/subtree_dir_merge.sh | 17 ---- aten/tools/subtree_dir_merge_all.sh | 10 --- setup.py | 17 ++-- third_party/README.md | 2 + {torch/lib => third_party}/gloo | 0 {torch/lib => third_party}/nanopb | 0 {torch/lib => third_party}/nccl/.gitignore | 0 .../lib => third_party}/nccl/CMakeLists.txt | 0 {torch/lib => third_party}/nccl/LICENSE.txt | 0 {torch/lib => third_party}/nccl/Makefile | 0 {torch/lib => third_party}/nccl/README.md | 0 .../nccl/debian/.gitignore | 0 .../nccl/debian/changelog.in | 0 {torch/lib => third_party}/nccl/debian/compat | 0 .../nccl/debian/control.in | 0 .../lib => third_party}/nccl/debian/copyright | 0 .../nccl/debian/libnccl-dev.install | 0 .../nccl/debian/libnccl-dev.manpages | 0 .../nccl/debian/libnccl1.install.in | 0 {torch/lib => third_party}/nccl/debian/nccl.7 | 0 {torch/lib => third_party}/nccl/debian/rules | 0 .../nccl/debian/shlibs.local.in | 0 .../nccl/debian/source/format | 0 .../lib => third_party}/nccl/fortran/Makefile | 0 .../nccl/fortran/src/cudafor.f90 | 0 .../nccl/fortran/src/ncclfor.f90 | 0 .../nccl/fortran/test/allgather_arr_out.f90 | 0 .../nccl/fortran/test/allgather_ptr_out.f90 | 0 .../nccl/fortran/test/allreduce_arr_out.f90 | 0 .../nccl/fortran/test/allreduce_ptr_out.f90 | 0 .../nccl/fortran/test/broadcast_arr.f90 | 0 .../nccl/fortran/test/broadcast_ptr.f90 | 0 .../nccl/fortran/test/reduce_arr_out.f90 | 0 .../nccl/fortran/test/reduce_ptr_out.f90 | 0 .../fortran/test/reducescatter_arr_out.f90 | 0 .../fortran/test/reducescatter_ptr_out.f90 | 0 .../nccl/src/all_gather.cu | 0 .../nccl/src/all_reduce.cu | 0 .../lib => third_party}/nccl/src/broadcast.cu | 0 .../nccl/src/common_coll.h | 0 .../nccl/src/common_kernel.h | 0 .../nccl/src/copy_kernel.h | 0 {torch/lib => third_party}/nccl/src/core.cu | 0 {torch/lib => third_party}/nccl/src/core.h | 0 {torch/lib => third_party}/nccl/src/enqueue.h | 0 .../lib => third_party}/nccl/src/libwrap.cu | 0 {torch/lib => third_party}/nccl/src/libwrap.h | 0 {torch/lib => third_party}/nccl/src/nccl.h | 0 .../lib => third_party}/nccl/src/primitives.h | 0 {torch/lib => third_party}/nccl/src/reduce.cu | 0 .../nccl/src/reduce_kernel.h | 0 .../nccl/src/reduce_scatter.cu | 0 .../nccl/test/include/test_utilities.h | 0 .../nccl/test/mpi/mpi_test.cu | 0 .../nccl/test/single/all_gather_scan.cu | 0 .../nccl/test/single/all_gather_test.cu | 0 .../nccl/test/single/all_reduce_scan.cu | 0 .../nccl/test/single/all_reduce_test.cu | 0 .../nccl/test/single/broadcast_scan.cu | 0 .../nccl/test/single/broadcast_test.cu | 0 .../nccl/test/single/reduce_scan.cu | 0 .../nccl/test/single/reduce_scatter_scan.cu | 0 .../nccl/test/single/reduce_scatter_test.cu | 0 .../nccl/test/single/reduce_test.cu | 0 {torch/lib => third_party}/pybind11 | 0 .../build_pytorch_libs.bat | 40 ++++++--- .../build_pytorch_libs.sh | 84 ++++++++++++------- tools/cpp_build/build_all.sh | 0 tools/cpp_build/libtorch/CMakeLists.txt | 2 +- tox.ini | 2 +- 73 files changed, 102 insertions(+), 101 deletions(-) rename {torch/lib => aten/src}/README.md (97%) delete mode 100755 aten/tools/subtree_dir_add.sh delete mode 100755 aten/tools/subtree_dir_merge.sh delete mode 100755 aten/tools/subtree_dir_merge_all.sh create mode 100644 third_party/README.md rename {torch/lib => third_party}/gloo (100%) rename {torch/lib => third_party}/nanopb (100%) rename {torch/lib => third_party}/nccl/.gitignore (100%) rename {torch/lib => third_party}/nccl/CMakeLists.txt (100%) rename {torch/lib => third_party}/nccl/LICENSE.txt (100%) rename {torch/lib => third_party}/nccl/Makefile (100%) rename {torch/lib => third_party}/nccl/README.md (100%) rename {torch/lib => third_party}/nccl/debian/.gitignore (100%) rename {torch/lib => third_party}/nccl/debian/changelog.in (100%) rename {torch/lib => third_party}/nccl/debian/compat (100%) rename {torch/lib => third_party}/nccl/debian/control.in (100%) rename {torch/lib => third_party}/nccl/debian/copyright (100%) rename {torch/lib => third_party}/nccl/debian/libnccl-dev.install (100%) rename {torch/lib => third_party}/nccl/debian/libnccl-dev.manpages (100%) rename {torch/lib => third_party}/nccl/debian/libnccl1.install.in (100%) rename {torch/lib => third_party}/nccl/debian/nccl.7 (100%) rename {torch/lib => third_party}/nccl/debian/rules (100%) rename {torch/lib => third_party}/nccl/debian/shlibs.local.in (100%) rename {torch/lib => third_party}/nccl/debian/source/format (100%) rename {torch/lib => third_party}/nccl/fortran/Makefile (100%) rename {torch/lib => third_party}/nccl/fortran/src/cudafor.f90 (100%) rename {torch/lib => third_party}/nccl/fortran/src/ncclfor.f90 (100%) rename {torch/lib => third_party}/nccl/fortran/test/allgather_arr_out.f90 (100%) rename {torch/lib => third_party}/nccl/fortran/test/allgather_ptr_out.f90 (100%) rename {torch/lib => third_party}/nccl/fortran/test/allreduce_arr_out.f90 (100%) rename {torch/lib => third_party}/nccl/fortran/test/allreduce_ptr_out.f90 (100%) rename {torch/lib => third_party}/nccl/fortran/test/broadcast_arr.f90 (100%) rename {torch/lib => third_party}/nccl/fortran/test/broadcast_ptr.f90 (100%) rename {torch/lib => third_party}/nccl/fortran/test/reduce_arr_out.f90 (100%) rename {torch/lib => third_party}/nccl/fortran/test/reduce_ptr_out.f90 (100%) rename {torch/lib => third_party}/nccl/fortran/test/reducescatter_arr_out.f90 (100%) rename {torch/lib => third_party}/nccl/fortran/test/reducescatter_ptr_out.f90 (100%) rename {torch/lib => third_party}/nccl/src/all_gather.cu (100%) rename {torch/lib => third_party}/nccl/src/all_reduce.cu (100%) rename {torch/lib => third_party}/nccl/src/broadcast.cu (100%) rename {torch/lib => third_party}/nccl/src/common_coll.h (100%) rename {torch/lib => third_party}/nccl/src/common_kernel.h (100%) rename {torch/lib => third_party}/nccl/src/copy_kernel.h (100%) rename {torch/lib => third_party}/nccl/src/core.cu (100%) rename {torch/lib => third_party}/nccl/src/core.h (100%) rename {torch/lib => third_party}/nccl/src/enqueue.h (100%) rename {torch/lib => third_party}/nccl/src/libwrap.cu (100%) rename {torch/lib => third_party}/nccl/src/libwrap.h (100%) rename {torch/lib => third_party}/nccl/src/nccl.h (100%) rename {torch/lib => third_party}/nccl/src/primitives.h (100%) rename {torch/lib => third_party}/nccl/src/reduce.cu (100%) rename {torch/lib => third_party}/nccl/src/reduce_kernel.h (100%) rename {torch/lib => third_party}/nccl/src/reduce_scatter.cu (100%) rename {torch/lib => third_party}/nccl/test/include/test_utilities.h (100%) rename {torch/lib => third_party}/nccl/test/mpi/mpi_test.cu (100%) rename {torch/lib => third_party}/nccl/test/single/all_gather_scan.cu (100%) rename {torch/lib => third_party}/nccl/test/single/all_gather_test.cu (100%) rename {torch/lib => third_party}/nccl/test/single/all_reduce_scan.cu (100%) rename {torch/lib => third_party}/nccl/test/single/all_reduce_test.cu (100%) rename {torch/lib => third_party}/nccl/test/single/broadcast_scan.cu (100%) rename {torch/lib => third_party}/nccl/test/single/broadcast_test.cu (100%) rename {torch/lib => third_party}/nccl/test/single/reduce_scan.cu (100%) rename {torch/lib => third_party}/nccl/test/single/reduce_scatter_scan.cu (100%) rename {torch/lib => third_party}/nccl/test/single/reduce_scatter_test.cu (100%) rename {torch/lib => third_party}/nccl/test/single/reduce_test.cu (100%) rename {torch/lib => third_party}/pybind11 (100%) rename torch/lib/build_libs.bat => tools/build_pytorch_libs.bat (89%) rename torch/lib/build_libs.sh => tools/build_pytorch_libs.sh (82%) mode change 100644 => 100755 tools/cpp_build/build_all.sh diff --git a/.gitmodules b/.gitmodules index b412d87c4c7..8720a6a9aa0 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,11 +1,11 @@ [submodule "torch/lib/gloo"] - path = torch/lib/gloo + path = third_party/gloo url = https://github.com/facebookincubator/gloo [submodule "torch/lib/pybind11"] - path = torch/lib/pybind11 + path = third_party/pybind11 url = https://github.com/pybind/pybind11 [submodule "torch/lib/nanopb"] - path = torch/lib/nanopb + path = third_party/nanopb url = https://github.com/nanopb/nanopb.git [submodule "aten/src/ATen/cpu/cpuinfo"] path = aten/src/ATen/cpu/cpuinfo diff --git a/torch/lib/README.md b/aten/src/README.md similarity index 97% rename from torch/lib/README.md rename to aten/src/README.md index f288e4d9bae..aa3002cb82f 100644 --- a/torch/lib/README.md +++ b/aten/src/README.md @@ -1,5 +1,7 @@ -This directory contains the low-level tensor libraries for PyTorch. -This code traces its lineage from the original Torch. There are +This directory contains the low-level tensor libraries for PyTorch, +as well as the new ATen C++ bindings. + +The low-level libraries trace their lineage from the original Torch. There are multiple variants of the library, summarized here: * TH = TorcH diff --git a/aten/tools/subtree_dir_add.sh b/aten/tools/subtree_dir_add.sh deleted file mode 100755 index 5eeb2959fb7..00000000000 --- a/aten/tools/subtree_dir_add.sh +++ /dev/null @@ -1,17 +0,0 @@ -SRC_BRANCH=$1 -SRC_PATH=$2 -DST_PATH=$3 - -CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD) -set -x -git branch -q -D temporary-split-branch -git checkout $SRC_BRANCH -git subtree split -P $SRC_PATH -b temporary-split-branch -git checkout $CURRENT_BRANCH -git subtree add -P $DST_PATH temporary-split-branch ${@:5} -git branch -D temporary-split-branch - -#./subtree_dir.sh add pytorch/master torch/lib/TH lib/TH -#./subtree_dir.sh add pytorch/master torch/lib/THC lib/THC -#./subtree_dir.sh add pytorch/master torch/lib/THNN lib/THNN -#./subtree_dir.sh add pytorch/master torch/lib/THCUNN lib/THCUNN diff --git a/aten/tools/subtree_dir_merge.sh b/aten/tools/subtree_dir_merge.sh deleted file mode 100755 index 0842c71254a..00000000000 --- a/aten/tools/subtree_dir_merge.sh +++ /dev/null @@ -1,17 +0,0 @@ -SRC_BRANCH=$1 -SRC_PATH=$2 -DST_PATH=$3 - -CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD) -set -x -git branch -q -D temporary-split-branch -git checkout $SRC_BRANCH -git subtree split -P $SRC_PATH -b temporary-split-branch -git checkout $CURRENT_BRANCH -git subtree merge -P $DST_PATH temporary-split-branch -m "Merge commit '`git rev-parse temporary-split-branch`'" -git branch -D temporary-split-branch - -#./subtree_dir.sh merge pytorch/master torch/lib/TH lib/TH -#./subtree_dir.sh merge pytorch/master torch/lib/THC lib/THC -#./subtree_dir.sh merge pytorch/master torch/lib/THNN lib/THNN -#./subtree_dir.sh merge pytorch/master torch/lib/THCUNN lib/THCUNN diff --git a/aten/tools/subtree_dir_merge_all.sh b/aten/tools/subtree_dir_merge_all.sh deleted file mode 100755 index 8573149c993..00000000000 --- a/aten/tools/subtree_dir_merge_all.sh +++ /dev/null @@ -1,10 +0,0 @@ -set -e -set -x -DIR=$(dirname $0) -git fetch pytorch -$DIR/subtree_dir_merge.sh pytorch/master torch/lib/TH lib/TH -$DIR/subtree_dir_merge.sh pytorch/master torch/lib/THC lib/THC -$DIR/subtree_dir_merge.sh pytorch/master torch/lib/THNN lib/THNN -$DIR/subtree_dir_merge.sh pytorch/master torch/lib/THCUNN lib/THCUNN -$DIR/subtree_dir_merge.sh pytorch/master torch/lib/THS lib/THS -$DIR/subtree_dir_merge.sh pytorch/master torch/lib/THCS lib/THCS diff --git a/setup.py b/setup.py index 6f0e459584d..6ac1597cd88 100644 --- a/setup.py +++ b/setup.py @@ -35,7 +35,7 @@ # # NO_SYSTEM_NCCL # disables use of system-wide nccl (we will use our submoduled -# copy in torch/lib/nccl) +# copy in third_party/nccl) # # WITH_GLOO_IBVERBS # toggle features related to distributed support @@ -190,9 +190,9 @@ def build_libs(libs): for lib in libs: assert lib in dep_libs, 'invalid lib: {}'.format(lib) if IS_WINDOWS: - build_libs_cmd = ['torch\\lib\\build_libs.bat'] + build_libs_cmd = ['tools\\build_pytorch_libs.bat'] else: - build_libs_cmd = ['bash', 'torch/lib/build_libs.sh'] + build_libs_cmd = ['bash', 'tools/build_pytorch_libs.sh'] my_env = os.environ.copy() my_env["PYTORCH_PYTHON"] = sys.executable my_env["NUM_JOBS"] = str(NUM_JOBS) @@ -250,9 +250,9 @@ class build_deps(Command): print("Could not find {}".format(f)) print("Did you run 'git submodule update --init'?") sys.exit(1) - check_file(os.path.join(lib_path, "gloo", "CMakeLists.txt")) - check_file(os.path.join(lib_path, "nanopb", "CMakeLists.txt")) - check_file(os.path.join(lib_path, "pybind11", "CMakeLists.txt")) + check_file(os.path.join(third_party_path, "gloo", "CMakeLists.txt")) + check_file(os.path.join(third_party_path, "nanopb", "CMakeLists.txt")) + check_file(os.path.join(third_party_path, "pybind11", "CMakeLists.txt")) check_file(os.path.join('aten', 'src', 'ATen', 'cpu', 'cpuinfo', 'CMakeLists.txt')) check_file(os.path.join('aten', 'src', 'ATen', 'cpu', 'tbb', 'tbb_remote', 'Makefile')) check_file(os.path.join('aten', 'src', 'ATen', 'utils', 'catch', 'CMakeLists.txt')) @@ -283,7 +283,7 @@ class build_deps(Command): # More information can be found in conversation thread of PR #5772 self.copy_tree('torch/csrc', 'torch/lib/include/torch/csrc/') - self.copy_tree('torch/lib/pybind11/include/pybind11/', + self.copy_tree('third_party/pybind11/include/pybind11/', 'torch/lib/include/pybind11') self.copy_file('torch/torch.h', 'torch/lib/include/torch/torch.h') @@ -487,13 +487,14 @@ else: cwd = os.path.dirname(os.path.abspath(__file__)) lib_path = os.path.join(cwd, "torch", "lib") +third_party_path = os.path.join(cwd, "third_party") tmp_install_path = lib_path + "/tmp_install" include_dirs += [ cwd, os.path.join(cwd, "torch", "csrc"), - lib_path + "/pybind11/include", + third_party_path + "/pybind11/include", tmp_install_path + "/include", tmp_install_path + "/include/TH", tmp_install_path + "/include/THNN", diff --git a/third_party/README.md b/third_party/README.md new file mode 100644 index 00000000000..d3a361f801a --- /dev/null +++ b/third_party/README.md @@ -0,0 +1,2 @@ +This folder contains vendored copies of third-party libraries that we +use. diff --git a/torch/lib/gloo b/third_party/gloo similarity index 100% rename from torch/lib/gloo rename to third_party/gloo diff --git a/torch/lib/nanopb b/third_party/nanopb similarity index 100% rename from torch/lib/nanopb rename to third_party/nanopb diff --git a/torch/lib/nccl/.gitignore b/third_party/nccl/.gitignore similarity index 100% rename from torch/lib/nccl/.gitignore rename to third_party/nccl/.gitignore diff --git a/torch/lib/nccl/CMakeLists.txt b/third_party/nccl/CMakeLists.txt similarity index 100% rename from torch/lib/nccl/CMakeLists.txt rename to third_party/nccl/CMakeLists.txt diff --git a/torch/lib/nccl/LICENSE.txt b/third_party/nccl/LICENSE.txt similarity index 100% rename from torch/lib/nccl/LICENSE.txt rename to third_party/nccl/LICENSE.txt diff --git a/torch/lib/nccl/Makefile b/third_party/nccl/Makefile similarity index 100% rename from torch/lib/nccl/Makefile rename to third_party/nccl/Makefile diff --git a/torch/lib/nccl/README.md b/third_party/nccl/README.md similarity index 100% rename from torch/lib/nccl/README.md rename to third_party/nccl/README.md diff --git a/torch/lib/nccl/debian/.gitignore b/third_party/nccl/debian/.gitignore similarity index 100% rename from torch/lib/nccl/debian/.gitignore rename to third_party/nccl/debian/.gitignore diff --git a/torch/lib/nccl/debian/changelog.in b/third_party/nccl/debian/changelog.in similarity index 100% rename from torch/lib/nccl/debian/changelog.in rename to third_party/nccl/debian/changelog.in diff --git a/torch/lib/nccl/debian/compat b/third_party/nccl/debian/compat similarity index 100% rename from torch/lib/nccl/debian/compat rename to third_party/nccl/debian/compat diff --git a/torch/lib/nccl/debian/control.in b/third_party/nccl/debian/control.in similarity index 100% rename from torch/lib/nccl/debian/control.in rename to third_party/nccl/debian/control.in diff --git a/torch/lib/nccl/debian/copyright b/third_party/nccl/debian/copyright similarity index 100% rename from torch/lib/nccl/debian/copyright rename to third_party/nccl/debian/copyright diff --git a/torch/lib/nccl/debian/libnccl-dev.install b/third_party/nccl/debian/libnccl-dev.install similarity index 100% rename from torch/lib/nccl/debian/libnccl-dev.install rename to third_party/nccl/debian/libnccl-dev.install diff --git a/torch/lib/nccl/debian/libnccl-dev.manpages b/third_party/nccl/debian/libnccl-dev.manpages similarity index 100% rename from torch/lib/nccl/debian/libnccl-dev.manpages rename to third_party/nccl/debian/libnccl-dev.manpages diff --git a/torch/lib/nccl/debian/libnccl1.install.in b/third_party/nccl/debian/libnccl1.install.in similarity index 100% rename from torch/lib/nccl/debian/libnccl1.install.in rename to third_party/nccl/debian/libnccl1.install.in diff --git a/torch/lib/nccl/debian/nccl.7 b/third_party/nccl/debian/nccl.7 similarity index 100% rename from torch/lib/nccl/debian/nccl.7 rename to third_party/nccl/debian/nccl.7 diff --git a/torch/lib/nccl/debian/rules b/third_party/nccl/debian/rules similarity index 100% rename from torch/lib/nccl/debian/rules rename to third_party/nccl/debian/rules diff --git a/torch/lib/nccl/debian/shlibs.local.in b/third_party/nccl/debian/shlibs.local.in similarity index 100% rename from torch/lib/nccl/debian/shlibs.local.in rename to third_party/nccl/debian/shlibs.local.in diff --git a/torch/lib/nccl/debian/source/format b/third_party/nccl/debian/source/format similarity index 100% rename from torch/lib/nccl/debian/source/format rename to third_party/nccl/debian/source/format diff --git a/torch/lib/nccl/fortran/Makefile b/third_party/nccl/fortran/Makefile similarity index 100% rename from torch/lib/nccl/fortran/Makefile rename to third_party/nccl/fortran/Makefile diff --git a/torch/lib/nccl/fortran/src/cudafor.f90 b/third_party/nccl/fortran/src/cudafor.f90 similarity index 100% rename from torch/lib/nccl/fortran/src/cudafor.f90 rename to third_party/nccl/fortran/src/cudafor.f90 diff --git a/torch/lib/nccl/fortran/src/ncclfor.f90 b/third_party/nccl/fortran/src/ncclfor.f90 similarity index 100% rename from torch/lib/nccl/fortran/src/ncclfor.f90 rename to third_party/nccl/fortran/src/ncclfor.f90 diff --git a/torch/lib/nccl/fortran/test/allgather_arr_out.f90 b/third_party/nccl/fortran/test/allgather_arr_out.f90 similarity index 100% rename from torch/lib/nccl/fortran/test/allgather_arr_out.f90 rename to third_party/nccl/fortran/test/allgather_arr_out.f90 diff --git a/torch/lib/nccl/fortran/test/allgather_ptr_out.f90 b/third_party/nccl/fortran/test/allgather_ptr_out.f90 similarity index 100% rename from torch/lib/nccl/fortran/test/allgather_ptr_out.f90 rename to third_party/nccl/fortran/test/allgather_ptr_out.f90 diff --git a/torch/lib/nccl/fortran/test/allreduce_arr_out.f90 b/third_party/nccl/fortran/test/allreduce_arr_out.f90 similarity index 100% rename from torch/lib/nccl/fortran/test/allreduce_arr_out.f90 rename to third_party/nccl/fortran/test/allreduce_arr_out.f90 diff --git a/torch/lib/nccl/fortran/test/allreduce_ptr_out.f90 b/third_party/nccl/fortran/test/allreduce_ptr_out.f90 similarity index 100% rename from torch/lib/nccl/fortran/test/allreduce_ptr_out.f90 rename to third_party/nccl/fortran/test/allreduce_ptr_out.f90 diff --git a/torch/lib/nccl/fortran/test/broadcast_arr.f90 b/third_party/nccl/fortran/test/broadcast_arr.f90 similarity index 100% rename from torch/lib/nccl/fortran/test/broadcast_arr.f90 rename to third_party/nccl/fortran/test/broadcast_arr.f90 diff --git a/torch/lib/nccl/fortran/test/broadcast_ptr.f90 b/third_party/nccl/fortran/test/broadcast_ptr.f90 similarity index 100% rename from torch/lib/nccl/fortran/test/broadcast_ptr.f90 rename to third_party/nccl/fortran/test/broadcast_ptr.f90 diff --git a/torch/lib/nccl/fortran/test/reduce_arr_out.f90 b/third_party/nccl/fortran/test/reduce_arr_out.f90 similarity index 100% rename from torch/lib/nccl/fortran/test/reduce_arr_out.f90 rename to third_party/nccl/fortran/test/reduce_arr_out.f90 diff --git a/torch/lib/nccl/fortran/test/reduce_ptr_out.f90 b/third_party/nccl/fortran/test/reduce_ptr_out.f90 similarity index 100% rename from torch/lib/nccl/fortran/test/reduce_ptr_out.f90 rename to third_party/nccl/fortran/test/reduce_ptr_out.f90 diff --git a/torch/lib/nccl/fortran/test/reducescatter_arr_out.f90 b/third_party/nccl/fortran/test/reducescatter_arr_out.f90 similarity index 100% rename from torch/lib/nccl/fortran/test/reducescatter_arr_out.f90 rename to third_party/nccl/fortran/test/reducescatter_arr_out.f90 diff --git a/torch/lib/nccl/fortran/test/reducescatter_ptr_out.f90 b/third_party/nccl/fortran/test/reducescatter_ptr_out.f90 similarity index 100% rename from torch/lib/nccl/fortran/test/reducescatter_ptr_out.f90 rename to third_party/nccl/fortran/test/reducescatter_ptr_out.f90 diff --git a/torch/lib/nccl/src/all_gather.cu b/third_party/nccl/src/all_gather.cu similarity index 100% rename from torch/lib/nccl/src/all_gather.cu rename to third_party/nccl/src/all_gather.cu diff --git a/torch/lib/nccl/src/all_reduce.cu b/third_party/nccl/src/all_reduce.cu similarity index 100% rename from torch/lib/nccl/src/all_reduce.cu rename to third_party/nccl/src/all_reduce.cu diff --git a/torch/lib/nccl/src/broadcast.cu b/third_party/nccl/src/broadcast.cu similarity index 100% rename from torch/lib/nccl/src/broadcast.cu rename to third_party/nccl/src/broadcast.cu diff --git a/torch/lib/nccl/src/common_coll.h b/third_party/nccl/src/common_coll.h similarity index 100% rename from torch/lib/nccl/src/common_coll.h rename to third_party/nccl/src/common_coll.h diff --git a/torch/lib/nccl/src/common_kernel.h b/third_party/nccl/src/common_kernel.h similarity index 100% rename from torch/lib/nccl/src/common_kernel.h rename to third_party/nccl/src/common_kernel.h diff --git a/torch/lib/nccl/src/copy_kernel.h b/third_party/nccl/src/copy_kernel.h similarity index 100% rename from torch/lib/nccl/src/copy_kernel.h rename to third_party/nccl/src/copy_kernel.h diff --git a/torch/lib/nccl/src/core.cu b/third_party/nccl/src/core.cu similarity index 100% rename from torch/lib/nccl/src/core.cu rename to third_party/nccl/src/core.cu diff --git a/torch/lib/nccl/src/core.h b/third_party/nccl/src/core.h similarity index 100% rename from torch/lib/nccl/src/core.h rename to third_party/nccl/src/core.h diff --git a/torch/lib/nccl/src/enqueue.h b/third_party/nccl/src/enqueue.h similarity index 100% rename from torch/lib/nccl/src/enqueue.h rename to third_party/nccl/src/enqueue.h diff --git a/torch/lib/nccl/src/libwrap.cu b/third_party/nccl/src/libwrap.cu similarity index 100% rename from torch/lib/nccl/src/libwrap.cu rename to third_party/nccl/src/libwrap.cu diff --git a/torch/lib/nccl/src/libwrap.h b/third_party/nccl/src/libwrap.h similarity index 100% rename from torch/lib/nccl/src/libwrap.h rename to third_party/nccl/src/libwrap.h diff --git a/torch/lib/nccl/src/nccl.h b/third_party/nccl/src/nccl.h similarity index 100% rename from torch/lib/nccl/src/nccl.h rename to third_party/nccl/src/nccl.h diff --git a/torch/lib/nccl/src/primitives.h b/third_party/nccl/src/primitives.h similarity index 100% rename from torch/lib/nccl/src/primitives.h rename to third_party/nccl/src/primitives.h diff --git a/torch/lib/nccl/src/reduce.cu b/third_party/nccl/src/reduce.cu similarity index 100% rename from torch/lib/nccl/src/reduce.cu rename to third_party/nccl/src/reduce.cu diff --git a/torch/lib/nccl/src/reduce_kernel.h b/third_party/nccl/src/reduce_kernel.h similarity index 100% rename from torch/lib/nccl/src/reduce_kernel.h rename to third_party/nccl/src/reduce_kernel.h diff --git a/torch/lib/nccl/src/reduce_scatter.cu b/third_party/nccl/src/reduce_scatter.cu similarity index 100% rename from torch/lib/nccl/src/reduce_scatter.cu rename to third_party/nccl/src/reduce_scatter.cu diff --git a/torch/lib/nccl/test/include/test_utilities.h b/third_party/nccl/test/include/test_utilities.h similarity index 100% rename from torch/lib/nccl/test/include/test_utilities.h rename to third_party/nccl/test/include/test_utilities.h diff --git a/torch/lib/nccl/test/mpi/mpi_test.cu b/third_party/nccl/test/mpi/mpi_test.cu similarity index 100% rename from torch/lib/nccl/test/mpi/mpi_test.cu rename to third_party/nccl/test/mpi/mpi_test.cu diff --git a/torch/lib/nccl/test/single/all_gather_scan.cu b/third_party/nccl/test/single/all_gather_scan.cu similarity index 100% rename from torch/lib/nccl/test/single/all_gather_scan.cu rename to third_party/nccl/test/single/all_gather_scan.cu diff --git a/torch/lib/nccl/test/single/all_gather_test.cu b/third_party/nccl/test/single/all_gather_test.cu similarity index 100% rename from torch/lib/nccl/test/single/all_gather_test.cu rename to third_party/nccl/test/single/all_gather_test.cu diff --git a/torch/lib/nccl/test/single/all_reduce_scan.cu b/third_party/nccl/test/single/all_reduce_scan.cu similarity index 100% rename from torch/lib/nccl/test/single/all_reduce_scan.cu rename to third_party/nccl/test/single/all_reduce_scan.cu diff --git a/torch/lib/nccl/test/single/all_reduce_test.cu b/third_party/nccl/test/single/all_reduce_test.cu similarity index 100% rename from torch/lib/nccl/test/single/all_reduce_test.cu rename to third_party/nccl/test/single/all_reduce_test.cu diff --git a/torch/lib/nccl/test/single/broadcast_scan.cu b/third_party/nccl/test/single/broadcast_scan.cu similarity index 100% rename from torch/lib/nccl/test/single/broadcast_scan.cu rename to third_party/nccl/test/single/broadcast_scan.cu diff --git a/torch/lib/nccl/test/single/broadcast_test.cu b/third_party/nccl/test/single/broadcast_test.cu similarity index 100% rename from torch/lib/nccl/test/single/broadcast_test.cu rename to third_party/nccl/test/single/broadcast_test.cu diff --git a/torch/lib/nccl/test/single/reduce_scan.cu b/third_party/nccl/test/single/reduce_scan.cu similarity index 100% rename from torch/lib/nccl/test/single/reduce_scan.cu rename to third_party/nccl/test/single/reduce_scan.cu diff --git a/torch/lib/nccl/test/single/reduce_scatter_scan.cu b/third_party/nccl/test/single/reduce_scatter_scan.cu similarity index 100% rename from torch/lib/nccl/test/single/reduce_scatter_scan.cu rename to third_party/nccl/test/single/reduce_scatter_scan.cu diff --git a/torch/lib/nccl/test/single/reduce_scatter_test.cu b/third_party/nccl/test/single/reduce_scatter_test.cu similarity index 100% rename from torch/lib/nccl/test/single/reduce_scatter_test.cu rename to third_party/nccl/test/single/reduce_scatter_test.cu diff --git a/torch/lib/nccl/test/single/reduce_test.cu b/third_party/nccl/test/single/reduce_test.cu similarity index 100% rename from torch/lib/nccl/test/single/reduce_test.cu rename to third_party/nccl/test/single/reduce_test.cu diff --git a/torch/lib/pybind11 b/third_party/pybind11 similarity index 100% rename from torch/lib/pybind11 rename to third_party/pybind11 diff --git a/torch/lib/build_libs.bat b/tools/build_pytorch_libs.bat similarity index 89% rename from torch/lib/build_libs.bat rename to tools/build_pytorch_libs.bat index fa0d7c83830..ce854d2eaed 100644 --- a/torch/lib/build_libs.bat +++ b/tools/build_pytorch_libs.bat @@ -1,11 +1,10 @@ -@echo off -cd "%~dp0" -cd "../.." +:: @echo off +cd "%~dp0/.." set BASE_DIR=%cd:\=/% -cd torch/lib - -set INSTALL_DIR=%cd:\=/%/tmp_install +set TORCH_LIB_DIR=%cd:\=/%/torch/lib +set INSTALL_DIR=%cd:\=/%/torch/lib/tmp_install +set THIRD_PARTY_DIR=%cd:\=/%/third_party set PATH=%INSTALL_DIR%/bin;%PATH% set BASIC_C_FLAGS= /DTH_INDEX_BASE=0 /I%INSTALL_DIR%/include /I%INSTALL_DIR%/include/TH /I%INSTALL_DIR%/include/THC /I%INSTALL_DIR%/include/THS /I%INSTALLDIR%/include/THCS /I%INSTALLDIR%/include/THPP /I%INSTALLDIR%/include/THNN /I%INSTALLDIR%/include/THCUNN set BASIC_CUDA_FLAGS= -DTH_INDEX_BASE=0 -I%INSTALL_DIR%/include -I%INSTALL_DIR%/include/TH -I%INSTALL_DIR%/include/THC -I%INSTALL_DIR%/include/THS -I%INSTALLDIR%/include/THCS -I%INSTALLDIR%/include/THPP -I%INSTALLDIR%/include/THNN -I%INSTALLDIR%/include/THCUNN @@ -16,7 +15,7 @@ set CWRAP_FILES=%BASE_DIR%/torch/lib/ATen/Declarations.cwrap;%BASE_DIR%/torch/li set C_FLAGS=%BASIC_C_FLAGS% /D_WIN32 /Z7 /EHa /DNOMINMAX set LINK_FLAGS=/DEBUG:FULL -mkdir tmp_install +mkdir torch/lib/tmp_install IF "%~1"=="--with-cuda" ( set /a NO_CUDA=0 @@ -60,15 +59,30 @@ IF "%CMAKE_GENERATOR%"=="" ( :read_loop if "%1"=="" goto after_loop if "%1"=="ATen" ( + cd aten call:build_aten %~1 + cd .. ) ELSE ( - call:build %~1 + set "IS_OURS=" + IF "%1"=="THD" set IS_OURS=1 + IF "%1"=="libshm_windows" set IS_OURS=1 + if defined IS_OURS ( + cd torch\lib + call:build %~1 + cd ..\.. + ) ELSE ( + cd third_party + call:build %~1 + cd .. + ) ) shift goto read_loop :after_loop +cd torch/lib + copy /Y tmp_install\lib\* . IF EXIST ".\tmp_install\bin" ( copy /Y tmp_install\bin\* . @@ -77,6 +91,8 @@ xcopy /Y /E tmp_install\include\*.* include\*.* xcopy /Y ..\..\aten\src\THNN\generic\THNN.h . xcopy /Y ..\..\aten\src\THCUNN\generic\THCUNN.h . +cd ..\.. + goto:eof :build @@ -121,9 +137,9 @@ goto:eof :build_aten @setlocal IF NOT "%PREBUILD_COMMAND%"=="" call "%PREBUILD_COMMAND%" %PREBUILD_COMMAND_ARGS% - mkdir build\%~1 - cd build/%~1 - cmake ../../../../%~1 %CMAKE_GENERATOR_COMMAND% ^ + mkdir build + cd build + cmake .. %CMAKE_GENERATOR_COMMAND% ^ -DCMAKE_INSTALL_PREFIX="%INSTALL_DIR%" ^ -DNO_CUDA=%NO_CUDA% ^ -DNO_NNPACK=%NO_NNPACK% ^ @@ -134,7 +150,7 @@ goto:eof %MAKE_COMMAND% IF NOT %ERRORLEVEL%==0 exit 1 - cd ../.. + cd .. @endlocal goto:eof diff --git a/torch/lib/build_libs.sh b/tools/build_pytorch_libs.sh similarity index 82% rename from torch/lib/build_libs.sh rename to tools/build_pytorch_libs.sh index 4ebeaaefdbe..cb403ca287e 100755 --- a/torch/lib/build_libs.sh +++ b/tools/build_pytorch_libs.sh @@ -1,10 +1,12 @@ #!/usr/bin/env bash -# Shell script used to build the torch/lib/* dependencies prior to +# Shell script used to build the aten/* and third_party/* dependencies prior to # linking the libraries and passing the headers to the Python extension # compilation stage. This file is used from setup.py, but can also be # called standalone to compile the libraries outside of the overall PyTorch # build process. +# +# TODO: Replace this with a CMakeLists.txt set -ex @@ -27,6 +29,8 @@ if [[ "$1" == "--with-gloo-ibverbs" ]]; then shift fi +CMAKE_INSTALL=${CMAKE_INSTALL-make install} + # Save user specified env vars, we will manually propagate them # to cmake. We copy distutils semantics, referring to # cpython/Lib/distutils/sysconfig.py as the source of truth @@ -45,11 +49,13 @@ if [[ -n "$CPPFLAGS" ]]; then USER_LDFLAGS="$USER_LDFLAGS $CPPFLAGS" fi -cd "$(dirname "$0")/../.." +cd "$(dirname "$0")/.." PWD=`printf "%q\n" "$(pwd)"` BASE_DIR="$PWD" -cd torch/lib -INSTALL_DIR="$PWD/tmp_install" +TORCH_LIB_DIR="$BASE_DIR/torch/lib" +INSTALL_DIR="$TORCH_LIB_DIR/tmp_install" +THIRD_PARTY_DIR="$BASE_DIR/third_party" + CMAKE_VERSION=${CMAKE_VERSION:="cmake"} C_FLAGS=" -DTH_INDEX_BASE=0 -I\"$INSTALL_DIR/include\" \ -I\"$INSTALL_DIR/include/TH\" -I\"$INSTALL_DIR/include/THC\" \ @@ -94,12 +100,12 @@ if [ -z "$NUM_JOBS" ]; then NUM_JOBS="$(getconf _NPROCESSORS_ONLN)" fi -# Used to build an individual library, e.g. build TH +# Used to build an individual library function build() { # We create a build directory for the library, which will # contain the cmake output mkdir -p build/$1 - cd build/$1 + pushd build/$1 BUILD_C_FLAGS='' case $1 in THCS | THCUNN ) BUILD_C_FLAGS=$C_FLAGS;; @@ -141,7 +147,7 @@ function build() { ${@:2} \ -DCMAKE_EXPORT_COMPILE_COMMANDS=1 ${CMAKE_INSTALL} -j"$NUM_JOBS" - cd ../.. + popd local lib_prefix=$INSTALL_DIR/lib/lib$1 if [ -f "$lib_prefix$LD_POSTFIX" ]; then @@ -149,32 +155,32 @@ function build() { fi if [[ $(uname) == 'Darwin' ]]; then - cd tmp_install/lib + pushd "$INSTALL_DIR/lib" for lib in *.dylib; do echo "Updating install_name for $lib" install_name_tool -id @rpath/$lib $lib done - cd ../.. + popd fi } function build_nccl() { - mkdir -p build/nccl - cd build/nccl - ${CMAKE_VERSION} ../../nccl -DCMAKE_MODULE_PATH="$BASE_DIR/cmake/FindCUDA" \ - ${CMAKE_GENERATOR} \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" \ - -DCMAKE_C_FLAGS="$C_FLAGS $USER_CFLAGS" \ - -DCMAKE_CXX_FLAGS="$C_FLAGS $CPP_FLAGS $USER_CFLAGS" \ - -DCMAKE_SHARED_LINKER_FLAGS="$USER_LDFLAGS" + mkdir -p build/nccl + pushd build/nccl + ${CMAKE_VERSION} ../../nccl -DCMAKE_MODULE_PATH="$BASE_DIR/cmake/FindCUDA" \ + ${CMAKE_GENERATOR} \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX="$INSTALL_DIR" \ + -DCMAKE_C_FLAGS="$C_FLAGS $USER_CFLAGS" \ + -DCMAKE_CXX_FLAGS="$C_FLAGS $CPP_FLAGS $USER_CFLAGS" \ + -DCMAKE_SHARED_LINKER_FLAGS="$USER_LDFLAGS" ${CMAKE_INSTALL} - mkdir -p ${INSTALL_DIR}/lib - cp "lib/libnccl.so.1" "${INSTALL_DIR}/lib/libnccl.so.1" - if [ ! -f "${INSTALL_DIR}/lib/libnccl.so" ]; then - ln -s "${INSTALL_DIR}/lib/libnccl.so.1" "${INSTALL_DIR}/lib/libnccl.so" - fi - cd ../.. + mkdir -p ${INSTALL_DIR}/lib + cp "lib/libnccl.so.1" "${INSTALL_DIR}/lib/libnccl.so.1" + if [ ! -f "${INSTALL_DIR}/lib/libnccl.so" ]; then + ln -s "${INSTALL_DIR}/lib/libnccl.so.1" "${INSTALL_DIR}/lib/libnccl.so" + fi + popd } # purpusefully not using build() because we need ATen to build the same @@ -186,9 +192,9 @@ function build_nccl() { # detected them (to ensure that we have a consistent view between the # PyTorch and ATen builds.) function build_aten() { - mkdir -p build/aten - cd build/aten - ${CMAKE_VERSION} ../../../../aten \ + mkdir -p build + pushd build + ${CMAKE_VERSION} .. \ ${CMAKE_GENERATOR} \ -DCMAKE_BUILD_TYPE=$([ $DEBUG ] && echo Debug || echo Release) \ -DNO_CUDA=$((1-$WITH_CUDA)) \ @@ -207,27 +213,43 @@ function build_aten() { # to aten/CMakeLists.txt, not here. We need the vanilla # cmake build to work. ${CMAKE_INSTALL} -j"$NUM_JOBS" - cd ../.. + popd } # In the torch/lib directory, create an installation directory -mkdir -p tmp_install +mkdir -p torch/lib/tmp_install # Build for arg in "$@"; do if [[ "$arg" == "nccl" ]]; then + pushd $THIRD_PARTY_DIR build_nccl + popd elif [[ "$arg" == "gloo" ]]; then + pushd "$THIRD_PARTY_DIR" build gloo $GLOO_FLAGS + popd elif [[ "$arg" == "ATen" ]]; then + pushd "$BASE_DIR/aten" build_aten + popd elif [[ "$arg" == "THD" ]]; then + pushd "$TORCH_LIB_DIR" build THD $THD_FLAGS - else + popd + elif [[ "$arg" == "libshm" ]] || [[ "$arg" == "libshm_windows" ]]; then + pushd "$TORCH_LIB_DIR" build $arg + popd + else + pushd "$THIRD_PARTY_DIR" + build $arg + popd fi done +pushd torch/lib + # If all the builds succeed we copy the libraries, headers, # binaries to torch/lib rm -rf "$INSTALL_DIR/lib/cmake" @@ -250,3 +272,5 @@ then # copy over dependency libraries into the current dir cp "$PYTORCH_SO_DEPS" . fi + +popd diff --git a/tools/cpp_build/build_all.sh b/tools/cpp_build/build_all.sh old mode 100644 new mode 100755 diff --git a/tools/cpp_build/libtorch/CMakeLists.txt b/tools/cpp_build/libtorch/CMakeLists.txt index 610137bf74c..a794d6b0192 100644 --- a/tools/cpp_build/libtorch/CMakeLists.txt +++ b/tools/cpp_build/libtorch/CMakeLists.txt @@ -147,7 +147,7 @@ TARGET_INCLUDE_DIRECTORIES(torch "${ATEN_INCLUDE_DIR}/TH" "${CMAKE_CURRENT_SOURCE_DIR}" "${TORCH_SRC_DIR}/../" - "${TORCH_SRC_DIR}/lib/nanopb" + "${TORCH_SRC_DIR}/../third_party/nanopb" "${CUDA_INCLUDE_DIRS}" ) diff --git a/tox.ini b/tox.ini index 793ccc2c3b2..39b260d4d46 100644 --- a/tox.ini +++ b/tox.ini @@ -1,4 +1,4 @@ [flake8] max-line-length = 120 ignore = E305,E402,E721,E741,F401,F403,F405,F821,F841,F999 -exclude = docs/src,venv,torch/lib/gloo,torch/lib/pybind11,torch/lib/nanopb +exclude = docs/src,venv,third_party