saymrwulf/pytorch
1
0
Fork 0
mirror of https://github.com/saymrwulf/pytorch.git synced 2026-05-14 20:57:59 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

[cmake] Add USE_SYSTEM_{GLOO,FP16,PTHREADPOOL,PSIMD,FXDIV,BENCHMARK} options (#14699) (#37277)

Browse source 
Summary:
These options are disabled by default, and are supposed to be used by
linux distro developers. With the existing shortcut option
USE_SYSTEM_LIBS toggled, these new options will be enabled as well.

Additionally, when USE_SYSTEM_LIBS is toggled, setup.py should
no longer check the existence of git submodules.

ezyang
Pull Request resolved: https://github.com/pytorch/pytorch/pull/37277

Differential Revision: D21256999

Pulled By: ezyang

fbshipit-source-id: 84f97d008db5a5e41a289cb7bce94906de3c52cf
This commit is contained in:
Mo Zhou 2020-04-27 09:34:52 -07:00 committed by Facebook GitHub Bot
parent 3a0ff3cd2f
commit 5b9f7f7b0e
5 changed files with 92 additions and 12 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
CMakeLists.txt
View file

@ -216,10 +216,23 @@ option(ONNX_ML "Enable traditional ONNX ML API." ON)
option(USE_SYSTEM_LIBS "Use all available system-provided libraries." OFF)
option(USE_SYSTEM_CPUINFO "Use system-provided cpuinfo." OFF)
option(USE_SYSTEM_SLEEF "Use system-provided sleef." OFF)
option(USE_SYSTEM_GLOO "Use system-provided gloo." OFF)
option(USE_SYSTEM_FP16 "Use system-provided fp16." OFF)
option(USE_SYSTEM_PTHREADPOOL "Use system-provided pthreadpool." OFF)
option(USE_SYSTEM_PSIMD "Use system-provided psimd." OFF)
option(USE_SYSTEM_FXDIV "Use system-provided fxdiv." OFF)
option(USE_SYSTEM_BENCHMARK "Use system-provided google benchmark." OFF)
if(USE_SYSTEM_LIBS)
set(USE_SYSTEM_CPUINFO ON)
set(USE_SYSTEM_SLEEF ON)
set(USE_SYSTEM_GLOO ON)
set(BUILD_CUSTOM_PROTOBUF OFF)
set(USE_SYSTEM_EIGEN_INSTALL ON)
set(USE_SYSTEM_FP16 ON)
set(USE_SYSTEM_PTHREADPOOL ON)
set(USE_SYSTEM_PSIMD ON)
set(USE_SYSTEM_FXDIV ON)
set(USE_SYSTEM_BENCHMARK ON)
endif()
# Used when building Caffe2 through setup.py

3
aten/src/ATen/CMakeLists.txt
View file

@ -255,6 +255,9 @@ if(NOT MSVC AND NOT EMSCRIPTEN AND NOT INTERN_BUILD_MOBILE)
else()
add_library(sleef SHARED IMPORTED)
find_library(SLEEF_LIBRARY sleef)
if(NOT SLEEF_LIBRARY)
message(FATAL_ERROR "Cannot find sleef")
endif()
message("Found sleef: ${SLEEF_LIBRARY}")
set_target_properties(sleef PROPERTIES IMPORTED_LOCATION "${SLEEF_LIBRARY}")
endif()

50
aten/src/ATen/native/quantized/cpu/qnnpack/CMakeLists.txt
View file

@ -72,7 +72,7 @@ if(NOT USE_SYSTEM_CPUINFO)
endif()
endif()
if(NOT DEFINED FP16_SOURCE_DIR)
if(NOT DEFINED FP16_SOURCE_DIR AND NOT USE_SYSTEM_FP16)
message(STATUS "Downloading FP16 to ${CONFU_DEPENDENCIES_SOURCE_DIR}/fp16 (define FP16_SOURCE_DIR to avoid it)")
configure_file(cmake/DownloadFP16.cmake "${CONFU_DEPENDENCIES_BINARY_DIR}/fp16-download/CMakeLists.txt")
execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" .
@ -82,7 +82,7 @@ if(NOT DEFINED FP16_SOURCE_DIR)
set(FP16_SOURCE_DIR "${CONFU_DEPENDENCIES_SOURCE_DIR}/fp16" CACHE STRING "FP16 source directory")
endif()
if(NOT DEFINED FXDIV_SOURCE_DIR)
if(NOT DEFINED FXDIV_SOURCE_DIR AND NOT USE_SYSTEM_FXDIV)
message(STATUS "Downloading FXdiv to ${CONFU_DEPENDENCIES_SOURCE_DIR}/fxdiv (define FXDIV_SOURCE_DIR to avoid it)")
configure_file(cmake/DownloadFXdiv.cmake "${CONFU_DEPENDENCIES_BINARY_DIR}/fxdiv-download/CMakeLists.txt")
execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" .
@ -92,7 +92,7 @@ if(NOT DEFINED FXDIV_SOURCE_DIR)
set(FXDIV_SOURCE_DIR "${CONFU_DEPENDENCIES_SOURCE_DIR}/fxdiv" CACHE STRING "FXdiv source directory")
endif()
if(NOT DEFINED PSIMD_SOURCE_DIR)
if(NOT DEFINED PSIMD_SOURCE_DIR AND NOT USE_SYSTEM_PSIMD)
message(STATUS "Downloading PSimd to ${CONFU_DEPENDENCIES_SOURCE_DIR}/psimd (define PSIMD_SOURCE_DIR to avoid it)")
configure_file(cmake/DownloadPSimd.cmake "${CONFU_DEPENDENCIES_BINARY_DIR}/psimd-download/CMakeLists.txt")
execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" .
@ -102,7 +102,7 @@ if(NOT DEFINED PSIMD_SOURCE_DIR)
set(PSIMD_SOURCE_DIR "${CONFU_DEPENDENCIES_SOURCE_DIR}/psimd" CACHE STRING "PSimd source directory")
endif()
if(NOT DEFINED PTHREADPOOL_SOURCE_DIR)
if(NOT DEFINED PTHREADPOOL_SOURCE_DIR AND NOT USE_SYSTEM_PTHREADPOOL)
message(STATUS "Downloading pthreadpool to ${CONFU_DEPENDENCIES_SOURCE_DIR}/pthreadpool (define PTHREADPOOL_SOURCE_DIR to avoid it)")
configure_file(cmake/DownloadPThreadPool.cmake "${CONFU_DEPENDENCIES_BINARY_DIR}/pthreadpool-download/CMakeLists.txt")
execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" .
@ -307,6 +307,9 @@ target_link_libraries(pytorch_qnnpack PUBLIC clog)
if(NOT TARGET cpuinfo AND USE_SYSTEM_CPUINFO)
add_library(cpuinfo SHARED IMPORTED)
find_library(CPUINFO_LIBRARY cpuinfo)
if(NOT CPUINFO_LIBRARY)
message(FATAL_ERROR "Cannot find cpuinfo")
endif()
message("Found cpuinfo: ${CPUINFO_LIBRARY}")
set_target_properties(cpuinfo PROPERTIES IMPORTED_LOCATION "${CPUINFO_LIBRARY}")
elseif(NOT TARGET cpuinfo)
@ -321,12 +324,22 @@ endif()
target_link_libraries(pytorch_qnnpack PRIVATE cpuinfo)
# ---[ Configure pthreadpool
if(NOT TARGET pthreadpool)
if(NOT TARGET pthreadpool AND NOT USE_SYSTEM_PTHREADPOOL)
set(PTHREADPOOL_BUILD_TESTS OFF CACHE BOOL "")
set(PTHREADPOOL_BUILD_BENCHMARKS OFF CACHE BOOL "")
add_subdirectory(
"${PTHREADPOOL_SOURCE_DIR}"
"${CONFU_DEPENDENCIES_BINARY_DIR}/pthreadpool")
elseif(NOT TARGET pthreadpool AND USE_SYSTEM_PTHREADPOOL)
add_library(pthreadpool SHARED IMPORTED)
find_library(PTHREADPOOL_LIBRARY pthreadpool)
if(NOT PTHREADPOOL_LIBRARY)
message(FATAL_ERROR "Cannot find pthreadpool")
endif()
message("-- Found pthreadpool: ${PTHREADPOOL_LIBRARY}")
set_target_properties(pthreadpool PROPERTIES
IMPORTED_LOCATION "${PTHREADPOOL_LIBRARY}")
add_library(pthreadpool_interface INTERFACE)
endif()
if(PYTORCH_QNNPACK_CUSTOM_THREADPOOL)
# Depend on pthreadpool interface, but not on implementation.
@ -337,30 +350,51 @@ else()
endif()
# ---[ Configure FXdiv
if(NOT TARGET fxdiv)
if(NOT TARGET fxdiv AND NOT USE_SYSTEM_FXDIV)
set(FXDIV_BUILD_TESTS OFF CACHE BOOL "")
set(FXDIV_BUILD_BENCHMARKS OFF CACHE BOOL "")
add_subdirectory(
"${FXDIV_SOURCE_DIR}"
"${CONFU_DEPENDENCIES_BINARY_DIR}/fxdiv")
elseif(NOT TARGET fxdiv AND USE_SYSTEM_FXDIV)
find_file(FXDIV_HDR fxdiv.h PATH_SUFFIXES include)
if(NOT FXDIV_HDR)
message(FATAL_ERROR "Cannot find fxdiv")
endif()
add_library(fxdiv STATIC "${FXDIV_HDR}")
set_property(TARGET fxdiv PROPERTY LINKER_LANGUAGE C)
endif()
target_link_libraries(pytorch_qnnpack PRIVATE fxdiv)
# ---[ Configure psimd
if(NOT TARGET psimd)
if(NOT TARGET psimd AND NOT USE_SYSTEM_PSIMD)
add_subdirectory(
"${PSIMD_SOURCE_DIR}"
"${CONFU_DEPENDENCIES_BINARY_DIR}/psimd")
elseif(NOT TARGET psimd AND USE_SYSTEM_PSIMD)
find_file(PSIMD_HDR psimd.h PATH_SUFFIXES include)
if(NOT PSIMD_HDR)
message(FATAL_ERROR "Cannot find psimd")
endif()
add_library(psimd STATIC "${PSIMD_HDR}")
set_property(TARGET psimd PROPERTY LINKER_LANGUAGE C)
endif()
target_link_libraries(pytorch_qnnpack PRIVATE psimd)
# ---[ Configure FP16
if(NOT TARGET fp16)
if(NOT TARGET fp16 AND NOT USE_SYSTEM_FP16)
set(FP16_BUILD_TESTS OFF CACHE BOOL "")
set(FP16_BUILD_BENCHMARKS OFF CACHE BOOL "")
add_subdirectory(
"${FP16_SOURCE_DIR}"
"${CONFU_DEPENDENCIES_BINARY_DIR}/fp16")
elseif(NOT TARGET fp16 AND USE_SYSTEM_FP16)
find_file(FP16_HDR fp16.h PATH_SUFFIXES include)
if(NOT FP16_HDR)
message(FATAL_ERROR "Cannot find fp16")
endif()
add_library(fp16 STATIC "${FP16_HDR}")
set_target_properties(fp16 PROPERTIES LINKER_LANGUAGE C)
endif()
target_link_libraries(pytorch_qnnpack PRIVATE fp16)

36
cmake/Dependencies.cmake
View file

@ -319,6 +319,9 @@ endif()
if(NOT TARGET cpuinfo AND USE_SYSTEM_CPUINFO)
add_library(cpuinfo SHARED IMPORTED)
find_library(CPUINFO_LIBRARY cpuinfo)
if(NOT CPUINFO_LIBRARY)
message(FATAL_ERROR "Cannot find cpuinfo")
endif()
message("Found cpuinfo: ${CPUINFO_LIBRARY}")
set_target_properties(cpuinfo PROPERTIES IMPORTED_LOCATION "${CPUINFO_LIBRARY}")
elseif(NOT TARGET cpuinfo)
@ -400,7 +403,9 @@ if(USE_PYTORCH_QNNPACK)
# We build static versions of QNNPACK and pthreadpool but link
# them into a shared library for Caffe2, so they need PIC.
set_property(TARGET pytorch_qnnpack PROPERTY POSITION_INDEPENDENT_CODE ON)
set_property(TARGET pthreadpool PROPERTY POSITION_INDEPENDENT_CODE ON)
if(NOT USE_SYSTEM_PTHREADPOOL)
set_property(TARGET pthreadpool PROPERTY POSITION_INDEPENDENT_CODE ON)
endif()
set_property(TARGET cpuinfo PROPERTY POSITION_INDEPENDENT_CODE ON)
endif()
@ -541,7 +546,17 @@ if(BUILD_TEST)
set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "Disable benchmark testing as we don't need it.")
# We will not need to install benchmark since we link it statically.
set(BENCHMARK_ENABLE_INSTALL OFF CACHE BOOL "Disable benchmark install to avoid overwriting vendor install.")
add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/../third_party/benchmark)
if(NOT USE_SYSTEM_BENCHMARK)
add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/../third_party/benchmark)
else()
add_library(benchmark SHARED IMPORTED)
find_library(BENCHMARK_LIBRARY benchmark)
if(NOT BENCHMARK_LIBRARY)
message(FATAL_ERROR "Cannot find google benchmark library")
endif()
message("-- Found benchmark: ${BENCHMARK_LIBRARY}")
set_property(TARGET benchmark PROPERTY IMPORTED_LOCATION ${BENCHMARK_LIBRARY})
endif()
include_directories(${CMAKE_CURRENT_LIST_DIR}/../third_party/benchmark/include)
# Recover build options.
@ -725,7 +740,7 @@ if(USE_FFMPEG)
endif()
# ---[ Caffe2 depends on FP16 library for half-precision conversions
if(NOT TARGET fp16)
if(NOT TARGET fp16 AND NOT USE_SYSTEM_FP16)
if(NOT DEFINED FP16_SOURCE_DIR)
set(FP16_SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}/../third_party/FP16" CACHE STRING "FP16 source directory")
endif()
@ -735,6 +750,9 @@ if(NOT TARGET fp16)
add_subdirectory(
"${FP16_SOURCE_DIR}"
"${CONFU_DEPENDENCIES_BINARY_DIR}/FP16")
elseif(NOT TARGET fp16 AND USE_SYSTEM_FP16)
add_library(fp16 STATIC "/usr/include/fp16.h")
set_target_properties(fp16 PROPERTIES LINKER_LANGUAGE C)
endif()
list(APPEND Caffe2_DEPENDENCY_LIBS fp16)
@ -1149,7 +1167,17 @@ if(USE_GLOO)
set(__BUILD_BENCHMARK ${BUILD_BENCHMARK})
set(BUILD_TEST OFF)
set(BUILD_BENCHMARK OFF)
add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/../third_party/gloo)
if(NOT USE_SYSTEM_GLOO)
add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/../third_party/gloo)
else()
add_library(gloo SHARED IMPORTED)
find_library(GLOO_LIBRARY gloo)
if(NOT GLOO_LIBRARY)
message(FATAL_ERROR "Cannot find gloo")
endif()
message("Found gloo: ${GLOO_LIBRARY}")
set_target_properties(gloo PROPERTIES IMPORTED_LOCATION ${GLOO_LIBRARY})
endif()
# Here is a little bit hacky. We have to put PROJECT_BINARY_DIR in front
# of PROJECT_SOURCE_DIR with/without conda system. The reason is that
# gloo generates a new config.h in the binary diretory.

2
setup.py
View file

@ -292,6 +292,8 @@ def build_deps():
report('-- Building version ' + version)
def check_file(f):
if bool(os.getenv("USE_SYSTEM_LIBS", False)):
return
if not os.path.exists(f):
report("Could not find {}".format(f))
report("Did you run 'git submodule update --init --recursive'?")