[cmake] Add USE_SYSTEM_{GLOO,FP16,PTHREADPOOL,PSIMD,FXDIV,BENCHMARK} options (#14699) (#37277)

Summary:
These options are disabled by default, and are supposed to be used by
linux distro developers. With the existing shortcut option
USE_SYSTEM_LIBS toggled, these new options will be enabled as well.

Additionally, when USE_SYSTEM_LIBS is toggled, setup.py should
no longer check the existence of git submodules.

ezyang
Pull Request resolved: https://github.com/pytorch/pytorch/pull/37277

Differential Revision: D21256999

Pulled By: ezyang

fbshipit-source-id: 84f97d008db5a5e41a289cb7bce94906de3c52cf

CMakeLists.txt

@@ -216,10 +216,23 @@ option(ONNX_ML "Enable traditional ONNX ML API." ON)
 option(USE_SYSTEM_LIBS "Use all available system-provided libraries." OFF)
 option(USE_SYSTEM_CPUINFO "Use system-provided cpuinfo." OFF)
 option(USE_SYSTEM_SLEEF "Use system-provided sleef." OFF)
+option(USE_SYSTEM_GLOO "Use system-provided gloo." OFF)
+option(USE_SYSTEM_FP16 "Use system-provided fp16." OFF)
+option(USE_SYSTEM_PTHREADPOOL "Use system-provided pthreadpool." OFF)
+option(USE_SYSTEM_PSIMD "Use system-provided psimd." OFF)
+option(USE_SYSTEM_FXDIV "Use system-provided fxdiv." OFF)
+option(USE_SYSTEM_BENCHMARK "Use system-provided google benchmark." OFF)
 if(USE_SYSTEM_LIBS)
   set(USE_SYSTEM_CPUINFO ON)
   set(USE_SYSTEM_SLEEF ON)
+  set(USE_SYSTEM_GLOO ON)
   set(BUILD_CUSTOM_PROTOBUF OFF)
+  set(USE_SYSTEM_EIGEN_INSTALL ON)
+  set(USE_SYSTEM_FP16 ON)
+  set(USE_SYSTEM_PTHREADPOOL ON)
+  set(USE_SYSTEM_PSIMD ON)
+  set(USE_SYSTEM_FXDIV ON)
+  set(USE_SYSTEM_BENCHMARK ON)
 endif()
 
 # Used when building Caffe2 through setup.py

aten/src/ATen/CMakeLists.txt

@@ -255,6 +255,9 @@ if(NOT MSVC AND NOT EMSCRIPTEN AND NOT INTERN_BUILD_MOBILE)
   else()
     add_library(sleef SHARED IMPORTED)
     find_library(SLEEF_LIBRARY sleef)
+    if(NOT SLEEF_LIBRARY)
+      message(FATAL_ERROR "Cannot find sleef")
+    endif()
     message("Found sleef: ${SLEEF_LIBRARY}")
     set_target_properties(sleef PROPERTIES IMPORTED_LOCATION "${SLEEF_LIBRARY}")
   endif()

aten/src/ATen/native/quantized/cpu/qnnpack/CMakeLists.txt

@@ -72,7 +72,7 @@ if(NOT USE_SYSTEM_CPUINFO)
   endif()
 endif()
 
-if(NOT DEFINED FP16_SOURCE_DIR)
+if(NOT DEFINED FP16_SOURCE_DIR AND NOT USE_SYSTEM_FP16)
   message(STATUS "Downloading FP16 to ${CONFU_DEPENDENCIES_SOURCE_DIR}/fp16 (define FP16_SOURCE_DIR to avoid it)")
   configure_file(cmake/DownloadFP16.cmake "${CONFU_DEPENDENCIES_BINARY_DIR}/fp16-download/CMakeLists.txt")
   execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" .
@@ -82,7 +82,7 @@ if(NOT DEFINED FP16_SOURCE_DIR)
   set(FP16_SOURCE_DIR "${CONFU_DEPENDENCIES_SOURCE_DIR}/fp16" CACHE STRING "FP16 source directory")
 endif()
 
-if(NOT DEFINED FXDIV_SOURCE_DIR)
+if(NOT DEFINED FXDIV_SOURCE_DIR AND NOT USE_SYSTEM_FXDIV)
   message(STATUS "Downloading FXdiv to ${CONFU_DEPENDENCIES_SOURCE_DIR}/fxdiv (define FXDIV_SOURCE_DIR to avoid it)")
   configure_file(cmake/DownloadFXdiv.cmake "${CONFU_DEPENDENCIES_BINARY_DIR}/fxdiv-download/CMakeLists.txt")
   execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" .
@@ -92,7 +92,7 @@ if(NOT DEFINED FXDIV_SOURCE_DIR)
   set(FXDIV_SOURCE_DIR "${CONFU_DEPENDENCIES_SOURCE_DIR}/fxdiv" CACHE STRING "FXdiv source directory")
 endif()
 
-if(NOT DEFINED PSIMD_SOURCE_DIR)
+if(NOT DEFINED PSIMD_SOURCE_DIR AND NOT USE_SYSTEM_PSIMD)
   message(STATUS "Downloading PSimd to ${CONFU_DEPENDENCIES_SOURCE_DIR}/psimd (define PSIMD_SOURCE_DIR to avoid it)")
   configure_file(cmake/DownloadPSimd.cmake "${CONFU_DEPENDENCIES_BINARY_DIR}/psimd-download/CMakeLists.txt")
   execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" .
@@ -102,7 +102,7 @@ if(NOT DEFINED PSIMD_SOURCE_DIR)
   set(PSIMD_SOURCE_DIR "${CONFU_DEPENDENCIES_SOURCE_DIR}/psimd" CACHE STRING "PSimd source directory")
 endif()
 
-if(NOT DEFINED PTHREADPOOL_SOURCE_DIR)
+if(NOT DEFINED PTHREADPOOL_SOURCE_DIR AND NOT USE_SYSTEM_PTHREADPOOL)
   message(STATUS "Downloading pthreadpool to ${CONFU_DEPENDENCIES_SOURCE_DIR}/pthreadpool (define PTHREADPOOL_SOURCE_DIR to avoid it)")
   configure_file(cmake/DownloadPThreadPool.cmake "${CONFU_DEPENDENCIES_BINARY_DIR}/pthreadpool-download/CMakeLists.txt")
   execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" .
@@ -307,6 +307,9 @@ target_link_libraries(pytorch_qnnpack PUBLIC clog)
 if(NOT TARGET cpuinfo AND USE_SYSTEM_CPUINFO)
   add_library(cpuinfo SHARED IMPORTED)
   find_library(CPUINFO_LIBRARY cpuinfo)
+  if(NOT CPUINFO_LIBRARY)
+    message(FATAL_ERROR "Cannot find cpuinfo")
+  endif()
   message("Found cpuinfo: ${CPUINFO_LIBRARY}")
   set_target_properties(cpuinfo PROPERTIES IMPORTED_LOCATION "${CPUINFO_LIBRARY}")
 elseif(NOT TARGET cpuinfo)
@@ -321,12 +324,22 @@ endif()
 target_link_libraries(pytorch_qnnpack PRIVATE cpuinfo)
 
 # ---[ Configure pthreadpool
-if(NOT TARGET pthreadpool)
+if(NOT TARGET pthreadpool AND NOT USE_SYSTEM_PTHREADPOOL)
   set(PTHREADPOOL_BUILD_TESTS OFF CACHE BOOL "")
   set(PTHREADPOOL_BUILD_BENCHMARKS OFF CACHE BOOL "")
   add_subdirectory(
     "${PTHREADPOOL_SOURCE_DIR}"
     "${CONFU_DEPENDENCIES_BINARY_DIR}/pthreadpool")
+elseif(NOT TARGET pthreadpool AND USE_SYSTEM_PTHREADPOOL)
+  add_library(pthreadpool SHARED IMPORTED)
+  find_library(PTHREADPOOL_LIBRARY pthreadpool)
+  if(NOT PTHREADPOOL_LIBRARY)
+    message(FATAL_ERROR "Cannot find pthreadpool")
+  endif()
+  message("-- Found pthreadpool: ${PTHREADPOOL_LIBRARY}")
+  set_target_properties(pthreadpool PROPERTIES
+    IMPORTED_LOCATION "${PTHREADPOOL_LIBRARY}")
+  add_library(pthreadpool_interface INTERFACE)
 endif()
 if(PYTORCH_QNNPACK_CUSTOM_THREADPOOL)
   # Depend on pthreadpool interface, but not on implementation.
@@ -337,30 +350,51 @@ else()
 endif()
 
 # ---[ Configure FXdiv
-if(NOT TARGET fxdiv)
+if(NOT TARGET fxdiv AND NOT USE_SYSTEM_FXDIV)
   set(FXDIV_BUILD_TESTS OFF CACHE BOOL "")
   set(FXDIV_BUILD_BENCHMARKS OFF CACHE BOOL "")
   add_subdirectory(
     "${FXDIV_SOURCE_DIR}"
     "${CONFU_DEPENDENCIES_BINARY_DIR}/fxdiv")
+elseif(NOT TARGET fxdiv AND USE_SYSTEM_FXDIV)
+  find_file(FXDIV_HDR fxdiv.h PATH_SUFFIXES include)
+  if(NOT FXDIV_HDR)
+    message(FATAL_ERROR "Cannot find fxdiv")
+  endif()
+  add_library(fxdiv STATIC "${FXDIV_HDR}")
+  set_property(TARGET fxdiv PROPERTY LINKER_LANGUAGE C)
 endif()
 target_link_libraries(pytorch_qnnpack PRIVATE fxdiv)
 
 # ---[ Configure psimd
-if(NOT TARGET psimd)
+if(NOT TARGET psimd AND NOT USE_SYSTEM_PSIMD)
   add_subdirectory(
     "${PSIMD_SOURCE_DIR}"
     "${CONFU_DEPENDENCIES_BINARY_DIR}/psimd")
+elseif(NOT TARGET psimd AND USE_SYSTEM_PSIMD)
+  find_file(PSIMD_HDR psimd.h PATH_SUFFIXES include)
+  if(NOT PSIMD_HDR)
+    message(FATAL_ERROR "Cannot find psimd")
+  endif()
+  add_library(psimd STATIC "${PSIMD_HDR}")
+  set_property(TARGET psimd PROPERTY LINKER_LANGUAGE C)
 endif()
 target_link_libraries(pytorch_qnnpack PRIVATE psimd)
 
 # ---[ Configure FP16
-if(NOT TARGET fp16)
+if(NOT TARGET fp16 AND NOT USE_SYSTEM_FP16)
   set(FP16_BUILD_TESTS OFF CACHE BOOL "")
   set(FP16_BUILD_BENCHMARKS OFF CACHE BOOL "")
   add_subdirectory(
     "${FP16_SOURCE_DIR}"
     "${CONFU_DEPENDENCIES_BINARY_DIR}/fp16")
+elseif(NOT TARGET fp16 AND USE_SYSTEM_FP16)
+  find_file(FP16_HDR fp16.h PATH_SUFFIXES include)
+  if(NOT FP16_HDR)
+    message(FATAL_ERROR "Cannot find fp16")
+  endif()
+  add_library(fp16 STATIC "${FP16_HDR}")
+  set_target_properties(fp16 PROPERTIES LINKER_LANGUAGE C)
 endif()
 target_link_libraries(pytorch_qnnpack PRIVATE fp16)
 

cmake/Dependencies.cmake

@@ -319,6 +319,9 @@ endif()
 if(NOT TARGET cpuinfo AND USE_SYSTEM_CPUINFO)
   add_library(cpuinfo SHARED IMPORTED)
   find_library(CPUINFO_LIBRARY cpuinfo)
+  if(NOT CPUINFO_LIBRARY)
+    message(FATAL_ERROR "Cannot find cpuinfo")
+  endif()
   message("Found cpuinfo: ${CPUINFO_LIBRARY}")
   set_target_properties(cpuinfo PROPERTIES IMPORTED_LOCATION "${CPUINFO_LIBRARY}")
 elseif(NOT TARGET cpuinfo)
@@ -400,7 +403,9 @@ if(USE_PYTORCH_QNNPACK)
       # We build static versions of QNNPACK and pthreadpool but link
       # them into a shared library for Caffe2, so they need PIC.
       set_property(TARGET pytorch_qnnpack PROPERTY POSITION_INDEPENDENT_CODE ON)
+      if(NOT USE_SYSTEM_PTHREADPOOL)
         set_property(TARGET pthreadpool PROPERTY POSITION_INDEPENDENT_CODE ON)
+      endif()
       set_property(TARGET cpuinfo PROPERTY POSITION_INDEPENDENT_CODE ON)
     endif()
 
@@ -541,7 +546,17 @@ if(BUILD_TEST)
   set(BENCHMARK_ENABLE_TESTING OFF CACHE BOOL "Disable benchmark testing as we don't need it.")
   # We will not need to install benchmark since we link it statically.
   set(BENCHMARK_ENABLE_INSTALL OFF CACHE BOOL "Disable benchmark install to avoid overwriting vendor install.")
+  if(NOT USE_SYSTEM_BENCHMARK)
     add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/../third_party/benchmark)
+  else()
+    add_library(benchmark SHARED IMPORTED)
+    find_library(BENCHMARK_LIBRARY benchmark)
+    if(NOT BENCHMARK_LIBRARY)
+      message(FATAL_ERROR "Cannot find google benchmark library")
+    endif()
+    message("-- Found benchmark: ${BENCHMARK_LIBRARY}")
+    set_property(TARGET benchmark PROPERTY IMPORTED_LOCATION ${BENCHMARK_LIBRARY})
+  endif()
   include_directories(${CMAKE_CURRENT_LIST_DIR}/../third_party/benchmark/include)
 
   # Recover build options.
@@ -725,7 +740,7 @@ if(USE_FFMPEG)
 endif()
 
 # ---[ Caffe2 depends on FP16 library for half-precision conversions
-if(NOT TARGET fp16)
+if(NOT TARGET fp16 AND NOT USE_SYSTEM_FP16)
   if(NOT DEFINED FP16_SOURCE_DIR)
     set(FP16_SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}/../third_party/FP16" CACHE STRING "FP16 source directory")
   endif()
@@ -735,6 +750,9 @@ if(NOT TARGET fp16)
   add_subdirectory(
     "${FP16_SOURCE_DIR}"
     "${CONFU_DEPENDENCIES_BINARY_DIR}/FP16")
+elseif(NOT TARGET fp16 AND USE_SYSTEM_FP16)
+  add_library(fp16 STATIC "/usr/include/fp16.h")
+  set_target_properties(fp16 PROPERTIES LINKER_LANGUAGE C)
 endif()
 list(APPEND Caffe2_DEPENDENCY_LIBS fp16)
 
@@ -1149,7 +1167,17 @@ if(USE_GLOO)
     set(__BUILD_BENCHMARK ${BUILD_BENCHMARK})
     set(BUILD_TEST OFF)
     set(BUILD_BENCHMARK OFF)
+    if(NOT USE_SYSTEM_GLOO)
       add_subdirectory(${CMAKE_CURRENT_LIST_DIR}/../third_party/gloo)
+    else()
+      add_library(gloo SHARED IMPORTED)
+      find_library(GLOO_LIBRARY gloo)
+      if(NOT GLOO_LIBRARY)
+        message(FATAL_ERROR "Cannot find gloo")
+      endif()
+      message("Found gloo: ${GLOO_LIBRARY}")
+      set_target_properties(gloo PROPERTIES IMPORTED_LOCATION ${GLOO_LIBRARY})
+    endif()
     # Here is a little bit hacky. We have to put PROJECT_BINARY_DIR in front
     # of PROJECT_SOURCE_DIR with/without conda system. The reason is that
     # gloo generates a new config.h in the binary diretory.

setup.py

@@ -292,6 +292,8 @@ def build_deps():
     report('-- Building version ' + version)
 
     def check_file(f):
+        if bool(os.getenv("USE_SYSTEM_LIBS", False)):
+            return
         if not os.path.exists(f):
             report("Could not find {}".format(f))
             report("Did you run 'git submodule update --init --recursive'?")