From ff21b156246e16e939b4f87fe20964e2dbb40dc8 Mon Sep 17 00:00:00 2001 From: Mo Zhou Date: Thu, 23 Apr 2020 20:40:16 -0700 Subject: [PATCH] cmake: add USE_SYSTEM_{LIBS,CPUINFO,SLEEF} options (#14699) (#37137) Summary: ezyang Pull Request resolved: https://github.com/pytorch/pytorch/pull/37137 Differential Revision: D21222632 Pulled By: ezyang fbshipit-source-id: 47624b30f8d07b31a40a26edf665bbec39e45202 --- CMakeLists.txt | 15 +++++++ aten/src/ATen/CMakeLists.txt | 45 +++++++++++-------- .../quantized/cpu/qnnpack/CMakeLists.txt | 25 +++++++---- cmake/Dependencies.cmake | 7 ++- setup.py | 4 ++ 5 files changed, 67 insertions(+), 29 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index e7ee06b1357..d859f3e7aca 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -207,6 +207,21 @@ cmake_dependent_option( option(USE_TBB "Use TBB" OFF) option(ONNX_ML "Enable traditional ONNX ML API." ON) +# Linux distributions do not want too many embedded sources, in that sense we +# need to be able to build pytorch with an (almost) empty third_party +# directory. +# USE_SYSTEM_LIBS is a shortcut variable to toggle all the # USE_SYSTEM_* +# variables on. Individual USE_SYSTEM_* variables can be toggled with +# USE_SYSTEM_LIBS being "OFF". +option(USE_SYSTEM_LIBS "Use all available system-provided libraries." OFF) +option(USE_SYSTEM_CPUINFO "Use system-provided cpuinfo." OFF) +option(USE_SYSTEM_SLEEF "Use system-provided sleef." OFF) +if(USE_SYSTEM_LIBS) + set(USE_SYSTEM_CPUINFO ON) + set(USE_SYSTEM_SLEEF ON) + set(BUILD_CUSTOM_PROTOBUF OFF) +endif() + # Used when building Caffe2 through setup.py option(BUILDING_WITH_TORCH_LIBS "Tell cmake if Caffe2 is being built alongside torch libs" ON) diff --git a/aten/src/ATen/CMakeLists.txt b/aten/src/ATen/CMakeLists.txt index 9f87de35069..fd3845171a3 100644 --- a/aten/src/ATen/CMakeLists.txt +++ b/aten/src/ATen/CMakeLists.txt @@ -230,27 +230,34 @@ if(NOT MSVC AND NOT EMSCRIPTEN AND NOT INTERN_BUILD_MOBILE) set(CMAKE_C_FLAGS_DEBUG "${CMAKE_C_FLAGS_DEBUG} -O1") endif() - set(BUILD_SHARED_LIBS OFF CACHE BOOL "Build sleef static" FORCE) - set(BUILD_DFT OFF CACHE BOOL "Don't build sleef DFT lib" FORCE) - set(BUILD_GNUABI_LIBS OFF CACHE BOOL "Don't build sleef gnuabi libs" FORCE) - set(BUILD_TESTS OFF CACHE BOOL "Don't build sleef tests" FORCE) - set(OLD_CMAKE_BUILD_TYPE ${CMAKE_BUILD_TYPE}) - if("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU" AND - CMAKE_C_COMPILER_VERSION VERSION_GREATER 6.9 AND CMAKE_C_COMPILER_VERSION VERSION_LESS 8) - set(GCC_7 True) + if(NOT USE_SYSTEM_SLEEF) + set(BUILD_SHARED_LIBS OFF CACHE BOOL "Build sleef static" FORCE) + set(BUILD_DFT OFF CACHE BOOL "Don't build sleef DFT lib" FORCE) + set(BUILD_GNUABI_LIBS OFF CACHE BOOL "Don't build sleef gnuabi libs" FORCE) + set(BUILD_TESTS OFF CACHE BOOL "Don't build sleef tests" FORCE) + set(OLD_CMAKE_BUILD_TYPE ${CMAKE_BUILD_TYPE}) + if("${CMAKE_C_COMPILER_ID}" STREQUAL "GNU" AND + CMAKE_C_COMPILER_VERSION VERSION_GREATER 6.9 AND CMAKE_C_COMPILER_VERSION VERSION_LESS 8) + set(GCC_7 True) + else() + set(GCC_7 False) + endif() + if(GCC_7) + set(CMAKE_BUILD_TYPE Release) # Always build Sleef as a Release build to work around a gcc-7 bug + endif() + add_subdirectory("${CMAKE_CURRENT_SOURCE_DIR}/../../../third_party/sleef" ${CMAKE_BINARY_DIR}/sleef) + if(GCC_7) + set(CMAKE_BUILD_TYPE ${OLD_CMAKE_BUILD_TYPE}) + endif() + set_property(TARGET sleef PROPERTY FOLDER "dependencies") + list(APPEND ATen_THIRD_PARTY_INCLUDE ${CMAKE_BINARY_DIR}/include) + link_directories(${CMAKE_BINARY_DIR}/sleef/lib) else() - set(GCC_7 False) + add_library(sleef SHARED IMPORTED) + find_library(SLEEF_LIBRARY sleef) + message("Found sleef: ${SLEEF_LIBRARY}") + set_target_properties(sleef PROPERTIES IMPORTED_LOCATION "${SLEEF_LIBRARY}") endif() - if(GCC_7) - set(CMAKE_BUILD_TYPE Release) # Always build Sleef as a Release build to work around a gcc-7 bug - endif() - add_subdirectory("${CMAKE_CURRENT_SOURCE_DIR}/../../../third_party/sleef" ${CMAKE_BINARY_DIR}/sleef) - if(GCC_7) - set(CMAKE_BUILD_TYPE ${OLD_CMAKE_BUILD_TYPE}) - endif() - set_property(TARGET sleef PROPERTY FOLDER "dependencies") - list(APPEND ATen_THIRD_PARTY_INCLUDE ${CMAKE_BINARY_DIR}/include) - link_directories(${CMAKE_BINARY_DIR}/sleef/lib) list(APPEND ATen_CPU_DEPENDENCY_LIBS sleef) set(CMAKE_C_FLAGS_DEBUG ${OLD_CMAKE_C_FLAGS_DEBUG}) diff --git a/aten/src/ATen/native/quantized/cpu/qnnpack/CMakeLists.txt b/aten/src/ATen/native/quantized/cpu/qnnpack/CMakeLists.txt index 352844dd8d2..d4fa607b875 100644 --- a/aten/src/ATen/native/quantized/cpu/qnnpack/CMakeLists.txt +++ b/aten/src/ATen/native/quantized/cpu/qnnpack/CMakeLists.txt @@ -60,14 +60,16 @@ if(NOT DEFINED CLOG_SOURCE_DIR) set(CLOG_SOURCE_DIR "${PROJECT_SOURCE_DIR}/deps/clog") endif() -if(NOT DEFINED CPUINFO_SOURCE_DIR) - message(STATUS "Downloading cpuinfo to ${CONFU_DEPENDENCIES_SOURCE_DIR}/cpuinfo (define CPUINFO_SOURCE_DIR to avoid it)") - configure_file(cmake/DownloadCpuinfo.cmake "${CONFU_DEPENDENCIES_BINARY_DIR}/cpuinfo-download/CMakeLists.txt") - execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . - WORKING_DIRECTORY "${CONFU_DEPENDENCIES_BINARY_DIR}/cpuinfo-download") - execute_process(COMMAND "${CMAKE_COMMAND}" --build . - WORKING_DIRECTORY "${CONFU_DEPENDENCIES_BINARY_DIR}/cpuinfo-download") - set(CPUINFO_SOURCE_DIR "${CONFU_DEPENDENCIES_SOURCE_DIR}/cpuinfo" CACHE STRING "cpuinfo source directory") +if(NOT USE_SYSTEM_CPUINFO) + if(NOT DEFINED CPUINFO_SOURCE_DIR) + message(STATUS "Downloading cpuinfo to ${CONFU_DEPENDENCIES_SOURCE_DIR}/cpuinfo (define CPUINFO_SOURCE_DIR to avoid it)") + configure_file(cmake/DownloadCpuinfo.cmake "${CONFU_DEPENDENCIES_BINARY_DIR}/cpuinfo-download/CMakeLists.txt") + execute_process(COMMAND "${CMAKE_COMMAND}" -G "${CMAKE_GENERATOR}" . + WORKING_DIRECTORY "${CONFU_DEPENDENCIES_BINARY_DIR}/cpuinfo-download") + execute_process(COMMAND "${CMAKE_COMMAND}" --build . + WORKING_DIRECTORY "${CONFU_DEPENDENCIES_BINARY_DIR}/cpuinfo-download") + set(CPUINFO_SOURCE_DIR "${CONFU_DEPENDENCIES_SOURCE_DIR}/cpuinfo" CACHE STRING "cpuinfo source directory") + endif() endif() if(NOT DEFINED FP16_SOURCE_DIR) @@ -302,7 +304,12 @@ endif() target_link_libraries(pytorch_qnnpack PUBLIC clog) # ---[ Configure cpuinfo -if(NOT TARGET cpuinfo) +if(NOT TARGET cpuinfo AND USE_SYSTEM_CPUINFO) + add_library(cpuinfo SHARED IMPORTED) + find_library(CPUINFO_LIBRARY cpuinfo) + message("Found cpuinfo: ${CPUINFO_LIBRARY}") + set_target_properties(cpuinfo PROPERTIES IMPORTED_LOCATION "${CPUINFO_LIBRARY}") +elseif(NOT TARGET cpuinfo) set(CPUINFO_BUILD_TOOLS OFF CACHE BOOL "") set(CPUINFO_BUILD_UNIT_TESTS OFF CACHE BOOL "") set(CPUINFO_BUILD_MOCK_TESTS OFF CACHE BOOL "") diff --git a/cmake/Dependencies.cmake b/cmake/Dependencies.cmake index 78e38b66c02..6d0f956bf17 100644 --- a/cmake/Dependencies.cmake +++ b/cmake/Dependencies.cmake @@ -310,7 +310,12 @@ if(USE_XNNPACK) endif() # ---[ Caffe2 uses cpuinfo library in the thread pool -if(NOT TARGET cpuinfo) +if(NOT TARGET cpuinfo AND USE_SYSTEM_CPUINFO) + add_library(cpuinfo SHARED IMPORTED) + find_library(CPUINFO_LIBRARY cpuinfo) + message("Found cpuinfo: ${CPUINFO_LIBRARY}") + set_target_properties(cpuinfo PROPERTIES IMPORTED_LOCATION "${CPUINFO_LIBRARY}") +elseif(NOT TARGET cpuinfo) if(NOT DEFINED CPUINFO_SOURCE_DIR) set(CPUINFO_SOURCE_DIR "${CMAKE_CURRENT_LIST_DIR}/../third_party/cpuinfo" CACHE STRING "cpuinfo source directory") endif() diff --git a/setup.py b/setup.py index 68db20d7c19..fbcd613dc82 100644 --- a/setup.py +++ b/setup.py @@ -157,6 +157,10 @@ # USE_TBB # enable TBB support # +# USE_SYSTEM_LIBS (work in progress) +# Use system-provided libraries to satisfy the build dependencies. +# When turned on, the following cmake variables will be toggled as well: +# USE_SYSTEM_CPUINFO=ON USE_SYSTEM_SLEEF=ON BUILD_CUSTOM_PROTOBUF=OFF from __future__ import print_function