Adapting conda build to work for ubuntu and adding a flag to control precedence of Anaconda include dirs

Summary:
This should fix Protobuf version problems on all Anaconda builds by putting include directories under Anaconda before all other include directories.
Closes https://github.com/caffe2/caffe2/pull/1728

Reviewed By: orionr

Differential Revision: D6698435

Pulled By: pjh5

fbshipit-source-id: f73f4a5ebb4ca91db14770a88a704ace69d37ba4

CMakeLists.txt

@@ -45,6 +45,7 @@ option(USE_SNPE "Use Qualcomm's SNPE library" OFF)
 option(USE_THREADS "Use Threads" ON)
 option(USE_ZMQ "Use ZMQ" OFF)
 option(USE_ZSTD "Use ZSTD" OFF)
+option(DEPRIORITIZE_ANACONDA "Search system include directories before Anaconda include directories" OFF)
 
 
 # ---[ CMake scripts + modules

caffe2/CMakeLists.txt

@@ -100,7 +100,6 @@ target_link_libraries(caffe2 PRIVATE ${Caffe2_DEPENDENCY_LIBS})
 target_include_directories(caffe2 INTERFACE $<INSTALL_INTERFACE:include>)
 target_compile_options(caffe2 INTERFACE "-std=c++11")
 install(TARGETS caffe2 EXPORT Caffe2Targets DESTINATION lib)
-link_directories(${CMAKE_INSTALL_PREFIX}/lib)
 caffe_add_linker_flag(caffe2 Caffe2_CPU_LINK)
 list(APPEND Caffe2_MAIN_LIBS_ORDER caffe2 Caffe2_PROTO)
 list(APPEND Caffe2_MAIN_LIBS ${Caffe2_CPU_LINK})

cmake/Summary.cmake

@@ -10,6 +10,9 @@ function (caffe2_print_configuration_summary)
   message(STATUS "  C++ compiler          : ${CMAKE_CXX_COMPILER}")
   message(STATUS "  C++ compiler version  : ${CMAKE_CXX_COMPILER_VERSION}")
   message(STATUS "  Protobuf compiler     : ${PROTOBUF_PROTOC_EXECUTABLE}")
+  message(STATUS "  Protoc libraries      : ${PROTOBUF_PROTOC_LIBRARIES}")
+  message(STATUS "  Protobuf libraries    : ${PROTOBUF_LIBRARIES}")
+  message(STATUS "  Protobuf include dirs : ${PROTOBUF_INCLUDE_DIR}")
   message(STATUS "  CXX flags             : ${CMAKE_CXX_FLAGS}")
   message(STATUS "  Build type            : ${CMAKE_BUILD_TYPE}")
   get_directory_property(tmp DIRECTORY ${PROJECT_SOURCE_DIR} COMPILE_DEFINITIONS)

cmake/Utils.cmake

@@ -254,8 +254,10 @@ endfunction()
 # Anaconda distributions typically contain a lot of packages and some
 # of those can conflict with headers/libraries that must be sourced
 # from elsewhere. This helper ensures that Anaconda paths are always
-# added AFTER other include paths, such that it does not accidentally
-# takes precedence when it shouldn't.
+# added BEFORE other include paths. This prevents a common case where
+# libraries and binaries are linked from Anaconda but headers are
+# included from system packages, since system include directories come
+# before Anaconda include directories by default. 
 #
 # This is just a heuristic and does not have any guarantees. We can
 # add other corner cases here (as long as they are generic enough).
@@ -264,10 +266,22 @@ endfunction()
 #
 function(caffe2_include_directories)
   foreach(path IN LISTS ARGN)
-    if (${path} MATCHES "/anaconda")
-      include_directories(AFTER SYSTEM ${path})
+    if (${DEPRIORITIZE_ANACONDA})
+      # When not preferring anaconda, always search system header files before
+      # anaconda include directories
+      if (${path} MATCHES "/anaconda")
+        include_directories(AFTER ${path})
+      else()
+        include_directories(BEFORE ${path})
+      endif()
     else()
-      include_directories(BEFORE SYSTEM ${path})
+      # When prefering Anaconda, always search anaconda for header files before
+      # system include directories
+      if (${path} MATCHES "/anaconda")
+        include_directories(BEFORE ${path})
+      else()
+        include_directories(AFTER ${path})
+      endif()
     endif()
   endforeach()
 endfunction()

conda/build.sh

@@ -16,24 +16,19 @@
 # Also, failed builds will accumulate those caffe2_<timestamp> directories. You
 # can remove them after a succesfull build with
 # $ conda build purge
+#
 
 set -ex
 
-if [ -z "$PREFIX" ]; then
-  PREFIX="$CONDA_PREFIX"
-fi
 echo "Installing caffe2 to ${PREFIX}"
 
-# conda build will copy everything over, including build directories.
-# Don't let this pollute the build!
-rm -rf build || true
+PYTHON_ARGS="$(python ./scripts/get_python_cmake_flags.py)"
+CMAKE_ARGS=()
 
 # Default leveldb from conda-forge doesn't work. If you want to use leveldb,
 # use this old pip version
 # pip install leveldb==0.18
-
-PYTHON_ARGS="$(python ./scripts/get_python_cmake_flags.py)"
-CMAKE_ARGS=()
+CMAKE_ARGS+=("-DUSE_LEVELDB=OFF")
 
 # This installation defaults to using MKL because it is much faster. If you
 # want to build without MKL then you should also remove mkl from meta.yaml in
@@ -54,13 +49,15 @@ CMAKE_ARGS+=("-DUSE_NCCL=OFF")
 # Install under specified prefix
 CMAKE_ARGS+=("-DCMAKE_INSTALL_PREFIX=$PREFIX")
 CMAKE_ARGS+=("-DCMAKE_PREFIX_PATH=$PREFIX")
-CMAKE_ARGS+=("-DUSE_LEVELDB=OFF")
 
-# Build. Note this assumes uname==Darwin as this script is meant for mac
 mkdir -p build
 cd build
 cmake "${CMAKE_ARGS[@]}"  $CONDA_CMAKE_ARGS $PYTHON_ARGS ..
-make VERBOSE=1 "-j$(sysctl -n hw.ncpu)"
+if [ "$(uname)" == 'Darwin' ]; then
+  make "-j$(sysctl -n hw.ncpu)"
+else
+  make "-j$(nproc)"
+fi
 
 make install/fast
 

conda/conda_build_config.yaml

@@ -0,0 +1,6 @@
+protobuf:
+  - 3.4.1
+pin_run_as_build:
+  protobuf:
+    min_pin: x.x
+    max_pin: x.x

conda/meta.yaml

@@ -8,8 +8,8 @@ source:
   path: ..
 
 build:
-  number: 1
-  skip: True  # [win or py3k]
+  number: 0
+  skip: True  # [win]
   script_env:
     - CONDA_CMAKE_ARGS
 
@@ -19,9 +19,8 @@ requirements:
     - glog
     - numpy
     - six
-    - eigen
     - python
-    - protobuf
+    - protobuf {{ protobuf }}
     - cmake
     - lmdb
     - snappy ==1.1.6
@@ -32,7 +31,6 @@ requirements:
     - glog
     - numpy
     - six
-    - eigen
     - python
     - protobuf
     - lmdb
@@ -41,11 +39,15 @@ requirements:
     - mkl-include
     - mkl
 
+test:
+  imports:
+    - caffe2.python.core
+
 about:
   home: https://caffe2.ai/
   license: BSD
-  summary: deep learning library
+  summary: Caffe2 is a lightweight, modular, and scalable deep learning framework.
 
 extra:
   recipe-maintainers:
-    - ezyang
+    - pjh5