From 5eec4f66edbfd6441056eed837432a42e191dcd2 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Mon, 17 Aug 2020 09:40:31 -0700 Subject: [PATCH] Refactor manylinux docker image and the related pipelines (#4751) 1. Publish the image ACR, instead of building it every time for every PR 2. Make USE_MKLML and USE_OPENMP be able to co-exist. Currently both of them are enabled in our Linux CI build but indeed only one of them is taking effect. 3. Split nuphar and DNNL to separated pipelines. 4. Fix two warnings in onnxruntime/core/optimizer/matmul_scale_fusion.cc and onnxruntime/test/tvm/tvm_basic_test.cc. 5. Update the manylinux2010_x86_64 image to the latest. --- BUILD.md | 4 +- cgmanifests/cgmanifest.json | 10 ++ cmake/CMakeLists.txt | 24 ++-- cmake/external/dnnl.cmake | 18 ++- cmake/onnxruntime_providers.cmake | 9 +- cmake/onnxruntime_unittests.cmake | 31 ++--- .../runtest-docker.sh | 22 +--- .../core/optimizer/matmul_scale_fusion.cc | 2 +- onnxruntime/test/tvm/tvm_basic_test.cc | 2 +- setup.py | 2 +- tools/ci_build/build.py | 77 ++++++------ .../c-api-packaging-pipelines.yml | 23 ++-- .../java-api-packaging-pipelines.yml | 22 ++-- .../azure-pipelines/linux-ci-pipeline.yml | 70 +++++++++-- .../linux-dnnl-ci-pipeline.yml | 63 ++++++++++ .../azure-pipelines/linux-gpu-ci-pipeline.yml | 69 ++++++++-- .../linux-nuphar-ci-pipeline.yml | 9 ++ .../azure-pipelines/nodejs/templates/cpu.yml | 22 +++- .../nuget/templates/cpu-mklml.yml | 26 ++-- .../templates/cpu-nocontribops-arm64.yml | 20 ++- .../azure-pipelines/nuget/templates/cpu.yml | 20 ++- .../nuget/templates/test_linux.yml | 20 +-- .../orttraining-py-packaging-pipeline.yml | 3 +- .../templates/py-packaging-stage.yml | 119 +++++++++++++++--- .../github/linux/docker/Dockerfile.centos6 | 14 --- .../linux/docker/Dockerfile.manylinux2010_cpu | 10 +- .../linux/docker/Dockerfile.manylinux2010_gpu | 10 +- .../linux/docker/manylinux/install_centos.sh | 40 ++++++ .../linux/docker/manylinux/install_deps.sh | 91 ++++++++++++++ .../linux/docker/manylinux/requirements.txt | 6 + .../docker/scripts/install_server_deps.sh | 9 +- .../linux/docker/scripts/requirements.txt | 3 +- 32 files changed, 650 insertions(+), 220 deletions(-) create mode 100644 tools/ci_build/github/azure-pipelines/linux-dnnl-ci-pipeline.yml create mode 100644 tools/ci_build/github/azure-pipelines/linux-nuphar-ci-pipeline.yml delete mode 100644 tools/ci_build/github/linux/docker/Dockerfile.centos6 create mode 100755 tools/ci_build/github/linux/docker/manylinux/install_centos.sh create mode 100755 tools/ci_build/github/linux/docker/manylinux/install_deps.sh create mode 100644 tools/ci_build/github/linux/docker/manylinux/requirements.txt diff --git a/BUILD.md b/BUILD.md index b97ef6fe43..f2a41d20ff 100644 --- a/BUILD.md +++ b/BUILD.md @@ -420,14 +420,14 @@ index 7dfa97c..6d99e71 100644 #### Build Instructions ##### Windows ``` -.\build.bat --use_tvm --use_llvm --llvm_path=\llvm\install\path\lib\cmake\llvm --use_mklml --use_nuphar --build_shared_lib --build_csharp --enable_pybind --config=Release +.\build.bat --llvm_path=\llvm\install\path\lib\cmake\llvm --use_mklml --use_nuphar --build_shared_lib --build_csharp --enable_pybind --config=Release ``` * These instructions build the release flavor. The Debug build of LLVM would be needed to build with the Debug flavor of ONNX Runtime. ##### Linux: ``` -./build.sh --use_tvm --use_llvm --llvm_path=/llvm/install/path/lib/cmake/llvm --use_mklml --use_nuphar --build_shared_lib --build_csharp --enable_pybind --config=Release +./build.sh --llvm_path=/llvm/install/path/lib/cmake/llvm --use_mklml --use_nuphar --build_shared_lib --build_csharp --enable_pybind --config=Release ``` Dockerfile instructions are available [here](./dockerfiles#nuphar). diff --git a/cgmanifests/cgmanifest.json b/cgmanifests/cgmanifest.json index c0cc258dc3..6543b2a554 100644 --- a/cgmanifests/cgmanifest.json +++ b/cgmanifests/cgmanifest.json @@ -337,6 +337,16 @@ }, "comments": "middleware between IB verbs and OpenMPI used by onnxruntime training image" } + }, + { + "component": { + "type": "git", + "git": { + "commitHash": "63d1e08e64e7e09408eb63cd8dd7c65ad766f277", + "repositoryUrl": "https://github.com/nodejs/node" + }, + "comments": "For Nodejs binding" + } } ], "Version": 1 diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index f0a07dfb93..f4c3d7d297 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -5,6 +5,8 @@ # Minimum CMake required cmake_minimum_required(VERSION 3.13) cmake_policy(SET CMP0069 NEW) +set(CMAKE_POLICY_DEFAULT_CMP0069 NEW) + if (${CMAKE_VERSION} VERSION_GREATER_EQUAL "3.15") cmake_policy(SET CMP0092 NEW) endif() @@ -178,7 +180,7 @@ if(onnxruntime_USE_OPENMP) include_directories(${OpenMP_CXX_INCLUDE_DIR}) # MKLML and NGraph depend on their own OpenMP library that may be different with the compiler's. # Disable the options to build mklml/NGraph and OpenMP together. - if(onnxruntime_USE_MKLML) + if((WIN32 OR APPLE) AND onnxruntime_USE_MKLML) message(FATAL_ERROR "Please use only one of onnxruntime_USE_MKLML, onnxruntime_USE_OPENMP") endif() if(onnxruntime_USE_NGRAPH) @@ -759,19 +761,17 @@ if (onnxruntime_USE_MKLML) if (WIN32 OR APPLE) list(APPEND onnxruntime_EXTERNAL_LIBRARIES mklml) else() - list(APPEND onnxruntime_EXTERNAL_LIBRARIES mklml_intel) + if(onnxruntime_USE_OPENMP) + list(APPEND onnxruntime_EXTERNAL_LIBRARIES mklml_gnu) + else() + list(APPEND onnxruntime_EXTERNAL_LIBRARIES mklml_intel) + endif() endif() list(APPEND onnxruntime_EXTERNAL_DEPENDENCIES project_mklml) include_directories(${MKLML_INCLUDE_DIR}) link_directories(${MKLML_LIB_DIR}) endif() -if (onnxruntime_USE_DNNL) - list(APPEND onnxruntime_EXTERNAL_LIBRARIES dnnl) - list(APPEND onnxruntime_EXTERNAL_DEPENDENCIES project_dnnl) - link_directories(${DNNL_LIB_DIR}) -endif() - if (onnxruntime_USE_NGRAPH) #if (onnxruntime_USE_OPENMP) # message(FATAL_ERROR "Please set onnxruntime_USE_OPENMP=OFF for nGraph execution provider.") @@ -941,7 +941,7 @@ if (onnxruntime_USE_TVM) # wd4456: declaration of X hides previous local declaration set(DISABLED_WARNINGS_FOR_TVM "/wd4100" "/wd4244" "/wd4251" "/wd4267" "/wd4275" "/wd4389" "/wd4456") else() - set(DISABLED_WARNINGS_FOR_TVM "-Wno-error=ignored-qualifiers") + set(DISABLED_WARNINGS_FOR_TVM "-Wno-error=ignored-qualifiers" "-Wno-unused-parameter" "-Wno-ignored-qualifiers") if(HAS_UNUSED_PARAMETER) list(APPEND DISABLED_WARNINGS_FOR_TVM "-Wno-error=unused-parameter") endif() @@ -1166,14 +1166,8 @@ endif() # built; hence the ordering if (onnxruntime_BUILD_UNIT_TESTS) if (onnxruntime_ENABLE_PYTHON) - if(UNIX) - set(CMAKE_SKIP_BUILD_RPATH ON) - endif() include(onnxruntime_python.cmake) endif() - # we need to make sure this is turned off since it - # turned ON by the previous step when building a shared lib - set(CMAKE_SKIP_BUILD_RPATH OFF) include(onnxruntime_unittests.cmake) endif() diff --git a/cmake/external/dnnl.cmake b/cmake/external/dnnl.cmake index f67ecbf49c..8975537b0d 100644 --- a/cmake/external/dnnl.cmake +++ b/cmake/external/dnnl.cmake @@ -23,14 +23,22 @@ else() if (APPLE) set(DNNL_SHARED_LIB libdnnl.1.dylib) set(MKLML_OS_VERSION_STR "mac") + if(onnxruntime_USE_MKLML) + set(MKLML_SHARED_LIB libmklml.dylib) + set(IOMP5MD_SHARED_LIB libiomp5.dylib) + endif() else() set(DNNL_SHARED_LIB libdnnl.so.1) set(MKLML_OS_VERSION_STR "lnx") - endif() - if(onnxruntime_USE_MKLML) - set(MKLML_SHARED_LIB libmklml_intel.so) - set(IOMP5MD_SHARED_LIB libiomp5.so) - endif() + if(onnxruntime_USE_MKLML) + if(onnxruntime_USE_OPENMP) + set(MKLML_SHARED_LIB libmklml_gnu.so) + else() + set(MKLML_SHARED_LIB libmklml_intel.so) + set(IOMP5MD_SHARED_LIB libiomp5.so) + endif() + endif() + endif() endif() if (onnxruntime_USE_MKLML) diff --git a/cmake/onnxruntime_providers.cmake b/cmake/onnxruntime_providers.cmake index 9172f8851a..659f4057da 100644 --- a/cmake/onnxruntime_providers.cmake +++ b/cmake/onnxruntime_providers.cmake @@ -35,10 +35,6 @@ file(GLOB onnxruntime_providers_common_srcs CONFIGURE_DEPENDS "${ONNXRUNTIME_ROOT}/core/providers/*.cc" ) -if(onnxruntime_USE_DNNL) - set(PROVIDERS_DNNL onnxruntime_providers_dnnl) - list(APPEND ONNXRUNTIME_PROVIDER_NAMES dnnl) -endif() if(onnxruntime_USE_NGRAPH) set(PROVIDERS_NGRAPH onnxruntime_providers_ngraph) list(APPEND ONNXRUNTIME_PROVIDER_NAMES ngraph) @@ -314,6 +310,8 @@ if (onnxruntime_USE_TENSORRT OR onnxruntime_USE_DNNL) endif() if (onnxruntime_USE_DNNL) + list(APPEND ONNXRUNTIME_PROVIDER_NAMES dnnl) + file(GLOB_RECURSE onnxruntime_providers_dnnl_cc_srcs CONFIGURE_DEPENDS "${ONNXRUNTIME_ROOT}/core/providers/dnnl/*.h" "${ONNXRUNTIME_ROOT}/core/providers/dnnl/*.cc" @@ -323,8 +321,9 @@ if (onnxruntime_USE_DNNL) source_group(TREE ${ONNXRUNTIME_ROOT}/core FILES ${onnxruntime_providers_dnnl_cc_srcs}) add_library(onnxruntime_providers_dnnl SHARED ${onnxruntime_providers_dnnl_cc_srcs}) + target_link_directories(onnxruntime_providers_dnnl PRIVATE ${DNNL_LIB_DIR}) onnxruntime_add_include_to_target(onnxruntime_providers_dnnl onnxruntime_common onnx) # onnx needed for stl_backports.h - add_dependencies(onnxruntime_providers_dnnl onnxruntime_providers_shared ${onnxruntime_EXTERNAL_DEPENDENCIES}) + add_dependencies(onnxruntime_providers_dnnl onnxruntime_providers_shared project_dnnl ${onnxruntime_EXTERNAL_DEPENDENCIES}) target_include_directories(onnxruntime_providers_dnnl PRIVATE ${ONNXRUNTIME_ROOT} ${eigen_INCLUDE_DIRS} ${DNNL_INCLUDE_DIR}) target_link_libraries(onnxruntime_providers_dnnl PRIVATE dnnl onnxruntime_providers_shared) install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/providers/dnnl DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core/providers) diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake index fcc40dd950..8726558543 100644 --- a/cmake/onnxruntime_unittests.cmake +++ b/cmake/onnxruntime_unittests.cmake @@ -560,22 +560,23 @@ add_custom_command( COMMAND ${CMAKE_COMMAND} -E copy_directory ${TEST_DATA_SRC} ${TEST_DATA_DES}) -if(WIN32) - if (onnxruntime_USE_DNNL) - list(APPEND onnx_test_libs dnnl) - add_custom_command( - TARGET ${test_data_target} POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy ${DNNL_DLL_PATH} $ - ) - endif() - if (onnxruntime_USE_MKLML) - add_custom_command( - TARGET ${test_data_target} POST_BUILD - COMMAND ${CMAKE_COMMAND} -E copy - ${MKLML_LIB_DIR}/${MKLML_SHARED_LIB} ${MKLML_LIB_DIR}/${IOMP5MD_SHARED_LIB} - $ + +if (onnxruntime_USE_DNNL) + list(APPEND onnx_test_libs dnnl) + add_custom_command( + TARGET ${test_data_target} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy ${DNNL_DLL_PATH} $ ) - endif() +endif() +if (onnxruntime_USE_MKLML) + add_custom_command( + TARGET ${test_data_target} POST_BUILD + COMMAND ${CMAKE_COMMAND} -E copy + ${MKLML_LIB_DIR}/${MKLML_SHARED_LIB} ${MKLML_LIB_DIR}/${IOMP5MD_SHARED_LIB} + $ + ) +endif() +if(WIN32) if (onnxruntime_USE_NGRAPH) add_custom_command( TARGET ${test_data_target} POST_BUILD diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker.sh b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker.sh index 4dc4a05a5e..0f18caf15e 100755 --- a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker.sh +++ b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker.sh @@ -22,26 +22,8 @@ OldDir=$(pwd) cd $SOURCE_ROOT/tools/ci_build/github/linux/docker -if [ $UseCentos7 = "false" ]; then - echo "Image used for testing is onnxruntime-$IMAGE" - if [ $Arch = "x86" ]; then - sudo docker build -t "onnxruntime-$IMAGE" --build-arg OS_VERSION=16.04 --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f Dockerfile.ubuntu_x86 . - else - sudo docker build -t "onnxruntime-$IMAGE" --build-arg OS_VERSION=16.04 --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f Dockerfile.ubuntu . - fi -else - IMAGE="centos7" - PYTHON_VER=3.6 - echo "Image used for testing is onnxruntime-$IMAGE" - sudo docker build -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f Dockerfile.centos . -fi - -sudo docker rm -f "onnxruntime-cpu" || true - - -sudo --preserve-env docker run -h $HOSTNAME \ - --rm \ +docker run --rm \ --name "onnxruntime-cpu" \ --volume "$SOURCE_ROOT:/onnxruntime_src" \ --volume "$BUILD_DIR:/home/onnxruntimedev" \ @@ -53,6 +35,6 @@ sudo --preserve-env docker run -h $HOSTNAME \ -e "DisableMlOps=$DISABLEMLOPS" \ -e "RunTestCsharp=$RunTestCsharp" \ -e "RunTestNative=$RunTestNative" \ - "onnxruntime-$IMAGE" \ + onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:chn \ /bin/bash /onnxruntime_src/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.sh \ /home/onnxruntimedev/$NUGET_REPO_DIRNAME /onnxruntime_src /home/onnxruntimedev $CurrentOnnxRuntimeVersion diff --git a/onnxruntime/core/optimizer/matmul_scale_fusion.cc b/onnxruntime/core/optimizer/matmul_scale_fusion.cc index 7356f55177..2b540d691b 100644 --- a/onnxruntime/core/optimizer/matmul_scale_fusion.cc +++ b/onnxruntime/core/optimizer/matmul_scale_fusion.cc @@ -43,7 +43,7 @@ optional GetScalarConstantInitializer(const Graph& graph, const NodeArg& return {}; } - float scalar; + float scalar=0.f; utils::MLTypeCallDispatcherRet< Status, ExtractScalarAsFloatDispatchTarget, uint32_t, uint64_t, int32_t, int64_t, MLFloat16, float, double> diff --git a/onnxruntime/test/tvm/tvm_basic_test.cc b/onnxruntime/test/tvm/tvm_basic_test.cc index 59784caadf..03b7a67d98 100644 --- a/onnxruntime/test/tvm/tvm_basic_test.cc +++ b/onnxruntime/test/tvm/tvm_basic_test.cc @@ -251,7 +251,7 @@ class FuseExecutionProviderX : public CPUExecutionProvider { tvm::TVMRetValue rvalue; try { evaluate_func_.CallPacked(tvm_args, &rvalue); - } catch (std::exception ex) { + } catch (std::exception& ex) { return Status(common::ONNXRUNTIME, common::FAIL); // TODO: Translate exception to error code } if (rvalue.type_code() != kNull) { diff --git a/setup.py b/setup.py index 5fe14996da..6ef3cff5c8 100644 --- a/setup.py +++ b/setup.py @@ -157,7 +157,7 @@ except ImportError as error: # Additional binaries if platform.system() == 'Linux': - libs = ['onnxruntime_pybind11_state.so', 'libdnnl.so.1', 'libmklml_intel.so', 'libiomp5.so', 'mimalloc.so'] + libs = ['onnxruntime_pybind11_state.so', 'libdnnl.so.1', 'libmklml_intel.so', 'libmklml_gnu.so', 'libiomp5.so', 'mimalloc.so'] # DNNL & TensorRT EPs are built as shared libs libs.extend(['libonnxruntime_providers_shared.so']) libs.extend(['libonnxruntime_providers_dnnl.so']) diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 207089c736..0eb506defa 100755 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -265,12 +265,8 @@ def parse_arguments(): "--use_preinstalled_eigen", action='store_true', help="Use pre-installed Eigen.") parser.add_argument("--eigen_path", help="Path to pre-installed Eigen.") - parser.add_argument( - "--use_tvm", action="store_true", help="Build with TVM") parser.add_argument( "--use_openmp", action='store_true', help="Build with OpenMP") - parser.add_argument( - "--use_llvm", action="store_true", help="Build TVM with LLVM") parser.add_argument( "--enable_msinternal", action="store_true", help="Enable for Microsoft internal builds only.") @@ -584,31 +580,16 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home "-Donnxruntime_USE_DNNL=" + ("ON" if args.use_dnnl else "OFF"), "-Donnxruntime_USE_MKLML=" + ("ON" if args.use_mklml else "OFF"), "-Donnxruntime_USE_NGRAPH=" + ("ON" if args.use_ngraph else "OFF"), - "-Donnxruntime_USE_OPENVINO=" + ("ON" if args.use_openvino else "OFF"), - "-Donnxruntime_USE_OPENVINO_MYRIAD=" + ( - "ON" if args.use_openvino == "MYRIAD_FP16" else "OFF"), - "-Donnxruntime_USE_OPENVINO_GPU_FP32=" + ( - "ON" if args.use_openvino == "GPU_FP32" else "OFF"), - "-Donnxruntime_USE_OPENVINO_GPU_FP16=" + ( - "ON" if args.use_openvino == "GPU_FP16" else "OFF"), - "-Donnxruntime_USE_OPENVINO_CPU_FP32=" + ( - "ON" if args.use_openvino == "CPU_FP32" else "OFF"), - "-Donnxruntime_USE_OPENVINO_VAD_M=" + ( - "ON" if args.use_openvino == "VAD-M_FP16" else "OFF"), - "-Donnxruntime_USE_OPENVINO_VAD_F=" + ( - "ON" if args.use_openvino == "VAD-F_FP32" else "OFF"), - "-Donnxruntime_USE_OPENVINO_BINARY=" + ( - "ON" if args.use_openvino else "OFF"), "-Donnxruntime_USE_NNAPI_BUILTIN=" + ("ON" if args.use_nnapi else "OFF"), "-Donnxruntime_USE_RKNPU=" + ("ON" if args.use_rknpu else "OFF"), "-Donnxruntime_USE_OPENMP=" + ( "ON" if args.use_openmp and not ( - args.use_nnapi or args.use_mklml or args.use_ngraph or + args.use_nnapi or (args.use_mklml and (is_macOS() or is_windows())) or args.use_ngraph or args.android or (args.ios and is_macOS()) or args.use_rknpu) else "OFF"), - "-Donnxruntime_USE_TVM=" + ("ON" if args.use_tvm else "OFF"), - "-Donnxruntime_USE_LLVM=" + ("ON" if args.use_llvm else "OFF"), + "-Donnxruntime_USE_TVM=" + ("ON" if args.use_nuphar else "OFF"), + "-Donnxruntime_USE_LLVM=" + ("ON" if args.use_nuphar else "OFF"), "-Donnxruntime_ENABLE_MICROSOFT_INTERNAL=" + ( "ON" if args.enable_msinternal else "OFF"), "-Donnxruntime_USE_VITISAI=" + ("ON" if args.use_vitisai else "OFF"), @@ -674,7 +655,22 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home if args.winml_root_namespace_override: cmake_args += ["-Donnxruntime_WINML_NAMESPACE_OVERRIDE=" + args.winml_root_namespace_override] - + if args.use_openvino: + cmake_args += ["-Donnxruntime_USE_OPENVINO=ON", + "-Donnxruntime_USE_OPENVINO_MYRIAD=" + ( + "ON" if args.use_openvino == "MYRIAD_FP16" else "OFF"), + "-Donnxruntime_USE_OPENVINO_GPU_FP32=" + ( + "ON" if args.use_openvino == "GPU_FP32" else "OFF"), + "-Donnxruntime_USE_OPENVINO_GPU_FP16=" + ( + "ON" if args.use_openvino == "GPU_FP16" else "OFF"), + "-Donnxruntime_USE_OPENVINO_CPU_FP32=" + ( + "ON" if args.use_openvino == "CPU_FP32" else "OFF"), + "-Donnxruntime_USE_OPENVINO_VAD_M=" + ( + "ON" if args.use_openvino == "VAD-M_FP16" else "OFF"), + "-Donnxruntime_USE_OPENVINO_VAD_F=" + ( + "ON" if args.use_openvino == "VAD-F_FP32" else "OFF"), + "-Donnxruntime_USE_OPENVINO_BINARY=" + ( + "ON" if args.use_openvino else "OFF")] # temp turn on only for linux gpu build if not is_windows(): if args.use_cuda: @@ -690,7 +686,7 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home "-DProtobuf_USE_STATIC_LIBS=ON" ] - if args.use_llvm: + if args.use_nuphar and args.llvm_path is not None: cmake_args += ["-DLLVM_DIR=%s" % args.llvm_path] if args.use_cuda and not is_windows(): @@ -709,12 +705,6 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home "-DANDROID_ABI=" + str(args.android_abi) ] - if is_macOS(): - if args.use_xcode: - cmake_args += ['-G', 'Xcode'] - elif args.cmake_generator is not None: - cmake_args += ['-G', args.cmake_generator] - if args.ios: if is_macOS(): needed_args = [ @@ -813,8 +803,7 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home cmake_args += ["-D{}".format(define) for define in cmake_extra_defines] - if is_windows(): - cmake_args += cmake_extra_args + cmake_args += cmake_extra_args # ADO pipelines will store the pipeline build number # (e.g. 191101-2300.1.master) and source version in environment @@ -858,16 +847,15 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home for config in configs: config_build_dir = get_config_build_dir(build_dir, config) os.makedirs(config_build_dir, exist_ok=True) - - if args.use_tvm: + if args.use_nuphar: os.environ["PATH"] = os.path.join( config_build_dir, "external", "tvm", - config) + os.pathsep + os.environ["PATH"] + config) + os.pathsep + os.path.dirname(sys.executable) + os.pathsep + os.environ["PATH"] run_subprocess( cmake_args + [ "-Donnxruntime_ENABLE_MEMLEAK_CHECKER=" + - ("ON" if config.lower() == 'debug' and not args.use_tvm and not + ("ON" if config.lower() == 'debug' and not args.use_nuphar and not args.use_ngraph and not args.use_openvino and not args.enable_msvc_static_runtime else "OFF"), "-DCMAKE_BUILD_TYPE={}".format(config)], @@ -906,7 +894,7 @@ def build_targets(args, cmake_path, build_dir, configs, parallel): elif (is_macOS() and args.use_xcode): # CMake will generate correct build tool args for Xcode cmd_args += ["--parallel", num_cores] - else: + elif args.cmake_generator != 'Ninja': build_tool_args += ["-j" + num_cores] if build_tool_args: @@ -1174,13 +1162,16 @@ def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs): 'cd /data/local/tmp && /data/local/tmp/onnx_test_runner /data/local/tmp/test') # noqa continue dll_path_list = [] - if args.use_tvm: + if args.use_nuphar: dll_path_list.append(os.path.join( build_dir, config, "external", "tvm", config)) if args.use_tensorrt: dll_path_list.append(os.path.join(args.tensorrt_home, 'lib')) if args.use_mklml: dll_path_list.append(os.path.join(build_dir, config, "mklml", "src", "project_mklml", "lib")) + if not is_windows(): + # A workaround for making libonnxruntime_providers_shared.so loadable. + dll_path_list.append(os.path.join(build_dir, config)) dll_path = None if len(dll_path_list) > 0: @@ -1407,16 +1398,14 @@ def build_protoc_for_host(cmake_path, source_dir, build_dir, args): ] is_ninja = args.cmake_generator == 'Ninja' - + if args.cmake_generator is not None and not (is_macOS() and args.use_xcode): + cmd_args += ['-G', args.cmake_generator] if is_windows(): if not is_ninja: cmd_args += ['-T', 'host=x64'] - cmd_args += ['-G', args.cmake_generator] elif is_macOS(): if args.use_xcode: cmd_args += ['-G', 'Xcode'] - elif args.cmake_generator is not None: - cmd_args += ['-G', args.cmake_generator] run_subprocess(cmd_args, cwd=protoc_build_dir) # Build step @@ -1614,6 +1603,10 @@ def main(): cmake_extra_args.append( '-DCMAKE_TOOLCHAIN_FILE=' + os.path.join( source_dir, 'cmake', 'wcos_toolchain.cmake')) + elif args.cmake_generator is not None and not (is_macOS() and args.use_xcode): + cmake_extra_args += ['-G', args.cmake_generator] + elif is_macOS() and args.use_xcode: + cmake_extra_args += ['-G', 'Xcode'] if (args.android or args.ios) and args.path_to_protoc_exe is None: # Cross-compiling for Android and iOS diff --git a/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml index 529aa3756f..cfde7c25d8 100644 --- a/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml +++ b/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml @@ -7,18 +7,27 @@ jobs: vmImage: 'ubuntu-latest' steps: - template: templates/set-version-number-variables-step.yml + - task: Docker@2 + displayName: login + inputs: + containerRegistry: onnxruntimeregistry + command: login + addPipelineData: false - task: CmdLine@2 inputs: script: | - docker build --pull -t onnxruntime-centos6 --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=3.6 -f Dockerfile.centos6 . - workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker - - task: CmdLine@2 - inputs: - script: | + mkdir -p $HOME/.onnx docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro \ - -e NIGHTLY_BUILD onnxruntime-centos6 /usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib \ - --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest + --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:chn /opt/python/cp37-cp37m/bin/python3 \ + /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --enable_lto \ + --skip_submodule_sync --parallel --build_shared_lib --use_openmp workingDirectory: $(Build.SourcesDirectory) + - task: Docker@2 + displayName: logout + inputs: + containerRegistry: onnxruntimeregistry + command: logout + addPipelineData: false - template: templates/c-api-artifacts-package-and-publish-steps-posix.yml parameters: buildConfig: 'Release' diff --git a/tools/ci_build/github/azure-pipelines/java-api-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/java-api-packaging-pipelines.yml index 9eaa40f772..291bd924f7 100644 --- a/tools/ci_build/github/azure-pipelines/java-api-packaging-pipelines.yml +++ b/tools/ci_build/github/azure-pipelines/java-api-packaging-pipelines.yml @@ -16,19 +16,25 @@ jobs: - template: templates/set-version-number-variables-step.yml - template: templates/linux-set-variables-and-download.yml + - task: Docker@2 + displayName: login + inputs: + containerRegistry: onnxruntimeregistry + command: login + addPipelineData: false - task: CmdLine@2 inputs: script: | - docker build --pull -t onnxruntime-centos6 --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=3.6 -f Dockerfile.centos6 . - workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker - displayName: 'Build CentoOS6 docker image' - - task: CmdLine@2 - inputs: - script: | - docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --build_java --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64" + mkdir -p $HOME/.onnx + docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:chn /bin/bash -c "/opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --build_java --use_openmp --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64" workingDirectory: $(Build.SourcesDirectory) displayName: 'Run build and test' - + - task: Docker@2 + displayName: logout + inputs: + containerRegistry: onnxruntimeregistry + command: logout + addPipelineData: false - template: templates/java-api-artifacts-package-and-publish-steps-posix.yml parameters: arch: 'linux-x64' diff --git a/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml index a65b0ad991..a4d75e8f38 100644 --- a/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml @@ -1,9 +1,63 @@ jobs: -- template: templates/linux-ci.yml - parameters: - AgentPool : 'Linux-CPU' - JobName: 'Linux_CI_Dev' - BuildCommand: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -x "--use_mklml --use_llvm --use_nuphar --use_dnnl --use_tvm --build_wheel --build_java --build_nodejs --enable_language_interop_ops"' - DoNugetPack: 'false' - ArtifactName: 'drop-linux' - TimeoutInMinutes: 180 +- job: Linux_Build + timeoutInMinutes: 120 + workspace: + clean: all + pool: Linux-CPU + steps: + - checkout: self + clean: true + submodules: recursive + + - task: Docker@2 + displayName: login + inputs: + containerRegistry: onnxruntimeregistry + command: login + addPipelineData: false + - task: CmdLine@2 + inputs: + script: | + mkdir -p $HOME/.onnx + docker run --rm \ + --volume /data/onnx:/data/onnx:ro \ + --volume $(Build.SourcesDirectory):/onnxruntime_src \ + --volume $(Build.BinariesDirectory):/build \ + --volume /data/models:/build/models:ro \ + --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \ + -e NIGHTLY_BUILD \ + -e BUILD_BUILDNUMBER \ + onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:chn \ + /opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py \ + --build_dir /build --cmake_generator Ninja \ + --config Debug Release \ + --skip_submodule_sync \ + --build_shared_lib \ + --parallel \ + --build_wheel \ + --use_openmp \ + --enable_onnx_tests \ + --use_mklml --enable_pybind --build_java --build_nodejs \ + --cmake_extra_defines PYTHON_INCLUDE_DIR=/opt/python/cp37-cp37m/include/python3.7m PYTHON_LIBRARY=/usr/lib64/librt.so + workingDirectory: $(Build.SourcesDirectory) + - task: Docker@2 + displayName: logout + inputs: + containerRegistry: onnxruntimeregistry + command: logout + addPipelineData: false + - task: PublishTestResults@2 + displayName: 'Publish unit test results' + inputs: + testResultsFiles: '**/*.results.xml' + searchFolder: '$(Build.BinariesDirectory)' + testRunTitle: 'Unit Test Run' + condition: succeededOrFailed() + + - template: templates/component-governance-component-detection-steps.yml + parameters: + condition: 'succeeded' + + - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 + displayName: 'Clean Agent Directories' + condition: always() diff --git a/tools/ci_build/github/azure-pipelines/linux-dnnl-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-dnnl-ci-pipeline.yml new file mode 100644 index 0000000000..1d660d99e4 --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/linux-dnnl-ci-pipeline.yml @@ -0,0 +1,63 @@ +jobs: +- job: Linux_py_Wheels + timeoutInMinutes: 180 + workspace: + clean: all + pool: Linux-CPU + steps: + - checkout: self + clean: true + submodules: recursive + + - task: Docker@2 + displayName: login + inputs: + containerRegistry: onnxruntimeregistry + command: login + addPipelineData: false + - task: CmdLine@2 + inputs: + script: | + mkdir -p $HOME/.onnx + docker run --rm \ + --volume /data/onnx:/data/onnx:ro \ + --volume $(Build.SourcesDirectory):/onnxruntime_src \ + --volume $(Build.BinariesDirectory):/build \ + --volume /data/models:/build/models:ro \ + --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \ + -e NIGHTLY_BUILD \ + -e BUILD_BUILDNUMBER \ + onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:chn \ + /opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py \ + --build_dir /build \ + --config Debug Release \ + --skip_submodule_sync \ + --build_shared_lib \ + --parallel \ + --enable_pybind \ + --use_openmp \ + --enable_onnx_tests \ + --use_dnnl \ + --cmake_extra_defines PYTHON_INCLUDE_DIR=/opt/python/cp37-cp37m/include/python3.7m PYTHON_LIBRARY=/usr/lib64/librt.so + workingDirectory: $(Build.SourcesDirectory) + - task: Docker@2 + displayName: logout + inputs: + containerRegistry: onnxruntimeregistry + command: logout + addPipelineData: false + - task: PublishTestResults@2 + displayName: 'Publish unit test results' + inputs: + testResultsFiles: '**/*.results.xml' + searchFolder: '$(Build.BinariesDirectory)' + testRunTitle: 'Unit Test Run' + condition: succeededOrFailed() + + - template: templates/component-governance-component-detection-steps.yml + parameters: + condition: 'succeeded' + + - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 + displayName: 'Clean Agent Directories' + condition: always() diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml index cbadf2c4e0..e2f8a4f08b 100644 --- a/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml @@ -1,8 +1,63 @@ jobs: -- template: templates/linux-ci.yml - parameters: - AgentPool : 'Linux-GPU-CUDA10' - JobName: 'Linux_CI_GPU_Dev' - BuildCommand: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d gpu -r $(Build.BinariesDirectory) -x "--build_wheel"' - DoNugetPack: 'false' - ArtifactName: 'drop-linux' +- job: Linux_Build + timeoutInMinutes: 120 + workspace: + clean: all + pool: Linux-GPU-CUDA10 + steps: + - checkout: self + clean: true + submodules: recursive + + - task: Docker@2 + displayName: login + inputs: + containerRegistry: onnxruntimeregistry + command: login + addPipelineData: false + - task: CmdLine@2 + inputs: + script: | + mkdir -p $HOME/.onnx + docker run --gpus all --rm \ + --volume /data/onnx:/data/onnx:ro \ + --volume $(Build.SourcesDirectory):/onnxruntime_src \ + --volume $(Build.BinariesDirectory):/build \ + --volume /data/models:/build/models:ro \ + --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \ + -e NIGHTLY_BUILD \ + -e BUILD_BUILDNUMBER \ + onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:chp \ + /opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py \ + --build_dir /build --cmake_generator Ninja \ + --config Debug Release \ + --skip_submodule_sync \ + --build_shared_lib \ + --parallel \ + --build_wheel \ + --use_openmp \ + --enable_onnx_tests --use_cuda --cuda_version=10.1 --cuda_home=/usr/local/cuda-10.1 --cudnn_home=/usr/local/cuda-10.1 \ + --use_mklml --enable_pybind --build_java --build_nodejs \ + --cmake_extra_defines PYTHON_INCLUDE_DIR=/opt/python/cp37-cp37m/include/python3.7m PYTHON_LIBRARY=/usr/lib64/librt.so + workingDirectory: $(Build.SourcesDirectory) + - task: Docker@2 + displayName: logout + inputs: + containerRegistry: onnxruntimeregistry + command: logout + addPipelineData: false + - task: PublishTestResults@2 + displayName: 'Publish unit test results' + inputs: + testResultsFiles: '**/*.results.xml' + searchFolder: '$(Build.BinariesDirectory)' + testRunTitle: 'Unit Test Run' + condition: succeededOrFailed() + + - template: templates/component-governance-component-detection-steps.yml + parameters: + condition: 'succeeded' + + - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 + displayName: 'Clean Agent Directories' + condition: always() diff --git a/tools/ci_build/github/azure-pipelines/linux-nuphar-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-nuphar-ci-pipeline.yml new file mode 100644 index 0000000000..5fb4244f3c --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/linux-nuphar-ci-pipeline.yml @@ -0,0 +1,9 @@ +jobs: +- template: templates/linux-ci.yml + parameters: + AgentPool : 'Linux-CPU' + JobName: 'Linux_CI_Dev' + BuildCommand: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -x "--enable_pybind --use_nuphar"' + DoNugetPack: 'false' + ArtifactName: 'drop-linux' + TimeoutInMinutes: 180 \ No newline at end of file diff --git a/tools/ci_build/github/azure-pipelines/nodejs/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nodejs/templates/cpu.yml index 729fdae8b5..6edf95bfbe 100644 --- a/tools/ci_build/github/azure-pipelines/nodejs/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nodejs/templates/cpu.yml @@ -38,16 +38,26 @@ jobs: steps: - template: ../../templates/set-version-number-variables-step.yml - template: ../../templates/linux-set-variables-and-download.yml + - task: Docker@2 + displayName: login + inputs: + containerRegistry: onnxruntimeregistry + command: login + addPipelineData: false - task: CmdLine@2 inputs: script: | - docker build --pull -t onnxruntime-centos7 --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=3.6 -f Dockerfile.centos . - workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker - - task: CmdLine@2 - inputs: - script: | - docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro -e NIGHTLY_BUILD onnxruntime-centos7 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_nodejs --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --enable_onnx_tests && cd /onnxruntime_src/nodejs && npm pack" + mkdir -p $HOME/.onnx && docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro \ + --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:chn /bin/bash -c "/opt/python/cp37-cp37m/bin/python3 \ + /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --enable_lto --build_nodejs \ + --skip_submodule_sync --parallel --build_shared_lib --use_openmp && cd /onnxruntime_src/nodejs && npm pack" workingDirectory: $(Build.SourcesDirectory) + - task: Docker@2 + displayName: logout + inputs: + containerRegistry: onnxruntimeregistry + command: logout + addPipelineData: false - script: | set -e -x cp $(Build.SourcesDirectory)/nodejs/onnxruntime-*.tgz $(Build.ArtifactStagingDirectory) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml index 635ac12a9b..5b33ee7481 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml @@ -30,21 +30,28 @@ jobs: steps: - template: ../../templates/set-version-number-variables-step.yml - template: ../../templates/linux-set-variables-and-download.yml + - task: Docker@2 + displayName: login + inputs: + containerRegistry: onnxruntimeregistry + command: login + addPipelineData: false - task: CmdLine@2 inputs: script: | - docker build --pull -t onnxruntime-centos6 --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=3.6 -f Dockerfile.centos6 . - workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker - - task: CmdLine@2 - inputs: - script: | - docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_mklml --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64" + mkdir -p $HOME/.onnx + docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:chn /bin/bash -c "/opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --enable_lto --skip_submodule_sync --parallel --build_shared_lib --use_openmp --enable_onnx_tests --use_mklml && cd /build/Release && make install DESTDIR=/build/linux-x64" workingDirectory: $(Build.SourcesDirectory) + - task: Docker@2 + displayName: logout + inputs: + containerRegistry: onnxruntimeregistry + command: logout + addPipelineData: false - script: | set -e -x mv $(Build.BinariesDirectory)/linux-x64/usr/local/lib64 $(Build.BinariesDirectory)/linux-x64/linux-x64 - cp $(Build.BinariesDirectory)/Release/mklml/src/project_mklml/lib/libiomp5.so $(Build.BinariesDirectory)/linux-x64/linux-x64 - cp $(Build.BinariesDirectory)/Release/mklml/src/project_mklml/lib/libmklml_intel.so $(Build.BinariesDirectory)/linux-x64/linux-x64 + cp $(Build.BinariesDirectory)/Release/mklml/src/project_mklml/lib/libmklml_gnu.so $(Build.BinariesDirectory)/linux-x64/linux-x64 ldd $(Build.BinariesDirectory)/linux-x64/linux-x64/libonnxruntime.so cd $(Build.BinariesDirectory)/linux-x64 zip -r linux-x64.zip linux-x64 @@ -140,8 +147,7 @@ jobs: mkdir %%~ni\runtimes\linux-x64 mkdir %%~ni\runtimes\linux-x64\native move linux-x64\linux-x64\libonnxruntime.so %%~ni\runtimes\linux-x64\native\libonnxruntime.so - move linux-x64\linux-x64\libiomp5.so %%~ni\runtimes\linux-x64\native\libiomp5.so - move linux-x64\linux-x64\libmklml_intel.so %%~ni\runtimes\linux-x64\native\libmklml_intel.so + move linux-x64\linux-x64\libmklml_gnu.so %%~ni\runtimes\linux-x64\native\libmklml_gnu.so unzip osx-x64.zip -d osx-x64 dir osx-x64 /s mkdir %%~ni\runtimes\osx-x64 diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml index f31670be80..bed1860fc8 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml @@ -72,16 +72,24 @@ jobs: steps: - template: ../../templates/set-version-number-variables-step.yml - template: ../../templates/linux-set-variables-and-download.yml + - task: Docker@2 + displayName: login + inputs: + containerRegistry: onnxruntimeregistry + command: login + addPipelineData: false - task: CmdLine@2 inputs: script: | - docker build --pull -t onnxruntime-centos6 --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=3.6 -f Dockerfile.centos6 . - workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker - - task: CmdLine@2 - inputs: - script: | - docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --enable_onnx_tests --disable_contrib_ops --disable_ml_ops && cd /build/Release && make install DESTDIR=/build/linux-x64" + mkdir -p $HOME/.onnx + docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:chn /bin/bash -c "/opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --enable_lto --skip_submodule_sync --parallel --build_shared_lib --enable_onnx_tests --disable_contrib_ops --disable_ml_ops && cd /build/Release && make install DESTDIR=/build/linux-x64" workingDirectory: $(Build.SourcesDirectory) + - task: Docker@2 + displayName: logout + inputs: + containerRegistry: onnxruntimeregistry + command: logout + addPipelineData: false - script: | set -e -x mv $(Build.BinariesDirectory)/linux-x64/usr/local/lib64 $(Build.BinariesDirectory)/linux-x64/linux-x64 diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index 42cc8e2ca4..c09acba23b 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -95,16 +95,24 @@ jobs: steps: - template: ../../templates/set-version-number-variables-step.yml - template: ../../templates/linux-set-variables-and-download.yml + - task: Docker@2 + displayName: login + inputs: + containerRegistry: onnxruntimeregistry + command: login + addPipelineData: false - task: CmdLine@2 inputs: script: | - docker build --pull -t onnxruntime-centos6 --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=3.6 -f Dockerfile.centos6 . - workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker - - task: CmdLine@2 - inputs: - script: | - docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64" + mkdir -p $HOME/.onnx + docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:chn /bin/bash -c "/opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --enable_lto --skip_submodule_sync --parallel --build_shared_lib --use_openmp --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64" workingDirectory: $(Build.SourcesDirectory) + - task: Docker@2 + displayName: logout + inputs: + containerRegistry: onnxruntimeregistry + command: logout + addPipelineData: false - script: | set -e -x mv $(Build.BinariesDirectory)/linux-x64/usr/local/lib64 $(Build.BinariesDirectory)/linux-x64/linux-x64 diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml index d37efd2cb2..2c10b8e8c8 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml @@ -34,6 +34,12 @@ jobs: - template: get-nuget-package-version-as-variable.yml parameters: packageFolder: '$(Build.BinariesDirectory)/nuget-artifact' + - task: Docker@2 + displayName: login + inputs: + containerRegistry: onnxruntimeregistry + command: login + addPipelineData: false - ${{ if eq(parameters['TestGPU'], 'false') }}: - script: | @@ -43,13 +49,6 @@ jobs: env: OnnxRuntimeBuildDirectory: $(Build.BinariesDirectory) - - ${{ if eq(parameters['TestGPU'], 'false') }}: - - script: | - set -e -x - $(Build.SourcesDirectory)/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker.sh $(Build.SourcesDirectory) $(Build.BinariesDirectory) nuget-artifact $(NuGetPackageVersionNumber) - displayName: 'Run Package Test (x64) on Ubuntu' - env: - OnnxRuntimeBuildDirectory: $(Build.BinariesDirectory) - ${{ if eq(parameters['TestGPU'], 'true') }}: - script: | @@ -59,6 +58,13 @@ jobs: env: OnnxRuntimeBuildDirectory: $(Build.BinariesDirectory) + - task: Docker@2 + displayName: logout + inputs: + containerRegistry: onnxruntimeregistry + command: logout + addPipelineData: false + - template: ../../templates/component-governance-component-detection-steps.yml parameters : condition : 'always' diff --git a/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline.yml index b3b5e84f49..c435a7f698 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline.yml @@ -5,7 +5,8 @@ stages: parameters: build_py_parameters: --enable_training --wheel_name_suffix=training enable_linux_cpu: false - enable_linux_gpu: true + enable_linux_gpu: false + enable_linux_gpu_training: true # Python 3.5 build has issues with nightly torch and torchvision dependencies enable_linux_gpu_py35: false enable_windows_cpu: false diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml index 7bc444daa7..f5334ea751 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml @@ -15,6 +15,11 @@ parameters: type: boolean default: true +- name: enable_linux_gpu_training + displayName: 'Whether Linux GPU package is built.' + type: boolean + default: false + - name: enable_linux_gpu_py35 displayName: > Whether Linux GPU package is built for Python 3.5. @@ -81,39 +86,43 @@ stages: - template: set-py-packaging-variables-step.yml + - task: Docker@2 + displayName: login + inputs: + containerRegistry: onnxruntimeregistry + command: login + addPipelineData: false - task: CmdLine@2 inputs: script: | - docker build --pull \ - -t ${{ parameters.docker_image_prefix }}-manylinux-$(python.version) \ - --build-arg BUILD_USER=onnxruntimedev \ - --build-arg BUILD_UID=$(id -u) \ - --build-arg PYTHON_VERSION=$(python.version) \ - -f Dockerfile.manylinux2010_cpu . - workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker - - task: CmdLine@2 - inputs: - script: | + mkdir -p $HOME/.onnx docker run --rm \ --volume /data/onnx:/data/onnx:ro \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ --volume $(Build.BinariesDirectory):/build \ --volume /data/models:/build/models:ro \ + --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \ -e NIGHTLY_BUILD \ -e BUILD_BUILDNUMBER \ - ${{ parameters.docker_image_prefix }}-manylinux-$(python.version) \ + onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:chn \ $(python.manylinux.dir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py \ - --build_dir /build \ + --build_dir /build --cmake_generator Ninja \ --config Release \ --skip_submodule_sync \ --parallel \ + --enable_lto \ --build_wheel \ --use_openmp \ --enable_onnx_tests \ ${{ parameters.build_py_parameters }} \ --cmake_extra_defines PYTHON_INCLUDE_DIR=$(python.manylinux.include.dir) PYTHON_LIBRARY=/usr/lib64/librt.so workingDirectory: $(Build.SourcesDirectory) - + - task: Docker@2 + displayName: logout + inputs: + containerRegistry: onnxruntimeregistry + command: logout + addPipelineData: false - task: CopyFiles@2 displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)' inputs: @@ -140,9 +149,85 @@ stages: pool: Linux-GPU-CUDA10 strategy: matrix: - ${{ if eq(parameters.enable_linux_gpu_py35, true) }}: - Python35: - python.version: '3.5' + Python35: + python.version: '3.5' + Python36: + python.version: '3.6' + Python37: + python.version: '3.7' + Python38: + python.version: '3.8' + steps: + - checkout: self + clean: true + submodules: recursive + + - template: set-py-packaging-variables-step.yml + + - task: Docker@2 + displayName: login + inputs: + containerRegistry: onnxruntimeregistry + command: login + addPipelineData: false + - task: CmdLine@2 + inputs: + script: | + mkdir -p $HOME/.onnx + docker run --gpus all --rm \ + --volume /data/onnx:/data/onnx:ro \ + --volume $(Build.SourcesDirectory):/onnxruntime_src \ + --volume $(Build.BinariesDirectory):/build \ + --volume /data/models:/build/models:ro \ + --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \ + -e NIGHTLY_BUILD \ + -e BUILD_BUILDNUMBER \ + onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:chp \ + $(python.manylinux.dir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py \ + --build_dir /build --cmake_generator Ninja \ + --config Release \ + --skip_submodule_sync \ + --parallel \ + --build_wheel \ + --use_openmp \ + --enable_onnx_tests --use_cuda --cuda_version=10.1 --cuda_home=/usr/local/cuda-10.1 --cudnn_home=/usr/local/cuda-10.1 \ + ${{ parameters.build_py_parameters }} \ + --cmake_extra_defines PYTHON_INCLUDE_DIR=$(python.manylinux.include.dir) PYTHON_LIBRARY=/usr/lib64/librt.so + workingDirectory: $(Build.SourcesDirectory) + + - task: Docker@2 + displayName: logout + inputs: + containerRegistry: onnxruntimeregistry + command: logout + addPipelineData: false + + - task: CopyFiles@2 + displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)' + inputs: + SourceFolder: '$(Build.BinariesDirectory)' + Contents: 'Release/dist/*.whl' + TargetFolder: '$(Build.ArtifactStagingDirectory)' + + - task: PublishBuildArtifacts@1 + displayName: 'Publish Artifact: ONNXRuntime python wheel' + inputs: + ArtifactName: onnxruntime_gpu + + - template: component-governance-component-detection-steps.yml + parameters: + condition: 'succeeded' + + - template: clean-agent-build-directory-step.yml + + - ${{ if eq(parameters.enable_linux_gpu_training, true) }}: + - job: Linux_py_GPU_Wheels + timeoutInMinutes: 120 + workspace: + clean: all + pool: Linux-GPU-CUDA10 + strategy: + matrix: Python36: python.version: '3.6' Python37: @@ -170,11 +255,13 @@ stages: - task: CmdLine@2 inputs: script: | + mkdir -p $HOME/.onnx docker run --rm --gpus all \ --volume /data/onnx:/data/onnx:ro \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ --volume $(Build.BinariesDirectory):/build \ --volume /data/models:/build/models:ro \ + --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \ -e NVIDIA_VISIBLE_DEVICES=all \ -e NIGHTLY_BUILD \ -e BUILD_BUILDNUMBER \ diff --git a/tools/ci_build/github/linux/docker/Dockerfile.centos6 b/tools/ci_build/github/linux/docker/Dockerfile.centos6 deleted file mode 100644 index a52cf31928..0000000000 --- a/tools/ci_build/github/linux/docker/Dockerfile.centos6 +++ /dev/null @@ -1,14 +0,0 @@ -FROM mcr.microsoft.com/dotnet-buildtools/prereqs:centos-6-50f0d02-20190918213956 - -ADD scripts /tmp/scripts -RUN cd /tmp/scripts && /tmp/scripts/install_centos.sh -ENV JAVA_HOME /usr/lib/jvm/jre-1.8.0-openjdk.x86_64 -ENV PATH /opt/rh/devtoolset-2/root/usr/bin:${JAVA_HOME}/bin:${PATH} -RUN /tmp/scripts/install_deps.sh -p 3.6 && rm -rf /tmp/scripts -ENV PATH ${PATH}:/usr/local/gradle/bin - -ARG BUILD_UID=1000 -ARG BUILD_USER=onnxruntimedev -RUN adduser --uid $BUILD_UID $BUILD_USER -WORKDIR /home/$BUILD_USER -USER $BUILD_USER diff --git a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2010_cpu b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2010_cpu index ab16f24892..24d330b900 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2010_cpu +++ b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2010_cpu @@ -1,11 +1,11 @@ -FROM quay.io/pypa/manylinux2010_x86_64:2020-04-04-f427f46 +FROM quay.io/pypa/manylinux2010_x86_64:latest -ARG PYTHON_VERSION -ADD scripts /tmp/scripts -RUN cd /tmp/scripts && /tmp/scripts/install_centos.sh && /tmp/scripts/install_deps.sh -p $PYTHON_VERSION && rm -rf /tmp/scripts +ADD manylinux /tmp/scripts +RUN cd /tmp/scripts && /tmp/scripts/install_centos.sh && /tmp/scripts/install_deps.sh && rm -rf /tmp/scripts ARG BUILD_UID=1000 ARG BUILD_USER=onnxruntimedev -RUN adduser --uid $BUILD_UID $BUILD_USER +RUN adduser --uid $BUILD_UID $BUILD_USER WORKDIR /home/$BUILD_USER USER $BUILD_USER +ENV PATH /usr/local/gradle/bin:/usr/local/dotnet:$PATH diff --git a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2010_gpu b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2010_gpu index c2fbb36d3a..fd4ac04d2a 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2010_gpu +++ b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2010_gpu @@ -1,14 +1,11 @@ -FROM quay.io/pypa/manylinux2010_x86_64:2020-04-04-f427f46 +FROM quay.io/pypa/manylinux2010_x86_64:latest -ARG PYTHON_VERSION=3.5 +ADD manylinux /tmp/scripts +RUN cd /tmp/scripts && /tmp/scripts/install_centos.sh && /tmp/scripts/install_deps.sh && rm -rf /tmp/scripts ENV NVIDIA_VISIBLE_DEVICES all ENV NVIDIA_DRIVER_CAPABILITIES compute,utility -ADD scripts /tmp/scripts -RUN cd /tmp/scripts && /tmp/scripts/install_centos.sh && /tmp/scripts/install_deps.sh -p $PYTHON_VERSION && \ -rm -rf /tmp/scripts - #Below are copied from https://gitlab.com/nvidia/container-images/cuda/tree/master/dist/centos6 RUN NVIDIA_GPGKEY_SUM=d1be581509378368edeec8c1eb2958702feedf3bc3d17011adbf24efacce4ab5 && \ @@ -67,3 +64,4 @@ ARG BUILD_USER=onnxruntimedev RUN adduser --comment 'onnxruntime Build User' --uid $BUILD_UID $BUILD_USER WORKDIR /home/$BUILD_USER USER $BUILD_USER +ENV PATH /usr/local/gradle/bin:/usr/local/dotnet:$PATH \ No newline at end of file diff --git a/tools/ci_build/github/linux/docker/manylinux/install_centos.sh b/tools/ci_build/github/linux/docker/manylinux/install_centos.sh new file mode 100755 index 0000000000..f4319a0c53 --- /dev/null +++ b/tools/ci_build/github/linux/docker/manylinux/install_centos.sh @@ -0,0 +1,40 @@ +#!/bin/bash +set -e + +os_major_version=$(cat /etc/redhat-release | tr -dc '0-9.'|cut -d \. -f1) + + +if ! rpm -q --quiet epel-release ; then + yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-$os_major_version.noarch.rpm +fi + +echo "installing for os major version : $os_major_version" +yum install -y redhat-lsb-core expat-devel libcurl-devel tar unzip curl zlib-devel make libunwind icu aria2 rsync bzip2 git bzip2-devel + +if [ "$os_major_version" == "6" ]; then + # See https://github.com/dotnet/core/blob/master/Documentation/build-and-install-rhel6-prerequisites.md + yum install -y epel-release libunwind openssl libnghttp2 libidn krb5-libs libuuid lttng-ust zlib + curl -sSL -o /tmp/1.tgz https://github.com/unicode-org/icu/releases/download/release-57-1/icu4c-57_1-RHEL6-x64.tgz + mkdir /tmp/icu + tar -zxf /tmp/1.tgz --strip=2 -C /tmp/icu + mv /tmp/icu/lib /tmp/icu/lib64 + /bin/cp -r /tmp/icu/* /usr/ + ldconfig /usr/lib64 + rm -rf /tmp/icu + rm /tmp/1.tgz + + curl -o /tmp/d.sh -sSL https://dot.net/v1/dotnet-install.sh + chmod +x /tmp/d.sh + /tmp/d.sh --install-dir /usr/local/dotnet -c 2.1 + rm /tmp/d.sh +fi +if [ "$os_major_version" == "7" ]; then + # install dotnet core dependencies + yum install -y lttng-ust openssl-libs krb5-libs libicu libuuid + # install dotnet runtimes + yum install -y https://packages.microsoft.com/config/centos/7/packages-microsoft-prod.rpm + yum install -y dotnet-sdk-2.1 +fi + +yum install -y java-1.8.0-openjdk-devel + diff --git a/tools/ci_build/github/linux/docker/manylinux/install_deps.sh b/tools/ci_build/github/linux/docker/manylinux/install_deps.sh new file mode 100755 index 0000000000..f6cefa9190 --- /dev/null +++ b/tools/ci_build/github/linux/docker/manylinux/install_deps.sh @@ -0,0 +1,91 @@ +#!/bin/bash +set -e -x + +#Download a file from internet +function GetFile { + local uri=$1 + local path=$2 + local force=${3:-false} + local download_retries=${4:-5} + local retry_wait_time_seconds=${5:-30} + + if [[ -f $path ]]; then + if [[ $force = false ]]; then + echo "File '$path' already exists. Skipping download" + return 0 + else + rm -rf $path + fi + fi + + if [[ -f $uri ]]; then + echo "'$uri' is a file path, copying file to '$path'" + cp $uri $path + return $? + fi + + echo "Downloading $uri" + # Use aria2c if available, otherwise use curl + if command -v aria2c > /dev/null; then + aria2c -q -d $(dirname $path) -o $(basename $path) "$uri" + else + curl "$uri" -sSL --retry $download_retries --retry-delay $retry_wait_time_seconds --create-dirs -o "$path" --fail + fi + + return $? +} + +PYTHON_EXES=("/opt/python/cp35-cp35m/bin/python3.5" "/opt/python/cp36-cp36m/bin/python3.6" "/opt/python/cp37-cp37m/bin/python3.7" "/opt/python/cp38-cp38/bin/python3.8") + +SYS_LONG_BIT=$(getconf LONG_BIT) +mkdir -p /tmp/src +GLIBC_VERSION=$(getconf GNU_LIBC_VERSION | cut -f 2 -d \.) + +DISTRIBUTOR=$(lsb_release -i -s) + +if [[ "$DISTRIBUTOR" = "CentOS" && $SYS_LONG_BIT = "64" ]]; then + LIBDIR="lib64" +else + LIBDIR="lib" +fi + +cd /tmp/src + + echo "Installing azcopy" + mkdir -p /tmp/azcopy + GetFile https://aka.ms/downloadazcopy-v10-linux /tmp/azcopy/azcopy.tar.gz + tar --strip 1 -xf /tmp/azcopy/azcopy.tar.gz -C /tmp/azcopy + cp /tmp/azcopy/azcopy /usr/bin + echo "Installing cmake" + GetFile https://github.com/Kitware/CMake/releases/download/v3.18.1/cmake-3.18.1-Linux-x86_64.tar.gz /tmp/src/cmake-3.18.1-Linux-x86_64.tar.gz + tar -zxf /tmp/src/cmake-3.18.1-Linux-x86_64.tar.gz --strip=1 -C /usr + echo "Installing Ninja" + GetFile https://github.com/ninja-build/ninja/archive/v1.10.0.tar.gz /tmp/src/ninja-linux.tar.gz + tar -zxf ninja-linux.tar.gz + cd ninja-1.10.0 + cmake -Bbuild-cmake -H. + cmake --build build-cmake + mv ./build-cmake/ninja /usr/bin + echo "Installing Node.js" + GetFile https://nodejs.org/dist/v12.16.3/node-v12.16.3.tar.xz /tmp/src/node-v12.16.3.tar.xz + tar -xf /tmp/src/node-v12.16.3.tar.xz + cd node-v12.16.3 + LDFLAGS=-lrt /opt/python/cp27-cp27m/bin/python configure --ninja + LDFLAGS=-lrt make -j$(getconf _NPROCESSORS_ONLN) + LDFLAGS=-lrt make install + +cd /tmp/src +GetFile https://downloads.gradle-dn.com/distributions/gradle-6.3-bin.zip /tmp/src/gradle-6.3-bin.zip +unzip /tmp/src/gradle-6.3-bin.zip +mv /tmp/src/gradle-6.3 /usr/local/gradle + + +for PYTHON_EXE in "${PYTHON_EXES[@]}" +do + ${PYTHON_EXE} -m pip install -r ${0/%install_deps\.sh/requirements\.txt} +done + + +cd / +rm -rf /tmp/src + diff --git a/tools/ci_build/github/linux/docker/manylinux/requirements.txt b/tools/ci_build/github/linux/docker/manylinux/requirements.txt new file mode 100644 index 0000000000..2be05ad9f1 --- /dev/null +++ b/tools/ci_build/github/linux/docker/manylinux/requirements.txt @@ -0,0 +1,6 @@ +numpy==1.16.6 +mypy +pytest +setuptools>=41.4.0 +wheel +onnx==1.7.0 \ No newline at end of file diff --git a/tools/ci_build/github/linux/docker/scripts/install_server_deps.sh b/tools/ci_build/github/linux/docker/scripts/install_server_deps.sh index ae88d26598..01d81ed68b 100755 --- a/tools/ci_build/github/linux/docker/scripts/install_server_deps.sh +++ b/tools/ci_build/github/linux/docker/scripts/install_server_deps.sh @@ -3,7 +3,6 @@ set -e SYS_LONG_BIT=$(getconf LONG_BIT) -apt-get remove -y libprotobuf-dev libprotobuf9v5 protobuf-compiler libprotobuf-lite9v5 apt-get update && apt-get install -y --no-install-recommends libre2-dev rm -rf /var/lib/apt/lists/* @@ -15,12 +14,12 @@ if [ $SYS_LONG_BIT = "64" ]; then tar -C /usr/local -vzxf /tmp/go/go1.12.6.linux-amd64.tar.gz echo "Installing CMAKE" - aria2c https://github.com/Kitware/CMake/releases/download/v3.16.2/cmake-3.16.2-Linux-x86_64.tar.gz - tar -zxf cmake-3.16.2-Linux-x86_64.tar.gz --strip=1 -C /usr + aria2c https://github.com/Kitware/CMake/releases/download/v3.18.1/cmake-3.18.1-Linux-x86_64.tar.gz + tar -zxf cmake-3.18.1-Linux-x86_64.tar.gz --strip=1 -C /usr echo "Installing onnxruntime" - aria2c https://github.com/microsoft/onnxruntime/releases/download/v1.2.0/onnxruntime-linux-x64-1.2.0.tgz - tar -zxf onnxruntime-linux-x64-1.2.0.tgz --strip=1 + aria2c https://github.com/microsoft/onnxruntime/releases/download/v1.4.0/onnxruntime-linux-x64-1.4.0.tgz + tar -zxf onnxruntime-linux-x64-1.4.0.tgz --strip=1 cp -r include/* /usr/include cp -r lib/* /usr/lib ldconfig /usr/lib diff --git a/tools/ci_build/github/linux/docker/scripts/requirements.txt b/tools/ci_build/github/linux/docker/scripts/requirements.txt index fc5f5e5175..be12b008b7 100644 --- a/tools/ci_build/github/linux/docker/scripts/requirements.txt +++ b/tools/ci_build/github/linux/docker/scripts/requirements.txt @@ -4,4 +4,5 @@ mypy pytest setuptools>=41.4.0 wheel -onnx==1.7.0 \ No newline at end of file +onnx==1.7.0 +argparse \ No newline at end of file