From 4161758058bc98f515d349d810a31a4e128b4cc1 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Fri, 12 Mar 2021 10:02:59 -0800 Subject: [PATCH] Remove openmp related packaging pipeline (#6991) 1. Remove openmp related packaging pipelines and build jobs. 2. Set continueOnError to true for the TSAUpload tasks. Their service is unstable recently. 3. Update Ubuntu 16 docker images to Ubuntu 18, in prepare for getting C++17 support 4. Cherry-pick the changes in 1.7.1 to the master: updating CFLAGS/CXXFLAGS to strip out debug symbols --- cmake/CMakeLists.txt | 4 - .../c-api-noopenmp-packaging-pipelines.yml | 2 +- .../c-api-packaging-pipelines.yml | 34 -- .../azure-pipelines/linux-ci-pipeline.yml | 11 +- .../linux-cpu-minimal-build-ci-pipeline.yml | 18 +- .../azure-pipelines/linux-gpu-ci-pipeline.yml | 2 +- .../linux-gpu-cuda-11-pipeline.yml | 2 +- .../linux-multi-gpu-ci-pipeline.yml | 2 +- .../linux-nocontribops-ci-pipeline.yml | 11 +- .../linux-openvino-nightly-pipeline.yml | 2 +- .../linux-ort-srv-ci-pipeline.yml | 4 +- .../azure-pipelines/nodejs/templates/cpu.yml | 6 +- .../nuget/cpu-esrp-noopenmp-pipeline.yml | 2 +- .../azure-pipelines/nuget/templates/gpu.yml | 2 +- .../nuget/templates/test_linux.yml | 6 +- .../orttraining-linux-gpu-ci-pipeline.yml | 2 +- ...inux-gpu-distributed-e2e-test-pipeline.yml | 2 +- ...linux-gpu-distributed-test-ci-pipeline.yml | 2 +- ...aining-linux-gpu-perf-test-ci-pipeline.yml | 2 +- .../templates/c-api-cpu-no-java.yml | 8 +- .../azure-pipelines/templates/c-api-cpu.yml | 11 +- .../azure-pipelines/templates/compliance.yml | 3 +- .../azure-pipelines/templates/linux-ci.yml | 2 +- .../templates/py-packaging-stage.yml | 310 +----------------- .../github/linux/docker/Dockerfile.centos | 11 - .../github/linux/docker/Dockerfile.fedora_gpu | 13 - .../build_full_ort_and_create_ort_files.sh | 3 +- .../build_minimal_ort_and_run_tests.sh | 2 +- ...rt_android_baseline_and_report_bin_size.sh | 2 +- 29 files changed, 58 insertions(+), 423 deletions(-) delete mode 100644 tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml delete mode 100644 tools/ci_build/github/linux/docker/Dockerfile.centos delete mode 100644 tools/ci_build/github/linux/docker/Dockerfile.fedora_gpu diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 8a5c86cc25..56e1a53ae1 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -30,10 +30,6 @@ set (CMAKE_C_STANDARD 99) include(CheckCXXCompilerFlag) include(CheckLanguage) -# CentOS compiler is old but it does allow certain C++14 features -# such as lambda captures and they are convenient -# On the other hand it does not allow some others. -# So we cant' regulate simply with the standard. set(CMAKE_CXX_STANDARD 14) set_property(GLOBAL PROPERTY USE_FOLDERS ON) diff --git a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml index f6a13fa8d8..bb1fd0c907 100644 --- a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml +++ b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml @@ -53,7 +53,7 @@ jobs: inputs: script: | mkdir -p $HOME/.onnx - docker run --gpus all -e CC=/opt/rh/devtoolset-8/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-8/root/usr/bin/c++ -e CFLAGS="-g -O3" -e CXXFLAGS="-g -O3" -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \ + docker run --gpus all -e CC=/opt/rh/devtoolset-8/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-8/root/usr/bin/c++ -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \ --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecuda11build \ /opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_java --build_dir /build --config Release \ --skip_submodule_sync --parallel --build_shared_lib --use_cuda --cuda_version=$(CUDA_VERSION) --cuda_home=/usr/local/cuda-$(CUDA_VERSION) --cudnn_home=/usr/local/cuda-$(CUDA_VERSION) --cmake_extra_defines CMAKE_CUDA_HOST_COMPILER=/opt/rh/devtoolset-8/root/usr/bin/cc diff --git a/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml deleted file mode 100644 index 0dc6d6ee95..0000000000 --- a/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml +++ /dev/null @@ -1,34 +0,0 @@ -parameters: -- name: RunOnnxRuntimeTests - displayName: Run Tests? - type: boolean - default: true - -- name: DoCompliance - displayName: Run Compliance Tasks? - type: boolean - default: true - -- name: DoEsrp - displayName: Run code sign tasks? Must be true if you are doing an Onnx Runtime release. - type: boolean - default: false - -- name: IsReleaseBuild - displayName: Is a release build? Set it to true if you are doing an Onnx Runtime release. - type: boolean - default: false - -variables: - PackageName: 'Microsoft.ML.OnnxRuntime.OpenMP' - -jobs: -- template: templates/c-api-cpu.yml - parameters: - RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }} - DoCompliance: ${{ parameters.DoCompliance }} - DoEsrp: ${{ parameters.DoEsrp }} - IsReleaseBuild: ${{ parameters.IsReleaseBuild }} - OrtNugetPackageId: 'Microsoft.ML.OnnxRuntime.OpenMP' - AdditionalBuildFlags: '--use_openmp' - AdditionalWinBuildFlags: '--enable_onnx_tests --enable_wcos' \ No newline at end of file diff --git a/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml index 7d548faf68..a403d2c4bc 100644 --- a/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml @@ -11,10 +11,10 @@ jobs: - template: templates/get-docker-image-steps.yml parameters: - Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.centos + Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cpu Context: tools/ci_build/github/linux/docker DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )" - Repository: onnxruntimecentoscpubuild + Repository: onnxruntimecpubuild - task: CmdLine@2 inputs: @@ -29,8 +29,8 @@ jobs: -e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \ -e NIGHTLY_BUILD \ -e BUILD_BUILDNUMBER \ - onnxruntimecentoscpubuild \ - python3 /onnxruntime_src/tools/ci_build/build.py \ + onnxruntimecpubuild \ + /opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py \ --build_dir /build --cmake_generator Ninja \ --config Debug Release \ --skip_submodule_sync \ @@ -39,7 +39,8 @@ jobs: --build_wheel \ --enable_onnx_tests \ --enable_symbolic_shape_infer_tests \ - --enable_pybind --build_java --build_nodejs + --build_java --build_nodejs \ + --cmake_extra_defines PYTHON_INCLUDE_DIR=/opt/python/cp37-cp37m/include/python3.7m PYTHON_LIBRARY=/usr/lib64/librt.so workingDirectory: $(Build.SourcesDirectory) - task: PublishTestResults@2 diff --git a/tools/ci_build/github/azure-pipelines/linux-cpu-minimal-build-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-cpu-minimal-build-ci-pipeline.yml index 9b45ad9e04..015602a561 100644 --- a/tools/ci_build/github/azure-pipelines/linux-cpu-minimal-build-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-cpu-minimal-build-ci-pipeline.yml @@ -38,10 +38,10 @@ jobs: - template: templates/get-docker-image-steps.yml parameters: - Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.centos + Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cpu Context: tools/ci_build/github/linux/docker DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )" - Repository: onnxruntimecentoscpubuild + Repository: onnxruntimecpubuild - task: CmdLine@2 displayName: Build full onnxruntime and generate ORT format test files @@ -54,7 +54,7 @@ jobs: -e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \ -e NIGHTLY_BUILD \ -e BUILD_BUILDNUMBER \ - onnxruntimecentoscpubuild \ + onnxruntimecpubuild \ /bin/bash /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_full_ort_and_create_ort_files.sh workingDirectory: $(Build.SourcesDirectory) @@ -70,8 +70,8 @@ jobs: -e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \ -e NIGHTLY_BUILD \ -e BUILD_BUILDNUMBER \ - onnxruntimecentoscpubuild \ - python3 /onnxruntime_src/tools/ci_build/build.py \ + onnxruntimecpubuild \ + /opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py \ --build_dir /build --cmake_generator Ninja \ --config Debug\ --skip_submodule_sync \ @@ -93,7 +93,7 @@ jobs: -e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \ -e NIGHTLY_BUILD \ -e BUILD_BUILDNUMBER \ - onnxruntimecentoscpubuild \ + onnxruntimecpubuild \ /bin/bash /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh \ --build-directory /build/without_type_reduction \ --reduced-ops-config /home/onnxruntimedev/.test_data/required_ops.ort_models.config \ @@ -115,7 +115,7 @@ jobs: -e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \ -e NIGHTLY_BUILD \ -e BUILD_BUILDNUMBER \ - onnxruntimecentoscpubuild \ + onnxruntimecpubuild \ /bin/bash /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh \ --build-directory /build/with_type_reduction \ --reduced-ops-config /home/onnxruntimedev/.test_data/required_ops_and_types.ort_models.config \ @@ -141,7 +141,7 @@ jobs: -e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \ -e NIGHTLY_BUILD \ -e BUILD_BUILDNUMBER \ - onnxruntimecentoscpubuild \ + onnxruntimecpubuild \ /bin/bash /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh \ --build-directory /build/with_type_reduction_globally_allowed_types \ --reduced-ops-config /home/onnxruntimedev/.test_data/globally_allowed_types.config \ @@ -170,7 +170,7 @@ jobs: -e BUILD_REASON=$(Build.Reason) \ -e BUILD_BRANCH=$(Build.SourceBranch) \ -e DASHBOARD_MYSQL_ORT_PASSWORD=$(dashboard-mysql-ort-password) \ - onnxruntimecentoscpubuild \ + onnxruntimecpubuild \ /bin/bash /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_android_baseline_and_report_bin_size.sh workingDirectory: $(Build.SourcesDirectory) diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml index bb00301dd9..b0b3542617 100644 --- a/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml @@ -20,7 +20,7 @@ jobs: inputs: script: | mkdir -p $HOME/.onnx - docker run --gpus all -e CC=/opt/rh/devtoolset-8/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-8/root/usr/bin/c++ -e CFLAGS="-g -O3" -e CXXFLAGS="-g -O3" --rm \ + docker run --gpus all -e CC=/opt/rh/devtoolset-8/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-8/root/usr/bin/c++ -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" --rm \ --volume /data/onnx:/data/onnx:ro \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ --volume $(Build.BinariesDirectory):/build \ diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-cuda-11-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-cuda-11-pipeline.yml index b140bb1959..95af9e90fb 100644 --- a/tools/ci_build/github/azure-pipelines/linux-gpu-cuda-11-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-cuda-11-pipeline.yml @@ -19,7 +19,7 @@ jobs: inputs: script: | mkdir -p $HOME/.onnx - docker run --gpus all -e CC=/opt/rh/devtoolset-8/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-8/root/usr/bin/c++ -e CFLAGS="-g -O3" -e CXXFLAGS="-g -O3" --rm \ + docker run --gpus all -e CC=/opt/rh/devtoolset-8/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-8/root/usr/bin/c++ -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" --rm \ --volume /data/onnx:/data/onnx:ro \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ --volume $(Build.BinariesDirectory):/build \ diff --git a/tools/ci_build/github/azure-pipelines/linux-multi-gpu-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-multi-gpu-ci-pipeline.yml index 4cb9de15a8..da957b534d 100644 --- a/tools/ci_build/github/azure-pipelines/linux-multi-gpu-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-multi-gpu-ci-pipeline.yml @@ -24,7 +24,7 @@ jobs: inputs: script: | mkdir -p $HOME/.onnx - docker run --gpus all -e CC=/opt/rh/devtoolset-8/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-8/root/usr/bin/c++ -e CFLAGS="-g -O3" -e CXXFLAGS="-g -O3" --rm \ + docker run --gpus all -e CC=/opt/rh/devtoolset-8/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-8/root/usr/bin/c++ -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" --rm \ --volume /data/onnx:/data/onnx:ro \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ --volume $(Build.BinariesDirectory):/build \ diff --git a/tools/ci_build/github/azure-pipelines/linux-nocontribops-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-nocontribops-ci-pipeline.yml index 992cb47e92..e135f75657 100644 --- a/tools/ci_build/github/azure-pipelines/linux-nocontribops-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-nocontribops-ci-pipeline.yml @@ -11,10 +11,10 @@ jobs: - template: templates/get-docker-image-steps.yml parameters: - Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.centos + Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cpu Context: tools/ci_build/github/linux/docker DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )" - Repository: onnxruntimecentoscpubuild + Repository: onnxruntimecpubuild - task: CmdLine@2 inputs: @@ -29,8 +29,8 @@ jobs: -e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \ -e NIGHTLY_BUILD \ -e BUILD_BUILDNUMBER \ - onnxruntimecentoscpubuild \ - python3 /onnxruntime_src/tools/ci_build/build.py \ + onnxruntimecpubuild \ + /opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py \ --build_dir /build --cmake_generator Ninja \ --config Debug Release \ --skip_submodule_sync \ @@ -38,7 +38,8 @@ jobs: --parallel \ --build_wheel \ --enable_onnx_tests \ - --disable_contrib_ops --disable_ml_ops + --disable_contrib_ops --disable_ml_ops \ + --cmake_extra_defines PYTHON_INCLUDE_DIR=/opt/python/cp37-cp37m/include/python3.7m PYTHON_LIBRARY=/usr/lib64/librt.so workingDirectory: $(Build.SourcesDirectory) - task: PublishTestResults@2 diff --git a/tools/ci_build/github/azure-pipelines/linux-openvino-nightly-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-openvino-nightly-pipeline.yml index 62a3dec8a6..a2e2ef0b2c 100644 --- a/tools/ci_build/github/azure-pipelines/linux-openvino-nightly-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-openvino-nightly-pipeline.yml @@ -23,7 +23,7 @@ jobs: - template: templates/run-docker-build-steps.yml parameters: - RunDockerBuildArgs: '-o ubuntu16.04 -d openvino -v 2020.2 -r $(Build.BinariesDirectory) -x "--use_openvino GPU_FP32 --build_wheel"' + RunDockerBuildArgs: '-o ubuntu18.04 -d openvino -v 2020.2 -r $(Build.BinariesDirectory) -x "--use_openvino GPU_FP32 --build_wheel"' - template: templates/component-governance-component-detection-steps.yml parameters : diff --git a/tools/ci_build/github/azure-pipelines/linux-ort-srv-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-ort-srv-ci-pipeline.yml index 718b53f104..e60b9c4c67 100644 --- a/tools/ci_build/github/azure-pipelines/linux-ort-srv-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-ort-srv-ci-pipeline.yml @@ -16,13 +16,13 @@ jobs: - task: CmdLine@2 displayName: 'Build docker image' inputs: - script: docker build --pull -t onnxruntime-server-ubuntu16.04 --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg OS_VERSION=16.04 --build-arg PYTHON_VERSION=3.5 -f Dockerfile.ubuntu_server . + script: docker build --pull -t onnxruntime-server-ubuntu18.04 --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg OS_VERSION=18.04 --build-arg PYTHON_VERSION=3.5 -f Dockerfile.ubuntu_server . workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker - task: CmdLine@2 displayName: 'Run docker image' inputs: - script: docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory)/server:/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro onnxruntime-server-ubuntu16.04 /bin/bash /onnxruntime_src/ci/run.sh + script: docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory)/server:/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro onnxruntime-server-ubuntu18.04 /bin/bash /onnxruntime_src/ci/run.sh workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker - template: templates/component-governance-component-detection-steps.yml diff --git a/tools/ci_build/github/azure-pipelines/nodejs/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nodejs/templates/cpu.yml index 746032b636..3fc5185210 100644 --- a/tools/ci_build/github/azure-pipelines/nodejs/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nodejs/templates/cpu.yml @@ -39,15 +39,15 @@ jobs: - template: ../../templates/set-version-number-variables-step.yml - template: ../../templates/get-docker-image-steps.yml parameters: - Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.centos + Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cpu Context: tools/ci_build/github/linux/docker DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )" - Repository: onnxruntimecentoscpubuild + Repository: onnxruntimecpubuild - task: CmdLine@2 inputs: script: | mkdir -p $HOME/.onnx && docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro \ - --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecentoscpubuild /bin/bash -c "python3 \ + --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecpubuild /bin/bash -c "/opt/python/cp37-cp37m/bin/python3 \ /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --build_nodejs \ --skip_submodule_sync --parallel --build_shared_lib && cd /onnxruntime_src/nodejs && npm pack" workingDirectory: $(Build.SourcesDirectory) diff --git a/tools/ci_build/github/azure-pipelines/nuget/cpu-esrp-noopenmp-pipeline.yml b/tools/ci_build/github/azure-pipelines/nuget/cpu-esrp-noopenmp-pipeline.yml index 445fca7294..e679b4edd0 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/cpu-esrp-noopenmp-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/cpu-esrp-noopenmp-pipeline.yml @@ -53,7 +53,7 @@ jobs: inputs: script: | mkdir -p $HOME/.onnx - docker run --gpus all -e CC=/opt/rh/devtoolset-8/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-8/root/usr/bin/c++ -e CFLAGS="-g -O3" -e CXXFLAGS="-g -O3" -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \ + docker run --gpus all -e CC=/opt/rh/devtoolset-8/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-8/root/usr/bin/c++ -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \ --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecuda11build \ /opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_java --build_dir /build --config Release \ --skip_submodule_sync --parallel --build_shared_lib --use_cuda --cuda_version=$(CUDA_VERSION) --cuda_home=/usr/local/cuda-$(CUDA_VERSION) --cudnn_home=/usr/local/cuda-$(CUDA_VERSION) --cmake_extra_defines CMAKE_CUDA_HOST_COMPILER=/opt/rh/devtoolset-8/root/usr/bin/cc diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml index 45f9593d3d..406cc8487f 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml @@ -88,7 +88,7 @@ jobs: inputs: script: | mkdir -p $HOME/.onnx - docker run --gpus all -e CC=/opt/rh/devtoolset-8/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-8/root/usr/bin/c++ -e CFLAGS="-g -O3" -e CXXFLAGS="-g -O3" -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecuda11build \ + docker run --gpus all -e CC=/opt/rh/devtoolset-8/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-8/root/usr/bin/c++ -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecuda11build \ /bin/bash -c "/opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_cuda --cuda_version=11.0 --cuda_home=/usr/local/cuda-11.0 --cudnn_home=/usr/local/cuda-11.0 --enable_onnx_tests --cmake_extra_defines CMAKE_CUDA_HOST_COMPILER=/opt/rh/devtoolset-8/root/usr/bin/cc && cd /build/Release && make install DESTDIR=/build/linux-x64" - script: | set -e -x diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml index 336d4ec025..08da20e43f 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml @@ -36,10 +36,10 @@ jobs: - ${{ if eq(parameters['TestGPU'], 'false') }}: - template: ../../templates/get-docker-image-steps.yml parameters: - Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.centos + Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cpu Context: tools/ci_build/github/linux/docker DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )" - Repository: onnxruntimecentoscpubuild + Repository: onnxruntimecpubuild - script: | set -e -x @@ -48,7 +48,7 @@ jobs: $(Build.BinariesDirectory) \ nuget-artifact \ $(NuGetPackageVersionNumber) \ - onnxruntimecentoscpubuild \ + onnxruntimecpubuild \ true displayName: 'Run Package Test (x64) on CentOS7' env: diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ci-pipeline.yml index 03fdff524c..22b1488f5e 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ci-pipeline.yml @@ -7,7 +7,7 @@ jobs: JobName: 'Onnxruntime_Linux_GPU_Training' SubmoduleCheckoutMode: 'recursive' RunDockerBuildArgs: > - -o ubuntu16.04 -d gpu -r $(Build.BinariesDirectory) + -o ubuntu18.04 -d gpu -r $(Build.BinariesDirectory) -x " --enable_training --config $(buildConfig) diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-distributed-e2e-test-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-distributed-e2e-test-pipeline.yml index a6d2081ad5..d2eebd962a 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-distributed-e2e-test-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-distributed-e2e-test-pipeline.yml @@ -14,7 +14,7 @@ jobs: - template: templates/run-docker-build-steps.yml parameters: RunDockerBuildArgs: | - -o ubuntu16.04 -d gpu -r $(Build.BinariesDirectory) \ + -o ubuntu18.04 -d gpu -r $(Build.BinariesDirectory) \ -t onnxruntime_e2e_test_image \ -x " \ --config RelWithDebInfo \ diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-distributed-test-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-distributed-test-ci-pipeline.yml index 66e41fd951..8aeb376ebc 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-distributed-test-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-distributed-test-ci-pipeline.yml @@ -14,7 +14,7 @@ jobs: - template: templates/run-docker-build-steps.yml parameters: RunDockerBuildArgs: | - -o ubuntu16.04 -d gpu -r $(Build.BinariesDirectory) \ + -o ubuntu18.04 -d gpu -r $(Build.BinariesDirectory) \ -t onnxruntime_distributed_tests_image \ -x " \ --config RelWithDebInfo \ diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-perf-test-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-perf-test-ci-pipeline.yml index d3d3a42c0a..05795f92cf 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-perf-test-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-perf-test-ci-pipeline.yml @@ -16,7 +16,7 @@ jobs: - template: templates/run-docker-build-steps.yml parameters: RunDockerBuildArgs: > - -o ubuntu16.04 -d gpu -r $(Build.BinariesDirectory) + -o ubuntu18.04 -d gpu -r $(Build.BinariesDirectory) -t onnxruntime_perf_test_image -x " --config RelWithDebInfo diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu-no-java.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu-no-java.yml index 157b92107a..b2c9ee8c1a 100644 --- a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu-no-java.yml +++ b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu-no-java.yml @@ -53,16 +53,16 @@ jobs: - template: set-version-number-variables-step.yml - template: get-docker-image-steps.yml parameters: - Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.centos + Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cpu Context: tools/ci_build/github/linux/docker DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )" - Repository: onnxruntimecentoscpubuild + Repository: onnxruntimecpubuild - task: CmdLine@2 inputs: script: | mkdir -p $HOME/.onnx - docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \ - --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecentoscpubuild /bin/bash -c "python3 \ + docker run --rm -e CC=/opt/rh/devtoolset-9/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-9/root/usr/bin/c++ -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \ + --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecpubuild /bin/bash -c "/opt/python/cp37-cp37m/bin/python3 \ /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release \ --skip_submodule_sync --parallel --build_shared_lib ${{ parameters.AdditionalBuildFlags }} && cd /build/Release && make install DESTDIR=/build/linux-x64" workingDirectory: $(Build.SourcesDirectory) diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml index 294278a4c3..7b7c5c82f6 100644 --- a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml +++ b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml @@ -53,16 +53,17 @@ jobs: - template: set-version-number-variables-step.yml - template: get-docker-image-steps.yml parameters: - Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.centos + Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cpu Context: tools/ci_build/github/linux/docker DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )" - Repository: onnxruntimecentoscpubuild + Repository: onnxruntimecpubuild + - task: CmdLine@2 inputs: script: | mkdir -p $HOME/.onnx - docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \ - --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecentoscpubuild /bin/bash -c "python3 \ + docker run --rm -e CC=/opt/rh/devtoolset-9/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-9/root/usr/bin/c++ -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \ + --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecpubuild /bin/bash -c "/opt/python/cp37-cp37m/bin/python3 \ /onnxruntime_src/tools/ci_build/build.py --build_java --build_dir /build --config Release \ --skip_submodule_sync --parallel --build_shared_lib ${{ parameters.AdditionalBuildFlags }} && cd /build/Release && make install DESTDIR=/build/linux-x64" workingDirectory: $(Build.SourcesDirectory) @@ -383,7 +384,7 @@ jobs: command: restore projects: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln' configuration: RelWithDebInfo - arguments: '--configuration RelWithDebInfo -p:Platform="Any CPU" -p:OrtPackageId=$(OrtPackageId) -p:OrtPackageId=$(OrtPackageId) -p:IsReleaseBuild=${{ parameters.IsReleaseBuild }}' + arguments: '--configuration RelWithDebInfo -p:Platform="Any CPU" -p:OrtPackageId=$(OrtPackageId) -p:IsReleaseBuild=${{ parameters.IsReleaseBuild }}' workingDirectory: '$(Build.SourcesDirectory)\csharp' - task: DotNetCoreCLI@2 diff --git a/tools/ci_build/github/azure-pipelines/templates/compliance.yml b/tools/ci_build/github/azure-pipelines/templates/compliance.yml index 31b88e0cea..cdf0cf1f26 100644 --- a/tools/ci_build/github/azure-pipelines/templates/compliance.yml +++ b/tools/ci_build/github/azure-pipelines/templates/compliance.yml @@ -54,4 +54,5 @@ steps: displayName: 'TSA Upload' inputs: tsaVersion: TsaV2 - codeBaseName: 'onnxruntime_master' \ No newline at end of file + codeBaseName: 'onnxruntime_master' + continueOnError: true \ No newline at end of file diff --git a/tools/ci_build/github/azure-pipelines/templates/linux-ci.yml b/tools/ci_build/github/azure-pipelines/templates/linux-ci.yml index 8a652cac25..1c188fb9b5 100644 --- a/tools/ci_build/github/azure-pipelines/templates/linux-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/linux-ci.yml @@ -2,7 +2,7 @@ parameters: AgentPool : 'Linux-CPU-2019' JobName : 'Linux_CI_Dev' SubmoduleCheckoutMode: '' - RunDockerBuildArgs: '-o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -x "--build_wheel"' + RunDockerBuildArgs: '-o ubuntu18.04 -d cpu -r $(Build.BinariesDirectory) -x "--build_wheel"' DoNodejsPack: 'false' DoNugetPack: 'false' NuPackScript: '' diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml index 5c176d7cb0..06bbf036dc 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml @@ -142,81 +142,6 @@ stages: - template: clean-agent-build-directory-step.yml - - job: Linux_py_Wheels_openmp - timeoutInMinutes: 90 - workspace: - clean: all - pool: Linux-CPU - strategy: - matrix: - Python36: - PythonVersion: '3.6' - Python37: - PythonVersion: '3.7' - Python38: - PythonVersion: '3.8' - Python39: - PythonVersion: '3.9' - steps: - - checkout: self - clean: true - submodules: recursive - - - template: set-python-manylinux-variables-step.yml - - - template: get-docker-image-steps.yml - parameters: - Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cpu - Context: tools/ci_build/github/linux/docker - DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )" - Repository: onnxruntimecpubuild - - - task: CmdLine@2 - inputs: - script: | - mkdir -p $HOME/.onnx - docker run --rm \ - --volume /data/onnx:/data/onnx:ro \ - --volume $(Build.SourcesDirectory):/onnxruntime_src \ - --volume $(Build.BinariesDirectory):/build \ - --volume /data/models:/build/models:ro \ - --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \ - -e NIGHTLY_BUILD \ - -e BUILD_BUILDNUMBER \ - -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" \ - -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" \ - onnxruntimecpubuild \ - $(PythonManylinuxDir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py \ - --build_dir /build --cmake_generator Ninja \ - --config Release \ - --skip_submodule_sync \ - --parallel \ - --enable_lto \ - --build_wheel \ - --use_openmp \ - --enable_onnx_tests \ - --wheel_name_suffix=openmp \ - ${{ parameters.build_py_parameters }} \ - --cmake_extra_defines PYTHON_INCLUDE_DIR=$(PythonManylinuxIncludeDir) PYTHON_LIBRARY=/usr/lib64/librt.so - workingDirectory: $(Build.SourcesDirectory) - - - task: CopyFiles@2 - displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)' - inputs: - SourceFolder: '$(Build.BinariesDirectory)' - Contents: 'Release/dist/*.whl' - TargetFolder: '$(Build.ArtifactStagingDirectory)' - - - task: PublishBuildArtifacts@1 - displayName: 'Publish Artifact: ONNXRuntime python wheel' - inputs: - ArtifactName: onnxruntime - - - template: component-governance-component-detection-steps.yml - parameters: - condition: 'succeeded' - - - template: clean-agent-build-directory-step.yml - ${{ if eq(parameters.enable_linux_gpu, true) }}: - job: Linux_py_GPU_Wheels @@ -496,183 +421,6 @@ stages: displayName: 'Clean Agent Directories' condition: always() - - job: Windows_py_Wheels_openmp - pool: 'Win-CPU-2021' - strategy: - matrix: - Python36: - PythonVersion: '3.6' - Python37: - PythonVersion: '3.7' - Python38: - PythonVersion: '3.8' - Python39: - PythonVersion: '3.9' - variables: - MsbuildArguments: '-maxcpucount' - OnnxRuntimeBuildDirectory: '$(Build.BinariesDirectory)' - EnvSetupScript: setup_env.bat - buildArch: x64 - setVcvars: true - BuildConfig: 'Release' - GDN_CODESIGN_TARGETDIRECTORY: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\dist' - timeoutInMinutes: 120 - workspace: - clean: all - - steps: - - checkout: self - clean: true - submodules: recursive - - - template: telemetry-steps.yml - - - task: UsePythonVersion@0 - inputs: - versionSpec: $(PythonVersion) - addToPath: true - architecture: 'x64' - - - template: set-nightly-build-option-variable-step.yml - - - task: BatchScript@1 - displayName: 'setup env' - inputs: - filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)' - modifyEnvironment: true - workingFolder: '$(Build.BinariesDirectory)' - - - script: | - python -m pip install -q pyopenssl setuptools wheel numpy==1.16.6 - workingDirectory: '$(Build.BinariesDirectory)' - displayName: 'Install python modules' - - - powershell: | - $Env:USE_MSVC_STATIC_RUNTIME=1 - $Env:ONNX_ML=1 - $Env:CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=OFF -DProtobuf_USE_STATIC_LIBS=ON -DONNX_USE_LITE_PROTO=ON -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=$(buildArch)-windows-static" - python setup.py bdist_wheel - python -m pip uninstall -y onnx -qq - Get-ChildItem -Path dist/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname} - workingDirectory: '$(Build.SourcesDirectory)\cmake\external\onnx' - displayName: 'Install ONNX' - - - task: PythonScript@0 - displayName: 'BUILD' - inputs: - scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: > - --config RelWithDebInfo - --enable_lto - --disable_rtti - --build_dir $(Build.BinariesDirectory) - --skip_submodule_sync - --cmake_generator "Visual Studio 16 2019" - --enable_pybind - --use_openmp - --enable_onnx_tests - ${{ parameters.build_py_parameters }} - --parallel - $(TelemetryOption) - workingDirectory: '$(Build.BinariesDirectory)' - - # Esrp signing - - template: win-esrp-dll.yml - parameters: - FolderPath: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\onnxruntime\capi' - DisplayName: 'ESRP - Sign Native dlls' - DoEsrp: true - Pattern: '*.pyd,*.dll' - - - task: PythonScript@0 - displayName: 'Build wheel' - inputs: - scriptPath: '$(Build.SourcesDirectory)\setup.py' - arguments: 'bdist_wheel ${{ parameters.build_py_parameters }} $(NightlyBuildOption) --wheel_name_suffix=openmp' - workingDirectory: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo' - - - task: CopyFiles@2 - displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)' - inputs: - SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\dist' - Contents: '*.whl' - TargetFolder: '$(Build.ArtifactStagingDirectory)' - - - script: | - rename *.whl *.zip - workingDirectory: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\dist' - displayName: 'Rename file extension for codesign validation' - - - task: DeleteFiles@1 - displayName: 'Delete files from $(Build.BinariesDirectory)\RelWithDebInfo' - condition: and (succeeded(), eq(variables['PythonVersion'], '3.7')) - inputs: - SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo' - Contents: | - **/*.obj - **/*.pdb - **/*.dll - - - task: PythonScript@0 - displayName: 'Regenerate cmake config with STATIC_ANALYSIS=ON' - condition: and (succeeded(), eq(variables['PythonVersion'], '3.7')) - inputs: - scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: > - --config RelWithDebInfo - --enable_lto - --disable_rtti - --build_dir $(Build.BinariesDirectory) - --skip_submodule_sync - --cmake_generator "Visual Studio 16 2019" - --enable_pybind - --use_openmp - --enable_onnx_tests - --update - --cmake_extra_defines onnxruntime_ENABLE_STATIC_ANALYSIS=ON - ${{ parameters.build_py_parameters }} - --parallel - $(TelemetryOption) - workingDirectory: '$(Build.BinariesDirectory)' - - #Manually set msBuildCommandline so that we can also set CAExcludePath - - task: SDLNativeRules@2 - displayName: 'Run the PREfast SDL Native Rules for MSBuild' - condition: and (succeeded(), eq(variables['PythonVersion'], '3.7')) - inputs: - userProvideBuildInfo: msBuildInfo - msBuildVersion: 16.0 - msBuildArchitecture: x64 - msBuildCommandline: '"C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\amd64\msbuild.exe" "$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln" /p:platform="x64" /p:configuration="RelWithDebInfo" /p:CAExcludePath="$(Build.BinariesDirectory);$(Build.SourcesDirectory)\cmake;C:\program files (x86)" /p:VisualStudioVersion="16.0" /m /p:PreferredToolArchitecture=x64' - continueOnError: true - - - task: TSAUpload@1 - condition: and(and (succeeded(), eq(variables['PythonVersion'], '3.7')), eq(variables['Build.SourceBranch'], 'refs/heads/master')) - displayName: 'TSA Upload' - inputs: - tsaVersion: TsaV2 - codeBaseName: 'onnxruntime_master' - - - task: PublishTestResults@2 - displayName: 'Publish unit test results' - inputs: - testResultsFiles: '**\*.results.xml' - searchFolder: '$(Build.BinariesDirectory)' - testRunTitle: 'Unit Test Run' - condition: succeededOrFailed() - - - task: PublishBuildArtifacts@1 - displayName: 'Publish Artifact: ONNXRuntime python wheel' - inputs: - ArtifactName: onnxruntime - - - template: component-governance-component-detection-steps.yml - parameters: - condition: 'succeeded' - - - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 - displayName: 'Clean Agent Directories' - condition: always() - ${{ if eq(parameters.enable_windows_gpu, true) }}: - job: Windows_py_GPU_Wheels workspace: @@ -839,6 +587,7 @@ stages: inputs: tsaVersion: TsaV2 codeBaseName: 'onnxruntime_master' + continueOnError: true - template: component-governance-component-detection-steps.yml parameters: @@ -906,63 +655,6 @@ stages: parameters: condition: 'succeeded' - - job: MacOS_py_Wheels_openmp - workspace: - clean: all - pool: - vmImage: 'macOS-10.14' - strategy: - matrix: - Python36: - PythonVersion: '3.6' - Python37: - PythonVersion: '3.7' - Python38: - PythonVersion: '3.8' - Python39: - PythonVersion: '3.9' - steps: - - checkout: self - clean: true - submodules: recursive - - - task: UsePythonVersion@0 - displayName: 'Use Python' - inputs: - versionSpec: $(PythonVersion) - - - script: | - set -e - pushd . - cd $(Build.SourcesDirectory)/cmake/external/protobuf - cmake ./cmake -DCMAKE_INSTALL_PREFIX=$(Build.BinariesDirectory)/protobuf -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Relwithdebinfo - make -j$(getconf _NPROCESSORS_ONLN) - make install - popd - export PATH=$(Build.BinariesDirectory)/protobuf/bin:$PATH - export ONNX_ML=1 - export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF" - sudo python3 -m pip install -r '$(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/requirements.txt' - sudo xcode-select --switch /Applications/Xcode_10.app/Contents/Developer - brew install libomp - python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --use_openmp --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --config Release --skip_onnx_tests --build_wheel ${{ parameters.build_py_parameters }} --wheel_name_suffix=openmp - displayName: 'Command Line Script' - - - task: CopyFiles@2 - displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)' - inputs: - SourceFolder: '$(Build.BinariesDirectory)/Release/dist' - Contents: '*.whl' - TargetFolder: '$(Build.ArtifactStagingDirectory)' - - - task: PublishBuildArtifacts@1 - displayName: 'Publish Artifact: ONNXRuntime python wheel' - inputs: - ArtifactName: onnxruntime - - - template: component-governance-component-detection-steps.yml - parameters: - condition: 'succeeded' - ${{ if eq(parameters.enable_linux_arm, true) }}: - job: Linux_ARM_py_Wheels diff --git a/tools/ci_build/github/linux/docker/Dockerfile.centos b/tools/ci_build/github/linux/docker/Dockerfile.centos deleted file mode 100644 index 2169682d3b..0000000000 --- a/tools/ci_build/github/linux/docker/Dockerfile.centos +++ /dev/null @@ -1,11 +0,0 @@ -FROM centos:7 - -ADD scripts /tmp/scripts -RUN cd /tmp/scripts && /tmp/scripts/manylinux/install_centos.sh && /tmp/scripts/manylinux/install_deps.sh && rm -rf /tmp/scripts - -ARG BUILD_UID=1001 -ARG BUILD_USER=onnxruntimedev -RUN adduser --uid $BUILD_UID $BUILD_USER -WORKDIR /home/$BUILD_USER -USER $BUILD_USER -ENV PATH /usr/local/gradle/bin:$PATH diff --git a/tools/ci_build/github/linux/docker/Dockerfile.fedora_gpu b/tools/ci_build/github/linux/docker/Dockerfile.fedora_gpu deleted file mode 100644 index 02cd456bc8..0000000000 --- a/tools/ci_build/github/linux/docker/Dockerfile.fedora_gpu +++ /dev/null @@ -1,13 +0,0 @@ -ARG OS_VERSION=27 -FROM fedora:${OS_VERSION} - -ARG PYTHON_VERSION=3.6 - -# nvidia-container-runtime -ENV NVIDIA_VISIBLE_DEVICES=all \ - NVIDIA_DRIVER_CAPABILITIES=compute,utility \ - NVIDIA_REQUIRE_CUDA="cuda>=9.2" \ - PATH="/usr/lib64/ccache:${PATH}" -ADD scripts /tmp/scripts -RUN cd /tmp/scripts && /tmp/scripts/install_fedora_gpu.sh && /tmp/scripts/install_deps.sh -p $PYTHON_VERSION && rm -rf /tmp/scripts - diff --git a/tools/ci_build/github/linux/ort_minimal/build_full_ort_and_create_ort_files.sh b/tools/ci_build/github/linux/ort_minimal/build_full_ort_and_create_ort_files.sh index 5c57e429f1..bc2a27a423 100644 --- a/tools/ci_build/github/linux/ort_minimal/build_full_ort_and_create_ort_files.sh +++ b/tools/ci_build/github/linux/ort_minimal/build_full_ort_and_create_ort_files.sh @@ -5,6 +5,7 @@ set -e set -x +export PATH=/opt/python/cp37-cp37m/bin:$PATH # Validate the operator kernel registrations, as the ORT model uses hashes of the kernel registration details # to find kernels. If the hashes from the registration details are incorrect we will produce a model that will break @@ -21,7 +22,7 @@ python3 /onnxruntime_src/tools/ci_build/build.py \ --parallel \ --build_wheel \ --skip_tests \ - --enable_pybind + --enable_pybind --cmake_extra_defines PYTHON_INCLUDE_DIR=/opt/python/cp37-cp37m/include/python3.7m PYTHON_LIBRARY=/usr/lib64/librt.so # Install the ORT python wheel python3 -m pip install --user /build/Debug/dist/* diff --git a/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh b/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh index 11c72752a8..3659f4f920 100644 --- a/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh +++ b/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh @@ -7,7 +7,7 @@ set -e set -x - +export PATH=/opt/python/cp37-cp37m/bin:$PATH USAGE_TEXT="Usage: -b|--build-directory Specifies the build directory. Required. diff --git a/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_android_baseline_and_report_bin_size.sh b/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_android_baseline_and_report_bin_size.sh index 6d4d04b18b..a7d7c2fea2 100644 --- a/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_android_baseline_and_report_bin_size.sh +++ b/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_android_baseline_and_report_bin_size.sh @@ -5,7 +5,7 @@ set -e set -x - +export PATH=/opt/python/cp37-cp37m/bin:$PATH # Create an empty file to be used with build --include_ops_by_config, which will include no operators at all echo -n > /home/onnxruntimedev/.test_data/include_no_operators.config