From c37fa7c2781f0a1d7d38fbc3b4609bef05834844 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Fri, 28 Aug 2020 09:56:52 -0700 Subject: [PATCH] Delete Dockerfile.centos6_gpu (#4851) --- .../c-api-packaging-pipelines.yml | 26 ++++-- .../java-api-packaging-pipelines-gpu.yml | 41 ++++++--- .../java-api-packaging-pipelines.yml | 4 +- .../azure-pipelines/linux-gpu-ci-pipeline.yml | 3 +- .../linux-multi-gpu-ci-pipeline.yml | 68 ++++++++++++-- .../linux-nocontribops-ci-pipeline.yml | 69 ++++++++++++-- .../azure-pipelines/nuget/templates/gpu.yml | 19 ++-- .../orttraining-linux-ci-pipeline.yml | 89 +++++++++++++------ .../templates/py-packaging-stage.yml | 1 - .../linux/docker/Dockerfile.centos6_gpu | 69 -------------- .../github/linux/docker/Dockerfile.fedora | 7 -- .../linux/docker/Dockerfile.manylinux2010_gpu | 28 +++++- .../linux/docker/scripts/install_centos.sh | 33 ++----- .../linux/docker/scripts/requirements.txt | 3 +- .../github/linux/java_linux_final_test.sh | 4 +- 15 files changed, 282 insertions(+), 182 deletions(-) delete mode 100644 tools/ci_build/github/linux/docker/Dockerfile.centos6_gpu delete mode 100644 tools/ci_build/github/linux/docker/Dockerfile.fedora diff --git a/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml index cfde7c25d8..b2d4bd8e6e 100644 --- a/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml +++ b/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml @@ -47,19 +47,27 @@ jobs: pool: 'Linux-GPU-CUDA10' steps: - template: templates/set-version-number-variables-step.yml + - task: Docker@2 + displayName: login + inputs: + containerRegistry: onnxruntimeregistry + command: login + addPipelineData: false - task: CmdLine@2 inputs: script: | - docker build --pull -t onnxruntime-centos6-gpu --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=$(python.version) -f Dockerfile.centos6_gpu . - workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker - - task: CmdLine@2 - inputs: - script: | + mkdir -p $HOME/.onnx docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \ - --volume /data/models:/build/models:ro -e NIGHTLY_BUILD onnxruntime-centos6-gpu /usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release \ - --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_cuda --cuda_version=10.1 --cuda_home=/usr/local/cuda-10.1 \ - --cudnn_home=/usr/local/cuda-10.1 - workingDirectory: $(Build.SourcesDirectory) + --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:chp \ + /opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release \ + --skip_submodule_sync --parallel --build_shared_lib --use_cuda --cuda_version=10.1 --cuda_home=/usr/local/cuda-10.1 --cudnn_home=/usr/local/cuda-10.1 + workingDirectory: $(Build.SourcesDirectory) + - task: Docker@2 + displayName: logout + inputs: + containerRegistry: onnxruntimeregistry + command: logout + addPipelineData: false - template: templates/c-api-artifacts-package-and-publish-steps-posix.yml parameters: buildConfig: 'Release' diff --git a/tools/ci_build/github/azure-pipelines/java-api-packaging-pipelines-gpu.yml b/tools/ci_build/github/azure-pipelines/java-api-packaging-pipelines-gpu.yml index 017b166a4e..561252b47a 100644 --- a/tools/ci_build/github/azure-pipelines/java-api-packaging-pipelines-gpu.yml +++ b/tools/ci_build/github/azure-pipelines/java-api-packaging-pipelines-gpu.yml @@ -14,16 +14,23 @@ jobs: pool: 'Linux-GPU-CUDA10' steps: - template: templates/set-version-number-variables-step.yml + - task: Docker@2 + displayName: login + inputs: + containerRegistry: onnxruntimeregistry + command: login + addPipelineData: false - task: CmdLine@2 inputs: script: | - docker build --pull -t onnxruntime-centos6-gpu --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=$(python.version) -f Dockerfile.centos6_gpu . - workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker - - task: CmdLine@2 - inputs: - script: | - docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro -e NIGHTLY_BUILD onnxruntime-centos6-gpu /usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_java --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_cuda --cuda_version=10.1 --cuda_home=/usr/local/cuda-10.1 --cudnn_home=/usr/local/cuda-10.1 + mkdir -p $HOME/.onnx + docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:chp /opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_java --build_shared_lib --use_cuda --cuda_version=10.1 --cuda_home=/usr/local/cuda-10.1 --cudnn_home=/usr/local/cuda-10.1 workingDirectory: $(Build.SourcesDirectory) + - task: Docker@2 + displayName: logout + inputs: + containerRegistry: onnxruntimeregistry + command: logout - template: templates/java-api-artifacts-package-and-publish-steps-posix.yml parameters: arch: 'linux-x64' @@ -85,7 +92,7 @@ jobs: displayName: 'Generate cmake config' inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: '--config RelWithDebInfo --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_java --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --enable_lto --disable_rtti --use_openmp --build_shared_lib --enable_onnx_tests $(buildparameter)' + arguments: '--config RelWithDebInfo --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_java --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --enable_lto --disable_rtti --build_shared_lib --enable_onnx_tests $(buildparameter)' workingDirectory: '$(Build.BinariesDirectory)' @@ -105,7 +112,7 @@ jobs: displayName: 'test' inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: '--config RelWithDebInfo --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --build_java --test --cmake_generator "Visual Studio 16 2019" --enable_lto --disable_rtti --use_openmp --build_shared_lib --enable_onnx_tests $(buildparameter)' + arguments: '--config RelWithDebInfo --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --build_java --test --cmake_generator "Visual Studio 16 2019" --enable_lto --disable_rtti --build_shared_lib --enable_onnx_tests $(buildparameter)' workingDirectory: '$(Build.BinariesDirectory)' - template: templates/java-api-artifacts-package-and-publish-steps-windows.yml @@ -239,14 +246,22 @@ jobs: artifactName: 'onnxruntime-java-gpu' targetPath: '$(Build.BinariesDirectory)/final-jar' - - task: CmdLine@2 + - task: Docker@2 + displayName: login inputs: - script: | - docker build --pull -t onnxruntime-centos6-gpu --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=$(python.version) -f Dockerfile.centos6_gpu . - workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker + containerRegistry: onnxruntimeregistry + command: login + addPipelineData: false - task: CmdLine@2 inputs: script: | - docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6-gpu /onnxruntime_src/tools/ci_build/github/linux/java_linux_final_test.sh -v $(OnnxRuntimeVersion) -r /build + docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:chp /onnxruntime_src/tools/ci_build/github/linux/java_linux_final_test.sh -v $(OnnxRuntimeVersion) -r /build workingDirectory: $(Build.BinariesDirectory)/final-jar + + - task: Docker@2 + displayName: logout + inputs: + containerRegistry: onnxruntimeregistry + command: logout + addPipelineData: false diff --git a/tools/ci_build/github/azure-pipelines/java-api-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/java-api-packaging-pipelines.yml index 291bd924f7..ad631c79af 100644 --- a/tools/ci_build/github/azure-pipelines/java-api-packaging-pipelines.yml +++ b/tools/ci_build/github/azure-pipelines/java-api-packaging-pipelines.yml @@ -137,7 +137,7 @@ jobs: displayName: 'Generate cmake config' inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: '--config RelWithDebInfo --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_wcos --build_java --update --cmake_generator "Visual Studio 16 2019" --enable_lto --disable_rtti --use_openmp --build_shared_lib --enable_onnx_tests $(buildparameter)' + arguments: '--config RelWithDebInfo --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --enable_wcos --build_java --update --cmake_generator "Visual Studio 16 2019" --enable_lto --disable_rtti --use_openmp --build_shared_lib --enable_onnx_tests $(buildparameter)' workingDirectory: '$(Build.BinariesDirectory)' @@ -158,7 +158,7 @@ jobs: displayName: 'test' inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: '--config RelWithDebInfo --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_java --build_shared_lib --enable_wcos --build_java --test --cmake_generator "Visual Studio 16 2019" --enable_lto --disable_rtti --use_openmp --build_shared_lib --enable_onnx_tests $(buildparameter)' + arguments: '--config RelWithDebInfo --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_java --enable_wcos --build_java --test --cmake_generator "Visual Studio 16 2019" --enable_lto --disable_rtti --use_openmp --build_shared_lib --enable_onnx_tests $(buildparameter)' workingDirectory: '$(Build.BinariesDirectory)' - template: templates/java-api-artifacts-package-and-publish-steps-windows.yml diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml index e2f8a4f08b..69fb626ab6 100644 --- a/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml @@ -35,9 +35,8 @@ jobs: --build_shared_lib \ --parallel \ --build_wheel \ - --use_openmp \ --enable_onnx_tests --use_cuda --cuda_version=10.1 --cuda_home=/usr/local/cuda-10.1 --cudnn_home=/usr/local/cuda-10.1 \ - --use_mklml --enable_pybind --build_java --build_nodejs \ + --enable_pybind --build_java --build_nodejs \ --cmake_extra_defines PYTHON_INCLUDE_DIR=/opt/python/cp37-cp37m/include/python3.7m PYTHON_LIBRARY=/usr/lib64/librt.so workingDirectory: $(Build.SourcesDirectory) - task: Docker@2 diff --git a/tools/ci_build/github/azure-pipelines/linux-multi-gpu-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-multi-gpu-ci-pipeline.yml index 96d6ca0159..2e2b07fb8f 100644 --- a/tools/ci_build/github/azure-pipelines/linux-multi-gpu-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-multi-gpu-ci-pipeline.yml @@ -1,8 +1,62 @@ jobs: -- template: templates/linux-ci.yml - parameters: - AgentPool : 'Linux-Multi-GPU' - JobName: 'Linux_CI_Multi_GPU_Dev' - BuildCommand: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d gpu -r $(Build.BinariesDirectory) -x "--enable_multi_device_test"' - DoNugetPack: 'false' - ArtifactName: 'drop-linux' +- job: Linux_CI_Multi_GPU_Dev + timeoutInMinutes: 120 + workspace: + clean: all + pool: Linux-Multi-GPU + steps: + - checkout: self + clean: true + submodules: recursive + + - task: Docker@2 + displayName: login + inputs: + containerRegistry: onnxruntimeregistry + command: login + addPipelineData: false + - task: CmdLine@2 + inputs: + script: | + mkdir -p $HOME/.onnx + docker run --gpus all --rm \ + --volume /data/onnx:/data/onnx:ro \ + --volume $(Build.SourcesDirectory):/onnxruntime_src \ + --volume $(Build.BinariesDirectory):/build \ + --volume /data/models:/build/models:ro \ + --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \ + -e NIGHTLY_BUILD \ + -e BUILD_BUILDNUMBER \ + onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:chp \ + /opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py \ + --build_dir /build --cmake_generator Ninja \ + --config Debug Release \ + --skip_submodule_sync \ + --build_shared_lib \ + --parallel \ + --build_wheel \ + --enable_onnx_tests --use_cuda --cuda_version=10.1 --cuda_home=/usr/local/cuda-10.1 --cudnn_home=/usr/local/cuda-10.1 \ + --enable_pybind --build_java --build_nodejs --enable_multi_device_test \ + --cmake_extra_defines PYTHON_INCLUDE_DIR=/opt/python/cp37-cp37m/include/python3.7m PYTHON_LIBRARY=/usr/lib64/librt.so + workingDirectory: $(Build.SourcesDirectory) + - task: Docker@2 + displayName: logout + inputs: + containerRegistry: onnxruntimeregistry + command: logout + addPipelineData: false + - task: PublishTestResults@2 + displayName: 'Publish unit test results' + inputs: + testResultsFiles: '**/*.results.xml' + searchFolder: '$(Build.BinariesDirectory)' + testRunTitle: 'Unit Test Run' + condition: succeededOrFailed() + + - template: templates/component-governance-component-detection-steps.yml + parameters: + condition: 'succeeded' + + - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 + displayName: 'Clean Agent Directories' + condition: always() diff --git a/tools/ci_build/github/azure-pipelines/linux-nocontribops-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-nocontribops-ci-pipeline.yml index 21bf1ed222..e11be47b9d 100644 --- a/tools/ci_build/github/azure-pipelines/linux-nocontribops-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-nocontribops-ci-pipeline.yml @@ -1,9 +1,62 @@ jobs: -- template: templates/linux-ci.yml - parameters: - AgentPool : 'Linux-CPU' - JobName: 'Linux_CI_Dev' - BuildCommand: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) -x "--disable_contrib_ops --disable_ml_ops"' - DoNugetPack: 'false' - ArtifactName: 'drop-linux' - TimeoutInMinutes: 120 +- job: Linux_Build + timeoutInMinutes: 120 + workspace: + clean: all + pool: Linux-CPU + steps: + - checkout: self + clean: true + submodules: recursive + + - task: Docker@2 + displayName: login + inputs: + containerRegistry: onnxruntimeregistry + command: login + addPipelineData: false + - task: CmdLine@2 + inputs: + script: | + mkdir -p $HOME/.onnx + docker run --rm \ + --volume /data/onnx:/data/onnx:ro \ + --volume $(Build.SourcesDirectory):/onnxruntime_src \ + --volume $(Build.BinariesDirectory):/build \ + --volume /data/models:/build/models:ro \ + --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \ + -e NIGHTLY_BUILD \ + -e BUILD_BUILDNUMBER \ + onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:chn \ + /opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py \ + --build_dir /build --cmake_generator Ninja \ + --config Debug Release \ + --skip_submodule_sync \ + --build_shared_lib \ + --parallel \ + --build_wheel \ + --enable_onnx_tests \ + --disable_contrib_ops --disable_ml_ops \ + --cmake_extra_defines PYTHON_INCLUDE_DIR=/opt/python/cp37-cp37m/include/python3.7m PYTHON_LIBRARY=/usr/lib64/librt.so + workingDirectory: $(Build.SourcesDirectory) + - task: Docker@2 + displayName: logout + inputs: + containerRegistry: onnxruntimeregistry + command: logout + addPipelineData: false + - task: PublishTestResults@2 + displayName: 'Publish unit test results' + inputs: + testResultsFiles: '**/*.results.xml' + searchFolder: '$(Build.BinariesDirectory)' + testRunTitle: 'Unit Test Run' + condition: succeededOrFailed() + + - template: templates/component-governance-component-detection-steps.yml + parameters: + condition: 'succeeded' + + - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 + displayName: 'Clean Agent Directories' + condition: always() diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml index db0f9dd5a4..3e04966340 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml @@ -79,15 +79,24 @@ jobs: steps: - template: ../../templates/set-version-number-variables-step.yml - template: ../../templates/linux-set-variables-and-download.yml + - task: Docker@2 + displayName: login + inputs: + containerRegistry: onnxruntimeregistry + command: login + addPipelineData: false - task: CmdLine@2 inputs: script: | - docker build --pull -t onnxruntime-centos6-gpu --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=$(python.version) -f Dockerfile.centos6_gpu . - workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker - - task: CmdLine@2 + mkdir -p $HOME/.onnx + docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:chp \ + /bin/bash -c "/opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_cuda --cuda_version=10.1 --cuda_home=/usr/local/cuda-10.1 --cudnn_home=/usr/local/cuda-10.1 --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64" + - task: Docker@2 + displayName: logout inputs: - script: | - docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro -e NIGHTLY_BUILD onnxruntime-centos6-gpu /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_cuda --cuda_version=10.1 --cuda_home=/usr/local/cuda-10.1 --cudnn_home=/usr/local/cuda-10.1 --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64" + containerRegistry: onnxruntimeregistry + command: logout + addPipelineData: false - script: | set -e -x mv $(Build.BinariesDirectory)/linux-x64/usr/local/lib64 $(Build.BinariesDirectory)/linux-x64/linux-x64 diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml index b1ce5a97f3..f0e8e0050b 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-ci-pipeline.yml @@ -1,28 +1,63 @@ -trigger: none - jobs: -- template: templates/linux-ci.yml - parameters: - AgentPool : 'Linux-CPU' - JobName: 'Onnxruntime_Linux_CPU_Training' - SubmoduleCheckoutMode: 'recursive' - BuildCommand: > - tools/ci_build/github/linux/run_dockerbuild.sh - -o ubuntu16.04 -d cpu -r $(Build.BinariesDirectory) - -x " - --build_wheel - --config $(buildConfig) - --enable_training - --use_dnnl - --use_mklml - " - DoNugetPack: 'false' - ArtifactName: 'drop-linux' - TimeoutInMinutes: 90 - Strategy: - maxParallel: 2 - matrix: - Debug: - buildConfig: Debug - Release: - buildConfig: Release +- job: Linux_Build + timeoutInMinutes: 120 + workspace: + clean: all + pool: Linux-CPU + steps: + - checkout: self + clean: true + submodules: recursive + + - task: Docker@2 + displayName: login + inputs: + containerRegistry: onnxruntimeregistry + command: login + addPipelineData: false + - task: CmdLine@2 + inputs: + script: | + mkdir -p $HOME/.onnx + docker run --rm \ + --volume /data/onnx:/data/onnx:ro \ + --volume $(Build.SourcesDirectory):/onnxruntime_src \ + --volume $(Build.BinariesDirectory):/build \ + --volume /data/models:/build/models:ro \ + --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \ + -e NIGHTLY_BUILD \ + -e BUILD_BUILDNUMBER \ + onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:chn \ + /opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py \ + --build_dir /build --cmake_generator Ninja \ + --config Debug Release \ + --skip_submodule_sync \ + --build_shared_lib \ + --parallel \ + --build_wheel \ + --use_openmp \ + --enable_onnx_tests \ + --use_mklml --enable_pybind --build_java --build_nodejs --enable_training \ + --cmake_extra_defines PYTHON_INCLUDE_DIR=/opt/python/cp37-cp37m/include/python3.7m PYTHON_LIBRARY=/usr/lib64/librt.so + workingDirectory: $(Build.SourcesDirectory) + - task: Docker@2 + displayName: logout + inputs: + containerRegistry: onnxruntimeregistry + command: logout + addPipelineData: false + - task: PublishTestResults@2 + displayName: 'Publish unit test results' + inputs: + testResultsFiles: '**/*.results.xml' + searchFolder: '$(Build.BinariesDirectory)' + testRunTitle: 'Unit Test Run' + condition: succeededOrFailed() + + - template: templates/component-governance-component-detection-steps.yml + parameters: + condition: 'succeeded' + + - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 + displayName: 'Clean Agent Directories' + condition: always() diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml index f350e24dd3..1e8e25e8d1 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml @@ -189,7 +189,6 @@ stages: --skip_submodule_sync \ --parallel \ --build_wheel \ - --use_openmp \ --enable_onnx_tests --use_cuda --cuda_version=10.1 --cuda_home=/usr/local/cuda-10.1 --cudnn_home=/usr/local/cuda-10.1 \ ${{ parameters.build_py_parameters }} \ --cmake_extra_defines PYTHON_INCLUDE_DIR=$(python.manylinux.include.dir) PYTHON_LIBRARY=/usr/lib64/librt.so diff --git a/tools/ci_build/github/linux/docker/Dockerfile.centos6_gpu b/tools/ci_build/github/linux/docker/Dockerfile.centos6_gpu deleted file mode 100644 index adcbaaf747..0000000000 --- a/tools/ci_build/github/linux/docker/Dockerfile.centos6_gpu +++ /dev/null @@ -1,69 +0,0 @@ -FROM mcr.microsoft.com/dotnet-buildtools/prereqs:centos-6-50f0d02-20190918213956 - -ENV NVIDIA_VISIBLE_DEVICES all -ENV NVIDIA_DRIVER_CAPABILITIES compute,utility - -ADD scripts /tmp/scripts -RUN cd /tmp/scripts && /tmp/scripts/install_centos.sh -ENV JAVA_HOME /usr/lib/jvm/jre-1.8.0-openjdk.x86_64 -ENV PATH /opt/rh/devtoolset-2/root/usr/bin:${JAVA_HOME}/bin:${PATH} -RUN /tmp/scripts/install_deps.sh -p 3.6 && rm -rf /tmp/scripts -ENV PATH ${PATH}:/usr/local/gradle/bin - -#Below are copied from https://gitlab.com/nvidia/container-images/cuda/tree/master/dist/centos6 - -RUN NVIDIA_GPGKEY_SUM=d1be581509378368edeec8c1eb2958702feedf3bc3d17011adbf24efacce4ab5 && \ - curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/rhel6/x86_64/7fa2af80.pub | sed '/^Version/d' > /etc/pki/rpm-gpg/RPM-GPG-KEY-NVIDIA && \ - echo "$NVIDIA_GPGKEY_SUM /etc/pki/rpm-gpg/RPM-GPG-KEY-NVIDIA" | sha256sum -c - - -COPY cuda_manylinux2010.repo /etc/yum.repos.d/cuda.repo - -ENV CUDA_VERSION 10.1.243 -ENV CUDA_PKG_VERSION 10-1-$CUDA_VERSION-1 -RUN yum install -y \ -cuda-cudart-$CUDA_PKG_VERSION \ -cuda-compat-10-1 \ -cuda-libraries-$CUDA_PKG_VERSION \ -cuda-nvtx-$CUDA_PKG_VERSION \ -libcublas10-10.2.1.243-1 \ -cuda-nvml-dev-$CUDA_PKG_VERSION \ -cuda-command-line-tools-$CUDA_PKG_VERSION \ -cuda-libraries-dev-$CUDA_PKG_VERSION \ -cuda-minimal-build-$CUDA_PKG_VERSION \ -libcublas-devel-10.2.1.243-1 \ -&& \ - ln -s cuda-10.1 /usr/local/cuda && \ - rpm -e --nodeps gcc gcc-c++ && \ - rm -rf /var/cache/yum/* - -# nvidia-docker 1.0 -RUN echo "/usr/local/nvidia/lib" >> /etc/ld.so.conf.d/nvidia.conf && \ - echo "/usr/local/nvidia/lib64" >> /etc/ld.so.conf.d/nvidia.conf - -ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:${PATH} -ENV LD_LIBRARY_PATH /usr/local/nvidia/lib:/usr/local/nvidia/lib64:${LD_LIBRARY_PATH} - -# nvidia-container-runtime -ENV NVIDIA_VISIBLE_DEVICES all -ENV NVIDIA_DRIVER_CAPABILITIES compute,utility -ENV NVIDIA_REQUIRE_CUDA "cuda>=10.1 brand=tesla,driver>=384,driver<385 brand=tesla,driver>=396,driver<397 brand=tesla,driver>=410,driver<411" - - -ENV LIBRARY_PATH /usr/local/cuda/lib64/stubs - -ENV CUDNN_VERSION 7.6.5.32 -LABEL com.nvidia.cudnn.version="${CUDNN_VERSION}" - -# cuDNN license: https://developer.nvidia.com/cudnn/license_agreement -RUN CUDNN_DOWNLOAD_SUM=7eaec8039a2c30ab0bc758d303588767693def6bf49b22485a2c00bf2e136cb3 && \ - curl -fsSL http://developer.download.nvidia.com/compute/redist/cudnn/v7.6.5/cudnn-10.1-linux-x64-v7.6.5.32.tgz -O && \ - echo "$CUDNN_DOWNLOAD_SUM cudnn-10.1-linux-x64-v7.6.5.32.tgz" | sha256sum -c - && \ - tar --no-same-owner -xzf cudnn-10.1-linux-x64-v7.6.5.32.tgz -C /usr/local && \ - rm cudnn-10.1-linux-x64-v7.6.5.32.tgz && \ - ldconfig - -ARG BUILD_UID=1000 -ARG BUILD_USER=onnxruntimedev -RUN /usr/sbin/adduser --uid $BUILD_UID $BUILD_USER -WORKDIR /home/$BUILD_USER -USER $BUILD_USER diff --git a/tools/ci_build/github/linux/docker/Dockerfile.fedora b/tools/ci_build/github/linux/docker/Dockerfile.fedora deleted file mode 100644 index 4bc2348dcf..0000000000 --- a/tools/ci_build/github/linux/docker/Dockerfile.fedora +++ /dev/null @@ -1,7 +0,0 @@ -FROM mcr.microsoft.com/dotnet-buildtools/prereqs:fedora-30-39ec319-20190806171718 - -ARG PYTHON_VERSION=3.7 - -ADD scripts /tmp/scripts -RUN cd /tmp/scripts && /tmp/scripts/install_fedora.sh && /tmp/scripts/install_deps.sh -p $PYTHON_VERSION && rm -rf /tmp/scripts - diff --git a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2010_gpu b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2010_gpu index fd4ac04d2a..9d0b46bb45 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2010_gpu +++ b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2010_gpu @@ -1,3 +1,29 @@ +#Copyright (c) 2019,2020 NVIDIA CORPORATION. All rights reserved. +# +#Redistribution and use in source and binary forms, with or without +#modification, are permitted provided that the following conditions +#are met: +# * Redistributions of source code must retain the above copyright +# notice, this list of conditions and the following disclaimer. +# * Redistributions in binary form must reproduce the above copyright +# notice, this list of conditions and the following disclaimer in the +# documentation and/or other materials provided with the distribution. +# * Neither the name of NVIDIA CORPORATION nor the names of its +# contributors may be used to endorse or promote products derived +# from this software without specific prior written permission. +# +#THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY +#EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE +#IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR +#PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR +#CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, +#EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, +#PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR +#PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY +#OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT +#(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE +#OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. + FROM quay.io/pypa/manylinux2010_x86_64:latest ADD manylinux /tmp/scripts @@ -64,4 +90,4 @@ ARG BUILD_USER=onnxruntimedev RUN adduser --comment 'onnxruntime Build User' --uid $BUILD_UID $BUILD_USER WORKDIR /home/$BUILD_USER USER $BUILD_USER -ENV PATH /usr/local/gradle/bin:/usr/local/dotnet:$PATH \ No newline at end of file +ENV PATH /usr/local/gradle/bin:/usr/local/dotnet:$PATH diff --git a/tools/ci_build/github/linux/docker/scripts/install_centos.sh b/tools/ci_build/github/linux/docker/scripts/install_centos.sh index f7c33e294c..13ae47eba0 100755 --- a/tools/ci_build/github/linux/docker/scripts/install_centos.sh +++ b/tools/ci_build/github/linux/docker/scripts/install_centos.sh @@ -9,32 +9,11 @@ if ! rpm -q --quiet epel-release ; then fi echo "installing for os major version : $os_major_version" -if [ "$os_major_version" == "5" ]; then - #Be careful, don't pull gcc into the base system, because we already have one in /opt/rh/devtoolset-2/root/usr/bin - yum install -y redhat-lsb expat-devel libcurl-devel tar unzip curl zlib-devel make icu rsync bzip2 git bzip2-devel -elif [ "$os_major_version" == "6" ] && [ ! -d "/opt/python/cp35-cp35m" ]; then - #The base image we are using already contains devtoolset-2 - yum install -y redhat-lsb-core expat-devel libcurl-devel tar unzip curl zlib-devel make libunwind icu aria2 rsync bzip2 git bzip2-devel - #Install python 3.6 - yum install -y https://repo.ius.io/ius-release-el6.rpm - yum --enablerepo=ius install -y python36u python36u-devel python36u-pip python36u-numpy python36u-setuptools python36u-wheel - /usr/bin/python3.6 -m pip install --upgrade pip -else - yum install -y redhat-lsb-core expat-devel libcurl-devel tar unzip curl zlib-devel make libunwind icu aria2 rsync bzip2 git bzip2-devel +yum install -y redhat-lsb-core expat-devel libcurl-devel tar unzip curl zlib-devel make libunwind icu aria2 rsync bzip2 git bzip2-devel - if [ "$os_major_version" == "7" ]; then - # install dotnet core dependencies - yum install -y lttng-ust openssl-libs krb5-libs libicu libuuid - # install dotnet runtimes - yum install -y https://packages.microsoft.com/config/centos/7/packages-microsoft-prod.rpm - yum install -y dotnet-sdk-2.2 - fi -fi +# install dotnet core dependencies +yum install -y lttng-ust openssl-libs krb5-libs libicu libuuid +# install dotnet runtimes +yum install -y https://packages.microsoft.com/config/centos/7/packages-microsoft-prod.rpm +yum install -y dotnet-sdk-2.2 java-1.8.0-openjdk-devel ccache gcc gcc-c++ python3 python3-devel python3-pip -yum install -y java-1.8.0-openjdk-devel - -#If the /opt/python folder exists, we assume this is the manylinux docker image -if [ "$os_major_version" != "6" ] && [ ! -d "/opt/python/cp35-cp35m" ] -then - yum install -y ccache gcc gcc-c++ python3 python3-devel python3-pip -fi diff --git a/tools/ci_build/github/linux/docker/scripts/requirements.txt b/tools/ci_build/github/linux/docker/scripts/requirements.txt index 6cd06cc2c3..00350822b0 100644 --- a/tools/ci_build/github/linux/docker/scripts/requirements.txt +++ b/tools/ci_build/github/linux/docker/scripts/requirements.txt @@ -1,9 +1,8 @@ cerberus numpy==1.16.6 -requests==2.24.0 mypy pytest setuptools>=41.4.0 wheel onnx==1.7.0 -argparse \ No newline at end of file +argparse diff --git a/tools/ci_build/github/linux/java_linux_final_test.sh b/tools/ci_build/github/linux/java_linux_final_test.sh index d25af6a791..068bd51b4e 100755 --- a/tools/ci_build/github/linux/java_linux_final_test.sh +++ b/tools/ci_build/github/linux/java_linux_final_test.sh @@ -26,8 +26,8 @@ pushd test jar xf $BINARY_DIR/final-jar/testing.jar popd -wget https://oss.sonatype.org/service/local/repositories/releases/content/org/junit/platform/junit-platform-console-standalone/1.6.2/junit-platform-console-standalone-1.6.2.jar -P ./ -wget https://oss.sonatype.org/service/local/repositories/google-releases/content/com/google/protobuf/protobuf-java/3.9.2/protobuf-java-3.9.2.jar -P ./ +curl -O -sSL https://oss.sonatype.org/service/local/repositories/releases/content/org/junit/platform/junit-platform-console-standalone/1.6.2/junit-platform-console-standalone-1.6.2.jar +curl -O -sSL https://oss.sonatype.org/service/local/repositories/google-releases/content/com/google/protobuf/protobuf-java/3.9.2/protobuf-java-3.9.2.jar java -DUSE_CUDA=1 -jar ./junit-platform-console-standalone-1.6.2.jar -cp .:./test:./protobuf-java-3.9.2.jar:./onnxruntime_gpu-${VERSION_NUMBER}.jar --scan-class-path --fail-if-no-tests --disable-banner EXIT_CODE=$?