From 01ca6392cb165f3ebad6fc3c28016adad5a26688 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Mon, 3 Aug 2020 10:18:10 -0700 Subject: [PATCH] Avoid building ONNX of every history ONNX versions in our CI (#4678) 1. Avoid building ONNX of every history ONNX versions in our CI, it is costly and easy to fail. 2. Run docker command without sudo. Previously the user is not in docker group, now Azure DevOps Service have added it in. --- .../c-api-packaging-pipelines.yml | 13 +++- .../azure-pipelines/centos-ci-pipeline.yml | 4 +- .../java-api-packaging-pipelines-gpu.yml | 8 +- .../java-api-packaging-pipelines.yml | 4 +- .../azure-pipelines/linux-arm-ci-pipeline.yml | 62 --------------- .../linux-ngraph-ci-pipeline.yml | 4 +- .../linux-openvino-nightly-pipeline.yml | 4 +- .../linux-ort-srv-ci-pipeline.yml | 8 +- .../linux-ort-srv-nightly-pipeline.yml | 4 +- .../linux-pytorch-custom-ops-ci-pipeline.yml | 6 +- .../azure-pipelines/nodejs/templates/cpu.yml | 4 +- .../nuget/templates/cpu-featurizers.yml | 4 +- .../nuget/templates/cpu-mklml.yml | 4 +- .../templates/cpu-nocontribops-arm64.yml | 4 +- .../azure-pipelines/nuget/templates/cpu.yml | 4 +- .../azure-pipelines/nuget/templates/gpu.yml | 4 +- .../linux-set-variables-and-download.yml | 6 +- .../orttraining-linux-gpu-ci-pipeline.yml | 4 - .../azure-pipelines/post-merge-jobs.yml | 2 +- .../linux-set-variables-and-download.yml | 6 +- .../templates/py-packaging-stage.yml | 13 ++-- .../linux/create_package_inside_docker.sh | 47 ------------ .../linux/docker/scripts/install_deps.sh | 21 ----- .../linux/docker/scripts/install_fedora.sh | 3 - .../docker/scripts/install_fedora_gpu.sh | 10 --- .../linux/docker/scripts/install_onnx.sh | 64 ---------------- .../linux/docker/scripts/install_ubuntu.sh | 2 - .../linux/docker/scripts/requirements.txt | 3 +- .../github/linux/ubuntu16.04/Dockerfile | 15 ---- .../github/linux/ubuntu16.04/install.sh | 76 ------------------- 30 files changed, 59 insertions(+), 354 deletions(-) delete mode 100644 tools/ci_build/github/azure-pipelines/linux-arm-ci-pipeline.yml delete mode 100755 tools/ci_build/github/linux/create_package_inside_docker.sh delete mode 100755 tools/ci_build/github/linux/docker/scripts/install_fedora.sh delete mode 100755 tools/ci_build/github/linux/docker/scripts/install_fedora_gpu.sh delete mode 100755 tools/ci_build/github/linux/docker/scripts/install_onnx.sh delete mode 100644 tools/ci_build/github/linux/ubuntu16.04/Dockerfile delete mode 100755 tools/ci_build/github/linux/ubuntu16.04/install.sh diff --git a/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml index 13caf609f4..529aa3756f 100644 --- a/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml +++ b/tools/ci_build/github/azure-pipelines/c-api-packaging-pipelines.yml @@ -10,12 +10,14 @@ jobs: - task: CmdLine@2 inputs: script: | - sudo docker build --pull -t onnxruntime-centos6 --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=3.6 -f Dockerfile.centos6 . + docker build --pull -t onnxruntime-centos6 --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=3.6 -f Dockerfile.centos6 . workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker - task: CmdLine@2 inputs: script: | - sudo --preserve-env docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro -e NIGHTLY_BUILD onnxruntime-centos6 /usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest + docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro \ + -e NIGHTLY_BUILD onnxruntime-centos6 /usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib \ + --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest workingDirectory: $(Build.SourcesDirectory) - template: templates/c-api-artifacts-package-and-publish-steps-posix.yml parameters: @@ -39,12 +41,15 @@ jobs: - task: CmdLine@2 inputs: script: | - sudo docker build --pull -t onnxruntime-centos6-gpu --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=$(python.version) -f Dockerfile.centos6_gpu . + docker build --pull -t onnxruntime-centos6-gpu --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=$(python.version) -f Dockerfile.centos6_gpu . workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker - task: CmdLine@2 inputs: script: | - sudo --preserve-env docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro -e NIGHTLY_BUILD onnxruntime-centos6-gpu /usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_cuda --cuda_version=10.1 --cuda_home=/usr/local/cuda-10.1 --cudnn_home=/usr/local/cuda-10.1 + docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \ + --volume /data/models:/build/models:ro -e NIGHTLY_BUILD onnxruntime-centos6-gpu /usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release \ + --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_cuda --cuda_version=10.1 --cuda_home=/usr/local/cuda-10.1 \ + --cudnn_home=/usr/local/cuda-10.1 workingDirectory: $(Build.SourcesDirectory) - template: templates/c-api-artifacts-package-and-publish-steps-posix.yml parameters: diff --git a/tools/ci_build/github/azure-pipelines/centos-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/centos-ci-pipeline.yml index f47c3db235..2a2018036a 100644 --- a/tools/ci_build/github/azure-pipelines/centos-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/centos-ci-pipeline.yml @@ -15,12 +15,12 @@ jobs: - task: CmdLine@2 inputs: script: | - sudo docker build --pull -t onnxruntime-centos7 --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=3.6 -f Dockerfile.centos . + docker build --pull -t onnxruntime-centos7 --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=3.6 -f Dockerfile.centos . workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker - task: CmdLine@2 inputs: script: | - sudo --preserve-env docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos7 /usr/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config $(BuildType) --skip_submodule_sync --parallel --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --build_wheel + docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos7 /usr/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config $(BuildType) --skip_submodule_sync --parallel --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --build_wheel workingDirectory: $(Build.SourcesDirectory) - template: templates/clean-agent-build-directory-step.yml diff --git a/tools/ci_build/github/azure-pipelines/java-api-packaging-pipelines-gpu.yml b/tools/ci_build/github/azure-pipelines/java-api-packaging-pipelines-gpu.yml index bfd62762be..0644ea7e6c 100644 --- a/tools/ci_build/github/azure-pipelines/java-api-packaging-pipelines-gpu.yml +++ b/tools/ci_build/github/azure-pipelines/java-api-packaging-pipelines-gpu.yml @@ -9,12 +9,12 @@ jobs: - task: CmdLine@2 inputs: script: | - sudo docker build --pull -t onnxruntime-centos6-gpu --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=$(python.version) -f Dockerfile.centos6_gpu . + docker build --pull -t onnxruntime-centos6-gpu --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=$(python.version) -f Dockerfile.centos6_gpu . workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker - task: CmdLine@2 inputs: script: | - sudo docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro -e NIGHTLY_BUILD onnxruntime-centos6-gpu /usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_java --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_cuda --cuda_version=10.1 --cuda_home=/usr/local/cuda-10.1 --cudnn_home=/usr/local/cuda-10.1 + docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro -e NIGHTLY_BUILD onnxruntime-centos6-gpu /usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_java --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_cuda --cuda_version=10.1 --cuda_home=/usr/local/cuda-10.1 --cudnn_home=/usr/local/cuda-10.1 workingDirectory: $(Build.SourcesDirectory) - template: templates/java-api-artifacts-package-and-publish-steps-posix.yml parameters: @@ -234,11 +234,11 @@ jobs: - task: CmdLine@2 inputs: script: | - sudo docker build --pull -t onnxruntime-centos6-gpu --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=$(python.version) -f Dockerfile.centos6_gpu . + docker build --pull -t onnxruntime-centos6-gpu --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=$(python.version) -f Dockerfile.centos6_gpu . workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker - task: CmdLine@2 inputs: script: | - sudo docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6-gpu /onnxruntime_src/tools/ci_build/github/linux/java_linux_final_test.sh -v $(OnnxRuntimeVersion) -r /build + docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-centos6-gpu /onnxruntime_src/tools/ci_build/github/linux/java_linux_final_test.sh -v $(OnnxRuntimeVersion) -r /build workingDirectory: $(Build.BinariesDirectory)/final-jar diff --git a/tools/ci_build/github/azure-pipelines/java-api-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/java-api-packaging-pipelines.yml index 1ecaf6cff5..aa0af46d55 100644 --- a/tools/ci_build/github/azure-pipelines/java-api-packaging-pipelines.yml +++ b/tools/ci_build/github/azure-pipelines/java-api-packaging-pipelines.yml @@ -11,13 +11,13 @@ jobs: - task: CmdLine@2 inputs: script: | - sudo docker build --pull -t onnxruntime-centos6 --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=3.6 -f Dockerfile.centos6 . + docker build --pull -t onnxruntime-centos6 --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=3.6 -f Dockerfile.centos6 . workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker displayName: 'Build CentoOS6 docker image' - task: CmdLine@2 inputs: script: | - sudo docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --build_java --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64" + docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --build_java --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64" workingDirectory: $(Build.SourcesDirectory) displayName: 'Run build and test' diff --git a/tools/ci_build/github/azure-pipelines/linux-arm-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-arm-ci-pipeline.yml deleted file mode 100644 index bcd0ea4622..0000000000 --- a/tools/ci_build/github/azure-pipelines/linux-arm-ci-pipeline.yml +++ /dev/null @@ -1,62 +0,0 @@ -jobs: - - job: Linux_ARM - timeoutInMinutes: 60 - workspace: - clean: all - pool: 'Linux-CPU' - strategy: - matrix: - Py37: - python.include: '3.7m' - cp.tag: 'cp37-cp37m' - Py36: - python.include: '3.6m' - cp.tag: 'cp36-cp36m' - Py35: - python.include: '3.5m' - cp.tag: 'cp35-cp35m' - steps: - - task: CmdLine@2 - inputs: - script: | - set -e -x - sudo rm -rf * - cd $(Build.SourcesDirectory) - git submodule update --init --recursive - cd - - sudo apt-get install -y qemu-user-static - sudo chmod a+x /usr/bin/azcopy - - cat << EOF > tool-chain.cmake - SET(CMAKE_SYSTEM_NAME Linux) - SET(CMAKE_SYSTEM_VERSION 1) - SET(CMAKE_C_COMPILER aarch64-linux-gnu-gcc) - SET(CMAKE_C_FLAGS "-march=armv8-a -mtune=generic -Wno-unused-parameter -Wno-type-limits") - SET(CMAKE_CXX_COMPILER aarch64-linux-gnu-g++) - SET(CMAKE_CXX_FLAGS "-march=armv8-a -mtune=generic -Wno-unused-parameter -Wno-type-limits") - SET(CMAKE_FIND_ROOT_PATH /mnt/toolchains/manylinux2014_aarch64) - SET(CMAKE_FIND_ROOT_PATH_MODE_PROGRAM NEVER) - SET(CMAKE_FIND_ROOT_PATH_MODE_LIBRARY ONLY) - SET(CMAKE_FIND_ROOT_PATH_MODE_INCLUDE ONLY) - SET(CMAKE_FIND_ROOT_PATH_MODE_PACKAGE ONLY) - EOF - export PATH=/mnt/toolchains/gcc-linaro-7.5.0-2019.12-x86_64_aarch64-linux-gnu/bin:$PATH - azcopy cp https://onnxruntimetestdata.blob.core.windows.net/models/toolchains.tar.xz $(Build.BinariesDirectory)/toolchains.tar.xz - sudo rm -rf /mnt/toolchains - sudo mkdir /mnt/toolchains - sudo tar -Jxf $(Build.BinariesDirectory)/toolchains.tar.xz -C /mnt/toolchains - aria2c -q https://github.com/protocolbuffers/protobuf/releases/download/v3.11.1/protoc-3.11.1-linux-x86_64.zip - unzip protoc-3.11.1-linux-x86_64.zip - aria2c -q https://github.com/Kitware/CMake/releases/download/v3.17.1/cmake-3.17.1-Linux-x86_64.tar.gz - tar --strip=1 -zxf cmake-3.17.1-Linux-x86_64.tar.gz - sudo cp /mnt/toolchains/manylinux2014_aarch64/usr/include/stdlib.h /mnt/toolchains/gcc-linaro-7.5.0-2019.12-x86_64_aarch64-linux-gnu/aarch64-linux-gnu/libc/usr/include/ - bin/cmake -Donnxruntime_GCC_STATIC_CPP_RUNTIME=ON -DCMAKE_BUILD_TYPE=Release -Dprotobuf_WITH_ZLIB=OFF -DCMAKE_TOOLCHAIN_FILE=tool-chain.cmake -Donnxruntime_ENABLE_PYTHON=ON -DPYTHON_LIBRARY=dl -DPYTHON_EXECUTABLE=/mnt/toolchains/manylinux2014_aarch64/opt/python/'$(cp.tag)'/bin/python3 -Donnxruntime_BUILD_SHARED_LIB=OFF -Donnxruntime_RUN_ONNX_TESTS=OFF -Donnxruntime_DEV_MODE=ON -DONNX_CUSTOM_PROTOC_EXECUTABLE=$(Build.BinariesDirectory)/bin/protoc "-DPYTHON_INCLUDE_DIR=/mnt/toolchains/manylinux2014_aarch64/usr/include;/mnt/toolchains/manylinux2014_aarch64/opt/python/$(cp.tag)/include/python$(python.include)" -DNUMPY_INCLUDE_DIR=/mnt/toolchains $(Build.SourcesDirectory)/cmake - make -j$(getconf _NPROCESSORS_ONLN) - sudo --preserve-env docker run -v /usr/bin/qemu-aarch64-static:/usr/bin/qemu-aarch64-static -v $(Build.BinariesDirectory):/tmp/a -v $(Build.SourcesDirectory):/tmp/b -w /tmp/a -u $(id -u ${USER}):$(id -g ${USER}) --rm quay.io/pypa/manylinux2014_aarch64 /opt/python/'$(cp.tag)'/bin/python3 /tmp/b/setup.py bdist_wheel - workingDirectory: $(Build.BinariesDirectory) - - task: PublishBuildArtifacts@1 - displayName: 'Publish Artifact: wheels' - inputs: - PathtoPublish: '$(Build.BinariesDirectory)/dist' - ArtifactName: wheels - - template: templates/clean-agent-build-directory-step.yml diff --git a/tools/ci_build/github/azure-pipelines/linux-ngraph-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-ngraph-ci-pipeline.yml index 66a04b74d2..a60bc4fe6a 100644 --- a/tools/ci_build/github/azure-pipelines/linux-ngraph-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-ngraph-ci-pipeline.yml @@ -9,8 +9,8 @@ jobs: displayName: 'Clean untagged docker images' inputs: script: | - sudo docker container prune -f - sudo docker image prune -f + docker container prune -f + docker image prune -f workingDirectory: $(Build.BinariesDirectory) continueOnError: true condition: always() diff --git a/tools/ci_build/github/azure-pipelines/linux-openvino-nightly-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-openvino-nightly-pipeline.yml index 9a9822e052..a1d128ab56 100644 --- a/tools/ci_build/github/azure-pipelines/linux-openvino-nightly-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-openvino-nightly-pipeline.yml @@ -7,8 +7,8 @@ jobs: displayName: 'Clean untagged docker images' inputs: script: | - sudo docker rm $(sudo docker ps -a | grep Exited | awk '{print $1;}') || true - sudo docker images -q --filter "dangling=true" | xargs -n1 -r sudo docker rmi + docker rm $(docker ps -a | grep Exited | awk '{print $1;}') || true + docker images -q --filter "dangling=true" | xargs -n1 -r docker rmi workingDirectory: $(Build.BinariesDirectory) continueOnError: true condition: always() diff --git a/tools/ci_build/github/azure-pipelines/linux-ort-srv-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-ort-srv-ci-pipeline.yml index cf48a75453..a8d7d7519e 100644 --- a/tools/ci_build/github/azure-pipelines/linux-ort-srv-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-ort-srv-ci-pipeline.yml @@ -8,8 +8,8 @@ jobs: displayName: 'Clean untagged docker images' inputs: script: | - sudo docker container prune -f - sudo docker image prune -f + docker container prune -f + docker image prune -f workingDirectory: $(Build.BinariesDirectory) continueOnError: true condition: always() @@ -17,13 +17,13 @@ jobs: - task: CmdLine@2 displayName: 'Build docker image' inputs: - script: sudo docker build --pull -t onnxruntime-server-ubuntu16.04 --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg OS_VERSION=16.04 --build-arg PYTHON_VERSION=3.5 -f Dockerfile.ubuntu_server . + script: docker build --pull -t onnxruntime-server-ubuntu16.04 --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg OS_VERSION=16.04 --build-arg PYTHON_VERSION=3.5 -f Dockerfile.ubuntu_server . workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker - task: CmdLine@2 displayName: 'Run docker image' inputs: - script: sudo --preserve-env docker run --rm --volume $(Build.SourcesDirectory)/server:/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro onnxruntime-server-ubuntu16.04 /bin/bash /onnxruntime_src/ci/run.sh + script: docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory)/server:/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro onnxruntime-server-ubuntu16.04 /bin/bash /onnxruntime_src/ci/run.sh workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker - template: templates/component-governance-component-detection-steps.yml diff --git a/tools/ci_build/github/azure-pipelines/linux-ort-srv-nightly-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-ort-srv-nightly-pipeline.yml index b6de177541..735ae95627 100644 --- a/tools/ci_build/github/azure-pipelines/linux-ort-srv-nightly-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-ort-srv-nightly-pipeline.yml @@ -8,8 +8,8 @@ jobs: displayName: 'Prune docker images' inputs: script: | - sudo docker container prune -f - sudo docker image prune -f + docker container prune -f + docker image prune -f workingDirectory: $(Build.BinariesDirectory) continueOnError: true condition: always() diff --git a/tools/ci_build/github/azure-pipelines/linux-pytorch-custom-ops-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-pytorch-custom-ops-ci-pipeline.yml index 95bd33bb1f..d5da2405ba 100644 --- a/tools/ci_build/github/azure-pipelines/linux-pytorch-custom-ops-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-pytorch-custom-ops-ci-pipeline.yml @@ -15,18 +15,18 @@ jobs: displayName: 'Docker build' inputs: script: | - sudo docker build --pull -t onnxruntime-ubuntu --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=3.6 -f Dockerfile.ubuntu . + docker build --pull -t onnxruntime-ubuntu --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=3.6 -f Dockerfile.ubuntu . workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker - task: CmdLine@2 displayName: 'ONNXRuntime build' inputs: script: | - sudo --preserve-env docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro -e NIGHTLY_BUILD onnxruntime-ubuntu /usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --build_shared_lib --cmake_path /usr/bin/cmake --build_wheel + docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro -e NIGHTLY_BUILD onnxruntime-ubuntu /usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --build_shared_lib --cmake_path /usr/bin/cmake --build_wheel workingDirectory: $(Build.SourcesDirectory) - task: CmdLine@2 displayName: 'Run export tests' inputs: - script: sudo --preserve-env docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build onnxruntime-ubuntu /bin/bash /onnxruntime_src/tools/ci_build/github/linux/test_custom_ops_pytorch_export.sh + script: docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build onnxruntime-ubuntu /bin/bash /onnxruntime_src/tools/ci_build/github/linux/test_custom_ops_pytorch_export.sh condition: succeededOrFailed() # ensure all tests are run diff --git a/tools/ci_build/github/azure-pipelines/nodejs/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nodejs/templates/cpu.yml index 7b7758bafd..729fdae8b5 100644 --- a/tools/ci_build/github/azure-pipelines/nodejs/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nodejs/templates/cpu.yml @@ -41,12 +41,12 @@ jobs: - task: CmdLine@2 inputs: script: | - sudo docker build --pull -t onnxruntime-centos7 --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=3.6 -f Dockerfile.centos . + docker build --pull -t onnxruntime-centos7 --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=3.6 -f Dockerfile.centos . workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker - task: CmdLine@2 inputs: script: | - sudo --preserve-env docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro -e NIGHTLY_BUILD onnxruntime-centos7 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_nodejs --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --enable_onnx_tests && cd /onnxruntime_src/nodejs && npm pack" + docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro -e NIGHTLY_BUILD onnxruntime-centos7 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_nodejs --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --enable_onnx_tests && cd /onnxruntime_src/nodejs && npm pack" workingDirectory: $(Build.SourcesDirectory) - script: | set -e -x diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-featurizers.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-featurizers.yml index 241e4f944a..28b8e20894 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-featurizers.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-featurizers.yml @@ -57,12 +57,12 @@ jobs: - task: CmdLine@2 inputs: script: | - sudo docker build --pull -t onnxruntime-centos6 --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=3.6 -f Dockerfile.centos6 . + docker build --pull -t onnxruntime-centos6 --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=3.6 -f Dockerfile.centos6 . workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker - task: CmdLine@2 inputs: script: | - sudo --preserve-env docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --use_featurizers --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64" + docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --use_featurizers --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64" workingDirectory: $(Build.SourcesDirectory) - script: | set -e -x diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml index 5aef7b622e..635ac12a9b 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-mklml.yml @@ -33,12 +33,12 @@ jobs: - task: CmdLine@2 inputs: script: | - sudo docker build --pull -t onnxruntime-centos6 --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=3.6 -f Dockerfile.centos6 . + docker build --pull -t onnxruntime-centos6 --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=3.6 -f Dockerfile.centos6 . workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker - task: CmdLine@2 inputs: script: | - sudo --preserve-env docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_mklml --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64" + docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_mklml --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64" workingDirectory: $(Build.SourcesDirectory) - script: | set -e -x diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml index ab4444f7f3..f31670be80 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu-nocontribops-arm64.yml @@ -75,12 +75,12 @@ jobs: - task: CmdLine@2 inputs: script: | - sudo docker build --pull -t onnxruntime-centos6 --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=3.6 -f Dockerfile.centos6 . + docker build --pull -t onnxruntime-centos6 --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=3.6 -f Dockerfile.centos6 . workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker - task: CmdLine@2 inputs: script: | - sudo --preserve-env docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --enable_onnx_tests --disable_contrib_ops --disable_ml_ops && cd /build/Release && make install DESTDIR=/build/linux-x64" + docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --enable_onnx_tests --disable_contrib_ops --disable_ml_ops && cd /build/Release && make install DESTDIR=/build/linux-x64" workingDirectory: $(Build.SourcesDirectory) - script: | set -e -x diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml index b8905bab9d..42cc8e2ca4 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/cpu.yml @@ -98,12 +98,12 @@ jobs: - task: CmdLine@2 inputs: script: | - sudo docker build --pull -t onnxruntime-centos6 --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=3.6 -f Dockerfile.centos6 . + docker build --pull -t onnxruntime-centos6 --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=3.6 -f Dockerfile.centos6 . workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker - task: CmdLine@2 inputs: script: | - sudo --preserve-env docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64" + docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro -e NIGHTLY_BUILD onnxruntime-centos6 /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64" workingDirectory: $(Build.SourcesDirectory) - script: | set -e -x diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml index a78793f43a..55fe83d520 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml @@ -82,12 +82,12 @@ jobs: - task: CmdLine@2 inputs: script: | - sudo docker build --pull -t onnxruntime-centos6-gpu --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=$(python.version) -f Dockerfile.centos6_gpu . + docker build --pull -t onnxruntime-centos6-gpu --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=$(python.version) -f Dockerfile.centos6_gpu . workingDirectory: $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker - task: CmdLine@2 inputs: script: | - sudo --preserve-env docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro -e NIGHTLY_BUILD onnxruntime-centos6-gpu /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_cuda --cuda_version=10.1 --cuda_home=/usr/local/cuda-10.1 --cudnn_home=/usr/local/cuda-10.1 --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64" + docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro -e NIGHTLY_BUILD onnxruntime-centos6-gpu /bin/bash -c "/usr/bin/python3.6 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_cuda --cuda_version=10.1 --cuda_home=/usr/local/cuda-10.1 --cudnn_home=/usr/local/cuda-10.1 --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64" - script: | set -e -x mv $(Build.BinariesDirectory)/linux-x64/usr/local/lib64 $(Build.BinariesDirectory)/linux-x64/linux-x64 diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/linux-set-variables-and-download.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/linux-set-variables-and-download.yml index 0138ee3d48..8d0b174eef 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/linux-set-variables-and-download.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/linux-set-variables-and-download.yml @@ -3,9 +3,9 @@ steps: displayName: 'Clean untagged docker images' inputs: script: | - sudo docker rm $(sudo docker ps -a | grep Exited | awk '{print $1;}') || true - sudo docker container prune -f - sudo docker image prune -f + docker rm $(docker ps -a | grep Exited | awk '{print $1;}') || true + docker container prune -f + docker image prune -f workingDirectory: $(Build.BinariesDirectory) continueOnError: true condition: always() diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ci-pipeline.yml index 928a3c3f65..974b466ecf 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ci-pipeline.yml @@ -6,16 +6,12 @@ jobs: AgentPool : 'Linux-GPU-CUDA10' JobName: 'Onnxruntime_Linux_GPU_Training' SubmoduleCheckoutMode: 'recursive' - # TODO --skip_onnx_tests is used to disable onnx_test_runner test. - # onnx_test_runner test occasionally causes "CUDA failure 77: an illegal memory access was encountered" - # remove --skip_onnx_tests when the test failures are fixed. BuildCommand: > tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d gpu -r $(Build.BinariesDirectory) -x " --enable_training --config $(buildConfig) - --skip_onnx_tests --build_wheel --enable_nvtx_profile " diff --git a/tools/ci_build/github/azure-pipelines/post-merge-jobs.yml b/tools/ci_build/github/azure-pipelines/post-merge-jobs.yml index ffcaf67b49..8dd0b82efc 100644 --- a/tools/ci_build/github/azure-pipelines/post-merge-jobs.yml +++ b/tools/ci_build/github/azure-pipelines/post-merge-jobs.yml @@ -24,7 +24,7 @@ jobs: #Merge the mulitple prof data into a single indexed profile data file llvm-profdata merge -sparse -o ort.profdata *.profraw #Create coverage report, output the result to 'report.json' - llvm-cov export -summary-only -instr-profile=ort.profdata onnx_test_runner -object onnxruntime_test_all $(Build.SourcesDirectory)/include/onnxruntime $(Build.SourcesDirectory)/onnxruntime/core $(Build.SourcesDirectory)/onnxruntime/contrib_ops > $(Build.BinariesDirectory)/report.json + llvm-cov export -summary-only -instr-profile=ort.profdata onnxruntime_test_all -object onnxruntime_mlas_test $(Build.SourcesDirectory)/include/onnxruntime $(Build.SourcesDirectory)/onnxruntime/core $(Build.SourcesDirectory)/onnxruntime/contrib_ops > $(Build.BinariesDirectory)/report.json workingDirectory: $(Build.BinariesDirectory) - task: CmdLine@2 diff --git a/tools/ci_build/github/azure-pipelines/templates/linux-set-variables-and-download.yml b/tools/ci_build/github/azure-pipelines/templates/linux-set-variables-and-download.yml index 58a49c506c..e521546d04 100644 --- a/tools/ci_build/github/azure-pipelines/templates/linux-set-variables-and-download.yml +++ b/tools/ci_build/github/azure-pipelines/templates/linux-set-variables-and-download.yml @@ -5,9 +5,9 @@ steps: displayName: 'Clean untagged docker images' inputs: script: | - sudo docker rm $(sudo docker ps -a | grep Exited | awk '{print $1;}') || true - sudo docker container prune -f - sudo docker image prune -f + docker rm $(docker ps -a | grep Exited | awk '{print $1;}') || true + docker container prune -f + docker image prune -f workingDirectory: $(Build.BinariesDirectory) continueOnError: true condition: always() diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml index 12eb30a1b7..7bc444daa7 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml @@ -84,7 +84,7 @@ stages: - task: CmdLine@2 inputs: script: | - sudo docker build --pull \ + docker build --pull \ -t ${{ parameters.docker_image_prefix }}-manylinux-$(python.version) \ --build-arg BUILD_USER=onnxruntimedev \ --build-arg BUILD_UID=$(id -u) \ @@ -94,7 +94,8 @@ stages: - task: CmdLine@2 inputs: script: | - sudo --preserve-env docker run --rm \ + docker run --rm \ + --volume /data/onnx:/data/onnx:ro \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ --volume $(Build.BinariesDirectory):/build \ --volume /data/models:/build/models:ro \ @@ -158,7 +159,7 @@ stages: - task: CmdLine@2 inputs: script: | - sudo docker build --pull \ + docker build --pull \ -t ${{ parameters.docker_image_prefix }}-manylinux-gpu-$(python.version) \ --build-arg BUILD_USER=onnxruntimedev \ --build-arg BUILD_UID=$(id -u) \ @@ -169,7 +170,8 @@ stages: - task: CmdLine@2 inputs: script: | - sudo --preserve-env docker run --rm --gpus all \ + docker run --rm --gpus all \ + --volume /data/onnx:/data/onnx:ro \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ --volume $(Build.BinariesDirectory):/build \ --volume /data/models:/build/models:ro \ @@ -572,8 +574,9 @@ stages: -DNUMPY_INCLUDE_DIR=/mnt/toolchains \ $(Build.SourcesDirectory)/cmake make -j$(getconf _NPROCESSORS_ONLN) - sudo --preserve-env docker run --rm \ + docker run --rm \ -e BUILD_BUILDNUMBER \ + -v /data/onnx:/data/onnx:ro \ -v /usr/bin/qemu-aarch64-static:/usr/bin/qemu-aarch64-static \ -v $(Build.BinariesDirectory):/tmp/a \ -v $(Build.SourcesDirectory):/tmp/b \ diff --git a/tools/ci_build/github/linux/create_package_inside_docker.sh b/tools/ci_build/github/linux/create_package_inside_docker.sh deleted file mode 100755 index ce0972551d..0000000000 --- a/tools/ci_build/github/linux/create_package_inside_docker.sh +++ /dev/null @@ -1,47 +0,0 @@ -#!/bin/bash -#This script uses system provided eigen and protobuf, not the ones in onnxruntime repo. -export PATH=/usr/lib64/ccache:$PATH -set -e -SCRIPT_DIR="$( dirname "${BASH_SOURCE[0]}" )" -TOP_SRC_DIR=$(realpath $SCRIPT_DIR/../../../..) -echo $TOP_SRC_DIR -VERSION_NUMBER=$(cat $TOP_SRC_DIR/VERSION_NUMBER) -rpmdev-setuptree -tmp_build_dir=$(mktemp -d) -export TMP_SOURCE_DIR=$tmp_build_dir/onnxruntime-$VERSION_NUMBER -mkdir -p $TMP_SOURCE_DIR -(cd $SCRIPT_DIR/../../../.. && git archive --format=tar HEAD | (cd $TMP_SOURCE_DIR && tar xf -)) -echo "exporting submodules..." -#TODO: support submodule in submodule -(cd $SCRIPT_DIR/../../../.. && git submodule foreach --recursive " - DEST_DIR=\$TMP_SOURCE_DIR/\$path; - echo \$DEST_DIR - mkdir -p \$DEST_DIR - git archive --format=tar HEAD | tar -C \$DEST_DIR -xf - -") -tar -cf ~/rpmbuild/SOURCES/onnxruntime.tar -C $tmp_build_dir onnxruntime-$VERSION_NUMBER -rm -rf $tmp_build_dir - -/usr/bin/cp /data/onnxruntime/package/rpm/onnxruntime.spec ~/rpmbuild/SPECS -rpmbuild -ba ~/rpmbuild/SPECS/onnxruntime.spec - -#Install the packages and test it -dnf install -y /root/rpmbuild/RPMS/x86_64/onnxruntime-*$VERSION_NUMBER-*.rpm -if [ -d /data/onnx ]; then - onnx_test_runner /data/onnx -fi - -/usr/bin/cp /root/rpmbuild/RPMS/x86_64/onnxruntime-*$VERSION_NUMBER-*.rpm /data/a - - -cd /data/a -#convert rpm to tar -for filename in onnxruntime-*.rpm; do - tmp_dir=$(mktemp -d) - #older cpio doesn't support -D - rpm2cpio $filename | (cd $tmp_dir && cpio -idmv); - dest_filename="${filename//.rpm/.tar.bz2}" - dest_filename="${dest_filename//\.fc[0-9][0-9]\./.}" - tar -jcf $dest_filename -C $tmp_dir . ; - rm -rf tmp_dir -done diff --git a/tools/ci_build/github/linux/docker/scripts/install_deps.sh b/tools/ci_build/github/linux/docker/scripts/install_deps.sh index 1f54119de1..5aaba20d7e 100755 --- a/tools/ci_build/github/linux/docker/scripts/install_deps.sh +++ b/tools/ci_build/github/linux/docker/scripts/install_deps.sh @@ -102,10 +102,6 @@ cd /tmp/src unzip gradle-6.3-bin.zip mv /tmp/src/gradle-6.3 /usr/local/gradle -if ! [ -x "$(command -v protoc)" ]; then - source ${0/%install_deps\.sh/install_protobuf\.sh} -fi - #Don't update 'wheel' to the latest version. see: https://github.com/pypa/auditwheel/issues/102 ${PYTHON_EXE} -m pip install -r ${0/%install_deps\.sh/requirements\.txt} @@ -122,23 +118,6 @@ elif [ $DEVICE_TYPE = "gpu" ]; then fi -#install onnx -export ONNX_ML=1 -if [ "$PYTHON_VER" = "3.4" ];then - echo "Python 3.5 and above is needed for running onnx tests!" 1>&2 -else - source ${0/%install_deps\.sh/install_onnx\.sh} $PYTHON_VER -fi - -#The last onnx version will be kept cd / rm -rf /tmp/src -if [ "$DISTRIBUTOR" = "Ubuntu" ]; then - apt-get -y remove libprotobuf-dev protobuf-compiler -elif [ "$DISTRIBUTOR" = "CentOS" ]; then - rm -rf /usr/include/google - rm -rf /usr/$LIBDIR/libproto* -else - dnf remove -y protobuf-devel protobuf-compiler -fi diff --git a/tools/ci_build/github/linux/docker/scripts/install_fedora.sh b/tools/ci_build/github/linux/docker/scripts/install_fedora.sh deleted file mode 100755 index 30b64d7c41..0000000000 --- a/tools/ci_build/github/linux/docker/scripts/install_fedora.sh +++ /dev/null @@ -1,3 +0,0 @@ -#!/bin/bash -set -e -dnf install -y --best --allowerasing redhat-lsb-core expat-devel libcurl-devel protobuf-compiler protobuf-devel protobuf rpmdevtools tar unzip ccache curl gcc gcc-c++ zlib-devel make git python2-devel python3-devel python3-numpy libunwind icu aria2 rsync python3-setuptools python3-wheel bzip2 diff --git a/tools/ci_build/github/linux/docker/scripts/install_fedora_gpu.sh b/tools/ci_build/github/linux/docker/scripts/install_fedora_gpu.sh deleted file mode 100755 index 8e70c44a9d..0000000000 --- a/tools/ci_build/github/linux/docker/scripts/install_fedora_gpu.sh +++ /dev/null @@ -1,10 +0,0 @@ -#!/bin/bash -set -e -dnf install -y --best --allowerasing redhat-lsb-core expat-devel libcurl-devel protobuf-compiler protobuf-devel protobuf compat-openssl10 lttng-ust libcurl openssl-libs krb5-libs libicu tar unzip ccache curl gcc gcc-c++ zlib-devel make git python2-devel python3-devel python3-numpy libunwind icu aria2 rsync python3-setuptools python3-wheel bzip2 -#install cuda 10.1 and cudnn 7 -aria2c -q -d /tmp 'http://developer.download.nvidia.com/compute/cuda/repos/fedora29/x86_64/cuda-repo-fedora29-10.1.243-1.x86_64.rpm' -dnf install -y /tmp/cuda-repo-fedora29-10.1.243-1.x86_64.rpm -dnf install -y cuda-10.1.243-1.x86_64 -tar -zxvf cudnn-10.1-linux-x64-v7.6.4.38.tgz --strip 1 -C /usr/local -ldconfig -v /usr/local/cuda/lib64 /usr/local/lib64 - diff --git a/tools/ci_build/github/linux/docker/scripts/install_onnx.sh b/tools/ci_build/github/linux/docker/scripts/install_onnx.sh deleted file mode 100755 index c24f918cbf..0000000000 --- a/tools/ci_build/github/linux/docker/scripts/install_onnx.sh +++ /dev/null @@ -1,64 +0,0 @@ -#!/bin/bash - -# The script is to generate all supported versions of onnx models which will be tested by onnx_test_runner -# in the end of ci build pipeline. The purpose is to make sure latest onnxruntime has no regressions. Note -# that the order of installation must be onnx123, onnx130, onnx141, onnx150 and onnxtip since we want -# to keep the tip of master on script exit for onnx backend test which is also a part of build pipeline. -# One possible improvement here is to keep the models saved to some public storage instead of generating -# on the fly every time. - -# The script build onnx from source because the prebuilt package doesn't conform to manylinux1 standard. - -set -e -PYTHON_VER=$1 - -if [[ "$PYTHON_VER" = "3.5" && -d "/opt/python/cp35-cp35m" ]]; then - PYTHON_EXE="/opt/python/cp35-cp35m/bin/python3.5" -elif [[ "$PYTHON_VER" = "3.6" && -d "/opt/python/cp36-cp36m" ]]; then - PYTHON_EXE="/opt/python/cp36-cp36m/bin/python3.6" -elif [[ "$PYTHON_VER" = "3.7" && -d "/opt/python/cp37-cp37m" ]]; then - PYTHON_EXE="/opt/python/cp37-cp37m/bin/python3.7" -elif [[ "$PYTHON_VER" = "3.8" && -d "/opt/python/cp38-cp38" ]]; then - PYTHON_EXE="/opt/python/cp38-cp38/bin/python3.8" -else - PYTHON_EXE="/usr/bin/python${PYTHON_VER}" -fi - -# We ping typing to the most recent version that does not deal -# with Python 3.5 version so we test reasonably modern version -${PYTHON_EXE} -m pip install protobuf typing==3.7.4.1 - -version2tag=(5af210ca8a1c73aa6bae8754c9346ec54d0a756e-onnx123 - bae6333e149a59a3faa9c4d9c44974373dcf5256-onnx130 - 9e55ace55aad1ada27516038dfbdc66a8a0763db-onnx141 - 7d7bc83d29a328233d3e8affa4c4ea8b3e3599ef-onnx150 - 1facb4c1bb9cc2107d4dbaf9fd647fefdbbeb0ab-onnx161 - 0c070abb0c40fec649f81a73a75b0098662ec486-onnx170) #1.7.0 -for v2t in ${version2tag[*]}; do - onnx_version="$(cut -d'-' -f1<<<${v2t})" - onnx_tag="$(cut -d'-' -f2<<<${v2t})" - if [ -z ${lastest_onnx_version+x} ]; then - echo "first pass"; - else - echo "deleting old onnx-${lastest_onnx_version}"; - ${PYTHON_EXE} -m pip uninstall -y onnx - fi - lastest_onnx_version=$onnx_version - GetFile https://github.com/onnx/onnx/archive/$onnx_version.tar.gz /tmp/src/$onnx_version.tar.gz - tar -xf /tmp/src/$onnx_version.tar.gz -C /tmp/src - cd /tmp/src/onnx-$onnx_version - if [ ! -d "third_party/pybind11/pybind11" ]; then - git clone https://github.com/pybind/pybind11.git third_party/pybind11 - fi - # We need to make the adjustment only for CentOS6 OR we substitue this only for - # ${PYTHON_EXE} where we'd need to escape slashes - # Make sure we do not hit pyhon2 as on CentOS 6 it does not work - ESCAPED_PY=$(echo "${PYTHON_EXE}" | sed 's/\//\\\//g') - sed "1,1 s/\/usr\/bin\/env python/$ESCAPED_PY/" /tmp/src/onnx-$onnx_version/tools/protoc-gen-mypy.py > /tmp/src/onnx-$onnx_version/tools/repl_protoc-gen-mypy.py - chmod a+w /tmp/src/onnx-$onnx_version/tools/protoc-gen-mypy.py - mv /tmp/src/onnx-$onnx_version/tools/repl_protoc-gen-mypy.py /tmp/src/onnx-$onnx_version/tools/protoc-gen-mypy.py - mkdir -p /data/onnx/${onnx_tag} - ${PYTHON_EXE} -m pip install . - cd /tmp - ${PYTHON_EXE} -m onnx.backend.test.cmd_tools generate-data -o /data/onnx/$onnx_tag -done diff --git a/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh b/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh index d589ccceef..a5d80ef786 100755 --- a/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh +++ b/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh @@ -54,7 +54,6 @@ if [ "$OS_VERSION" = "16.04" ]; then zip \ rsync libunwind8 libpng16-dev libexpat1-dev \ python3-setuptools python3-numpy python3-wheel python python3-pip python3-pytest \ - libprotobuf-dev libprotobuf9v5 protobuf-compiler \ openjdk-8-jdk" else # ubuntu18.04 PACKAGE_LIST="autotools-dev \ @@ -88,7 +87,6 @@ else # ubuntu18.04 zip \ rsync libunwind8 libpng-dev libexpat1-dev \ python3-setuptools python3-numpy python3-wheel python python3-pip python3-pytest \ - libprotobuf-dev libprotobuf10 protobuf-compiler \ openjdk-11-jdk" fi diff --git a/tools/ci_build/github/linux/docker/scripts/requirements.txt b/tools/ci_build/github/linux/docker/scripts/requirements.txt index b8e92516dd..fc5f5e5175 100644 --- a/tools/ci_build/github/linux/docker/scripts/requirements.txt +++ b/tools/ci_build/github/linux/docker/scripts/requirements.txt @@ -1,6 +1,7 @@ numpy==1.16.6 -requests==2.21.0 +requests==2.24.0 mypy pytest setuptools>=41.4.0 wheel +onnx==1.7.0 \ No newline at end of file diff --git a/tools/ci_build/github/linux/ubuntu16.04/Dockerfile b/tools/ci_build/github/linux/ubuntu16.04/Dockerfile deleted file mode 100644 index 6ec132b370..0000000000 --- a/tools/ci_build/github/linux/ubuntu16.04/Dockerfile +++ /dev/null @@ -1,15 +0,0 @@ -FROM ubuntu:16.04 - -ADD install.sh /tmp/install.sh -ENV PATH="/opt/cmake/bin:${PATH}" -RUN /tmp/install.sh && rm /tmp/install.sh - -WORKDIR /root - -ENV LD_LIBRARY_PATH /usr/local/openblas/lib:$LD_LIBRARY_PATH - -ARG BUILD_UID=1000 -ARG BUILD_USER=onnxruntimedev -RUN adduser --gecos 'ONNXRuntime Build User' --disabled-password $BUILD_USER --uid $BUILD_UID -USER $BUILD_USER -WORKDIR /home/$BUILD_USER diff --git a/tools/ci_build/github/linux/ubuntu16.04/install.sh b/tools/ci_build/github/linux/ubuntu16.04/install.sh deleted file mode 100755 index 06813b9114..0000000000 --- a/tools/ci_build/github/linux/ubuntu16.04/install.sh +++ /dev/null @@ -1,76 +0,0 @@ -#!/bin/bash -set -e -DEBIAN_FRONTEND=noninteractive -apt-get update && apt-get install -y --no-install-recommends \ - autotools-dev \ - build-essential \ - git ca-certificates \ - ca-certificates \ - pkg-config \ - wget \ - zlib1g \ - zlib1g-dev \ - libssl-dev \ - curl \ - autoconf \ - sudo \ - gfortran \ - python3-dev \ - language-pack-en \ - libopenblas-dev \ - liblttng-ust0 \ - libcurl3 \ - libssl1.0.0 \ - libkrb5-3 \ - libicu55 \ - aria2 \ - bzip2 \ - unzip \ - rsync libunwind8 \ - python3-setuptools python3-numpy python3-wheel python python3-pip - -locale-gen en_US.UTF-8 -update-locale LANG=en_US.UTF-8 - -rm -rf /var/lib/apt/lists/* - -aria2c -q -d /tmp https://github.com/ninja-build/ninja/releases/download/v1.8.2/ninja-linux.zip -unzip -q /tmp/ninja-linux.zip -d /usr/bin -mkdir -p /tmp/azcopy -aria2c -q -d /tmp/azcopy -o azcopy.tar.gz https://aka.ms/downloadazcopylinux64 -tar -xf /tmp/azcopy/azcopy.tar.gz -C /tmp/azcopy -/tmp/azcopy/install.sh -rm -rf /tmp/azcopy -#install protobuf -mkdir -p /tmp/src -mkdir -p /opt/cmake -aria2c -q -d /tmp/src https://github.com/Kitware/CMake/releases/download/v3.13.2/cmake-3.13.2-Linux-x86_64.tar.gz -tar -xf /tmp/src/cmake-3.13.2-Linux-x86_64.tar.gz --strip 1 -C /opt/cmake -aria2c -q -d /tmp/src https://github.com/protocolbuffers/protobuf/archive/v3.6.1.tar.gz -tar -xf /tmp/src/protobuf-3.6.1.tar.gz -C /tmp/src -cd /tmp/src/protobuf-3.6.1 -for build_type in 'Debug' 'Relwithdebinfo'; do - pushd . - mkdir build_$build_type - cd build_$build_type - cmake -G Ninja ../cmake -DCMAKE_INSTALL_PREFIX=/usr -DCMAKE_INSTALL_LIBDIR=lib -DCMAKE_INSTALL_SYSCONFDIR=/etc -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=$build_type - ninja - ninja install - popd -done -export ONNX_ML=1 -#3376d4438aaadfba483399fa249b841153152bc0 is v1.2.2 -for onnx_version in "3376d4438aaadfba483399fa249b841153152bc0" "6f91908b6a894278377e2767dc9ce75ce197fb88" ; do - aria2c -q -d /tmp/src https://github.com/onnx/onnx/archive/$onnx_version.tar.gz - tar -xf /tmp/src/onnx-$onnx_version.tar.gz -C /tmp/src - cd /tmp/src/onnx-$onnx_version - git clone https://github.com/pybind/pybind11.git third_party/pybind11 - python3 setup.py bdist_wheel - pip3 install -q dist/* - mkdir -p /data/onnx/$onnx_version - backend-test-tools generate-data -o /data/onnx/$onnx_version - pip3 uninstall -y onnx -done - -chmod 0777 /data/onnx -rm -rf /tmp/src