From 6e2b064aec3ab2dd5d6d5a92d234f8c7d1afe52e Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Tue, 22 Jun 2021 18:43:33 -0700 Subject: [PATCH] Delete some unused code in run_dockerbuild.sh and Enable Nuget CUDA tests (#8089) 1. Remove some unused code and simplify tools/ci_build/github/linux/run_dockerbuild.sh. 2. Enable Nuget CUDA tests. The original design was we could leverage Directory.Build.props and let cmake generate the required properties(USE_CUDA/...) there. However, in nuget packaging pipeline we test the package on a different host that doesn't run cmake command and doesn't have the auto-generated Directory.Build.props file. --- .../runtest-docker-gpu.sh | 35 ----- .../runtest-docker.sh | 43 ------ .../runtest.sh | 11 +- .../linux-multi-gpu-tensorrt-ci-pipeline.yml | 2 +- .../linux-nuphar-ci-pipeline.yml | 2 +- .../linux-openvino-ci-pipeline.yml | 2 +- .../linux-openvino-nightly-pipeline.yml | 2 +- .../azure-pipelines/nuget/templates/gpu.yml | 2 +- .../nuget/templates/test_linux.yml | 60 +++---- .../nuget/templates/test_macos.yml | 3 - .../orttraining-linux-gpu-ci-pipeline.yml | 2 +- ...inux-gpu-distributed-e2e-test-pipeline.yml | 2 +- ...linux-gpu-distributed-test-ci-pipeline.yml | 2 +- ...ortmodule-distributed-test-ci-pipeline.yml | 2 +- ...aining-linux-gpu-perf-test-ci-pipeline.yml | 2 +- .../azure-pipelines/templates/linux-ci.yml | 2 +- .../linux/docker/scripts/install_centos.sh | 21 --- .../linux/docker/scripts/install_ubuntu.sh | 146 ++++-------------- .../scripts/manylinux/install_centos.sh | 9 +- .../ci_build/github/linux/run_dockerbuild.sh | 66 ++++---- 20 files changed, 106 insertions(+), 310 deletions(-) delete mode 100755 csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker-gpu.sh delete mode 100755 csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker.sh delete mode 100755 tools/ci_build/github/linux/docker/scripts/install_centos.sh diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker-gpu.sh b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker-gpu.sh deleted file mode 100755 index 028eab989b..0000000000 --- a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker-gpu.sh +++ /dev/null @@ -1,35 +0,0 @@ -#!/bin/bash -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -# build docker image for CPU - -set -x -e - -SOURCE_ROOT=$1 -BUILD_DIR=$2 -NUGET_REPO_DIRNAME=$3 # path relative to BUILD_DIR -CurrentOnnxRuntimeVersion=$4 -DockerImage=$5 -PACKAGENAME=${PACKAGENAME:-Microsoft.ML.OnnxRuntime.Gpu} -RunTestCsharp=${RunTestCsharp:-true} -RunTestNative=${RunTestNative:-true} - -OldDir=$(pwd) - -cd $SOURCE_ROOT/tools/ci_build/github/linux/docker - -docker run --gpus all --rm \ - --volume "$SOURCE_ROOT:/onnxruntime_src" \ - --volume "$BUILD_DIR:/home/onnxruntimedev" \ - --volume /data/models:/home/onnxruntimedev/models:ro \ - -e "OnnxRuntimeBuildDirectory=/home/onnxruntimedev" \ - -e "IsReleaseBuild=$ISRELEASEBUILD" \ - -e "PACKAGENAME=$PACKAGENAME" \ - -e "RunTestCsharp=$RunTestCsharp" \ - -e "RunTestNative=$RunTestNative" \ - -e "BUILD_BINARIESDIRECTORY=/home/onnxruntimedev" \ - -e "BUILD_SOURCESDIRECTORY=/onnxruntime_src" \ - "$DockerImage" \ - /bin/bash /onnxruntime_src/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.sh \ - /home/onnxruntimedev/$NUGET_REPO_DIRNAME /onnxruntime_src /home/onnxruntimedev $CurrentOnnxRuntimeVersion diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker.sh b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker.sh deleted file mode 100755 index fb79501c77..0000000000 --- a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker.sh +++ /dev/null @@ -1,43 +0,0 @@ -#!/bin/bash -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. - -# build docker image for CPU - -set -x -e - -SOURCE_ROOT=$1 -BUILD_DIR=$2 -NUGET_REPO_DIRNAME=$3 # path relative to BUILD_DIR -CurrentOnnxRuntimeVersion=$4 -DockerImage=$5 -UseCentos7=${6:-false} -Arch=${7:-x64} # x32, x64 -PACKAGENAME=${PACKAGENAME:-Microsoft.ML.OnnxRuntime} -RunTestCsharp=${RunTestCsharp:-true} -RunTestNative=${RunTestNative:-true} -PYTHON_VER=3.5 -IMAGE="ubuntu16.04_$Arch" - -OldDir=$(pwd) - -cd $SOURCE_ROOT/tools/ci_build/github/linux/docker - - -docker run --rm \ - --name "onnxruntime-cpu" \ - --volume "$SOURCE_ROOT:/onnxruntime_src" \ - --volume "$BUILD_DIR:/home/onnxruntimedev" \ - --volume /data/models:/home/onnxruntimedev/models:ro \ - -e "OnnxRuntimeBuildDirectory=/home/onnxruntimedev" \ - -e "IsReleaseBuild=$ISRELEASEBUILD" \ - -e "PACKAGENAME=$PACKAGENAME" \ - -e "DisableContribOps=$DISABLECONTRIBOPS" \ - -e "DisableMlOps=$DISABLEMLOPS" \ - -e "RunTestCsharp=$RunTestCsharp" \ - -e "RunTestNative=$RunTestNative" \ - -e "BUILD_BINARIESDIRECTORY=/home/onnxruntimedev" \ - -e "BUILD_SOURCESDIRECTORY=/onnxruntime_src" \ - "$DockerImage" \ - /bin/bash /onnxruntime_src/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.sh \ - /home/onnxruntimedev/$NUGET_REPO_DIRNAME /onnxruntime_src /home/onnxruntimedev $CurrentOnnxRuntimeVersion diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.sh b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.sh index 40baae40c6..7e2ff0ea64 100755 --- a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.sh +++ b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.sh @@ -3,9 +3,8 @@ # Licensed under the MIT License. LocalNuGetRepo=$1 -BuildDir=$3 -export CurrentOnnxRuntimeVersion=$4 -IsMacOS=${5:-false} +export CurrentOnnxRuntimeVersion=$2 +IsMacOS=${3:-false} PACKAGENAME=${PACKAGENAME:-Microsoft.ML.OnnxRuntime} RunTestCsharp=${RunTestCsharp:-true} RunTestNative=${RunTestNative:-true} @@ -29,7 +28,11 @@ if [ $RunTestCsharp = "true" ]; then exit 1 fi - dotnet test $BUILD_SOURCESDIRECTORY/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/Microsoft.ML.OnnxRuntime.EndToEndTests.csproj --no-restore --verbosity detailed + if [ $PACKAGENAME = "Microsoft.ML.OnnxRuntime.Gpu" ]; then + dotnet test -p:DefineConstants=USE_CUDA $BUILD_SOURCESDIRECTORY/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/Microsoft.ML.OnnxRuntime.EndToEndTests.csproj --no-restore --verbosity detailed + else + dotnet test $BUILD_SOURCESDIRECTORY/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/Microsoft.ML.OnnxRuntime.EndToEndTests.csproj --no-restore --verbosity detailed + fi if [ $? -ne 0 ]; then echo "Failed to build or execute the end-to-end test" exit 1 diff --git a/tools/ci_build/github/azure-pipelines/linux-multi-gpu-tensorrt-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-multi-gpu-tensorrt-ci-pipeline.yml index c9f2fcb2d5..f0e298073b 100644 --- a/tools/ci_build/github/azure-pipelines/linux-multi-gpu-tensorrt-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-multi-gpu-tensorrt-ci-pipeline.yml @@ -4,6 +4,6 @@ jobs: AgentPool : 'Linux-Multi-GPU' JobName: 'Linux_CI_Multi_GPU_TensorRT_Dev' # The latest TensorRT container only supports ubuntu20.04 and python 3.8 - RunDockerBuildArgs: '-o ubuntu20.04 -p 3.8 -d tensorrt -r $(Build.BinariesDirectory) -p 3.8 -x "--enable_multi_device_test"' + RunDockerBuildArgs: '-o ubuntu20.04 -d tensorrt -x "--enable_multi_device_test"' DoNugetPack: 'false' ArtifactName: 'drop-linux' diff --git a/tools/ci_build/github/azure-pipelines/linux-nuphar-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-nuphar-ci-pipeline.yml index 13d5177ed6..90181db7a3 100644 --- a/tools/ci_build/github/azure-pipelines/linux-nuphar-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-nuphar-ci-pipeline.yml @@ -3,7 +3,7 @@ jobs: parameters: AgentPool : 'Linux-CPU-2019' JobName: 'Linux_CI_Dev' - RunDockerBuildArgs: '-o ubuntu20.04 -p 3.8 -d cpu -r $(Build.BinariesDirectory) -x "--enable_pybind --use_nuphar"' + RunDockerBuildArgs: '-o ubuntu20.04 -d cpu -x "--enable_pybind --use_nuphar"' DoNugetPack: 'false' ArtifactName: 'drop-linux' TimeoutInMinutes: 180 diff --git a/tools/ci_build/github/azure-pipelines/linux-openvino-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-openvino-ci-pipeline.yml index 2e689a3818..ec35166baf 100644 --- a/tools/ci_build/github/azure-pipelines/linux-openvino-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-openvino-ci-pipeline.yml @@ -3,7 +3,7 @@ jobs: parameters: AgentPool : 'Linux-CPU-2019' JobName: 'Linux_CI_Dev' - RunDockerBuildArgs: '-o ubuntu20.04 -p 3.8 -d openvino -v 2021.3 -r $(Build.BinariesDirectory) -x "--use_openvino CPU_FP32 --build_wheel"' + RunDockerBuildArgs: '-o ubuntu20.04 -d openvino -v 2021.3 -x "--use_openvino CPU_FP32 --build_wheel"' DoNugetPack: 'false' ArtifactName: 'drop-linux' TimeoutInMinutes: 120 diff --git a/tools/ci_build/github/azure-pipelines/linux-openvino-nightly-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-openvino-nightly-pipeline.yml index b3a0e71e51..689322ed36 100644 --- a/tools/ci_build/github/azure-pipelines/linux-openvino-nightly-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-openvino-nightly-pipeline.yml @@ -23,7 +23,7 @@ jobs: - template: templates/run-docker-build-steps.yml parameters: - RunDockerBuildArgs: '-o ubuntu20.04 -p 3.8 -d openvino -v 2020.2 -r $(Build.BinariesDirectory) -x "--use_openvino GPU_FP32 --build_wheel"' + RunDockerBuildArgs: '-o ubuntu20.04 -d openvino -v 2020.2 -x "--use_openvino GPU_FP32 --build_wheel"' - template: templates/component-governance-component-detection-steps.yml parameters : diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml index cd3cc069ff..73e495de19 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml @@ -315,7 +315,7 @@ jobs: parameters: AgentPool : 'Onnxruntime-Linux-GPU' TestGPU : 'true' - + - job: Publish_NuGet_Package_And_Report workspace: clean: all diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml index 97db32c85f..d7a7e4cdf4 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml @@ -1,6 +1,5 @@ parameters: AgentPool: 'Linux-CPU' - TestGPU: 'false' jobs: - job: NuGet_Test_Linux @@ -32,48 +31,25 @@ jobs: parameters: packageFolder: '$(Build.BinariesDirectory)/nuget-artifact' - - ${{ if eq(parameters['TestGPU'], 'false') }}: - - template: ../../templates/get-docker-image-steps.yml - parameters: - Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cpu - Context: tools/ci_build/github/linux/docker - DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )" - Repository: onnxruntimecpubuild - - - script: | - set -e -x - $(Build.SourcesDirectory)/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker.sh \ - $(Build.SourcesDirectory) \ - $(Build.BinariesDirectory) \ - nuget-artifact \ - $(NuGetPackageVersionNumber) \ - onnxruntimecpubuild \ - true - displayName: 'Run Package Test (x64) on CentOS7' - env: - OnnxRuntimeBuildDirectory: $(Build.BinariesDirectory) - - - - ${{ if eq(parameters['TestGPU'], 'true') }}: - - template: ../../templates/get-docker-image-steps.yml - parameters: - Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11 - Context: tools/ci_build/github/linux/docker - DockerBuildArgs: "--network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64 --build-arg BASEIMAGE=nvcr.io/nvidia/cuda:11.1-cudnn8-devel-centos7 --build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-9/root --build-arg PREPEND_PATH=/opt/rh/devtoolset-9/root/usr/bin: --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-9/root/usr/lib64:/opt/rh/devtoolset-9/root/usr/lib:/opt/rh/devtoolset-9/root/usr/lib64/dyninst:/opt/rh/devtoolset-9/root/usr/lib/dyninst:/usr/local/lib64 --build-arg BUILD_UID=$( id -u )" - Repository: onnxruntimecuda11build - - - script: | - set -e -x - $(Build.SourcesDirectory)/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker-gpu.sh \ - $(Build.SourcesDirectory) \ - $(Build.BinariesDirectory) \ - nuget-artifact \ - $(NuGetPackageVersionNumber) \ - onnxruntimecuda11build - displayName: 'Run Package Test GPU (x64)' - env: - OnnxRuntimeBuildDirectory: $(Build.BinariesDirectory) + - task: CmdLine@2 + displayName: 'Create symlink for test models' + inputs: + script: | + ln -sf /data/models $(Build.BinariesDirectory) + - task: Bash@3 + displayName: 'Run Package Test' + inputs: + targetType: filePath + filePath: '$(Build.SourcesDirectory)/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.sh' + arguments: '$(Build.BinariesDirectory)/nuget-artifact $(NuGetPackageVersionNumber)' + workingDirectory: $(Build.BinariesDirectory) + env: + OnnxRuntimeBuildDirectory: $(Build.BinariesDirectory) + DisableContribOps: $(DisableContribOps) + DisableMlOps: $(DisableMlOps) + IsReleaseBuild: $(IsReleaseBuild) + - template: ../../templates/component-governance-component-detection-steps.yml parameters : condition : 'always' diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/test_macos.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/test_macos.yml index 9cc9fe4c6c..0bb1852e5a 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/test_macos.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/test_macos.yml @@ -31,11 +31,8 @@ jobs: packageFolder: '$(Build.BinariesDirectory)/nuget-artifact' - script: | - brew install libomp $(Build.SourcesDirectory)/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.sh \ $(Build.BinariesDirectory)/nuget-artifact \ - $(Build.SourcesDirectory) \ - $(Build.BinariesDirectory) \ $(NuGetPackageVersionNumber) \ ${{ parameters.IsMacOS }} diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ci-pipeline.yml index 27360851cc..59b18847bc 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ci-pipeline.yml @@ -7,7 +7,7 @@ jobs: JobName: 'Onnxruntime_Linux_GPU_Training' SubmoduleCheckoutMode: 'recursive' RunDockerBuildArgs: > - -o ubuntu20.04 -p 3.8 -d gpu -r $(Build.BinariesDirectory) + -o ubuntu20.04 -d gpu -t onnxruntime_orttraining_ortmodule_tests_image -e -x " diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-distributed-e2e-test-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-distributed-e2e-test-pipeline.yml index 9f0f42ead6..6135f751f2 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-distributed-e2e-test-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-distributed-e2e-test-pipeline.yml @@ -14,7 +14,7 @@ jobs: - template: templates/run-docker-build-steps.yml parameters: RunDockerBuildArgs: | - -o ubuntu20.04 -p 3.8 -d gpu -r $(Build.BinariesDirectory) \ + -o ubuntu20.04 -d gpu \ -t onnxruntime_e2e_test_image \ -x " \ --config RelWithDebInfo \ diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-distributed-test-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-distributed-test-ci-pipeline.yml index dd7058d66b..f11877cd50 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-distributed-test-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-distributed-test-ci-pipeline.yml @@ -14,7 +14,7 @@ jobs: - template: templates/run-docker-build-steps.yml parameters: RunDockerBuildArgs: | - -o ubuntu20.04 -p 3.8 -d gpu -r $(Build.BinariesDirectory) \ + -o ubuntu20.04 -d gpu \ -t onnxruntime_distributed_tests_image \ -x " \ --config RelWithDebInfo \ diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ortmodule-distributed-test-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ortmodule-distributed-test-ci-pipeline.yml index 23754a3ee6..23464bf3d9 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ortmodule-distributed-test-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ortmodule-distributed-test-ci-pipeline.yml @@ -14,7 +14,7 @@ jobs: - template: templates/run-docker-build-steps.yml parameters: RunDockerBuildArgs: | - -o ubuntu20.04 -p 3.8 -d gpu -r $(Build.BinariesDirectory) \ + -o ubuntu20.04 -d gpu \ -t onnxruntime_ortmodule_distributed_tests_image \ -x " \ --config RelWithDebInfo \ diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-perf-test-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-perf-test-ci-pipeline.yml index 2808ffddac..2ea8bc4068 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-perf-test-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-perf-test-ci-pipeline.yml @@ -16,7 +16,7 @@ jobs: - template: templates/run-docker-build-steps.yml parameters: RunDockerBuildArgs: > - -o ubuntu20.04 -p 3.8 -d gpu -r $(Build.BinariesDirectory) + -o ubuntu20.04 -d gpu -t onnxruntime_perf_test_image -x " --config RelWithDebInfo diff --git a/tools/ci_build/github/azure-pipelines/templates/linux-ci.yml b/tools/ci_build/github/azure-pipelines/templates/linux-ci.yml index c0635d0fee..88b4cc5411 100644 --- a/tools/ci_build/github/azure-pipelines/templates/linux-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/linux-ci.yml @@ -2,7 +2,7 @@ parameters: AgentPool : 'Linux-CPU-2019' JobName : 'Linux_CI_Dev' SubmoduleCheckoutMode: '' - RunDockerBuildArgs: '-o ubuntu20.04 -p 3.8 -d cpu -r $(Build.BinariesDirectory) -x "--build_wheel"' + RunDockerBuildArgs: '-o ubuntu20.04 -d cpu -x "--build_wheel"' DoNodejsPack: 'false' DoNugetPack: 'false' NuPackScript: '' diff --git a/tools/ci_build/github/linux/docker/scripts/install_centos.sh b/tools/ci_build/github/linux/docker/scripts/install_centos.sh deleted file mode 100755 index ffc2d7882e..0000000000 --- a/tools/ci_build/github/linux/docker/scripts/install_centos.sh +++ /dev/null @@ -1,21 +0,0 @@ -#!/bin/bash -set -e - -os_major_version=$(cat /etc/redhat-release | tr -dc '0-9.'|cut -d \. -f1) - - -if ! rpm -q --quiet epel-release ; then - yum install -y https://dl.fedoraproject.org/pub/epel/epel-release-latest-$os_major_version.noarch.rpm -fi - -echo "installing for os major version : $os_major_version" -yum install -y redhat-lsb-core expat-devel libcurl-devel tar unzip curl zlib-devel make libunwind icu aria2 rsync bzip2 git bzip2-devel - -# install dotnet core dependencies -yum install -y lttng-ust openssl-libs krb5-libs libicu libuuid -# install dotnet runtimes -yum install -y https://packages.microsoft.com/config/centos/7/packages-microsoft-prod.rpm -yum install -y dotnet-sdk-2.2 java-1.8.0-openjdk-devel ccache gcc gcc-c++ python3 python3-devel python3-pip - -# install automatic documentation generation dependencies -yum install -y graphviz diff --git a/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh b/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh index 345df8e3eb..f6d2fdc595 100755 --- a/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh +++ b/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh @@ -3,12 +3,10 @@ set -e while getopts p:d: parameter_Option do case "${parameter_Option}" in -p) PYTHON_VER=${OPTARG};; d) DEVICE_TYPE=${OPTARG};; esac done -PYTHON_VER=${PYTHON_VER:=3.5} # Some Edge devices only have limited disk space, use this option to exclude some package DEVICE_TYPE=${DEVICE_TYPE:=Normal} @@ -21,126 +19,48 @@ OS_VERSION=$(lsb_release -r -s) SYS_LONG_BIT=$(getconf LONG_BIT) +PACKAGE_LIST="autotools-dev \ + automake \ + build-essential \ + git apt-transport-https apt-utils \ + ca-certificates \ + pkg-config \ + wget \ + zlib1g \ + zlib1g-dev \ + libssl-dev \ + curl libcurl4-openssl-dev \ + autoconf \ + sudo \ + gfortran \ + python3-dev \ + language-pack-en \ + liblttng-ust0 \ + libcurl4 \ + libssl1.1 \ + libkrb5-3 \ + libicu66 \ + libtinfo-dev \ + libtool \ + openssh-server \ + aria2 \ + bzip2 \ + unzip \ + zip \ + rsync libunwind8 libpng-dev libexpat1-dev \ + python3-setuptools python3-numpy python3-wheel python python3-pip python3-pytest \ + openjdk-11-jdk \ + graphviz" -#see: https://docs.microsoft.com/en-us/dotnet/core/linux-prerequisites?tabs=netcore21 -if [ "$OS_VERSION" = "18.04" ]; then - PACKAGE_LIST="autotools-dev \ - automake \ - build-essential \ - git apt-transport-https apt-utils \ - ca-certificates \ - pkg-config \ - wget \ - zlib1g \ - zlib1g-dev \ - libssl-dev \ - curl libcurl4-openssl-dev \ - autoconf \ - sudo \ - gfortran \ - python3-dev \ - language-pack-en \ - liblttng-ust0 \ - libcurl4 \ - libssl1.0.0 \ - libkrb5-3 \ - libicu60 \ - libtinfo-dev \ - libtool \ - openssh-server \ - aria2 \ - bzip2 \ - unzip \ - zip \ - rsync libunwind8 libpng-dev libexpat1-dev \ - python3-setuptools python3-numpy python3-wheel python python3-pip python3-pytest \ - openjdk-11-jdk \ - graphviz" -else # ubuntu20.04 - PACKAGE_LIST="autotools-dev \ - automake \ - build-essential \ - git apt-transport-https apt-utils \ - ca-certificates \ - pkg-config \ - wget \ - zlib1g \ - zlib1g-dev \ - libssl-dev \ - curl libcurl4-openssl-dev \ - autoconf \ - sudo \ - gfortran \ - python3-dev \ - language-pack-en \ - liblttng-ust0 \ - libcurl4 \ - libssl1.1 \ - libkrb5-3 \ - libicu66 \ - libtinfo-dev \ - libtool \ - openssh-server \ - aria2 \ - bzip2 \ - unzip \ - zip \ - rsync libunwind8 libpng-dev libexpat1-dev \ - python3-setuptools python3-numpy python3-wheel python python3-pip python3-pytest \ - openjdk-11-jdk \ - graphviz" -fi if [ $DEVICE_TYPE = "Normal" ]; then PACKAGE_LIST="$PACKAGE_LIST libedit-dev libxml2-dev python3-packaging" fi -apt-get update && apt-get install -y --no-install-recommends $PACKAGE_LIST +apt-get install -y --no-install-recommends $PACKAGE_LIST locale-gen en_US.UTF-8 update-locale LANG=en_US.UTF-8 -if [ "$SYS_LONG_BIT" = "64" ]; then - mkdir -p /tmp/dotnet - aria2c -q -d /tmp/dotnet https://packages.microsoft.com/config/ubuntu/${OS_VERSION}/packages-microsoft-prod.deb - dpkg -i /tmp/dotnet/packages-microsoft-prod.deb - apt-get update - apt-get install -y dotnet-sdk-2.1 - rm -rf /tmp/dotnet -fi - -if [ "$OS_VERSION" = "16.04" ]; then - exit 1 -elif [ "$OS_VERSION" = "18.04" ]; then - if [ "$PYTHON_VER" != "3.6" ]; then - add-apt-repository -y ppa:deadsnakes/ppa - apt-get update - apt-get install -y --no-install-recommends \ - python${PYTHON_VER} \ - python${PYTHON_VER}-dev - update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VER} 1 - update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.6 2 - update-alternatives --set python3 /usr/bin/python${PYTHON_VER} - #TODO: the old one(/usr/bin/pip3) should be uninstalled first. Because the one will be - #put at /usr/local/. Then there will be two pips. - /usr/bin/python${PYTHON_VER} -m pip install --upgrade --force-reinstall pip==19.0.3 - fi - -else # ubuntu20.04 - if [ "$PYTHON_VER" != "3.8" ]; then - add-apt-repository -y ppa:deadsnakes/ppa - apt-get update - apt-get install -y --no-install-recommends \ - python${PYTHON_VER} \ - python${PYTHON_VER}-dev - update-alternatives --install /usr/bin/python3 python3 /usr/bin/python${PYTHON_VER} 1 - update-alternatives --install /usr/bin/python3 python3 /usr/bin/python3.8 2 - update-alternatives --set python3 /usr/bin/python${PYTHON_VER} - #TODO: the old one(/usr/bin/pip3) should be uninstalled first. Because the one will be - #put at /usr/local/. Then there will be two pips. - /usr/bin/python${PYTHON_VER} -m pip install --upgrade --force-reinstall pip==19.0.3 - fi -fi - rm -rf /var/lib/apt/lists/* if [ "$SYS_LONG_BIT" = "64" ]; then diff --git a/tools/ci_build/github/linux/docker/scripts/manylinux/install_centos.sh b/tools/ci_build/github/linux/docker/scripts/manylinux/install_centos.sh index 7940c12e31..92b85c2e7a 100755 --- a/tools/ci_build/github/linux/docker/scripts/manylinux/install_centos.sh +++ b/tools/ci_build/github/linux/docker/scripts/manylinux/install_centos.sh @@ -9,15 +9,8 @@ if ! rpm -q --quiet epel-release ; then fi echo "installing for os major version : $os_major_version" -yum install -y which gdb redhat-lsb-core expat-devel libcurl-devel tar unzip curl zlib-devel make libunwind icu aria2 rsync bzip2 git bzip2-devel +yum install -y which gdb redhat-lsb-core expat-devel tar unzip zlib-devel make libunwind aria2 bzip2 bzip2-devel -if [ "$os_major_version" == "7" ]; then - # install dotnet core dependencies - yum install -y lttng-ust openssl-libs krb5-libs libicu libuuid - # install dotnet runtimes - yum install -y https://packages.microsoft.com/config/centos/7/packages-microsoft-prod.rpm - yum install -y dotnet-sdk-2.1 -fi # Install Java # Install automatic documentation generation dependencies diff --git a/tools/ci_build/github/linux/run_dockerbuild.sh b/tools/ci_build/github/linux/run_dockerbuild.sh index ec8a68e077..117b77ebbc 100755 --- a/tools/ci_build/github/linux/run_dockerbuild.sh +++ b/tools/ci_build/github/linux/run_dockerbuild.sh @@ -1,29 +1,34 @@ #!/bin/bash +#This file is only for Linux pipelines that build on ubuntu. All the docker images here are based on ubuntu. +#Please don't put CentOS or manylinux2014 related stuffs here. set -e -o -x id -SCRIPT_DIR="$( dirname "${BASH_SOURCE[0]}" )" -SOURCE_ROOT=$(realpath $SCRIPT_DIR/../../../../) +SOURCE_ROOT=$BUILD_SOURCESDIRECTORY +SCRIPT_DIR=$BUILD_SOURCESDIRECTORY/tools/ci_build/github/linux +BUILD_DIR=$BUILD_BINARIESDIRECTORY + + YOCTO_VERSION="4.19" +#Training only INSTALL_DEPS_DISTRIBUTED_SETUP=false +#Training only ORTMODULE_BUILD=false +#Training only USE_CONDA=false ALLOW_RELEASED_ONNX_OPSET_ONLY_ENV="ALLOW_RELEASED_ONNX_OPSET_ONLY="$ALLOW_RELEASED_ONNX_OPSET_ONLY echo "ALLOW_RELEASED_ONNX_OPSET_ONLY environment variable is set as "$ALLOW_RELEASED_ONNX_OPSET_ONLY_ENV -while getopts c:o:d:r:p:x:a:v:y:t:i:mue parameter_Option +while getopts o:d:p:x:v:y:t:i:mue parameter_Option do case "${parameter_Option}" in -#android, ubuntu16.04, ubuntu18.04, CentOS7 +#android, yocto, ubuntu20.04 o) BUILD_OS=${OPTARG};; -#cpu, gpu, tensorrt +#gpu, tensorrt or openvino. It is ignored when BUILD_OS is android or yocto. d) BUILD_DEVICE=${OPTARG};; -r) BUILD_DIR=${OPTARG};; #python version: 3.6 3.7 (absence means default 3.6) p) PYTHON_VER=${OPTARG};; # "--build_wheel --use_openblas" x) BUILD_EXTR_PAR=${OPTARG};; -# x86 or other, only for ubuntu16.04 os -a) BUILD_ARCH=${OPTARG};; # openvino version tag: 2020.3 (OpenVINO EP 2.0 supports version starting 2020.3) v) OPENVINO_VERSION=${OPTARG};; # YOCTO 4.19 + ACL 19.05, YOCTO 4.14 + ACL 19.02 @@ -94,30 +99,32 @@ elif [[ $BUILD_DEVICE = "tensorrt"* ]]; then DOCKER_FILE=Dockerfile.ubuntu_tensorrt fi $GET_DOCKER_IMAGE_CMD --repository "onnxruntime-$IMAGE" \ - --docker-build-args="--build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} --network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64 --build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-9/root --build-arg PREPEND_PATH=/opt/rh/devtoolset-9/root/usr/bin: --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-9/root/usr/lib64:/opt/rh/devtoolset-9/root/usr/lib:/opt/rh/devtoolset-9/root/usr/lib64/dyninst:/opt/rh/devtoolset-9/root/usr/lib/dyninst:/usr/local/lib64" \ - --dockerfile $DOCKER_FILE --context . -elif [ $BUILD_DEVICE = "openvino" ]; then - IMAGE="$BUILD_OS-openvino" - DOCKER_FILE=Dockerfile.ubuntu_openvino - $GET_DOCKER_IMAGE_CMD --repository "onnxruntime-$IMAGE" \ - --docker-build-args="--build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} --build-arg OPENVINO_VERSION=${OPENVINO_VERSION}" \ + --docker-build-args="--build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER}" \ --dockerfile $DOCKER_FILE --context . else - IMAGE="$BUILD_OS" - IMAGE_OS_VERSION="" - if [ $BUILD_OS = "ubuntu18.04" ]; then - IMAGE_OS_VERSION="18.04" - PYTHON_VER="3.6" - elif [ $BUILD_OS = "ubuntu20.04" ]; then - IMAGE_OS_VERSION="20.04" - PYTHON_VER="3.8" - else - exit 1 - fi - + IMAGE_OS_VERSION="" + if [ $BUILD_OS = "ubuntu18.04" ]; then + IMAGE_OS_VERSION="18.04" + PYTHON_VER="3.6" + elif [ $BUILD_OS = "ubuntu20.04" ]; then + IMAGE_OS_VERSION="20.04" + PYTHON_VER="3.8" + else + exit 1 + fi + BUILD_ARGS="--build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} --build-arg OS_VERSION=${IMAGE_OS_VERSION}" + + if [ $BUILD_DEVICE = "openvino" ]; then + IMAGE="$BUILD_OS-openvino" + DOCKER_FILE=Dockerfile.ubuntu_openvino + BUILD_ARGS+=" --build-arg OPENVINO_VERSION=${OPENVINO_VERSION}" + else + IMAGE="$BUILD_OS" + DOCKER_FILE=Dockerfile.ubuntu + fi $GET_DOCKER_IMAGE_CMD --repository "onnxruntime-$IMAGE" \ - --docker-build-args="--build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} --build-arg OS_VERSION=${IMAGE_OS_VERSION}" \ - --dockerfile Dockerfile.ubuntu --context . + --docker-build-args="${BUILD_ARGS}" \ + --dockerfile $DOCKER_FILE --context . fi if [ -v EXTRA_IMAGE_TAG ]; then @@ -125,7 +132,6 @@ if [ -v EXTRA_IMAGE_TAG ]; then fi set +e -mkdir -p ~/.cache/onnxruntime mkdir -p ~/.onnx if [ -z "$NIGHTLY_BUILD" ]; then