diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 482b39cd5a..ccba55b0ea 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -1551,11 +1551,6 @@ def nuphar_run_python_tests(build_dir, configs): if is_windows(): cwd = os.path.join(cwd, config) dll_path = os.path.join(build_dir, config, "external", "tvm", config) - # install onnx for shape inference in testing Nuphar scripts - # this needs to happen after onnx_test_data preparation which - # uses onnx 1.3.0 - run_subprocess( - [sys.executable, '-m', 'pip', 'install', '--user', 'onnx==1.5.0']) run_subprocess( [sys.executable, 'onnxruntime_test_python_nuphar.py'], cwd=cwd, dll_path=dll_path) diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml index a1355483ef..c0898350de 100644 --- a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml @@ -39,7 +39,7 @@ jobs: --enable_onnx_tests --use_cuda --cuda_version=11.1 --cuda_home=/usr/local/cuda-11.1 --cudnn_home=/usr/local/cuda-11.1 \ --enable_pybind --build_java --build_nodejs \ --use_tensorrt --tensorrt_home /usr \ - --cmake_extra_defines CMAKE_CUDA_HOST_COMPILER=/opt/rh/devtoolset-9/root/usr/bin/cc PYTHON_INCLUDE_DIR=/opt/python/cp37-cp37m/include/python3.7m PYTHON_LIBRARY=/usr/lib64/librt.so CMAKE_CUDA_ARCHITECTURES=52 + --cmake_extra_defines CMAKE_CUDA_HOST_COMPILER=/opt/rh/devtoolset-9/root/usr/bin/cc CMAKE_CUDA_ARCHITECTURES=52 workingDirectory: $(Build.SourcesDirectory) - task: PublishTestResults@2 displayName: 'Publish unit test results' diff --git a/tools/ci_build/github/azure-pipelines/linux-multi-gpu-tensorrt-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-multi-gpu-tensorrt-ci-pipeline.yml index fdf252cdcc..c9f2fcb2d5 100644 --- a/tools/ci_build/github/azure-pipelines/linux-multi-gpu-tensorrt-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-multi-gpu-tensorrt-ci-pipeline.yml @@ -4,6 +4,6 @@ jobs: AgentPool : 'Linux-Multi-GPU' JobName: 'Linux_CI_Multi_GPU_TensorRT_Dev' # The latest TensorRT container only supports ubuntu20.04 and python 3.8 - RunDockerBuildArgs: '-o ubuntu20.04 -d tensorrt -r $(Build.BinariesDirectory) -p 3.8 -x "--enable_multi_device_test"' + RunDockerBuildArgs: '-o ubuntu20.04 -p 3.8 -d tensorrt -r $(Build.BinariesDirectory) -p 3.8 -x "--enable_multi_device_test"' DoNugetPack: 'false' ArtifactName: 'drop-linux' diff --git a/tools/ci_build/github/azure-pipelines/linux-nuphar-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-nuphar-ci-pipeline.yml index fadd8efc7b..13d5177ed6 100644 --- a/tools/ci_build/github/azure-pipelines/linux-nuphar-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-nuphar-ci-pipeline.yml @@ -3,7 +3,7 @@ jobs: parameters: AgentPool : 'Linux-CPU-2019' JobName: 'Linux_CI_Dev' - RunDockerBuildArgs: '-o ubuntu20.04 -d cpu -r $(Build.BinariesDirectory) -x "--enable_pybind --use_nuphar"' + RunDockerBuildArgs: '-o ubuntu20.04 -p 3.8 -d cpu -r $(Build.BinariesDirectory) -x "--enable_pybind --use_nuphar"' DoNugetPack: 'false' ArtifactName: 'drop-linux' TimeoutInMinutes: 180 diff --git a/tools/ci_build/github/azure-pipelines/linux-openvino-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-openvino-ci-pipeline.yml index ab2502a5be..2e689a3818 100644 --- a/tools/ci_build/github/azure-pipelines/linux-openvino-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-openvino-ci-pipeline.yml @@ -3,7 +3,7 @@ jobs: parameters: AgentPool : 'Linux-CPU-2019' JobName: 'Linux_CI_Dev' - RunDockerBuildArgs: '-o ubuntu20.04 -d openvino -v 2021.3 -r $(Build.BinariesDirectory) -x "--use_openvino CPU_FP32 --build_wheel"' + RunDockerBuildArgs: '-o ubuntu20.04 -p 3.8 -d openvino -v 2021.3 -r $(Build.BinariesDirectory) -x "--use_openvino CPU_FP32 --build_wheel"' DoNugetPack: 'false' ArtifactName: 'drop-linux' TimeoutInMinutes: 120 diff --git a/tools/ci_build/github/azure-pipelines/linux-openvino-nightly-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-openvino-nightly-pipeline.yml index c3e2b79bdd..b3a0e71e51 100644 --- a/tools/ci_build/github/azure-pipelines/linux-openvino-nightly-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-openvino-nightly-pipeline.yml @@ -23,7 +23,7 @@ jobs: - template: templates/run-docker-build-steps.yml parameters: - RunDockerBuildArgs: '-o ubuntu20.04 -d openvino -v 2020.2 -r $(Build.BinariesDirectory) -x "--use_openvino GPU_FP32 --build_wheel"' + RunDockerBuildArgs: '-o ubuntu20.04 -p 3.8 -d openvino -v 2020.2 -r $(Build.BinariesDirectory) -x "--use_openvino GPU_FP32 --build_wheel"' - template: templates/component-governance-component-detection-steps.yml parameters : diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ci-pipeline.yml index df2e6e5d00..ae9b2fe2bd 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ci-pipeline.yml @@ -7,7 +7,7 @@ jobs: JobName: 'Onnxruntime_Linux_GPU_Training' SubmoduleCheckoutMode: 'recursive' RunDockerBuildArgs: > - -o ubuntu20.04 -d gpu -r $(Build.BinariesDirectory) + -o ubuntu20.04 -p 3.8 -d gpu -r $(Build.BinariesDirectory) -x " --enable_training --config $(buildConfig) diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-distributed-e2e-test-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-distributed-e2e-test-pipeline.yml index 66a8ddfad3..9f0f42ead6 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-distributed-e2e-test-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-distributed-e2e-test-pipeline.yml @@ -14,13 +14,13 @@ jobs: - template: templates/run-docker-build-steps.yml parameters: RunDockerBuildArgs: | - -o ubuntu20.04 -d gpu -r $(Build.BinariesDirectory) \ + -o ubuntu20.04 -p 3.8 -d gpu -r $(Build.BinariesDirectory) \ -t onnxruntime_e2e_test_image \ -x " \ --config RelWithDebInfo \ --enable_training \ --update --build \ - --build_wheel \ + --build_wheel --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=70 \ " \ -m DisplayName: 'Build' diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-distributed-test-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-distributed-test-ci-pipeline.yml index 2597ec6d2b..dd7058d66b 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-distributed-test-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-distributed-test-ci-pipeline.yml @@ -14,7 +14,7 @@ jobs: - template: templates/run-docker-build-steps.yml parameters: RunDockerBuildArgs: | - -o ubuntu20.04 -d gpu -r $(Build.BinariesDirectory) \ + -o ubuntu20.04 -p 3.8 -d gpu -r $(Build.BinariesDirectory) \ -t onnxruntime_distributed_tests_image \ -x " \ --config RelWithDebInfo \ diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ortmodule-distributed-test-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ortmodule-distributed-test-ci-pipeline.yml index 5c9c7d6901..db290f3a75 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ortmodule-distributed-test-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ortmodule-distributed-test-ci-pipeline.yml @@ -14,7 +14,7 @@ jobs: - template: templates/run-docker-build-steps.yml parameters: RunDockerBuildArgs: | - -o ubuntu20.04 -d gpu -r $(Build.BinariesDirectory) \ + -o ubuntu20.04 -p 3.8 -d gpu -r $(Build.BinariesDirectory) \ -t onnxruntime_ortmodule_distributed_tests_image \ -x " \ --config RelWithDebInfo \ diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ortmodule-test-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ortmodule-test-ci-pipeline.yml index e243f5a014..8da4fd73d8 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ortmodule-test-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ortmodule-test-ci-pipeline.yml @@ -14,7 +14,7 @@ jobs: - template: templates/run-docker-build-steps.yml parameters: RunDockerBuildArgs: | - -o ubuntu20.04 -d gpu -r $(Build.BinariesDirectory) \ + -o ubuntu20.04 -p 3.8 -p 3.8 -d gpu -r $(Build.BinariesDirectory) \ -t onnxruntime_ortmodule_tests_image \ -x " \ --config RelWithDebInfo \ diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-perf-test-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-perf-test-ci-pipeline.yml index 3c389698a7..a1215a27ac 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-perf-test-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-perf-test-ci-pipeline.yml @@ -16,12 +16,12 @@ jobs: - template: templates/run-docker-build-steps.yml parameters: RunDockerBuildArgs: > - -o ubuntu20.04 -d gpu -r $(Build.BinariesDirectory) + -o ubuntu20.04 -p 3.8 -d gpu -r $(Build.BinariesDirectory) -t onnxruntime_perf_test_image -x " --config RelWithDebInfo --enable_training - --update --build + --update --build --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=70 " DisplayName: 'Build performance tests' diff --git a/tools/ci_build/github/azure-pipelines/templates/linux-ci.yml b/tools/ci_build/github/azure-pipelines/templates/linux-ci.yml index 42abc40e50..5ebe5a4895 100644 --- a/tools/ci_build/github/azure-pipelines/templates/linux-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/linux-ci.yml @@ -2,7 +2,7 @@ parameters: AgentPool : 'Linux-CPU-2019' JobName : 'Linux_CI_Dev' SubmoduleCheckoutMode: '' - RunDockerBuildArgs: '-o ubuntu20.04 -d cpu -r $(Build.BinariesDirectory) -x "--build_wheel"' + RunDockerBuildArgs: '-o ubuntu20.04 -p 3.8 -d cpu -r $(Build.BinariesDirectory) -x "--build_wheel"' DoNodejsPack: 'false' DoNugetPack: 'false' NuPackScript: '' diff --git a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_gpu_training b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_gpu_training index c50a5442aa..1b5e6e6a4a 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_gpu_training +++ b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_gpu_training @@ -1,4 +1,6 @@ -FROM nvidia/cuda:11.1.1-cudnn8-devel-ubuntu18.04 +ARG BASEIMAGE=nvcr.io/nvidia/cuda:11.1.1-cudnn8-devel-ubuntu18.04 + +FROM $BASEIMAGE ARG PYTHON_VERSION=3.6 ARG INSTALL_DEPS_EXTRA_ARGS diff --git a/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh b/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh index d9cdb68b30..345df8e3eb 100755 --- a/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh +++ b/tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh @@ -12,11 +12,12 @@ PYTHON_VER=${PYTHON_VER:=3.5} # Some Edge devices only have limited disk space, use this option to exclude some package DEVICE_TYPE=${DEVICE_TYPE:=Normal} +DEBIAN_FRONTEND=noninteractive +echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections + apt-get update && apt-get install -y software-properties-common lsb-release - OS_VERSION=$(lsb_release -r -s) -DEBIAN_FRONTEND=noninteractive SYS_LONG_BIT=$(getconf LONG_BIT) @@ -144,11 +145,11 @@ rm -rf /var/lib/apt/lists/* if [ "$SYS_LONG_BIT" = "64" ]; then if [ "$DEVICE_TYPE" = "Normal" ]; then - if [ "$OS_VERSION" = "20.04" ]; then - aria2c -q -d /tmp -o llvm.tar.xz https://github.com/llvm/llvm-project/releases/download/llvmorg-11.0.0/llvm-project-11.0.0.tar.xz - else - aria2c -q -d /tmp -o llvm.tar.xz http://releases.llvm.org/9.0.0/clang+llvm-9.0.0-x86_64-linux-gnu-ubuntu-${OS_VERSION}.tar.xz - fi - tar --strip 1 -Jxf /tmp/llvm.tar.xz -C /usr + if [ "$OS_VERSION" = "20.04" ]; then + #llvm 9.0 doesn't have a release for 20.04, but the binaries for 18.04 should work well. + OS_VERSION="18.04" + fi + aria2c -q -d /tmp -o llvm.tar.xz http://releases.llvm.org/9.0.0/clang+llvm-9.0.0-x86_64-linux-gnu-ubuntu-${OS_VERSION}.tar.xz + tar --strip 1 -Jxf /tmp/llvm.tar.xz -C /usr fi fi diff --git a/tools/ci_build/github/linux/run_dockerbuild.sh b/tools/ci_build/github/linux/run_dockerbuild.sh index 4e514249e7..a6e81fd9e0 100755 --- a/tools/ci_build/github/linux/run_dockerbuild.sh +++ b/tools/ci_build/github/linux/run_dockerbuild.sh @@ -59,12 +59,6 @@ if [ $BUILD_OS = "android" ]; then $GET_DOCKER_IMAGE_CMD --repository "onnxruntime-$IMAGE" \ --docker-build-args="--build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER}" \ --dockerfile $DOCKER_FILE --context . -elif [ $BUILD_OS = "centos7" ]; then - IMAGE="centos7" - DOCKER_FILE=Dockerfile.centos - $GET_DOCKER_IMAGE_CMD --repository "onnxruntime-$IMAGE" \ - --docker-build-args="--build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER}" \ - --dockerfile $DOCKER_FILE --context . elif [ $BUILD_OS = "yocto" ]; then IMAGE="arm-yocto-$YOCTO_VERSION" DOCKER_FILE=Dockerfile.ubuntu_for_arm @@ -76,10 +70,9 @@ elif [ $BUILD_OS = "yocto" ]; then $GET_DOCKER_IMAGE_CMD --repository "onnxruntime-$IMAGE" \ --docker-build-args="--build-arg TOOL_CHAIN=$TOOL_CHAIN_SCRIPT --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER}" \ --dockerfile $DOCKER_FILE --context . -else - if [ $BUILD_DEVICE = "gpu" ]; then +elif [ $BUILD_DEVICE = "gpu" ]; then #This code path is only for training. Inferecing pipeline uses CentOS - IMAGE="ubuntu_gpu_training" + IMAGE="$BUILD_OS-gpu_training" INSTALL_DEPS_EXTRA_ARGS="${INSTALL_DEPS_EXTRA_ARGS} -t" if [[ $INSTALL_DEPS_DISTRIBUTED_SETUP = true ]]; then INSTALL_DEPS_EXTRA_ARGS="${INSTALL_DEPS_EXTRA_ARGS} -m" @@ -88,27 +81,37 @@ else INSTALL_DEPS_EXTRA_ARGS="${INSTALL_DEPS_EXTRA_ARGS} -u" fi $GET_DOCKER_IMAGE_CMD --repository "onnxruntime-$IMAGE" \ - --docker-build-args="--build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} --build-arg INSTALL_DEPS_EXTRA_ARGS=\"${INSTALL_DEPS_EXTRA_ARGS}\" --build-arg USE_CONDA=${USE_CONDA} --network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64 --build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-9/root --build-arg PREPEND_PATH=/opt/rh/devtoolset-9/root/usr/bin: --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-9/root/usr/lib64:/opt/rh/devtoolset-9/root/usr/lib:/opt/rh/devtoolset-9/root/usr/lib64/dyninst:/opt/rh/devtoolset-9/root/usr/lib/dyninst:/usr/local/lib64" \ + --docker-build-args="--build-arg BASEIMAGE=nvcr.io/nvidia/cuda:11.1.1-cudnn8-devel-${BUILD_OS} --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} --build-arg INSTALL_DEPS_EXTRA_ARGS=\"${INSTALL_DEPS_EXTRA_ARGS}\" --build-arg USE_CONDA=${USE_CONDA} --network=host" \ --dockerfile Dockerfile.ubuntu_gpu_training --context . - elif [ $BUILD_DEVICE = "tensorrt" ]; then +elif [ $BUILD_DEVICE = "tensorrt" ]; then # TensorRT container release 20.12 IMAGE="$BUILD_OS-cuda11.1-cudnn8.0-tensorrt7.2" DOCKER_FILE=Dockerfile.ubuntu_tensorrt $GET_DOCKER_IMAGE_CMD --repository "onnxruntime-$IMAGE" \ --docker-build-args="--build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} --network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64 --build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-9/root --build-arg PREPEND_PATH=/opt/rh/devtoolset-9/root/usr/bin: --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-9/root/usr/lib64:/opt/rh/devtoolset-9/root/usr/lib:/opt/rh/devtoolset-9/root/usr/lib64/dyninst:/opt/rh/devtoolset-9/root/usr/lib/dyninst:/usr/local/lib64" \ --dockerfile $DOCKER_FILE --context . - elif [ $BUILD_DEVICE = "openvino" ]; then +elif [ $BUILD_DEVICE = "openvino" ]; then IMAGE="$BUILD_OS-openvino" DOCKER_FILE=Dockerfile.ubuntu_openvino $GET_DOCKER_IMAGE_CMD --repository "onnxruntime-$IMAGE" \ --docker-build-args="--build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} --build-arg OPENVINO_VERSION=${OPENVINO_VERSION}" \ --dockerfile $DOCKER_FILE --context . - else +else IMAGE="$BUILD_OS" + IMAGE_OS_VERSION="" + if [ $BUILD_OS = "ubuntu18.04" ]; then + IMAGE_OS_VERSION="18.04" + PYTHON_VER="3.6" + elif [ $BUILD_OS = "ubuntu20.04" ]; then + IMAGE_OS_VERSION="20.04" + PYTHON_VER="3.8" + else + exit 1 + fi + $GET_DOCKER_IMAGE_CMD --repository "onnxruntime-$IMAGE" \ - --docker-build-args="--build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER}" \ + --docker-build-args="--build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} --build-arg OS_VERSION=${IMAGE_OS_VERSION}" \ --dockerfile Dockerfile.ubuntu --context . - fi fi if [ -v EXTRA_IMAGE_TAG ]; then