Update Linux GPU build pipelines to CUDA 10.2 (#5120)

* Update Linux GPU build pipelines to CUDA 10.2
This commit is contained in:
Changming Sun 2020-09-10 17:40:51 -07:00 committed by GitHub
parent a8557b3f0f
commit c5efb0085d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 16 additions and 208 deletions

View file

@ -14,19 +14,11 @@ PackageName=${PACKAGENAME:-Microsoft.ML.OnnxRuntime.Gpu}
RunTestCsharp=${RunTestCsharp:-true}
RunTestNative=${RunTestNative:-true}
PYTHON_VER=3.5
IMAGE="ubuntu16.04-$CUDA_VER"
OldDir=$(pwd)
cd $SOURCE_ROOT/tools/ci_build/github/linux/docker
sudo docker build -t onnxruntime-ubuntu16.04-cuda10.1-cudnn7.6 --build-arg OS_VERSION=16.04 --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f Dockerfile.ubuntu_gpu .
sudo docker rm -f "onnxruntime-gpu-container" || true
sudo --preserve-env docker run -h $HOSTNAME \
--rm \
--name "onnxruntime-gpu-container" \
docker run --gpus all --rm \
--volume "$SOURCE_ROOT:/onnxruntime_src" \
--volume "$BUILD_DIR:/home/onnxruntimedev" \
--volume /data/models:/home/onnxruntimedev/models:ro \
@ -35,6 +27,6 @@ sudo --preserve-env docker run -h $HOSTNAME \
-e "PackageName=$PackageName" \
-e "RunTestCsharp=$RunTestCsharp" \
-e "RunTestNative=$RunTestNative" \
onnxruntime-ubuntu16.04-cuda10.1-cudnn7.6 \
onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecuda102build:ch2q \
/bin/bash /onnxruntime_src/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.sh \
/home/onnxruntimedev/$NUGET_REPO_DIRNAME /onnxruntime_src /home/onnxruntimedev $CurrentOnnxRuntimeVersion

View file

@ -623,6 +623,7 @@ namespace Microsoft.ML.OnnxRuntime.Tests
{
skipModels["test_vgg19"] = "Get preallocated buffer for initializer conv4_4_b_0 failed";
skipModels["GPT2_LM_HEAD"] = "System out of memory";
skipModels["GPT2"] = "System out of memory";
skipModels["tf_pnasnet_large"] = "Get preallocated buffer for initializer ConvBnFusion_BN_B_cell_5/comb_iter_1/left/bn_sep_7x7_1/beta:0_203 failed";
skipModels["tf_nasnet_large"] = "Get preallocated buffer for initializer ConvBnFusion_BN_B_cell_11/beginning_bn/beta:0_331 failed";
skipModels["test_zfnet512"] = "System out of memory";

View file

@ -58,9 +58,9 @@ jobs:
script: |
mkdir -p $HOME/.onnx
docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \
--volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:chp \
--volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecuda102build:ch2q \
/opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release \
--skip_submodule_sync --parallel --build_shared_lib --use_cuda --cuda_version=10.1 --cuda_home=/usr/local/cuda-10.1 --cudnn_home=/usr/local/cuda-10.1
--skip_submodule_sync --parallel --build_shared_lib --use_cuda --cuda_version=10.2 --cuda_home=/usr/local/cuda-10.2 --cudnn_home=/usr/local/cuda-10.2
workingDirectory: $(Build.SourcesDirectory)
- task: Docker@2
displayName: logout

View file

@ -24,7 +24,7 @@ jobs:
inputs:
script: |
mkdir -p $HOME/.onnx
docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:chp /opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_java --build_shared_lib --use_cuda --cuda_version=10.1 --cuda_home=/usr/local/cuda-10.1 --cudnn_home=/usr/local/cuda-10.1
docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecuda102build:ch2q /opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_java --build_shared_lib --use_cuda --cuda_version=10.2 --cuda_home=/usr/local/cuda-10.2 --cudnn_home=/usr/local/cuda-10.2
workingDirectory: $(Build.SourcesDirectory)
- task: Docker@2
displayName: logout
@ -256,7 +256,7 @@ jobs:
- task: CmdLine@2
inputs:
script: |
docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:chp /onnxruntime_src/tools/ci_build/github/linux/java_linux_final_test.sh -v $(OnnxRuntimeVersion) -r /build
docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecuda102build:ch2q /onnxruntime_src/tools/ci_build/github/linux/java_linux_final_test.sh -v $(OnnxRuntimeVersion) -r /build
workingDirectory: $(Build.BinariesDirectory)/final-jar
- task: Docker@2

View file

@ -28,7 +28,7 @@ jobs:
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \
-e NIGHTLY_BUILD \
-e BUILD_BUILDNUMBER \
onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:chp \
onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecuda102build:ch2q \
/opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py \
--build_dir /build --cmake_generator Ninja \
--config Debug Release \
@ -36,7 +36,7 @@ jobs:
--build_shared_lib \
--parallel \
--build_wheel \
--enable_onnx_tests --use_cuda --cuda_version=10.1 --cuda_home=/usr/local/cuda-10.1 --cudnn_home=/usr/local/cuda-10.1 \
--enable_onnx_tests --use_cuda --cuda_version=10.2 --cuda_home=/usr/local/cuda-10.2 --cudnn_home=/usr/local/cuda-10.2 \
--enable_pybind --build_java --build_nodejs \
--cmake_extra_defines PYTHON_INCLUDE_DIR=/opt/python/cp37-cp37m/include/python3.7m PYTHON_LIBRARY=/usr/lib64/librt.so CMAKE_CUDA_ARCHITECTURES=52
workingDirectory: $(Build.SourcesDirectory)

View file

@ -1,64 +0,0 @@
jobs:
- job: Linux_Build
timeoutInMinutes: 120
workspace:
clean: all
pool: Linux-GPU-CUDA10
steps:
- checkout: self
clean: true
submodules: recursive
- task: Docker@2
displayName: login
inputs:
containerRegistry: onnxruntimeregistry
command: login
addPipelineData: false
- task: CmdLine@2
inputs:
script: |
mkdir -p $HOME/.onnx
docker run --gpus all --rm \
--volume /data/onnx:/data/onnx:ro \
--volume $(Build.SourcesDirectory):/onnxruntime_src \
--volume $(Build.BinariesDirectory):/build \
--volume /data/models:/build/models:ro \
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
-e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \
-e NIGHTLY_BUILD \
-e BUILD_BUILDNUMBER \
onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecuda102build:ch2e \
/opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py \
--build_dir /build --cmake_generator Ninja \
--config Debug Release \
--skip_submodule_sync \
--build_shared_lib \
--parallel \
--build_wheel \
--use_openmp \
--enable_onnx_tests --use_cuda --cuda_version=10.2 --cuda_home=/usr/local/cuda-10.2 --cudnn_home=/usr/local/cuda-10.2 \
--use_mklml --enable_pybind --build_java --build_nodejs \
--cmake_extra_defines PYTHON_INCLUDE_DIR=/opt/python/cp37-cp37m/include/python3.7m PYTHON_LIBRARY=/usr/lib64/librt.so CMAKE_CUDA_ARCHITECTURES=52
workingDirectory: $(Build.SourcesDirectory)
- task: Docker@2
displayName: logout
inputs:
containerRegistry: onnxruntimeregistry
command: logout
addPipelineData: false
- task: PublishTestResults@2
displayName: 'Publish unit test results'
inputs:
testResultsFiles: '**/*.results.xml'
searchFolder: '$(Build.BinariesDirectory)'
testRunTitle: 'Unit Test Run'
condition: succeededOrFailed()
- template: templates/component-governance-component-detection-steps.yml
parameters:
condition: 'succeeded'
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
displayName: 'Clean Agent Directories'
condition: always()

View file

@ -27,7 +27,7 @@ jobs:
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
-e NIGHTLY_BUILD \
-e BUILD_BUILDNUMBER \
onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:chp \
onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecuda102build:ch2q \
/opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py \
--build_dir /build --cmake_generator Ninja \
--config Debug Release \
@ -35,7 +35,7 @@ jobs:
--build_shared_lib \
--parallel \
--build_wheel \
--enable_onnx_tests --use_cuda --cuda_version=10.1 --cuda_home=/usr/local/cuda-10.1 --cudnn_home=/usr/local/cuda-10.1 \
--enable_onnx_tests --use_cuda --cuda_version=10.2 --cuda_home=/usr/local/cuda-10.2 --cudnn_home=/usr/local/cuda-10.2 \
--enable_pybind --build_java --build_nodejs --enable_multi_device_test \
--cmake_extra_defines PYTHON_INCLUDE_DIR=/opt/python/cp37-cp37m/include/python3.7m PYTHON_LIBRARY=/usr/lib64/librt.so
workingDirectory: $(Build.SourcesDirectory)

View file

@ -89,8 +89,8 @@ jobs:
inputs:
script: |
mkdir -p $HOME/.onnx
docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:chp \
/bin/bash -c "/opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_cuda --cuda_version=10.1 --cuda_home=/usr/local/cuda-10.1 --cudnn_home=/usr/local/cuda-10.1 --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64"
docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecuda102build:ch2q \
/bin/bash -c "/opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_cuda --cuda_version=10.2 --cuda_home=/usr/local/cuda-10.2 --cudnn_home=/usr/local/cuda-10.2 --enable_onnx_tests && cd /build/Release && make install DESTDIR=/build/linux-x64"
- task: Docker@2
displayName: logout
inputs:

View file

@ -182,14 +182,14 @@ stages:
--volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
-e NIGHTLY_BUILD \
-e BUILD_BUILDNUMBER \
onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:chp \
onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecuda102build:ch2q \
$(python.manylinux.dir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py \
--build_dir /build --cmake_generator Ninja \
--config Release \
--skip_submodule_sync \
--parallel \
--build_wheel \
--enable_onnx_tests --use_cuda --cuda_version=10.1 --cuda_home=/usr/local/cuda-10.1 --cudnn_home=/usr/local/cuda-10.1 \
--enable_onnx_tests --use_cuda --cuda_version=10.2 --cuda_home=/usr/local/cuda-10.2 --cudnn_home=/usr/local/cuda-10.2 \
${{ parameters.build_py_parameters }} \
--cmake_extra_defines PYTHON_INCLUDE_DIR=$(python.manylinux.include.dir) PYTHON_LIBRARY=/usr/lib64/librt.so
workingDirectory: $(Build.SourcesDirectory)
@ -274,7 +274,7 @@ stages:
--enable_onnx_tests \
${{ parameters.build_py_parameters }} \
--cmake_extra_defines PYTHON_INCLUDE_DIR=$(python.manylinux.include.dir) PYTHON_LIBRARY=/usr/lib64/librt.so \
--use_cuda --cuda_version=10.1 --cuda_home=/usr/local/cuda-10.1 --cudnn_home=/usr/local/cuda-10.1
--use_cuda --cuda_version=10.2 --cuda_home=/usr/local/cuda-10.2 --cudnn_home=/usr/local/cuda-10.2
workingDirectory: $(Build.SourcesDirectory)
- task: CopyFiles@2

View file

@ -1,28 +0,0 @@
FROM quay.io/pypa/manylinux2010_x86_64:2020-04-04-f427f46
ARG PYTHON_VERSION=3.5
ADD scripts /tmp/scripts
RUN /tmp/scripts/install_manylinux2010.sh -p $PYTHON_VERSION && \
(source /opt/onnxruntime-python/bin/activate; /tmp/scripts/install_deps.sh -p $PYTHON_VERSION) && \
rm -rf /tmp/scripts # not useful at all except not to see the scripts
RUN echo "#!/bin/bash" > /opt/entrypoint.sh && \
echo "set -e" >> /opt/entrypoint.sh && \
echo "source /opt/onnxruntime-python/bin/activate" >> /opt/entrypoint.sh && \
echo "exec \"\$@\"" >> /opt/entrypoint.sh
RUN cat /opt/entrypoint.sh
RUN chmod +x /opt/entrypoint.sh
WORKDIR /root
ENV LD_LIBRARY_PATH /usr/local/openblas/lib:$LD_LIBRARY_PATH
ARG BUILD_UID=1000
ARG BUILD_USER=onnxruntimedev
WORKDIR /home/$BUILD_USER
# --disabled-password
RUN adduser --comment 'onnxruntime Build User' $BUILD_USER --uid $BUILD_UID
USER $BUILD_USER
ENTRYPOINT ["/opt/entrypoint.sh"]

View file

@ -1,93 +0,0 @@
#Copyright (c) 2019,2020 NVIDIA CORPORATION. All rights reserved.
#
#Redistribution and use in source and binary forms, with or without
#modification, are permitted provided that the following conditions
#are met:
# * Redistributions of source code must retain the above copyright
# notice, this list of conditions and the following disclaimer.
# * Redistributions in binary form must reproduce the above copyright
# notice, this list of conditions and the following disclaimer in the
# documentation and/or other materials provided with the distribution.
# * Neither the name of NVIDIA CORPORATION nor the names of its
# contributors may be used to endorse or promote products derived
# from this software without specific prior written permission.
#
#THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS ``AS IS'' AND ANY
#EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
#IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
#PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR
#CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL,
#EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO,
#PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR
#PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY
#OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT
#(INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE
#OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
FROM quay.io/pypa/manylinux2010_x86_64:latest
ADD manylinux /tmp/scripts
RUN cd /tmp/scripts && /tmp/scripts/install_centos.sh && /tmp/scripts/install_deps.sh && rm -rf /tmp/scripts
ENV NVIDIA_VISIBLE_DEVICES all
ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
#Below are copied from https://gitlab.com/nvidia/container-images/cuda/tree/master/dist/centos6
RUN NVIDIA_GPGKEY_SUM=d1be581509378368edeec8c1eb2958702feedf3bc3d17011adbf24efacce4ab5 && \
curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/rhel6/x86_64/7fa2af80.pub | sed '/^Version/d' > /etc/pki/rpm-gpg/RPM-GPG-KEY-NVIDIA && \
echo "$NVIDIA_GPGKEY_SUM /etc/pki/rpm-gpg/RPM-GPG-KEY-NVIDIA" | sha256sum -c -
COPY cuda_manylinux2010.repo /etc/yum.repos.d/cuda.repo
ENV CUDA_VERSION 10.1.243
ENV CUDA_PKG_VERSION 10-1-$CUDA_VERSION-1
RUN yum install -y \
cuda-cudart-$CUDA_PKG_VERSION \
cuda-compat-10-1 \
cuda-libraries-$CUDA_PKG_VERSION \
cuda-nvtx-$CUDA_PKG_VERSION \
libcublas10-10.2.1.243-1 \
cuda-nvml-dev-$CUDA_PKG_VERSION \
cuda-command-line-tools-$CUDA_PKG_VERSION \
cuda-libraries-dev-$CUDA_PKG_VERSION \
cuda-minimal-build-$CUDA_PKG_VERSION \
libcublas-devel-10.2.1.243-1 \
&& \
ln -s cuda-10.1 /usr/local/cuda && \
rpm -e --nodeps gcc gcc-c++ && \
rm -rf /var/cache/yum/*
# nvidia-docker 1.0
RUN echo "/usr/local/nvidia/lib" >> /etc/ld.so.conf.d/nvidia.conf && \
echo "/usr/local/nvidia/lib64" >> /etc/ld.so.conf.d/nvidia.conf
ENV PATH /usr/local/nvidia/bin:/usr/local/cuda/bin:/opt/rh/devtoolset-7/root/usr/bin:${PATH}
ENV LD_LIBRARY_PATH /usr/local/nvidia/lib:/usr/local/nvidia/lib64:${LD_LIBRARY_PATH}
# nvidia-container-runtime
ENV NVIDIA_VISIBLE_DEVICES all
ENV NVIDIA_DRIVER_CAPABILITIES compute,utility
ENV NVIDIA_REQUIRE_CUDA "cuda>=10.1 brand=tesla,driver>=384,driver<385 brand=tesla,driver>=396,driver<397 brand=tesla,driver>=410,driver<411"
ENV LIBRARY_PATH /usr/local/cuda/lib64/stubs
ENV CUDNN_VERSION 7.6.5.32
LABEL com.nvidia.cudnn.version="${CUDNN_VERSION}"
# cuDNN license: https://developer.nvidia.com/cudnn/license_agreement
RUN CUDNN_DOWNLOAD_SUM=7eaec8039a2c30ab0bc758d303588767693def6bf49b22485a2c00bf2e136cb3 && \
curl -fsSL http://developer.download.nvidia.com/compute/redist/cudnn/v7.6.5/cudnn-10.1-linux-x64-v7.6.5.32.tgz -O && \
echo "$CUDNN_DOWNLOAD_SUM cudnn-10.1-linux-x64-v7.6.5.32.tgz" | sha256sum -c - && \
tar --no-same-owner -xzf cudnn-10.1-linux-x64-v7.6.5.32.tgz -C /usr/local && \
rm cudnn-10.1-linux-x64-v7.6.5.32.tgz && \
ldconfig
ARG BUILD_UID=1000
ARG BUILD_USER=onnxruntimedev
RUN adduser --comment 'onnxruntime Build User' --uid $BUILD_UID $BUILD_USER
WORKDIR /home/$BUILD_USER
USER $BUILD_USER
ENV PATH /usr/local/gradle/bin:/usr/local/dotnet:$PATH