Upgrade gpu build to CUDA 10 + cudnn 7.3 (#112)

* Upgrade gpu build to CUDA 10 + cudnn 7.3 * update the yaml file for python package building * switch to the cuda9.1 docker file if the CUDA_VER is cuda9.1-cudnn7.1
2026-06-20 02:07:56 +00:00 · 2018-12-05 17:49:16 -08:00 · 2018-12-05 17:49:16 -08:00 · a68f5ccfd9
commit a68f5ccfd9
parent 4801e67104
8 changed files with 117 additions and 14 deletions
--- a/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml
+++ b/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml
@ -33,7 +33,7 @@ jobs:
      condition: always()

 - job: Ubuntu1604_py_GPU_Wheels
-  pool: Linux-GPU
+  pool: Linux-GPU-CUDA10
  strategy:
    matrix:
      Python35:
@ -47,7 +47,7 @@ jobs:
      displayName: 'Run build script'
      inputs:
        scriptPath: 'tools/ci_build/github/linux/run_dockerbuild.sh'
-        args: '-c Release -o ubuntu16.04 -d gpu -r $(Build.BinariesDirectory) -p $(python.version) -x "--build_wheel --use_mkldnn"'
+        args: '-c Release -o ubuntu16.04 -d gpu -c cuda9.1-cudnn7.1 -r $(Build.BinariesDirectory) -p $(python.version) -x "--build_wheel --use_mkldnn"'

    - task: CopyFiles@2
      displayName: 'Copy Python Wheel to:  $(Build.ArtifactStagingDirectory)'
@ -118,7 +118,7 @@ jobs:
      condition: always()

 - job:  Windows_py_GPU_Wheels
-  pool: Win-GPU
+  pool: Win-GPU-CUDA10
  variables:
    buildDirectory: '$(Build.SourcesDirectory)\build'
    CUDA_VERSION: '9.1'
--- a/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline-cuda9.yml
+++ b/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline-cuda9.yml
@ -0,0 +1,12 @@
+jobs:
+- job: Linux_CI_GPU_Dev
+  pool: Linux-GPU-CUDA10
+  steps:
+    - script: 'tools/ci_build/github/linux/run_dockerbuild.sh -o ubuntu16.04 -d gpu -c cuda9.1-cudnn7.1 -r $(Build.BinariesDirectory)'
+      displayName: 'Command Line Script'
+      env:
+        AZURE_BLOB_KEY: $(onnxruntime-storage-key)
+
+    - script: 'sudo rm -rf $(Agent.BuildDirectory)'
+      displayName: 'Clean build folders/files'
+      condition: always()
--- a/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline-cuda9.yml
+++ b/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline-cuda9.yml
@ -0,0 +1,41 @@
+jobs:
+- job: Windows_CI_GPU_Dev
+  pool: Win-GPU-CUDA10
+  variables:
+    CUDA_VERSION: '9.1'
+  steps:
+    - task: PowerShell@1
+      displayName: 'Set CUDA path'
+      inputs:
+        scriptName: 'tools/ci_build/github/windows/set_cuda_path.ps1'
+        arguments: '-CudaMsbuildPath C:\local\cudaMsbuildIntegration-9.1.85-windows10-x64-0 -CudaVersion $(CUDA_VERSION)'
+    - task: BatchScript@1
+      displayName: 'Setup VS2017 env vars'
+      inputs:
+        filename: 'C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvars64.bat'
+        arguments: 'amd64 -vcvars_ver=14.11'
+        modifyEnvironment: true
+    - task: CmdLine@1
+      displayName: 'Get ONNX testdata'
+      inputs:
+        filename: azcopy
+        arguments: ' /S /Source:https://onnxruntimetestdata.blob.core.windows.net/onnx-model-zoo-20181018 /Dest:$(Build.SourcesDirectory)\build\Windows\Debug\models /SourceKey:%AZURE_BLOB_KEY%'
+      env:
+        AZURE_BLOB_KEY: $(onnxruntime-storage-key)
+    - task: BatchScript@1
+      inputs:
+        filename: build.bat
+        arguments: ' --use_cuda --cuda_home="C:\local\cuda-9.1.85-windows10-x64-0" --cudnn_home="C:\local\cudnn-9.1-windows10-x64-v7.1\cuda"'
+        workingFolder: "$(Build.SourcesDirectory)"
+    - task: PowerShell@1
+      displayName: 'Clean up CUDA props files'
+      inputs:
+        scriptName: 'tools/ci_build/github/windows/clean_up_cuda_prop_files.ps1'
+        arguments: '-CudaVersion $(CUDA_VERSION)'
+    - task: CmdLine@1
+      displayName: 'Clean build folders/files'
+      inputs:
+        filename: rd
+        arguments: '/s /q $(Agent.BuildDirectory)'
+      continueOnError: true
+      condition: always()
--- a/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml
@ -2,13 +2,13 @@ jobs:
 - job: Windows_CI_GPU_Dev
  pool: Win-GPU-CUDA10
  variables:
-    CUDA_VERSION: '9.1'
+    CUDA_VERSION: '10.0'
  steps:
    - task: PowerShell@1
      displayName: 'Set CUDA path'
      inputs:
        scriptName: 'tools/ci_build/github/windows/set_cuda_path.ps1'
-        arguments: '-CudaMsbuildPath C:\local\cudaMsbuildIntegration-9.1.85-windows10-x64-0 -CudaVersion $(CUDA_VERSION)'
+        arguments: '-CudaMsbuildPath C:\local\cudaMsbuildIntegration-10.0.130-win10 -CudaVersion $(CUDA_VERSION)'
    - task: BatchScript@1
      displayName: 'Setup VS2017 env vars'
      inputs:
@ -25,7 +25,7 @@ jobs:
    - task: BatchScript@1
      inputs:
        filename: build.bat
-        arguments: ' --use_cuda --cuda_home="C:\local\cuda-9.1.85-windows10-x64-0" --cudnn_home="C:\local\cudnn-9.1-windows10-x64-v7.1\cuda"'
+        arguments: ' --enable_onnx_tests --use_cuda --cuda_home="C:\local\cuda_10.0.130_win10" --cudnn_home="C:\local\cudnn-10.0-windows10-x64-v7.3.1.20\cuda"'
        workingFolder: "$(Build.SourcesDirectory)"
    - task: PowerShell@1
      displayName: 'Clean up CUDA props files'
--- a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_gpu
+++ b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_gpu
@ -1,9 +1,8 @@
-# Tag: nvidia/cuda:9.1-cudnn7-devel-ubuntu16.04
-# Label: com.nvidia.cuda.version: 9.1.85
-# Label: com.nvidia.cudnn.version: 7.1.2.21
-# Label: com.nvidia.nccl.version: 2.2.12
+# Tag: nvidia/cuda:10.0-cudnn7-devel-ubuntu16.04
+# Label: com.nvidia.cuda.version: 10.0.130
+# Label: com.nvidia.cudnn.version: 7.3.1.20
 # Ubuntu 16.04.5
-FROM nvidia/cuda:9.1-cudnn7-devel-ubuntu16.04
+FROM nvidia/cuda@sha256:362e4e25aa46a18dfa834360140e91b61cdb0a3a2796c8e09dadb268b9de3f6b

 ARG PYTHON_VERSION=3.5

--- a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_gpu_cuda9
+++ b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_gpu_cuda9
@ -0,0 +1,43 @@
+# Tag: nvidia/cuda:9.1-cudnn7-devel-ubuntu16.04
+# Label: com.nvidia.cuda.version: 9.1.85
+# Label: com.nvidia.cudnn.version: 7.1.2.21
+# Ubuntu 16.04.5
+FROM nvidia/cuda@sha256:e48777124a0217001be8533123fcb8cc12ace38a4add2774b34295e611c99f10
+
+ARG PYTHON_VERSION=3.5
+
+ADD scripts /tmp/scripts
+ENV PATH="/opt/cmake/bin:${PATH}"
+RUN /tmp/scripts/install_ubuntu.sh -p ${PYTHON_VERSION} && /tmp/scripts/install_deps.sh && rm -rf /tmp/scripts
+
+WORKDIR /root
+
+# Allow configure to pick up GDK and CuDNN where it expects it.
+# (Note: $CUDNN_VERSION is defined by NVidia's base image)
+RUN _CUDNN_VERSION=$(echo $CUDNN_VERSION | cut -d. -f1-2) && \
+    mkdir -p /usr/local/cudnn-$_CUDNN_VERSION/cuda/include && \
+    ln -s /usr/include/cudnn.h /usr/local/cudnn-$_CUDNN_VERSION/cuda/include/cudnn.h && \
+    mkdir -p /usr/local/cudnn-$_CUDNN_VERSION/cuda/lib64 && \
+    ln -s /etc/alternatives/libcudnn_so /usr/local/cudnn-$_CUDNN_VERSION/cuda/lib64/libcudnn.so && \
+    ln -s /usr/local/cudnn{-$_CUDNN_VERSION,}
+
+# Build and Install LLVM
+ARG LLVM_VERSION=6.0.1
+RUN cd /tmp && \
+    wget --no-verbose http://releases.llvm.org/$LLVM_VERSION/llvm-$LLVM_VERSION.src.tar.xz && \
+    xz -d llvm-$LLVM_VERSION.src.tar.xz && \
+    tar xvf llvm-$LLVM_VERSION.src.tar && \
+    cd llvm-$LLVM_VERSION.src && \
+    mkdir -p build && \
+    cd build && \
+    cmake .. -DCMAKE_BUILD_TYPE=Release && \
+    cmake --build . -- -j$(nproc) && \
+    cmake -DCMAKE_INSTALL_PREFIX=/usr/local/llvm-$LLVM_VERSION -DBUILD_TYPE=Release -P cmake_install.cmake && \
+    cd /tmp && \
+    rm -rf llvm*
+
+ENV LD_LIBRARY_PATH /usr/local/openblas/lib:$LD_LIBRARY_PATH
+
+ARG BUILD_USER=onnxruntimedev
+WORKDIR /home/$BUILD_USER
+
--- a/tools/ci_build/github/linux/run_build.sh
+++ b/tools/ci_build/github/linux/run_build.sh
@ -25,13 +25,14 @@ else
 fi

 if [ $BUILD_DEVICE = "gpu" ]; then
+    _CUDNN_VERSION=$(echo $CUDNN_VERSION | cut -d. -f1-2)
    python3 $SCRIPT_DIR/../../build.py --build_dir /home/onnxruntimedev \
        --config Debug Release \
        --skip_submodule_sync \
        --parallel --build_shared_lib \
        --use_cuda \
        --cuda_home /usr/local/cuda \
-        --cudnn_home /usr/local/cudnn-7.1/cuda --build_shared_lib $BUILD_EXTR_PAR
+        --cudnn_home /usr/local/cudnn-$_CUDNN_VERSION/cuda --build_shared_lib $BUILD_EXTR_PAR
    /home/onnxruntimedev/Release/onnx_test_runner -e cuda /data/onnx
 else
    python3 $SCRIPT_DIR/../../build.py --build_dir /home/onnxruntimedev \
--- a/tools/ci_build/github/linux/run_dockerbuild.sh
+++ b/tools/ci_build/github/linux/run_dockerbuild.sh
@ -3,6 +3,7 @@ set -e -o -x

 SCRIPT_DIR="$( dirname "${BASH_SOURCE[0]}" )"
 SOURCE_ROOT=$(realpath $SCRIPT_DIR/../../../../)
+CUDA_VER=cuda10.0-cudnn7.3

 while getopts c:o:d:r:p:x: parameter_Option
 do case "${parameter_Option}"
@ -16,6 +17,8 @@ r) BUILD_DIR=${OPTARG};;
 p) PYTHON_VER=${OPTARG};;
 # "--build_wheel --use_openblas"
 x) BUILD_EXTR_PAR=${OPTARG};;
+# "cuda10.0-cudnn7.3, cuda9.1-cudnn7.1"
+c) CUDA_VER=${OPTARG};;
 esac
 done

@ -25,8 +28,12 @@ echo "bo=$BUILD_OS bd=$BUILD_DEVICE bdir=$BUILD_DIR pv=$PYTHON_VER bex=$BUILD_EX

 cd $SCRIPT_DIR/docker
 if [ $BUILD_DEVICE = "gpu" ]; then
-    IMAGE="ubuntu16.04-cuda9.1-cudnn7.1"
-    docker build -t "onnxruntime-$IMAGE" --build-arg PYTHON_VERSION=${PYTHON_VER} -f Dockerfile.ubuntu_gpu .
+    IMAGE="ubuntu16.04-$CUDA_VER"
+    DOCKER_FILE=Dockerfile.ubuntu_gpu
+    if [ $CUDA_VER = "cuda9.1-cudnn7.1" ]; then
+    DOCKER_FILE=Dockerfile.ubuntu_gpu_cuda9
+    fi
+    docker build -t "onnxruntime-$IMAGE" --build-arg PYTHON_VERSION=${PYTHON_VER} -f $DOCKER_FILE .
 else
    IMAGE="ubuntu16.04"
    docker build -t "onnxruntime-$IMAGE" --build-arg OS_VERSION=16.04 --build-arg PYTHON_VERSION=${PYTHON_VER} -f Dockerfile.ubuntu .