From b20daeda81d008eb0a76485ab233c1a1e4c4e762 Mon Sep 17 00:00:00 2001
From: Adrian Lizarraga <adlizarraga@microsoft.com>
Date: Tue, 21 Jun 2022 07:59:11 -0700
Subject: [PATCH] Update Linux Multi GPU TensorRT pipeline to TensorRT 8.4
 (#11923)

* Try manually installing trt8.4 in multi-gpu pipeline

* Remove stmts that clean up cmake, ctest. Update tensorrt repository name passed to get_docker_image.py

* Update trt and cudnn home

* Don't install trtexec cli tool.

* Increase job timeout

* Revert timeout change and use trt placeholder builder build option
---
 .../linux/docker/Dockerfile.ubuntu_tensorrt   | 19 +++++++++++++------
 tools/ci_build/github/linux/run_build.sh      |  5 +++--
 .../ci_build/github/linux/run_dockerbuild.sh  |  3 +--
 3 files changed, 17 insertions(+), 10 deletions(-)

diff --git a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_tensorrt b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_tensorrt
index b3d1d8c81b..e260a20b2d 100644
--- a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_tensorrt
+++ b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_tensorrt
@@ -1,15 +1,22 @@
-# Tag: nvcr.io/nvidia/tensorrt:21.12-py3
-# Label: com.nvidia.cuda.version: 11.5.0
-# Label: com.nvidia.cudnn.version: 8.3.1.22
+# Tag: nvidia/cuda:11.6.1-cudnn8-devel-ubuntu20.04
+# Label: com.nvidia.cuda.version: 11.6.1
+# Label: com.nvidia.cudnn.version: 8.4.0
 # Ubuntu 20.04
-FROM nvcr.io/nvidia/tensorrt:21.12-py3
+FROM nvidia/cuda:11.6.1-cudnn8-devel-ubuntu20.04
 
 ARG PYTHON_VERSION=3.8
 ARG DEBIAN_FRONTEND=noninteractive
 
 ADD scripts /tmp/scripts
-RUN /tmp/scripts/install_ubuntu.sh -p $PYTHON_VERSION && /tmp/scripts/install_os_deps.sh && /tmp/scripts/install_python_deps.sh -p $PYTHON_VERSION && rm -rf /tmp/scripts \
-    && rm /usr/local/bin/cmake && rm /usr/local/bin/ctest && rm -r /usr/local/share/cmake-3.14
+RUN /tmp/scripts/install_ubuntu.sh -p $PYTHON_VERSION && /tmp/scripts/install_os_deps.sh && /tmp/scripts/install_python_deps.sh -p $PYTHON_VERSION && rm -rf /tmp/scripts
+
+# Install TensorRT
+RUN v="8.4.1-1+cuda11.6" &&\
+    apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/7fa2af80.pub &&\
+    apt-get update &&\
+    sudo apt-get install -y libnvinfer8=${v} libnvonnxparsers8=${v} libnvparsers8=${v} libnvinfer-plugin8=${v} \
+        libnvinfer-dev=${v} libnvonnxparsers-dev=${v} libnvparsers-dev=${v} libnvinfer-plugin-dev=${v} \
+        python3-libnvinfer=${v}
 
 WORKDIR /root
 
diff --git a/tools/ci_build/github/linux/run_build.sh b/tools/ci_build/github/linux/run_build.sh
index ab54379901..2a4fee76db 100755
--- a/tools/ci_build/github/linux/run_build.sh
+++ b/tools/ci_build/github/linux/run_build.sh
@@ -60,9 +60,10 @@ else
         _CUDNN_VERSION=$(echo $CUDNN_VERSION | cut -d. -f1-2)
         python3 $SCRIPT_DIR/../../build.py --build_dir /build \
             --config Release $COMMON_BUILD_ARGS \
-            --use_tensorrt --tensorrt_home /workspace/tensorrt \
+            --tensorrt_placeholder_builder \
+            --use_tensorrt --tensorrt_home /usr/lib/x86_64-linux-gnu/ \
             --cuda_home /usr/local/cuda \
-            --cudnn_home /usr/local/cuda $BUILD_EXTR_PAR
+            --cudnn_home /usr/lib/x86_64-linux-gnu/ $BUILD_EXTR_PAR
     else #cpu and openvino
         python3 $SCRIPT_DIR/../../build.py --build_dir /build \
             --config Release $COMMON_BUILD_ARGS $BUILD_EXTR_PAR
diff --git a/tools/ci_build/github/linux/run_dockerbuild.sh b/tools/ci_build/github/linux/run_dockerbuild.sh
index 82353fc5af..d9e607f52c 100755
--- a/tools/ci_build/github/linux/run_dockerbuild.sh
+++ b/tools/ci_build/github/linux/run_dockerbuild.sh
@@ -96,8 +96,7 @@ elif [ $BUILD_DEVICE = "gpu" ]; then
             --docker-build-args="--build-arg BASEIMAGE=nvcr.io/nvidia/cuda:11.3.1-cudnn8-devel-${BUILD_OS} --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} --build-arg INSTALL_DEPS_EXTRA_ARGS=\"${INSTALL_DEPS_EXTRA_ARGS}\" --build-arg USE_CONDA=${USE_CONDA} --network=host" \
             --dockerfile Dockerfile.ubuntu_gpu_training --context .
 elif [[ $BUILD_DEVICE = "tensorrt"* ]]; then
-        # TensorRT container release 21.12
-        IMAGE="$BUILD_OS-cuda11.5-cudnn8.3-tensorrt8.2"
+        IMAGE="$BUILD_OS-cuda11.6-cudnn8.4-tensorrt8.4"
         DOCKER_FILE=Dockerfile.ubuntu_tensorrt
 
         $GET_DOCKER_IMAGE_CMD --repository "onnxruntime-$IMAGE" \