Fix Multi GPU TensorRT tests (#17269)

### Description * Integrate `trt_multi_gpu` test stage in ORT post merge CI (Win-2xA10 vm) * Deprecate Linux MultiGPU TRT CI (This vm will be deprecated soon) * Add multi gpu support to existing C# test cases * Deprecate unfunctional flag `--enable_multi_device_tests` ### Motivation and Context * Two contexts of replacing Linux MultiGPU TRT CI: * Flag `--enable_multi_device_tests` is not functional, which cannot detect issues like #17036 * The Linux-2xM60 VM of this CI pool is about to be deprecated 9/6/23. Need to enable this test in other dualGPU vm pool.
2026-07-13 18:08:13 +00:00 · 2023-08-25 20:30:45 -07:00 · 2023-08-25 20:30:45 -07:00 · 808215366d
commit 808215366d
parent c262879214
7 changed files with 50 additions and 90 deletions
--- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/InferenceTest.netcore.cs
+++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/InferenceTest.netcore.cs
@ -73,13 +73,21 @@ namespace Microsoft.ML.OnnxRuntime.Tests
        {
            string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "squeezenet.onnx");

+            string defaultDeviceId = "0";
+            string deviceIdFromEnv = System.Environment.GetEnvironmentVariable("OnnxruntimeTestGpuDeviceId");
+            if (!string.IsNullOrEmpty(deviceIdFromEnv) && int.TryParse(deviceIdFromEnv, out int deviceId) && deviceId >= 0)
+            {
+                defaultDeviceId = deviceIdFromEnv;
+                output.WriteLine($"Parsed ID: {deviceIdFromEnv}");
+            }
+
            using (var cleanUp = new DisposableListTest<IDisposable>())
            {
                var cudaProviderOptions = new OrtCUDAProviderOptions();
                cleanUp.Add(cudaProviderOptions);

                var providerOptionsDict = new Dictionary<string, string>();
-                providerOptionsDict["device_id"] = "0";
+                providerOptionsDict["device_id"] = defaultDeviceId;
                // 256MB
                providerOptionsDict["gpu_mem_limit"] = "268435456";
                providerOptionsDict["arena_extend_strategy"] = "kSameAsRequested";
@ -137,10 +145,18 @@ namespace Microsoft.ML.OnnxRuntime.Tests
        private void CanRunInferenceOnAModelWithTensorRT()
        {
            string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "squeezenet.onnx");
+            
+            int deviceId = 0;
+            string deviceIdStr = System.Environment.GetEnvironmentVariable("ONNXRUNTIME_TEST_GPU_DEVICE_ID");
+            if (!string.IsNullOrEmpty(deviceIdStr) && int.TryParse(deviceIdStr, out int parsedValue) && parsedValue >= 0)
+            {
+                deviceId = parsedValue;
+                output.WriteLine($"Parsed ID: {parsedValue}");
+            }

            using (var cleanUp = new DisposableListTest<IDisposable>())
            {
-                SessionOptions options = SessionOptions.MakeSessionOptionWithTensorrtProvider(0);
+                SessionOptions options = SessionOptions.MakeSessionOptionWithTensorrtProvider(deviceId);
                cleanUp.Add(options);

                var session = new InferenceSession(modelPath, options);
@ -172,6 +188,13 @@ namespace Microsoft.ML.OnnxRuntime.Tests
            string calTablePath = "squeezenet_calibration.flatbuffers";
            string enginePath = "./";
            string engineDecrptLibPath = "engine_decryp";
+            string defaultDeviceId = "0";
+            string deviceIdFromEnv = System.Environment.GetEnvironmentVariable("OnnxruntimeTestGpuDeviceId");
+            if (!string.IsNullOrEmpty(deviceIdFromEnv) && int.TryParse(deviceIdFromEnv, out int deviceId) && deviceId >= 0)
+            {
+                defaultDeviceId = deviceIdFromEnv;
+                output.WriteLine($"Parsed ID: {deviceIdFromEnv}");
+            }

            using (var cleanUp = new DisposableListTest<IDisposable>())
            {
@ -179,7 +202,7 @@ namespace Microsoft.ML.OnnxRuntime.Tests
                cleanUp.Add(trtProviderOptions);

                var providerOptionsDict = new Dictionary<string, string>();
-                providerOptionsDict["device_id"] = "0";
+                providerOptionsDict["device_id"] = defaultDeviceId;
                providerOptionsDict["trt_fp16_enable"] = "1";
                providerOptionsDict["trt_int8_enable"] = "1";
                providerOptionsDict["trt_int8_calibration_table_name"] = calTablePath;
@ -195,7 +218,7 @@ namespace Microsoft.ML.OnnxRuntime.Tests
                // test provider options configuration
                string value;
                value = resultProviderOptionsDict["device_id"];
-                Assert.Equal("0", value);
+                Assert.Equal(defaultDeviceId, value);
                value = resultProviderOptionsDict["trt_fp16_enable"];
                Assert.Equal("1", value);
                value = resultProviderOptionsDict["trt_int8_enable"];
--- a/tools/ci_build/build.py
+++ b/tools/ci_build/build.py
@ -550,11 +550,6 @@ def parse_arguments():
        default=None,
        help="Specify the generator that CMake invokes.",
    )
-    parser.add_argument(
-        "--enable_multi_device_test",
-        action="store_true",
-        help="Test with multi-device. Mostly used for multi-device GPU",
-    )
    parser.add_argument("--use_dml", action="store_true", help="Build with DirectML.")
    parser.add_argument(
        "--dml_path",
--- a/tools/ci_build/github/azure-pipelines/linux-multi-gpu-tensorrt-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/linux-multi-gpu-tensorrt-ci-pipeline.yml
@ -1,37 +0,0 @@
-trigger:
-  branches:
-    include:
-    - main
-    - rel-*
-  paths:
-    exclude:
-    - docs/**
-    - README.md
-    - CONTRIBUTING.md
-    - BUILD.md
-    - 'js/web'
-    - 'js/node'
-    - 'onnxruntime/core/providers/js'
-pr:
-  branches:
-    include:
-    - main
-    - rel-*
-  paths:
-    exclude:
-    - docs/**
-    - README.md
-    - CONTRIBUTING.md
-    - BUILD.md
-    - 'js/web'
-    - 'js/node'
-    - 'onnxruntime/core/providers/js'
-jobs:
- template: templates/linux-ci.yml
-  parameters:
-    AgentPool : 'Linux-Multi-GPU'
-    JobName: 'Linux_CI_Multi_GPU_TensorRT_Dev'
-    # The latest TensorRT container only supports ubuntu20.04 and python 3.8
-    RunDockerBuildArgs: '-o ubuntu20.04 -d tensorrt -x "--enable_multi_device_test"'
-    DoNugetPack:  'false'
-    ArtifactName: 'drop-linux'
--- a/tools/ci_build/github/azure-pipelines/post-merge-jobs.yml
+++ b/tools/ci_build/github/azure-pipelines/post-merge-jobs.yml
@ -80,6 +80,27 @@ stages:
        MachinePool: onnxruntime-Win2022-GPU-MultiA10
        OnnxruntimeTestGpuDeviceId: 1

+- ${{ if or(startsWith(variables['System.CollectionUri'], 'https://dev.azure.com/aiinfra/'),startsWith(variables['System.CollectionUri'], 'https://aiinfra.visualstudio.com/')) }}:
+  # The settings below is the same as Windows GPU CI pipeline's CUDA job except here we set OnnxruntimeTestGpuDeviceId to 1
+  - stage: trt_multi_gpu
+    dependsOn: []
+    jobs:
+    - template: templates/jobs/win-ci-vs-2022-job.yml
+      parameters:
+        BuildConfig: 'RelWithDebInfo'
+        EnvSetupScript: setup_env_trt.bat
+        buildArch: x64
+        additionalBuildFlags: --enable_pybind --build_java --build_nodejs --use_cuda --cuda_home="$(Agent.TempDirectory)\v11.8" --enable_cuda_profiling --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.6.1.6.Windows10.x86_64.cuda-11.8" --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=86
+        msbuildPlatform: x64
+        isX86: false
+        job_name_suffix: x64_RelWithDebInfo
+        RunOnnxRuntimeTests: true
+        RunStaticCodeAnalysis: false
+        ORT_EP_NAME: TRT
+        WITH_CACHE: true
+        MachinePool: onnxruntime-Win2022-GPU-MultiA10
+        OnnxruntimeTestGpuDeviceId: 1
+
 - stage: Mimalloc
  dependsOn: [ ]
  jobs:
--- a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_tensorrt
+++ b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_tensorrt
@ -1,37 +0,0 @@
-# Tag: nvidia/cuda:11.8.0-cudnn8-devel-ubuntu20.04
-# Label: com.nvidia.cuda.version: 11.8.0
-# Label: com.nvidia.cudnn.version: 8.7.0
-# Ubuntu 20.04
-FROM nvidia/cuda:11.8.0-cudnn8-devel-ubuntu20.04@sha256:b754c43fe9d62e88862d168c4ab9282618a376dbc54871467870366cacfa456e
-
-ARG PYTHON_VERSION=3.8
-ARG DEBIAN_FRONTEND=noninteractive
-
-ADD scripts /tmp/scripts
-RUN /tmp/scripts/install_ubuntu.sh -p $PYTHON_VERSION && /tmp/scripts/install_os_deps.sh && /tmp/scripts/install_python_deps.sh -p $PYTHON_VERSION && rm -rf /tmp/scripts
-
-# Install TensorRT
-RUN v="8.5.1-1+cuda11.8" &&\
-    apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/7fa2af80.pub &&\
-    apt-get update &&\
-    sudo apt-get install -y libnvinfer8=${v} libnvonnxparsers8=${v} libnvparsers8=${v} libnvinfer-plugin8=${v} \
-        libnvinfer-dev=${v} libnvonnxparsers-dev=${v} libnvparsers-dev=${v} libnvinfer-plugin-dev=${v} \
-        python3-libnvinfer=${v} libnvinfer-samples=${v}
-
-WORKDIR /root
-
-# Allow configure to pick up GDK and CuDNN where it expects it.
-# (Note: $CUDNN_VERSION is defined by NVidia's base image)
-RUN _CUDNN_VERSION=$(echo $CUDNN_VERSION | cut -d. -f1-2) && \
-    mkdir -p /usr/local/cudnn-$_CUDNN_VERSION/cuda/include && \
-    ln -s /usr/include/cudnn.h /usr/local/cudnn-$_CUDNN_VERSION/cuda/include/cudnn.h && \
-    mkdir -p /usr/local/cudnn-$_CUDNN_VERSION/cuda/lib64 && \
-    ln -s /etc/alternatives/libcudnn_so /usr/local/cudnn-$_CUDNN_VERSION/cuda/lib64/libcudnn.so && \
-    ln -s /usr/local/cudnn{-$_CUDNN_VERSION,}
-
-ARG BUILD_USER=onnxruntimedev
-ARG BUILD_UID=1000
-WORKDIR /home/$BUILD_USER
-RUN adduser --gecos 'onnxruntime Build User' --disabled-password $BUILD_USER --uid $BUILD_UID
-USER $BUILD_USER
-ENV CUDA_MODULE_LOADING "LAZY"
--- a/tools/ci_build/github/linux/run_dockerbuild.sh
+++ b/tools/ci_build/github/linux/run_dockerbuild.sh
@ -95,13 +95,6 @@ elif [ $BUILD_DEVICE = "gpu" ]; then
        $GET_DOCKER_IMAGE_CMD --repository "onnxruntime-$IMAGE" \
            --docker-build-args="--build-arg BASEIMAGE=nvcr.io/nvidia/cuda:11.8.0-cudnn8-devel-${BUILD_OS} --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} --build-arg INSTALL_DEPS_EXTRA_ARGS=\"${INSTALL_DEPS_EXTRA_ARGS}\" --build-arg USE_CONDA=${USE_CONDA} --network=host" \
            --dockerfile Dockerfile.ubuntu_gpu_training --context .
-elif [[ $BUILD_DEVICE = "tensorrt"* ]]; then
-        IMAGE="$BUILD_OS-cuda11.8-cudnn8.7-tensorrt8.5"
-        DOCKER_FILE=Dockerfile.ubuntu_tensorrt
-
-        $GET_DOCKER_IMAGE_CMD --repository "onnxruntime-$IMAGE" \
-            --docker-build-args="--build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER}" \
-            --dockerfile $DOCKER_FILE --context .
 elif [[ $BUILD_DEVICE = "openvino"* ]]; then
        BUILD_ARGS="--build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=3.8"
        IMAGE="$BUILD_OS-openvino"
--- a/tools/ci_build/github/windows/setup_env_trt.bat
+++ b/tools/ci_build/github/windows/setup_env_trt.bat
@ -6,4 +6,6 @@ if exist PATH=%AGENT_TEMPDIRECTORY%\v11.8\ {
 } else {
    set PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8\bin;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8\extras\CUPTI\lib64;%PATH%
 }
+set PATH=C:\local\TensorRT-8.6.1.6.Windows10.x86_64.cuda-11.8\lib;%PATH%
 set GRADLE_OPTS=-Dorg.gradle.daemon=false
+set CUDA_MODULE_LOADING=LAZY