From 808215366d1aedb0567994c2f11acb2b9b6cbc24 Mon Sep 17 00:00:00 2001 From: Yifan Li <109183385+yf711@users.noreply.github.com> Date: Fri, 25 Aug 2023 20:30:45 -0700 Subject: [PATCH] Fix Multi GPU TensorRT tests (#17269) ### Description * Integrate `trt_multi_gpu` test stage in ORT post merge CI (Win-2xA10 vm) * Deprecate Linux MultiGPU TRT CI (This vm will be deprecated soon) * Add multi gpu support to existing C# test cases * Deprecate unfunctional flag `--enable_multi_device_tests` ### Motivation and Context * Two contexts of replacing Linux MultiGPU TRT CI: * Flag `--enable_multi_device_tests` is not functional, which cannot detect issues like #17036 * The Linux-2xM60 VM of this CI pool is about to be deprecated 9/6/23. Need to enable this test in other dualGPU vm pool. --- .../InferenceTest.netcore.cs | 31 ++++++++++++++-- tools/ci_build/build.py | 5 --- .../linux-multi-gpu-tensorrt-ci-pipeline.yml | 37 ------------------- .../azure-pipelines/post-merge-jobs.yml | 21 +++++++++++ .../linux/docker/Dockerfile.ubuntu_tensorrt | 37 ------------------- .../ci_build/github/linux/run_dockerbuild.sh | 7 ---- .../ci_build/github/windows/setup_env_trt.bat | 2 + 7 files changed, 50 insertions(+), 90 deletions(-) delete mode 100644 tools/ci_build/github/azure-pipelines/linux-multi-gpu-tensorrt-ci-pipeline.yml delete mode 100644 tools/ci_build/github/linux/docker/Dockerfile.ubuntu_tensorrt diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/InferenceTest.netcore.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/InferenceTest.netcore.cs index 13417c32cd..715aed7e1d 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/InferenceTest.netcore.cs +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.NetCoreApp/InferenceTest.netcore.cs @@ -73,13 +73,21 @@ namespace Microsoft.ML.OnnxRuntime.Tests { string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "squeezenet.onnx"); + string defaultDeviceId = "0"; + string deviceIdFromEnv = System.Environment.GetEnvironmentVariable("OnnxruntimeTestGpuDeviceId"); + if (!string.IsNullOrEmpty(deviceIdFromEnv) && int.TryParse(deviceIdFromEnv, out int deviceId) && deviceId >= 0) + { + defaultDeviceId = deviceIdFromEnv; + output.WriteLine($"Parsed ID: {deviceIdFromEnv}"); + } + using (var cleanUp = new DisposableListTest()) { var cudaProviderOptions = new OrtCUDAProviderOptions(); cleanUp.Add(cudaProviderOptions); var providerOptionsDict = new Dictionary(); - providerOptionsDict["device_id"] = "0"; + providerOptionsDict["device_id"] = defaultDeviceId; // 256MB providerOptionsDict["gpu_mem_limit"] = "268435456"; providerOptionsDict["arena_extend_strategy"] = "kSameAsRequested"; @@ -137,10 +145,18 @@ namespace Microsoft.ML.OnnxRuntime.Tests private void CanRunInferenceOnAModelWithTensorRT() { string modelPath = Path.Combine(Directory.GetCurrentDirectory(), "squeezenet.onnx"); + + int deviceId = 0; + string deviceIdStr = System.Environment.GetEnvironmentVariable("ONNXRUNTIME_TEST_GPU_DEVICE_ID"); + if (!string.IsNullOrEmpty(deviceIdStr) && int.TryParse(deviceIdStr, out int parsedValue) && parsedValue >= 0) + { + deviceId = parsedValue; + output.WriteLine($"Parsed ID: {parsedValue}"); + } using (var cleanUp = new DisposableListTest()) { - SessionOptions options = SessionOptions.MakeSessionOptionWithTensorrtProvider(0); + SessionOptions options = SessionOptions.MakeSessionOptionWithTensorrtProvider(deviceId); cleanUp.Add(options); var session = new InferenceSession(modelPath, options); @@ -172,6 +188,13 @@ namespace Microsoft.ML.OnnxRuntime.Tests string calTablePath = "squeezenet_calibration.flatbuffers"; string enginePath = "./"; string engineDecrptLibPath = "engine_decryp"; + string defaultDeviceId = "0"; + string deviceIdFromEnv = System.Environment.GetEnvironmentVariable("OnnxruntimeTestGpuDeviceId"); + if (!string.IsNullOrEmpty(deviceIdFromEnv) && int.TryParse(deviceIdFromEnv, out int deviceId) && deviceId >= 0) + { + defaultDeviceId = deviceIdFromEnv; + output.WriteLine($"Parsed ID: {deviceIdFromEnv}"); + } using (var cleanUp = new DisposableListTest()) { @@ -179,7 +202,7 @@ namespace Microsoft.ML.OnnxRuntime.Tests cleanUp.Add(trtProviderOptions); var providerOptionsDict = new Dictionary(); - providerOptionsDict["device_id"] = "0"; + providerOptionsDict["device_id"] = defaultDeviceId; providerOptionsDict["trt_fp16_enable"] = "1"; providerOptionsDict["trt_int8_enable"] = "1"; providerOptionsDict["trt_int8_calibration_table_name"] = calTablePath; @@ -195,7 +218,7 @@ namespace Microsoft.ML.OnnxRuntime.Tests // test provider options configuration string value; value = resultProviderOptionsDict["device_id"]; - Assert.Equal("0", value); + Assert.Equal(defaultDeviceId, value); value = resultProviderOptionsDict["trt_fp16_enable"]; Assert.Equal("1", value); value = resultProviderOptionsDict["trt_int8_enable"]; diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 64dae354a9..2951362f65 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -550,11 +550,6 @@ def parse_arguments(): default=None, help="Specify the generator that CMake invokes.", ) - parser.add_argument( - "--enable_multi_device_test", - action="store_true", - help="Test with multi-device. Mostly used for multi-device GPU", - ) parser.add_argument("--use_dml", action="store_true", help="Build with DirectML.") parser.add_argument( "--dml_path", diff --git a/tools/ci_build/github/azure-pipelines/linux-multi-gpu-tensorrt-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-multi-gpu-tensorrt-ci-pipeline.yml deleted file mode 100644 index c1f1c39c85..0000000000 --- a/tools/ci_build/github/azure-pipelines/linux-multi-gpu-tensorrt-ci-pipeline.yml +++ /dev/null @@ -1,37 +0,0 @@ -trigger: - branches: - include: - - main - - rel-* - paths: - exclude: - - docs/** - - README.md - - CONTRIBUTING.md - - BUILD.md - - 'js/web' - - 'js/node' - - 'onnxruntime/core/providers/js' -pr: - branches: - include: - - main - - rel-* - paths: - exclude: - - docs/** - - README.md - - CONTRIBUTING.md - - BUILD.md - - 'js/web' - - 'js/node' - - 'onnxruntime/core/providers/js' -jobs: -- template: templates/linux-ci.yml - parameters: - AgentPool : 'Linux-Multi-GPU' - JobName: 'Linux_CI_Multi_GPU_TensorRT_Dev' - # The latest TensorRT container only supports ubuntu20.04 and python 3.8 - RunDockerBuildArgs: '-o ubuntu20.04 -d tensorrt -x "--enable_multi_device_test"' - DoNugetPack: 'false' - ArtifactName: 'drop-linux' diff --git a/tools/ci_build/github/azure-pipelines/post-merge-jobs.yml b/tools/ci_build/github/azure-pipelines/post-merge-jobs.yml index 872812a6ae..113b24f757 100644 --- a/tools/ci_build/github/azure-pipelines/post-merge-jobs.yml +++ b/tools/ci_build/github/azure-pipelines/post-merge-jobs.yml @@ -80,6 +80,27 @@ stages: MachinePool: onnxruntime-Win2022-GPU-MultiA10 OnnxruntimeTestGpuDeviceId: 1 +- ${{ if or(startsWith(variables['System.CollectionUri'], 'https://dev.azure.com/aiinfra/'),startsWith(variables['System.CollectionUri'], 'https://aiinfra.visualstudio.com/')) }}: + # The settings below is the same as Windows GPU CI pipeline's CUDA job except here we set OnnxruntimeTestGpuDeviceId to 1 + - stage: trt_multi_gpu + dependsOn: [] + jobs: + - template: templates/jobs/win-ci-vs-2022-job.yml + parameters: + BuildConfig: 'RelWithDebInfo' + EnvSetupScript: setup_env_trt.bat + buildArch: x64 + additionalBuildFlags: --enable_pybind --build_java --build_nodejs --use_cuda --cuda_home="$(Agent.TempDirectory)\v11.8" --enable_cuda_profiling --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.6.1.6.Windows10.x86_64.cuda-11.8" --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=86 + msbuildPlatform: x64 + isX86: false + job_name_suffix: x64_RelWithDebInfo + RunOnnxRuntimeTests: true + RunStaticCodeAnalysis: false + ORT_EP_NAME: TRT + WITH_CACHE: true + MachinePool: onnxruntime-Win2022-GPU-MultiA10 + OnnxruntimeTestGpuDeviceId: 1 + - stage: Mimalloc dependsOn: [ ] jobs: diff --git a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_tensorrt b/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_tensorrt deleted file mode 100644 index 7ab781925b..0000000000 --- a/tools/ci_build/github/linux/docker/Dockerfile.ubuntu_tensorrt +++ /dev/null @@ -1,37 +0,0 @@ -# Tag: nvidia/cuda:11.8.0-cudnn8-devel-ubuntu20.04 -# Label: com.nvidia.cuda.version: 11.8.0 -# Label: com.nvidia.cudnn.version: 8.7.0 -# Ubuntu 20.04 -FROM nvidia/cuda:11.8.0-cudnn8-devel-ubuntu20.04@sha256:b754c43fe9d62e88862d168c4ab9282618a376dbc54871467870366cacfa456e - -ARG PYTHON_VERSION=3.8 -ARG DEBIAN_FRONTEND=noninteractive - -ADD scripts /tmp/scripts -RUN /tmp/scripts/install_ubuntu.sh -p $PYTHON_VERSION && /tmp/scripts/install_os_deps.sh && /tmp/scripts/install_python_deps.sh -p $PYTHON_VERSION && rm -rf /tmp/scripts - -# Install TensorRT -RUN v="8.5.1-1+cuda11.8" &&\ - apt-key adv --fetch-keys https://developer.download.nvidia.com/compute/cuda/repos/ubuntu2004/x86_64/7fa2af80.pub &&\ - apt-get update &&\ - sudo apt-get install -y libnvinfer8=${v} libnvonnxparsers8=${v} libnvparsers8=${v} libnvinfer-plugin8=${v} \ - libnvinfer-dev=${v} libnvonnxparsers-dev=${v} libnvparsers-dev=${v} libnvinfer-plugin-dev=${v} \ - python3-libnvinfer=${v} libnvinfer-samples=${v} - -WORKDIR /root - -# Allow configure to pick up GDK and CuDNN where it expects it. -# (Note: $CUDNN_VERSION is defined by NVidia's base image) -RUN _CUDNN_VERSION=$(echo $CUDNN_VERSION | cut -d. -f1-2) && \ - mkdir -p /usr/local/cudnn-$_CUDNN_VERSION/cuda/include && \ - ln -s /usr/include/cudnn.h /usr/local/cudnn-$_CUDNN_VERSION/cuda/include/cudnn.h && \ - mkdir -p /usr/local/cudnn-$_CUDNN_VERSION/cuda/lib64 && \ - ln -s /etc/alternatives/libcudnn_so /usr/local/cudnn-$_CUDNN_VERSION/cuda/lib64/libcudnn.so && \ - ln -s /usr/local/cudnn{-$_CUDNN_VERSION,} - -ARG BUILD_USER=onnxruntimedev -ARG BUILD_UID=1000 -WORKDIR /home/$BUILD_USER -RUN adduser --gecos 'onnxruntime Build User' --disabled-password $BUILD_USER --uid $BUILD_UID -USER $BUILD_USER -ENV CUDA_MODULE_LOADING "LAZY" diff --git a/tools/ci_build/github/linux/run_dockerbuild.sh b/tools/ci_build/github/linux/run_dockerbuild.sh index 0c1850e518..440752bc81 100755 --- a/tools/ci_build/github/linux/run_dockerbuild.sh +++ b/tools/ci_build/github/linux/run_dockerbuild.sh @@ -95,13 +95,6 @@ elif [ $BUILD_DEVICE = "gpu" ]; then $GET_DOCKER_IMAGE_CMD --repository "onnxruntime-$IMAGE" \ --docker-build-args="--build-arg BASEIMAGE=nvcr.io/nvidia/cuda:11.8.0-cudnn8-devel-${BUILD_OS} --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} --build-arg INSTALL_DEPS_EXTRA_ARGS=\"${INSTALL_DEPS_EXTRA_ARGS}\" --build-arg USE_CONDA=${USE_CONDA} --network=host" \ --dockerfile Dockerfile.ubuntu_gpu_training --context . -elif [[ $BUILD_DEVICE = "tensorrt"* ]]; then - IMAGE="$BUILD_OS-cuda11.8-cudnn8.7-tensorrt8.5" - DOCKER_FILE=Dockerfile.ubuntu_tensorrt - - $GET_DOCKER_IMAGE_CMD --repository "onnxruntime-$IMAGE" \ - --docker-build-args="--build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER}" \ - --dockerfile $DOCKER_FILE --context . elif [[ $BUILD_DEVICE = "openvino"* ]]; then BUILD_ARGS="--build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=3.8" IMAGE="$BUILD_OS-openvino" diff --git a/tools/ci_build/github/windows/setup_env_trt.bat b/tools/ci_build/github/windows/setup_env_trt.bat index 1308e43a4f..d9b77c76fc 100644 --- a/tools/ci_build/github/windows/setup_env_trt.bat +++ b/tools/ci_build/github/windows/setup_env_trt.bat @@ -6,4 +6,6 @@ if exist PATH=%AGENT_TEMPDIRECTORY%\v11.8\ { } else { set PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8\bin;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8\extras\CUPTI\lib64;%PATH% } +set PATH=C:\local\TensorRT-8.6.1.6.Windows10.x86_64.cuda-11.8\lib;%PATH% set GRADLE_OPTS=-Dorg.gradle.daemon=false +set CUDA_MODULE_LOADING=LAZY \ No newline at end of file