Integrate TensorRT EP libs into existing GPU Nuget Package (Approach#1) (#8727)

* Merge CPU/GPU nuget pipeline * Include TensorRT EP libraries into existing GPU nuget package pipeline * modify to use correct YAML * Modify for test * modify for test * Add depedance * Add depedance (cont.) * modify for test * Add create TensorRT nuget package * modify for test * modify for test * Merge CPU/GPU nuget pipeline * Include TensorRT EP libraries into existing GPU nuget package pipeline * modify to use correct YAML * Modify for test * modify for test * Add depedance * Add depedance (cont.) * modify for test * Add create TensorRT nuget package * modify for test * fix merge bug * code refactor * code refactor * modify for test * modify for test * modify for test * modify for test * modify for test * modify for test * cleanup * modify for test * fix bug * modify for test * refactor * fix bug and test * Modify for test * Modify for test * Modify for test * Modify for test * Prepare for PR * Prepare for PR * code refacotr from review * Remove naming 'Microsoft.ML.OnnxRuntime.TensorRT' to avoid confusion * Add linux TensorRT libraries * Remove redundant variable in YMAL * revert file * undo revert file * Modify regular expression so that it can capture the correct file * Remove newline at end of file * small fix * Revert to CUDA11.1 on Windows * Add unit tests for nuget package on Linux Co-authored-by: Changming Sun <chasun@microsoft.com>
2026-07-03 03:58:54 +00:00 · 2021-08-18 17:26:34 -07:00 · 2021-08-18 17:26:34 -07:00 · 51152e1aaa
commit 51152e1aaa
parent fe5046f48e
9 changed files with 86 additions and 14 deletions
--- a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.bat
+++ b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.bat
@ -7,6 +7,7 @@ SETLOCAL EnableDelayedExpansion
 SET TargetFramework=netcoreapp2.1
 SET TargetArch=x64
 SET dn="C:\Program Files\dotnet\dotnet"
+SET CurrentOnnxRuntimeVersion=""

 SET LocalNuGetRepo=%1
 IF NOT "%2"=="" (SET TargetFramework=%2)
@ -37,6 +38,7 @@ IF EXIST test\Microsoft.ML.OnnxRuntime.EndToEndTests\packages RMDIR /S /Q test\M
 IF EXIST test\Microsoft.ML.OnnxRuntime.EndToEndTests\bin RMDIR /S /Q test\Microsoft.ML.OnnxRuntime.EndToEndTests\bin
 IF EXIST test\Microsoft.ML.OnnxRuntime.EndToEndTests\obj RMDIR /S /Q test\Microsoft.ML.OnnxRuntime.EndToEndTests\obj

+@echo %PackageName%
@echo %CurrentOnnxRuntimeVersion%
 %dn% clean test\Microsoft.ML.OnnxRuntime.EndToEndTests\Microsoft.ML.OnnxRuntime.EndToEndTests.csproj
 %dn% restore -v detailed test\Microsoft.ML.OnnxRuntime.EndToEndTests\Microsoft.ML.OnnxRuntime.EndToEndTests.csproj --configfile .\Nuget.CSharp.config --no-cache --packages test\Microsoft.ML.OnnxRuntime.EndToEndTests\packages --source https://api.nuget.org/v3/index.json --source  %LocalNuGetRepo%
@ -46,12 +48,23 @@ IF NOT errorlevel 0 (
    EXIT 1
 )

+%dn% list test\Microsoft.ML.OnnxRuntime.EndToEndTests\Microsoft.ML.OnnxRuntime.EndToEndTests.csproj package
+dir test\Microsoft.ML.OnnxRuntime.EndToEndTests\packages\
+
 IF "%PACKAGENAME%"=="Microsoft.ML.OnnxRuntime.Gpu" (
  set TESTONGPU=ON
+  %dn% test -p:DefineConstants=USE_TENSORRT test\Microsoft.ML.OnnxRuntime.EndToEndTests\Microsoft.ML.OnnxRuntime.EndToEndTests.csproj --no-restore --filter TensorRT 
+
+  IF NOT errorlevel 0 (
+    @echo "Failed to build or execute the end-to-end test"
+    EXIT 1
+  )
+
  %dn% test -p:DefineConstants=USE_CUDA test\Microsoft.ML.OnnxRuntime.EndToEndTests\Microsoft.ML.OnnxRuntime.EndToEndTests.csproj --no-restore
 ) ELSE (
  %dn% test test\Microsoft.ML.OnnxRuntime.EndToEndTests\Microsoft.ML.OnnxRuntime.EndToEndTests.csproj --no-restore
 )
+
 IF NOT errorlevel 0 (
    @echo "Failed to build or execute the end-to-end test"
    EXIT 1
--- a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.sh
+++ b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.sh
@ -31,6 +31,11 @@ if [ $RunTestCsharp = "true" ]; then
  if [ $PACKAGENAME = "Microsoft.ML.OnnxRuntime.Gpu" ]; then
    export TESTONGPU=ON 
    dotnet test -p:DefineConstants=USE_CUDA $BUILD_SOURCESDIRECTORY/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/Microsoft.ML.OnnxRuntime.EndToEndTests.csproj --no-restore --verbosity detailed
+    if [ $? -ne 0 ]; then
+      echo "Failed to build or execute the end-to-end test"
+      exit 1
+    fi
+    dotnet test -p:DefineConstants=USE_TENSORRT $BUILD_SOURCESDIRECTORY/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/Microsoft.ML.OnnxRuntime.EndToEndTests.csproj --no-restore --verbosity detailed
  else
    dotnet test $BUILD_SOURCESDIRECTORY/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/Microsoft.ML.OnnxRuntime.EndToEndTests.csproj --no-restore --verbosity detailed
  fi
@ -41,4 +46,4 @@ if [ $RunTestCsharp = "true" ]; then
 fi

 cd $OldDir
-popd
+popd
--- a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml
+++ b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml
@ -78,6 +78,7 @@ jobs:
        condition : 'succeeded'
    - template: templates/clean-agent-build-directory-step.yml

+- template: linux-gpu-tensorrt-packaging-pipeline.yml

 - template: templates/win-ci.yml
  parameters:
@ -85,15 +86,28 @@ jobs:
    DoCompliance: ${{ parameters.DoCompliance }}
    DoEsrp: ${{ parameters.DoEsrp }}
    job_name_suffix: gpu
-    EnvSetupScript: setup_env_cuda_11.bat
+    EnvSetupScript: setup_env_gpu.bat
    buildArch: x64
    msbuildPlatform: x64
    packageName: gpu-x64
-    buildparameter: --use_cuda --cuda_version=11.4 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.4" --cudnn_home="C:\local\cudnn-11.4-windows-x64-v8.2.2.26\cuda" --enable_onnx_tests --enable_wcos --build_java --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=37;50;52;60;61;70;75;80"
+    buildparameter: --use_cuda --cuda_version=11.1 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.1" --cudnn_home="C:\local\cudnn-11.4-windows-x64-v8.2.2.26\cuda" --enable_onnx_tests --enable_wcos --build_java --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=37;52;60;61;70;75;80"
    runTests: ${{ parameters.RunOnnxRuntimeTests }}
    buildJava: true
    java_artifact_id: onnxruntime_gpu

+- template: templates/win-ci.yml
+  parameters:
+    ort_build_pool_name: 'onnxruntime-gpu-tensorrt8-winbuild'
+    DoCompliance: ${{ parameters.DoCompliance }}
+    DoEsrp: ${{ parameters.DoEsrp }}
+    job_name_suffix: tensorrt 
+    EnvSetupScript: setup_env_gpu.bat
+    buildArch: x64
+    msbuildPlatform: x64
+    packageName: tensorrt-x64
+    buildparameter: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.0.1.6.Windows10.x86_64.cuda-11.3.cudnn8.2" --cuda_version=11.1 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.1" --cudnn_home="C:\local\cudnn-11.4-windows-x64-v8.2.2.26\cuda" --enable_onnx_tests --enable_wcos --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=37;52;60;61;70;75;80"
+    runTests: ${{ parameters.RunOnnxRuntimeTests }}
+    buildJava: false 

 - template: templates/android-java-api-aar.yml
  parameters:
@ -248,7 +262,9 @@ jobs:
    breakCodesignValidationInjection: ${{ parameters.DoEsrp }}
  dependsOn:
  - Windows_Packaging_gpu
+  - Windows_Packaging_tensorrt
  - Linux_C_API_Packaging_GPU_x64
+  - Linux_C_API_Packaging_GPU_TensorRT_x64
  condition: succeeded()
  steps:
  - checkout: self
@ -259,12 +275,24 @@ jobs:
      artifactName: 'onnxruntime-win-gpu-x64'
      targetPath: '$(Build.BinariesDirectory)/nuget-artifact'

+  - task: DownloadPipelineArtifact@2
+    displayName: 'Download Pipeline Artifact - NuGet'
+    inputs:
+      artifactName: 'onnxruntime-win-tensorrt-x64'
+      targetPath: '$(Build.BinariesDirectory)/nuget-artifact'
+
  - task: DownloadPipelineArtifact@2
    displayName: 'Download Pipeline Artifact - NuGet'
    inputs:
      artifactName: 'onnxruntime-linux-x64-gpu'
      targetPath: '$(Build.BinariesDirectory)/nuget-artifact'

+  - task: DownloadPipelineArtifact@2
+    displayName: 'Download Pipeline Artifact - NuGet'
+    inputs:
+      artifactName: 'onnxruntime-linux-x64-gpu-tensorrt'
+      targetPath: '$(Build.BinariesDirectory)/nuget-artifact'
+
  - task: DownloadPipelineArtifact@2
    displayName: 'Download Pipeline Artifact - NuGet'
    inputs:
@ -403,7 +431,7 @@ jobs:
    
 - template: nuget/templates/test_win.yml
  parameters:
-    AgentPool : 'onnxruntime-gpu-winbuild'
+    AgentPool : 'onnxruntime-gpu-tensorrt8-winbuild'
    NugetPackageName : 'Microsoft.ML.OnnxRuntime.Gpu'
    ArtifactSuffix: 'GPU'
    Skipx86Tests: 'true'
@ -412,4 +440,4 @@ jobs:
  parameters:
    AgentPool : Onnxruntime-Linux-GPU
    ArtifactSuffix: 'GPU'
-    NugetPackageName : 'Microsoft.ML.OnnxRuntime.Gpu'
+    NugetPackageName : 'Microsoft.ML.OnnxRuntime.Gpu'
--- a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-packaging-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-packaging-pipeline.yml
@ -1,6 +1,3 @@
-variables:
-  PackageName: 'Microsoft.ML.OnnxRuntime.Gpu.Tensorrt'
-
 jobs:

 - job: Linux_C_API_Packaging_GPU_TensorRT_x64
--- a/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml
+++ b/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml
@ -35,7 +35,7 @@ jobs:
  - task: BatchScript@1
    displayName: 'setup env'
    inputs:
-      filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\setup_env_cuda_11.bat'
+      filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\setup_env_gpu.bat'
      modifyEnvironment: true
      workingFolder: '$(Build.BinariesDirectory)'

--- a/tools/ci_build/github/azure-pipelines/templates/c-api-artifacts-package-and-publish-steps-windows.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/c-api-artifacts-package-and-publish-steps-windows.yml
@ -25,6 +25,11 @@ parameters:
  type: string
  default: ''

+- name: trtEnabled  
+  displayName: Include TRT EP libraries? 
+  type: boolean
+  default: true
+
 steps:
    - task: CmdLine@2
      displayName: 'Copy build artifacts for zipping'
@ -59,6 +64,19 @@ steps:

        workingDirectory: '$(Build.BinariesDirectory)\${{parameters.buildConfig}}'

+    - task: CmdLine@2
+      displayName: 'Copy TensorRT EP build artifacts for zipping'
+      condition: and(succeeded(), eq('${{ parameters.trtEnabled}}', true))
+      inputs:
+        script: |
+          mkdir $(Build.BinariesDirectory)\${{parameters.artifactName}}
+          mkdir $(Build.BinariesDirectory)\${{parameters.artifactName}}\lib
+          mkdir $(Build.BinariesDirectory)\${{parameters.artifactName}}\include
+
+          copy $(Build.BinariesDirectory)\${{parameters.buildConfig}}\${{parameters.buildConfig}}\onnxruntime_providers_tensorrt.dll $(Build.BinariesDirectory)\${{parameters.artifactName}}\lib
+          copy $(Build.BinariesDirectory)\${{parameters.buildConfig}}\${{parameters.buildConfig}}\onnxruntime.dll $(Build.BinariesDirectory)\${{parameters.artifactName}}\lib
+          copy $(Build.BinariesDirectory)\${{parameters.buildConfig}}\${{parameters.buildConfig}}\onnxruntime.pdb $(Build.BinariesDirectory)\${{parameters.artifactName}}\lib
+          copy $(Build.BinariesDirectory)\${{parameters.buildConfig}}\${{parameters.buildConfig}}\onnxruntime.lib $(Build.BinariesDirectory)\${{parameters.artifactName}}\lib

    - template: win-esrp-dll.yml
      parameters:
--- a/tools/ci_build/github/windows/bundle_dlls_gpu.bat
+++ b/tools/ci_build/github/windows/bundle_dlls_gpu.bat
@ -8,8 +8,15 @@ FOR /R %%i IN (*.nupkg) do (
   set filename=%%~ni
   IF NOT "!filename:~25,7!"=="Managed" (
       mkdir runtimes\linux-x64\native
-       move onnxruntime-linux-x64\lib\libonnxruntime.so.1* runtimes\linux-x64\native\libonnxruntime.so
-       move onnxruntime-linux-x64\lib\libonnxruntime_providers_* runtimes\linux-x64\native
+       move onnxruntime-linux-x64-gpu\lib\libonnxruntime_providers_* runtimes\linux-x64\native
+       move onnxruntime-linux-x64-gpu-tensorrt\lib\libonnxruntime.so.1* runtimes\linux-x64\native\libonnxruntime.so
+       move onnxruntime-linux-x64-gpu-tensorrt\lib\libonnxruntime_providers_shared.so runtimes\linux-x64\native\libonnxruntime_providers_shared.so
+       move onnxruntime-linux-x64-gpu-tensorrt\lib\libonnxruntime_providers_tensorrt.so runtimes\linux-x64\native\libonnxruntime_providers_tensorrt.so
+       mkdir runtimes\win-x64\native
+       move onnxruntime-win-tensorrt-x64\lib\onnxruntime_providers_tensorrt.dll runtimes\win-x64\native\onnxruntime_providers_tensorrt.dll
+       move onnxruntime-win-tensorrt-x64\lib\onnxruntime.dll runtimes\win-x64\native\onnxruntime.dll
+       move onnxruntime-win-tensorrt-x64\lib\onnxruntime.lib runtimes\win-x64\native\onnxruntime.lib
+       move onnxruntime-win-tensorrt-x64\lib\onnxruntime.pdb runtimes\win-x64\native\onnxruntime.pdb
       7z a  %%~ni.nupkg runtimes
   )
-)
+) 
--- a/tools/ci_build/github/windows/extract_nuget_files_gpu.ps1
+++ b/tools/ci_build/github/windows/extract_nuget_files_gpu.ps1
@ -12,7 +12,7 @@ Foreach-Object {

 Get-ChildItem $Env:BUILD_BINARIESDIRECTORY\nuget-artifact -Filter *.tgz | 
 Foreach-Object {
- $cmd = "7z.exe x $($_.FullName) -y -o$Env:BUILD_BINARIESDIRECTORY\nuget-artifact"
+ $cmd = "7z.exe x $($_.FullName) -y -o$Env:BUILD_BINARIESDIRECTORY\nuget-artifact" # *.tar will be created after *.tgz is extracted 
 Write-Output $cmd
 Invoke-Expression -Command $cmd
 }
@ -31,4 +31,6 @@ Copy-Item -Path $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget
 Copy-Item -Path $Env:BUILD_BINARIESDIRECTORY\extra-artifact\protoc.exe $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\external\protobuf\cmake\RelWithDebInfo


-Get-ChildItem $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts\onnxruntime-linux-x64-* | Rename-Item -NewName $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts\onnxruntime-linux-x64
+Get-ChildItem $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts | Where-Object { $_.Name -match 'onnxruntime-linux-x64-gpu-\d{1,}\.\d{1,}\.\d{1,}$' } | Rename-Item -NewName $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts\onnxruntime-linux-x64-gpu
+Get-ChildItem $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts\onnxruntime-linux-x64-gpu-tensorrt-* | Rename-Item -NewName $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts\onnxruntime-linux-x64-gpu-tensorrt
+Get-ChildItem $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts\onnxruntime-win-tensorrt-x64-* | Rename-Item -NewName $Env:BUILD_BINARIESDIRECTORY\RelWithDebInfo\RelWithDebInfo\nuget-artifacts\onnxruntime-win-tensorrt-x64
--- a/tools/ci_build/github/windows/setup_env_gpu.bat
+++ b/tools/ci_build/github/windows/setup_env_gpu.bat
@ -0,0 +1,2 @@
+set PATH=C:\azcopy;C:\local\TensorRT-8.0.1.6.Windows10.x86_64.cuda-11.3.cudnn8.2\lib;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.1\bin;C:\local\cudnn-11.4-windows-x64-v8.2.2.26\cuda\bin;%PATH%
+set GRADLE_OPTS=-Dorg.gradle.daemon=false