Integrate C-API tests into Pipelines for release packages (#10794)

* add c-api test for package

* fix bug for running c-api test for package

* refine run application script

* remove redundant code

* include CUDA test

* Remove testing CUDA EP temporarily

* fix bug

* Code refactor

* try to fix YAML bug

* try to fix YAML bug

* try to fix YAML bug

* fix bug for multiple directories in Pipelines

* fix bug

* add comments and fix bug

* Update c-api-noopenmp-packaging-pipelines.yml

* Remove failOnStandardError flag in Pipelines

tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml

@@ -19,6 +19,13 @@ parameters:
   type: boolean
   default: false
 
+resources:
+  repositories:
+  - repository: onnxruntime-inference-examples # The name used to reference this repository in the checkout step
+    type: github
+    endpoint: ort-examples 
+    name: microsoft/onnxruntime-inference-examples 
+
 jobs:
 - template: templates/c-api-cpu.yml
   parameters:
@@ -269,9 +276,14 @@ jobs:
   - Linux_C_API_Packaging_GPU_TensorRT_x64
   condition: succeeded()
   steps:
-  - checkout: self
+  - checkout: self                           # due to checkout multiple repos, the root directory is $(Build.SourcesDirectory)/onnxruntime
+  - checkout: onnxruntime-inference-examples # due to checkout multiple repos, the root directory is $(Build.SourcesDirectory)/onnxruntime-inference-examples
     submodules: false
+  - script: dir $(Build.SourcesDirectory)
   - template: templates/set-version-number-variables-step.yml
+    parameters:
+      versionFileDirectory: '$(Build.SourcesDirectory)/onnxruntime'
+      workingDirectory: '$(Build.SourcesDirectory)/onnxruntime'
   - task: DownloadPipelineArtifact@2
     displayName: 'Download Pipeline Artifact - Combined GPU'
     inputs:
@@ -287,7 +299,7 @@ jobs:
   - task: ShellScript@2
     displayName: 'Shell Script'
     inputs:
-      scriptPath: 'tools/ci_build/github/linux/extract_and_bundle_gpu_package.sh'
+      scriptPath: 'onnxruntime/tools/ci_build/github/linux/extract_and_bundle_gpu_package.sh'
       args: '-a $(Build.BinariesDirectory)/tgz-artifacts'
       workingDirectory: '$(Build.BinariesDirectory)/tgz-artifacts'
 
@@ -305,10 +317,27 @@ jobs:
       PackageType: 'tarball'
       PackagePath: '$(Build.ArtifactStagingDirectory)'
       PackageName: 'onnxruntime-linux-x64-gpu-$(OnnxRuntimeVersion).tgz'
+      ScriptPath: '$(Build.SourcesDirectory)/onnxruntime/tools/nuget/validate_package.py'
       PlatformsSupported: 'linux-x64'
       VerifyNugetSigning: false
       workingDirectory: '$(Build.ArtifactStagingDirectory)'
 
+  - template: templates/get-docker-image-steps.yml
+    parameters:
+      ScriptName: onnxruntime/tools/ci_build/get_docker_image.py 
+      Dockerfile: onnxruntime/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11_4_tensorrt8_2
+      Context: onnxruntime/tools/ci_build/github/linux/docker
+      DockerBuildArgs: "--network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64 --build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-10/root --build-arg PREPEND_PATH=/opt/rh/devtoolset-10/root/usr/bin: --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-10/root/usr/lib64:/opt/rh/devtoolset-10/root/usr/lib:/opt/rh/devtoolset-10/root/usr/lib64/dyninst:/opt/rh/devtoolset-10/root/usr/lib/dyninst:/usr/local/lib64 --build-arg BUILD_UID=$( id -u )"
+      Repository: onnxruntimecuda114xtrt82build
+  - task: CmdLine@2
+    displayName: 'Test C API application for GPU package'
+    inputs:
+      script: |
+        docker run --gpus all -e CC=/opt/rh/devtoolset-10/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-10/root/usr/bin/c++ -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/src_dir \
+        --volume $(Build.ArtifactStagingDirectory):/artifact_src -e NIGHTLY_BUILD onnxruntimecuda114xtrt82build \
+        /src_dir/onnxruntime-inference-examples/c_cxx/squeezenet/run_capi_application.sh -o /src_dir/onnxruntime -p /artifact_src/onnxruntime-linux-x64-gpu-$(OnnxRuntimeVersion).tgz -w /src_dir/onnxruntime-inference-examples/c_cxx/squeezenet 
+      workingDirectory: '$(Build.ArtifactStagingDirectory)'
+
   - task: PublishPipelineArtifact@1
     inputs:
       targetPath: '$(Build.ArtifactStagingDirectory)/onnxruntime-linux-x64-gpu-$(OnnxRuntimeVersion).tgz'
@@ -317,15 +346,26 @@ jobs:
 - job: Windows_Packaging_combined_GPU
   workspace:
     clean: all
-  pool: 'Win-CPU-2021'
+  pool: 'onnxruntime-gpu-tensorrt8-winbuild'
   dependsOn:
   - Windows_Packaging_gpu
   - Windows_Packaging_tensorrt
   condition: succeeded()
   steps:
-  - checkout: self
+  - checkout: self                           # due to checkout multiple repos, the root directory is $(Build.SourcesDirectory)/onnxruntime
+  - checkout: onnxruntime-inference-examples # due to checkout multiple repos, the root directory is $(Build.SourcesDirectory)/onnxruntime-inference-examples
     submodules: false
+  - script: dir $(Build.SourcesDirectory)
+  - task: BatchScript@1
+    displayName: 'setup env'
+    inputs:
+        filename: '$(Build.SourcesDirectory)\onnxruntime\tools\ci_build\github\windows\setup_env_gpu.bat'
+        modifyEnvironment: true
+        workingFolder: '$(Build.BinariesDirectory)'
   - template: templates/set-version-number-variables-step.yml
+    parameters:
+      versionFileDirectory: '$(Build.SourcesDirectory)\onnxruntime'
+      workingDirectory: '$(Build.SourcesDirectory)\onnxruntime'
   - task: DownloadPipelineArtifact@2
     displayName: 'Download Pipeline Artifact - Combined GPU'
     inputs:
@@ -342,7 +382,7 @@ jobs:
     displayName: 'PowerShell Script'
     inputs:
       targetType: filePath
-      filePath: $(Build.SourcesDirectory)\tools\ci_build\github\windows\extract_zip_files_gpu.ps1
+      filePath: $(Build.SourcesDirectory)\onnxruntime\tools\ci_build\github\windows\extract_zip_files_gpu.ps1
 
   - script: |
      dir
@@ -352,7 +392,7 @@ jobs:
   - task: BatchScript@1
     displayName: 'Bundle CUDA/TRT EP binaries'
     inputs:
-      filename: $(Build.SourcesDirectory)\tools\ci_build\github\windows\bundle_dlls_gpu.bat
+      filename: $(Build.SourcesDirectory)\onnxruntime\tools\ci_build\github\windows\bundle_dlls_gpu.bat
       workingFolder: $(Build.BinariesDirectory)\zip-artifacts
 
   - task: CopyFiles@2
@@ -367,10 +407,18 @@ jobs:
       PackageType: 'zip'
       PackagePath: '$(Build.ArtifactStagingDirectory)'
       PackageName: 'onnxruntime-win-x64-gpu-$(OnnxRuntimeVersion).zip'
+      ScriptPath: '$(Build.SourcesDirectory)\onnxruntime\tools\nuget\validate_package.py'
       PlatformsSupported: 'win-x64'
       VerifyNugetSigning: false
       workingDirectory: '$(Build.ArtifactStagingDirectory)'
 
+  - task: BatchScript@1
+    displayName: 'Test C API application for GPU package'
+    inputs:
+      filename: $(Build.SourcesDirectory)\onnxruntime-inference-examples\c_cxx\squeezenet\run_capi_application.bat
+      arguments: $(Build.SourcesDirectory)\onnxruntime $(Build.ArtifactStagingDirectory)\onnxruntime-win-x64-gpu-$(OnnxRuntimeVersion).zip $(Build.SourcesDirectory)\onnxruntime-inference-examples\c_cxx\squeezenet 
+      workingFolder: '$(Build.ArtifactStagingDirectory)' 
+
   - task: PublishPipelineArtifact@0
     displayName: 'Publish Pipeline Combined GPU Package Artifact'
     inputs:

tools/ci_build/github/azure-pipelines/templates/get-docker-image-steps.yml

@@ -13,6 +13,9 @@ parameters:
 - name: UseImageCacheContainerRegistry
   type: boolean
   default: true
+- name: ScriptName
+  type: string
+  default: "tools/ci_build/get_docker_image.py" 
 
 steps:
 - ${{ if eq(parameters.UseImageCacheContainerRegistry, true) }}:
@@ -20,7 +23,7 @@ steps:
     parameters:
       Steps:
       - script: |
-          tools/ci_build/get_docker_image.py \
+          ${{ parameters.ScriptName }} \
             --dockerfile "${{ parameters.Dockerfile }}" \
             --context "${{ parameters.Context }}" \
             --docker-build-args "${{ parameters.DockerBuildArgs }}" \
@@ -30,7 +33,7 @@ steps:
       ContainerRegistry: onnxruntimebuildcache
 - ${{ if eq(parameters.UseImageCacheContainerRegistry, false) }}:
   - script: |
-      tools/ci_build/get_docker_image.py \
+      ${{ parameters.ScriptName }} \
         --dockerfile "${{ parameters.Dockerfile }}" \
         --context "${{ parameters.Context }}" \
         --docker-build-args "${{ parameters.DockerBuildArgs }}" \

tools/ci_build/github/azure-pipelines/templates/set-version-number-variables-step.yml

@@ -1,4 +1,8 @@
 # Sets version number from VERSION.txt into a variable. As well as the git commit hash.
+parameters:
+  versionFileDirectory: "$(Build.SourcesDirectory)"
+  workingDirectory: "$(Build.SourcesDirectory)"
+
 
 steps:
 - task: CmdLine@2
@@ -6,7 +10,7 @@ steps:
   inputs:
     script: |
         SETLOCAL EnableDelayedExpansion
-        set /p _OnnxRuntimeVersion=<$(Build.SourcesDirectory)\VERSION_NUMBER 
+        set /p _OnnxRuntimeVersion=<${{parameters.versionFileDirectory}}\VERSION_NUMBER 
         @echo ##vso[task.setvariable variable=OnnxRuntimeVersion;]%_OnnxRuntimeVersion%
         
         FOR /F "tokens=* USEBACKQ" %%F IN (`git rev-parse HEAD`) DO (
@@ -17,14 +21,14 @@ steps:
             @echo ##vso[task.setvariable variable=OnnxRuntimeGitCommitHashShort;]%%F
         )
         
-    workingDirectory: '$(Build.SourcesDirectory)'
+    workingDirectory: ${{parameters.workingDirectory}} 
   condition: eq(variables['Agent.OS'], 'Windows_NT') 
 
 - task: CmdLine@2
   displayName: 'Set version number variables for Unix'
   inputs:
     script: |
-        _OnnxRuntimeVersion=$(head -1 $(Build.SourcesDirectory)/VERSION_NUMBER) 
+        _OnnxRuntimeVersion=$(head -1 ${{parameters.versionFileDirectory}}/VERSION_NUMBER) 
         echo "##vso[task.setvariable variable=OnnxRuntimeVersion;]$_OnnxRuntimeVersion"
         
         _OnnxRuntimeGitCommitHash=$(git rev-parse HEAD)
@@ -33,5 +37,5 @@ steps:
         _OnnxRuntimeGitCommitHash=$(git rev-parse --short=8 HEAD)
         echo "##vso[task.setvariable variable=OnnxRuntimeGitCommitHashShort;]$_OnnxRuntimeGitCommitHash"
 
-    workingDirectory: '$(Build.SourcesDirectory)'
-  condition: not(eq(variables['Agent.OS'], 'Windows_NT'))
+    workingDirectory: ${{parameters.workingDirectory}} 
+  condition: not(eq(variables['Agent.OS'], 'Windows_NT'))

tools/ci_build/github/azure-pipelines/templates/validate-package.yml

@@ -4,6 +4,7 @@ parameters:
   PackageType: ''
   PackageName: ''
   PackagePath: ''
+  ScriptPath: '$(Build.SourcesDirectory)/tools/nuget/validate_package.py'
   workingDirectory: "$(Build.BinariesDirectory)"
 
 steps:
@@ -15,6 +16,6 @@ steps:
     - task: PythonScript@0
       displayName: 'Validate Package'
       inputs:
-        scriptPath: '$(Build.SourcesDirectory)/tools/nuget/validate_package.py'
+        scriptPath: '${{parameters.ScriptPath}}'
         arguments: '--package_type ${{parameters.PackageType}} --package_name ${{parameters.PackageName}} --package_path ${{parameters.PackagePath}} --platforms_supported ${{parameters.PlatformsSupported}} --verify_nuget_signing ${{parameters.VerifyNugetSigning}}'
         workingDirectory: ${{parameters.workingDirectory}} 

tools/ci_build/github/windows/setup_env_gpu.bat

@@ -1,2 +1,2 @@
-set PATH=C:\azcopy;C:\local\TensorRT-8.2.1.8.Windows10.x86_64.cuda-11.4.cudnn8.2\lib;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.4\bin;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.4\extras\CUPTI\lib64;%PATH%
+set PATH=C:\azcopy;C:\local\TensorRT-8.2.1.8.Windows10.x86_64.cuda-11.4.cudnn8.2\lib;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.4\bin;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.4\extras\CUPTI\lib64;C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin;%PATH%
 set GRADLE_OPTS=-Dorg.gradle.daemon=false