onnxruntime/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml

parameters:
- name: RunOnnxRuntimeTests
  displayName: Run Tests?
  type: boolean
  default: true

- name: DoCompliance
  displayName: Run Compliance Tasks?
  type: boolean
  default: true

- name: DoEsrp
  displayName: Run code sign tasks? Must be true if you are doing an Onnx Runtime release.
  type: boolean
  default: false

- name: IsReleaseBuild
  displayName: Is a release build? Set it to true if you are doing an Onnx Runtime release.
  type: boolean
  default: false

resources:
  repositories:
  - repository: onnxruntime-inference-examples # The name used to reference this repository in the checkout step
    type: github
    endpoint: ort-examples 
    name: microsoft/onnxruntime-inference-examples 

jobs:
- template: templates/c-api-cpu.yml
  parameters:
    RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }}
    DoCompliance: ${{ parameters.DoCompliance }}
    DoEsrp: ${{ parameters.DoEsrp }}
    IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
    OrtNugetPackageId: 'Microsoft.ML.OnnxRuntime'
    AdditionalBuildFlags: ''
    AdditionalWinBuildFlags: '--enable_onnx_tests --enable_wcos'
    BuildVariant: 'default'

- job: Linux_C_API_Packaging_GPU_x64
  workspace:
    clean: all
  timeoutInMinutes:  120
  pool: 'Onnxruntime-Linux-GPU'
  variables:
    CUDA_VERSION: '11.4'
  steps:
    - template: templates/set-version-number-variables-step.yml
    - template: templates/get-docker-image-steps.yml
      parameters:
        Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11
        Context: tools/ci_build/github/linux/docker
        DockerBuildArgs: "--network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64 --build-arg BASEIMAGE=nvidia/cuda:11.4.0-cudnn8-devel-centos7 --build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-10/root --build-arg PREPEND_PATH=/opt/rh/devtoolset-10/root/usr/bin: --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-10/root/usr/lib64:/opt/rh/devtoolset-10/root/usr/lib:/opt/rh/devtoolset-10/root/usr/lib64/dyninst:/opt/rh/devtoolset-10/root/usr/lib/dyninst:/usr/local/lib64 --build-arg BUILD_UID=$( id -u )"
        Repository: onnxruntimecuda11build
    - task: CmdLine@2
      inputs:
        script: |
          mkdir -p $HOME/.onnx
          docker run --gpus all -e CC=/opt/rh/devtoolset-10/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-10/root/usr/bin/c++ -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \
          --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecuda11build \
          /opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_java --build_dir /build --config Release \
          --skip_submodule_sync  --parallel --build_shared_lib --use_cuda --cuda_version=$(CUDA_VERSION) --cuda_home=/usr/local/cuda-$(CUDA_VERSION) --cudnn_home=/usr/local/cuda-$(CUDA_VERSION) --cmake_extra_defines CMAKE_CUDA_HOST_COMPILER=/opt/rh/devtoolset-10/root/usr/bin/cc 'CMAKE_CUDA_ARCHITECTURES=37;50;52;60;61;70;75;80'          
        workingDirectory: $(Build.SourcesDirectory)

    - template: templates/java-api-artifacts-package-and-publish-steps-posix.yml
      parameters:
          arch: 'linux-x64'
          buildConfig: 'Release'
          artifactName: 'onnxruntime-java-linux-x64-cuda'
          version: '$(OnnxRuntimeVersion)'
          libraryName: 'libonnxruntime.so'
          nativeLibraryName: 'libonnxruntime4j_jni.so'

    - template: templates/c-api-artifacts-package-and-publish-steps-posix.yml
      parameters:
          buildConfig: 'Release'
          artifactName: 'onnxruntime-linux-x64-cuda-$(OnnxRuntimeVersion)'
          artifactNameNoVersionString:  'onnxruntime-linux-x64-cuda'
          libraryName: 'libonnxruntime.so.$(OnnxRuntimeVersion)'

    - template: templates/component-governance-component-detection-steps.yml
      parameters :
        condition : 'succeeded'
    - template: templates/clean-agent-build-directory-step.yml

- template: linux-gpu-tensorrt-packaging-pipeline.yml
  parameters:
      artifactName: 'onnxruntime-linux-x64-tensorrt-$(OnnxRuntimeVersion)'
      artifactNameNoVersionString: 'onnxruntime-linux-x64-tensorrt'
      buildJava: true
      buildJavaOption: '--build_java'

- template: templates/win-ci.yml
  parameters:
    ort_build_pool_name: 'onnxruntime-gpu-tensorrt8-winbuild-t4'
    DoCompliance: ${{ parameters.DoCompliance }}
    DoEsrp: ${{ parameters.DoEsrp }}
    job_name_suffix: gpu
    EnvSetupScript: setup_env_gpu.bat
    buildArch: x64
    msbuildPlatform: x64
    packageName: x64-cuda
    buildparameter: --use_cuda --cuda_version=11.4 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.4" --cudnn_home="C:\local\cudnn-11.4-windows-x64-v8.2.2.26\cuda" --enable_onnx_tests --enable_wcos --build_java --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=37;52;60;61;70;75;80"
    runTests: ${{ parameters.RunOnnxRuntimeTests }}
    buildJava: true
    java_artifact_id: onnxruntime_gpu

- template: templates/win-ci.yml
  parameters:
    ort_build_pool_name: 'onnxruntime-gpu-tensorrt8-winbuild-t4'
    DoCompliance: ${{ parameters.DoCompliance }}
    DoEsrp: ${{ parameters.DoEsrp }}
    job_name_suffix: tensorrt
    EnvSetupScript: setup_env_gpu.bat
    buildArch: x64
    msbuildPlatform: x64
    packageName: x64-tensorrt
    buildparameter: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.2.1.8.Windows10.x86_64.cuda-11.4.cudnn8.2" --cuda_version=11.4 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.4" --cudnn_home="C:\local\cudnn-11.4-windows-x64-v8.2.2.26\cuda" --enable_onnx_tests --enable_wcos --build_java --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=37;52;60;61;70;75;80"
    runTests: ${{ parameters.RunOnnxRuntimeTests }}
    buildJava: true
    java_artifact_id: onnxruntime_gpu

- job: Jar_Packaging_GPU
  workspace:
    clean: all
  pool: 'Win-CPU-2021'
  dependsOn:
  - Linux_C_API_Packaging_GPU_x64
  - Linux_C_API_Packaging_GPU_TensorRT_x64
  - Windows_Packaging_gpu
  - Windows_Packaging_tensorrt
  condition: succeeded()
  steps:
  - checkout: self
    submodules: false
  - template: templates/set-version-number-variables-step.yml

  - task: DownloadPipelineArtifact@2
    displayName: 'Download Pipeline Artifact - Win x64'
    inputs:
      buildType: 'current'
      artifactName: 'drop-onnxruntime-java-win-x64-tensorrt'
      targetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64'

  - task: DownloadPipelineArtifact@2
    displayName: 'Download Pipeline Artifact - Linux x64'
    inputs:
      buildType: 'current'
      artifactName: 'drop-onnxruntime-java-linux-x64-cuda'
      targetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-linux-x64'

  - task: DownloadPipelineArtifact@2
    displayName: 'Download Pipeline Artifact - Linux x64'
    inputs:
      buildType: 'current'
      artifactName: 'drop-onnxruntime-java-linux-x64-tensorrt'
      targetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-linux-x64-tensorrt'

  - task: PowerShell@2
    displayName: 'PowerShell Script'
    inputs:
      targetType: filePath
      filePath: $(Build.SourcesDirectory)\tools\ci_build\github\windows\jar_gpu_packaging.ps1
      failOnStderr: true
      showWarnings: true
      workingDirectory: '$(Build.BinariesDirectory)\java-artifact'

  - task: CopyFiles@2
    displayName: 'Copy Java Files to Artifact Staging Directory'
    inputs:
      SourceFolder: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64'
      TargetFolder: '$(Build.ArtifactStagingDirectory)'

  - task: PublishPipelineArtifact@1
    displayName: 'Publish Pipeline Artifact'
    inputs:
      targetPath: '$(Build.ArtifactStagingDirectory)'
      artifact: 'onnxruntime-java-gpu'

  - template: templates/component-governance-component-detection-steps.yml
    parameters :
      condition : 'succeeded'
  - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
    displayName: 'Clean Agent Directories'
    condition: always()


- job: Final_Jar_Testing_Windows_GPU
  workspace:
    clean: all
  pool: 'onnxruntime-gpu-winbuild'
  timeoutInMinutes: 60
  variables:
  - name: runCodesignValidationInjection
    value: false
  dependsOn:
    Jar_Packaging_GPU
  steps:
  - template: templates/set-version-number-variables-step.yml

  - task: BatchScript@1
    displayName: 'setup env'
    inputs:
        filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\setup_env_cuda_11.bat'
        modifyEnvironment: true
        workingFolder: '$(Build.BinariesDirectory)'

  - task: DownloadPipelineArtifact@2
    displayName: 'Download Final Jar'
    inputs:
        buildType: 'current'
        artifactName: 'onnxruntime-java-gpu'
        targetPath: '$(Build.BinariesDirectory)\final-jar'

  - task: CmdLine@2
    inputs:
      script: |
        mkdir test
        pushd test
        jar xf $(Build.BinariesDirectory)\final-jar\testing.jar
        popd
        powershell -Command "Invoke-WebRequest https://oss.sonatype.org/service/local/repositories/releases/content/org/junit/platform/junit-platform-console-standalone/1.6.2/junit-platform-console-standalone-1.6.2.jar -OutFile junit-platform-console-standalone-1.6.2.jar"
        powershell -Command "Invoke-WebRequest https://oss.sonatype.org/service/local/repositories/releases/content/com/google/protobuf/protobuf-java/3.20.1/protobuf-java-3.20.1.jar -OutFile protobuf-java-3.20.1.jar"
        java -DUSE_CUDA=1 -jar junit-platform-console-standalone-1.6.2.jar -cp .;.\test;protobuf-java-3.20.1.jar;onnxruntime_gpu-$(OnnxRuntimeVersion).jar --scan-class-path --fail-if-no-tests --disable-banner        
      workingDirectory: '$(Build.BinariesDirectory)\final-jar'

  - template: templates/component-governance-component-detection-steps.yml
    parameters :
      condition : 'succeeded'
  - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
    displayName: 'Clean Agent Directories'
    condition: always()

- job: Final_Jar_Testing_Linux_GPU
  workspace:
    clean: all
  pool: 'Onnxruntime-Linux-GPU'
  variables:
  - name: runCodesignValidationInjection
    value: false
  timeoutInMinutes: 60
  dependsOn:
    Jar_Packaging_GPU
  steps:
  - checkout: self
    submodules: false
  - template: templates/set-version-number-variables-step.yml
  - task: DownloadPipelineArtifact@2
    displayName: 'Download Final Jar'
    inputs:
      buildType: 'current'
      artifactName: 'onnxruntime-java-gpu'
      targetPath: '$(Build.BinariesDirectory)/final-jar'

  - task: Bash@3
    displayName: 'Test'
    inputs:
      targetType: filePath
      filePath: 'tools/ci_build/github/linux/java_linux_final_test.sh'
      arguments: '-r $(Build.BinariesDirectory) -v $(OnnxRuntimeVersion)'

  - template: templates/component-governance-component-detection-steps.yml
    parameters :
      condition : 'succeeded'
  - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
    displayName: 'Clean Agent Directories'
    condition: always()

- job: Linux_Packaging_combined_GPU
  workspace:
    clean: all
  pool: 'Onnxruntime-Linux-GPU'
  dependsOn:
  - Linux_C_API_Packaging_GPU_x64
  - Linux_C_API_Packaging_GPU_TensorRT_x64
  condition: succeeded()
  steps:
  - checkout: self                           # due to checkout multiple repos, the root directory is $(Build.SourcesDirectory)/onnxruntime
  - checkout: onnxruntime-inference-examples # due to checkout multiple repos, the root directory is $(Build.SourcesDirectory)/onnxruntime-inference-examples
    submodules: false
  - script: dir $(Build.SourcesDirectory)
  - template: templates/set-version-number-variables-step.yml
    parameters:
      versionFileDirectory: '$(Build.SourcesDirectory)/onnxruntime'
      workingDirectory: '$(Build.SourcesDirectory)/onnxruntime'
  - task: DownloadPipelineArtifact@2
    displayName: 'Download Pipeline Artifact - Combined GPU'
    inputs:
      artifactName: 'onnxruntime-linux-x64-cuda'
      targetPath: '$(Build.BinariesDirectory)/tgz-artifacts'

  - task: DownloadPipelineArtifact@2
    displayName: 'Download Pipeline Artifact - Combined GPU'
    inputs:
      artifactName: 'onnxruntime-linux-x64-tensorrt'
      targetPath: '$(Build.BinariesDirectory)/tgz-artifacts'

  - task: ShellScript@2
    displayName: 'Shell Script'
    inputs:
      scriptPath: 'onnxruntime/tools/ci_build/github/linux/extract_and_bundle_gpu_package.sh'
      args: '-a $(Build.BinariesDirectory)/tgz-artifacts'
      workingDirectory: '$(Build.BinariesDirectory)/tgz-artifacts'

  - task: ArchiveFiles@2
    inputs:
      rootFolderOrFile: '$(Build.BinariesDirectory)/tgz-artifacts/onnxruntime-linux-x64-gpu'
      includeRootFolder: false
      archiveType: 'tar' # Options: zip, 7z, tar, wim
      tarCompression: 'gz'
      archiveFile: '$(Build.ArtifactStagingDirectory)/onnxruntime-linux-x64-gpu-$(OnnxRuntimeVersion).tgz'
      replaceExistingArchive: true

  - template: templates/validate-package.yml
    parameters:
      PackageType: 'tarball'
      PackagePath: '$(Build.ArtifactStagingDirectory)'
      PackageName: 'onnxruntime-linux-x64-gpu-$(OnnxRuntimeVersion).tgz'
      ScriptPath: '$(Build.SourcesDirectory)/onnxruntime/tools/nuget/validate_package.py'
      PlatformsSupported: 'linux-x64'
      VerifyNugetSigning: false
      workingDirectory: '$(Build.ArtifactStagingDirectory)'

  - template: templates/get-docker-image-steps.yml
    parameters:
      ScriptName: onnxruntime/tools/ci_build/get_docker_image.py 
      Dockerfile: onnxruntime/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11_4_tensorrt8_2
      Context: onnxruntime/tools/ci_build/github/linux/docker
      DockerBuildArgs: "--network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64 --build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-10/root --build-arg PREPEND_PATH=/opt/rh/devtoolset-10/root/usr/bin: --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-10/root/usr/lib64:/opt/rh/devtoolset-10/root/usr/lib:/opt/rh/devtoolset-10/root/usr/lib64/dyninst:/opt/rh/devtoolset-10/root/usr/lib/dyninst:/usr/local/lib64 --build-arg BUILD_UID=$( id -u )"
      Repository: onnxruntimecuda114xtrt82build
  - task: CmdLine@2
    displayName: 'Test C API application for GPU package'
    inputs:
      script: |
        docker run --gpus all -e CC=/opt/rh/devtoolset-10/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-10/root/usr/bin/c++ -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/src_dir \
        --volume $(Build.ArtifactStagingDirectory):/artifact_src -e NIGHTLY_BUILD onnxruntimecuda114xtrt82build \
        /src_dir/onnxruntime-inference-examples/c_cxx/squeezenet/run_capi_application.sh -o /src_dir/onnxruntime -p /artifact_src/onnxruntime-linux-x64-gpu-$(OnnxRuntimeVersion).tgz -w /src_dir/onnxruntime-inference-examples/c_cxx/squeezenet         
      workingDirectory: '$(Build.ArtifactStagingDirectory)'

  - task: PublishPipelineArtifact@1
    inputs:
      targetPath: '$(Build.ArtifactStagingDirectory)/onnxruntime-linux-x64-gpu-$(OnnxRuntimeVersion).tgz'
      artifactName: 'onnxruntime-linux-x64-gpu'

- job: Windows_Packaging_combined_GPU
  workspace:
    clean: all
  pool: 'onnxruntime-gpu-tensorrt8-winbuild'
  dependsOn:
  - Windows_Packaging_gpu
  - Windows_Packaging_tensorrt
  condition: succeeded()
  steps:
  - checkout: self                           # due to checkout multiple repos, the root directory is $(Build.SourcesDirectory)/onnxruntime
  - checkout: onnxruntime-inference-examples # due to checkout multiple repos, the root directory is $(Build.SourcesDirectory)/onnxruntime-inference-examples
    submodules: false
  - script: dir $(Build.SourcesDirectory)
  - task: BatchScript@1
    displayName: 'setup env'
    inputs:
        filename: '$(Build.SourcesDirectory)\onnxruntime\tools\ci_build\github\windows\setup_env_gpu.bat'
        modifyEnvironment: true
        workingFolder: '$(Build.BinariesDirectory)'
  - template: templates/set-version-number-variables-step.yml
    parameters:
      versionFileDirectory: '$(Build.SourcesDirectory)\onnxruntime'
      workingDirectory: '$(Build.SourcesDirectory)\onnxruntime'
  - task: DownloadPipelineArtifact@2
    displayName: 'Download Pipeline Artifact - Combined GPU'
    inputs:
      artifactName: 'onnxruntime-win-x64-cuda'
      targetPath: '$(Build.BinariesDirectory)/zip-artifacts'

  - task: DownloadPipelineArtifact@2
    displayName: 'Download Pipeline Artifact - Combined GPU'
    inputs:
      artifactName: 'onnxruntime-win-x64-tensorrt'
      targetPath: '$(Build.BinariesDirectory)/zip-artifacts'

  - task: PowerShell@2
    displayName: 'PowerShell Script'
    inputs:
      targetType: filePath
      filePath: $(Build.SourcesDirectory)\onnxruntime\tools\ci_build\github\windows\extract_zip_files_gpu.ps1

  - script: |
     dir     
    workingDirectory: '$(Build.BinariesDirectory)/zip-artifacts'
    displayName: 'List artifacts'

  - task: BatchScript@1
    displayName: 'Bundle CUDA/TRT EP binaries'
    inputs:
      filename: $(Build.SourcesDirectory)\onnxruntime\tools\ci_build\github\windows\bundle_dlls_gpu.bat
      workingFolder: $(Build.BinariesDirectory)\zip-artifacts

  - task: CopyFiles@2
    displayName: 'Copy zip file to: $(Build.ArtifactStagingDirectory)'
    inputs:
      SourceFolder: '$(Build.BinariesDirectory)\zip-artifacts'
      Contents: 'onnxruntime-win-x64-gpu-*.zip'
      TargetFolder: '$(Build.ArtifactStagingDirectory)'

  - template: templates/validate-package.yml
    parameters:
      PackageType: 'zip'
      PackagePath: '$(Build.ArtifactStagingDirectory)'
      PackageName: 'onnxruntime-win-x64-gpu-$(OnnxRuntimeVersion).zip'
      ScriptPath: '$(Build.SourcesDirectory)\onnxruntime\tools\nuget\validate_package.py'
      PlatformsSupported: 'win-x64'
      VerifyNugetSigning: false
      workingDirectory: '$(Build.ArtifactStagingDirectory)'

  - task: BatchScript@1
    displayName: 'Test C API application for GPU package'
    inputs:
      filename: $(Build.SourcesDirectory)\onnxruntime-inference-examples\c_cxx\squeezenet\run_capi_application.bat
      arguments: $(Build.SourcesDirectory)\onnxruntime $(Build.ArtifactStagingDirectory)\onnxruntime-win-x64-gpu-$(OnnxRuntimeVersion).zip $(Build.SourcesDirectory)\onnxruntime-inference-examples\c_cxx\squeezenet 
      workingFolder: '$(Build.ArtifactStagingDirectory)' 

  - task: PublishPipelineArtifact@0
    displayName: 'Publish Pipeline Combined GPU Package Artifact'
    inputs:
      artifactName: 'onnxruntime-win-x64-gpu'
      targetPath: '$(Build.ArtifactStagingDirectory)'

- job: NuGet_Packaging_GPU
  workspace:
    clean: all
  pool: 'Win-CPU-2021'
  variables:
    breakCodesignValidationInjection: ${{ parameters.DoEsrp }}
  dependsOn:
  - Windows_Packaging_gpu
  - Windows_Packaging_tensorrt
  - Linux_C_API_Packaging_GPU_x64
  - Linux_C_API_Packaging_GPU_TensorRT_x64
  condition: succeeded()
  steps:
  - checkout: self
    submodules: true
  - task: DownloadPipelineArtifact@2
    displayName: 'Download Pipeline Artifact - NuGet'
    inputs:
      artifactName: 'onnxruntime-win-x64-cuda'
      targetPath: '$(Build.BinariesDirectory)/nuget-artifact'

  - task: DownloadPipelineArtifact@2
    displayName: 'Download Pipeline Artifact - NuGet'
    inputs:
      artifactName: 'onnxruntime-win-x64-tensorrt'
      targetPath: '$(Build.BinariesDirectory)/nuget-artifact'

  - task: DownloadPipelineArtifact@2
    displayName: 'Download Pipeline Artifact - NuGet'
    inputs:
      artifactName: 'onnxruntime-linux-x64-cuda'
      targetPath: '$(Build.BinariesDirectory)/nuget-artifact'

  - task: DownloadPipelineArtifact@2
    displayName: 'Download Pipeline Artifact - NuGet'
    inputs:
      artifactName: 'onnxruntime-linux-x64-tensorrt'
      targetPath: '$(Build.BinariesDirectory)/nuget-artifact'

  - task: DownloadPipelineArtifact@2
    displayName: 'Download Pipeline Artifact - NuGet'
    inputs:
      artifactName: 'drop-extra'
      targetPath: '$(Build.BinariesDirectory)/extra-artifact'

  #Reconstruct the build dir
  - task: PowerShell@2
    displayName: 'PowerShell Script'
    inputs:
      targetType: filePath
      filePath: $(Build.SourcesDirectory)\tools\ci_build\github\windows\extract_nuget_files_gpu.ps1

  - script: |
     dir     
    workingDirectory: '$(Build.BinariesDirectory)/nuget-artifact'
    displayName: 'List artifacts'

  - script: |
     mklink /D /J models C:\local\models     
    workingDirectory: '$(Build.BinariesDirectory)'
    displayName: 'Create models link'

  - task: NuGetToolInstaller@0
    displayName: Use Nuget 5.7.0
    inputs:
      versionSpec: 5.7.0

  - task: MSBuild@1
    displayName: 'Restore NuGet Packages'
    inputs:
      solution: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln'
      platform: 'Any CPU'
      configuration: RelWithDebInfo
      msbuildArguments: '-t:restore -p:OrtPackageId="Microsoft.ML.OnnxRuntime.Gpu"'
      workingDirectory: '$(Build.SourcesDirectory)\csharp'

  - task: MSBuild@1
    displayName: 'Build C#'
    inputs:
      solution: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln'
      configuration: RelWithDebInfo
      platform: 'Any CPU'
      msbuildArguments: '-p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId="Microsoft.ML.OnnxRuntime.Gpu" -p:IsReleaseBuild=${{ parameters.IsReleaseBuild }}'
      workingDirectory: '$(Build.SourcesDirectory)\csharp'

  - template: templates/win-esrp-dll.yml
    parameters:
      FolderPath: '$(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo'
      DisplayName: 'ESRP - Sign C# dlls'
      DoEsrp: ${{ parameters.DoEsrp }}

  - task: MSBuild@1
    displayName: 'Build Nuget Packages'
    inputs:
      solution: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj'
      configuration: RelWithDebInfo
      platform: 'Any CPU'
      msbuildArguments: '-t:CreatePackage -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=Microsoft.ML.OnnxRuntime.Gpu -p:IsReleaseBuild=${{ parameters.IsReleaseBuild }}'
      workingDirectory: '$(Build.SourcesDirectory)\csharp'

  - task: BatchScript@1
    displayName: 'Add TensorRT header file to the native nuGet package'
    inputs:
      filename: $(Build.SourcesDirectory)\tools\ci_build\github\windows\bundle_nuget_with_native_headers.bat
      workingFolder: $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo

  - task: CopyFiles@2
    displayName: 'Copy nuget packages to: $(Build.ArtifactStagingDirectory)'
    inputs:
      SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo'
      Contents: '*.snupkg'
      TargetFolder: '$(Build.ArtifactStagingDirectory)'

  - task: CopyFiles@2
    displayName: 'Copy nuget packages to: $(Build.ArtifactStagingDirectory)'
    inputs:
      SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo'
      Contents: '*.nupkg'
      TargetFolder: '$(Build.ArtifactStagingDirectory)'

  - task: CopyFiles@2
    displayName: 'Copy nuget packages to: $(Build.ArtifactStagingDirectory)'
    inputs:
      SourceFolder: '$(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo'
      Contents: '*.nupkg'
      TargetFolder: '$(Build.ArtifactStagingDirectory)'

  - template: templates/esrp_nuget.yml
    parameters:
      DisplayName: 'ESRP - sign NuGet package'
      FolderPath: '$(Build.ArtifactStagingDirectory)'
      DoEsrp: ${{ parameters.DoEsrp }}

  - template: templates/validate-package.yml
    parameters:
      PackageType: 'nuget'
      PackagePath: '$(Build.ArtifactStagingDirectory)'
      PackageName: 'Microsoft.ML.OnnxRuntime.*nupkg'
      PlatformsSupported: 'win-x64,linux-x64'
      VerifyNugetSigning: false

  - task: PublishPipelineArtifact@0
    displayName: 'Publish Pipeline NuGet Artifact'
    inputs:
      artifactName: 'drop-signed-nuget-GPU'
      targetPath: '$(Build.ArtifactStagingDirectory)'


  - task: MSBuild@1
    displayName: 'Clean C#'
    inputs:
      solution: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln'
      platform: 'Any CPU'
      configuration: RelWithDebInfo
      msbuildArguments: '-t:Clean -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=Microsoft.ML.OnnxRuntime.Gpu'
      workingDirectory: '$(Build.SourcesDirectory)\csharp'


  - task: RoslynAnalyzers@2
    displayName: 'Run Roslyn Analyzers'
    inputs:
      userProvideBuildInfo: msBuildInfo
      msBuildCommandline: '"C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\msbuild.exe" $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln -p:configuration="RelWithDebInfo" -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=Microsoft.ML.OnnxRuntime.Gpu'
    condition: and(succeeded(), eq('${{ parameters.DoCompliance }}', true))

  - template: templates/component-governance-component-detection-steps.yml
    parameters :
      condition : 'succeeded'

  - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
    displayName: 'Clean Agent Directories'
    condition: always()

- template: nuget/templates/test_win.yml
  parameters:
    AgentPool : 'onnxruntime-gpu-tensorrt8-winbuild'
    NugetPackageName : 'Microsoft.ML.OnnxRuntime.Gpu'
    ArtifactSuffix: 'GPU'
    Skipx86Tests: 'true'

- template: nuget/templates/test_linux.yml
  parameters:
    AgentPool : Onnxruntime-Linux-GPU
    ArtifactSuffix: 'GPU'
    NugetPackageName : 'Microsoft.ML.OnnxRuntime.Gpu'