onnxruntime/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml

parameters:
- name: RunOnnxRuntimeTests
  displayName: Run Tests?
  type: boolean
  default: true

- name: DoCompliance
  displayName: Run Compliance Tasks?
  type: boolean
  default: true

- name: DoEsrp
  displayName: Run code sign tasks? Must be true if you are doing an Onnx Runtime release.
  type: boolean
  default: false

- name: IsReleaseBuild
  displayName: Is a release build? Set it to true if you are doing an Onnx Runtime release.
  type: boolean
  default: false

jobs:
- template: templates/c-api-cpu.yml
  parameters:
    RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }}
    DoCompliance: ${{ parameters.DoCompliance }}
    DoEsrp: ${{ parameters.DoEsrp }}
    IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
    OrtNugetPackageId: 'Microsoft.ML.OnnxRuntime'
    AdditionalBuildFlags: ''
    AdditionalWinBuildFlags: '--enable_onnx_tests --enable_wcos'
    BuildVariant: 'default'

- job: Linux_C_API_Packaging_GPU_x64
  workspace:
    clean: all
  timeoutInMinutes:  120
  pool: 'Onnxruntime-Linux-GPU'
  variables:
    CUDA_VERSION: '11.4'
  steps:
    - template: templates/set-version-number-variables-step.yml
    - template: templates/get-docker-image-steps.yml
      parameters:
        Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11
        Context: tools/ci_build/github/linux/docker
        DockerBuildArgs: "--network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64 --build-arg BASEIMAGE=nvidia/cuda:11.4.0-cudnn8-devel-centos7 --build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-10/root --build-arg PREPEND_PATH=/opt/rh/devtoolset-10/root/usr/bin: --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-10/root/usr/lib64:/opt/rh/devtoolset-10/root/usr/lib:/opt/rh/devtoolset-10/root/usr/lib64/dyninst:/opt/rh/devtoolset-10/root/usr/lib/dyninst:/usr/local/lib64 --build-arg BUILD_UID=$( id -u )"
        Repository: onnxruntimecuda11build
    - task: CmdLine@2
      inputs:
        script: |
          mkdir -p $HOME/.onnx
          docker run --gpus all -e CC=/opt/rh/devtoolset-10/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-10/root/usr/bin/c++ -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \
          --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecuda11build \
          /opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_java --build_dir /build --config Release \
          --skip_submodule_sync  --parallel --build_shared_lib --use_cuda --cuda_version=$(CUDA_VERSION) --cuda_home=/usr/local/cuda-$(CUDA_VERSION) --cudnn_home=/usr/local/cuda-$(CUDA_VERSION) --cmake_extra_defines CMAKE_CUDA_HOST_COMPILER=/opt/rh/devtoolset-10/root/usr/bin/cc 'CMAKE_CUDA_ARCHITECTURES=37;50;52;60;61;70;75;80'          
        workingDirectory: $(Build.SourcesDirectory)

    - template: templates/java-api-artifacts-package-and-publish-steps-posix.yml
      parameters:
          arch: 'linux-x64'
          buildConfig: 'Release'
          artifactName: 'onnxruntime-java-linux-gpu-x64'
          version: '$(OnnxRuntimeVersion)'
          libraryName: 'libonnxruntime.so'
          nativeLibraryName: 'libonnxruntime4j_jni.so'

    - template: templates/c-api-artifacts-package-and-publish-steps-posix.yml
      parameters:
          buildConfig: 'Release'
          artifactName: 'onnxruntime-linux-x64-gpu-$(OnnxRuntimeVersion)'
          artifactNameNoVersionString:  'onnxruntime-linux-x64-gpu'
          libraryName: 'libonnxruntime.so.$(OnnxRuntimeVersion)'
          commitId: $(OnnxRuntimeGitCommitHash)

    - template: templates/component-governance-component-detection-steps.yml
      parameters :
        condition : 'succeeded'
    - template: templates/clean-agent-build-directory-step.yml

- template: linux-gpu-tensorrt-packaging-pipeline.yml

- template: templates/win-ci.yml
  parameters:
    ort_build_pool_name: 'onnxruntime-gpu-winbuild'
    DoCompliance: ${{ parameters.DoCompliance }}
    DoEsrp: ${{ parameters.DoEsrp }}
    job_name_suffix: gpu
    EnvSetupScript: setup_env_gpu.bat
    buildArch: x64
    msbuildPlatform: x64
    packageName: gpu-x64
    buildparameter: --use_cuda --cuda_version=11.1 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.1" --cudnn_home="C:\local\cudnn-11.4-windows-x64-v8.2.2.26\cuda" --enable_onnx_tests --enable_wcos --build_java --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=37;52;60;61;70;75;80"
    runTests: ${{ parameters.RunOnnxRuntimeTests }}
    buildJava: true
    java_artifact_id: onnxruntime_gpu

- template: templates/win-ci.yml
  parameters:
    ort_build_pool_name: 'onnxruntime-gpu-tensorrt8-winbuild'
    DoCompliance: ${{ parameters.DoCompliance }}
    DoEsrp: ${{ parameters.DoEsrp }}
    job_name_suffix: tensorrt 
    EnvSetupScript: setup_env_gpu.bat
    buildArch: x64
    msbuildPlatform: x64
    packageName: tensorrt-x64
    buildparameter: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.0.1.6.Windows10.x86_64.cuda-11.3.cudnn8.2" --cuda_version=11.1 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.1" --cudnn_home="C:\local\cudnn-11.4-windows-x64-v8.2.2.26\cuda" --enable_onnx_tests --enable_wcos --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=37;52;60;61;70;75;80"
    runTests: ${{ parameters.RunOnnxRuntimeTests }}
    buildJava: false 

- template: templates/android-java-api-aar.yml
  parameters:
    buildConfig: 'Release'

- job: Jar_Packaging_GPU
  workspace:
    clean: all
  pool: 'Win-CPU-2021'
  dependsOn:
  - Linux_C_API_Packaging_GPU_x64
  - Windows_Packaging_gpu
  condition: succeeded()
  steps:
  - checkout: self
    submodules: false
  - template: templates/set-version-number-variables-step.yml

  - task: DownloadPipelineArtifact@2
    displayName: 'Download Pipeline Artifact - Win x64'
    inputs:
      buildType: 'current'
      artifactName: 'drop-onnxruntime-java-win-gpu-x64'
      targetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64'

  - task: DownloadPipelineArtifact@2
    displayName: 'Download Pipeline Artifact - Linux x64'
    inputs:
      buildType: 'current'
      artifactName: 'drop-onnxruntime-java-linux-gpu-x64'
      targetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-linux-x64'

  - task: PowerShell@2
    displayName: 'PowerShell Script'
    inputs:
      targetType: filePath
      filePath: $(Build.SourcesDirectory)\tools\ci_build\github\windows\jar_gpu_packaging.ps1
      failOnStderr: true
      showWarnings: true
      workingDirectory: '$(Build.BinariesDirectory)\java-artifact'



  - task: CopyFiles@2
    displayName: 'Copy Java Files to Artifact Staging Directory'
    inputs:
      SourceFolder: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64'
      TargetFolder: '$(Build.ArtifactStagingDirectory)'

  - task: PublishPipelineArtifact@1
    displayName: 'Publish Pipeline Artifact'
    inputs:
      targetPath: '$(Build.ArtifactStagingDirectory)'
      artifact: 'onnxruntime-java-gpu'

  - template: templates/component-governance-component-detection-steps.yml
    parameters :
      condition : 'succeeded'
  - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
    displayName: 'Clean Agent Directories'
    condition: always()


- job: Final_Jar_Testing_Windows_GPU
  workspace:
    clean: all
  pool: 'onnxruntime-gpu-winbuild'
  timeoutInMinutes: 60
  variables:
  - name: runCodesignValidationInjection
    value: false
  dependsOn:
    Jar_Packaging_GPU
  steps:
  - template: templates/set-version-number-variables-step.yml

  - task: BatchScript@1
    displayName: 'setup env'
    inputs:
        filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\setup_env_cuda_11.bat'
        modifyEnvironment: true
        workingFolder: '$(Build.BinariesDirectory)'

  - task: DownloadPipelineArtifact@2
    displayName: 'Download Final Jar'
    inputs:
        buildType: 'current'
        artifactName: 'onnxruntime-java-gpu'
        targetPath: '$(Build.BinariesDirectory)\final-jar'

  - task: CmdLine@2
    inputs:
      script: |
        mkdir test
        pushd test
        jar xf $(Build.BinariesDirectory)\final-jar\testing.jar
        popd
        powershell -Command "Invoke-WebRequest https://oss.sonatype.org/service/local/repositories/releases/content/org/junit/platform/junit-platform-console-standalone/1.6.2/junit-platform-console-standalone-1.6.2.jar -OutFile junit-platform-console-standalone-1.6.2.jar"
        powershell -Command "Invoke-WebRequest https://oss.sonatype.org/service/local/repositories/google-releases/content/com/google/protobuf/protobuf-java/3.9.2/protobuf-java-3.9.2.jar -OutFile protobuf-java-3.9.2.jar"
        java -DUSE_CUDA=1 -jar junit-platform-console-standalone-1.6.2.jar -cp .;.\test;protobuf-java-3.9.2.jar;onnxruntime_gpu-$(OnnxRuntimeVersion).jar --scan-class-path --fail-if-no-tests --disable-banner        
      workingDirectory: '$(Build.BinariesDirectory)\final-jar'

  - template: templates/component-governance-component-detection-steps.yml
    parameters :
      condition : 'succeeded'
  - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
    displayName: 'Clean Agent Directories'
    condition: always()

- job: Final_Jar_Testing_Linux_GPU
  workspace:
    clean: all
  pool: 'Onnxruntime-Linux-GPU'
  variables:
  - name: runCodesignValidationInjection
    value: false
  timeoutInMinutes: 60
  dependsOn:
    Jar_Packaging_GPU
  steps:
  - checkout: self
    submodules: false
  - template: templates/set-version-number-variables-step.yml
  - task: DownloadPipelineArtifact@2
    displayName: 'Download Final Jar'
    inputs:
      buildType: 'current'
      artifactName: 'onnxruntime-java-gpu'
      targetPath: '$(Build.BinariesDirectory)/final-jar'

  - task: Bash@3
    displayName: 'Test'
    inputs:
      targetType: filePath
      filePath: 'tools/ci_build/github/linux/java_linux_final_test.sh'
      arguments: '-r $(Build.BinariesDirectory) -v $(OnnxRuntimeVersion)'

  - template: templates/component-governance-component-detection-steps.yml
    parameters :
      condition : 'succeeded'
  - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
    displayName: 'Clean Agent Directories'
    condition: always()



- job: NuGet_Packaging_GPU
  workspace:
    clean: all
  pool: 'Win-CPU-2021'
  variables:
    breakCodesignValidationInjection: ${{ parameters.DoEsrp }}
  dependsOn:
  - Windows_Packaging_gpu
  - Windows_Packaging_tensorrt
  - Linux_C_API_Packaging_GPU_x64
  - Linux_C_API_Packaging_GPU_TensorRT_x64
  condition: succeeded()
  steps:
  - checkout: self
    submodules: true
  - task: DownloadPipelineArtifact@2
    displayName: 'Download Pipeline Artifact - NuGet'
    inputs:
      artifactName: 'onnxruntime-win-gpu-x64'
      targetPath: '$(Build.BinariesDirectory)/nuget-artifact'

  - task: DownloadPipelineArtifact@2
    displayName: 'Download Pipeline Artifact - NuGet'
    inputs:
      artifactName: 'onnxruntime-win-tensorrt-x64'
      targetPath: '$(Build.BinariesDirectory)/nuget-artifact'

  - task: DownloadPipelineArtifact@2
    displayName: 'Download Pipeline Artifact - NuGet'
    inputs:
      artifactName: 'onnxruntime-linux-x64-gpu'
      targetPath: '$(Build.BinariesDirectory)/nuget-artifact'

  - task: DownloadPipelineArtifact@2
    displayName: 'Download Pipeline Artifact - NuGet'
    inputs:
      artifactName: 'onnxruntime-linux-x64-gpu-tensorrt'
      targetPath: '$(Build.BinariesDirectory)/nuget-artifact'

  - task: DownloadPipelineArtifact@2
    displayName: 'Download Pipeline Artifact - NuGet'
    inputs:
      artifactName: 'drop-extra'
      targetPath: '$(Build.BinariesDirectory)/extra-artifact'

  #Reconstruct the build dir
  - task: PowerShell@2
    displayName: 'PowerShell Script'
    inputs:
      targetType: filePath
      filePath: $(Build.SourcesDirectory)\tools\ci_build\github\windows\extract_nuget_files_gpu.ps1

  - script: |
     dir     
    workingDirectory: '$(Build.BinariesDirectory)/nuget-artifact'
    displayName: 'List artifacts'

  - script: |
     mklink /D /J models C:\local\models     
    workingDirectory: '$(Build.BinariesDirectory)'
    displayName: 'Create models link'

  - task: NuGetToolInstaller@0
    displayName: Use Nuget 5.7.0
    inputs:
      versionSpec: 5.7.0

  - task: DotNetCoreCLI@2
    displayName: 'Restore nuget packages'
    inputs:
      command: restore
      projects: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln'
      configuration: RelWithDebInfo
      arguments: '--configuration RelWithDebInfo -p:Platform="Any CPU" -p:OrtPackageId="Microsoft.ML.OnnxRuntime.Gpu" -p:IsReleaseBuild=${{ parameters.IsReleaseBuild }}'
      workingDirectory: '$(Build.SourcesDirectory)\csharp'

  - task: DotNetCoreCLI@2
    displayName: 'Build C#'
    inputs:
      command: build
      projects: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln'
      configuration: RelWithDebInfo
      arguments: '--configuration RelWithDebInfo -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId="Microsoft.ML.OnnxRuntime.Gpu" -p:IsReleaseBuild=${{ parameters.IsReleaseBuild }}'
      workingDirectory: '$(Build.SourcesDirectory)\csharp'

  - template: templates/win-esrp-dll.yml
    parameters:
      FolderPath: '$(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo'
      DisplayName: 'ESRP - Sign C# dlls'
      DoEsrp: ${{ parameters.DoEsrp }}

  - task: DotNetCoreCLI@2
    displayName: 'Build Nuget Packages'
    inputs:
      command: build
      projects: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj'
      configuration: RelWithDebInfo
      arguments: '--configuration RelWithDebInfo -t:CreatePackage -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=Microsoft.ML.OnnxRuntime.Gpu -p:IsReleaseBuild=${{ parameters.IsReleaseBuild }}'
      workingDirectory: '$(Build.SourcesDirectory)\csharp'

  - task: BatchScript@1
    displayName: 'Bundle Native NuGet and other binaries'
    inputs:
      filename: $(Build.SourcesDirectory)\tools\ci_build\github\windows\bundle_dlls_gpu.bat
      workingFolder: $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\nuget-artifacts

  - task: CopyFiles@2
    displayName: 'Copy nuget packages to: $(Build.ArtifactStagingDirectory)'
    inputs:
      SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\nuget-artifacts'
      Contents: '*.nupkg'
      TargetFolder: '$(Build.ArtifactStagingDirectory)'

  - task: CopyFiles@2
    displayName: 'Copy nuget packages to: $(Build.ArtifactStagingDirectory)'
    inputs:
      SourceFolder: '$(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo'
      Contents: '*.nupkg'
      TargetFolder: '$(Build.ArtifactStagingDirectory)'

  - template: templates/esrp_nuget.yml
    parameters:
      DisplayName: 'ESRP - sign NuGet package'
      FolderPath: '$(Build.ArtifactStagingDirectory)'
      DoEsrp: ${{ parameters.DoEsrp }}

  - template: templates/validate-nuget.yml
    parameters:
      NugetPath: '$(Build.ArtifactStagingDirectory)'
      NugetPackage: 'Microsoft.ML.OnnxRuntime.*nupkg'
      PlatformsSupported: 'win-x64,linux-x64'
      VerifyNugetSigning: false

  - task: PublishPipelineArtifact@0
    displayName: 'Publish Pipeline NuGet Artifact'
    inputs:
      artifactName: 'drop-signed-nuget-GPU'
      targetPath: '$(Build.ArtifactStagingDirectory)'

  - powershell: |
     $counter = 0
     Get-ChildItem -Path $(Build.ArtifactStagingDirectory)/*.nupkg | foreach {7z x -y -o$counter $_.fullname;$counter++}     
    workingDirectory: '$(Build.ArtifactStagingDirectory)'
    displayName: 'Unzip files to sanitize the PE files'

  - task: DotNetCoreCLI@2
    displayName: 'Build C#'
    inputs:
      command: custom
      custom: clean
      projects: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln'
      configuration: RelWithDebInfo
      arguments: '--configuration RelWithDebInfo -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=Microsoft.ML.OnnxRuntime.Gpu'
      workingDirectory: '$(Build.SourcesDirectory)\csharp'

  #TODO: enable binskim
  #- task: BinSkim@4
  #  displayName: 'Run BinSkim'

  - task: RoslynAnalyzers@2
    displayName: 'Run Roslyn Analyzers'
    inputs:
      userProvideBuildInfo: msBuildInfo
      msBuildArchitecture: DotNetCore
      msBuildCommandline: '"C:\Program Files\dotnet\dotnet.exe" build $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln --configuration RelWithDebInfo -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=Microsoft.ML.OnnxRuntime.Gpu'
    condition: and(succeeded(), eq('${{ parameters.DoCompliance }}', true))

  - template: templates/component-governance-component-detection-steps.yml
    parameters :
      condition : 'succeeded'

  - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
    displayName: 'Clean Agent Directories'
    condition: always()
    
- template: nuget/templates/test_win.yml
  parameters:
    AgentPool : 'onnxruntime-gpu-tensorrt8-winbuild'
    NugetPackageName : 'Microsoft.ML.OnnxRuntime.Gpu'
    ArtifactSuffix: 'GPU'
    Skipx86Tests: 'true'
    
- template: nuget/templates/test_linux.yml
  parameters:
    AgentPool : Onnxruntime-Linux-GPU
    ArtifactSuffix: 'GPU'
    NugetPackageName : 'Microsoft.ML.OnnxRuntime.Gpu'