onnxruntime/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml

parameters:
- name: RunOnnxRuntimeTests
  displayName: Run Tests?
  type: boolean
  default: true

- name: UseIncreasedTimeoutForTests
  displayName: Increase timeout for tests? Set it to false if you are doing an Onnx Runtime release.
  type: boolean
  default: false

- name: DoCompliance
  displayName: Run Compliance Tasks?
  type: boolean
  default: true

- name: DoEsrp
  displayName: Run code sign tasks? Must be true if you are doing an ONNX Runtime release
  type: boolean
  default: true

- name: IsReleaseBuild
  displayName: Is a release build? Set it to true if you are doing an ONNX Runtime release.
  type: boolean
  default: false

- name: PreReleaseVersionSuffixString
  displayName: Suffix added to pre-release package version. Only used if IsReleaseBuild is true. Denotes the type of pre-release package.
  type: string
  values:
  - alpha
  - beta
  - rc
  - none
  default: none

- name: PreReleaseVersionSuffixNumber
  displayName: Number added to pre-release package version. Only used if IsReleaseBuild is true. Denotes the sequence of a pre-release package.
  type: number
  default: 0

# these 2 parameters are used for debugging.
- name: SpecificArtifact
  displayName: Use Specific Artifact (Debugging only)
  type: boolean
  default: false

- name: BuildId
  displayName: Pipeline BuildId, you could find it in the URL
  type: string
  default: '0'

- name: NugetPackageSuffix
  displayName: Suffix to append to nuget package
  type: string
  default: 'NONE'

- name: AdditionalBuildFlag
  displayName: Build flags to append to build command
  type: string
  default: '--use_azure'
- name: QnnSdk
  displayName: QNN SDK Version
  type: string
  default: 2.23.0.240531

resources:
  repositories:
  - repository: onnxruntime-inference-examples # The name used to reference this repository in the checkout step
    type: github
    endpoint: ort-examples
    name: microsoft/onnxruntime-inference-examples
  - repository: manylinux
    type: Github
    endpoint: Microsoft
    name: pypa/manylinux
    ref: 5eda9aded5462201e6310105728d33016e637ea7

variables:
- name: ReleaseVersionSuffix
  value: ''
- name: win_trt_version
  value: 11.8

- name: win_trt_home
  value: $(Agent.TempDirectory)\TensorRT-10.0.1.6.Windows10.x86_64.cuda-11.8
- name: win_cuda_home
  value: $(Agent.TempDirectory)\v11.8

stages:
- template: stages/set_packaging_variables_stage.yml
  parameters:
    IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
    PreReleaseVersionSuffixString: ${{ parameters.PreReleaseVersionSuffixString }}
    PreReleaseVersionSuffixNumber: ${{ parameters.PreReleaseVersionSuffixNumber }}

- stage: Debug
  dependsOn: Setup
  jobs:
  - job: D1
    pool:
      name: 'onnxruntime-Ubuntu2204-AMD-CPU'
    variables:
      MyVar: $[stageDependencies.Setup.Set_Variables.outputs['Set_Release_Version_Suffix.ReleaseVersionSuffix']]
      BuildDate: $[stageDependencies.Setup.Set_Variables.outputs['Set_Build_Date.BuildDate']]
      BuildTime: $[stageDependencies.Setup.Set_Variables.outputs['Set_Build_Time.BuildTime']]
    steps:
    - checkout: none
    - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
      displayName: 'Clean Agent Directories'
      condition: always()
    - bash: echo $(MyVar)
    - bash: echo $(BuildTime)
    - bash: echo $(BuildDate)
    - template: templates/component-governance-component-detection-steps.yml
      parameters :
        condition : 'succeeded'

- template: stages/download-java-tools-stage.yml

- template: templates/c-api-cpu.yml
  parameters:
    RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }}
    DoCompliance: ${{ parameters.DoCompliance }}
    DoEsrp: ${{ parameters.DoEsrp }}
    IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
    ${{ if eq(parameters.NugetPackageSuffix, 'NONE') }}:
      OrtNugetPackageId: 'Microsoft.ML.OnnxRuntime'
    ${{ else }}:
      OrtNugetPackageId: 'Microsoft.ML.OnnxRuntime${{ parameters.NugetPackageSuffix }}'
    AdditionalBuildFlags: ''
    AdditionalWinBuildFlags: '--enable_onnx_tests --enable_wcos ${{parameters.AdditionalBuildFlag}}'
    BuildVariant: 'default'
    SpecificArtifact: ${{ parameters.SpecificArtifact }}
    BuildId: ${{ parameters.BuildId }}

- template: templates/ondevice-training-cpu-packaging-pipeline.yml
  parameters:
    RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }}
    DoCompliance: ${{ parameters.DoCompliance }}
    DoEsrp: ${{ parameters.DoEsrp }}
    IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
    OrtNugetPackageId: 'Microsoft.ML.OnnxRuntime.Training'
    AdditionalBuildFlags: '--enable_training_apis'
    AdditionalWinBuildFlags: '--enable_onnx_tests --enable_wcos'
    BuildVariant: 'default'

- template: stages/java-cuda-packaging-stage.yml
  parameters:
    CudaVersion: 11.8
    SpecificArtifact: ${{ parameters.SpecificArtifact }}
    BuildId: ${{ parameters.BuildId }}

- template: stages/nuget-combine-cuda-stage.yml
  parameters:
      DoCompliance: ${{ parameters.DoCompliance }}
      CudaVersion: 11.8
      docker_base_image: 'nvidia/cuda:11.8.0-cudnn8-devel-ubi8'
      RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }}
      UseIncreasedTimeoutForTests: ${{ parameters.UseIncreasedTimeoutForTests }}
      win_trt_home: ${{ variables.win_trt_home }}
      win_cuda_home: ${{ variables.win_cuda_home }}
      DoEsrp: ${{ parameters.DoEsrp }}
      IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
      buildJava: true
      buildNodejs: true
      SpecificArtifact: ${{ parameters.SpecificArtifact }}
      BuildId: ${{ parameters.BuildId }}

# ROCm
- stage: Linux_C_API_Packaging_ROCm_x64
  dependsOn: []
  jobs:
  - job: Linux_C_API_Packaging_ROCm_x64
    workspace:
      clean: all
    timeoutInMinutes: 240
    pool: onnxruntime-Ubuntu2204-AMD-CPU
    variables:
      RocmVersion: '5.6'
      RocmVersionPatchSuffix: ''
    steps:
    - checkout: self                           # due to checkout multiple repos, the root directory is $(Build.SourcesDirectory)/onnxruntime
      submodules: recursive
    - checkout: manylinux                      # due to checkout multiple repos, the root directory is $(Build.SourcesDirectory)/manylinux, for get-docker-image-steps.yml
      submodules: false

    # get-docker-image-steps.yml will move the $(Build.SourcesDirectory)/manylinux into $(Build.SourcesDirectory)/onnxruntime,
    # then rename $(Build.SourcesDirectory)/onnxruntime as $(Build.SourcesDirectory)
    - template: templates/get-docker-image-steps.yml
      parameters:
        Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_rocm
        Context: tools/ci_build/github/linux/docker
        DockerBuildArgs: >-
          --build-arg INSTALL_DEPS_EXTRA_ARGS=-tmur
          --build-arg BUILD_UID=$(id -u)
          --network=host --build-arg POLICY=manylinux_2_28 --build-arg PLATFORM=x86_64
          --build-arg ROCM_VERSION=$(RocmVersion)$(RocmVersionPatchSuffix)
          --build-arg DEVTOOLSET_ROOTPATH=/opt/rh/gcc-toolset-12/root
          --build-arg PREPEND_PATH=/opt/rh/gcc-toolset-12/root/usr/bin:
          --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/gcc-toolset-12/root/usr/lib64:/opt/rh/gcc-toolset-12/root/usr/lib:/opt/rh/gcc-toolset-12/root/usr/lib64/dyninst:/opt/rh/gcc-toolset-12/root/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib          
        Repository: onnxruntimetrainingrocmbuild-rocm$(RocmVersion)
        CheckOutManyLinux: true

    - template: templates/set-version-number-variables-step.yml

    - task: Bash@3
      displayName: 'Build'
      inputs:
        targetType: filePath
        filePath: tools/ci_build/github/linux/build_rocm_c_api_package.sh
        arguments: >-
          -S $(Build.SourcesDirectory)
          -B $(Build.BinariesDirectory)
          -V $(RocmVersion)
          -I onnxruntimetrainingrocmbuild-rocm$(RocmVersion)
          -P python3.10          

    - script: |
       set -e -x
       mkdir $(Build.ArtifactStagingDirectory)/testdata
       cp $(Build.BinariesDirectory)/Release/libcustom_op_library.so* $(Build.ArtifactStagingDirectory)/testdata
       ls -al $(Build.ArtifactStagingDirectory)       
      displayName: 'Create Artifacts for CustomOp'  # libcustom_op_library.so from cpu build is built with fp8, ROCm does not support it.

    - template: templates/c-api-artifacts-package-and-publish-steps-posix.yml
      parameters:
        buildConfig: 'Release'
        artifactName: 'onnxruntime-linux-x64-rocm-$(OnnxRuntimeVersion)'
        artifactNameNoVersionString: 'onnxruntime-linux-x64-rocm'
        libraryName: 'libonnxruntime.so.$(OnnxRuntimeVersion)'

    - template: templates/component-governance-component-detection-steps.yml
      parameters:
        condition: 'succeeded'
    - template: templates/clean-agent-build-directory-step.yml


- stage: NuGet_Packaging_ROCm
  dependsOn:
  - Setup
  - Linux_C_API_Packaging_ROCm_x64
  condition: succeeded()
  jobs:
  - job: NuGet_Packaging_ROCm
    workspace:
      clean: all
    # we need to use a 2022 pool to create the nuget package with MAUI targets.
    # VS2019 has no support for net6/MAUI and we need to use msbuild (from the VS install) to do the packing
    pool: 'Onnxruntime-Win-CPU-2022'
    variables:
      breakCodesignValidationInjection: ${{ parameters.DoEsrp }}
      ReleaseVersionSuffix: $[stageDependencies.Setup.Set_Variables.outputs['Set_Release_Version_Suffix.ReleaseVersionSuffix']]
      BuildDate : $[stageDependencies.Setup.Set_Variables.outputs['Set_Build_Date.BuildDate']]
      BuildTime : $[stageDependencies.Setup.Set_Variables.outputs['Set_Build_Time.BuildTime']]

    steps:
    - checkout: self
      submodules: true
      fetchDepth: 1

    - template: templates/flex-downloadPipelineArtifact.yml
      parameters:
        StepName: 'Download Pipeline Artifact - NuGet'
        ArtifactName: 'onnxruntime-linux-x64-rocm'
        targetPath: '$(Build.BinariesDirectory)/nuget-artifact'
        SpecificArtifact: ${{ parameters.specificArtifact }}
        BuildId: ${{ parameters.BuildId }}

    - task: PowerShell@2
      displayName: 'Reconstruct Build Directory'
      inputs:
        targetType: inline
        script: |
          Get-ChildItem $(Build.BinariesDirectory)\nuget-artifact -Filter *.tgz | % {
            # *.tar will be created after *.tgz is extracted
            $cmd = "7z.exe x $($_.FullName) -y -o$(Build.BinariesDirectory)\nuget-artifact"
            Write-Output $cmd
            Invoke-Expression -Command $cmd
          }

          Get-ChildItem $(Build.BinariesDirectory)\nuget-artifact -Filter *.tar | % {
            $cmd = "7z.exe x $($_.FullName) -y -o$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\nuget-artifacts"
            Write-Output $cmd
            Invoke-Expression -Command $cmd
          }

          $ort_dirs = Get-ChildItem -Path $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\nuget-artifacts\onnxruntime-* -Directory
          foreach ($ort_dir in $ort_dirs)
          {
            $dirname = Split-Path -Path $ort_dir -Leaf
            $dirname = $dirname.SubString(0, $dirname.LastIndexOf('-'))
            Write-Output "Renaming $ort_dir to $dirname"
            Rename-Item -Path $ort_dir -NewName $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\nuget-artifacts\$dirname
          }

          Copy-Item -Path $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\nuget-artifacts\onnxruntime-linux-x64-rocm\lib\* -Destination $(Build.BinariesDirectory)\RelWithDebInfo          

    - script: |
       tree /F       
      workingDirectory: '$(Build.BinariesDirectory)'
      displayName: 'Inspect Build Binaries Directory'

    - script: |
       mklink /D /J models C:\local\models       
      workingDirectory: '$(Build.BinariesDirectory)'
      displayName: 'Create models link'

    - task: NuGetToolInstaller@0
      displayName: Use Nuget 6.2.1
      inputs:
        versionSpec: 6.2.1

    - task: MSBuild@1
      displayName: 'Restore NuGet Packages and create project.assets.json'
      inputs:
        solution: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln'
        platform: 'Any CPU'
        configuration: RelWithDebInfo
        msbuildArguments: '-t:restore -p:OrtPackageId="Microsoft.ML.OnnxRuntime.ROCm"'
        workingDirectory: '$(Build.SourcesDirectory)\csharp'

    - task: MSBuild@1
      displayName: 'Build C# bindings'
      inputs:
        solution: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln'
        platform: 'Any CPU'
        configuration: RelWithDebInfo
        msbuildArguments: >
          -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)"
          -p:OrtPackageId="Microsoft.ML.OnnxRuntime.ROCm"
          -p:IsReleaseBuild=${{ parameters.IsReleaseBuild }}
          -p:ReleaseVersionSuffix=$(ReleaseVersionSuffix)
          -p:IsLinuxBuild=true
          -p:IsWindowsBuild=false
          -p:IsMacOSBuild=false          
        workingDirectory: '$(Build.SourcesDirectory)\csharp'

    - template: templates/win-esrp-dll.yml
      parameters:
        FolderPath: '$(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo'
        DisplayName: 'ESRP - Sign C# dlls'
        DoEsrp: ${{ parameters.DoEsrp }}

    - task: UsePythonVersion@0
      displayName: 'Use Python'
      inputs:
        versionSpec: 3.8

    - task: MSBuild@1
      displayName: 'Build Nuget Packages'
      inputs:
        solution: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj'
        configuration: RelWithDebInfo
        platform: 'Any CPU'
        msbuildArguments: >
          -t:CreatePackage
          -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)"
          -p:OrtPackageId=Microsoft.ML.OnnxRuntime.ROCm
          -p:IsReleaseBuild=${{ parameters.IsReleaseBuild }}
          -p:ReleaseVersionSuffix=$(ReleaseVersionSuffix)
          -p:CurrentTime=$(BuildTime)
          -p:CurrentDate=$(BuildDate)          
        workingDirectory: '$(Build.SourcesDirectory)\csharp'

    - task: CopyFiles@2
      displayName: 'Copy nuget packages to: $(Build.ArtifactStagingDirectory)'
      inputs:
        SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo'
        Contents: '*.snupkg'
        TargetFolder: '$(Build.ArtifactStagingDirectory)'

    - task: CopyFiles@2
      displayName: 'Copy nuget packages to: $(Build.ArtifactStagingDirectory)'
      inputs:
        SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo'
        Contents: '*.nupkg'
        TargetFolder: '$(Build.ArtifactStagingDirectory)'

    - task: CopyFiles@2
      displayName: 'Copy nuget packages to: $(Build.ArtifactStagingDirectory)'
      inputs:
        SourceFolder: '$(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo'
        Contents: '*.nupkg'
        TargetFolder: '$(Build.ArtifactStagingDirectory)'

    - template: templates/esrp_nuget.yml
      parameters:
        DisplayName: 'ESRP - sign NuGet package'
        FolderPath: '$(Build.ArtifactStagingDirectory)'
        DoEsrp: ${{ parameters.DoEsrp }}

    - template: templates/validate-package.yml
      parameters:
        PackageType: 'nuget'
        PackagePath: '$(Build.ArtifactStagingDirectory)'
        PackageName: 'Microsoft.ML.OnnxRuntime.*nupkg'
        PlatformsSupported: 'linux-x64'
        VerifyNugetSigning: false

    - task: PublishPipelineArtifact@0
      displayName: 'Publish Pipeline NuGet Artifact'
      inputs:
        artifactName: 'drop-signed-nuget-ROCm'
        targetPath: '$(Build.ArtifactStagingDirectory)'

    - task: MSBuild@1
      displayName: 'Clean C#'
      inputs:
        solution: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln'
        platform: 'Any CPU'
        configuration: RelWithDebInfo
        msbuildArguments: '-t:Clean -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=Microsoft.ML.OnnxRuntime.ROCm'
        workingDirectory: '$(Build.SourcesDirectory)\csharp'

    - template: templates/component-governance-component-detection-steps.yml
      parameters :
        condition : 'succeeded'

    - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
      displayName: 'Clean Agent Directories'
      condition: always()

- template: nuget/templates/test_linux.yml
  parameters:
    AgentPool: AMD-GPU
    ArtifactSuffix: 'ROCm'
    StageSuffix: 'ROCm'
    NugetPackageName: 'Microsoft.ML.OnnxRuntime.ROCm'
    SpecificArtifact: ${{ parameters.specificArtifact }}
    CustomOpArtifactName: 'onnxruntime-linux-x64-rocm'
    BuildId: ${{ parameters.BuildId }}

- template: nuget/templates/dml-vs-2022.yml
  parameters:
    AgentPool: 'onnxruntime-Win2022-GPU-dml-A10'
    IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
    ArtifactName: 'drop-nuget-dml'
    StageName: 'Windows_CI_GPU_DML_Dev'
    BuildCommand: --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos --use_telemetry --use_dml --use_winml --build_nodejs --cmake_generator "Visual Studio 17 2022"
    BuildArch: 'x64'
    msbuildArchitecture: 'amd64'
    EnvSetupScript: 'setup_env.bat'
    sln_platform: 'x64'
    DoDebugBuild: 'false'
    DoNugetPack: 'true'
    DoCompliance: 'false'
    DoEsrp: ${{ parameters.DoEsrp }}
    NuPackScript: |
     msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /t:CreatePackage /p:OrtPackageId=Microsoft.ML.OnnxRuntime.DirectML /p:IsReleaseBuild=${{ parameters.IsReleaseBuild }} /p:CurrentData=$(BuildDate) /p:CurrentTime=$(BuildTime)
     copy $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory)
     copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory)
     mkdir $(Build.ArtifactStagingDirectory)\testdata
     copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\custom_op_library.* $(Build.ArtifactStagingDirectory)\testdata     

- template: nuget/templates/dml-vs-2022.yml
  parameters:
    AgentPool: 'onnxruntime-Win2022-GPU-dml-A10'
    IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
    ArtifactName: 'drop-win-dml-x86-zip'
    StageName: 'Windows_CI_GPU_DML_Dev_x86'
    BuildCommand: --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos --use_telemetry --use_dml --use_winml --cmake_generator "Visual Studio 17 2022"
    BuildArch: 'x86'
    EnvSetupScript: 'setup_env_x86.bat'
    sln_platform: 'Win32'
    DoDebugBuild: 'false'
    DoNugetPack: 'true'
    DoCompliance: ${{ parameters.DoCompliance }}
    DoEsrp: ${{ parameters.DoEsrp }}
    RunTests: 'false'
    NuPackScript: |
     msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /p:TargetArchitecture=x86 /t:CreatePackage /p:OrtPackageId=Microsoft.ML.OnnxRuntime.DirectML /p:IsReleaseBuild=${{ parameters.IsReleaseBuild }}
     cd $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\
     ren Microsoft.ML.OnnxRuntime.DirectML.* win-dml-x86.zip
     copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\win-dml-x86.zip $(Build.ArtifactStagingDirectory)
     mkdir $(Build.ArtifactStagingDirectory)\testdata
     copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\custom_op_library.* $(Build.ArtifactStagingDirectory)\testdata     

- template: nuget/templates/dml-vs-2022.yml
  parameters:
    AgentPool: 'onnxruntime-Win2022-GPU-dml-A10'
    IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
    ArtifactName: 'drop-win-dml-arm64-zip'
    StageName: 'Windows_CI_GPU_DML_Dev_arm64'
    BuildCommand: --build_dir $(Build.BinariesDirectory) --arm64 --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos --use_telemetry --use_dml --use_winml --build_nodejs --cmake_generator "Visual Studio 17 2022"
    BuildArch: 'x64'
    EnvSetupScript: 'setup_env.bat'
    sln_platform: 'arm64'
    DoDebugBuild: 'false'
    DoNugetPack: 'true'
    DoCompliance: ${{ parameters.DoCompliance }}
    DoEsrp: ${{ parameters.DoEsrp }}
    RunTests: 'false'
    NuPackScript: |
     msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /p:TargetArchitecture=arm64 /t:CreatePackage /p:OrtPackageId=Microsoft.ML.OnnxRuntime.DirectML /p:IsReleaseBuild=${{ parameters.IsReleaseBuild }}
     cd $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\
     ren Microsoft.ML.OnnxRuntime.DirectML.* win-dml-arm64.zip
     copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\win-dml-arm64.zip $(Build.ArtifactStagingDirectory)
     mkdir $(Build.ArtifactStagingDirectory)\testdata
     copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\custom_op_library.* $(Build.ArtifactStagingDirectory)\testdata     

- stage: NuGet_Packaging_DML
  dependsOn:
  - Windows_CI_GPU_DML_Dev
  - Windows_CI_GPU_DML_Dev_x86
  - Windows_CI_GPU_DML_Dev_arm64
  condition: succeeded()
  jobs:
  - job: NuGet_Packaging_DML
    workspace:
      clean: all
    pool: 'onnxruntime-Win2022-GPU-dml-A10'
    steps:
    - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
      displayName: 'Clean Agent Directories'
      condition: always()
    - task: DownloadPipelineArtifact@0
      displayName: 'Download Pipeline Artifact - NuGet DirectML'
      inputs:
        artifactName: 'drop-nuget-dml'
        targetPath: '$(Build.BinariesDirectory)/nuget-artifact-dml'

    - task: DownloadPipelineArtifact@0
      displayName: 'Download Pipeline Artifact - NuGet DirectML x86'
      inputs:
        artifactName: 'drop-win-dml-x86-zip'
        targetPath: '$(Build.BinariesDirectory)/nuget-artifact-dml'

    - task: DownloadPipelineArtifact@0
      displayName: 'Download Pipeline Artifact - NuGet DirectML arm64'
      inputs:
        artifactName: 'drop-win-dml-arm64-zip'
        targetPath: '$(Build.BinariesDirectory)/nuget-artifact-dml'

    - script: |
        pushd $(Build.BinariesDirectory)\nuget-artifact-dml
        dir
        powershell -Command "Invoke-WebRequest http://stahlworks.com/dev/unzip.exe -OutFile unzip.exe"
        powershell -Command "Invoke-WebRequest http://stahlworks.com/dev/zip.exe -OutFile zip.exe"
        set PATH=%CD%;%PATH%
        SETLOCAL EnableDelayedExpansion
        FOR /R %%i IN (*.nupkg) do (
            set filename=%%~ni
            IF NOT "!filename:~25,7!"=="Managed" (
                rename %%~ni.nupkg %%~ni.zip
                unzip %%~ni.zip -d %%~ni
                del /Q %%~ni.zip

                unzip win-dml-x86.zip -d win-x86
                mkdir %%~ni\runtimes\win-x86
                mkdir %%~ni\runtimes\win-x86\native

                move win-x86\runtimes\win-x86\native\onnxruntime.dll %%~ni\runtimes\win-x86\native\onnxruntime.dll
                move win-x86\runtimes\win-x86\native\onnxruntime.lib %%~ni\runtimes\win-x86\native\onnxruntime.lib
                move win-x86\runtimes\win-x86\native\onnxruntime.pdb %%~ni\runtimes\win-x86\native\onnxruntime.pdb

                unzip win-dml-arm64.zip -d win-arm64
                mkdir %%~ni\runtimes\win-arm64
                mkdir %%~ni\runtimes\win-arm64\native

                move win-arm64\runtimes\win-arm64\native\onnxruntime.dll %%~ni\runtimes\win-arm64\native\onnxruntime.dll
                move win-arm64\runtimes\win-arm64\native\onnxruntime.lib %%~ni\runtimes\win-arm64\native\onnxruntime.lib
                move win-arm64\runtimes\win-arm64\native\onnxruntime.pdb %%~ni\runtimes\win-arm64\native\onnxruntime.pdb


                pushd %%~ni
                zip -r ..\%%~ni.zip .
                popd
                move %%~ni.zip %%~ni.nupkg
            )
        )
        popd
        copy $(Build.BinariesDirectory)\nuget-artifact-dml\Microsoft.ML.OnnxRuntime.DirectML*nupkg $(Build.ArtifactStagingDirectory)        
      displayName: 'Bundle DML NuGet and other binaries'

    - template: templates/esrp_nuget.yml
      parameters:
        DisplayName: 'ESRP - sign NuGet package'
        FolderPath: '$(Build.ArtifactStagingDirectory)'
        DoEsrp: ${{ parameters.DoEsrp }}

    - template: templates/validate-package.yml
      parameters:
        PackageType: 'nuget'
        PackagePath: '$(Build.ArtifactStagingDirectory)'
        PackageName: 'Microsoft.ML.OnnxRuntime.DirectML*nupkg'
        PlatformsSupported: 'win-x64,win-x86,win-arm64'
        VerifyNugetSigning: ${{ parameters.DoEsrp }}

    - task: PublishPipelineArtifact@0
      displayName: 'Publish Pipeline NuGet Artifact'
      inputs:
        artifactName: 'drop-signed-nuget-dml'
        targetPath: '$(Build.ArtifactStagingDirectory)'
    - template: templates/component-governance-component-detection-steps.yml
      parameters:
        condition: 'succeeded'

- template: templates/qnn-ep-win.yml
  parameters:
    qnn_ep_build_pool_name: 'Onnxruntime-QNNEP-Windows-2022-CPU'
    QnnSdk: ${{ parameters.QnnSdk }}
    IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
    DoEsrp: ${{ parameters.DoEsrp }}
    ArtifactName: 'drop-nuget-qnn-x64'
    StageName: 'OnnxRuntime_QNN_Nuget_Win_x64'
    build_config: 'RelWithDebInfo'
- template: templates/qnn-ep-win.yml
  parameters:
    qnn_ep_build_pool_name: 'Onnxruntime-QNNEP-Windows-2022-CPU'
    QnnSdk: ${{ parameters.QnnSdk }}
    IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
    DoEsrp: ${{ parameters.DoEsrp }}
    ArtifactName: 'drop-nuget-qnn-arm64'
    buildParameter: '--arm64'
    buildPlatform: 'ARM64'
    buildArch: 'ARM64'
    StageName: 'OnnxRuntime_QNN_Nuget_Win_Arm64'
    build_config: 'RelWithDebInfo'

- stage: NuGet_Packaging_QNN
  pool: 'Onnxruntime-QNNEP-Windows-2022-CPU'
  dependsOn:
  - OnnxRuntime_QNN_Nuget_Win_x64
  - OnnxRuntime_QNN_Nuget_Win_Arm64
  condition: succeeded()
  jobs:
  - job: NuGet_Packaging_QNN
    workspace:
      clean: all
    steps:
    - task: DownloadPipelineArtifact@0
      displayName: 'Download Pipeline Artifact - QNN NuGet x64'
      inputs:
        artifactName: 'drop-nuget-qnn-x64'
        targetPath: '$(Build.BinariesDirectory)/nuget-artifact-x64'

    - task: DownloadPipelineArtifact@0
      displayName: 'Download Pipeline Artifact - QNN NuGet arm64'
      inputs:
        artifactName: 'drop-nuget-qnn-arm64'
        targetPath: '$(Build.BinariesDirectory)/nuget-artifact-arm64'

    - task: PowerShell@2
      displayName: 'Bundle NuGet'
      inputs:
        targetType: 'inline'
        script: |

          $x64_nupkgs = (Get-ChildItem $(Build.BinariesDirectory)/nuget-artifact-x64 -Filter Microsoft.ML.OnnxRuntime.QNN*.nupkg -Recurse)
          $nuget_package_name = $x64_nupkgs[0].Name
          $x64_nuget_package = $x64_nupkgs[0].FullName

          $nupkg_unzipped_directory = [System.IO.Path]::Combine($Env:BUILD_ARTIFACTSTAGINGDIRECTORY, 'nuget_unzip_merged', [System.IO.Path]::GetFileNameWithoutExtension($nuget_package_name))

          $x64_unzip_cmd = "7z.exe x $x64_nuget_package -y -o$nupkg_unzipped_directory"
          Invoke-Expression -Command $x64_unzip_cmd

          $arm64_nupkgs = (Get-ChildItem $(Build.BinariesDirectory)/nuget-artifact-arm64 -Filter Microsoft.ML.OnnxRuntime.QNN*.nupkg -Recurse)
          $arm64_nuget_package = $arm64_nupkgs[0].FullName

          $arm64_unzip_cmd = "7z.exe x $arm64_nuget_package -y -o$nupkg_unzipped_directory"
          Invoke-Expression -Command $arm64_unzip_cmd

          $merged_nuget_path = [System.IO.Path]::Combine($Env:BUILD_ARTIFACTSTAGINGDIRECTORY, 'nuget-artifact-merged')
          if (!(Test-Path $merged_nuget_path)) {
              New-Item -Path $merged_nuget_path -ItemType Directory
          }

          $merged_zip = [System.IO.Path]::Combine($merged_nuget_path, 'qnn_nuget.zip')
          $zip_cmd = "7z.exe a -r $merged_zip $nupkg_unzipped_directory/*"
          Invoke-Expression -Command $zip_cmd

          $merged_nuget = [System.IO.Path]::Combine($merged_nuget_path, $nuget_package_name)
          move $merged_zip $merged_nuget
        workingDirectory: $(Build.BinariesDirectory)

    - template: templates/esrp_nuget.yml
      parameters:
        DisplayName: 'ESRP - sign NuGet package'
        FolderPath: '$(Build.ArtifactStagingDirectory)/nuget-artifact-merged'
        DoEsrp: ${{ parameters.DoEsrp }}

    - task: PublishPipelineArtifact@0
      displayName: 'Publish Pipeline NuGet Artifact'
      inputs:
        artifactName: 'drop-signed-nuget-qnn'
        targetPath: '$(Build.ArtifactStagingDirectory)/nuget-artifact-merged'