From 41ea47be1eca597bbe80872d3dbf8a4451ddfd44 Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Fri, 12 Jul 2024 00:07:23 +0800 Subject: [PATCH] Move QNN nuget package stages out of the big Nuget packaging pipeline. (#21306) ### Description 1. remove QNN stages from the big packaging pipeline 2. Add publish nightly package in the current [QNN Nuget pipeline](https://dev.azure.com/aiinfra/Lotus/_builddefinitionId=1234]) ### Motivation and Context Reduce the complexity of the big Nuget packaging pipelines. --------- Co-authored-by: Yi Zhang --- .../c-api-noopenmp-packaging-pipelines.yml | 91 ------------------- .../github/azure-pipelines/publish-nuget.yml | 6 +- .../qnn-ep-nuget-packaging-pipeline.yml | 35 +++++-- .../templates/publish-nuget-steps.yml | 7 +- 4 files changed, 34 insertions(+), 105 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml index abfe4df3f4..8b386dde7d 100644 --- a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml +++ b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml @@ -310,94 +310,3 @@ stages: - template: templates/component-governance-component-detection-steps.yml parameters: condition: 'succeeded' - -- template: templates/qnn-ep-win.yml - parameters: - qnn_ep_build_pool_name: 'Onnxruntime-QNNEP-Windows-2022-CPU' - QnnSdk: ${{ parameters.QnnSdk }} - IsReleaseBuild: ${{ parameters.IsReleaseBuild }} - DoEsrp: ${{ parameters.DoEsrp }} - ArtifactName: 'drop-nuget-qnn-x64' - StageName: 'OnnxRuntime_QNN_Nuget_Win_x64' - build_config: 'RelWithDebInfo' -- template: templates/qnn-ep-win.yml - parameters: - qnn_ep_build_pool_name: 'Onnxruntime-QNNEP-Windows-2022-CPU' - QnnSdk: ${{ parameters.QnnSdk }} - IsReleaseBuild: ${{ parameters.IsReleaseBuild }} - DoEsrp: ${{ parameters.DoEsrp }} - ArtifactName: 'drop-nuget-qnn-arm64' - buildParameter: '--arm64' - buildPlatform: 'ARM64' - buildArch: 'ARM64' - StageName: 'OnnxRuntime_QNN_Nuget_Win_Arm64' - build_config: 'RelWithDebInfo' - -- stage: NuGet_Packaging_QNN - pool: 'Onnxruntime-QNNEP-Windows-2022-CPU' - dependsOn: - - OnnxRuntime_QNN_Nuget_Win_x64 - - OnnxRuntime_QNN_Nuget_Win_Arm64 - condition: succeeded() - jobs: - - job: NuGet_Packaging_QNN - workspace: - clean: all - steps: - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - QNN NuGet x64' - inputs: - artifactName: 'drop-nuget-qnn-x64' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact-x64' - - - task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - QNN NuGet arm64' - inputs: - artifactName: 'drop-nuget-qnn-arm64' - targetPath: '$(Build.BinariesDirectory)/nuget-artifact-arm64' - - - task: PowerShell@2 - displayName: 'Bundle NuGet' - inputs: - targetType: 'inline' - script: | - - $x64_nupkgs = (Get-ChildItem $(Build.BinariesDirectory)/nuget-artifact-x64 -Filter Microsoft.ML.OnnxRuntime.QNN*.nupkg -Recurse) - $nuget_package_name = $x64_nupkgs[0].Name - $x64_nuget_package = $x64_nupkgs[0].FullName - - $nupkg_unzipped_directory = [System.IO.Path]::Combine($Env:BUILD_ARTIFACTSTAGINGDIRECTORY, 'nuget_unzip_merged', [System.IO.Path]::GetFileNameWithoutExtension($nuget_package_name)) - - $x64_unzip_cmd = "7z.exe x $x64_nuget_package -y -o$nupkg_unzipped_directory" - Invoke-Expression -Command $x64_unzip_cmd - - $arm64_nupkgs = (Get-ChildItem $(Build.BinariesDirectory)/nuget-artifact-arm64 -Filter Microsoft.ML.OnnxRuntime.QNN*.nupkg -Recurse) - $arm64_nuget_package = $arm64_nupkgs[0].FullName - - $arm64_unzip_cmd = "7z.exe x $arm64_nuget_package -y -o$nupkg_unzipped_directory" - Invoke-Expression -Command $arm64_unzip_cmd - - $merged_nuget_path = [System.IO.Path]::Combine($Env:BUILD_ARTIFACTSTAGINGDIRECTORY, 'nuget-artifact-merged') - if (!(Test-Path $merged_nuget_path)) { - New-Item -Path $merged_nuget_path -ItemType Directory - } - - $merged_zip = [System.IO.Path]::Combine($merged_nuget_path, 'qnn_nuget.zip') - $zip_cmd = "7z.exe a -r $merged_zip $nupkg_unzipped_directory/*" - Invoke-Expression -Command $zip_cmd - - $merged_nuget = [System.IO.Path]::Combine($merged_nuget_path, $nuget_package_name) - move $merged_zip $merged_nuget - workingDirectory: $(Build.BinariesDirectory) - - - template: templates/esrp_nuget.yml - parameters: - DisplayName: 'ESRP - sign NuGet package' - FolderPath: '$(Build.ArtifactStagingDirectory)/nuget-artifact-merged' - DoEsrp: ${{ parameters.DoEsrp }} - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Pipeline NuGet Artifact' - inputs: - artifactName: 'drop-signed-nuget-qnn' - targetPath: '$(Build.ArtifactStagingDirectory)/nuget-artifact-merged' diff --git a/tools/ci_build/github/azure-pipelines/publish-nuget.yml b/tools/ci_build/github/azure-pipelines/publish-nuget.yml index 01957a6eec..e0c5884134 100644 --- a/tools/ci_build/github/azure-pipelines/publish-nuget.yml +++ b/tools/ci_build/github/azure-pipelines/publish-nuget.yml @@ -12,6 +12,7 @@ resources: stages: - template: templates/publish-nuget-steps.yml parameters: + stage_name: 'Publish_NuGet_Packag_And_Report' include_cpu_ep: true download_artifacts_steps: - download: build @@ -28,8 +29,3 @@ stages: displayName: 'Download Pipeline Artifact - Signed NuGet Package' artifact: 'drop-signed-nuget-GPU' - script: move "$(Pipeline.Workspace)\build\drop-signed-nuget-GPU\*" $(Build.BinariesDirectory)\nuget-artifact\final-package - - - download: build - displayName: 'Download Pipeline Artifact - Signed NuGet Qnn Package' - artifact: 'drop-signed-nuget-qnn' - - script: move "$(Pipeline.Workspace)\build\drop-signed-nuget-qnn\*" $(Build.BinariesDirectory)\nuget-artifact\final-package diff --git a/tools/ci_build/github/azure-pipelines/qnn-ep-nuget-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/qnn-ep-nuget-packaging-pipeline.yml index 11e11384f2..a8b12637b7 100644 --- a/tools/ci_build/github/azure-pipelines/qnn-ep-nuget-packaging-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/qnn-ep-nuget-packaging-pipeline.yml @@ -8,7 +8,7 @@ parameters: displayName: Build Configuration type: string default: 'RelWithDebInfo' - + - name: IsReleaseBuild displayName: Is a release build? Set it to true if you are doing an Onnx Runtime release. type: boolean @@ -17,8 +17,19 @@ parameters: - name: DoEsrp displayName: Run code sign tasks? Must be true if you are doing an Onnx Runtime release. type: boolean + default: true + +# these 2 parameters are used for debugging. +- name: SpecificArtifact + displayName: Use Specific Artifact (Debugging only) + type: boolean default: false +- name: BuildId + displayName: Pipeline BuildId, you could find it in the URL + type: string + default: '0' + stages: - template: templates/qnn-ep-win.yml @@ -60,25 +71,25 @@ stages: inputs: artifactName: 'drop-nuget-qnn-x64' targetPath: '$(Build.BinariesDirectory)/nuget-artifact-x64' - + - task: DownloadPipelineArtifact@0 displayName: 'Download Pipeline Artifact - QNN NuGet arm64' inputs: artifactName: 'drop-nuget-qnn-arm64' targetPath: '$(Build.BinariesDirectory)/nuget-artifact-arm64' - + - task: PowerShell@2 displayName: 'Bundle NuGet' inputs: targetType: 'inline' script: | - + $x64_nupkgs = (Get-ChildItem $(Build.BinariesDirectory)/nuget-artifact-x64 -Filter Microsoft.ML.OnnxRuntime.QNN*.nupkg -Recurse) $nuget_package_name = $x64_nupkgs[0].Name $x64_nuget_package = $x64_nupkgs[0].FullName $nupkg_unzipped_directory = [System.IO.Path]::Combine($Env:BUILD_ARTIFACTSTAGINGDIRECTORY, 'nuget_unzip_merged', [System.IO.Path]::GetFileNameWithoutExtension($nuget_package_name)) - + $x64_unzip_cmd = "7z.exe x $x64_nuget_package -y -o$nupkg_unzipped_directory" Invoke-Expression -Command $x64_unzip_cmd @@ -94,9 +105,9 @@ stages: } $merged_zip = [System.IO.Path]::Combine($merged_nuget_path, 'qnn_nuget.zip') - $zip_cmd = "7z.exe a -r $merged_zip $nupkg_unzipped_directory/*" + $zip_cmd = "7z.exe a -r $merged_zip $nupkg_unzipped_directory/*" Invoke-Expression -Command $zip_cmd - + $merged_nuget = [System.IO.Path]::Combine($merged_nuget_path, $nuget_package_name) move $merged_zip $merged_nuget workingDirectory: $(Build.BinariesDirectory) @@ -113,3 +124,13 @@ stages: artifactName: 'drop-signed-nuget-qnn' targetPath: '$(Build.ArtifactStagingDirectory)/nuget-artifact-merged' +- template: templates/publish-nuget-steps.yml + parameters: + download_artifacts_steps: + - template: templates/flex-downloadPipelineArtifact.yml + parameters: + StepName: 'Download Pipeline Artifact - Signed NuGet Qnn Package' + ArtifactName: 'drop-signed-nuget-qnn' + targetPath: '$(Build.BinariesDirectory)/nuget-artifact/final-package' + SpecificArtifact: ${{ parameters.specificArtifact }} + BuildId: ${{ parameters.BuildId }} diff --git a/tools/ci_build/github/azure-pipelines/templates/publish-nuget-steps.yml b/tools/ci_build/github/azure-pipelines/templates/publish-nuget-steps.yml index 6698501e74..8639a5ca0a 100644 --- a/tools/ci_build/github/azure-pipelines/templates/publish-nuget-steps.yml +++ b/tools/ci_build/github/azure-pipelines/templates/publish-nuget-steps.yml @@ -4,11 +4,14 @@ parameters: default: false - name: download_artifacts_steps type: stepList +- name: stage_name + type: string + default: 'Publish_NuGet_Package' stages: -- stage: Publish_NuGet_Package_And_Report +- stage: ${{ parameters.stage_name }} jobs: - - job: Publish_NuGet_Package_And_Report + - job: ${{ parameters.stage_name }} workspace: clean: all variables: