From 045c623415b7cc2057c876c4173d83d320fe04d2 Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Mon, 8 May 2023 20:53:08 +0800 Subject: [PATCH] Make Nuget workflow easy to debug (#15808) ### Description Fix the bug in #15693 ### Motivation and Context --- .../c-api-noopenmp-packaging-pipelines.yml | 48 +++++++++++++------ .../nuget/templates/test_linux.yml | 28 ++++++----- .../azure-pipelines/templates/c-api-cpu.yml | 12 +++++ .../flex-downloadPipelineArtifact.yml | 32 +++++++++++++ 4 files changed, 94 insertions(+), 26 deletions(-) create mode 100644 tools/ci_build/github/azure-pipelines/templates/flex-downloadPipelineArtifact.yml diff --git a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml index dcf1661464..c83290f7cb 100644 --- a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml +++ b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml @@ -39,6 +39,17 @@ parameters: type: number default: 0 +# these 2 parameters are used for debugging. +- name: SpecificArtifact + displayName: Use Specific Artifact (Debugging only) + type: boolean + default: false + +- name: BuildId + displayName: Pipeline BuildId, you could find it in the URL + type: string + default: '0' + resources: repositories: - repository: onnxruntime-inference-examples # The name used to reference this repository in the checkout step @@ -92,6 +103,8 @@ stages: AdditionalBuildFlags: '' AdditionalWinBuildFlags: '--enable_onnx_tests --enable_wcos' BuildVariant: 'default' + SpecificArtifact: ${{ parameters.SpecificArtifact }} + BuildId: ${{ parameters.BuildId }} - template: templates/ondevice-training-cpu-packaging-pipeline.yml parameters: @@ -207,26 +220,29 @@ stages: submodules: false - template: templates/set-version-number-variables-step.yml - - task: DownloadPipelineArtifact@2 - displayName: 'Download Pipeline Artifact - Win x64' - inputs: - buildType: 'current' - artifactName: 'drop-onnxruntime-java-win-x64-tensorrt' - targetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64' + - template: templates/flex-downloadPipelineArtifact.yml + parameters: + StepName: 'Download Pipeline Artifact - Win x64' + ArtifactName: 'drop-onnxruntime-java-win-x64-tensorrt' + TargetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64' + SpecificArtifact: ${{ parameters.specificArtifact }} + BuildId: ${{ parameters.BuildId }} - - task: DownloadPipelineArtifact@2 - displayName: 'Download Pipeline Artifact - Linux x64' - inputs: - buildType: 'current' + - template: templates/flex-downloadPipelineArtifact.yml + parameters: + stepName: 'Download Pipeline Artifact - Linux x64' artifactName: 'drop-onnxruntime-java-linux-x64-cuda' targetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-linux-x64' + SpecificArtifact: ${{ parameters.specificArtifact }} + BuildId: ${{ parameters.BuildId }} - - task: DownloadPipelineArtifact@2 - displayName: 'Download Pipeline Artifact - Linux x64' - inputs: - buildType: 'current' - artifactName: 'drop-onnxruntime-java-linux-x64-tensorrt' + - template: templates/flex-downloadPipelineArtifact.yml + parameters: + StepName: 'Download Pipeline Artifact - Linux x64' + ArtifactName: 'drop-onnxruntime-java-linux-x64-tensorrt' targetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-linux-x64-tensorrt' + SpecificArtifact: ${{ parameters.specificArtifact }} + BuildId: ${{ parameters.BuildId }} - task: PowerShell@2 displayName: 'PowerShell Script' @@ -752,6 +768,8 @@ stages: ArtifactSuffix: 'GPU' StageSuffix: 'GPU' NugetPackageName : 'Microsoft.ML.OnnxRuntime.Gpu' + SpecificArtifact: ${{ parameters.specificArtifact }} + BuildId: ${{ parameters.BuildId }} - template: nuget/templates/dml-vs-2019.yml parameters: diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml index dfced9c706..e33b2d34d9 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml @@ -4,6 +4,8 @@ parameters: NugetPackageName : '' StageSuffix: 'CPU' NativePackagePrefix: 'onnxruntime' + SpecificArtifact: false + BuildId: '0' stages: - stage: NuGet_Test_Linux_${{ parameters.StageSuffix }} @@ -20,19 +22,23 @@ stages: variables: - name: OnnxRuntimeBuildDirectory value: '$(Build.BinariesDirectory)' + steps: + - template: ../../templates/flex-downloadPipelineArtifact.yml + parameters: + StepName: 'Download Signed NuGet' + ArtifactName: drop-signed-nuget-${{ parameters.ArtifactSuffix }} + TargetPath: '$(Build.BinariesDirectory)/nuget-artifact' + SpecificArtifact: ${{ parameters.SpecificArtifact }} + BuildId: ${{ parameters.BuildId }} - - task: DownloadPipelineArtifact@0 - displayName: 'Download Signed NuGet' - inputs: - artifactName: drop-signed-nuget-${{ parameters.ArtifactSuffix }} - targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - - - task: DownloadPipelineArtifact@0 - displayName: 'Download Linux CustomOp TestData' - inputs: - artifactName: '${{ parameters.NativePackagePrefix }}-linux-x64' - targetPath: '$(Build.BinariesDirectory)/testdata' + - template: ../../templates/flex-downloadPipelineArtifact.yml + parameters: + StepName: 'Download Linux CustomOp TestData' + ArtifactName: 'onnxruntime-linux-x64' + TargetPath: '$(Build.BinariesDirectory)/testdata' + SpecificArtifact: ${{ parameters.specificArtifact }} + BuildId: ${{ parameters.BuildId }} - template: get-nuget-package-version-as-variable.yml parameters: diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml index 75a189bdc0..d4577fdc1c 100644 --- a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml +++ b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml @@ -43,6 +43,16 @@ parameters: type: string default: 'default' +- name: SpecificArtifact + displayName: Use Specific Artifact + type: boolean + default: false + +- name: BuildId + displayName: Specific Artifact's BuildId + type: string + default: '0' + stages: - template: linux-cpu-packaging-pipeline.yml parameters: @@ -775,6 +785,8 @@ stages: AgentPool : aiinfra-Linux-CPU NugetPackageName : 'Microsoft.ML.OnnxRuntime' ArtifactSuffix: 'CPU' + SpecificArtifact: ${{ parameters.SpecificArtifact }} + BuildId: ${{ parameters.BuildId }} - template: ../nuget/templates/test_macos.yml parameters: diff --git a/tools/ci_build/github/azure-pipelines/templates/flex-downloadPipelineArtifact.yml b/tools/ci_build/github/azure-pipelines/templates/flex-downloadPipelineArtifact.yml new file mode 100644 index 0000000000..0f4e0553d0 --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/templates/flex-downloadPipelineArtifact.yml @@ -0,0 +1,32 @@ +parameters: + - name: StepName + type: string + + - name: ArtifactName + type: string + + - name: TargetPath + type: string + + - name: SpecificArtifact + type: boolean + default: false + + - name: BuildId + type: string + default: '0' + +steps: + - task: DownloadPipelineArtifact@2 + displayName: ${{ parameters.StepName }}} + inputs: + artifactName: ${{ parameters.ArtifactName}} + targetPath: '${{ parameters.TargetPath }}' + ${{ if eq(parameters.SpecificArtifact, false)}}: + buildType: 'current' + ${{ else }}: + source: 'specific' + project: $(System.TeamProject) + pipeline: $(Build.DefinitionName) + runVersion: 'specific' + buildId: ${{ parameters.BuildId }}