mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-26 19:52:38 +00:00
Templatize publishing nuget package (#21199)
### Description It's the prerequisite step of reducing complexity of current zip-nuget pipeline. Some packaging tasks could be cut from the most complex nuget pipline and easily be published ### Motivation and Context <!-- - Why is this change required? What problem does it solve? - If it fixes an open issue, please link to the issue here. -->
This commit is contained in:
Yi Zhang
GitHub
parent
8c2689877f
commit
beb2496748
2 changed files with 160 additions and 147 deletions
|
|
@ -10,154 +10,31 @@ resources:
|
|||
branch: main
|
||||
|
||||
stages:
|
||||
- stage: Publish_NuGet_Package_And_Report
|
||||
jobs:
|
||||
- job: Publish_NuGet_Package_And_Report
|
||||
workspace:
|
||||
clean: all
|
||||
variables:
|
||||
- name: GDN_CODESIGN_TARGETDIRECTORY
|
||||
value: '$(Agent.TempDirectory)\binfiles'
|
||||
pool: 'onnxruntime-Win-CPU-2022'
|
||||
- template: templates/publish-nuget-steps.yml
|
||||
parameters:
|
||||
include_cpu_ep: true
|
||||
download_artifacts_steps:
|
||||
- download: build
|
||||
displayName: 'Download Pipeline Artifact - Signed NuGet Package'
|
||||
artifact: 'drop-signed-nuget-dml'
|
||||
- script: move "$(Pipeline.Workspace)\build\drop-signed-nuget-dml\*" $(Build.BinariesDirectory)\nuget-artifact\final-package
|
||||
|
||||
steps:
|
||||
# https://learn.microsoft.com/en-us/azure/devops/pipelines/yaml-schema/resources-pipelines-pipeline?view=azure-pipelines#pipeline-resource-metadata-as-predefined-variables
|
||||
- script: |
|
||||
echo $(resources.pipeline.build.sourceBranch)
|
||||
echo $(Build.Reason)
|
||||
displayName: 'Print triggering sourceBranch Name in resources'
|
||||
- download: build
|
||||
displayName: 'Download Pipeline Artifact - Signed NuGet Package'
|
||||
artifact: 'drop-signed-nuget-Training-CPU'
|
||||
- script: move "$(Pipeline.Workspace)\build\drop-signed-nuget-Training-CPU\*" $(Build.BinariesDirectory)\nuget-artifact\final-package
|
||||
|
||||
- checkout: self
|
||||
submodules: false
|
||||
- download: build
|
||||
displayName: 'Download Pipeline Artifact - Signed NuGet Package'
|
||||
artifact: 'drop-signed-nuget-GPU'
|
||||
- script: move "$(Pipeline.Workspace)\build\drop-signed-nuget-GPU\*" $(Build.BinariesDirectory)\nuget-artifact\final-package
|
||||
|
||||
- task: UsePythonVersion@0
|
||||
inputs:
|
||||
versionSpec: '3.9'
|
||||
addToPath: true
|
||||
- download: build
|
||||
displayName: 'Download Pipeline Artifact - Signed NuGet ROCm Package'
|
||||
artifact: 'drop-signed-nuget-ROCm'
|
||||
- script: move "$(Pipeline.Workspace)\build\drop-signed-nuget-ROCm\*" $(Build.BinariesDirectory)\nuget-artifact\final-package
|
||||
|
||||
- template: templates/set-version-number-variables-step.yml
|
||||
|
||||
- script: mkdir "$(Build.BinariesDirectory)\nuget-artifact\final-package"
|
||||
|
||||
- download: build
|
||||
displayName: 'Download Pipeline Artifact - Signed NuGet Package'
|
||||
artifact: 'drop-signed-nuget-CPU'
|
||||
|
||||
- script: move "$(Pipeline.Workspace)\build\drop-signed-nuget-CPU\*" "$(Build.BinariesDirectory)\nuget-artifact\final-package"
|
||||
|
||||
- template: nuget/templates/get-nuget-package-version-as-variable.yml
|
||||
parameters:
|
||||
packageFolder: '$(Build.BinariesDirectory)/nuget-artifact/final-package'
|
||||
|
||||
- task: CmdLine@2
|
||||
displayName: 'Post binary sizes to the dashboard database using command line'
|
||||
inputs:
|
||||
script: |
|
||||
echo changing directory to artifact download path
|
||||
cd $(Build.BinariesDirectory)/nuget-artifact/final-package
|
||||
echo processing nupkg
|
||||
SETLOCAL EnableDelayedExpansion
|
||||
FOR /R %%i IN (*.nupkg) do (
|
||||
set filename=%%~ni
|
||||
IF NOT "!filename:~25,7!"=="Managed" (
|
||||
echo processing %%~ni.nupkg
|
||||
copy %%~ni.nupkg %%~ni.zip
|
||||
echo copied to zip
|
||||
echo listing lib files in the zip
|
||||
REM use a single .csv file to put the data
|
||||
echo os,arch,build_config,size > $(Build.BinariesDirectory)\binary_size_data.txt
|
||||
7z.exe l -slt %%~ni.zip runtimes\linux-arm64\native\libonnxruntime.so | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo linux,aarch64,default,%%a >> $(Build.BinariesDirectory)\binary_size_data.txt
|
||||
7z.exe l -slt %%~ni.zip runtimes\osx-x64\native\libonnxruntime.dylib | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo osx,x64,default,%%a >> $(Build.BinariesDirectory)\binary_size_data.txt
|
||||
7z.exe l -slt %%~ni.zip runtimes\win-x64\native\onnxruntime.dll | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo win,x64,default,%%a >> $(Build.BinariesDirectory)\binary_size_data.txt
|
||||
7z.exe l -slt %%~ni.zip runtimes\win-x86\native\onnxruntime.dll | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo win,x86,default,%%a >> $(Build.BinariesDirectory)\binary_size_data.txt
|
||||
)
|
||||
)
|
||||
|
||||
- task: AzureCLI@2
|
||||
displayName: 'Azure CLI'
|
||||
#Only report binary sizes to database if the build build was auto-triggered from the main branch
|
||||
condition: and (succeeded(), and(eq(variables['resources.pipeline.build.sourceBranch'], 'refs/heads/main'), eq(variables['Build.Reason'], 'ResourceTrigger')))
|
||||
inputs:
|
||||
azureSubscription: AIInfraBuildOnnxRuntimeOSS
|
||||
scriptLocation: inlineScript
|
||||
scriptType: batch
|
||||
inlineScript: |
|
||||
python.exe -m pip install -r $(Build.SourcesDirectory)\tools\ci_build\github\windows\post_to_dashboard\requirements.txt && ^
|
||||
python.exe $(Build.SourcesDirectory)\tools\ci_build\github\windows\post_binary_sizes_to_dashboard.py --commit_hash=$(Build.SourceVersion) --size_data_file=binary_size_data.txt --build_project=Lotus --build_id=$(Build.BuildId)
|
||||
workingDirectory: '$(Build.BinariesDirectory)'
|
||||
|
||||
- download: build
|
||||
displayName: 'Download Pipeline Artifact - Signed NuGet Package'
|
||||
artifact: 'drop-signed-nuget-dml'
|
||||
|
||||
- script: move "$(Pipeline.Workspace)\build\drop-signed-nuget-dml\*" $(Build.BinariesDirectory)\nuget-artifact\final-package
|
||||
|
||||
- download: build
|
||||
displayName: 'Download Pipeline Artifact - Signed NuGet Package'
|
||||
artifact: 'drop-signed-nuget-Training-CPU'
|
||||
- script: move "$(Pipeline.Workspace)\build\drop-signed-nuget-Training-CPU\*" $(Build.BinariesDirectory)\nuget-artifact\final-package
|
||||
|
||||
- download: build
|
||||
displayName: 'Download Pipeline Artifact - Signed NuGet Package'
|
||||
artifact: 'drop-signed-nuget-GPU'
|
||||
- script: move "$(Pipeline.Workspace)\build\drop-signed-nuget-GPU\*" $(Build.BinariesDirectory)\nuget-artifact\final-package
|
||||
|
||||
- download: build
|
||||
displayName: 'Download Pipeline Artifact - Signed NuGet ROCm Package'
|
||||
artifact: 'drop-signed-nuget-ROCm'
|
||||
- script: move "$(Pipeline.Workspace)\build\drop-signed-nuget-ROCm\*" $(Build.BinariesDirectory)\nuget-artifact\final-package
|
||||
|
||||
- download: build
|
||||
displayName: 'Download Pipeline Artifact - Signed NuGet Qnn Package'
|
||||
artifact: 'drop-signed-nuget-qnn'
|
||||
- script: move "$(Pipeline.Workspace)\build\drop-signed-nuget-qnn\*" $(Build.BinariesDirectory)\nuget-artifact\final-package
|
||||
|
||||
- script: |
|
||||
dir $(Build.BinariesDirectory)\nuget-artifact\final-package
|
||||
cd $(Build.BinariesDirectory)\nuget-artifact\final-package
|
||||
nuget verify -Signatures *.nupkg
|
||||
displayName: List Downloaded Package
|
||||
|
||||
- powershell: |
|
||||
New-Item -Path $(Agent.TempDirectory) -Name "binfiles" -ItemType "directory"
|
||||
$base_path_name = Join-Path -Path $(Agent.TempDirectory) -ChildPath "binfiles"
|
||||
Get-ChildItem $Env:BUILD_BINARIESDIRECTORY\nuget-artifact\final-package -Filter *.nupkg |
|
||||
Foreach-Object {
|
||||
$dir_name = Join-Path -Path $base_path_name -ChildPath $_.Basename
|
||||
$cmd = "7z.exe x $($_.FullName) -y -o$dir_name"
|
||||
Write-Output $cmd
|
||||
Invoke-Expression -Command $cmd
|
||||
}
|
||||
dir $(Agent.TempDirectory)
|
||||
tree $(Agent.TempDirectory)
|
||||
workingDirectory: '$(Agent.TempDirectory)'
|
||||
|
||||
- task: CodeSign@1
|
||||
displayName: 'Run Codesign Validation'
|
||||
|
||||
|
||||
- task: PublishSecurityAnalysisLogs@3
|
||||
displayName: 'Publish Security Analysis Logs'
|
||||
continueOnError: true
|
||||
|
||||
- task: PostAnalysis@2
|
||||
inputs:
|
||||
GdnBreakAllTools: true
|
||||
GdnBreakPolicy: M365
|
||||
GdnBreakPolicyMinSev: Error
|
||||
|
||||
#TODO: allow choosing different feeds
|
||||
- task: NuGetCommand@2
|
||||
displayName: 'Copy Signed Native NuGet Package to ORT-NIGHTLY'
|
||||
inputs:
|
||||
command: 'push'
|
||||
packagesToPush: '$(Build.BinariesDirectory)/nuget-artifact/final-package/*.nupkg'
|
||||
publishVstsFeed: '2692857e-05ef-43b4-ba9c-ccf1c22c437c/7982ae20-ed19-4a35-a362-a96ac99897b7'
|
||||
allowPackageConflicts: true
|
||||
|
||||
- template: templates/component-governance-component-detection-steps.yml
|
||||
parameters :
|
||||
condition : 'succeeded'
|
||||
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
|
||||
displayName: 'Clean Agent Directories'
|
||||
condition: always()
|
||||
- download: build
|
||||
displayName: 'Download Pipeline Artifact - Signed NuGet Qnn Package'
|
||||
artifact: 'drop-signed-nuget-qnn'
|
||||
- script: move "$(Pipeline.Workspace)\build\drop-signed-nuget-qnn\*" $(Build.BinariesDirectory)\nuget-artifact\final-package
|
||||
|
|
|
|||
|
|
@ -0,0 +1,136 @@
|
|||
parameters:
|
||||
- name: include_cpu_ep
|
||||
type: boolean
|
||||
default: false
|
||||
- name: download_artifacts_steps
|
||||
type: stepList
|
||||
|
||||
stages:
|
||||
- stage: Publish_NuGet_Package_And_Report
|
||||
jobs:
|
||||
- job: Publish_NuGet_Package_And_Report
|
||||
workspace:
|
||||
clean: all
|
||||
variables:
|
||||
- name: GDN_CODESIGN_TARGETDIRECTORY
|
||||
value: '$(Agent.TempDirectory)\binfiles'
|
||||
pool: 'onnxruntime-Win-CPU-2022'
|
||||
|
||||
steps:
|
||||
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
|
||||
displayName: 'Clean Agent Directories'
|
||||
condition: always()
|
||||
# https://learn.microsoft.com/en-us/azure/devops/pipelines/yaml-schema/resources-pipelines-pipeline?view=azure-pipelines#pipeline-resource-metadata-as-predefined-variables
|
||||
- script: |
|
||||
echo $(resources.pipeline.build.sourceBranch)
|
||||
echo $(Build.Reason)
|
||||
displayName: 'Print triggering sourceBranch Name in resources'
|
||||
|
||||
- checkout: self
|
||||
submodules: false
|
||||
|
||||
- task: UsePythonVersion@0
|
||||
inputs:
|
||||
versionSpec: '3.9'
|
||||
addToPath: true
|
||||
|
||||
- template: set-version-number-variables-step.yml
|
||||
|
||||
- script: mkdir "$(Build.BinariesDirectory)\nuget-artifact\final-package"
|
||||
|
||||
- template: ../nuget/templates/get-nuget-package-version-as-variable.yml
|
||||
parameters:
|
||||
packageFolder: '$(Build.BinariesDirectory)/nuget-artifact/final-package'
|
||||
|
||||
- ${{if eq(parameters.include_cpu_ep, true)}}:
|
||||
- download: build
|
||||
displayName: 'Download Pipeline Artifact - Signed NuGet Package'
|
||||
artifact: 'drop-signed-nuget-CPU'
|
||||
|
||||
- script: move "$(Pipeline.Workspace)\build\drop-signed-nuget-CPU\*" "$(Build.BinariesDirectory)\nuget-artifact\final-package"
|
||||
|
||||
- task: CmdLine@2
|
||||
displayName: 'Post binary sizes to the dashboard database using command line'
|
||||
inputs:
|
||||
script: |
|
||||
echo changing directory to artifact download path
|
||||
cd $(Build.BinariesDirectory)/nuget-artifact/final-package
|
||||
echo processing nupkg
|
||||
SETLOCAL EnableDelayedExpansion
|
||||
FOR /R %%i IN (*.nupkg) do (
|
||||
set filename=%%~ni
|
||||
IF NOT "!filename:~25,7!"=="Managed" (
|
||||
echo processing %%~ni.nupkg
|
||||
copy %%~ni.nupkg %%~ni.zip
|
||||
echo copied to zip
|
||||
echo listing lib files in the zip
|
||||
REM use a single .csv file to put the data
|
||||
echo os,arch,build_config,size > $(Build.BinariesDirectory)\binary_size_data.txt
|
||||
7z.exe l -slt %%~ni.zip runtimes\linux-arm64\native\libonnxruntime.so | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo linux,aarch64,default,%%a >> $(Build.BinariesDirectory)\binary_size_data.txt
|
||||
7z.exe l -slt %%~ni.zip runtimes\osx-x64\native\libonnxruntime.dylib | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo osx,x64,default,%%a >> $(Build.BinariesDirectory)\binary_size_data.txt
|
||||
7z.exe l -slt %%~ni.zip runtimes\win-x64\native\onnxruntime.dll | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo win,x64,default,%%a >> $(Build.BinariesDirectory)\binary_size_data.txt
|
||||
7z.exe l -slt %%~ni.zip runtimes\win-x86\native\onnxruntime.dll | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo win,x86,default,%%a >> $(Build.BinariesDirectory)\binary_size_data.txt
|
||||
)
|
||||
)
|
||||
|
||||
- task: AzureCLI@2
|
||||
displayName: 'Azure CLI'
|
||||
#Only report binary sizes to database if the build build was auto-triggered from the main branch
|
||||
condition: and (succeeded(), and(eq(variables['resources.pipeline.build.sourceBranch'], 'refs/heads/main'), eq(variables['Build.Reason'], 'ResourceTrigger')))
|
||||
inputs:
|
||||
azureSubscription: AIInfraBuildOnnxRuntimeOSS
|
||||
scriptLocation: inlineScript
|
||||
scriptType: batch
|
||||
inlineScript: |
|
||||
python.exe -m pip install -r $(Build.SourcesDirectory)\tools\ci_build\github\windows\post_to_dashboard\requirements.txt && ^
|
||||
python.exe $(Build.SourcesDirectory)\tools\ci_build\github\windows\post_binary_sizes_to_dashboard.py --commit_hash=$(Build.SourceVersion) --size_data_file=binary_size_data.txt --build_project=Lotus --build_id=$(Build.BuildId)
|
||||
workingDirectory: '$(Build.BinariesDirectory)'
|
||||
|
||||
- ${{ parameters.download_artifacts_steps }}
|
||||
|
||||
- script: |
|
||||
dir $(Build.BinariesDirectory)\nuget-artifact\final-package
|
||||
cd $(Build.BinariesDirectory)\nuget-artifact\final-package
|
||||
nuget verify -Signatures *.nupkg
|
||||
displayName: List Downloaded Package
|
||||
|
||||
- powershell: |
|
||||
New-Item -Path $(Agent.TempDirectory) -Name "binfiles" -ItemType "directory"
|
||||
$base_path_name = Join-Path -Path $(Agent.TempDirectory) -ChildPath "binfiles"
|
||||
Get-ChildItem $Env:BUILD_BINARIESDIRECTORY\nuget-artifact\final-package -Filter *.nupkg |
|
||||
Foreach-Object {
|
||||
$dir_name = Join-Path -Path $base_path_name -ChildPath $_.Basename
|
||||
$cmd = "7z.exe x $($_.FullName) -y -o$dir_name"
|
||||
Write-Output $cmd
|
||||
Invoke-Expression -Command $cmd
|
||||
}
|
||||
dir $(Agent.TempDirectory)
|
||||
tree $(Agent.TempDirectory)
|
||||
workingDirectory: '$(Agent.TempDirectory)'
|
||||
|
||||
- task: CodeSign@1
|
||||
displayName: 'Run Codesign Validation'
|
||||
|
||||
|
||||
- task: PublishSecurityAnalysisLogs@3
|
||||
displayName: 'Publish Security Analysis Logs'
|
||||
continueOnError: true
|
||||
|
||||
- task: PostAnalysis@2
|
||||
inputs:
|
||||
GdnBreakAllTools: true
|
||||
GdnBreakPolicy: M365
|
||||
GdnBreakPolicyMinSev: Error
|
||||
|
||||
#TODO: allow choosing different feeds
|
||||
- task: NuGetCommand@2
|
||||
displayName: 'Copy Signed Native NuGet Package to ORT-NIGHTLY'
|
||||
inputs:
|
||||
command: 'push'
|
||||
packagesToPush: '$(Build.BinariesDirectory)/nuget-artifact/final-package/*.nupkg'
|
||||
publishVstsFeed: '2692857e-05ef-43b4-ba9c-ccf1c22c437c/7982ae20-ed19-4a35-a362-a96ac99897b7'
|
||||
allowPackageConflicts: true
|
||||
|
||||
- template: component-governance-component-detection-steps.yml
|
||||
parameters :
|
||||
condition : 'succeeded'
|
||||
Loading…
Reference in a new issue