mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-29 20:14:01 +00:00
30b6e82e7d
### Description Make current ROCm packaging stages to a single workflow. Reduce the possibility of all nightly packages can't be generated by one failed stage ### Motivation and Context Our plan is to reduce the complexity of the current zip-nuget pipeline to improve the stability and performance of nightly packages generation. ROCm packaging stages has no dependencies with other packaging jobs and it's the most time-consuming route. After this change, the most used CPU/CUDA/Mobile packaging workflow duration can be reduced roughly from 3h20m to 2h30m.
403 lines
16 KiB
YAML
403 lines
16 KiB
YAML
parameters:
|
|
- name: RunOnnxRuntimeTests
|
|
displayName: Run Tests?
|
|
type: boolean
|
|
default: true
|
|
|
|
- name: UseIncreasedTimeoutForTests
|
|
displayName: Increase timeout for tests? Set it to false if you are doing an Onnx Runtime release.
|
|
type: boolean
|
|
default: false
|
|
|
|
- name: DoCompliance
|
|
displayName: Run Compliance Tasks?
|
|
type: boolean
|
|
default: true
|
|
|
|
- name: DoEsrp
|
|
displayName: Run code sign tasks? Must be true if you are doing an ONNX Runtime release
|
|
type: boolean
|
|
default: true
|
|
|
|
- name: IsReleaseBuild
|
|
displayName: Is a release build? Set it to true if you are doing an ONNX Runtime release.
|
|
type: boolean
|
|
default: false
|
|
|
|
- name: PreReleaseVersionSuffixString
|
|
displayName: Suffix added to pre-release package version. Only used if IsReleaseBuild is true. Denotes the type of pre-release package.
|
|
type: string
|
|
values:
|
|
- alpha
|
|
- beta
|
|
- rc
|
|
- none
|
|
default: none
|
|
|
|
- name: PreReleaseVersionSuffixNumber
|
|
displayName: Number added to pre-release package version. Only used if IsReleaseBuild is true. Denotes the sequence of a pre-release package.
|
|
type: number
|
|
default: 0
|
|
|
|
# these 2 parameters are used for debugging.
|
|
- name: SpecificArtifact
|
|
displayName: Use Specific Artifact (Debugging only)
|
|
type: boolean
|
|
default: false
|
|
|
|
- name: BuildId
|
|
displayName: Pipeline BuildId, you could find it in the URL
|
|
type: string
|
|
default: '0'
|
|
|
|
- name: NugetPackageSuffix
|
|
displayName: Suffix to append to nuget package
|
|
type: string
|
|
default: 'NONE'
|
|
|
|
- name: AdditionalBuildFlag
|
|
displayName: Build flags to append to build command
|
|
type: string
|
|
default: '--use_azure'
|
|
- name: QnnSdk
|
|
displayName: QNN SDK Version
|
|
type: string
|
|
default: 2.23.0.240531
|
|
|
|
resources:
|
|
repositories:
|
|
- repository: onnxruntime-inference-examples # The name used to reference this repository in the checkout step
|
|
type: github
|
|
endpoint: ort-examples
|
|
name: microsoft/onnxruntime-inference-examples
|
|
- repository: manylinux
|
|
type: Github
|
|
endpoint: Microsoft
|
|
name: pypa/manylinux
|
|
ref: 5eda9aded5462201e6310105728d33016e637ea7
|
|
|
|
variables:
|
|
- name: ReleaseVersionSuffix
|
|
value: ''
|
|
- name: win_trt_version
|
|
value: 11.8
|
|
|
|
- name: win_trt_home
|
|
value: $(Agent.TempDirectory)\TensorRT-10.0.1.6.Windows10.x86_64.cuda-11.8
|
|
- name: win_cuda_home
|
|
value: $(Agent.TempDirectory)\v11.8
|
|
|
|
stages:
|
|
- template: stages/set_packaging_variables_stage.yml
|
|
parameters:
|
|
IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
|
|
PreReleaseVersionSuffixString: ${{ parameters.PreReleaseVersionSuffixString }}
|
|
PreReleaseVersionSuffixNumber: ${{ parameters.PreReleaseVersionSuffixNumber }}
|
|
|
|
- template: stages/download-java-tools-stage.yml
|
|
|
|
- template: templates/c-api-cpu.yml
|
|
parameters:
|
|
RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }}
|
|
DoCompliance: ${{ parameters.DoCompliance }}
|
|
DoEsrp: ${{ parameters.DoEsrp }}
|
|
IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
|
|
${{ if eq(parameters.NugetPackageSuffix, 'NONE') }}:
|
|
OrtNugetPackageId: 'Microsoft.ML.OnnxRuntime'
|
|
${{ else }}:
|
|
OrtNugetPackageId: 'Microsoft.ML.OnnxRuntime${{ parameters.NugetPackageSuffix }}'
|
|
AdditionalBuildFlags: ''
|
|
AdditionalWinBuildFlags: '--enable_onnx_tests --enable_wcos ${{parameters.AdditionalBuildFlag}}'
|
|
BuildVariant: 'default'
|
|
SpecificArtifact: ${{ parameters.SpecificArtifact }}
|
|
BuildId: ${{ parameters.BuildId }}
|
|
|
|
- template: templates/ondevice-training-cpu-packaging-pipeline.yml
|
|
parameters:
|
|
RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }}
|
|
DoCompliance: ${{ parameters.DoCompliance }}
|
|
DoEsrp: ${{ parameters.DoEsrp }}
|
|
IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
|
|
OrtNugetPackageId: 'Microsoft.ML.OnnxRuntime.Training'
|
|
AdditionalBuildFlags: '--enable_training_apis'
|
|
AdditionalWinBuildFlags: '--enable_onnx_tests --enable_wcos'
|
|
BuildVariant: 'default'
|
|
|
|
- template: stages/java-cuda-packaging-stage.yml
|
|
parameters:
|
|
CudaVersion: 11.8
|
|
SpecificArtifact: ${{ parameters.SpecificArtifact }}
|
|
BuildId: ${{ parameters.BuildId }}
|
|
|
|
- template: stages/nuget-combine-cuda-stage.yml
|
|
parameters:
|
|
DoCompliance: ${{ parameters.DoCompliance }}
|
|
CudaVersion: 11.8
|
|
docker_base_image: 'nvidia/cuda:11.8.0-cudnn8-devel-ubi8'
|
|
RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }}
|
|
UseIncreasedTimeoutForTests: ${{ parameters.UseIncreasedTimeoutForTests }}
|
|
win_trt_home: ${{ variables.win_trt_home }}
|
|
win_cuda_home: ${{ variables.win_cuda_home }}
|
|
DoEsrp: ${{ parameters.DoEsrp }}
|
|
IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
|
|
buildJava: true
|
|
buildNodejs: true
|
|
SpecificArtifact: ${{ parameters.SpecificArtifact }}
|
|
BuildId: ${{ parameters.BuildId }}
|
|
|
|
|
|
- template: nuget/templates/dml-vs-2022.yml
|
|
parameters:
|
|
AgentPool: 'onnxruntime-Win2022-GPU-dml-A10'
|
|
IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
|
|
ArtifactName: 'drop-nuget-dml'
|
|
StageName: 'Windows_CI_GPU_DML_Dev'
|
|
BuildCommand: --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos --use_telemetry --use_dml --use_winml --build_nodejs --cmake_generator "Visual Studio 17 2022"
|
|
BuildArch: 'x64'
|
|
msbuildArchitecture: 'amd64'
|
|
EnvSetupScript: 'setup_env.bat'
|
|
sln_platform: 'x64'
|
|
DoDebugBuild: 'false'
|
|
DoNugetPack: 'true'
|
|
DoCompliance: 'false'
|
|
DoEsrp: ${{ parameters.DoEsrp }}
|
|
NuPackScript: |
|
|
msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /t:CreatePackage /p:OrtPackageId=Microsoft.ML.OnnxRuntime.DirectML /p:IsReleaseBuild=${{ parameters.IsReleaseBuild }} /p:CurrentData=$(BuildDate) /p:CurrentTime=$(BuildTime)
|
|
copy $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory)
|
|
copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory)
|
|
mkdir $(Build.ArtifactStagingDirectory)\testdata
|
|
copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\custom_op_library.* $(Build.ArtifactStagingDirectory)\testdata
|
|
|
|
- template: nuget/templates/dml-vs-2022.yml
|
|
parameters:
|
|
AgentPool: 'onnxruntime-Win2022-GPU-dml-A10'
|
|
IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
|
|
ArtifactName: 'drop-win-dml-x86-zip'
|
|
StageName: 'Windows_CI_GPU_DML_Dev_x86'
|
|
BuildCommand: --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos --use_telemetry --use_dml --use_winml --cmake_generator "Visual Studio 17 2022"
|
|
BuildArch: 'x86'
|
|
EnvSetupScript: 'setup_env_x86.bat'
|
|
sln_platform: 'Win32'
|
|
DoDebugBuild: 'false'
|
|
DoNugetPack: 'true'
|
|
DoCompliance: ${{ parameters.DoCompliance }}
|
|
DoEsrp: ${{ parameters.DoEsrp }}
|
|
RunTests: 'false'
|
|
NuPackScript: |
|
|
msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /p:TargetArchitecture=x86 /t:CreatePackage /p:OrtPackageId=Microsoft.ML.OnnxRuntime.DirectML /p:IsReleaseBuild=${{ parameters.IsReleaseBuild }}
|
|
cd $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\
|
|
ren Microsoft.ML.OnnxRuntime.DirectML.* win-dml-x86.zip
|
|
copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\win-dml-x86.zip $(Build.ArtifactStagingDirectory)
|
|
mkdir $(Build.ArtifactStagingDirectory)\testdata
|
|
copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\custom_op_library.* $(Build.ArtifactStagingDirectory)\testdata
|
|
|
|
- template: nuget/templates/dml-vs-2022.yml
|
|
parameters:
|
|
AgentPool: 'onnxruntime-Win2022-GPU-dml-A10'
|
|
IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
|
|
ArtifactName: 'drop-win-dml-arm64-zip'
|
|
StageName: 'Windows_CI_GPU_DML_Dev_arm64'
|
|
BuildCommand: --build_dir $(Build.BinariesDirectory) --arm64 --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos --use_telemetry --use_dml --use_winml --build_nodejs --cmake_generator "Visual Studio 17 2022"
|
|
BuildArch: 'x64'
|
|
EnvSetupScript: 'setup_env.bat'
|
|
sln_platform: 'arm64'
|
|
DoDebugBuild: 'false'
|
|
DoNugetPack: 'true'
|
|
DoCompliance: ${{ parameters.DoCompliance }}
|
|
DoEsrp: ${{ parameters.DoEsrp }}
|
|
RunTests: 'false'
|
|
NuPackScript: |
|
|
msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /p:TargetArchitecture=arm64 /t:CreatePackage /p:OrtPackageId=Microsoft.ML.OnnxRuntime.DirectML /p:IsReleaseBuild=${{ parameters.IsReleaseBuild }}
|
|
cd $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\
|
|
ren Microsoft.ML.OnnxRuntime.DirectML.* win-dml-arm64.zip
|
|
copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\win-dml-arm64.zip $(Build.ArtifactStagingDirectory)
|
|
mkdir $(Build.ArtifactStagingDirectory)\testdata
|
|
copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\custom_op_library.* $(Build.ArtifactStagingDirectory)\testdata
|
|
|
|
- stage: NuGet_Packaging_DML
|
|
dependsOn:
|
|
- Windows_CI_GPU_DML_Dev
|
|
- Windows_CI_GPU_DML_Dev_x86
|
|
- Windows_CI_GPU_DML_Dev_arm64
|
|
condition: succeeded()
|
|
jobs:
|
|
- job: NuGet_Packaging_DML
|
|
workspace:
|
|
clean: all
|
|
pool: 'onnxruntime-Win2022-GPU-dml-A10'
|
|
steps:
|
|
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
|
|
displayName: 'Clean Agent Directories'
|
|
condition: always()
|
|
- task: DownloadPipelineArtifact@0
|
|
displayName: 'Download Pipeline Artifact - NuGet DirectML'
|
|
inputs:
|
|
artifactName: 'drop-nuget-dml'
|
|
targetPath: '$(Build.BinariesDirectory)/nuget-artifact-dml'
|
|
|
|
- task: DownloadPipelineArtifact@0
|
|
displayName: 'Download Pipeline Artifact - NuGet DirectML x86'
|
|
inputs:
|
|
artifactName: 'drop-win-dml-x86-zip'
|
|
targetPath: '$(Build.BinariesDirectory)/nuget-artifact-dml'
|
|
|
|
- task: DownloadPipelineArtifact@0
|
|
displayName: 'Download Pipeline Artifact - NuGet DirectML arm64'
|
|
inputs:
|
|
artifactName: 'drop-win-dml-arm64-zip'
|
|
targetPath: '$(Build.BinariesDirectory)/nuget-artifact-dml'
|
|
|
|
- script: |
|
|
pushd $(Build.BinariesDirectory)\nuget-artifact-dml
|
|
dir
|
|
powershell -Command "Invoke-WebRequest http://stahlworks.com/dev/unzip.exe -OutFile unzip.exe"
|
|
powershell -Command "Invoke-WebRequest http://stahlworks.com/dev/zip.exe -OutFile zip.exe"
|
|
set PATH=%CD%;%PATH%
|
|
SETLOCAL EnableDelayedExpansion
|
|
FOR /R %%i IN (*.nupkg) do (
|
|
set filename=%%~ni
|
|
IF NOT "!filename:~25,7!"=="Managed" (
|
|
rename %%~ni.nupkg %%~ni.zip
|
|
unzip %%~ni.zip -d %%~ni
|
|
del /Q %%~ni.zip
|
|
|
|
unzip win-dml-x86.zip -d win-x86
|
|
mkdir %%~ni\runtimes\win-x86
|
|
mkdir %%~ni\runtimes\win-x86\native
|
|
|
|
move win-x86\runtimes\win-x86\native\onnxruntime.dll %%~ni\runtimes\win-x86\native\onnxruntime.dll
|
|
move win-x86\runtimes\win-x86\native\onnxruntime.lib %%~ni\runtimes\win-x86\native\onnxruntime.lib
|
|
move win-x86\runtimes\win-x86\native\onnxruntime.pdb %%~ni\runtimes\win-x86\native\onnxruntime.pdb
|
|
|
|
unzip win-dml-arm64.zip -d win-arm64
|
|
mkdir %%~ni\runtimes\win-arm64
|
|
mkdir %%~ni\runtimes\win-arm64\native
|
|
|
|
move win-arm64\runtimes\win-arm64\native\onnxruntime.dll %%~ni\runtimes\win-arm64\native\onnxruntime.dll
|
|
move win-arm64\runtimes\win-arm64\native\onnxruntime.lib %%~ni\runtimes\win-arm64\native\onnxruntime.lib
|
|
move win-arm64\runtimes\win-arm64\native\onnxruntime.pdb %%~ni\runtimes\win-arm64\native\onnxruntime.pdb
|
|
|
|
|
|
pushd %%~ni
|
|
zip -r ..\%%~ni.zip .
|
|
popd
|
|
move %%~ni.zip %%~ni.nupkg
|
|
)
|
|
)
|
|
popd
|
|
copy $(Build.BinariesDirectory)\nuget-artifact-dml\Microsoft.ML.OnnxRuntime.DirectML*nupkg $(Build.ArtifactStagingDirectory)
|
|
displayName: 'Bundle DML NuGet and other binaries'
|
|
|
|
- template: templates/esrp_nuget.yml
|
|
parameters:
|
|
DisplayName: 'ESRP - sign NuGet package'
|
|
FolderPath: '$(Build.ArtifactStagingDirectory)'
|
|
DoEsrp: ${{ parameters.DoEsrp }}
|
|
|
|
- template: templates/validate-package.yml
|
|
parameters:
|
|
PackageType: 'nuget'
|
|
PackagePath: '$(Build.ArtifactStagingDirectory)'
|
|
PackageName: 'Microsoft.ML.OnnxRuntime.DirectML*nupkg'
|
|
PlatformsSupported: 'win-x64,win-x86,win-arm64'
|
|
VerifyNugetSigning: ${{ parameters.DoEsrp }}
|
|
|
|
- task: PublishPipelineArtifact@0
|
|
displayName: 'Publish Pipeline NuGet Artifact'
|
|
inputs:
|
|
artifactName: 'drop-signed-nuget-dml'
|
|
targetPath: '$(Build.ArtifactStagingDirectory)'
|
|
- template: templates/component-governance-component-detection-steps.yml
|
|
parameters:
|
|
condition: 'succeeded'
|
|
|
|
- template: templates/qnn-ep-win.yml
|
|
parameters:
|
|
qnn_ep_build_pool_name: 'Onnxruntime-QNNEP-Windows-2022-CPU'
|
|
QnnSdk: ${{ parameters.QnnSdk }}
|
|
IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
|
|
DoEsrp: ${{ parameters.DoEsrp }}
|
|
ArtifactName: 'drop-nuget-qnn-x64'
|
|
StageName: 'OnnxRuntime_QNN_Nuget_Win_x64'
|
|
build_config: 'RelWithDebInfo'
|
|
- template: templates/qnn-ep-win.yml
|
|
parameters:
|
|
qnn_ep_build_pool_name: 'Onnxruntime-QNNEP-Windows-2022-CPU'
|
|
QnnSdk: ${{ parameters.QnnSdk }}
|
|
IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
|
|
DoEsrp: ${{ parameters.DoEsrp }}
|
|
ArtifactName: 'drop-nuget-qnn-arm64'
|
|
buildParameter: '--arm64'
|
|
buildPlatform: 'ARM64'
|
|
buildArch: 'ARM64'
|
|
StageName: 'OnnxRuntime_QNN_Nuget_Win_Arm64'
|
|
build_config: 'RelWithDebInfo'
|
|
|
|
- stage: NuGet_Packaging_QNN
|
|
pool: 'Onnxruntime-QNNEP-Windows-2022-CPU'
|
|
dependsOn:
|
|
- OnnxRuntime_QNN_Nuget_Win_x64
|
|
- OnnxRuntime_QNN_Nuget_Win_Arm64
|
|
condition: succeeded()
|
|
jobs:
|
|
- job: NuGet_Packaging_QNN
|
|
workspace:
|
|
clean: all
|
|
steps:
|
|
- task: DownloadPipelineArtifact@0
|
|
displayName: 'Download Pipeline Artifact - QNN NuGet x64'
|
|
inputs:
|
|
artifactName: 'drop-nuget-qnn-x64'
|
|
targetPath: '$(Build.BinariesDirectory)/nuget-artifact-x64'
|
|
|
|
- task: DownloadPipelineArtifact@0
|
|
displayName: 'Download Pipeline Artifact - QNN NuGet arm64'
|
|
inputs:
|
|
artifactName: 'drop-nuget-qnn-arm64'
|
|
targetPath: '$(Build.BinariesDirectory)/nuget-artifact-arm64'
|
|
|
|
- task: PowerShell@2
|
|
displayName: 'Bundle NuGet'
|
|
inputs:
|
|
targetType: 'inline'
|
|
script: |
|
|
|
|
$x64_nupkgs = (Get-ChildItem $(Build.BinariesDirectory)/nuget-artifact-x64 -Filter Microsoft.ML.OnnxRuntime.QNN*.nupkg -Recurse)
|
|
$nuget_package_name = $x64_nupkgs[0].Name
|
|
$x64_nuget_package = $x64_nupkgs[0].FullName
|
|
|
|
$nupkg_unzipped_directory = [System.IO.Path]::Combine($Env:BUILD_ARTIFACTSTAGINGDIRECTORY, 'nuget_unzip_merged', [System.IO.Path]::GetFileNameWithoutExtension($nuget_package_name))
|
|
|
|
$x64_unzip_cmd = "7z.exe x $x64_nuget_package -y -o$nupkg_unzipped_directory"
|
|
Invoke-Expression -Command $x64_unzip_cmd
|
|
|
|
$arm64_nupkgs = (Get-ChildItem $(Build.BinariesDirectory)/nuget-artifact-arm64 -Filter Microsoft.ML.OnnxRuntime.QNN*.nupkg -Recurse)
|
|
$arm64_nuget_package = $arm64_nupkgs[0].FullName
|
|
|
|
$arm64_unzip_cmd = "7z.exe x $arm64_nuget_package -y -o$nupkg_unzipped_directory"
|
|
Invoke-Expression -Command $arm64_unzip_cmd
|
|
|
|
$merged_nuget_path = [System.IO.Path]::Combine($Env:BUILD_ARTIFACTSTAGINGDIRECTORY, 'nuget-artifact-merged')
|
|
if (!(Test-Path $merged_nuget_path)) {
|
|
New-Item -Path $merged_nuget_path -ItemType Directory
|
|
}
|
|
|
|
$merged_zip = [System.IO.Path]::Combine($merged_nuget_path, 'qnn_nuget.zip')
|
|
$zip_cmd = "7z.exe a -r $merged_zip $nupkg_unzipped_directory/*"
|
|
Invoke-Expression -Command $zip_cmd
|
|
|
|
$merged_nuget = [System.IO.Path]::Combine($merged_nuget_path, $nuget_package_name)
|
|
move $merged_zip $merged_nuget
|
|
workingDirectory: $(Build.BinariesDirectory)
|
|
|
|
- template: templates/esrp_nuget.yml
|
|
parameters:
|
|
DisplayName: 'ESRP - sign NuGet package'
|
|
FolderPath: '$(Build.ArtifactStagingDirectory)/nuget-artifact-merged'
|
|
DoEsrp: ${{ parameters.DoEsrp }}
|
|
|
|
- task: PublishPipelineArtifact@0
|
|
displayName: 'Publish Pipeline NuGet Artifact'
|
|
inputs:
|
|
artifactName: 'drop-signed-nuget-qnn'
|
|
targetPath: '$(Build.ArtifactStagingDirectory)/nuget-artifact-merged'
|