mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-05-28 22:56:32 +00:00
4167b68abf
### Description Right now our "Zip-Nuget-Java-Nodejs Packaging Pipeline" is too big. This OnDevice training part is independent of the others, so it can be split out. Then our NPM Packaging pipeline will not depends on this training stuff. ### Motivation and Context Similar to #21235 Also, this PR fixed a problem that: "NuGet_Test_Linux_Training_CPU" job downloads artifacts from "onnxruntime-linux-x64" for getting customop shared libs, but the job forget to declare it depends on the "Linux_C_API_Packaging_CPU_x64" which produces the artifact. Such problems can be hard to find when a pipeline goes big.
301 lines
12 KiB
YAML
301 lines
12 KiB
YAML
parameters:
|
|
- name: RunOnnxRuntimeTests
|
|
displayName: Run Tests?
|
|
type: boolean
|
|
default: true
|
|
|
|
- name: UseIncreasedTimeoutForTests
|
|
displayName: Increase timeout for tests? Set it to false if you are doing an Onnx Runtime release.
|
|
type: boolean
|
|
default: false
|
|
|
|
- name: DoCompliance
|
|
displayName: Run Compliance Tasks?
|
|
type: boolean
|
|
default: true
|
|
|
|
- name: DoEsrp
|
|
displayName: Run code sign tasks? Must be true if you are doing an ONNX Runtime release
|
|
type: boolean
|
|
default: true
|
|
|
|
- name: IsReleaseBuild
|
|
displayName: Is a release build? Set it to true if you are doing an ONNX Runtime release.
|
|
type: boolean
|
|
default: false
|
|
|
|
- name: PreReleaseVersionSuffixString
|
|
displayName: Suffix added to pre-release package version. Only used if IsReleaseBuild is true. Denotes the type of pre-release package.
|
|
type: string
|
|
values:
|
|
- alpha
|
|
- beta
|
|
- rc
|
|
- none
|
|
default: none
|
|
|
|
- name: PreReleaseVersionSuffixNumber
|
|
displayName: Number added to pre-release package version. Only used if IsReleaseBuild is true. Denotes the sequence of a pre-release package.
|
|
type: number
|
|
default: 0
|
|
|
|
# these 2 parameters are used for debugging.
|
|
- name: SpecificArtifact
|
|
displayName: Use Specific Artifact (Debugging only)
|
|
type: boolean
|
|
default: false
|
|
|
|
- name: BuildId
|
|
displayName: Pipeline BuildId, you could find it in the URL
|
|
type: string
|
|
default: '0'
|
|
|
|
- name: NugetPackageSuffix
|
|
displayName: Suffix to append to nuget package
|
|
type: string
|
|
default: 'NONE'
|
|
|
|
- name: AdditionalBuildFlag
|
|
displayName: Build flags to append to build command
|
|
type: string
|
|
default: '--use_azure'
|
|
- name: QnnSdk
|
|
displayName: QNN SDK Version
|
|
type: string
|
|
default: 2.24.0.240626
|
|
|
|
resources:
|
|
repositories:
|
|
- repository: onnxruntime-inference-examples # The name used to reference this repository in the checkout step
|
|
type: github
|
|
endpoint: ort-examples
|
|
name: microsoft/onnxruntime-inference-examples
|
|
- repository: manylinux
|
|
type: Github
|
|
endpoint: Microsoft
|
|
name: pypa/manylinux
|
|
ref: 5eda9aded5462201e6310105728d33016e637ea7
|
|
|
|
variables:
|
|
- name: ReleaseVersionSuffix
|
|
value: ''
|
|
- name: win_trt_version
|
|
value: 11.8
|
|
|
|
- name: win_trt_home
|
|
value: $(Agent.TempDirectory)\TensorRT-10.2.0.19.Windows10.x86_64.cuda-11.8
|
|
- name: win_cuda_home
|
|
value: $(Agent.TempDirectory)\v11.8
|
|
|
|
stages:
|
|
- template: stages/set_packaging_variables_stage.yml
|
|
parameters:
|
|
IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
|
|
PreReleaseVersionSuffixString: ${{ parameters.PreReleaseVersionSuffixString }}
|
|
PreReleaseVersionSuffixNumber: ${{ parameters.PreReleaseVersionSuffixNumber }}
|
|
|
|
- template: stages/download-java-tools-stage.yml
|
|
|
|
- template: templates/c-api-cpu.yml
|
|
parameters:
|
|
RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }}
|
|
DoCompliance: ${{ parameters.DoCompliance }}
|
|
DoEsrp: ${{ parameters.DoEsrp }}
|
|
IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
|
|
${{ if eq(parameters.NugetPackageSuffix, 'NONE') }}:
|
|
OrtNugetPackageId: 'Microsoft.ML.OnnxRuntime'
|
|
${{ else }}:
|
|
OrtNugetPackageId: 'Microsoft.ML.OnnxRuntime${{ parameters.NugetPackageSuffix }}'
|
|
AdditionalBuildFlags: ''
|
|
AdditionalWinBuildFlags: '--enable_onnx_tests --enable_wcos ${{parameters.AdditionalBuildFlag}}'
|
|
BuildVariant: 'default'
|
|
SpecificArtifact: ${{ parameters.SpecificArtifact }}
|
|
BuildId: ${{ parameters.BuildId }}
|
|
|
|
- template: stages/java-cuda-packaging-stage.yml
|
|
parameters:
|
|
CudaVersion: 11.8
|
|
SpecificArtifact: ${{ parameters.SpecificArtifact }}
|
|
BuildId: ${{ parameters.BuildId }}
|
|
|
|
- template: stages/nuget-combine-cuda-stage.yml
|
|
parameters:
|
|
DoCompliance: ${{ parameters.DoCompliance }}
|
|
CudaVersion: 11.8
|
|
docker_base_image: 'nvidia/cuda:11.8.0-cudnn8-devel-ubi8'
|
|
RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }}
|
|
UseIncreasedTimeoutForTests: ${{ parameters.UseIncreasedTimeoutForTests }}
|
|
win_trt_home: ${{ variables.win_trt_home }}
|
|
win_cuda_home: ${{ variables.win_cuda_home }}
|
|
DoEsrp: ${{ parameters.DoEsrp }}
|
|
IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
|
|
buildJava: true
|
|
buildNodejs: true
|
|
SpecificArtifact: ${{ parameters.SpecificArtifact }}
|
|
BuildId: ${{ parameters.BuildId }}
|
|
|
|
|
|
- template: nuget/templates/dml-vs-2022.yml
|
|
parameters:
|
|
AgentPool: 'onnxruntime-Win2022-GPU-dml-A10'
|
|
IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
|
|
ArtifactName: 'drop-nuget-dml'
|
|
StageName: 'Windows_CI_GPU_DML_Dev'
|
|
BuildCommand: --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos --use_telemetry --use_dml --use_winml --build_nodejs --cmake_generator "Visual Studio 17 2022"
|
|
BuildArch: 'x64'
|
|
msbuildArchitecture: 'amd64'
|
|
EnvSetupScript: 'setup_env.bat'
|
|
sln_platform: 'x64'
|
|
DoDebugBuild: 'false'
|
|
DoNugetPack: 'true'
|
|
DoCompliance: 'false'
|
|
DoEsrp: ${{ parameters.DoEsrp }}
|
|
NuPackScript: |
|
|
msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /t:CreatePackage /p:OrtPackageId=Microsoft.ML.OnnxRuntime.DirectML /p:IsReleaseBuild=${{ parameters.IsReleaseBuild }} /p:CurrentData=$(BuildDate) /p:CurrentTime=$(BuildTime)
|
|
copy $(Build.SourcesDirectory)\csharp\src\Microsoft.ML.OnnxRuntime\bin\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory)
|
|
copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\*.nupkg $(Build.ArtifactStagingDirectory)
|
|
mkdir $(Build.ArtifactStagingDirectory)\testdata
|
|
copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\custom_op_library.* $(Build.ArtifactStagingDirectory)\testdata
|
|
|
|
- template: nuget/templates/dml-vs-2022.yml
|
|
parameters:
|
|
AgentPool: 'onnxruntime-Win2022-GPU-dml-A10'
|
|
IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
|
|
ArtifactName: 'drop-win-dml-x86-zip'
|
|
StageName: 'Windows_CI_GPU_DML_Dev_x86'
|
|
BuildCommand: --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos --use_telemetry --use_dml --use_winml --cmake_generator "Visual Studio 17 2022"
|
|
BuildArch: 'x86'
|
|
EnvSetupScript: 'setup_env_x86.bat'
|
|
sln_platform: 'Win32'
|
|
DoDebugBuild: 'false'
|
|
DoNugetPack: 'true'
|
|
DoCompliance: ${{ parameters.DoCompliance }}
|
|
DoEsrp: ${{ parameters.DoEsrp }}
|
|
RunTests: 'false'
|
|
NuPackScript: |
|
|
msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /p:TargetArchitecture=x86 /t:CreatePackage /p:OrtPackageId=Microsoft.ML.OnnxRuntime.DirectML /p:IsReleaseBuild=${{ parameters.IsReleaseBuild }}
|
|
cd $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\
|
|
ren Microsoft.ML.OnnxRuntime.DirectML.* win-dml-x86.zip
|
|
copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\win-dml-x86.zip $(Build.ArtifactStagingDirectory)
|
|
mkdir $(Build.ArtifactStagingDirectory)\testdata
|
|
copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\custom_op_library.* $(Build.ArtifactStagingDirectory)\testdata
|
|
|
|
- template: nuget/templates/dml-vs-2022.yml
|
|
parameters:
|
|
AgentPool: 'onnxruntime-Win2022-GPU-dml-A10'
|
|
IsReleaseBuild: ${{ parameters.IsReleaseBuild }}
|
|
ArtifactName: 'drop-win-dml-arm64-zip'
|
|
StageName: 'Windows_CI_GPU_DML_Dev_arm64'
|
|
BuildCommand: --build_dir $(Build.BinariesDirectory) --arm64 --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos --use_telemetry --use_dml --use_winml --build_nodejs --cmake_generator "Visual Studio 17 2022"
|
|
BuildArch: 'x64'
|
|
EnvSetupScript: 'setup_env.bat'
|
|
sln_platform: 'arm64'
|
|
DoDebugBuild: 'false'
|
|
DoNugetPack: 'true'
|
|
DoCompliance: ${{ parameters.DoCompliance }}
|
|
DoEsrp: ${{ parameters.DoEsrp }}
|
|
RunTests: 'false'
|
|
NuPackScript: |
|
|
msbuild $(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.proj /p:Configuration=RelWithDebInfo /p:TargetArchitecture=arm64 /t:CreatePackage /p:OrtPackageId=Microsoft.ML.OnnxRuntime.DirectML /p:IsReleaseBuild=${{ parameters.IsReleaseBuild }}
|
|
cd $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\
|
|
ren Microsoft.ML.OnnxRuntime.DirectML.* win-dml-arm64.zip
|
|
copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\win-dml-arm64.zip $(Build.ArtifactStagingDirectory)
|
|
mkdir $(Build.ArtifactStagingDirectory)\testdata
|
|
copy $(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\custom_op_library.* $(Build.ArtifactStagingDirectory)\testdata
|
|
|
|
- stage: NuGet_Packaging_DML
|
|
dependsOn:
|
|
- Windows_CI_GPU_DML_Dev
|
|
- Windows_CI_GPU_DML_Dev_x86
|
|
- Windows_CI_GPU_DML_Dev_arm64
|
|
condition: succeeded()
|
|
jobs:
|
|
- job: NuGet_Packaging_DML
|
|
workspace:
|
|
clean: all
|
|
pool: 'onnxruntime-Win2022-GPU-dml-A10'
|
|
steps:
|
|
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
|
|
displayName: 'Clean Agent Directories'
|
|
condition: always()
|
|
- task: DownloadPipelineArtifact@0
|
|
displayName: 'Download Pipeline Artifact - NuGet DirectML'
|
|
inputs:
|
|
artifactName: 'drop-nuget-dml'
|
|
targetPath: '$(Build.BinariesDirectory)/nuget-artifact-dml'
|
|
|
|
- task: DownloadPipelineArtifact@0
|
|
displayName: 'Download Pipeline Artifact - NuGet DirectML x86'
|
|
inputs:
|
|
artifactName: 'drop-win-dml-x86-zip'
|
|
targetPath: '$(Build.BinariesDirectory)/nuget-artifact-dml'
|
|
|
|
- task: DownloadPipelineArtifact@0
|
|
displayName: 'Download Pipeline Artifact - NuGet DirectML arm64'
|
|
inputs:
|
|
artifactName: 'drop-win-dml-arm64-zip'
|
|
targetPath: '$(Build.BinariesDirectory)/nuget-artifact-dml'
|
|
|
|
- script: |
|
|
pushd $(Build.BinariesDirectory)\nuget-artifact-dml
|
|
dir
|
|
powershell -Command "Invoke-WebRequest http://stahlworks.com/dev/unzip.exe -OutFile unzip.exe"
|
|
powershell -Command "Invoke-WebRequest http://stahlworks.com/dev/zip.exe -OutFile zip.exe"
|
|
set PATH=%CD%;%PATH%
|
|
SETLOCAL EnableDelayedExpansion
|
|
FOR /R %%i IN (*.nupkg) do (
|
|
set filename=%%~ni
|
|
IF NOT "!filename:~25,7!"=="Managed" (
|
|
rename %%~ni.nupkg %%~ni.zip
|
|
unzip %%~ni.zip -d %%~ni
|
|
del /Q %%~ni.zip
|
|
|
|
unzip win-dml-x86.zip -d win-x86
|
|
mkdir %%~ni\runtimes\win-x86
|
|
mkdir %%~ni\runtimes\win-x86\native
|
|
|
|
move win-x86\runtimes\win-x86\native\onnxruntime.dll %%~ni\runtimes\win-x86\native\onnxruntime.dll
|
|
move win-x86\runtimes\win-x86\native\onnxruntime.lib %%~ni\runtimes\win-x86\native\onnxruntime.lib
|
|
move win-x86\runtimes\win-x86\native\onnxruntime.pdb %%~ni\runtimes\win-x86\native\onnxruntime.pdb
|
|
|
|
unzip win-dml-arm64.zip -d win-arm64
|
|
mkdir %%~ni\runtimes\win-arm64
|
|
mkdir %%~ni\runtimes\win-arm64\native
|
|
|
|
move win-arm64\runtimes\win-arm64\native\onnxruntime.dll %%~ni\runtimes\win-arm64\native\onnxruntime.dll
|
|
move win-arm64\runtimes\win-arm64\native\onnxruntime.lib %%~ni\runtimes\win-arm64\native\onnxruntime.lib
|
|
move win-arm64\runtimes\win-arm64\native\onnxruntime.pdb %%~ni\runtimes\win-arm64\native\onnxruntime.pdb
|
|
|
|
|
|
pushd %%~ni
|
|
zip -r ..\%%~ni.zip .
|
|
popd
|
|
move %%~ni.zip %%~ni.nupkg
|
|
)
|
|
)
|
|
popd
|
|
copy $(Build.BinariesDirectory)\nuget-artifact-dml\Microsoft.ML.OnnxRuntime.DirectML*nupkg $(Build.ArtifactStagingDirectory)
|
|
displayName: 'Bundle DML NuGet and other binaries'
|
|
|
|
- template: templates/esrp_nuget.yml
|
|
parameters:
|
|
DisplayName: 'ESRP - sign NuGet package'
|
|
FolderPath: '$(Build.ArtifactStagingDirectory)'
|
|
DoEsrp: ${{ parameters.DoEsrp }}
|
|
|
|
- template: templates/validate-package.yml
|
|
parameters:
|
|
PackageType: 'nuget'
|
|
PackagePath: '$(Build.ArtifactStagingDirectory)'
|
|
PackageName: 'Microsoft.ML.OnnxRuntime.DirectML*nupkg'
|
|
PlatformsSupported: 'win-x64,win-x86,win-arm64'
|
|
VerifyNugetSigning: ${{ parameters.DoEsrp }}
|
|
|
|
- task: PublishPipelineArtifact@0
|
|
displayName: 'Publish Pipeline NuGet Artifact'
|
|
inputs:
|
|
artifactName: 'drop-signed-nuget-dml'
|
|
targetPath: '$(Build.ArtifactStagingDirectory)'
|
|
- template: templates/component-governance-component-detection-steps.yml
|
|
parameters:
|
|
condition: 'succeeded'
|