mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-26 19:52:38 +00:00
4af593a722
1. Add python 3.13 to our python packaging pipelines 2. Because numpy 2.0.0 doesn't support thread free python, this PR also upgrades numpy to the latest 3. Delete some unused files.
400 lines
No EOL
16 KiB
YAML
400 lines
No EOL
16 KiB
YAML
parameters:
|
|
- name: DoCompliance
|
|
displayName: Run Compliance Tasks?
|
|
type: boolean
|
|
default: true
|
|
|
|
- name: DoEsrp
|
|
displayName: Run code sign tasks? Must be true if you are doing an Onnx Runtime release.
|
|
type: boolean
|
|
default: false
|
|
|
|
- name: buildArch
|
|
type: string
|
|
|
|
- name: msbuildPlatform
|
|
type: string
|
|
|
|
- name: packageName
|
|
displayName: Package name for nuget
|
|
type: string
|
|
|
|
- name: buildparameter
|
|
displayName: Additional build commandline parameter
|
|
type: string
|
|
|
|
- name: runTests
|
|
type: boolean
|
|
default: true
|
|
|
|
- name: buildJava
|
|
type: boolean
|
|
default: true
|
|
|
|
- name: buildNodejs
|
|
type: boolean
|
|
default: false
|
|
|
|
- name: stage_name_suffix
|
|
displayName: job name for nuget
|
|
type: string
|
|
default: ''
|
|
|
|
# for inference packages '', for training packages '-training'
|
|
# used for drop-extra and c api artifacts (onnxruntime-win-* or onnxrutime-training-win-*)
|
|
- name: artifact_name_suffix
|
|
type: string
|
|
default: ''
|
|
|
|
- name: ort_build_pool_name
|
|
type: string
|
|
default: 'onnxruntime-Win-CPU-2022'
|
|
|
|
#'onnxruntime' or 'onnxruntime_gpu'
|
|
- name: java_artifact_id
|
|
type: string
|
|
default: 'onnxruntime'
|
|
|
|
- name: UseIncreasedTimeoutForTests
|
|
displayName: Increase timeout for tests? Set it to false if you are doing an Onnx Runtime release.
|
|
type: boolean
|
|
default: false
|
|
|
|
- name: PublishProtoc
|
|
type: boolean
|
|
default: false
|
|
|
|
- name: CudaVersion
|
|
type: string
|
|
default: '11.8'
|
|
values:
|
|
- 11.8
|
|
- 12.2
|
|
|
|
- name: SpecificArtifact
|
|
displayName: Use Specific Artifact
|
|
type: boolean
|
|
default: false
|
|
|
|
- name: BuildId
|
|
displayName: Specific Artifact's BuildId
|
|
type: string
|
|
default: '0'
|
|
|
|
stages:
|
|
- stage: Windows_Packaging_${{ parameters.stage_name_suffix }}
|
|
dependsOn: []
|
|
variables:
|
|
GRADLE_OPTS: '-Dorg.gradle.daemon=false'
|
|
VSGenerator: 'Visual Studio 17 2022'
|
|
CUDA_MODULE_LOADING: 'LAZY'
|
|
${{ if eq(parameters['buildJava'], 'true') }}:
|
|
buildJavaParameter: '--build_java'
|
|
${{ else }}:
|
|
buildJavaParameter: ''
|
|
${{ if eq(parameters['UseIncreasedTimeoutForTests'], 'true') }}:
|
|
timeoutParameter: '--test_all_timeout 72000'
|
|
${{ else }}:
|
|
timeoutParameter: ''
|
|
jobs:
|
|
- job: Windows_Packaging_${{ parameters.stage_name_suffix }}
|
|
workspace:
|
|
clean: all
|
|
${{ if contains(parameters.ort_build_pool_name, 'GPU') }}:
|
|
pool: onnxruntime-Win-CPU-2022
|
|
${{ else }}:
|
|
pool: ${{ parameters.ort_build_pool_name }}
|
|
${{ if eq(parameters['UseIncreasedTimeoutForTests'], 'true') }}:
|
|
timeoutInMinutes: 1200
|
|
${{ else }}:
|
|
timeoutInMinutes: 360
|
|
|
|
steps:
|
|
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
|
|
displayName: 'Clean Agent Directories'
|
|
condition: always()
|
|
|
|
- checkout: self
|
|
clean: true
|
|
submodules: none
|
|
|
|
- template: telemetry-steps.yml
|
|
|
|
- ${{ if eq(parameters['buildJava'], 'true') }}:
|
|
- task: JavaToolInstaller@0
|
|
inputs:
|
|
versionSpec: "11"
|
|
jdkArchitectureOption: ${{ parameters.buildArch }}
|
|
jdkSourceOption: 'PreInstalled'
|
|
|
|
- task: UsePythonVersion@0
|
|
inputs:
|
|
versionSpec: '3.12'
|
|
addToPath: true
|
|
architecture: ${{ parameters.buildArch }}
|
|
|
|
- task: NodeTool@0
|
|
condition: and(succeeded(), eq('${{ parameters.buildNodejs}}', true))
|
|
inputs:
|
|
versionSpec: '18.x'
|
|
|
|
- ${{ if ne(parameters.CudaVersion, '') }}:
|
|
- template: jobs/download_win_gpu_library.yml
|
|
parameters:
|
|
CudaVersion: ${{ parameters.CudaVersion }}
|
|
${{ if contains(parameters.buildparameter, 'use_cuda') }}:
|
|
DownloadCUDA: true
|
|
${{ if contains(parameters.buildparameter, 'use_tensorrt') }}:
|
|
DownloadCUDA: true
|
|
DownloadTRT: true
|
|
- powershell: |
|
|
Write-Host "##vso[task.prependpath]C:\Program Files (x86)\dotnet"
|
|
displayName: 'Append dotnet x86 Directory to PATH'
|
|
condition: and(succeeded(), eq('${{ parameters.buildArch}}', 'x86'))
|
|
|
|
- template: download-deps.yml
|
|
|
|
- task: PythonScript@0
|
|
displayName: 'Update deps.txt'
|
|
inputs:
|
|
scriptPath: $(Build.SourcesDirectory)/tools/ci_build/replace_urls_in_deps.py
|
|
arguments: --new_dir $(Build.BinariesDirectory)/deps
|
|
workingDirectory: $(Build.BinariesDirectory)
|
|
|
|
- task: PowerShell@2
|
|
displayName: 'Install ONNX'
|
|
inputs:
|
|
filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/windows/install_third_party_deps.ps1'
|
|
workingDirectory: '$(Build.BinariesDirectory)'
|
|
arguments: -cpu_arch ${{ parameters.buildArch }} -install_prefix $(Build.BinariesDirectory)\RelWithDebInfo\installed -build_config RelWithDebInfo
|
|
|
|
- template: set-version-number-variables-step.yml
|
|
|
|
- task: PythonScript@0
|
|
displayName: 'Generate cmake config'
|
|
inputs:
|
|
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
|
|
arguments: '--config RelWithDebInfo --use_binskim_compliant_compile_flags --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "$(VSGenerator)" --enable_onnx_tests $(TelemetryOption) ${{ parameters.buildparameter }} $(timeoutParameter) $(buildJavaParameter)'
|
|
workingDirectory: '$(Build.BinariesDirectory)'
|
|
|
|
|
|
- task: VSBuild@1
|
|
displayName: 'Build'
|
|
inputs:
|
|
solution: '$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln'
|
|
platform: ${{ parameters.msbuildPlatform }}
|
|
configuration: RelWithDebInfo
|
|
msbuildArchitecture: ${{ parameters.buildArch }}
|
|
maximumCpuCount: true # default is num logical cores worth of projects building concurrently
|
|
logProjectEvents: true
|
|
workingFolder: '$(Build.BinariesDirectory)\RelWithDebInfo'
|
|
createLogFile: true
|
|
|
|
# For CPU job, tests are run in the same machine as building
|
|
- ${{ if eq(parameters.buildJava, 'true') }}:
|
|
- template: make_java_win_binaries.yml
|
|
parameters:
|
|
msbuildPlatform: ${{ parameters.msbuildPlatform }}
|
|
java_artifact_id: ${{ parameters.java_artifact_id }}
|
|
${{ if or(contains(parameters.buildparameter, 'use_cuda'), contains(parameters.buildparameter, 'use_tensorrt')) }}:
|
|
# When it is a GPU build, we only assemble the java binaries, testing will be done in the later stage with GPU machine
|
|
buildOnly: true
|
|
${{ else }}:
|
|
buildOnly: false
|
|
|
|
- task: PublishBuildArtifacts@1
|
|
displayName: 'Publish Java temp binaries'
|
|
inputs:
|
|
pathtoPublish: '$(Build.BinariesDirectory)\onnxruntime-java-win-${{ parameters.msbuildPlatform }}'
|
|
artifactName: 'drop-onnxruntime-java-win-${{ parameters.packageName }}${{parameters.artifact_name_suffix}}'
|
|
# All GPU builds will be tested in the next stage with GPU machine
|
|
- ${{ if contains(parameters.ort_build_pool_name, 'CPU') }}:
|
|
- task: PythonScript@0
|
|
displayName: 'test'
|
|
condition: and(succeeded(), eq('${{ parameters.runTests}}', true))
|
|
inputs:
|
|
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
|
|
arguments: '--config RelWithDebInfo --use_binskim_compliant_compile_flags --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --test --cmake_generator "$(VSGenerator)" --enable_onnx_tests $(TelemetryOption) ${{ parameters.buildparameter }}'
|
|
workingDirectory: '$(Build.BinariesDirectory)'
|
|
- ${{ else }}:
|
|
- powershell: |
|
|
New-Item $(Agent.TempDirectory)/RelWithDebInfo -Force -ItemType Directory
|
|
Copy-Item -Path "$(Build.BinariesDirectory)/RelWithDebInfo/CTestTestfile.cmake" -Destination $(Agent.TempDirectory)/RelWithDebInfo/ -Force
|
|
Copy-Item -Path "$(Build.BinariesDirectory)/RelWithDebInfo/RelWithDebInfo" -Destination $(Agent.TempDirectory)/RelWithDebInfo/ -Recurse -Force
|
|
Get-ChildItem -Path "$(Agent.TempDirectory)/RelWithDebInfo" -Include *.pdb -File -Recurse | ForEach-Object { $_.Delete() }
|
|
Get-ChildItem -Path "$(Agent.TempDirectory)/RelWithDebInfo" -Include *.lib -File -Recurse | ForEach-Object { $_.Delete() }
|
|
Copy-Item -Path $(Build.BinariesDirectory)/RelWithDebInfo/RelWithDebInfo/onnxruntime.pdb -Destination $(Agent.TempDirectory)/RelWithDebInfo/RelWithDebInfo -Force
|
|
cd $(Agent.TempDirectory)/RelWithDebInfo
|
|
tree /f
|
|
displayName: 'Copy native test needs files'
|
|
workingDirectory: '$(Build.BinariesDirectory)'
|
|
|
|
- ${{ if eq(parameters['buildJava'], 'true') }}:
|
|
- powershell: |
|
|
Copy-Item -Path $(Build.BinariesDirectory)/RelWithDebInfo/RelWithDebInfo/onnxruntime4j_jni.pdb -Destination $(Agent.TempDirectory)/RelWithDebInfo/RelWithDebInfo -Force
|
|
Copy-Item -Path "$(Build.BinariesDirectory)/RelWithDebInfo/java" -Destination $(Agent.TempDirectory)/RelWithDebInfo/ -Recurse -Force
|
|
cd $(Agent.TempDirectory)/RelWithDebInfo
|
|
tree /f
|
|
displayName: 'Copy java pad and folder for java test'
|
|
workingDirectory: '$(Build.BinariesDirectory)'
|
|
|
|
- task: PublishPipelineArtifact@1
|
|
inputs:
|
|
targetPath: '$(Agent.TempDirectory)/RelWithDebInfo'
|
|
artifactName: 'Windows_Packaging_${{ parameters.stage_name_suffix }}_build_artifacts'
|
|
publishLocation: 'pipeline'
|
|
|
|
- script: |
|
|
dir *.dll
|
|
mkdir $(Build.ArtifactStagingDirectory)\testdata
|
|
workingDirectory: '$(Build.BinariesDirectory)/RelWithDebInfo/RelWithDebInfo'
|
|
displayName: 'List built DLLs'
|
|
|
|
- template: c-api-artifacts-package-and-publish-steps-windows.yml
|
|
parameters:
|
|
buildConfig: RelWithDebInfo
|
|
artifactName: 'onnxruntime${{ parameters.artifact_name_suffix }}-win-${{ parameters.packageName }}-$(OnnxRuntimeVersion)'
|
|
artifactNameNoVersionString: 'onnxruntime${{ parameters.artifact_name_suffix }}-win-${{ parameters.packageName }}'
|
|
commitId: $(OnnxRuntimeGitCommitHash)
|
|
DoEsrp: ${{ parameters.DoEsrp }}
|
|
|
|
- ${{ if eq(parameters.buildNodejs, true) }}:
|
|
- template: nodejs-artifacts-package-and-publish-steps-windows.yml
|
|
parameters:
|
|
arch: ${{ parameters.packageName }}
|
|
artifactName: 'drop-onnxruntime-nodejs-win-${{ parameters.packageName }}${{ parameters.artifact_name_suffix }}'
|
|
DoEsrp: ${{ parameters.DoEsrp }}
|
|
|
|
#Upload protoc.exe, which will be used in nuget build for generating C# files
|
|
- task: PublishPipelineArtifact@1
|
|
displayName: Publish protoc as drop-extra
|
|
condition: and(succeeded(), or(eq('${{ parameters.packageName}}', 'x64'), eq('${{ parameters.PublishProtoc}}', true)))
|
|
inputs:
|
|
targetPath: '$(Build.BinariesDirectory)\RelWithDebInfo\installed\bin\protoc.exe'
|
|
artifactName: 'drop-extra${{ parameters.artifact_name_suffix }}'
|
|
|
|
- task: CopyFiles@2
|
|
displayName: 'Copy custom_op_library to: $(Build.ArtifactStagingDirectory)'
|
|
condition: and(succeeded(), eq('${{ parameters.packageName}}', 'x64'))
|
|
inputs:
|
|
SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo'
|
|
Contents: 'custom_op_library.dll'
|
|
TargetFolder: '$(Build.ArtifactStagingDirectory)/testdata'
|
|
|
|
- ${{ if eq(parameters['DoCompliance'], 'true') }}:
|
|
- task: CredScan@3
|
|
displayName: 'Run CredScan'
|
|
inputs:
|
|
debugMode: false
|
|
continueOnError: true
|
|
|
|
- task: BinSkim@4
|
|
displayName: 'Run BinSkim'
|
|
inputs:
|
|
AnalyzeTargetGlob: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\**\*.dll'
|
|
continueOnError: true
|
|
|
|
- task: PostAnalysis@2
|
|
inputs:
|
|
GdnBreakAllTools: false
|
|
GdnBreakGdnToolBinSkim: true
|
|
GdnBreakPolicy: M365
|
|
GdnBreakPolicyMinSev: Error
|
|
|
|
- task: TSAUpload@2
|
|
displayName: 'TSA upload'
|
|
condition: and (succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main'))
|
|
inputs:
|
|
GdnPublishTsaOnboard: false
|
|
GdnPublishTsaConfigFile: '$(Build.sourcesDirectory)\.gdn\.gdntsa'
|
|
continueOnError: true
|
|
|
|
- template: component-governance-component-detection-steps.yml
|
|
parameters :
|
|
condition : 'succeeded'
|
|
- ${{ if contains(parameters.ort_build_pool_name, 'GPU') }}:
|
|
- stage: Windows_Packaging_${{ parameters.stage_name_suffix }}_Testing
|
|
dependsOn: Windows_Packaging_${{ parameters.stage_name_suffix }}
|
|
jobs:
|
|
- job: Windows_Packaging_${{ parameters.stage_name_suffix }}_Testing
|
|
workspace:
|
|
clean: all
|
|
pool: ${{ parameters.ort_build_pool_name }}
|
|
timeoutInMinutes: 180
|
|
steps:
|
|
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
|
|
displayName: 'Clean Agent Directories'
|
|
condition: always()
|
|
|
|
- checkout: self
|
|
clean: true
|
|
submodules: none
|
|
|
|
- template: flex-downloadPipelineArtifact.yml
|
|
parameters:
|
|
ArtifactName: "Windows_Packaging_${{ parameters.stage_name_suffix }}_build_artifacts"
|
|
StepName: 'Download Pipeline Artifact - Windows GPU Packages Build'
|
|
TargetPath: '$(Build.BinariesDirectory)/RelWithDebInfo/'
|
|
SpecificArtifact: ${{ parameters.SpecificArtifact }}
|
|
BuildId: ${{ parameters.BuildId }}
|
|
|
|
- powershell: |
|
|
tree /f
|
|
$drive = (Get-Location).Drive.Name
|
|
$file = 'CTestTestfile.cmake'
|
|
(Get-Content $file ) -replace 'C:\\a', -join($drive, ':\\a') | Set-Content $file
|
|
(Get-Content $file ) -replace 'C:/a', -join($drive, ':/a') | Set-Content $file
|
|
displayName: 'List built files and update CTestTestfile.cmake drive letter'
|
|
workingDirectory: '$(Build.BinariesDirectory)/RelWithDebInfo/'
|
|
|
|
- template: telemetry-steps.yml
|
|
|
|
- template: set-version-number-variables-step.yml
|
|
|
|
- ${{ if eq(parameters['buildJava'], 'true') }}:
|
|
- task: JavaToolInstaller@0
|
|
inputs:
|
|
versionSpec: "11"
|
|
jdkArchitectureOption: ${{ parameters.buildArch }}
|
|
jdkSourceOption: 'PreInstalled'
|
|
|
|
- task: UsePythonVersion@0
|
|
inputs:
|
|
versionSpec: '3.8'
|
|
addToPath: true
|
|
architecture: ${{ parameters.buildArch }}
|
|
|
|
- task: NodeTool@0
|
|
condition: and(succeeded(), eq('${{ parameters.buildNodejs}}', true))
|
|
inputs:
|
|
versionSpec: '18.x'
|
|
|
|
- ${{ if ne(parameters.CudaVersion, '') }}:
|
|
- template: jobs/download_win_gpu_library.yml
|
|
parameters:
|
|
CudaVersion: ${{ parameters.CudaVersion }}
|
|
${{ if contains(parameters.buildparameter, 'use_cuda') }}:
|
|
DownloadCUDA: true
|
|
${{ if contains(parameters.buildparameter, 'use_tensorrt') }}:
|
|
DownloadCUDA: true
|
|
DownloadTRT: true
|
|
|
|
- powershell: |
|
|
Write-Host "##vso[task.prependpath]C:\Program Files (x86)\dotnet"
|
|
displayName: 'Append dotnet x86 Directory to PATH'
|
|
condition: and(succeeded(), eq('${{ parameters.buildArch}}', 'x86'))
|
|
|
|
- task: PythonScript@0
|
|
displayName: 'test'
|
|
condition: and(succeeded(), eq('${{ parameters.runTests}}', true))
|
|
inputs:
|
|
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
|
|
arguments: '--config RelWithDebInfo --use_binskim_compliant_compile_flags --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --test --enable_onnx_tests $(TelemetryOption) '
|
|
workingDirectory: '$(Build.BinariesDirectory)'
|
|
# Previous stage only assembles the java binaries, testing will be done in this stage with GPU machine
|
|
- ${{ if eq(parameters.buildJava, 'true') }}:
|
|
- template: make_java_win_binaries.yml
|
|
parameters:
|
|
msbuildPlatform: ${{ parameters.msbuildPlatform }}
|
|
java_artifact_id: ${{ parameters.java_artifact_id }}
|
|
buildOnly: false |