mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-05-31 23:27:43 +00:00
440 lines
18 KiB
YAML
440 lines
18 KiB
YAML
parameters:
|
|
- name: DoCompliance
|
|
displayName: Run Compliance Tasks?
|
|
type: boolean
|
|
default: true
|
|
|
|
- name: DoEsrp
|
|
displayName: Run code sign tasks? Must be true if you are doing an Onnx Runtime release.
|
|
type: boolean
|
|
default: false
|
|
|
|
- name: buildArch
|
|
type: string
|
|
|
|
- name: msbuildPlatform
|
|
type: string
|
|
|
|
- name: packageName
|
|
displayName: Package name for nuget
|
|
type: string
|
|
|
|
- name: buildparameter
|
|
displayName: Additional build commandline parameter
|
|
type: string
|
|
|
|
- name: runTests
|
|
type: boolean
|
|
default: false
|
|
|
|
- name: buildJava
|
|
type: boolean
|
|
default: true
|
|
|
|
- name: buildNodejs
|
|
type: boolean
|
|
default: false
|
|
|
|
- name: stage_name_suffix
|
|
displayName: job name for nuget
|
|
type: string
|
|
default: ''
|
|
|
|
# for inference packages '', for training packages '-training'
|
|
# used for drop-extra and c api artifacts (onnxruntime-win-* or onnxrutime-training-win-*)
|
|
- name: artifact_name_suffix
|
|
type: string
|
|
default: ''
|
|
|
|
- name: ort_build_pool_name
|
|
type: string
|
|
default: 'onnxruntime-Win-CPU-2022'
|
|
|
|
#'onnxruntime' or 'onnxruntime_gpu'
|
|
- name: java_artifact_id
|
|
type: string
|
|
default: 'onnxruntime'
|
|
|
|
- name: UseIncreasedTimeoutForTests
|
|
displayName: Increase timeout for tests? Set it to false if you are doing an Onnx Runtime release.
|
|
type: boolean
|
|
default: false
|
|
|
|
- name: PublishProtoc
|
|
type: boolean
|
|
default: false
|
|
|
|
- name: CudaVersion
|
|
type: string
|
|
default: '11.8'
|
|
values:
|
|
- 11.8
|
|
- 12.2
|
|
|
|
- name: ComboTests
|
|
type: boolean
|
|
default: false
|
|
|
|
- name: SpecificArtifact
|
|
displayName: Use Specific Artifact
|
|
type: boolean
|
|
default: false
|
|
|
|
- name: BuildId
|
|
displayName: Specific Artifact's BuildId
|
|
type: string
|
|
default: '0'
|
|
|
|
stages:
|
|
- stage: Windows_Packaging_${{ parameters.stage_name_suffix }}
|
|
dependsOn: []
|
|
variables:
|
|
GRADLE_OPTS: '-Dorg.gradle.daemon=false'
|
|
VSGenerator: 'Visual Studio 17 2022'
|
|
CUDA_MODULE_LOADING: 'LAZY'
|
|
${{ if eq(parameters['buildJava'], 'true') }}:
|
|
buildJavaParameter: '--build_java'
|
|
${{ else }}:
|
|
buildJavaParameter: ''
|
|
${{ if eq(parameters['UseIncreasedTimeoutForTests'], 'true') }}:
|
|
timeoutParameter: '--test_all_timeout 72000'
|
|
${{ else }}:
|
|
timeoutParameter: ''
|
|
jobs:
|
|
- job: Windows_Packaging_${{ parameters.stage_name_suffix }}
|
|
workspace:
|
|
clean: all
|
|
${{ if contains(parameters.ort_build_pool_name, 'GPU') }}:
|
|
pool: onnxruntime-Win-CPU-2022
|
|
${{ else }}:
|
|
pool: ${{ parameters.ort_build_pool_name }}
|
|
${{ if eq(parameters['UseIncreasedTimeoutForTests'], 'true') }}:
|
|
timeoutInMinutes: 1200
|
|
${{ else }}:
|
|
timeoutInMinutes: 360
|
|
|
|
steps:
|
|
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
|
|
displayName: 'Clean Agent Directories'
|
|
condition: always()
|
|
|
|
- checkout: self
|
|
clean: true
|
|
submodules: none
|
|
|
|
- task: UsePythonVersion@0
|
|
inputs:
|
|
versionSpec: '3.12'
|
|
addToPath: true
|
|
architecture: ${{ parameters.buildArch }}
|
|
|
|
- template: telemetry-steps.yml
|
|
|
|
- task: PipAuthenticate@1
|
|
displayName: 'Pip Authenticate'
|
|
inputs:
|
|
artifactFeeds: 'Lotus'
|
|
|
|
- ${{ if eq(parameters['buildJava'], 'true') }}:
|
|
- task: JavaToolInstaller@0
|
|
inputs:
|
|
versionSpec: "17"
|
|
jdkArchitectureOption: ${{ parameters.buildArch }}
|
|
jdkSourceOption: 'PreInstalled'
|
|
|
|
|
|
- task: NodeTool@0
|
|
condition: and(succeeded(), eq('${{ parameters.buildNodejs}}', true))
|
|
inputs:
|
|
versionSpec: '20.x'
|
|
|
|
- ${{ if ne(parameters.CudaVersion, '') }}:
|
|
- template: jobs/download_win_gpu_library.yml
|
|
parameters:
|
|
CudaVersion: ${{ parameters.CudaVersion }}
|
|
${{ if contains(parameters.buildparameter, 'use_cuda') }}:
|
|
DownloadCUDA: true
|
|
${{ if contains(parameters.buildparameter, 'use_tensorrt') }}:
|
|
DownloadCUDA: true
|
|
DownloadTRT: true
|
|
- powershell: |
|
|
Write-Host "##vso[task.prependpath]C:\Program Files (x86)\dotnet"
|
|
displayName: 'Append dotnet x86 Directory to PATH'
|
|
condition: and(succeeded(), eq('${{ parameters.buildArch}}', 'x86'))
|
|
|
|
- template: download-deps.yml
|
|
|
|
- task: PythonScript@0
|
|
displayName: 'Update deps.txt'
|
|
inputs:
|
|
scriptPath: $(Build.SourcesDirectory)/tools/ci_build/replace_urls_in_deps.py
|
|
arguments: --new_dir $(Build.BinariesDirectory)/deps
|
|
workingDirectory: $(Build.BinariesDirectory)
|
|
|
|
- task: PowerShell@2
|
|
displayName: 'Install ONNX'
|
|
inputs:
|
|
filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/windows/install_third_party_deps.ps1'
|
|
workingDirectory: '$(Build.BinariesDirectory)'
|
|
arguments: -cpu_arch ${{ parameters.buildArch }} -install_prefix $(Build.BinariesDirectory)\RelWithDebInfo\installed -build_config RelWithDebInfo
|
|
|
|
- template: set-version-number-variables-step.yml
|
|
|
|
- task: PythonScript@0
|
|
displayName: 'Generate cmake config'
|
|
inputs:
|
|
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
|
|
arguments: '--config RelWithDebInfo --use_binskim_compliant_compile_flags --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "$(VSGenerator)" --enable_onnx_tests $(TelemetryOption) ${{ parameters.buildparameter }} $(timeoutParameter) $(buildJavaParameter)'
|
|
workingDirectory: '$(Build.BinariesDirectory)'
|
|
|
|
|
|
- task: VSBuild@1
|
|
displayName: 'Build'
|
|
inputs:
|
|
solution: '$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln'
|
|
platform: ${{ parameters.msbuildPlatform }}
|
|
configuration: RelWithDebInfo
|
|
msbuildArchitecture: ${{ parameters.buildArch }}
|
|
maximumCpuCount: true # default is num logical cores worth of projects building concurrently
|
|
logProjectEvents: true
|
|
workingFolder: '$(Build.BinariesDirectory)\RelWithDebInfo'
|
|
createLogFile: true
|
|
|
|
# For CPU job, tests are run in the same machine as building
|
|
- ${{ if eq(parameters.buildJava, 'true') }}:
|
|
- template: make_java_win_binaries.yml
|
|
parameters:
|
|
msbuildPlatform: ${{ parameters.msbuildPlatform }}
|
|
java_artifact_id: ${{ parameters.java_artifact_id }}
|
|
${{ if or(contains(parameters.buildparameter, 'use_cuda'), contains(parameters.buildparameter, 'use_tensorrt')) }}:
|
|
# When it is a GPU build, we only assemble the java binaries, testing will be done in the later stage with GPU machine
|
|
buildOnly: true
|
|
${{ else }}:
|
|
buildOnly: false
|
|
|
|
- task: PublishBuildArtifacts@1
|
|
displayName: 'Publish Java temp binaries'
|
|
inputs:
|
|
pathtoPublish: '$(Build.BinariesDirectory)\onnxruntime-java-win-${{ parameters.msbuildPlatform }}'
|
|
artifactName: 'drop-onnxruntime-java-win-${{ parameters.packageName }}${{parameters.artifact_name_suffix}}'
|
|
# All GPU builds will be tested in the next stage with GPU machine
|
|
- ${{ if contains(parameters.ort_build_pool_name, 'CPU') }}:
|
|
- task: PythonScript@0
|
|
displayName: 'test'
|
|
condition: and(succeeded(), eq('${{ parameters.runTests}}', true))
|
|
inputs:
|
|
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
|
|
arguments: '--config RelWithDebInfo --use_binskim_compliant_compile_flags --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --test --skip_submodule_sync --build_shared_lib --cmake_generator "$(VSGenerator)" --enable_onnx_tests $(TelemetryOption) ${{ parameters.buildparameter }}'
|
|
workingDirectory: '$(Build.BinariesDirectory)'
|
|
- ${{ else }}:
|
|
- powershell: |
|
|
New-Item $(Agent.TempDirectory)/RelWithDebInfo -Force -ItemType Directory
|
|
Copy-Item -Path "$(Build.BinariesDirectory)/RelWithDebInfo/CTestTestfile.cmake" -Destination $(Agent.TempDirectory)/RelWithDebInfo/ -Force
|
|
Copy-Item -Path "$(Build.BinariesDirectory)/RelWithDebInfo/RelWithDebInfo" -Destination $(Agent.TempDirectory)/RelWithDebInfo/ -Recurse -Force
|
|
Get-ChildItem -Path "$(Agent.TempDirectory)/RelWithDebInfo" -Include *.pdb -File -Recurse | ForEach-Object { $_.Delete() }
|
|
Get-ChildItem -Path "$(Agent.TempDirectory)/RelWithDebInfo" -Include *.lib -File -Recurse | ForEach-Object { $_.Delete() }
|
|
Copy-Item -Path $(Build.BinariesDirectory)/RelWithDebInfo/RelWithDebInfo/onnxruntime.pdb -Destination $(Agent.TempDirectory)/RelWithDebInfo/RelWithDebInfo -Force
|
|
cd $(Agent.TempDirectory)/RelWithDebInfo
|
|
tree /f
|
|
displayName: 'Copy native test needs files'
|
|
workingDirectory: '$(Build.BinariesDirectory)'
|
|
|
|
- ${{ if eq(parameters['buildJava'], 'true') }}:
|
|
- powershell: |
|
|
Copy-Item -Path $(Build.BinariesDirectory)/RelWithDebInfo/RelWithDebInfo/onnxruntime4j_jni.pdb -Destination $(Agent.TempDirectory)/RelWithDebInfo/RelWithDebInfo -Force
|
|
Copy-Item -Path "$(Build.BinariesDirectory)/RelWithDebInfo/java" -Destination $(Agent.TempDirectory)/RelWithDebInfo/ -Recurse -Force
|
|
cd $(Agent.TempDirectory)/RelWithDebInfo
|
|
tree /f
|
|
displayName: 'Copy java pad and folder for java test'
|
|
workingDirectory: '$(Build.BinariesDirectory)'
|
|
|
|
- task: PublishPipelineArtifact@1
|
|
inputs:
|
|
targetPath: '$(Agent.TempDirectory)/RelWithDebInfo'
|
|
artifactName: 'Windows_Packaging_${{ parameters.stage_name_suffix }}_build_artifacts'
|
|
publishLocation: 'pipeline'
|
|
|
|
- script: |
|
|
dir *.dll
|
|
mkdir $(Build.ArtifactStagingDirectory)\testdata
|
|
workingDirectory: '$(Build.BinariesDirectory)/RelWithDebInfo/RelWithDebInfo'
|
|
displayName: 'List built DLLs'
|
|
|
|
- template: c-api-artifacts-package-and-publish-steps-windows.yml
|
|
parameters:
|
|
buildConfig: RelWithDebInfo
|
|
artifactName: 'onnxruntime${{ parameters.artifact_name_suffix }}-win-${{ parameters.packageName }}-$(OnnxRuntimeVersion)'
|
|
artifactNameNoVersionString: 'onnxruntime${{ parameters.artifact_name_suffix }}-win-${{ parameters.packageName }}'
|
|
commitId: $(OnnxRuntimeGitCommitHash)
|
|
DoEsrp: ${{ parameters.DoEsrp }}
|
|
|
|
- ${{ if eq(parameters.buildNodejs, true) }}:
|
|
- template: nodejs-artifacts-package-and-publish-steps-windows.yml
|
|
parameters:
|
|
arch: ${{ parameters.packageName }}
|
|
artifactName: 'drop-onnxruntime-nodejs-win-${{ parameters.packageName }}${{ parameters.artifact_name_suffix }}'
|
|
DoEsrp: ${{ parameters.DoEsrp }}
|
|
|
|
#Upload protoc.exe, which will be used in nuget build for generating C# files
|
|
- task: PublishPipelineArtifact@1
|
|
displayName: Publish protoc as drop-extra
|
|
condition: and(succeeded(), or(eq('${{ parameters.packageName}}', 'x64'), eq('${{ parameters.PublishProtoc}}', true)))
|
|
inputs:
|
|
targetPath: '$(Build.BinariesDirectory)\RelWithDebInfo\installed\bin\protoc.exe'
|
|
artifactName: 'drop-extra${{ parameters.artifact_name_suffix }}'
|
|
|
|
- task: CopyFiles@2
|
|
displayName: 'Copy custom_op_library to: $(Build.ArtifactStagingDirectory)'
|
|
condition: and(succeeded(), eq('${{ parameters.packageName}}', 'x64'))
|
|
inputs:
|
|
SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo'
|
|
Contents: 'custom_op_library.dll'
|
|
TargetFolder: '$(Build.ArtifactStagingDirectory)/testdata'
|
|
|
|
- ${{ if eq(parameters['DoCompliance'], 'true') }}:
|
|
- task: CredScan@3
|
|
displayName: 'Run CredScan'
|
|
inputs:
|
|
debugMode: false
|
|
continueOnError: true
|
|
|
|
- task: BinSkim@4
|
|
displayName: 'Run BinSkim'
|
|
inputs:
|
|
AnalyzeTargetGlob: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\**\*.dll'
|
|
continueOnError: true
|
|
|
|
- task: PostAnalysis@2
|
|
inputs:
|
|
GdnBreakAllTools: false
|
|
GdnBreakGdnToolBinSkim: true
|
|
GdnBreakPolicy: M365
|
|
GdnBreakPolicyMinSev: Error
|
|
|
|
- task: TSAUpload@2
|
|
displayName: 'TSA upload'
|
|
condition: and (succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main'))
|
|
inputs:
|
|
GdnPublishTsaOnboard: false
|
|
GdnPublishTsaConfigFile: '$(Build.sourcesDirectory)\.gdn\.gdntsa'
|
|
continueOnError: true
|
|
|
|
- template: component-governance-component-detection-steps.yml
|
|
parameters :
|
|
condition : 'succeeded'
|
|
- ${{ if contains(parameters.ort_build_pool_name, 'GPU') }}:
|
|
- stage: Windows_Packaging_${{ parameters.stage_name_suffix }}_Testing
|
|
dependsOn: Windows_Packaging_${{ parameters.stage_name_suffix }}
|
|
jobs:
|
|
- job: Windows_Packaging_${{ parameters.stage_name_suffix }}_Testing
|
|
workspace:
|
|
clean: all
|
|
pool: ${{ parameters.ort_build_pool_name }}
|
|
timeoutInMinutes: 180
|
|
steps:
|
|
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
|
|
displayName: 'Clean Agent Directories'
|
|
condition: always()
|
|
|
|
- script:
|
|
echo ${{ parameters.SpecificArtifact }}
|
|
displayName: 'Print Specific Artifact'
|
|
|
|
- checkout: self
|
|
clean: true
|
|
submodules: none
|
|
|
|
- template: flex-downloadPipelineArtifact.yml
|
|
parameters:
|
|
ArtifactName: "Windows_Packaging_${{ parameters.stage_name_suffix }}_build_artifacts"
|
|
StepName: 'Download Pipeline Artifact - Windows GPU Packages Build'
|
|
TargetPath: '$(Build.BinariesDirectory)/RelWithDebInfo/'
|
|
SpecificArtifact: ${{ parameters.SpecificArtifact }}
|
|
BuildId: ${{ parameters.BuildId }}
|
|
|
|
- powershell: |
|
|
tree /f
|
|
$drive = (Get-Location).Drive.Name
|
|
$file = 'CTestTestfile.cmake'
|
|
(Get-Content $file ) -replace 'C:\\a', -join($drive, ':\\a') | Set-Content $file
|
|
(Get-Content $file ) -replace 'C:/a', -join($drive, ':/a') | Set-Content $file
|
|
displayName: 'List built files and update CTestTestfile.cmake drive letter'
|
|
workingDirectory: '$(Build.BinariesDirectory)/RelWithDebInfo/'
|
|
|
|
- template: telemetry-steps.yml
|
|
|
|
- template: set-version-number-variables-step.yml
|
|
|
|
- ${{ if eq(parameters['buildJava'], 'true') }}:
|
|
- task: JavaToolInstaller@0
|
|
inputs:
|
|
versionSpec: "17"
|
|
jdkArchitectureOption: ${{ parameters.buildArch }}
|
|
jdkSourceOption: 'PreInstalled'
|
|
|
|
- task: UsePythonVersion@0
|
|
inputs:
|
|
versionSpec: '3.12'
|
|
addToPath: true
|
|
architecture: ${{ parameters.buildArch }}
|
|
|
|
- task: PipAuthenticate@1
|
|
displayName: 'Pip Authenticate'
|
|
inputs:
|
|
artifactFeeds: 'Lotus'
|
|
|
|
- task: NodeTool@0
|
|
condition: and(succeeded(), eq('${{ parameters.buildNodejs}}', true))
|
|
inputs:
|
|
versionSpec: '20.x'
|
|
|
|
- ${{ if ne(parameters.CudaVersion, '') }}:
|
|
- template: jobs/download_win_gpu_library.yml
|
|
parameters:
|
|
CudaVersion: ${{ parameters.CudaVersion }}
|
|
${{ if contains(parameters.buildparameter, 'use_cuda') }}:
|
|
DownloadCUDA: true
|
|
${{ if contains(parameters.buildparameter, 'use_tensorrt') }}:
|
|
DownloadCUDA: true
|
|
DownloadTRT: true
|
|
|
|
- powershell: |
|
|
Write-Host "##vso[task.prependpath]C:\Program Files (x86)\dotnet"
|
|
displayName: 'Append dotnet x86 Directory to PATH'
|
|
condition: and(succeeded(), eq('${{ parameters.buildArch}}', 'x86'))
|
|
|
|
- ${{ if eq(parameters.ComboTests, 'true') }}:
|
|
- task: PythonScript@0
|
|
displayName: 'test excludes CUDA'
|
|
condition: and(succeeded(), eq('${{ parameters.runTests}}', true))
|
|
inputs:
|
|
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
|
|
arguments: '--config RelWithDebInfo --use_binskim_compliant_compile_flags --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --test --enable_onnx_tests $(TelemetryOption) '
|
|
workingDirectory: '$(Build.BinariesDirectory)'
|
|
env:
|
|
NO_CUDA_TEST: '1'
|
|
- task: PythonScript@0
|
|
displayName: 'test excludes DML'
|
|
condition: and(succeeded(), eq('${{ parameters.runTests}}', true))
|
|
inputs:
|
|
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
|
|
arguments: '--config RelWithDebInfo --use_binskim_compliant_compile_flags --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --test --enable_onnx_tests $(TelemetryOption) '
|
|
workingDirectory: '$(Build.BinariesDirectory)'
|
|
env:
|
|
NO_DML_TEST: '1'
|
|
- ${{ else }}:
|
|
- task: PythonScript@0
|
|
displayName: 'test'
|
|
condition: and(succeeded(), eq('${{ parameters.runTests}}', true))
|
|
inputs:
|
|
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
|
|
arguments: '--config RelWithDebInfo --use_binskim_compliant_compile_flags --enable_lto --disable_rtti --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --test --enable_onnx_tests $(TelemetryOption) '
|
|
workingDirectory: '$(Build.BinariesDirectory)'
|
|
|
|
# Previous stage only assembles the java binaries, testing will be done in this stage with GPU machine
|
|
- ${{ if eq(parameters.buildJava, 'true') }}:
|
|
- template: make_java_win_binaries.yml
|
|
parameters:
|
|
msbuildPlatform: ${{ parameters.msbuildPlatform }}
|
|
java_artifact_id: ${{ parameters.java_artifact_id }}
|
|
buildOnly: false
|