mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-05-18 21:21:17 +00:00
Windows CI build steps template (#17263)
### Description 1. New windows ci build steps template. 2. Remove useless variables. ### Motivation and Context 1. Make it easier to apply build cache to all windows CIs. 2. Other team's devs only need to take care of build options ###Comparision Before:9f21f694cf/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml (L19-L82)After:b4c1f2261b/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml (L35-L54)
This commit is contained in:
parent
680fac64ed
commit
756eda2cc4
4 changed files with 124 additions and 77 deletions
|
|
@ -0,0 +1,93 @@
|
|||
parameters:
|
||||
- name: WithCache
|
||||
displayName: Build with Cache
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
- name: Today
|
||||
type: string
|
||||
default: ""
|
||||
|
||||
- name: CacheDir
|
||||
type: string
|
||||
default: "$(Agent.TempDirectory)/ort_ccache"
|
||||
|
||||
- name: AdditionalKey
|
||||
type: string
|
||||
default: ""
|
||||
|
||||
# it is used to pass additional arguments to build.py
|
||||
- name: BuildPyArguments
|
||||
type: string
|
||||
default: ""
|
||||
|
||||
# it is used to pass msbuild arguments to VSBuild@1
|
||||
- name: MsbuildArguments
|
||||
type: string
|
||||
default: ""
|
||||
|
||||
# it is used to pass platform arguments to VSBuild@1
|
||||
- name: Platform
|
||||
type: string
|
||||
default: "x64"
|
||||
|
||||
# it is used to pass configuration arguments to VSBuild@1
|
||||
- name: BuildConfig
|
||||
type: string
|
||||
|
||||
# it is used to pass msbuildArchitecture arguments to VSBuild@1
|
||||
- name: BuildArch
|
||||
type: string
|
||||
|
||||
- name: CacheArg
|
||||
type: string
|
||||
default: '/p:CLToolExe=cl.exe /p:CLToolPath=C:\ProgramData\chocolatey\bin /p:TrackFileAccess=false /p:UseMultiToolTask=true /p:DebugInformationFormat=OldStyle'
|
||||
|
||||
steps:
|
||||
- ${{ if eq(parameters.WithCache, true) }}:
|
||||
- task: Cache@2
|
||||
inputs:
|
||||
${{if eq(variables['Build.SourceBranchName'], 'merge')}}:
|
||||
key: ' "$(TODAY)" | ${{parameters.AdditionalKey}} | merge '
|
||||
${{else}}:
|
||||
key: '"$(TODAY)" | onnxruntime | ${{parameters.AdditionalKey}} | $(Build.SourceVersion) '
|
||||
path: ${{parameters.CacheDir}}
|
||||
restoreKeys: |
|
||||
"$(TODAY)" | onnxruntime | ${{parameters.AdditionalKey}}
|
||||
displayName: Cache Task
|
||||
|
||||
- task: PythonScript@0
|
||||
displayName: 'Generate cmake config'
|
||||
inputs:
|
||||
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
|
||||
${{ if eq(parameters.WithCache, true) }}:
|
||||
arguments: '${{parameters.BuildPyArguments}} --use_cache'
|
||||
${{ else }}:
|
||||
arguments: '${{parameters.BuildPyArguments}}'
|
||||
workingDirectory: '$(Build.BinariesDirectory)'
|
||||
|
||||
- task: VSBuild@1
|
||||
displayName: 'Build'
|
||||
inputs:
|
||||
solution: '$(Build.BinariesDirectory)\${{parameters.BuildConfig}}\onnxruntime.sln'
|
||||
platform: ${{parameters.Platform}}
|
||||
configuration: ${{parameters.BuildConfig}}
|
||||
${{ if eq(parameters.WithCache, true) }}:
|
||||
msbuildArgs: '${{parameters.MsbuildArguments}} ${{parameters.CacheArg}}'
|
||||
${{ else }}:
|
||||
arguments: '${{parameters.CMakeArguments}}'
|
||||
msbuildArchitecture: ${{parameters.BuildArch}}
|
||||
maximumCpuCount: true
|
||||
logProjectEvents: false
|
||||
workingFolder: '$(Build.BinariesDirectory)\${{parameters.BuildConfig}}'
|
||||
createLogFile: true
|
||||
env:
|
||||
CCACHE_DIR: ${{parameters.CacheDir}}
|
||||
|
||||
- ${{ if eq(parameters.WithCache, true) }}:
|
||||
- powershell: |
|
||||
ccache -sv
|
||||
ccache -z
|
||||
displayName: cache stat
|
||||
env:
|
||||
CCACHE_DIR: ${{parameters.CacheDir}}
|
||||
|
|
@ -78,7 +78,6 @@ jobs:
|
|||
${{ if eq(parameters.WITH_CACHE, true) }}:
|
||||
PS_CACHE_ARG: '-use_cache'
|
||||
PY_CACHE_ARG: '--use_cache'
|
||||
MSBUILD_CACHE_ARG: '/p:CLToolExe=cl.exe /p:CLToolPath=C:\ProgramData\chocolatey\bin /p:TrackFileAccess=false /p:UseMultiToolTask=true /p:DebugInformationFormat=OldStyle'
|
||||
workspace:
|
||||
clean: all
|
||||
pool: ${{ parameters.MachinePool }}
|
||||
|
|
@ -137,48 +136,17 @@ jobs:
|
|||
nugetConfigPath: '$(Build.SourcesDirectory)\NuGet.config'
|
||||
restoreDirectory: '$(Build.BinariesDirectory)\${{ parameters.BuildConfig }}'
|
||||
|
||||
- ${{ if eq(parameters.WITH_CACHE, true) }}:
|
||||
- task: Cache@2
|
||||
inputs:
|
||||
${{if eq(variables['Build.SourceBranchName'], 'merge')}}:
|
||||
key: ' "$(TODAY)" | $(System.StageName) | ${{ parameters.BuildConfig }} | merge '
|
||||
${{else}}:
|
||||
key: '"$(TODAY)" | onnxruntime | $(System.StageName) | ${{ parameters.BuildConfig }} | $(Build.SourceVersion) '
|
||||
path: $(ORT_CACHE_DIR)
|
||||
restoreKeys: |
|
||||
"$(TODAY)" | onnxruntime | $(System.StageName) | ${{ parameters.BuildConfig }}
|
||||
displayName: Cache Task
|
||||
|
||||
- task: PythonScript@0
|
||||
displayName: 'Generate cmake config'
|
||||
inputs:
|
||||
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
|
||||
arguments: '--config ${{ parameters.BuildConfig }} --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_csharp --update --parallel --cmake_generator "Visual Studio 17 2022" --build_shared_lib --enable_onnx_tests ${{ variables.PY_CACHE_ARG }} ${{ parameters.additionalBuildFlags }}'
|
||||
workingDirectory: '$(Build.BinariesDirectory)'
|
||||
|
||||
- task: VSBuild@1
|
||||
displayName: 'Build'
|
||||
inputs:
|
||||
solution: '$(Build.BinariesDirectory)\${{ parameters.BuildConfig }}\onnxruntime.sln'
|
||||
platform: ${{ parameters.msbuildPlatform }}
|
||||
configuration: ${{ parameters.BuildConfig }}
|
||||
msbuildArgs: '-maxcpucount ${{ variables.MSBUILD_CACHE_ARG }}'
|
||||
msbuildArchitecture: ${{ parameters.buildArch }}
|
||||
maximumCpuCount: true
|
||||
logProjectEvents: false
|
||||
workingFolder: '$(Build.BinariesDirectory)\${{ parameters.BuildConfig }}'
|
||||
createLogFile: true
|
||||
${{ if eq(parameters.WITH_CACHE, true) }}:
|
||||
env:
|
||||
CCACHE_DIR: $(ORT_CACHE_DIR)
|
||||
|
||||
- ${{ if eq(parameters.WITH_CACHE, true) }}:
|
||||
- powershell: |
|
||||
ccache -sv
|
||||
ccache -z
|
||||
displayName: cache stat
|
||||
env:
|
||||
CCACHE_DIR: $(ORT_CACHE_DIR)
|
||||
- template: win-ci-build-steps.yml
|
||||
parameters:
|
||||
WithCache: ${{ parameters.WITH_CACHE }}
|
||||
Today: $(TODAY)
|
||||
CacheDir: $(ORT_CACHE_DIR)
|
||||
AdditionalKey: " $(System.StageName) | ${{ parameters.BuildConfig }} "
|
||||
BuildPyArguments: '--config ${{ parameters.BuildConfig }} --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_csharp --update --parallel --cmake_generator "Visual Studio 17 2022" --build_shared_lib --enable_onnx_tests ${{ parameters.additionalBuildFlags }}'
|
||||
MsbuildArguments: '-maxcpucount'
|
||||
BuildArch: ${{ parameters.buildArch }}
|
||||
Platform: ${{ parameters.msbuildPlatform }}
|
||||
BuildConfig: ${{ parameters.BuildConfig }}
|
||||
|
||||
- powershell: |
|
||||
Get-Volume $("$(Build.BinariesDirectory)")[0]
|
||||
|
|
|
|||
|
|
@ -9,13 +9,9 @@ jobs:
|
|||
minsizerel:
|
||||
BuildConfig: 'MinSizeRel'
|
||||
variables:
|
||||
OrtPackageId: 'Microsoft.ML.OnnxRuntime'
|
||||
MsbuildArguments: '-detailedsummary -maxcpucount -consoleloggerparameters:PerformanceSummary'
|
||||
OnnxRuntimeBuildDirectory: '$(Build.BinariesDirectory)'
|
||||
DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
|
||||
EnvSetupScript: setup_env_cuda_11.bat
|
||||
buildArch: x64
|
||||
setVcvars: true
|
||||
TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)]
|
||||
timeoutInMinutes: 120
|
||||
workspace:
|
||||
|
|
@ -31,12 +27,16 @@ jobs:
|
|||
WithCache: true
|
||||
Today: $(Today)
|
||||
|
||||
- task: PythonScript@0
|
||||
displayName: 'Build and test'
|
||||
inputs:
|
||||
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
|
||||
arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_generator "Visual Studio 17 2022" --build_wheel --use_cuda --cuda_home="$(Agent.TempDirectory)\v11.8" --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=75" --include_ops_by_config="$(Build.SourcesDirectory)\onnxruntime\test\testdata\required_ops.config"'
|
||||
workingDirectory: '$(Build.BinariesDirectory)'
|
||||
- template: templates/jobs/win-ci-build-steps.yml
|
||||
parameters:
|
||||
WithCache: True
|
||||
Today: $(TODAY)
|
||||
AdditionalKey: "gpu-reduced-ops | $(BuildConfig)"
|
||||
BuildPyArguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --update --skip_submodule_sync --cmake_generator "Visual Studio 17 2022" --build_wheel --use_cuda --cuda_home="$(Agent.TempDirectory)\v11.8" --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=75" --include_ops_by_config="$(Build.SourcesDirectory)\onnxruntime\test\testdata\required_ops.config"'
|
||||
MsbuildArguments: $(MsbuildArguments)
|
||||
BuildArch: $(buildArch)
|
||||
Platform: 'x64'
|
||||
BuildConfig: $(BuildConfig)
|
||||
|
||||
- template: templates/component-governance-component-detection-steps.yml
|
||||
parameters :
|
||||
|
|
|
|||
|
|
@ -22,15 +22,10 @@ jobs:
|
|||
- job: 'build'
|
||||
pool: 'onnxruntime-Win2022-GPU-T4'
|
||||
variables:
|
||||
OrtPackageId: 'Microsoft.ML.OnnxRuntime'
|
||||
MsbuildArguments: '-detailedsummary -maxcpucount -consoleloggerparameters:PerformanceSummary'
|
||||
OnnxRuntimeBuildDirectory: '$(Build.BinariesDirectory)'
|
||||
DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
|
||||
EnvSetupScript: setup_env_trt.bat
|
||||
buildArch: x64
|
||||
setVcvars: true
|
||||
BuildConfig: 'RelWithDebInfo'
|
||||
ALLOW_RELEASED_ONNX_OPSET_ONLY: '1'
|
||||
skipComponentGovernanceDetection: true
|
||||
TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)]
|
||||
timeoutInMinutes: 150
|
||||
|
|
@ -47,25 +42,16 @@ jobs:
|
|||
WithCache: true
|
||||
Today: $(Today)
|
||||
|
||||
- task: PythonScript@0
|
||||
displayName: 'Generate cmake config'
|
||||
inputs:
|
||||
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
|
||||
arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 17 2022" --build_wheel --enable_onnx_tests --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.6.1.6.Windows10.x86_64.cuda-11.8" --cuda_version=11.8 --cuda_home="$(Agent.TempDirectory)\v11.8" --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=75'
|
||||
workingDirectory: '$(Build.BinariesDirectory)'
|
||||
|
||||
- task: VSBuild@1
|
||||
displayName: 'Build'
|
||||
inputs:
|
||||
solution: '$(Build.BinariesDirectory)\$(BuildConfig)\onnxruntime.sln'
|
||||
platform: 'x64'
|
||||
configuration: $(BuildConfig)
|
||||
msbuildArgs: $(MsbuildArguments)
|
||||
msbuildArchitecture: $(buildArch)
|
||||
maximumCpuCount: true
|
||||
logProjectEvents: false
|
||||
workingFolder: '$(Build.BinariesDirectory)\$(BuildConfig)'
|
||||
createLogFile: true
|
||||
- template: templates/jobs/win-ci-build-steps.yml
|
||||
parameters:
|
||||
WithCache: True
|
||||
Today: $(TODAY)
|
||||
AdditionalKey: "gpu-tensorrt | $(BuildConfig)"
|
||||
BuildPyArguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 17 2022" --build_wheel --enable_onnx_tests --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.6.1.6.Windows10.x86_64.cuda-11.8" --cuda_version=11.8 --cuda_home="$(Agent.TempDirectory)\v11.8" --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=75'
|
||||
MsbuildArguments: $(MsbuildArguments)
|
||||
BuildArch: $(buildArch)
|
||||
Platform: 'x64'
|
||||
BuildConfig: $(BuildConfig)
|
||||
|
||||
- task: PythonScript@0
|
||||
displayName: 'Build wheel'
|
||||
|
|
|
|||
Loading…
Reference in a new issue