saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-30 20:18:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Improve windows build cache steps (#15537)

Browse source 
### Description
1. Split deps' compilation cache and ort's
2. reduce the caches generation in merge branch.

### Motivation and Context
Reduce pipeline cache stage.
This commit is contained in:
Yi Zhang 2023-04-20 09:42:22 +08:00 committed by GitHub
parent 29d00fb776
commit 5b6f79e79b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 57 additions and 35 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

92
tools/ci_build/github/azure-pipelines/templates/win-ci-vs-2019.yml
View file

@ -65,7 +65,8 @@ jobs:
ALLOW_RELEASED_ONNX_OPSET_ONLY: '0'
DocUpdateNeeded: false # Set to true during document generation if there are diffs
skipComponentGovernanceDetection: true
CCACHE_DIR: $(Pipeline.Workspace)/ccache
DEPS_CACHE_DIR: $(Agent.TempDirectory)/deps_ccache
ORT_CACHE_DIR: $(Agent.TempDirectory)/ort_ccache
TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)]
${{ if eq(parameters.WITH_CACHE, true) }}:
PS_CACHE_ARG: '-use_cache'
@ -130,44 +131,48 @@ jobs:
workingDirectory: '$(Build.BinariesDirectory)'
displayName: 'Install python modules'
- powershell: |
if ([string]::IsNullOrEmpty((Get-Command ccache -errorAction SilentlyContinue)))
{
choco install ccache -y --version 4.7.4
$ccache_path = (Get-Command ccache).Source
$ccache_parent_dir = (Split-Path -parent $ccache_path)
Copy-Item "C:\ProgramData\chocolatey\lib\ccache\tools\ccache-4.7.4-windows-x86_64\ccache.exe" -Destination "C:\ProgramData\chocolatey\bin\cl.exe"
Get-ChildItem $ccache_parent_dir
ccache --version
}
displayName: Install ccache and update PATH to use linked versions of gcc, cc, etc
condition: eq(${{ parameters.WITH_CACHE }}, true)
- task: Cache@2
inputs:
key: '"$(TODAY)" | ccache | "$(System.StageName)" | "$(Build.SourceBranch)" | "$(Build.SourceVersion)" '
path: $(CCACHE_DIR)
restoreKeys: |
"$(TODAY)" | ccache | "$(System.StageName)" | "$(Build.SourceBranch)"
"$(TODAY)" | ccache | "$(System.StageName)"
"$(TODAY)" | ccache |
displayName: Cache Task
condition: eq(${{ parameters.WITH_CACHE }}, true)
- ${{ if eq(parameters.WITH_CACHE, true) }}:
- powershell: |
if ([string]::IsNullOrEmpty((Get-Command ccache -errorAction SilentlyContinue)))
{
choco install ccache -y --version 4.7.4
$ccache_path = (Get-Command ccache).Source
$ccache_parent_dir = (Split-Path -parent $ccache_path)
Copy-Item "C:\ProgramData\chocolatey\lib\ccache\tools\ccache-4.7.4-windows-x86_64\ccache.exe" -Destination "C:\ProgramData\chocolatey\bin\cl.exe"
Get-ChildItem $ccache_parent_dir
ccache --version
}
displayName: Install ccache and update PATH to use linked versions of gcc, cc, etc
- ${{ if or(eq(parameters.RunOnnxRuntimeTests, true), eq(parameters.GenerateDocumentation, true)) }}:
- ${{ if eq(parameters.WITH_CACHE, true) }}:
- task: Cache@2
# machinepool is used to ensure the compiler is same
inputs:
key: '"$(TODAY)" | ${{ parameters.buildArch }} | ${{ parameters.BuildConfig }} | ${{ parameters.MachinePool }} | $(Build.SourcesDirectory)/cmake/deps.txt, $(Build.SourcesDirectory)/tools/ci_build/github/windows/install_third_party_deps.ps1, $(Build.SourcesDirectory)/tools/ci_build/github/windows/helpers.ps1'
path: $(DEPS_CACHE_DIR)
restoreKeys: |
"$(TODAY)" | ${{ parameters.buildArch }} | ${{ parameters.BuildConfig }} | ${{ parameters.MachinePool }}
displayName: Cache Task
- task: PowerShell@2
displayName: 'Install ONNX'
inputs:
filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/windows/install_third_party_deps.ps1'
workingDirectory: '$(Build.BinariesDirectory)'
arguments: -cpu_arch ${{ parameters.buildArch }} -install_prefix $(Build.BinariesDirectory)\${{ parameters.BuildConfig }}\installed -build_config ${{ parameters.BuildConfig }} ${{ variables['PS_CACHE_ARG'] }}
${{ if eq(parameters.WITH_CACHE, true) }}:
env:
CCACHE_DIR: $(DEPS_CACHE_DIR)
CCACHE_COMPILERCHECK: content
- powershell: |
ccache -sv
ccache -z
displayName: cache stat
condition: eq(${{ parameters.WITH_CACHE }}, true)
- ${{ if eq(parameters.WITH_CACHE, true) }}:
- powershell: |
ccache -sv
ccache -z
displayName: cache stat
env:
CCACHE_DIR: $(DEPS_CACHE_DIR)
- task: NuGetToolInstaller@0
displayName: Use Nuget 5.7.0
@ -183,6 +188,18 @@ jobs:
nugetConfigPath: '$(Build.SourcesDirectory)\NuGet.config'
restoreDirectory: '$(Build.BinariesDirectory)\${{ parameters.BuildConfig }}'
- ${{ if eq(parameters.WITH_CACHE, true) }}:
- task: Cache@2
inputs:
${{if eq(variables['Build.SourceBranchName'], 'merge')}}:
key: ' "$(TODAY)" | $(System.StageName) | ${{ parameters.BuildConfig }} | merge '
${{else}}:
key: '"$(TODAY)" | onnxruntime | $(System.StageName) | ${{ parameters.BuildConfig }} | $(Build.SourceVersion) '
path: $(ORT_CACHE_DIR)
restoreKeys: |
"$(TODAY)" | onnxruntime | $(System.StageName) | ${{ parameters.BuildConfig }}
displayName: Cache Task
- task: PythonScript@0
displayName: 'Generate cmake config'
inputs:
@ -202,12 +219,17 @@ jobs:
logProjectEvents: false
workingFolder: '$(Build.BinariesDirectory)\${{ parameters.BuildConfig }}'
createLogFile: true
${{ if eq(parameters.WITH_CACHE, true) }}:
env:
CCACHE_DIR: $(ORT_CACHE_DIR)
- powershell: |
ccache -sv
ccache -z
displayName: cache stat
condition: eq(${{ parameters.WITH_CACHE }}, true)
- ${{ if eq(parameters.WITH_CACHE, true) }}:
- powershell: |
ccache -sv
ccache -z
displayName: cache stat
env:
CCACHE_DIR: $(ORT_CACHE_DIR)
- powershell: |
Get-Volume D