mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-02 03:55:34 +00:00
Improve 2 cache tasks in one pipeline yaml (#15267)
### Description
1. Make 2 cache tasks in one pipeline really works
2. Each building step has its own environment variable CCACHE_DIR
instead of job variables.
3. Extenal Protobuf compilation cache only updates with deps.txt. It
doesn't generate new cache in every commit.
### Motivation and Context
The simple workflow is as below
```
--------build with ccache-------
|
cache
|
{CCACHE_DIR}-----cache stat.
```
```
-------Cache@2------
|
download cache
|
{path}--------upload cache
```
1. {XXX} means environment variable or task input.
2. {CCACHE_DIR} must be consistent with {path}. Ccache produces caches
in {CCACHE_DIR} and Cache@2 download cache into {path} and tar {path}
and upload it.
3. Protobuf changes with deps.txt so that it would reduce the storage
size.
4. Next step, we may split the compilation into 2 steps, one for
external dependencies and another for ORT.
This commit is contained in:
parent
aab3c15585
commit
c5f5e3ec5e
3 changed files with 58 additions and 17 deletions
|
|
@ -0,0 +1,15 @@
|
|||
# it's used to check the cache stats after the build with ccache
|
||||
parameters:
|
||||
- name: CacheDir
|
||||
displayName: Cache Directory
|
||||
type: string
|
||||
|
||||
steps:
|
||||
- script: |
|
||||
ccache -sv
|
||||
ccache -z
|
||||
ls -l $CCACHE_DIR
|
||||
du -sh $CCACHE_DIR
|
||||
displayName: Show Cache stats
|
||||
env:
|
||||
CCACHE_DIR: ${{ parameters.CacheDir }}
|
||||
|
|
@ -22,11 +22,24 @@ parameters:
|
|||
type: boolean
|
||||
default: false
|
||||
|
||||
- name: WithCache
|
||||
displayName: Build with Cache
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
- name: CacheDir
|
||||
displayName: Cache Directory
|
||||
type: string
|
||||
default: ''
|
||||
|
||||
steps:
|
||||
- script: |
|
||||
rm -rf $(Build.BinariesDirectory)/Release
|
||||
python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --update --build ${{ parameters.AdditionalBuildFlags }} --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_shared_lib --config Release
|
||||
displayName: 'Build ${{ parameters.MacosArch }}'
|
||||
${{ if eq(parameters.WithCache, true) }}:
|
||||
env:
|
||||
CCACHE_DIR: ${{ parameters.CacheDir }}
|
||||
|
||||
- ${{ if eq(parameters.MacosArch, 'x86_64') }}:
|
||||
- script: |
|
||||
|
|
@ -50,7 +63,7 @@ steps:
|
|||
replaceExistingArchive: true
|
||||
|
||||
- script: |
|
||||
set -e -x
|
||||
set -e -x
|
||||
mkdir -p $(Build.ArtifactStagingDirectory)/testdata
|
||||
cp $(Build.BinariesDirectory)/Release/libcustom_op_library.dylib $(Build.ArtifactStagingDirectory)/testdata
|
||||
displayName: 'Copy libcustom_op_library.dylib to ArtifactStagingDirectory'
|
||||
|
|
|
|||
|
|
@ -33,8 +33,9 @@ jobs:
|
|||
variables:
|
||||
MACOSX_DEPLOYMENT_TARGET: '10.14'
|
||||
ALLOW_RELEASED_ONNX_OPSET_ONLY: ${{ parameters.AllowReleasedOpsetOnly }}
|
||||
CCACHE_DIR: $(Pipeline.Workspace)/ccache
|
||||
TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)]
|
||||
PROTO_CACHE_DIR: $(Pipeline.Workspace)/ccache_proto
|
||||
ORT_CACHE_DIR: $(Pipeline.Workspace)/ccache_ort
|
||||
pool:
|
||||
vmImage: 'macOS-12'
|
||||
timeoutInMinutes: 300
|
||||
|
|
@ -57,16 +58,18 @@ jobs:
|
|||
- script: |
|
||||
brew install ccache
|
||||
echo "##vso[task.prependpath]/usr/local/opt/ccache/libexec"
|
||||
mkdir -p $(PROTO_CACHE_DIR)
|
||||
mkdir -p $(ORT_CACHE_DIR)
|
||||
displayName: Install ccache and update PATH to use linked versions of gcc, cc, etc
|
||||
condition: eq(${{ parameters.WithCache }}, true)
|
||||
|
||||
- task: Cache@2
|
||||
inputs:
|
||||
key: ' "$(TODAY)" | ccache | protocol | "$(Build.SourceVersion)" '
|
||||
path: $(CCACHE_DIR)
|
||||
key: ' protobuf | "$(Agent.OS)" | $(Build.SourcesDirectory)/cmake/deps.txt, $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_protobuf.sh'
|
||||
path: $(PROTO_CACHE_DIR)
|
||||
restoreKeys: |
|
||||
"$(TODAY)" | ccache | protocol
|
||||
displayName: ccache protocol
|
||||
protobuf | "$(Agent.OS)"
|
||||
displayName: mac protobuf
|
||||
condition: eq(${{ parameters.WithCache }}, true)
|
||||
|
||||
- script: |
|
||||
|
|
@ -80,17 +83,19 @@ jobs:
|
|||
python3 -m pip install -r '$(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/requirements.txt'
|
||||
sudo xcode-select --switch /Applications/Xcode_13.1.app/Contents/Developer
|
||||
displayName: 'Install dependencies'
|
||||
${{ if eq(parameters.WithCache, true) }}:
|
||||
env:
|
||||
CCACHE_DIR: $(PROTO_CACHE_DIR)
|
||||
|
||||
- script: |
|
||||
ccache -s
|
||||
ccache -z
|
||||
displayName: Show Cache stats and Clear protolbuf stats.
|
||||
condition: eq(${{ parameters.WithCache }}, true)
|
||||
- ${{ if eq(parameters.WithCache, true) }}:
|
||||
- template: check-cache-stats.yml
|
||||
parameters:
|
||||
CacheDir: $(PROTO_CACHE_DIR)
|
||||
|
||||
- task: Cache@2
|
||||
inputs:
|
||||
key: ' "$(TODAY)" | ccache | onnxruntime | "$(Build.SourceVersion)" '
|
||||
path: $(CCACHE_DIR)
|
||||
path: $(ORT_CACHE_DIR)
|
||||
restoreKeys: |
|
||||
"$(TODAY)" | ccache | onnxruntime
|
||||
displayName: ccache MacOS Build
|
||||
|
|
@ -103,6 +108,9 @@ jobs:
|
|||
AdditionalBuildFlags: ${{ parameters.AdditionalBuildFlags }} --use_coreml --cmake_extra_defines CMAKE_OSX_ARCHITECTURES="arm64;x86_64"
|
||||
BuildJava: false
|
||||
BuildNodejs: false
|
||||
WithCache: ${{ parameters.WithCache }}
|
||||
${{ if eq(parameters.WithCache, true) }}:
|
||||
CacheDir: $(ORT_CACHE_DIR)
|
||||
|
||||
- ${{ if eq(parameters.MacosArch, 'arm64') }}:
|
||||
- template: mac-cpu-packaging-steps.yml
|
||||
|
|
@ -111,6 +119,9 @@ jobs:
|
|||
AdditionalBuildFlags: ${{ parameters.AdditionalBuildFlags }} --build_nodejs --build_java --use_coreml --cmake_extra_defines CMAKE_OSX_ARCHITECTURES=arm64
|
||||
BuildJava: true
|
||||
BuildNodejs: true
|
||||
WithCache: ${{ parameters.WithCache }}
|
||||
${{ if eq(parameters.WithCache, true) }}:
|
||||
CacheDir: $(ORT_CACHE_DIR)
|
||||
|
||||
- ${{ if eq(parameters.MacosArch, 'x86_64') }}:
|
||||
- template: mac-cpu-packaging-steps.yml
|
||||
|
|
@ -119,9 +130,11 @@ jobs:
|
|||
AdditionalBuildFlags: ${{ parameters.AdditionalBuildFlags }} --build_nodejs --build_java --use_coreml
|
||||
BuildJava: true
|
||||
BuildNodejs: true
|
||||
WithCache: ${{ parameters.WithCache }}
|
||||
${{ if eq(parameters.WithCache, true) }}:
|
||||
CacheDir: $(ORT_CACHE_DIR)
|
||||
|
||||
- script: |
|
||||
ccache -s
|
||||
ccache -z
|
||||
displayName: Show Cache stats
|
||||
condition: eq(${{ parameters.WithCache }}, true)
|
||||
- ${{ if eq(parameters.WithCache, true) }}:
|
||||
- template: check-cache-stats.yml
|
||||
parameters:
|
||||
CacheDir: $(ORT_CACHE_DIR)
|
||||
|
|
|
|||
Loading…
Reference in a new issue