Improve 2 cache tasks in one pipeline yaml (#15267)

### Description
1. Make 2 cache tasks in one pipeline really works
2. Each building step has its own environment variable CCACHE_DIR
instead of job variables.
3. Extenal Protobuf compilation cache only updates with deps.txt. It
doesn't generate new cache in every commit.


### Motivation and Context
The simple workflow is as below
```
--------build with ccache-------             
         |                       
        cache                    
         |                       
      {CCACHE_DIR}-----cache stat.
```

```
-------Cache@2------
           |
    download cache           
           |                         
          {path}--------upload cache
```

1. {XXX} means environment variable or task input.
2. {CCACHE_DIR} must be consistent with {path}. Ccache produces caches
in {CCACHE_DIR} and Cache@2 download cache into {path} and tar {path}
and upload it.
3. Protobuf changes with deps.txt so that it would reduce the storage
size.
4. Next step, we may split the compilation into 2 steps, one for
external dependencies and another for ORT.
This commit is contained in:
Yi Zhang 2023-03-30 23:22:11 +08:00 committed by GitHub
parent aab3c15585
commit c5f5e3ec5e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 58 additions and 17 deletions

View file

@ -0,0 +1,15 @@
# it's used to check the cache stats after the build with ccache
parameters:
- name: CacheDir
displayName: Cache Directory
type: string
steps:
- script: |
ccache -sv
ccache -z
ls -l $CCACHE_DIR
du -sh $CCACHE_DIR
displayName: Show Cache stats
env:
CCACHE_DIR: ${{ parameters.CacheDir }}

View file

@ -22,11 +22,24 @@ parameters:
type: boolean
default: false
- name: WithCache
displayName: Build with Cache
type: boolean
default: false
- name: CacheDir
displayName: Cache Directory
type: string
default: ''
steps:
- script: |
rm -rf $(Build.BinariesDirectory)/Release
python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --update --build ${{ parameters.AdditionalBuildFlags }} --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_shared_lib --config Release
displayName: 'Build ${{ parameters.MacosArch }}'
${{ if eq(parameters.WithCache, true) }}:
env:
CCACHE_DIR: ${{ parameters.CacheDir }}
- ${{ if eq(parameters.MacosArch, 'x86_64') }}:
- script: |
@ -50,7 +63,7 @@ steps:
replaceExistingArchive: true
- script: |
set -e -x
set -e -x
mkdir -p $(Build.ArtifactStagingDirectory)/testdata
cp $(Build.BinariesDirectory)/Release/libcustom_op_library.dylib $(Build.ArtifactStagingDirectory)/testdata
displayName: 'Copy libcustom_op_library.dylib to ArtifactStagingDirectory'

View file

@ -33,8 +33,9 @@ jobs:
variables:
MACOSX_DEPLOYMENT_TARGET: '10.14'
ALLOW_RELEASED_ONNX_OPSET_ONLY: ${{ parameters.AllowReleasedOpsetOnly }}
CCACHE_DIR: $(Pipeline.Workspace)/ccache
TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)]
PROTO_CACHE_DIR: $(Pipeline.Workspace)/ccache_proto
ORT_CACHE_DIR: $(Pipeline.Workspace)/ccache_ort
pool:
vmImage: 'macOS-12'
timeoutInMinutes: 300
@ -57,16 +58,18 @@ jobs:
- script: |
brew install ccache
echo "##vso[task.prependpath]/usr/local/opt/ccache/libexec"
mkdir -p $(PROTO_CACHE_DIR)
mkdir -p $(ORT_CACHE_DIR)
displayName: Install ccache and update PATH to use linked versions of gcc, cc, etc
condition: eq(${{ parameters.WithCache }}, true)
- task: Cache@2
inputs:
key: ' "$(TODAY)" | ccache | protocol | "$(Build.SourceVersion)" '
path: $(CCACHE_DIR)
key: ' protobuf | "$(Agent.OS)" | $(Build.SourcesDirectory)/cmake/deps.txt, $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_protobuf.sh'
path: $(PROTO_CACHE_DIR)
restoreKeys: |
"$(TODAY)" | ccache | protocol
displayName: ccache protocol
protobuf | "$(Agent.OS)"
displayName: mac protobuf
condition: eq(${{ parameters.WithCache }}, true)
- script: |
@ -80,17 +83,19 @@ jobs:
python3 -m pip install -r '$(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/requirements.txt'
sudo xcode-select --switch /Applications/Xcode_13.1.app/Contents/Developer
displayName: 'Install dependencies'
${{ if eq(parameters.WithCache, true) }}:
env:
CCACHE_DIR: $(PROTO_CACHE_DIR)
- script: |
ccache -s
ccache -z
displayName: Show Cache stats and Clear protolbuf stats.
condition: eq(${{ parameters.WithCache }}, true)
- ${{ if eq(parameters.WithCache, true) }}:
- template: check-cache-stats.yml
parameters:
CacheDir: $(PROTO_CACHE_DIR)
- task: Cache@2
inputs:
key: ' "$(TODAY)" | ccache | onnxruntime | "$(Build.SourceVersion)" '
path: $(CCACHE_DIR)
path: $(ORT_CACHE_DIR)
restoreKeys: |
"$(TODAY)" | ccache | onnxruntime
displayName: ccache MacOS Build
@ -103,6 +108,9 @@ jobs:
AdditionalBuildFlags: ${{ parameters.AdditionalBuildFlags }} --use_coreml --cmake_extra_defines CMAKE_OSX_ARCHITECTURES="arm64;x86_64"
BuildJava: false
BuildNodejs: false
WithCache: ${{ parameters.WithCache }}
${{ if eq(parameters.WithCache, true) }}:
CacheDir: $(ORT_CACHE_DIR)
- ${{ if eq(parameters.MacosArch, 'arm64') }}:
- template: mac-cpu-packaging-steps.yml
@ -111,6 +119,9 @@ jobs:
AdditionalBuildFlags: ${{ parameters.AdditionalBuildFlags }} --build_nodejs --build_java --use_coreml --cmake_extra_defines CMAKE_OSX_ARCHITECTURES=arm64
BuildJava: true
BuildNodejs: true
WithCache: ${{ parameters.WithCache }}
${{ if eq(parameters.WithCache, true) }}:
CacheDir: $(ORT_CACHE_DIR)
- ${{ if eq(parameters.MacosArch, 'x86_64') }}:
- template: mac-cpu-packaging-steps.yml
@ -119,9 +130,11 @@ jobs:
AdditionalBuildFlags: ${{ parameters.AdditionalBuildFlags }} --build_nodejs --build_java --use_coreml
BuildJava: true
BuildNodejs: true
WithCache: ${{ parameters.WithCache }}
${{ if eq(parameters.WithCache, true) }}:
CacheDir: $(ORT_CACHE_DIR)
- script: |
ccache -s
ccache -z
displayName: Show Cache stats
condition: eq(${{ parameters.WithCache }}, true)
- ${{ if eq(parameters.WithCache, true) }}:
- template: check-cache-stats.yml
parameters:
CacheDir: $(ORT_CACHE_DIR)