From feafbc42637588aeef9a074bdabe3320f0579b70 Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Tue, 11 Apr 2023 12:12:46 +0800 Subject: [PATCH] Refactor all Mac build steps (#15440) ### Description ### Motivation and Context Make the compilation cache steps easy to use and maintain Reduce cache storage. --- .../mac-coreml-ci-pipeline.yml | 3 +- .../azure-pipelines/mac-ios-ci-pipeline.yml | 109 ++++++++---------- .../templates/mac-build-step-with-cache.yml | 30 +++-- .../templates/mac-cpu-packaging-steps.yml | 25 ++-- .../templates/mac-cpu-packing-jobs.yml | 78 ++++--------- .../templates/react-native-ci.yml | 6 +- 6 files changed, 120 insertions(+), 131 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/mac-coreml-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/mac-coreml-ci-pipeline.yml index 8aca46e232..a359bbd834 100644 --- a/tools/ci_build/github/azure-pipelines/mac-coreml-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/mac-coreml-ci-pipeline.yml @@ -15,7 +15,8 @@ jobs: - template: templates/mac-build-step-with-cache.yml parameters: - TODAY: $(TODAY) + WithCache: true + Today: $(TODAY) AdditionalKey: coreml CacheDir: $(CCACHE_DIR) BuildStep: diff --git a/tools/ci_build/github/azure-pipelines/mac-ios-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/mac-ios-ci-pipeline.yml index ba78461f87..7aaa713d8e 100644 --- a/tools/ci_build/github/azure-pipelines/mac-ios-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/mac-ios-ci-pipeline.yml @@ -4,67 +4,56 @@ jobs: vmImage: 'macOS-12' variables: MACOSX_DEPLOYMENT_TARGET: '10.14' - CCACHE_DIR: $(Pipeline.Workspace)/ccache + PROTO_CACHE_DIR: $(Pipeline.Workspace)/proto_ccache + ORT_CACHE_DIR: $(Pipeline.Workspace)/ort_ccache TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)] timeoutInMinutes: 150 steps: - - script: | - brew install ccache - echo "##vso[task.prependpath]/usr/local/opt/ccache/libexec" - displayName: Install ccache + - template: templates/mac-build-step-with-cache.yml + parameters: + WithCache: true + Today: $(TODAY) + AdditionalKey: ' protobuf | "$(Agent.OS)" | $(Build.SourcesDirectory)/cmake/deps.txt, $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_protobuf.sh' + CacheDir: $(PROTO_CACHE_DIR) + ChangeEveryCommit: false + BuildStep: + - script: | + $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_protobuf.sh \ + -p $(Build.BinariesDirectory)/protobuf_install -d $(Build.SourcesDirectory)/cmake/deps.txt + displayName: Install protobuf + env: + CCACHE_DIR: $(PROTO_CACHE_DIR) - - task: Cache@2 - inputs: - key: ' "$(TODAY)" | ccache | protocol | "$(Build.SourceVersion)" ' - path: $(CCACHE_DIR) - restoreKeys: | - "$(TODAY)" | ccache | protocol - displayName: ccache protocol - - - script: | - $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_protobuf.sh \ - -p $(Build.BinariesDirectory)/protobuf_install -d $(Build.SourcesDirectory)/cmake/deps.txt - displayName: Install protobuf - - - script: | - ccache -sv - ccache -z - displayName: Show Cache stats and Clear protolbuf stats - - - task: Cache@2 - inputs: - key: ' "$(TODAY)" | ccache | onnxruntime | "$(Build.SourceVersion)" ' - path: $(CCACHE_DIR) - restoreKeys: | - "$(TODAY)" | ccache | onnxruntime - displayName: ccache MacOS Build - - - script: | - sudo xcode-select --switch /Applications/Xcode_13.1.app/Contents/Developer - python3 $(Build.SourcesDirectory)/tools/ci_build/build.py \ - --skip_submodule_sync \ - --build_dir $(Build.BinariesDirectory)/iOS \ - --build_shared \ - --use_coreml \ - --use_xnnpack \ - --ios \ - --ios_sysroot iphonesimulator \ - --osx_arch x86_64 \ - --apple_deploy_target 11.0 \ - --use_xcode \ - --config RelWithDebInfo \ - --build_apple_framework \ - --path_to_protoc_exe $(Build.BinariesDirectory)/protobuf_install/bin/protoc \ - --parallel - displayName: (CPU, CoreML, XNNPACK EPs) Build onnxruntime for iOS x86_64 and run tests using simulator - env: - CC: clang - CXX: clang++ - CCACHE_CPP2: 1 - CCACHE_DEPEND: 1 - CCACHE_SLOPPINESS: modules - - - script: | - ccache -sv - ccache -z - displayName: Show Cache stats + - template: templates/mac-build-step-with-cache.yml + parameters: + WithCache: true + Today: $(TODAY) + AdditionalKey: onnxruntime + CacheDir: $(ORT_CACHE_DIR) + ChangeEveryCommit: true + BuildStep: + - script: | + sudo xcode-select --switch /Applications/Xcode_13.1.app/Contents/Developer + python3 $(Build.SourcesDirectory)/tools/ci_build/build.py \ + --skip_submodule_sync \ + --build_dir $(Build.BinariesDirectory)/iOS \ + --build_shared \ + --use_coreml \ + --use_xnnpack \ + --ios \ + --ios_sysroot iphonesimulator \ + --osx_arch x86_64 \ + --apple_deploy_target 11.0 \ + --use_xcode \ + --config RelWithDebInfo \ + --build_apple_framework \ + --path_to_protoc_exe $(Build.BinariesDirectory)/protobuf_install/bin/protoc \ + --parallel + displayName: (CPU, CoreML, XNNPACK EPs) Build onnxruntime for iOS x86_64 and run tests using simulator + env: + CC: clang + CXX: clang++ + CCACHE_CPP2: 1 + CCACHE_DEPEND: 1 + CCACHE_SLOPPINESS: modules + CCACHE_DIR: $(ORT_CACHE_DIR) diff --git a/tools/ci_build/github/azure-pipelines/templates/mac-build-step-with-cache.yml b/tools/ci_build/github/azure-pipelines/templates/mac-build-step-with-cache.yml index 7cf1fcb509..f1aa581b75 100644 --- a/tools/ci_build/github/azure-pipelines/templates/mac-build-step-with-cache.yml +++ b/tools/ci_build/github/azure-pipelines/templates/mac-build-step-with-cache.yml @@ -1,17 +1,26 @@ # It's used for compilation with cache in Mac Host. parameters: -- name: TODAY - type: string +- name: WithCache + displayName: Build with Cache + type: boolean + default: false - name: BuildStep type: stepList +# if WithCahe is false, the following parameters are not used. +- name: Today + type: string + default: "" + - name: AdditionalKey type: string + default: "" - name: CacheDir type: string + default: "" - name: ChangeEveryCommit type: boolean @@ -23,22 +32,29 @@ steps: echo "##vso[task.prependpath]/usr/local/opt/ccache/libexec" mkdir -p "${{ parameters.CacheDir }}" displayName: Install ccache and update PATH to use linked versions of gcc, cc, etc + condition: eq(${{ parameters.WithCache }}, true) - task: Cache@2 inputs: ${{ if eq(parameters.ChangeEveryCommit, true) }}: - key: ' "${{parameters.TODAY}}" | ${{parameters.AdditionalKey}} | "$(Build.SourceVersion)" ' + # If it's a merge build, the run will read the cache generated by main build. + ${{if eq(variables['Build.SourceBranchName'], 'merge')}}: + key: ' "$(TODAY)" | ccache | merge ' + ${{else}}: + key: ' "${{parameters.Today}}" | ${{parameters.AdditionalKey}} | "$(Build.SourceVersion)" ' restoreKeys: | - "${{parameters.TODAY}}" | ${{parameters.AdditionalKey}} + "${{parameters.Today}}" | ${{parameters.AdditionalKey}} ${{ else }}: key: ' ${{parameters.AdditionalKey}} ' restoreKeys: | ${{parameters.AdditionalKey}} path: ${{ parameters.CacheDir }} displayName: ccache task + condition: eq(${{ parameters.WithCache }}, true) - ${{ parameters.BuildStep }} - - template: check-cache-stats.yml - parameters: - CacheDir: ${{ parameters.CacheDir }} + - ${{ if eq(parameters.WithCache, true) }}: + - template: check-cache-stats.yml + parameters: + CacheDir: ${{ parameters.CacheDir }} diff --git a/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packaging-steps.yml b/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packaging-steps.yml index d44142cdb8..0cb77e222a 100644 --- a/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packaging-steps.yml +++ b/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packaging-steps.yml @@ -32,14 +32,25 @@ parameters: type: string default: '' +- name: Today + type: string + default: "" + steps: -- script: | - rm -rf $(Build.BinariesDirectory)/Release - python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --update --build ${{ parameters.AdditionalBuildFlags }} --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_shared_lib --config Release - displayName: 'Build ${{ parameters.MacosArch }}' - ${{ if eq(parameters.WithCache, true) }}: - env: - CCACHE_DIR: ${{ parameters.CacheDir }} +- template: mac-build-step-with-cache.yml + parameters: + WithCache: ${{ parameters.WithCache }} + Today: ${{ parameters.Today }} + AdditionalKey: onnxruntime_${{ parameters.MacosArch }} + CacheDir: ${{ parameters.CacheDir }} + ChangeEveryCommit: true + BuildStep: + - script: | + rm -rf $(Build.BinariesDirectory)/Release + python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --update --build ${{ parameters.AdditionalBuildFlags }} --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_shared_lib --config Release + displayName: 'Build ${{ parameters.MacosArch }}' + env: + CCACHE_DIR: ${{ parameters.CacheDir }} - ${{ if eq(parameters.MacosArch, 'x86_64') }}: - script: | diff --git a/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packing-jobs.yml b/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packing-jobs.yml index 411c2277e7..e4de00f2fc 100644 --- a/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packing-jobs.yml +++ b/tools/ci_build/github/azure-pipelines/templates/mac-cpu-packing-jobs.yml @@ -55,55 +55,27 @@ jobs: - template: set-version-number-variables-step.yml - - script: | - brew install ccache - echo "##vso[task.prependpath]/usr/local/opt/ccache/libexec" - mkdir -p $(PROTO_CACHE_DIR) - mkdir -p $(ORT_CACHE_DIR) - displayName: Install ccache and update PATH to use linked versions of gcc, cc, etc - condition: eq(${{ parameters.WithCache }}, true) - - - task: Cache@2 - inputs: - key: ' protobuf | "$(Agent.OS)" | $(Build.SourcesDirectory)/cmake/deps.txt, $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_protobuf.sh' - path: $(PROTO_CACHE_DIR) - restoreKeys: | - protobuf | "$(Agent.OS)" - displayName: mac protobuf - condition: eq(${{ parameters.WithCache }}, true) - - - script: | - set -e -x - pushd . - $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_protobuf.sh -d $(Build.SourcesDirectory)/cmake/deps.txt -p $(Build.BinariesDirectory)/installed - popd - export PATH=$(Build.BinariesDirectory)/installed/bin:$PATH - export ONNX_ML=1 - export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF" - python3 -m pip install -r '$(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/requirements.txt' - sudo xcode-select --switch /Applications/Xcode_13.1.app/Contents/Developer - displayName: 'Install dependencies' - ${{ if eq(parameters.WithCache, true) }}: - env: - CCACHE_DIR: $(PROTO_CACHE_DIR) - - - ${{ if eq(parameters.WithCache, true) }}: - - template: check-cache-stats.yml - parameters: - CacheDir: $(PROTO_CACHE_DIR) - - # If it's a merge build, the run will read the cache generated by main build. - - task: Cache@2 - inputs: - ${{if eq(variables['Build.SourceBranchName'], 'merge')}}: - key: ' "$(TODAY)" | ccache | merge ' - ${{else}}: - key: ' "$(TODAY)" | ccache | onnxruntime | $(Build.SourceVersion) ' - path: $(ORT_CACHE_DIR) - restoreKeys: | - "$(TODAY)" | ccache | onnxruntime - displayName: ccache MacOS Build - condition: eq(${{ parameters.WithCache }}, true) + - template: mac-build-step-with-cache.yml + parameters: + WithCache: ${{ parameters.WithCache }} + Today: $(TODAY) + AdditionalKey: ' protobuf | "$(Agent.OS)" | $(Build.SourcesDirectory)/cmake/deps.txt, $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_protobuf.sh' + CacheDir: $(PROTO_CACHE_DIR) + ChangeEveryCommit: false + BuildStep: + - script: | + set -e -x + pushd . + $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_protobuf.sh -d $(Build.SourcesDirectory)/cmake/deps.txt -p $(Build.BinariesDirectory)/installed + popd + export PATH=$(Build.BinariesDirectory)/installed/bin:$PATH + export ONNX_ML=1 + export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF" + python3 -m pip install -r '$(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/requirements.txt' + sudo xcode-select --switch /Applications/Xcode_13.1.app/Contents/Developer + displayName: 'Install dependencies' + env: + CCACHE_DIR: $(PROTO_CACHE_DIR) - ${{ if eq(parameters.MacosArch, 'universal2') }}: - template: mac-cpu-packaging-steps.yml @@ -114,6 +86,7 @@ jobs: BuildNodejs: false WithCache: ${{ parameters.WithCache }} ${{ if eq(parameters.WithCache, true) }}: + Today: $(TODAY) CacheDir: $(ORT_CACHE_DIR) - ${{ if eq(parameters.MacosArch, 'arm64') }}: @@ -125,6 +98,7 @@ jobs: BuildNodejs: true WithCache: ${{ parameters.WithCache }} ${{ if eq(parameters.WithCache, true) }}: + Today: $(TODAY) CacheDir: $(ORT_CACHE_DIR) - ${{ if eq(parameters.MacosArch, 'x86_64') }}: @@ -136,9 +110,5 @@ jobs: BuildNodejs: true WithCache: ${{ parameters.WithCache }} ${{ if eq(parameters.WithCache, true) }}: + Today: $(TODAY) CacheDir: $(ORT_CACHE_DIR) - - - ${{ if eq(parameters.WithCache, true) }}: - - template: check-cache-stats.yml - parameters: - CacheDir: $(ORT_CACHE_DIR) diff --git a/tools/ci_build/github/azure-pipelines/templates/react-native-ci.yml b/tools/ci_build/github/azure-pipelines/templates/react-native-ci.yml index a4a741b333..1691639773 100644 --- a/tools/ci_build/github/azure-pipelines/templates/react-native-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/react-native-ci.yml @@ -84,7 +84,8 @@ stages: - template: mac-build-step-with-cache.yml parameters: - TODAY: $(TODAY) + WithCache: true + Today: $(TODAY) AdditionalKey: ' protobuf | "$(Agent.OS)" | $(Build.SourcesDirectory)/cmake/deps.txt, $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_protobuf.sh' CacheDir: $(PROTO_CACHE_DIR) ChangeEveryCommit: false @@ -98,7 +99,8 @@ stages: - template: mac-build-step-with-cache.yml parameters: - TODAY: $(TODAY) + WithCache: true + Today: $(TODAY) AdditionalKey: react_${{parameters.BuildConfig}} CacheDir: $(ORT_CACHE_DIR) BuildStep: