Refactor all Mac build steps (#15440)

### Description


### Motivation and Context
Make the compilation cache steps easy to use and maintain
Reduce cache storage.

tools/ci_build/github/azure-pipelines/mac-coreml-ci-pipeline.yml

@@ -15,7 +15,8 @@ jobs:
 
   - template: templates/mac-build-step-with-cache.yml
     parameters:
-      TODAY: $(TODAY)
+      WithCache: true
+      Today: $(TODAY)
       AdditionalKey: coreml
       CacheDir: $(CCACHE_DIR)
       BuildStep:

tools/ci_build/github/azure-pipelines/mac-ios-ci-pipeline.yml

@@ -4,67 +4,56 @@ jobs:
     vmImage: 'macOS-12'
   variables:
     MACOSX_DEPLOYMENT_TARGET: '10.14'
-    CCACHE_DIR: $(Pipeline.Workspace)/ccache
+    PROTO_CACHE_DIR: $(Pipeline.Workspace)/proto_ccache
+    ORT_CACHE_DIR: $(Pipeline.Workspace)/ort_ccache
     TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)]
   timeoutInMinutes: 150
   steps:
-    - script: |
-        brew install ccache
-        echo "##vso[task.prependpath]/usr/local/opt/ccache/libexec"
-      displayName: Install ccache
+    - template: templates/mac-build-step-with-cache.yml
+      parameters:
+        WithCache: true
+        Today: $(TODAY)
+        AdditionalKey: ' protobuf | "$(Agent.OS)" | $(Build.SourcesDirectory)/cmake/deps.txt, $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_protobuf.sh'
+        CacheDir: $(PROTO_CACHE_DIR)
+        ChangeEveryCommit: false
+        BuildStep:
+          - script: |
+              $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_protobuf.sh \
+                -p $(Build.BinariesDirectory)/protobuf_install -d $(Build.SourcesDirectory)/cmake/deps.txt
+            displayName: Install protobuf
+            env:
+              CCACHE_DIR: $(PROTO_CACHE_DIR)
 
-    - task: Cache@2
-      inputs:
-        key:  ' "$(TODAY)" | ccache | protocol | "$(Build.SourceVersion)" '
-        path: $(CCACHE_DIR)
-        restoreKeys: |
-          "$(TODAY)" | ccache | protocol
-      displayName: ccache protocol
-
-    - script: |
-        $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_protobuf.sh \
-          -p $(Build.BinariesDirectory)/protobuf_install -d $(Build.SourcesDirectory)/cmake/deps.txt
-      displayName: Install protobuf
-
-    - script: |
-        ccache -sv
-        ccache -z
-      displayName: Show Cache stats and Clear protolbuf stats
-
-    - task: Cache@2
-      inputs:
-        key: ' "$(TODAY)" | ccache  | onnxruntime | "$(Build.SourceVersion)" '
-        path: $(CCACHE_DIR)
-        restoreKeys: |
-          "$(TODAY)" | ccache | onnxruntime
-      displayName: ccache MacOS Build
-
-    - script: |
-        sudo xcode-select --switch /Applications/Xcode_13.1.app/Contents/Developer
-        python3 $(Build.SourcesDirectory)/tools/ci_build/build.py \
-          --skip_submodule_sync \
-          --build_dir $(Build.BinariesDirectory)/iOS \
-          --build_shared \
-          --use_coreml \
-          --use_xnnpack \
-          --ios \
-          --ios_sysroot iphonesimulator  \
-          --osx_arch x86_64 \
-          --apple_deploy_target 11.0 \
-          --use_xcode \
-          --config RelWithDebInfo \
-          --build_apple_framework \
-          --path_to_protoc_exe $(Build.BinariesDirectory)/protobuf_install/bin/protoc \
-          --parallel
-      displayName: (CPU, CoreML, XNNPACK EPs) Build onnxruntime for iOS x86_64 and run tests using simulator
-      env:
-        CC: clang
-        CXX: clang++
-        CCACHE_CPP2: 1
-        CCACHE_DEPEND: 1
-        CCACHE_SLOPPINESS: modules
-
-    - script: |
-        ccache -sv
-        ccache -z
-      displayName: Show Cache stats
+    - template: templates/mac-build-step-with-cache.yml
+      parameters:
+        WithCache: true
+        Today: $(TODAY)
+        AdditionalKey: onnxruntime
+        CacheDir: $(ORT_CACHE_DIR)
+        ChangeEveryCommit: true
+        BuildStep:
+          - script: |
+              sudo xcode-select --switch /Applications/Xcode_13.1.app/Contents/Developer
+              python3 $(Build.SourcesDirectory)/tools/ci_build/build.py \
+                --skip_submodule_sync \
+                --build_dir $(Build.BinariesDirectory)/iOS \
+                --build_shared \
+                --use_coreml \
+                --use_xnnpack \
+                --ios \
+                --ios_sysroot iphonesimulator  \
+                --osx_arch x86_64 \
+                --apple_deploy_target 11.0 \
+                --use_xcode \
+                --config RelWithDebInfo \
+                --build_apple_framework \
+                --path_to_protoc_exe $(Build.BinariesDirectory)/protobuf_install/bin/protoc \
+                --parallel
+            displayName: (CPU, CoreML, XNNPACK EPs) Build onnxruntime for iOS x86_64 and run tests using simulator
+            env:
+              CC: clang
+              CXX: clang++
+              CCACHE_CPP2: 1
+              CCACHE_DEPEND: 1
+              CCACHE_SLOPPINESS: modules
+              CCACHE_DIR: $(ORT_CACHE_DIR)

tools/ci_build/github/azure-pipelines/templates/mac-build-step-with-cache.yml

@@ -1,17 +1,26 @@
 # It's used for compilation with cache in Mac Host.
 
 parameters:
-- name: TODAY
-  type: string
+- name: WithCache
+  displayName: Build with Cache
+  type: boolean
+  default: false
 
 - name: BuildStep
   type: stepList
 
+# if WithCahe is false, the following parameters are not used.
+- name: Today
+  type: string
+  default: ""
+
 - name: AdditionalKey
   type: string
+  default: ""
 
 - name: CacheDir
   type: string
+  default: ""
 
 - name: ChangeEveryCommit
   type: boolean
@@ -23,22 +32,29 @@ steps:
       echo "##vso[task.prependpath]/usr/local/opt/ccache/libexec"
       mkdir -p "${{ parameters.CacheDir }}"
     displayName: Install ccache and update PATH to use linked versions of gcc, cc, etc
+    condition: eq(${{ parameters.WithCache }}, true)
 
   - task: Cache@2
     inputs:
       ${{ if eq(parameters.ChangeEveryCommit, true) }}:
-        key:  ' "${{parameters.TODAY}}" | ${{parameters.AdditionalKey}} | "$(Build.SourceVersion)" '
+        # If it's a merge build, the run will read the cache generated by main build.
+        ${{if eq(variables['Build.SourceBranchName'], 'merge')}}:
+          key: ' "$(TODAY)" | ccache  | merge '
+        ${{else}}:
+          key:  ' "${{parameters.Today}}" | ${{parameters.AdditionalKey}} | "$(Build.SourceVersion)" '
         restoreKeys: |
-          "${{parameters.TODAY}}" | ${{parameters.AdditionalKey}}
+          "${{parameters.Today}}" | ${{parameters.AdditionalKey}}
       ${{ else }}:
         key:  ' ${{parameters.AdditionalKey}} '
         restoreKeys: |
           ${{parameters.AdditionalKey}}
       path: ${{ parameters.CacheDir }}
     displayName: ccache task
+    condition: eq(${{ parameters.WithCache }}, true)
 
   - ${{ parameters.BuildStep }}
 
-  - template: check-cache-stats.yml
-    parameters:
-      CacheDir: ${{ parameters.CacheDir }}
+  - ${{ if eq(parameters.WithCache, true) }}:
+    - template: check-cache-stats.yml
+      parameters:
+        CacheDir: ${{ parameters.CacheDir }}

tools/ci_build/github/azure-pipelines/templates/mac-cpu-packaging-steps.yml

@@ -32,14 +32,25 @@ parameters:
   type: string
   default: ''
 
+- name: Today
+  type: string
+  default: ""
+
 steps:
-- script: |
-    rm -rf $(Build.BinariesDirectory)/Release
-    python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --update --build ${{ parameters.AdditionalBuildFlags }} --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_shared_lib --config Release
-  displayName: 'Build ${{ parameters.MacosArch }}'
-  ${{ if eq(parameters.WithCache, true) }}:
-    env:
-      CCACHE_DIR: ${{ parameters.CacheDir }}
+- template: mac-build-step-with-cache.yml
+  parameters:
+    WithCache: ${{ parameters.WithCache }}
+    Today: ${{ parameters.Today }}
+    AdditionalKey: onnxruntime_${{ parameters.MacosArch }}
+    CacheDir: ${{ parameters.CacheDir }}
+    ChangeEveryCommit: true
+    BuildStep:
+      - script: |
+          rm -rf $(Build.BinariesDirectory)/Release
+          python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --update --build ${{ parameters.AdditionalBuildFlags }} --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --parallel --build_shared_lib --config Release
+        displayName: 'Build ${{ parameters.MacosArch }}'
+        env:
+          CCACHE_DIR: ${{ parameters.CacheDir }}
 
 - ${{ if eq(parameters.MacosArch, 'x86_64') }}:
   - script: |

tools/ci_build/github/azure-pipelines/templates/mac-cpu-packing-jobs.yml

@@ -55,55 +55,27 @@ jobs:
 
   - template: set-version-number-variables-step.yml
 
-  - script: |
-      brew install ccache
-      echo "##vso[task.prependpath]/usr/local/opt/ccache/libexec"
-      mkdir -p $(PROTO_CACHE_DIR)
-      mkdir -p $(ORT_CACHE_DIR)
-    displayName: Install ccache and update PATH to use linked versions of gcc, cc, etc
-    condition: eq(${{ parameters.WithCache }}, true)
-
-  - task: Cache@2
-    inputs:
-      key: ' protobuf | "$(Agent.OS)" | $(Build.SourcesDirectory)/cmake/deps.txt, $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_protobuf.sh'
-      path: $(PROTO_CACHE_DIR)
-      restoreKeys: |
-         protobuf | "$(Agent.OS)"
-    displayName: mac protobuf
-    condition: eq(${{ parameters.WithCache }}, true)
-
-  - script: |
-      set -e -x
-      pushd .
-      $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_protobuf.sh -d $(Build.SourcesDirectory)/cmake/deps.txt -p $(Build.BinariesDirectory)/installed
-      popd
-      export PATH=$(Build.BinariesDirectory)/installed/bin:$PATH
-      export ONNX_ML=1
-      export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF"
-      python3 -m pip install -r '$(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/requirements.txt'
-      sudo xcode-select --switch /Applications/Xcode_13.1.app/Contents/Developer
-    displayName: 'Install dependencies'
-    ${{ if eq(parameters.WithCache, true) }}:
-      env:
-        CCACHE_DIR: $(PROTO_CACHE_DIR)
-
-  - ${{ if eq(parameters.WithCache, true) }}:
-    - template: check-cache-stats.yml
-      parameters:
-        CacheDir: $(PROTO_CACHE_DIR)
-
-  # If it's a merge build, the run will read the cache generated by main build.
-  - task: Cache@2
-    inputs:
-      ${{if eq(variables['Build.SourceBranchName'], 'merge')}}:
-        key: ' "$(TODAY)" | ccache  | merge '
-      ${{else}}:
-        key: ' "$(TODAY)" | ccache  | onnxruntime | $(Build.SourceVersion) '
-      path: $(ORT_CACHE_DIR)
-      restoreKeys: |
-        "$(TODAY)" | ccache | onnxruntime
-    displayName: ccache MacOS Build
-    condition: eq(${{ parameters.WithCache }}, true)
+  - template: mac-build-step-with-cache.yml
+    parameters:
+      WithCache: ${{ parameters.WithCache }}
+      Today: $(TODAY)
+      AdditionalKey: ' protobuf | "$(Agent.OS)" | $(Build.SourcesDirectory)/cmake/deps.txt, $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_protobuf.sh'
+      CacheDir: $(PROTO_CACHE_DIR)
+      ChangeEveryCommit: false
+      BuildStep:
+        - script: |
+            set -e -x
+            pushd .
+            $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_protobuf.sh -d $(Build.SourcesDirectory)/cmake/deps.txt -p $(Build.BinariesDirectory)/installed
+            popd
+            export PATH=$(Build.BinariesDirectory)/installed/bin:$PATH
+            export ONNX_ML=1
+            export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF"
+            python3 -m pip install -r '$(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/requirements.txt'
+            sudo xcode-select --switch /Applications/Xcode_13.1.app/Contents/Developer
+          displayName: 'Install dependencies'
+          env:
+            CCACHE_DIR: $(PROTO_CACHE_DIR)
 
   - ${{ if eq(parameters.MacosArch, 'universal2') }}:
     - template: mac-cpu-packaging-steps.yml
@@ -114,6 +86,7 @@ jobs:
         BuildNodejs: false
         WithCache: ${{ parameters.WithCache }}
         ${{ if eq(parameters.WithCache, true) }}:
+          Today: $(TODAY)
           CacheDir: $(ORT_CACHE_DIR)
 
   - ${{ if eq(parameters.MacosArch, 'arm64') }}:
@@ -125,6 +98,7 @@ jobs:
         BuildNodejs: true
         WithCache: ${{ parameters.WithCache }}
         ${{ if eq(parameters.WithCache, true) }}:
+          Today: $(TODAY)
           CacheDir: $(ORT_CACHE_DIR)
 
   - ${{ if eq(parameters.MacosArch, 'x86_64') }}:
@@ -136,9 +110,5 @@ jobs:
         BuildNodejs: true
         WithCache: ${{ parameters.WithCache }}
         ${{ if eq(parameters.WithCache, true) }}:
+          Today: $(TODAY)
           CacheDir: $(ORT_CACHE_DIR)
-
-  - ${{ if eq(parameters.WithCache, true) }}:
-    - template: check-cache-stats.yml
-      parameters:
-        CacheDir: $(ORT_CACHE_DIR)

tools/ci_build/github/azure-pipelines/templates/react-native-ci.yml

@@ -84,7 +84,8 @@ stages:
 
     - template: mac-build-step-with-cache.yml
       parameters:
-        TODAY: $(TODAY)
+        WithCache: true
+        Today: $(TODAY)
         AdditionalKey: ' protobuf | "$(Agent.OS)" | $(Build.SourcesDirectory)/cmake/deps.txt, $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_protobuf.sh'
         CacheDir: $(PROTO_CACHE_DIR)
         ChangeEveryCommit: false
@@ -98,7 +99,8 @@ stages:
 
     - template: mac-build-step-with-cache.yml
       parameters:
-        TODAY: $(TODAY)
+        WithCache: true
+        Today: $(TODAY)
         AdditionalKey: react_${{parameters.BuildConfig}}
         CacheDir: $(ORT_CACHE_DIR)
         BuildStep: