diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py
index 5afb9c7377..9155098105 100644
--- a/tools/ci_build/build.py
+++ b/tools/ci_build/build.py
@@ -2271,6 +2271,9 @@ def main():
 
     args = parse_arguments()
 
+    if os.getenv("ORT_BUILD_WITH_CACHE") == "1":
+        args.use_cache = True
+
     if not is_windows():
         if not args.allow_running_as_root:
             is_root_user = os.geteuid() == 0
diff --git a/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml
index 049fe13219..98d1bdfdf5 100644
--- a/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml
@@ -15,7 +15,7 @@ stages:
         clean: all
       variables:
         skipComponentGovernanceDetection: true
-        CCACHE_DIR: $(Pipeline.Workspace)/ccache
+        ORT_CACHE_DIR: $(Agent.TempDirectory)/ort_ccache
         TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)]
       pool: onnxruntime-Linux-CPU-2019
       steps:
@@ -43,50 +43,48 @@ stages:
           DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )"
           Repository: onnxruntimecpubuild
 
-      - task: Cache@2
-        inputs:
-          key: '"$(TODAY)" | "$(Build.SourceBranch)" | "$(Build.SourceVersion)"'
-          path: $(CCACHE_DIR)
-          cacheHitVar: CACHE_RESTORED
-          restoreKeys: |
-            "$(TODAY)" | "$(Build.SourceBranch)"
-            "$(TODAY)" |
-        displayName: Cach Task
-
-      - task: CmdLine@2
-        displayName: 'build'
-        inputs:
-          script: |
-            mkdir -p $HOME/.onnx
-            mkdir -p $(Pipeline.Workspace)/ccache
-            docker run --rm \
-              --volume /data/onnx:/data/onnx:ro \
-              --volume $(Build.SourcesDirectory):/onnxruntime_src \
-              --volume $(Build.BinariesDirectory):/build \
-              --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
-              --volume $(Pipeline.Workspace)/ccache:/cache \
-              -e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \
-              -e NIGHTLY_BUILD \
-              -e BUILD_BUILDNUMBER \
-              -e CCACHE_DIR=/cache \
-              onnxruntimecpubuild \
-              /bin/bash -c "
-                set -ex; \
-                ccache -s; \
-                /opt/python/cp38-cp38/bin/python3 /onnxruntime_src/tools/ci_build/build.py \
-                  --build_dir /build --cmake_generator Ninja \
-                  --config Debug Release \
-                  --skip_submodule_sync \
-                  --build_shared_lib \
-                  --parallel \
-                  --build_wheel \
-                  --enable_onnx_tests \
-                  --enable_transformers_tool_test \
-                  --use_cache \
-                  --build_java --build_nodejs --update --build --cmake_extra_defines onnxruntime_BUILD_BENCHMARKS=ON; \
-                ccache -sv; \
-                ccache -z"
-          workingDirectory: $(Build.SourcesDirectory)
+      - template: templates/linux-build-step-with-cache.yml
+        parameters:
+          WithCache: true
+          Today: $(TODAY)
+          AdditionalKey: onnxruntime
+          CacheDir: $(ORT_CACHE_DIR)
+          ChangeEveryCommit: true
+          BuildStep:
+            - task: CmdLine@2
+              displayName: 'build'
+              inputs:
+                script: |
+                  mkdir -p $HOME/.onnx
+                  mkdir -p $(Pipeline.Workspace)/ccache
+                  docker run --rm \
+                    --volume /data/onnx:/data/onnx:ro \
+                    --volume $(Build.SourcesDirectory):/onnxruntime_src \
+                    --volume $(Build.BinariesDirectory):/build \
+                    --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
+                    --volume $(ORT_CACHE_DIR):/cache \
+                    -e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \
+                    -e NIGHTLY_BUILD \
+                    -e BUILD_BUILDNUMBER \
+                    -e CCACHE_DIR=/cache \
+                    onnxruntimecpubuild \
+                    /bin/bash -c "
+                      set -ex; \
+                      ccache -s; \
+                      /opt/python/cp38-cp38/bin/python3 /onnxruntime_src/tools/ci_build/build.py \
+                        --build_dir /build --cmake_generator Ninja \
+                        --config Debug Release \
+                        --skip_submodule_sync \
+                        --build_shared_lib \
+                        --parallel \
+                        --build_wheel \
+                        --enable_onnx_tests \
+                        --enable_transformers_tool_test \
+                        --use_cache \
+                        --build_java --build_nodejs --update --build --cmake_extra_defines onnxruntime_BUILD_BENCHMARKS=ON; \
+                      ccache -sv; \
+                      ccache -z"
+                workingDirectory: $(Build.SourcesDirectory)
 
       - task: CmdLine@2
         displayName: 'Install python deps and run java tests'
@@ -184,6 +182,7 @@ stages:
       devtoolset_rootpath: /opt/rh/devtoolset-10/root
       ld_library_path_arg: /opt/rh/devtoolset-10/root/usr/lib64:/opt/rh/devtoolset-10/root/usr/lib:/opt/rh/devtoolset-10/root/usr/lib64/dyninst:/opt/rh/devtoolset-10/root/usr/lib/dyninst:/usr/local/lib64
       prepend_path: '/opt/rh/devtoolset-10/root/usr/bin:'
+      with_cache: true
 
 - stage: arm64_test
   dependsOn: ['arm64_build']
diff --git a/tools/ci_build/github/azure-pipelines/linux-cpu-aten-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-cpu-aten-pipeline.yml
index 6aa09da375..5e715b9a61 100644
--- a/tools/ci_build/github/azure-pipelines/linux-cpu-aten-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/linux-cpu-aten-pipeline.yml
@@ -12,7 +12,7 @@ jobs:
   workspace:
     clean: all
   variables:
-    CCACHE_DIR: $(Pipeline.Workspace)/ccache
+    CCACHE_DIR: $(Agent.TempDirectory)/ccache
     TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)]
   pool: onnxruntime-Linux-CPU-2019
   steps:
@@ -37,53 +37,46 @@ jobs:
       UseImageCacheContainerRegistry: true
       UsePipelineCache: false
 
-  - task: Cache@2
-    inputs:
-      key: '"$(TODAY)" | "$(Build.SourceBranch)" | "$(Build.SourceVersion)"'
-      path: $(CCACHE_DIR)
-      cacheHitVar: CACHE_RESTORED
-      restoreKeys: |
-        "$(TODAY)" | "$(Build.SourceBranch)"
-        "$(TODAY)" |
-    displayName: Cach Task
-
-  - script: |
-      sudo mkdir -p $(Pipeline.Workspace)/ccache
-    condition: ne(variables.CACHE_RESTORED, 'true')
-    displayName: Create Cache Dir
-
-  - task: CmdLine@2
-    displayName: 'build'
-    inputs:
-      script: |
-        mkdir -p $HOME/.onnx
-        docker run --rm \
-          --volume /data/onnx:/data/onnx:ro \
-          --volume $(Build.SourcesDirectory):/onnxruntime_src \
-          --volume $(Build.BinariesDirectory):/build \
-          --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
-          --volume $(Pipeline.Workspace)/ccache:/cache \
-          -e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \
-          -e NIGHTLY_BUILD \
-          -e BUILD_BUILDNUMBER \
-          -e CCACHE_DIR=/cache \
-          onnxruntimecpubuildaten \
-          /bin/bash -c "
-              set -ex; \
-              ccache -s; \
-              /opt/python/cp38-cp38/bin/python3 /onnxruntime_src/tools/ci_build/build.py \
-                --build_dir /build --cmake_generator Ninja \
-                --config Release \
-                --skip_submodule_sync \
-                --build_shared_lib \
-                --parallel \
-                --build_wheel \
-                --skip_tests \
-                --cmake_extra_defines onnxruntime_ENABLE_ATEN=ON \
-                --use_cache; \
-                ccache -sv; \
-                ccache -z"
-      workingDirectory: $(Build.SourcesDirectory)
+  - template: templates/linux-build-step-with-cache.yml
+    parameters:
+      WithCache: true
+      Today: $(TODAY)
+      AdditionalKey: ort_aten
+      CacheDir: $(CCACHE_DIR)
+      ChangeEveryCommit: true
+      BuildStep:
+        - task: CmdLine@2
+          displayName: 'build'
+          inputs:
+            script: |
+              mkdir -p $HOME/.onnx
+              docker run --rm \
+                --volume /data/onnx:/data/onnx:ro \
+                --volume $(Build.SourcesDirectory):/onnxruntime_src \
+                --volume $(Build.BinariesDirectory):/build \
+                --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
+                --volume $(CCACHE_DIR):/cache \
+                -e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \
+                -e NIGHTLY_BUILD \
+                -e BUILD_BUILDNUMBER \
+                -e CCACHE_DIR=/cache \
+                onnxruntimecpubuildaten \
+                /bin/bash -c "
+                    set -ex; \
+                    ccache -s; \
+                    /opt/python/cp38-cp38/bin/python3 /onnxruntime_src/tools/ci_build/build.py \
+                      --build_dir /build --cmake_generator Ninja \
+                      --config Release \
+                      --skip_submodule_sync \
+                      --build_shared_lib \
+                      --parallel \
+                      --build_wheel \
+                      --skip_tests \
+                      --cmake_extra_defines onnxruntime_ENABLE_ATEN=ON \
+                      --use_cache; \
+                      ccache -sv; \
+                      ccache -z"
+            workingDirectory: $(Build.SourcesDirectory)
 
   - task: CmdLine@2
     displayName: 'install ort_torch_ext and launch test'
diff --git a/tools/ci_build/github/azure-pipelines/linux-cpu-minimal-build-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-cpu-minimal-build-ci-pipeline.yml
index 1f4f1a4980..9765deffea 100644
--- a/tools/ci_build/github/azure-pipelines/linux-cpu-minimal-build-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/linux-cpu-minimal-build-ci-pipeline.yml
@@ -36,9 +36,15 @@ jobs:
     clean: all
   pool: onnxruntime-Linux-CPU-2019
   variables:
+    ORT_CACHE_DIR: $(Pipeline.Workspace)/ort_ccache
+    TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)]
     test_data_directory: $(Build.SourcesDirectory)/.test_data
 
   steps:
+  - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
+    displayName: 'Clean Agent Directories'
+    condition: always()
+
   - checkout: self
     clean: true
     submodules: none
@@ -60,21 +66,34 @@ jobs:
         touch $(test_data_directory)/include_no_operators.config
       workingDirectory: $(Build.SourcesDirectory)
 
-  - task: CmdLine@2
-    displayName: 1. Build full onnxruntime and generate ORT format test files
-    inputs:
-      script: |
-        docker run --rm \
-          --volume $(Build.SourcesDirectory):/onnxruntime_src \
-          --volume $(Build.BinariesDirectory):/build \
-          --volume $(test_data_directory):/home/onnxruntimedev/.test_data \
-          -e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \
-          -e NIGHTLY_BUILD \
-          -e BUILD_BUILDNUMBER \
-          onnxruntimecpubuild \
-            /bin/bash /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_full_ort_and_create_ort_files.sh \
-              /build/1
-      workingDirectory: $(Build.SourcesDirectory)
+  - template: templates/linux-build-step-with-cache.yml
+    parameters:
+      WithCache: true
+      Today: $(TODAY)
+      AdditionalKey: onnxruntime__full
+      CacheDir: $(ORT_CACHE_DIR)
+      ChangeEveryCommit: true
+      BuildStep:
+        - task: CmdLine@2
+          displayName: 1. Build full onnxruntime and generate ORT format test files
+          inputs:
+            script: |
+              docker run --rm \
+                --volume $(Build.SourcesDirectory):/onnxruntime_src \
+                --volume $(Build.BinariesDirectory):/build \
+                --volume $(test_data_directory):/home/onnxruntimedev/.test_data \
+                --volume $(ORT_CACHE_DIR):/cache \
+                -e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \
+                -e NIGHTLY_BUILD \
+                -e BUILD_BUILDNUMBER \
+                -e CCACHE_DIR=/cache \
+                -e ORT_BUILD_WITH_CACHE=1 \
+                onnxruntimecpubuild \
+                  /bin/bash -c "
+                    /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_full_ort_and_create_ort_files.sh /build/1; \
+                    ccache -sv; \
+                    ccache -z;"
+            workingDirectory: $(Build.SourcesDirectory)
 
   - task: CmdLine@2
     displayName: 2. Build minimal onnxruntime [exceptions DISABLED, type reduction DISABLED, training ops ENABLED]
@@ -300,4 +319,4 @@ jobs:
               --skip_tests --path_to_protoc_exe /usr/bin/protoc
       workingDirectory: $(Build.SourcesDirectory)
 
-  - template: templates/clean-agent-build-directory-step.yml
+  - template: templates/explicitly-defined-final-tasks.yml
diff --git a/tools/ci_build/github/azure-pipelines/templates/linux-build-step-with-cache.yml b/tools/ci_build/github/azure-pipelines/templates/linux-build-step-with-cache.yml
new file mode 100644
index 0000000000..4acaa0d027
--- /dev/null
+++ b/tools/ci_build/github/azure-pipelines/templates/linux-build-step-with-cache.yml
@@ -0,0 +1,50 @@
+parameters:
+- name: WithCache
+  displayName: Build with Cache
+  type: boolean
+  default: false
+
+- name: BuildStep
+  type: stepList
+
+- name: Today
+  type: string
+  default: ""
+
+# It'd be different in different jobs in one workflow.
+- name: AdditionalKey
+  type: string
+  default: ""
+
+- name: ChangeEveryCommit
+  type: boolean
+  default: true
+
+- name: CacheDir
+  type: string
+  default: ""
+
+steps:
+  - ${{if eq(parameters.WithCache, true)}}:
+    - script: |
+        mkdir -p ${{ parameters.CacheDir }}
+      displayName: Create Cache Dir
+
+  - ${{if eq(parameters.WithCache, true)}}:
+    - task: Cache@2
+      inputs:
+        ${{ if eq(parameters.ChangeEveryCommit, true) }}:
+          ${{if eq(variables['Build.SourceBranchName'], 'merge')}}:
+            key: ' "${{parameters.Today}}" | merge | ${{parameters.AdditionalKey}} '
+          ${{else}}:
+            key:  ' "${{parameters.Today}}" | ${{parameters.AdditionalKey}} | "$(Build.SourceVersion)" '
+          restoreKeys: |
+            "${{parameters.Today}}" | ${{parameters.AdditionalKey}}
+        ${{ else }}:
+          key:  ' ${{parameters.AdditionalKey}} '
+          restoreKeys: |
+            ${{parameters.AdditionalKey}}
+        path: ${{ parameters.CacheDir }}
+      displayName: ccache task
+
+  - ${{ parameters.BuildStep }}
diff --git a/tools/ci_build/github/azure-pipelines/templates/py-linux.yml b/tools/ci_build/github/azure-pipelines/templates/py-linux.yml
index a42e005a3f..3645b0730f 100644
--- a/tools/ci_build/github/azure-pipelines/templates/py-linux.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/py-linux.yml
@@ -21,6 +21,10 @@ parameters:
   type: string
   default: '-d CPU'
 
+- name: with_cache
+  type: boolean
+  default: false
+
 jobs:
 - job: Linux_py_Wheels_${{ parameters.arch }}
   timeoutInMinutes: 240
@@ -30,7 +34,13 @@ jobs:
   variables:
     # The build machine pool doesn't have dotnet, so it can't run CG.
     skipComponentGovernanceDetection: true
+    ORT_CACHE_DIR: $(Agent.TempDirectory)/ort_ccache
+    TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)]
   steps:
+    - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
+      displayName: 'Clean Agent Directories'
+      condition: always()
+
     - checkout: self
       clean: true
       submodules: none
@@ -46,12 +56,23 @@ jobs:
         ${{ if eq(parameters.arch, 'aarch64') }}:
             UpdateDepsTxt: false
 
-    - task: Bash@3
-      displayName: 'Build Python Wheel'
-      inputs:
-        targetType: filePath
-        filePath: tools/ci_build/github/linux/run_python_dockerbuild.sh
-        arguments: -i onnxruntimecpubuilpython${{ parameters.arch }} -x "${{ parameters.device }}"
+    - template: linux-build-step-with-cache.yml
+      parameters:
+        WithCache: ${{parameters.with_cache}}
+        Today: $(TODAY)
+        AdditionalKey: Linux_py_Wheels_${{ parameters.arch }}
+        CacheDir: $(ORT_CACHE_DIR)
+        ChangeEveryCommit: true
+        BuildStep:
+          - task: Bash@3
+            displayName: 'Build Python Wheel'
+            inputs:
+              targetType: filePath
+              filePath: tools/ci_build/github/linux/run_python_dockerbuild.sh
+              arguments: -i onnxruntimecpubuilpython${{ parameters.arch }} -x "${{ parameters.device }}"
+            ${{ if eq(parameters.with_cache, 'true') }}:
+              env:
+                ADDITIONAL_DOCKER_PARAMETER: "--volume $(ORT_CACHE_DIR):/cache -e CCACHE_DIR=/cache -e ORT_BUILD_WITH_CACHE=1"
 
     - task: PublishBuildArtifacts@1
       displayName: 'Publish Artifact: ONNXRuntime python wheel'
@@ -64,8 +85,3 @@ jobs:
       inputs:
         artifactName: 'drop-linux-cpu-${{ parameters.arch }}'
         targetPath: '$(Build.BinariesDirectory)/Release'
-
-
-    - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
-      displayName: 'Clean Agent Directories'
-      condition: always()
diff --git a/tools/ci_build/github/linux/build_linux_arm64_python_package.sh b/tools/ci_build/github/linux/build_linux_arm64_python_package.sh
index 14b72dcfc7..9af2f40a42 100755
--- a/tools/ci_build/github/linux/build_linux_arm64_python_package.sh
+++ b/tools/ci_build/github/linux/build_linux_arm64_python_package.sh
@@ -64,3 +64,5 @@ do
 
   cp /build/$BUILD_CONFIG/dist/*.whl /build/dist
 done
+
+which ccache && ccache -sv && ccache -z
diff --git a/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_centos.sh b/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_centos.sh
index ab262a614c..db4b617744 100755
--- a/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_centos.sh
+++ b/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_centos.sh
@@ -4,4 +4,4 @@ set -e
 os_major_version=$(cat /etc/redhat-release | tr -dc '0-9.'|cut -d \. -f1)
 
 echo "installing for os major version : $os_major_version"
-yum install -y which gdb redhat-lsb-core expat-devel tar unzip zlib-devel make libunwind bzip2 bzip2-devel
\ No newline at end of file
+yum install -y which gdb redhat-lsb-core expat-devel tar unzip zlib-devel make libunwind bzip2 bzip2-devel
diff --git a/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_deps.sh b/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_deps.sh
index ec37598d39..7bf031ee78 100755
--- a/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_deps.sh
+++ b/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_deps.sh
@@ -15,4 +15,22 @@ do
   ${PYTHON_EXE} -m pip install -r requirements.txt
 done
 
+# No release binary for ccache aarch64, so we need to build it from source.
+if ! [ -x "$(command -v ccache)" ]; then
+    ccache_url="https://github.com/ccache/ccache/archive/refs/tags/v4.8.tar.gz"
+    pushd .
+    curl -sSL --retry 5 --retry-delay 10 --create-dirs --fail -L -o ccache_src.tar.gz $ccache_url
+    mkdir ccache_main
+    cd ccache_main
+    tar -zxf ../ccache_src.tar.gz --strip=1
 
+    mkdir build
+    cd build
+    cmake -DCMAKE_INSTALL_PREFIX=/usr/local _DCMAKE_BUILD_TYPE=Release ..
+    make
+    make install
+    which ccache
+    popd
+    rm -f ccache_src.tar.gz
+    rm -rf ccache_src
+fi
diff --git a/tools/ci_build/github/linux/run_python_dockerbuild.sh b/tools/ci_build/github/linux/run_python_dockerbuild.sh
index 14b86bf4e7..a9c406a551 100755
--- a/tools/ci_build/github/linux/run_python_dockerbuild.sh
+++ b/tools/ci_build/github/linux/run_python_dockerbuild.sh
@@ -18,6 +18,7 @@ docker run --rm \
     -w /onnxruntime_src \
     -e NIGHTLY_BUILD \
     -e BUILD_BUILDNUMBER \
+    $ADDITIONAL_DOCKER_PARAMETER \
     $DOCKER_IMAGE tools/ci_build/github/linux/build_linux_arm64_python_package.sh $BUILD_EXTR_PAR
 
 sudo rm -rf $BUILD_BINARIESDIRECTORY/Release/onnxruntime $BUILD_BINARIESDIRECTORY/Release/pybind11 \