Add compilation cache in 2 Linux CPU pipelines and refactor the Linux build step with cache (#15484)

### Description 1. Add compilation cache in Linux CPU ARM and Linux Minimal Build. 2. Integrate 4 Linux CPU build step with cache into one. 3. install ccache from source code in Linux ARM64 image. ### Motivation and Context 1. Enable more build steps with compilation cache. 2. Make it easier to add cache. It could save 40 more minutes of compilation time in Linux ARM64. https://dev.azure.com/onnxruntime/onnxruntime/_build/results?buildId=959619&view=logs&j=1e0830bb-fd74-5d0a-5029-1c63b4266d7b&t=75260ed7-7566-5947-2095-566660191920
2026-07-29 20:14:01 +00:00 · 2023-04-14 23:56:59 +08:00 · 2023-04-14 23:56:59 +08:00 · 4e1f75810c
commit 4e1f75810c
parent bf32dbbd9b
10 changed files with 222 additions and 121 deletions
--- a/tools/ci_build/build.py
+++ b/tools/ci_build/build.py
@ -2271,6 +2271,9 @@ def main():

    args = parse_arguments()

+    if os.getenv("ORT_BUILD_WITH_CACHE") == "1":
+        args.use_cache = True
+
    if not is_windows():
        if not args.allow_running_as_root:
            is_root_user = os.geteuid() == 0
--- a/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml
@ -15,7 +15,7 @@ stages:
        clean: all
      variables:
        skipComponentGovernanceDetection: true
-        CCACHE_DIR: $(Pipeline.Workspace)/ccache
+        ORT_CACHE_DIR: $(Agent.TempDirectory)/ort_ccache
        TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)]
      pool: onnxruntime-Linux-CPU-2019
      steps:
@ -43,50 +43,48 @@ stages:
          DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )"
          Repository: onnxruntimecpubuild

-      - task: Cache@2
-        inputs:
-          key: '"$(TODAY)" | "$(Build.SourceBranch)" | "$(Build.SourceVersion)"'
-          path: $(CCACHE_DIR)
-          cacheHitVar: CACHE_RESTORED
-          restoreKeys: |
-            "$(TODAY)" | "$(Build.SourceBranch)"
-            "$(TODAY)" |
-        displayName: Cach Task
-
-      - task: CmdLine@2
-        displayName: 'build'
-        inputs:
-          script: |
-            mkdir -p $HOME/.onnx
-            mkdir -p $(Pipeline.Workspace)/ccache
-            docker run --rm \
-              --volume /data/onnx:/data/onnx:ro \
-              --volume $(Build.SourcesDirectory):/onnxruntime_src \
-              --volume $(Build.BinariesDirectory):/build \
-              --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
-              --volume $(Pipeline.Workspace)/ccache:/cache \
-              -e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \
-              -e NIGHTLY_BUILD \
-              -e BUILD_BUILDNUMBER \
-              -e CCACHE_DIR=/cache \
-              onnxruntimecpubuild \
-              /bin/bash -c "
-                set -ex; \
-                ccache -s; \
-                /opt/python/cp38-cp38/bin/python3 /onnxruntime_src/tools/ci_build/build.py \
-                  --build_dir /build --cmake_generator Ninja \
-                  --config Debug Release \
-                  --skip_submodule_sync \
-                  --build_shared_lib \
-                  --parallel \
-                  --build_wheel \
-                  --enable_onnx_tests \
-                  --enable_transformers_tool_test \
-                  --use_cache \
-                  --build_java --build_nodejs --update --build --cmake_extra_defines onnxruntime_BUILD_BENCHMARKS=ON; \
-                ccache -sv; \
-                ccache -z"
-          workingDirectory: $(Build.SourcesDirectory)
+      - template: templates/linux-build-step-with-cache.yml
+        parameters:
+          WithCache: true
+          Today: $(TODAY)
+          AdditionalKey: onnxruntime
+          CacheDir: $(ORT_CACHE_DIR)
+          ChangeEveryCommit: true
+          BuildStep:
+            - task: CmdLine@2
+              displayName: 'build'
+              inputs:
+                script: |
+                  mkdir -p $HOME/.onnx
+                  mkdir -p $(Pipeline.Workspace)/ccache
+                  docker run --rm \
+                    --volume /data/onnx:/data/onnx:ro \
+                    --volume $(Build.SourcesDirectory):/onnxruntime_src \
+                    --volume $(Build.BinariesDirectory):/build \
+                    --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
+                    --volume $(ORT_CACHE_DIR):/cache \
+                    -e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \
+                    -e NIGHTLY_BUILD \
+                    -e BUILD_BUILDNUMBER \
+                    -e CCACHE_DIR=/cache \
+                    onnxruntimecpubuild \
+                    /bin/bash -c "
+                      set -ex; \
+                      ccache -s; \
+                      /opt/python/cp38-cp38/bin/python3 /onnxruntime_src/tools/ci_build/build.py \
+                        --build_dir /build --cmake_generator Ninja \
+                        --config Debug Release \
+                        --skip_submodule_sync \
+                        --build_shared_lib \
+                        --parallel \
+                        --build_wheel \
+                        --enable_onnx_tests \
+                        --enable_transformers_tool_test \
+                        --use_cache \
+                        --build_java --build_nodejs --update --build --cmake_extra_defines onnxruntime_BUILD_BENCHMARKS=ON; \
+                      ccache -sv; \
+                      ccache -z"
+                workingDirectory: $(Build.SourcesDirectory)

      - task: CmdLine@2
        displayName: 'Install python deps and run java tests'
@ -184,6 +182,7 @@ stages:
      devtoolset_rootpath: /opt/rh/devtoolset-10/root
      ld_library_path_arg: /opt/rh/devtoolset-10/root/usr/lib64:/opt/rh/devtoolset-10/root/usr/lib:/opt/rh/devtoolset-10/root/usr/lib64/dyninst:/opt/rh/devtoolset-10/root/usr/lib/dyninst:/usr/local/lib64
      prepend_path: '/opt/rh/devtoolset-10/root/usr/bin:'
+      with_cache: true

 - stage: arm64_test
  dependsOn: ['arm64_build']
--- a/tools/ci_build/github/azure-pipelines/linux-cpu-aten-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/linux-cpu-aten-pipeline.yml
@ -12,7 +12,7 @@ jobs:
  workspace:
    clean: all
  variables:
-    CCACHE_DIR: $(Pipeline.Workspace)/ccache
+    CCACHE_DIR: $(Agent.TempDirectory)/ccache
    TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)]
  pool: onnxruntime-Linux-CPU-2019
  steps:
@ -37,53 +37,46 @@ jobs:
      UseImageCacheContainerRegistry: true
      UsePipelineCache: false

-  - task: Cache@2
-    inputs:
-      key: '"$(TODAY)" | "$(Build.SourceBranch)" | "$(Build.SourceVersion)"'
-      path: $(CCACHE_DIR)
-      cacheHitVar: CACHE_RESTORED
-      restoreKeys: |
-        "$(TODAY)" | "$(Build.SourceBranch)"
-        "$(TODAY)" |
-    displayName: Cach Task
-
-  - script: |
-      sudo mkdir -p $(Pipeline.Workspace)/ccache
-    condition: ne(variables.CACHE_RESTORED, 'true')
-    displayName: Create Cache Dir
-
-  - task: CmdLine@2
-    displayName: 'build'
-    inputs:
-      script: |
-        mkdir -p $HOME/.onnx
-        docker run --rm \
-          --volume /data/onnx:/data/onnx:ro \
-          --volume $(Build.SourcesDirectory):/onnxruntime_src \
-          --volume $(Build.BinariesDirectory):/build \
-          --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
-          --volume $(Pipeline.Workspace)/ccache:/cache \
-          -e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \
-          -e NIGHTLY_BUILD \
-          -e BUILD_BUILDNUMBER \
-          -e CCACHE_DIR=/cache \
-          onnxruntimecpubuildaten \
-          /bin/bash -c "
-              set -ex; \
-              ccache -s; \
-              /opt/python/cp38-cp38/bin/python3 /onnxruntime_src/tools/ci_build/build.py \
-                --build_dir /build --cmake_generator Ninja \
-                --config Release \
-                --skip_submodule_sync \
-                --build_shared_lib \
-                --parallel \
-                --build_wheel \
-                --skip_tests \
-                --cmake_extra_defines onnxruntime_ENABLE_ATEN=ON \
-                --use_cache; \
-                ccache -sv; \
-                ccache -z"
-      workingDirectory: $(Build.SourcesDirectory)
+  - template: templates/linux-build-step-with-cache.yml
+    parameters:
+      WithCache: true
+      Today: $(TODAY)
+      AdditionalKey: ort_aten
+      CacheDir: $(CCACHE_DIR)
+      ChangeEveryCommit: true
+      BuildStep:
+        - task: CmdLine@2
+          displayName: 'build'
+          inputs:
+            script: |
+              mkdir -p $HOME/.onnx
+              docker run --rm \
+                --volume /data/onnx:/data/onnx:ro \
+                --volume $(Build.SourcesDirectory):/onnxruntime_src \
+                --volume $(Build.BinariesDirectory):/build \
+                --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
+                --volume $(CCACHE_DIR):/cache \
+                -e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \
+                -e NIGHTLY_BUILD \
+                -e BUILD_BUILDNUMBER \
+                -e CCACHE_DIR=/cache \
+                onnxruntimecpubuildaten \
+                /bin/bash -c "
+                    set -ex; \
+                    ccache -s; \
+                    /opt/python/cp38-cp38/bin/python3 /onnxruntime_src/tools/ci_build/build.py \
+                      --build_dir /build --cmake_generator Ninja \
+                      --config Release \
+                      --skip_submodule_sync \
+                      --build_shared_lib \
+                      --parallel \
+                      --build_wheel \
+                      --skip_tests \
+                      --cmake_extra_defines onnxruntime_ENABLE_ATEN=ON \
+                      --use_cache; \
+                      ccache -sv; \
+                      ccache -z"
+            workingDirectory: $(Build.SourcesDirectory)

  - task: CmdLine@2
    displayName: 'install ort_torch_ext and launch test'
--- a/tools/ci_build/github/azure-pipelines/linux-cpu-minimal-build-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/linux-cpu-minimal-build-ci-pipeline.yml
@ -36,9 +36,15 @@ jobs:
    clean: all
  pool: onnxruntime-Linux-CPU-2019
  variables:
+    ORT_CACHE_DIR: $(Pipeline.Workspace)/ort_ccache
+    TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)]
    test_data_directory: $(Build.SourcesDirectory)/.test_data

  steps:
+  - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
+    displayName: 'Clean Agent Directories'
+    condition: always()
+
  - checkout: self
    clean: true
    submodules: none
@ -60,21 +66,34 @@ jobs:
        touch $(test_data_directory)/include_no_operators.config
      workingDirectory: $(Build.SourcesDirectory)

-  - task: CmdLine@2
-    displayName: 1. Build full onnxruntime and generate ORT format test files
-    inputs:
-      script: |
-        docker run --rm \
-          --volume $(Build.SourcesDirectory):/onnxruntime_src \
-          --volume $(Build.BinariesDirectory):/build \
-          --volume $(test_data_directory):/home/onnxruntimedev/.test_data \
-          -e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \
-          -e NIGHTLY_BUILD \
-          -e BUILD_BUILDNUMBER \
-          onnxruntimecpubuild \
-            /bin/bash /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_full_ort_and_create_ort_files.sh \
-              /build/1
-      workingDirectory: $(Build.SourcesDirectory)
+  - template: templates/linux-build-step-with-cache.yml
+    parameters:
+      WithCache: true
+      Today: $(TODAY)
+      AdditionalKey: onnxruntime__full
+      CacheDir: $(ORT_CACHE_DIR)
+      ChangeEveryCommit: true
+      BuildStep:
+        - task: CmdLine@2
+          displayName: 1. Build full onnxruntime and generate ORT format test files
+          inputs:
+            script: |
+              docker run --rm \
+                --volume $(Build.SourcesDirectory):/onnxruntime_src \
+                --volume $(Build.BinariesDirectory):/build \
+                --volume $(test_data_directory):/home/onnxruntimedev/.test_data \
+                --volume $(ORT_CACHE_DIR):/cache \
+                -e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \
+                -e NIGHTLY_BUILD \
+                -e BUILD_BUILDNUMBER \
+                -e CCACHE_DIR=/cache \
+                -e ORT_BUILD_WITH_CACHE=1 \
+                onnxruntimecpubuild \
+                  /bin/bash -c "
+                    /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_full_ort_and_create_ort_files.sh /build/1; \
+                    ccache -sv; \
+                    ccache -z;"
+            workingDirectory: $(Build.SourcesDirectory)

  - task: CmdLine@2
    displayName: 2. Build minimal onnxruntime [exceptions DISABLED, type reduction DISABLED, training ops ENABLED]
@ -300,4 +319,4 @@ jobs:
              --skip_tests --path_to_protoc_exe /usr/bin/protoc
      workingDirectory: $(Build.SourcesDirectory)

-  - template: templates/clean-agent-build-directory-step.yml
+  - template: templates/explicitly-defined-final-tasks.yml
--- a/tools/ci_build/github/azure-pipelines/templates/linux-build-step-with-cache.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/linux-build-step-with-cache.yml
@ -0,0 +1,50 @@
+parameters:
+- name: WithCache
+  displayName: Build with Cache
+  type: boolean
+  default: false
+
+- name: BuildStep
+  type: stepList
+
+- name: Today
+  type: string
+  default: ""
+
+# It'd be different in different jobs in one workflow.
+- name: AdditionalKey
+  type: string
+  default: ""
+
+- name: ChangeEveryCommit
+  type: boolean
+  default: true
+
+- name: CacheDir
+  type: string
+  default: ""
+
+steps:
+  - ${{if eq(parameters.WithCache, true)}}:
+    - script: |
+        mkdir -p ${{ parameters.CacheDir }}
+      displayName: Create Cache Dir
+
+  - ${{if eq(parameters.WithCache, true)}}:
+    - task: Cache@2
+      inputs:
+        ${{ if eq(parameters.ChangeEveryCommit, true) }}:
+          ${{if eq(variables['Build.SourceBranchName'], 'merge')}}:
+            key: ' "${{parameters.Today}}" | merge | ${{parameters.AdditionalKey}} '
+          ${{else}}:
+            key:  ' "${{parameters.Today}}" | ${{parameters.AdditionalKey}} | "$(Build.SourceVersion)" '
+          restoreKeys: |
+            "${{parameters.Today}}" | ${{parameters.AdditionalKey}}
+        ${{ else }}:
+          key:  ' ${{parameters.AdditionalKey}} '
+          restoreKeys: |
+            ${{parameters.AdditionalKey}}
+        path: ${{ parameters.CacheDir }}
+      displayName: ccache task
+
+  - ${{ parameters.BuildStep }}
--- a/tools/ci_build/github/azure-pipelines/templates/py-linux.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/py-linux.yml
@ -21,6 +21,10 @@ parameters:
  type: string
  default: '-d CPU'

+- name: with_cache
+  type: boolean
+  default: false
+
 jobs:
 - job: Linux_py_Wheels_${{ parameters.arch }}
  timeoutInMinutes: 240
@ -30,7 +34,13 @@ jobs:
  variables:
    # The build machine pool doesn't have dotnet, so it can't run CG.
    skipComponentGovernanceDetection: true
+    ORT_CACHE_DIR: $(Agent.TempDirectory)/ort_ccache
+    TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)]
  steps:
+    - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
+      displayName: 'Clean Agent Directories'
+      condition: always()
+
    - checkout: self
      clean: true
      submodules: none
@ -46,12 +56,23 @@ jobs:
        ${{ if eq(parameters.arch, 'aarch64') }}:
            UpdateDepsTxt: false

-    - task: Bash@3
-      displayName: 'Build Python Wheel'
-      inputs:
-        targetType: filePath
-        filePath: tools/ci_build/github/linux/run_python_dockerbuild.sh
-        arguments: -i onnxruntimecpubuilpython${{ parameters.arch }} -x "${{ parameters.device }}"
+    - template: linux-build-step-with-cache.yml
+      parameters:
+        WithCache: ${{parameters.with_cache}}
+        Today: $(TODAY)
+        AdditionalKey: Linux_py_Wheels_${{ parameters.arch }}
+        CacheDir: $(ORT_CACHE_DIR)
+        ChangeEveryCommit: true
+        BuildStep:
+          - task: Bash@3
+            displayName: 'Build Python Wheel'
+            inputs:
+              targetType: filePath
+              filePath: tools/ci_build/github/linux/run_python_dockerbuild.sh
+              arguments: -i onnxruntimecpubuilpython${{ parameters.arch }} -x "${{ parameters.device }}"
+            ${{ if eq(parameters.with_cache, 'true') }}:
+              env:
+                ADDITIONAL_DOCKER_PARAMETER: "--volume $(ORT_CACHE_DIR):/cache -e CCACHE_DIR=/cache -e ORT_BUILD_WITH_CACHE=1"

    - task: PublishBuildArtifacts@1
      displayName: 'Publish Artifact: ONNXRuntime python wheel'
@ -64,8 +85,3 @@ jobs:
      inputs:
        artifactName: 'drop-linux-cpu-${{ parameters.arch }}'
        targetPath: '$(Build.BinariesDirectory)/Release'
-
-
-    - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
-      displayName: 'Clean Agent Directories'
-      condition: always()
--- a/tools/ci_build/github/linux/build_linux_arm64_python_package.sh
+++ b/tools/ci_build/github/linux/build_linux_arm64_python_package.sh
@ -64,3 +64,5 @@ do

  cp /build/$BUILD_CONFIG/dist/*.whl /build/dist
 done
+
+which ccache && ccache -sv && ccache -z
--- a/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_centos.sh
+++ b/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_centos.sh
@ -4,4 +4,4 @@ set -e
 os_major_version=$(cat /etc/redhat-release | tr -dc '0-9.'|cut -d \. -f1)

 echo "installing for os major version : $os_major_version"
-yum install -y which gdb redhat-lsb-core expat-devel tar unzip zlib-devel make libunwind bzip2 bzip2-devel
+yum install -y which gdb redhat-lsb-core expat-devel tar unzip zlib-devel make libunwind bzip2 bzip2-devel
--- a/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_deps.sh
+++ b/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_deps.sh
@ -15,4 +15,22 @@ do
  ${PYTHON_EXE} -m pip install -r requirements.txt
 done

+# No release binary for ccache aarch64, so we need to build it from source.
+if ! [ -x "$(command -v ccache)" ]; then
+    ccache_url="https://github.com/ccache/ccache/archive/refs/tags/v4.8.tar.gz"
+    pushd .
+    curl -sSL --retry 5 --retry-delay 10 --create-dirs --fail -L -o ccache_src.tar.gz $ccache_url
+    mkdir ccache_main
+    cd ccache_main
+    tar -zxf ../ccache_src.tar.gz --strip=1

+    mkdir build
+    cd build
+    cmake -DCMAKE_INSTALL_PREFIX=/usr/local _DCMAKE_BUILD_TYPE=Release ..
+    make
+    make install
+    which ccache
+    popd
+    rm -f ccache_src.tar.gz
+    rm -rf ccache_src
+fi
--- a/tools/ci_build/github/linux/run_python_dockerbuild.sh
+++ b/tools/ci_build/github/linux/run_python_dockerbuild.sh
@ -18,6 +18,7 @@ docker run --rm \
    -w /onnxruntime_src \
    -e NIGHTLY_BUILD \
    -e BUILD_BUILDNUMBER \
+    $ADDITIONAL_DOCKER_PARAMETER \
    $DOCKER_IMAGE tools/ci_build/github/linux/build_linux_arm64_python_package.sh $BUILD_EXTR_PAR

 sudo rm -rf $BUILD_BINARIESDIRECTORY/Release/onnxruntime $BUILD_BINARIESDIRECTORY/Release/pybind11 \