Enable cache for msbuild (#14085)

### Description Enable ccache in windows CPU compilation. The windows compilation in CI could be reduced to 1 more minute at most. ![image](https://user-images.githubusercontent.com/16190118/210294061-86742cf4-65c7-4cc2-9725-e102c3c64abd.png)
2026-05-28 22:56:32 +00:00 · 2023-01-06 11:19:57 +08:00 · 2023-01-06 11:19:57 +08:00 · 2ce7b1c1dc
commit 2ce7b1c1dc
parent d460c01b8c
8 changed files with 134 additions and 18 deletions
--- a/cmake/CMakeLists.txt
+++ b/cmake/CMakeLists.txt
@ -196,6 +196,10 @@ option(onnxruntime_ENABLE_ATEN "Enable ATen fallback" OFF)

 option(onnxruntime_BUILD_KERNEL_EXPLORER "Build Kernel Explorer for testing and profiling GPU kernels" OFF)

+option(onnxruntime_BUILD_CACHE "onnxruntime build with cache" OFF)
+# https://zeux.io/2010/11/22/z7-everything-old-is-new-again/
+cmake_dependent_option(MSVC_Z7_OVERRIDE "replacing /Zi and /ZI with /Z7 when using MSVC with CCache" ON "onnxruntime_BUILD_CACHE; MSVC" OFF)
+
 option(onnxruntime_USE_CLOUD "Build with cloud inferencing support" OFF)

 # ENABLE_TRAINING includes all training functionality
@ -681,6 +685,18 @@ if (onnxruntime_ENABLE_LAZY_TENSOR)
  # cleaner.
 endif()

+if(MSVC)
+  # Replace /Zi and /ZI with /Z7
+  if(MSVC_Z7_OVERRIDE)
+    foreach(flag_var
+      CMAKE_C_FLAGS CMAKE_C_FLAGS_DEBUG  CMAKE_C_FLAGS_RELWITHDEBINFO
+      CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG  CMAKE_CXX_FLAGS_RELWITHDEBINFO)
+      if(${flag_var} MATCHES "/Z[iI]")
+        string(REGEX REPLACE "/Z[iI]" "/Z7" ${flag_var} "${${flag_var}}")
+      endif(${flag_var} MATCHES "/Z[iI]")
+    endforeach(flag_var)
+  endif(MSVC_Z7_OVERRIDE)
+endif()

 function(onnxruntime_set_compile_flags target_name)
    if (CPUINFO_SUPPORTED)
@ -851,6 +867,7 @@ function(onnxruntime_configure_target target_name)
  if(MSVC AND NOT onnxruntime_target_platform MATCHES "ARM")
    target_link_options(${target_name} PRIVATE "/CETCOMPAT")
  endif()
+
 endfunction()

 function(onnxruntime_add_shared_library target_name)
--- a/cmake/external/helper_functions.cmake
+++ b/cmake/external/helper_functions.cmake
@ -163,11 +163,11 @@ macro(onnxruntime_fetchcontent_makeavailable)
          get_property(subdir_import_targets DIRECTORY "${__cmake_srcdir}" PROPERTY BUILDSYSTEM_TARGETS)
          foreach(subdir_target ${subdir_import_targets})
            if(TARGET ${subdir_target})
-              get_target_property(subdir_target_type ${subdir_target} TYPE)            
+              get_target_property(subdir_target_type ${subdir_target} TYPE)
              if(subdir_target_type STREQUAL "EXECUTABLE")
                get_target_property(subdir_target_osx_arch ${subdir_target} OSX_ARCHITECTURES)
                if (subdir_target_osx_arch)
-                  if (NOT ${CMAKE_HOST_SYSTEM_PROCESSOR} IN_LIST subdir_target_osx_arch)                              
+                  if (NOT ${CMAKE_HOST_SYSTEM_PROCESSOR} IN_LIST subdir_target_osx_arch)
                    message("Added an executable target ${subdir_target} but it can not run natively on ${CMAKE_HOST_SYSTEM_PROCESSOR}, we will try to modify it")
                  endif()
                endif()
@ -175,7 +175,19 @@ macro(onnxruntime_fetchcontent_makeavailable)
              set_target_properties(${subdir_target} PROPERTIES FOLDER "External")
              set_target_properties(${subdir_target} PROPERTIES COMPILE_WARNING_AS_ERROR OFF)
            endif()
-          endforeach()          
+          endforeach()
+          if(MSVC)
+            # Replace /Zi and /ZI with /Z7
+            if(MSVC_Z7_OVERRIDE)
+              foreach(flag_var
+                CMAKE_C_FLAGS CMAKE_C_FLAGS_DEBUG  CMAKE_C_FLAGS_RELWITHDEBINFO
+                CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG  CMAKE_CXX_FLAGS_RELWITHDEBINFO)
+                if(${flag_var} MATCHES "/Z[iI]")
+                  string(REGEX REPLACE "/Z[iI]" "/Z7" ${flag_var} "${${flag_var}}")
+                endif(${flag_var} MATCHES "/Z[iI]")
+              endforeach(flag_var)
+            endif(MSVC_Z7_OVERRIDE)
+        endif()
      endif()

      unset(__cmake_srcdir)
@ -196,4 +208,4 @@ macro(onnxruntime_fetchcontent_makeavailable)
  unset(__cmake_contentNameUpper)
  unset(__cmake_providerCommand)
  unset(__cmake_original_verify_setting)
-endmacro()
+endmacro()
--- a/tools/ci_build/build.py
+++ b/tools/ci_build/build.py
@ -965,10 +965,12 @@ def generate_build_tree(
        "-Donnxruntime_USE_CANN=" + ("ON" if args.use_cann else "OFF"),
    ]
    if args.use_cache:
-        cmake_args.append("-DCMAKE_CXX_COMPILER_LAUNCHER=ccache")
-        cmake_args.append("-DCMAKE_C_COMPILER_LAUNCHER=ccache")
-        if args.use_cuda:
+        cmake_args.append("-Donnxruntime_BUILD_CACHE=ON")
+        if not (is_windows() and args.cmake_generator != "Ninja"):
+            cmake_args.append("-DCMAKE_CXX_COMPILER_LAUNCHER=ccache")
            cmake_args.append("-DCMAKE_C_COMPILER_LAUNCHER=ccache")
+            if args.use_cuda:
+                cmake_args.append("-DCMAKE_CUDA_COMPILER_LAUNCHER=ccache")
    # By default cmake does not check TLS/SSL certificates. Here we turn it on.
    # But, in some cases you may also need to supply a CA file.
    add_default_definition(cmake_extra_defines, "CMAKE_TLS_VERIFY", "ON")
@ -1369,9 +1371,10 @@ def build_targets(args, cmake_path, build_dir, configs, num_parallel_jobs, targe
        if num_parallel_jobs != 1:
            if is_windows() and args.cmake_generator != "Ninja" and not args.build_wasm:
                build_tool_args += [
-                    "/maxcpucount:{}".format(num_parallel_jobs),
+                    f"/maxcpucount:{num_parallel_jobs}",
                    # if nodeReuse is true, msbuild processes will stay around for a bit after the build completes
                    "/nodeReuse:False",
+                    f"/p:CL_MPCount={num_parallel_jobs}",
                ]
            elif is_macOS() and args.use_xcode:
                # CMake will generate correct build tool args for Xcode
--- a/tools/ci_build/github/azure-pipelines/templates/win-ci-vs-2019.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/win-ci-vs-2019.yml
@ -50,6 +50,11 @@ parameters:
  type: boolean
  default: false

+- name: WITH_CACHE
+  displayName: Use Cache to acclerate compilation
+  type: boolean
+  default: false
+
 jobs:
 - job: build_${{ parameters.job_name_suffix }}
  variables:
@ -60,6 +65,12 @@ jobs:
    ALLOW_RELEASED_ONNX_OPSET_ONLY: '0'
    DocUpdateNeeded: false  # Set to true during document generation if there are diffs
    skipComponentGovernanceDetection: true
+    CCACHE_DIR: $(Pipeline.Workspace)/ccache
+    TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)]
+    ${{ if eq(parameters.WITH_CACHE, true) }}:
+      PS_CACHE_ARG: '-use_cache'
+      PY_CACHE_ARG: '--use_cache'
+      MSBUILD_CACHE_ARG: '/p:CLToolExe=cl.exe /p:CLToolPath=C:\ProgramData\chocolatey\bin /p:TrackFileAccess=false /p:UseMultiToolTask=true /p:DebugInformationFormat=OldStyle'
  workspace:
    clean: all
  pool: ${{ parameters.MachinePool }}
@ -119,13 +130,44 @@ jobs:
    workingDirectory: '$(Build.BinariesDirectory)'
    displayName: 'Install python modules'

+  - powershell: |
+      if ([string]::IsNullOrEmpty((Get-Command ccache -errorAction SilentlyContinue)))
+      {
+        choco install ccache -y --version 4.7.4
+        $ccache_path = (Get-Command ccache).Source
+        $ccache_parent_dir = (Split-Path -parent $ccache_path)
+        Copy-Item "C:\ProgramData\chocolatey\lib\ccache\tools\ccache-4.7.4-windows-x86_64\ccache.exe" -Destination "C:\ProgramData\chocolatey\bin\cl.exe"
+        Get-ChildItem $ccache_parent_dir
+        ccache --version
+      }
+    displayName: Install ccache and update PATH to use linked versions of gcc, cc, etc
+    condition: eq(${{ parameters.WITH_CACHE }}, true)
+
+  - task: Cache@2
+    inputs:
+      key: '"$(TODAY)" | ccache | "$(System.StageName)" | "$(Build.SourceBranch)" | "$(Build.SourceVersion)" '
+      path: $(CCACHE_DIR)
+      restoreKeys: |
+        "$(TODAY)" | ccache | "$(System.StageName)" | "$(Build.SourceBranch)"
+        "$(TODAY)" | ccache | "$(System.StageName)"
+        "$(TODAY)" | ccache |
+    displayName: Cache Task
+    condition: eq(${{ parameters.WITH_CACHE }}, true)
+
  - ${{ if or(eq(parameters.RunOnnxRuntimeTests, true), eq(parameters.GenerateDocumentation, true)) }}:
      - task: PowerShell@2
        displayName: 'Install ONNX'
        inputs:
          filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/windows/install_third_party_deps.ps1'
          workingDirectory: '$(Build.BinariesDirectory)'
-          arguments: -cpu_arch ${{ parameters.buildArch }} -install_prefix $(Build.BinariesDirectory)\${{ parameters.BuildConfig }}\installed -build_config ${{ parameters.BuildConfig }}
+          arguments: -cpu_arch ${{ parameters.buildArch }} -install_prefix $(Build.BinariesDirectory)\${{ parameters.BuildConfig }}\installed -build_config ${{ parameters.BuildConfig }} ${{ variables['PS_CACHE_ARG'] }}
+
+      - powershell: |
+          ccache -sv
+          ccache -z
+        displayName: cache stat
+        condition: eq(${{ parameters.WITH_CACHE }}, true)
+

  - task: NuGetToolInstaller@0
    displayName: Use Nuget 5.7.0
@ -145,7 +187,7 @@ jobs:
    displayName: 'Generate cmake config'
    inputs:
      scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
-      arguments: '--config ${{ parameters.BuildConfig }} --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --build_csharp --update --cmake_generator "Visual Studio 16 2019"  --build_shared_lib --enable_onnx_tests ${{ parameters.additionalBuildFlags }}'
+      arguments: '--config ${{ parameters.BuildConfig }} --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_csharp --update --parallel --cmake_generator "Visual Studio 16 2019" --build_shared_lib --enable_onnx_tests ${{ variables.PY_CACHE_ARG }} ${{ parameters.additionalBuildFlags }}'
      workingDirectory: '$(Build.BinariesDirectory)'

  - task: VSBuild@1
@ -154,13 +196,20 @@ jobs:
      solution: '$(Build.BinariesDirectory)\${{ parameters.BuildConfig }}\onnxruntime.sln'
      platform: ${{ parameters.msbuildPlatform }}
      configuration: ${{ parameters.BuildConfig }}
-      msbuildArgs: -maxcpucount
+      msbuildArgs: '-maxcpucount ${{ variables.MSBUILD_CACHE_ARG }}'
      msbuildArchitecture: ${{ parameters.buildArch }}
      maximumCpuCount: true
      logProjectEvents: false
      workingFolder: '$(Build.BinariesDirectory)\${{ parameters.BuildConfig }}'
      createLogFile: true

+  - powershell: |
+      ccache -sv
+      ccache -z
+    displayName: cache stat
+    condition: eq(${{ parameters.WITH_CACHE }}, true)
+
+
  - ${{ if eq(parameters.EnablePython, true) }}:
      - task: PythonScript@0
        displayName: 'Build wheel'
@ -274,7 +323,7 @@ jobs:
      displayName: 'Generate documentation'
      inputs:
        scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
-        arguments: '--config ${{ parameters.BuildConfig }} --build_dir $(Build.BinariesDirectory) --gen_doc validate'
+        arguments: '--config ${{ parameters.BuildConfig }} --build_dir $(Build.BinariesDirectory) --gen_doc validate ${{ variables.PY_CACHE_ARG }}'
        workingDirectory: '$(Build.BinariesDirectory)'

  # if the validation from --gen_doc failed it sets DocUpdateNeeded so we can publish the latest version of the docs
--- a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml
@ -22,6 +22,7 @@ stages:
        isTraining: false
        ORT_EP_NAME: CPU
        GenerateDocumentation: false
+        WITH_CACHE: true
        MachinePool: 'Win-CPU-2019'

 - stage: x64_release
@ -43,6 +44,7 @@ stages:
        isTraining: false
        ORT_EP_NAME: CPU
        GenerateDocumentation: false
+        WITH_CACHE: true
        MachinePool: 'Win-CPU-2019'

 - stage: x64_release_dnnl
@ -62,6 +64,7 @@ stages:
        isTraining: false
        ORT_EP_NAME: DNNL
        GenerateDocumentation: false
+        WITH_CACHE: true
        MachinePool: 'Win-CPU-2019'

 - stage: x64_release_xnnpack
@ -81,6 +84,7 @@ stages:
        isTraining: false
        ORT_EP_NAME: XNNPACK
        GenerateDocumentation: false
+        WITH_CACHE: true
        MachinePool: 'Win-CPU-2019'

 - stage: x64_release_winml
@ -102,6 +106,7 @@ stages:
        isTraining: false
        ORT_EP_NAME: CPU
        GenerateDocumentation: false
+        WITH_CACHE: true
        MachinePool: 'Win-CPU-2019'

 - stage: x86_release
@ -121,6 +126,7 @@ stages:
        isTraining: false
        ORT_EP_NAME: CPU
        GenerateDocumentation: false
+        WITH_CACHE: true
        MachinePool: 'Win-CPU-2019'

 - stage: training_x64_debug
@ -140,6 +146,7 @@ stages:
        isTraining: true
        ORT_EP_NAME: CPU
        GenerateDocumentation: false
+        WITH_CACHE: true
        MachinePool: 'onnxruntime-Win2019-CPU-training'

 - stage: training_x64_release
@ -159,6 +166,7 @@ stages:
        isTraining: true
        ORT_EP_NAME: CPU
        GenerateDocumentation: false
+        WITH_CACHE: true
        MachinePool: 'onnxruntime-Win2019-CPU-training'

 - stage: ort_training_apis_x64_release
@ -179,8 +187,10 @@ stages:
        isTraining: true
        ORT_EP_NAME: CPU
        GenerateDocumentation: false
+        WITH_CACHE: true
        MachinePool: 'onnxruntime-Win2019-CPU-training'

+
 - stage: x64_release_cloud
  dependsOn: []
  jobs:
@ -199,4 +209,5 @@ stages:
        isTraining: false
        ORT_EP_NAME: CPU
        GenerateDocumentation: false
+        WITH_CACHE: true
        MachinePool: 'Win-CPU-2019'
--- a/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml
@ -43,6 +43,7 @@ stages:
        RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }}
        RunStaticCodeAnalysis: false
        ORT_EP_NAME: CUDA
+        WITH_CACHE: true
        MachinePool: onnxruntime-Win2019-GPU-T4

 - stage: training
@ -60,6 +61,7 @@ stages:
        RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }}
        RunStaticCodeAnalysis: false
        ORT_EP_NAME: CUDA
+        WITH_CACHE: true
        MachinePool: onnxruntime-Win2019-GPU-training-T4
        isTraining: true

@ -78,6 +80,7 @@ stages:
        RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }}
        RunStaticCodeAnalysis: false
        ORT_EP_NAME: DML
+        WITH_CACHE: true
        # DirectML cannot run on T4 GPUs.
        MachinePool: onnxruntime-Win2019-GPU-dml

@ -98,4 +101,5 @@ stages:
        RunStaticCodeAnalysis: false
        GenerateDocumentation: true
        ORT_EP_NAME: CUDA # It doesn't really matter which EP is selected here since this stage is for documentation.
+        WITH_CACHE: true
        MachinePool: onnxruntime-Win2019-GPU-T4
--- a/tools/ci_build/github/windows/helpers.ps1
+++ b/tools/ci_build/github/windows/helpers.ps1
@ -392,6 +392,9 @@ function Install-Protobuf {
        exit $exitCode
    }
    $cmake_args = "--build", ".",  "--parallel", "--config", $build_config, "--target", "INSTALL"
+    if ($use_cache) {
+      $cmake_args += "--", "/p:CLToolExe=cl.exe /p:CLToolPath=C:\ProgramData\chocolatey\bin /p:TrackFileAccess=false /p:UseMultiToolTask=true"
+    }
    $p = Start-Process -FilePath $cmake_path -ArgumentList $cmake_args -NoNewWindow -Wait -PassThru
    $exitCode = $p.ExitCode
    if ($exitCode -ne 0) {
@ -458,4 +461,4 @@ function Install-ONNX {
    }
    Write-Host "Finished installing onnx"
    popd
-}
+}
--- a/tools/ci_build/github/windows/install_third_party_deps.ps1
+++ b/tools/ci_build/github/windows/install_third_party_deps.ps1
@ -7,7 +7,8 @@
 param (
    [string]$cpu_arch = "x64",
    [string]$build_config = "RelWithDebInfo",
-    [string]$install_prefix = "."
+    [string]$install_prefix = ".",
+    [switch]$use_cache
 )

 . "$PSScriptRoot\helpers.ps1"
@ -27,15 +28,23 @@ New-Item -Path "$install_prefix" -ItemType Directory -Force
 $compile_flags = '/MP /guard:cf /Qspectre /DWIN32 /D_WINDOWS /DWINVER=0x0601 /D_WIN32_WINNT=0x0601 /DNTDDI_VERSION=0x06010000 /W3 '
 $linker_flags=@('/guard:cf')

+if ($use_cache) {
+  $debug_info_format = "/Z7"
+}
+else {
+  $debug_info_format = "/Zi"
+}
+
 if($build_config -eq 'Release'){
  $compile_flags += "/O2", "/Ob2", "/DNDEBUG", "/Gw", "/GL"
 } elseif($build_config -eq 'RelWithDebInfo'){
-  $compile_flags += "/Zi", "/O2", "/Ob1", "/DNDEBUG", "/Gw", "/GL"
+    $compile_flags += "$debug_info_format", "/O2", "/Ob1", "/DNDEBUG", "/Gw", "/GL"
 } elseif($build_config -eq 'Debug'){
-  $compile_flags += "/Zi", "/Ob0", "/Od", "/RTC1"
+  $compile_flags += "$debug_info_format", "/Ob0", "/Od", "/RTC1"
 } elseif($build_config -eq 'MinSizeRel'){
  $compile_flags += "/O1", "/Ob1", "/DNDEBUG", "/Gw", "/GL"
 }
+Write-Host $compile_flags
 # cmake args that applies to every 3rd-party library
 [string[]]$cmake_extra_args="-DCMAKE_CXX_STANDARD=17 `"-DCMAKE_CXX_FLAGS=$compile_flags /EHsc`" ", "`"-DCMAKE_C_FLAGS=$compile_flags`"", "--compile-no-warning-as-error", "--fresh", "-Wno-dev"
 if($cpu_arch -eq 'x86'){
@ -47,11 +56,19 @@ if($cpu_arch -eq 'x86'){
  throw "$cpu_arch is not supported"
 }

+if ($use_cache) {
+  if ($build_config -eq 'RelWithDebInfo') {
+    $cmake_extra_args += "-DCMAKE_CXX_FLAGS_RELWITHDEBINFO=`"/MD /Z7 /O2 /Ob1 /DNDEBUG`""
+  }
+  elseif ($build_config -eq 'Debug') {
+    $cmake_extra_args += "-DCMAKE_CXX_FLAGS_DEBUG=`"/MDd /Z7 /Ob0 /Od /RTC1`""
+  }
+}

 $cmake_extra_args += "-DCMAKE_EXE_LINKER_FLAGS=`"$linker_flags`""

 # Find the full path of cmake.exe
-$cmake_command = Get-Command -CommandType Application cmake
+$cmake_command = (Get-Command -CommandType Application cmake)[0]
 $cmake_path = $cmake_command.Path

 Install-Pybind -cmake_path $cmake_path -src_root $ort_src_root -build_config $build_config  -cmake_extra_args $cmake_extra_args
@ -62,4 +79,4 @@ Install-Protobuf -cmake_path $cmake_path -src_root $ort_src_root -build_config $
 $protobuf_version="3.18.3"

 # ONNX doesn't allow us to specify CMake's path
-Install-ONNX -build_config $build_config -src_root $ort_src_root -protobuf_version $protobuf_version
+Install-ONNX -build_config $build_config -src_root $ort_src_root -protobuf_version $protobuf_version