From 2ce7b1c1dcc2e21a79be672d2fcf2c480fafd8b0 Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Fri, 6 Jan 2023 11:19:57 +0800 Subject: [PATCH] Enable cache for msbuild (#14085) ### Description Enable ccache in windows CPU compilation. The windows compilation in CI could be reduced to 1 more minute at most. ![image](https://user-images.githubusercontent.com/16190118/210294061-86742cf4-65c7-4cc2-9725-e102c3c64abd.png) --- cmake/CMakeLists.txt | 17 ++++++ cmake/external/helper_functions.cmake | 20 +++++-- tools/ci_build/build.py | 11 ++-- .../templates/win-ci-vs-2019.yml | 57 +++++++++++++++++-- .../azure-pipelines/win-ci-pipeline.yml | 11 ++++ .../azure-pipelines/win-gpu-ci-pipeline.yml | 4 ++ tools/ci_build/github/windows/helpers.ps1 | 5 +- .../windows/install_third_party_deps.ps1 | 27 +++++++-- 8 files changed, 134 insertions(+), 18 deletions(-) diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 8a07e7eddc..b5bda37a19 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -196,6 +196,10 @@ option(onnxruntime_ENABLE_ATEN "Enable ATen fallback" OFF) option(onnxruntime_BUILD_KERNEL_EXPLORER "Build Kernel Explorer for testing and profiling GPU kernels" OFF) +option(onnxruntime_BUILD_CACHE "onnxruntime build with cache" OFF) +# https://zeux.io/2010/11/22/z7-everything-old-is-new-again/ +cmake_dependent_option(MSVC_Z7_OVERRIDE "replacing /Zi and /ZI with /Z7 when using MSVC with CCache" ON "onnxruntime_BUILD_CACHE; MSVC" OFF) + option(onnxruntime_USE_CLOUD "Build with cloud inferencing support" OFF) # ENABLE_TRAINING includes all training functionality @@ -681,6 +685,18 @@ if (onnxruntime_ENABLE_LAZY_TENSOR) # cleaner. endif() +if(MSVC) + # Replace /Zi and /ZI with /Z7 + if(MSVC_Z7_OVERRIDE) + foreach(flag_var + CMAKE_C_FLAGS CMAKE_C_FLAGS_DEBUG CMAKE_C_FLAGS_RELWITHDEBINFO + CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELWITHDEBINFO) + if(${flag_var} MATCHES "/Z[iI]") + string(REGEX REPLACE "/Z[iI]" "/Z7" ${flag_var} "${${flag_var}}") + endif(${flag_var} MATCHES "/Z[iI]") + endforeach(flag_var) + endif(MSVC_Z7_OVERRIDE) +endif() function(onnxruntime_set_compile_flags target_name) if (CPUINFO_SUPPORTED) @@ -851,6 +867,7 @@ function(onnxruntime_configure_target target_name) if(MSVC AND NOT onnxruntime_target_platform MATCHES "ARM") target_link_options(${target_name} PRIVATE "/CETCOMPAT") endif() + endfunction() function(onnxruntime_add_shared_library target_name) diff --git a/cmake/external/helper_functions.cmake b/cmake/external/helper_functions.cmake index cbd06755c4..88b46890b7 100644 --- a/cmake/external/helper_functions.cmake +++ b/cmake/external/helper_functions.cmake @@ -163,11 +163,11 @@ macro(onnxruntime_fetchcontent_makeavailable) get_property(subdir_import_targets DIRECTORY "${__cmake_srcdir}" PROPERTY BUILDSYSTEM_TARGETS) foreach(subdir_target ${subdir_import_targets}) if(TARGET ${subdir_target}) - get_target_property(subdir_target_type ${subdir_target} TYPE) + get_target_property(subdir_target_type ${subdir_target} TYPE) if(subdir_target_type STREQUAL "EXECUTABLE") get_target_property(subdir_target_osx_arch ${subdir_target} OSX_ARCHITECTURES) if (subdir_target_osx_arch) - if (NOT ${CMAKE_HOST_SYSTEM_PROCESSOR} IN_LIST subdir_target_osx_arch) + if (NOT ${CMAKE_HOST_SYSTEM_PROCESSOR} IN_LIST subdir_target_osx_arch) message("Added an executable target ${subdir_target} but it can not run natively on ${CMAKE_HOST_SYSTEM_PROCESSOR}, we will try to modify it") endif() endif() @@ -175,7 +175,19 @@ macro(onnxruntime_fetchcontent_makeavailable) set_target_properties(${subdir_target} PROPERTIES FOLDER "External") set_target_properties(${subdir_target} PROPERTIES COMPILE_WARNING_AS_ERROR OFF) endif() - endforeach() + endforeach() + if(MSVC) + # Replace /Zi and /ZI with /Z7 + if(MSVC_Z7_OVERRIDE) + foreach(flag_var + CMAKE_C_FLAGS CMAKE_C_FLAGS_DEBUG CMAKE_C_FLAGS_RELWITHDEBINFO + CMAKE_CXX_FLAGS CMAKE_CXX_FLAGS_DEBUG CMAKE_CXX_FLAGS_RELWITHDEBINFO) + if(${flag_var} MATCHES "/Z[iI]") + string(REGEX REPLACE "/Z[iI]" "/Z7" ${flag_var} "${${flag_var}}") + endif(${flag_var} MATCHES "/Z[iI]") + endforeach(flag_var) + endif(MSVC_Z7_OVERRIDE) + endif() endif() unset(__cmake_srcdir) @@ -196,4 +208,4 @@ macro(onnxruntime_fetchcontent_makeavailable) unset(__cmake_contentNameUpper) unset(__cmake_providerCommand) unset(__cmake_original_verify_setting) -endmacro() \ No newline at end of file +endmacro() diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 98af43b403..c6bd645831 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -965,10 +965,12 @@ def generate_build_tree( "-Donnxruntime_USE_CANN=" + ("ON" if args.use_cann else "OFF"), ] if args.use_cache: - cmake_args.append("-DCMAKE_CXX_COMPILER_LAUNCHER=ccache") - cmake_args.append("-DCMAKE_C_COMPILER_LAUNCHER=ccache") - if args.use_cuda: + cmake_args.append("-Donnxruntime_BUILD_CACHE=ON") + if not (is_windows() and args.cmake_generator != "Ninja"): + cmake_args.append("-DCMAKE_CXX_COMPILER_LAUNCHER=ccache") cmake_args.append("-DCMAKE_C_COMPILER_LAUNCHER=ccache") + if args.use_cuda: + cmake_args.append("-DCMAKE_CUDA_COMPILER_LAUNCHER=ccache") # By default cmake does not check TLS/SSL certificates. Here we turn it on. # But, in some cases you may also need to supply a CA file. add_default_definition(cmake_extra_defines, "CMAKE_TLS_VERIFY", "ON") @@ -1369,9 +1371,10 @@ def build_targets(args, cmake_path, build_dir, configs, num_parallel_jobs, targe if num_parallel_jobs != 1: if is_windows() and args.cmake_generator != "Ninja" and not args.build_wasm: build_tool_args += [ - "/maxcpucount:{}".format(num_parallel_jobs), + f"/maxcpucount:{num_parallel_jobs}", # if nodeReuse is true, msbuild processes will stay around for a bit after the build completes "/nodeReuse:False", + f"/p:CL_MPCount={num_parallel_jobs}", ] elif is_macOS() and args.use_xcode: # CMake will generate correct build tool args for Xcode diff --git a/tools/ci_build/github/azure-pipelines/templates/win-ci-vs-2019.yml b/tools/ci_build/github/azure-pipelines/templates/win-ci-vs-2019.yml index 90d996c0d7..d479323d7a 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-ci-vs-2019.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-ci-vs-2019.yml @@ -50,6 +50,11 @@ parameters: type: boolean default: false +- name: WITH_CACHE + displayName: Use Cache to acclerate compilation + type: boolean + default: false + jobs: - job: build_${{ parameters.job_name_suffix }} variables: @@ -60,6 +65,12 @@ jobs: ALLOW_RELEASED_ONNX_OPSET_ONLY: '0' DocUpdateNeeded: false # Set to true during document generation if there are diffs skipComponentGovernanceDetection: true + CCACHE_DIR: $(Pipeline.Workspace)/ccache + TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)] + ${{ if eq(parameters.WITH_CACHE, true) }}: + PS_CACHE_ARG: '-use_cache' + PY_CACHE_ARG: '--use_cache' + MSBUILD_CACHE_ARG: '/p:CLToolExe=cl.exe /p:CLToolPath=C:\ProgramData\chocolatey\bin /p:TrackFileAccess=false /p:UseMultiToolTask=true /p:DebugInformationFormat=OldStyle' workspace: clean: all pool: ${{ parameters.MachinePool }} @@ -119,13 +130,44 @@ jobs: workingDirectory: '$(Build.BinariesDirectory)' displayName: 'Install python modules' + - powershell: | + if ([string]::IsNullOrEmpty((Get-Command ccache -errorAction SilentlyContinue))) + { + choco install ccache -y --version 4.7.4 + $ccache_path = (Get-Command ccache).Source + $ccache_parent_dir = (Split-Path -parent $ccache_path) + Copy-Item "C:\ProgramData\chocolatey\lib\ccache\tools\ccache-4.7.4-windows-x86_64\ccache.exe" -Destination "C:\ProgramData\chocolatey\bin\cl.exe" + Get-ChildItem $ccache_parent_dir + ccache --version + } + displayName: Install ccache and update PATH to use linked versions of gcc, cc, etc + condition: eq(${{ parameters.WITH_CACHE }}, true) + + - task: Cache@2 + inputs: + key: '"$(TODAY)" | ccache | "$(System.StageName)" | "$(Build.SourceBranch)" | "$(Build.SourceVersion)" ' + path: $(CCACHE_DIR) + restoreKeys: | + "$(TODAY)" | ccache | "$(System.StageName)" | "$(Build.SourceBranch)" + "$(TODAY)" | ccache | "$(System.StageName)" + "$(TODAY)" | ccache | + displayName: Cache Task + condition: eq(${{ parameters.WITH_CACHE }}, true) + - ${{ if or(eq(parameters.RunOnnxRuntimeTests, true), eq(parameters.GenerateDocumentation, true)) }}: - task: PowerShell@2 displayName: 'Install ONNX' inputs: filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/windows/install_third_party_deps.ps1' workingDirectory: '$(Build.BinariesDirectory)' - arguments: -cpu_arch ${{ parameters.buildArch }} -install_prefix $(Build.BinariesDirectory)\${{ parameters.BuildConfig }}\installed -build_config ${{ parameters.BuildConfig }} + arguments: -cpu_arch ${{ parameters.buildArch }} -install_prefix $(Build.BinariesDirectory)\${{ parameters.BuildConfig }}\installed -build_config ${{ parameters.BuildConfig }} ${{ variables['PS_CACHE_ARG'] }} + + - powershell: | + ccache -sv + ccache -z + displayName: cache stat + condition: eq(${{ parameters.WITH_CACHE }}, true) + - task: NuGetToolInstaller@0 displayName: Use Nuget 5.7.0 @@ -145,7 +187,7 @@ jobs: displayName: 'Generate cmake config' inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: '--config ${{ parameters.BuildConfig }} --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --build_csharp --update --cmake_generator "Visual Studio 16 2019" --build_shared_lib --enable_onnx_tests ${{ parameters.additionalBuildFlags }}' + arguments: '--config ${{ parameters.BuildConfig }} --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_csharp --update --parallel --cmake_generator "Visual Studio 16 2019" --build_shared_lib --enable_onnx_tests ${{ variables.PY_CACHE_ARG }} ${{ parameters.additionalBuildFlags }}' workingDirectory: '$(Build.BinariesDirectory)' - task: VSBuild@1 @@ -154,13 +196,20 @@ jobs: solution: '$(Build.BinariesDirectory)\${{ parameters.BuildConfig }}\onnxruntime.sln' platform: ${{ parameters.msbuildPlatform }} configuration: ${{ parameters.BuildConfig }} - msbuildArgs: -maxcpucount + msbuildArgs: '-maxcpucount ${{ variables.MSBUILD_CACHE_ARG }}' msbuildArchitecture: ${{ parameters.buildArch }} maximumCpuCount: true logProjectEvents: false workingFolder: '$(Build.BinariesDirectory)\${{ parameters.BuildConfig }}' createLogFile: true + - powershell: | + ccache -sv + ccache -z + displayName: cache stat + condition: eq(${{ parameters.WITH_CACHE }}, true) + + - ${{ if eq(parameters.EnablePython, true) }}: - task: PythonScript@0 displayName: 'Build wheel' @@ -274,7 +323,7 @@ jobs: displayName: 'Generate documentation' inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: '--config ${{ parameters.BuildConfig }} --build_dir $(Build.BinariesDirectory) --gen_doc validate' + arguments: '--config ${{ parameters.BuildConfig }} --build_dir $(Build.BinariesDirectory) --gen_doc validate ${{ variables.PY_CACHE_ARG }}' workingDirectory: '$(Build.BinariesDirectory)' # if the validation from --gen_doc failed it sets DocUpdateNeeded so we can publish the latest version of the docs diff --git a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml index e8d1ba8ee1..c646cb42c7 100644 --- a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml @@ -22,6 +22,7 @@ stages: isTraining: false ORT_EP_NAME: CPU GenerateDocumentation: false + WITH_CACHE: true MachinePool: 'Win-CPU-2019' - stage: x64_release @@ -43,6 +44,7 @@ stages: isTraining: false ORT_EP_NAME: CPU GenerateDocumentation: false + WITH_CACHE: true MachinePool: 'Win-CPU-2019' - stage: x64_release_dnnl @@ -62,6 +64,7 @@ stages: isTraining: false ORT_EP_NAME: DNNL GenerateDocumentation: false + WITH_CACHE: true MachinePool: 'Win-CPU-2019' - stage: x64_release_xnnpack @@ -81,6 +84,7 @@ stages: isTraining: false ORT_EP_NAME: XNNPACK GenerateDocumentation: false + WITH_CACHE: true MachinePool: 'Win-CPU-2019' - stage: x64_release_winml @@ -102,6 +106,7 @@ stages: isTraining: false ORT_EP_NAME: CPU GenerateDocumentation: false + WITH_CACHE: true MachinePool: 'Win-CPU-2019' - stage: x86_release @@ -121,6 +126,7 @@ stages: isTraining: false ORT_EP_NAME: CPU GenerateDocumentation: false + WITH_CACHE: true MachinePool: 'Win-CPU-2019' - stage: training_x64_debug @@ -140,6 +146,7 @@ stages: isTraining: true ORT_EP_NAME: CPU GenerateDocumentation: false + WITH_CACHE: true MachinePool: 'onnxruntime-Win2019-CPU-training' - stage: training_x64_release @@ -159,6 +166,7 @@ stages: isTraining: true ORT_EP_NAME: CPU GenerateDocumentation: false + WITH_CACHE: true MachinePool: 'onnxruntime-Win2019-CPU-training' - stage: ort_training_apis_x64_release @@ -179,8 +187,10 @@ stages: isTraining: true ORT_EP_NAME: CPU GenerateDocumentation: false + WITH_CACHE: true MachinePool: 'onnxruntime-Win2019-CPU-training' + - stage: x64_release_cloud dependsOn: [] jobs: @@ -199,4 +209,5 @@ stages: isTraining: false ORT_EP_NAME: CPU GenerateDocumentation: false + WITH_CACHE: true MachinePool: 'Win-CPU-2019' diff --git a/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml index d00cd91fbb..9a8954012c 100644 --- a/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml @@ -43,6 +43,7 @@ stages: RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }} RunStaticCodeAnalysis: false ORT_EP_NAME: CUDA + WITH_CACHE: true MachinePool: onnxruntime-Win2019-GPU-T4 - stage: training @@ -60,6 +61,7 @@ stages: RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }} RunStaticCodeAnalysis: false ORT_EP_NAME: CUDA + WITH_CACHE: true MachinePool: onnxruntime-Win2019-GPU-training-T4 isTraining: true @@ -78,6 +80,7 @@ stages: RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }} RunStaticCodeAnalysis: false ORT_EP_NAME: DML + WITH_CACHE: true # DirectML cannot run on T4 GPUs. MachinePool: onnxruntime-Win2019-GPU-dml @@ -98,4 +101,5 @@ stages: RunStaticCodeAnalysis: false GenerateDocumentation: true ORT_EP_NAME: CUDA # It doesn't really matter which EP is selected here since this stage is for documentation. + WITH_CACHE: true MachinePool: onnxruntime-Win2019-GPU-T4 diff --git a/tools/ci_build/github/windows/helpers.ps1 b/tools/ci_build/github/windows/helpers.ps1 index 95cc67f0e8..cf65d645a7 100644 --- a/tools/ci_build/github/windows/helpers.ps1 +++ b/tools/ci_build/github/windows/helpers.ps1 @@ -392,6 +392,9 @@ function Install-Protobuf { exit $exitCode } $cmake_args = "--build", ".", "--parallel", "--config", $build_config, "--target", "INSTALL" + if ($use_cache) { + $cmake_args += "--", "/p:CLToolExe=cl.exe /p:CLToolPath=C:\ProgramData\chocolatey\bin /p:TrackFileAccess=false /p:UseMultiToolTask=true" + } $p = Start-Process -FilePath $cmake_path -ArgumentList $cmake_args -NoNewWindow -Wait -PassThru $exitCode = $p.ExitCode if ($exitCode -ne 0) { @@ -458,4 +461,4 @@ function Install-ONNX { } Write-Host "Finished installing onnx" popd -} \ No newline at end of file +} diff --git a/tools/ci_build/github/windows/install_third_party_deps.ps1 b/tools/ci_build/github/windows/install_third_party_deps.ps1 index d0177f1c68..bd3158f547 100644 --- a/tools/ci_build/github/windows/install_third_party_deps.ps1 +++ b/tools/ci_build/github/windows/install_third_party_deps.ps1 @@ -7,7 +7,8 @@ param ( [string]$cpu_arch = "x64", [string]$build_config = "RelWithDebInfo", - [string]$install_prefix = "." + [string]$install_prefix = ".", + [switch]$use_cache ) . "$PSScriptRoot\helpers.ps1" @@ -27,15 +28,23 @@ New-Item -Path "$install_prefix" -ItemType Directory -Force $compile_flags = '/MP /guard:cf /Qspectre /DWIN32 /D_WINDOWS /DWINVER=0x0601 /D_WIN32_WINNT=0x0601 /DNTDDI_VERSION=0x06010000 /W3 ' $linker_flags=@('/guard:cf') +if ($use_cache) { + $debug_info_format = "/Z7" +} +else { + $debug_info_format = "/Zi" +} + if($build_config -eq 'Release'){ $compile_flags += "/O2", "/Ob2", "/DNDEBUG", "/Gw", "/GL" } elseif($build_config -eq 'RelWithDebInfo'){ - $compile_flags += "/Zi", "/O2", "/Ob1", "/DNDEBUG", "/Gw", "/GL" + $compile_flags += "$debug_info_format", "/O2", "/Ob1", "/DNDEBUG", "/Gw", "/GL" } elseif($build_config -eq 'Debug'){ - $compile_flags += "/Zi", "/Ob0", "/Od", "/RTC1" + $compile_flags += "$debug_info_format", "/Ob0", "/Od", "/RTC1" } elseif($build_config -eq 'MinSizeRel'){ $compile_flags += "/O1", "/Ob1", "/DNDEBUG", "/Gw", "/GL" } +Write-Host $compile_flags # cmake args that applies to every 3rd-party library [string[]]$cmake_extra_args="-DCMAKE_CXX_STANDARD=17 `"-DCMAKE_CXX_FLAGS=$compile_flags /EHsc`" ", "`"-DCMAKE_C_FLAGS=$compile_flags`"", "--compile-no-warning-as-error", "--fresh", "-Wno-dev" if($cpu_arch -eq 'x86'){ @@ -47,11 +56,19 @@ if($cpu_arch -eq 'x86'){ throw "$cpu_arch is not supported" } +if ($use_cache) { + if ($build_config -eq 'RelWithDebInfo') { + $cmake_extra_args += "-DCMAKE_CXX_FLAGS_RELWITHDEBINFO=`"/MD /Z7 /O2 /Ob1 /DNDEBUG`"" + } + elseif ($build_config -eq 'Debug') { + $cmake_extra_args += "-DCMAKE_CXX_FLAGS_DEBUG=`"/MDd /Z7 /Ob0 /Od /RTC1`"" + } +} $cmake_extra_args += "-DCMAKE_EXE_LINKER_FLAGS=`"$linker_flags`"" # Find the full path of cmake.exe -$cmake_command = Get-Command -CommandType Application cmake +$cmake_command = (Get-Command -CommandType Application cmake)[0] $cmake_path = $cmake_command.Path Install-Pybind -cmake_path $cmake_path -src_root $ort_src_root -build_config $build_config -cmake_extra_args $cmake_extra_args @@ -62,4 +79,4 @@ Install-Protobuf -cmake_path $cmake_path -src_root $ort_src_root -build_config $ $protobuf_version="3.18.3" # ONNX doesn't allow us to specify CMake's path -Install-ONNX -build_config $build_config -src_root $ort_src_root -protobuf_version $protobuf_version \ No newline at end of file +Install-ONNX -build_config $build_config -src_root $ort_src_root -protobuf_version $protobuf_version