From 4e1f75810c9aa115f922515a4577a9ac0fcf837c Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Fri, 14 Apr 2023 23:56:59 +0800 Subject: [PATCH] Add compilation cache in 2 Linux CPU pipelines and refactor the Linux build step with cache (#15484) ### Description 1. Add compilation cache in Linux CPU ARM and Linux Minimal Build. 2. Integrate 4 Linux CPU build step with cache into one. 3. install ccache from source code in Linux ARM64 image. ### Motivation and Context 1. Enable more build steps with compilation cache. 2. Make it easier to add cache. It could save 40 more minutes of compilation time in Linux ARM64. https://dev.azure.com/onnxruntime/onnxruntime/_build/results?buildId=959619&view=logs&j=1e0830bb-fd74-5d0a-5029-1c63b4266d7b&t=75260ed7-7566-5947-2095-566660191920 --- tools/ci_build/build.py | 3 + .../azure-pipelines/linux-ci-pipeline.yml | 89 +++++++++---------- .../linux-cpu-aten-pipeline.yml | 89 +++++++++---------- .../linux-cpu-minimal-build-ci-pipeline.yml | 51 +++++++---- .../templates/linux-build-step-with-cache.yml | 50 +++++++++++ .../azure-pipelines/templates/py-linux.yml | 38 +++++--- .../linux/build_linux_arm64_python_package.sh | 2 + .../x64/python/cpu/scripts/install_centos.sh | 2 +- .../x64/python/cpu/scripts/install_deps.sh | 18 ++++ .../github/linux/run_python_dockerbuild.sh | 1 + 10 files changed, 222 insertions(+), 121 deletions(-) create mode 100644 tools/ci_build/github/azure-pipelines/templates/linux-build-step-with-cache.yml diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 5afb9c7377..9155098105 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -2271,6 +2271,9 @@ def main(): args = parse_arguments() + if os.getenv("ORT_BUILD_WITH_CACHE") == "1": + args.use_cache = True + if not is_windows(): if not args.allow_running_as_root: is_root_user = os.geteuid() == 0 diff --git a/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml index 049fe13219..98d1bdfdf5 100644 --- a/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml @@ -15,7 +15,7 @@ stages: clean: all variables: skipComponentGovernanceDetection: true - CCACHE_DIR: $(Pipeline.Workspace)/ccache + ORT_CACHE_DIR: $(Agent.TempDirectory)/ort_ccache TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)] pool: onnxruntime-Linux-CPU-2019 steps: @@ -43,50 +43,48 @@ stages: DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )" Repository: onnxruntimecpubuild - - task: Cache@2 - inputs: - key: '"$(TODAY)" | "$(Build.SourceBranch)" | "$(Build.SourceVersion)"' - path: $(CCACHE_DIR) - cacheHitVar: CACHE_RESTORED - restoreKeys: | - "$(TODAY)" | "$(Build.SourceBranch)" - "$(TODAY)" | - displayName: Cach Task - - - task: CmdLine@2 - displayName: 'build' - inputs: - script: | - mkdir -p $HOME/.onnx - mkdir -p $(Pipeline.Workspace)/ccache - docker run --rm \ - --volume /data/onnx:/data/onnx:ro \ - --volume $(Build.SourcesDirectory):/onnxruntime_src \ - --volume $(Build.BinariesDirectory):/build \ - --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \ - --volume $(Pipeline.Workspace)/ccache:/cache \ - -e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \ - -e NIGHTLY_BUILD \ - -e BUILD_BUILDNUMBER \ - -e CCACHE_DIR=/cache \ - onnxruntimecpubuild \ - /bin/bash -c " - set -ex; \ - ccache -s; \ - /opt/python/cp38-cp38/bin/python3 /onnxruntime_src/tools/ci_build/build.py \ - --build_dir /build --cmake_generator Ninja \ - --config Debug Release \ - --skip_submodule_sync \ - --build_shared_lib \ - --parallel \ - --build_wheel \ - --enable_onnx_tests \ - --enable_transformers_tool_test \ - --use_cache \ - --build_java --build_nodejs --update --build --cmake_extra_defines onnxruntime_BUILD_BENCHMARKS=ON; \ - ccache -sv; \ - ccache -z" - workingDirectory: $(Build.SourcesDirectory) + - template: templates/linux-build-step-with-cache.yml + parameters: + WithCache: true + Today: $(TODAY) + AdditionalKey: onnxruntime + CacheDir: $(ORT_CACHE_DIR) + ChangeEveryCommit: true + BuildStep: + - task: CmdLine@2 + displayName: 'build' + inputs: + script: | + mkdir -p $HOME/.onnx + mkdir -p $(Pipeline.Workspace)/ccache + docker run --rm \ + --volume /data/onnx:/data/onnx:ro \ + --volume $(Build.SourcesDirectory):/onnxruntime_src \ + --volume $(Build.BinariesDirectory):/build \ + --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \ + --volume $(ORT_CACHE_DIR):/cache \ + -e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \ + -e NIGHTLY_BUILD \ + -e BUILD_BUILDNUMBER \ + -e CCACHE_DIR=/cache \ + onnxruntimecpubuild \ + /bin/bash -c " + set -ex; \ + ccache -s; \ + /opt/python/cp38-cp38/bin/python3 /onnxruntime_src/tools/ci_build/build.py \ + --build_dir /build --cmake_generator Ninja \ + --config Debug Release \ + --skip_submodule_sync \ + --build_shared_lib \ + --parallel \ + --build_wheel \ + --enable_onnx_tests \ + --enable_transformers_tool_test \ + --use_cache \ + --build_java --build_nodejs --update --build --cmake_extra_defines onnxruntime_BUILD_BENCHMARKS=ON; \ + ccache -sv; \ + ccache -z" + workingDirectory: $(Build.SourcesDirectory) - task: CmdLine@2 displayName: 'Install python deps and run java tests' @@ -184,6 +182,7 @@ stages: devtoolset_rootpath: /opt/rh/devtoolset-10/root ld_library_path_arg: /opt/rh/devtoolset-10/root/usr/lib64:/opt/rh/devtoolset-10/root/usr/lib:/opt/rh/devtoolset-10/root/usr/lib64/dyninst:/opt/rh/devtoolset-10/root/usr/lib/dyninst:/usr/local/lib64 prepend_path: '/opt/rh/devtoolset-10/root/usr/bin:' + with_cache: true - stage: arm64_test dependsOn: ['arm64_build'] diff --git a/tools/ci_build/github/azure-pipelines/linux-cpu-aten-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-cpu-aten-pipeline.yml index 6aa09da375..5e715b9a61 100644 --- a/tools/ci_build/github/azure-pipelines/linux-cpu-aten-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-cpu-aten-pipeline.yml @@ -12,7 +12,7 @@ jobs: workspace: clean: all variables: - CCACHE_DIR: $(Pipeline.Workspace)/ccache + CCACHE_DIR: $(Agent.TempDirectory)/ccache TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)] pool: onnxruntime-Linux-CPU-2019 steps: @@ -37,53 +37,46 @@ jobs: UseImageCacheContainerRegistry: true UsePipelineCache: false - - task: Cache@2 - inputs: - key: '"$(TODAY)" | "$(Build.SourceBranch)" | "$(Build.SourceVersion)"' - path: $(CCACHE_DIR) - cacheHitVar: CACHE_RESTORED - restoreKeys: | - "$(TODAY)" | "$(Build.SourceBranch)" - "$(TODAY)" | - displayName: Cach Task - - - script: | - sudo mkdir -p $(Pipeline.Workspace)/ccache - condition: ne(variables.CACHE_RESTORED, 'true') - displayName: Create Cache Dir - - - task: CmdLine@2 - displayName: 'build' - inputs: - script: | - mkdir -p $HOME/.onnx - docker run --rm \ - --volume /data/onnx:/data/onnx:ro \ - --volume $(Build.SourcesDirectory):/onnxruntime_src \ - --volume $(Build.BinariesDirectory):/build \ - --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \ - --volume $(Pipeline.Workspace)/ccache:/cache \ - -e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \ - -e NIGHTLY_BUILD \ - -e BUILD_BUILDNUMBER \ - -e CCACHE_DIR=/cache \ - onnxruntimecpubuildaten \ - /bin/bash -c " - set -ex; \ - ccache -s; \ - /opt/python/cp38-cp38/bin/python3 /onnxruntime_src/tools/ci_build/build.py \ - --build_dir /build --cmake_generator Ninja \ - --config Release \ - --skip_submodule_sync \ - --build_shared_lib \ - --parallel \ - --build_wheel \ - --skip_tests \ - --cmake_extra_defines onnxruntime_ENABLE_ATEN=ON \ - --use_cache; \ - ccache -sv; \ - ccache -z" - workingDirectory: $(Build.SourcesDirectory) + - template: templates/linux-build-step-with-cache.yml + parameters: + WithCache: true + Today: $(TODAY) + AdditionalKey: ort_aten + CacheDir: $(CCACHE_DIR) + ChangeEveryCommit: true + BuildStep: + - task: CmdLine@2 + displayName: 'build' + inputs: + script: | + mkdir -p $HOME/.onnx + docker run --rm \ + --volume /data/onnx:/data/onnx:ro \ + --volume $(Build.SourcesDirectory):/onnxruntime_src \ + --volume $(Build.BinariesDirectory):/build \ + --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \ + --volume $(CCACHE_DIR):/cache \ + -e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \ + -e NIGHTLY_BUILD \ + -e BUILD_BUILDNUMBER \ + -e CCACHE_DIR=/cache \ + onnxruntimecpubuildaten \ + /bin/bash -c " + set -ex; \ + ccache -s; \ + /opt/python/cp38-cp38/bin/python3 /onnxruntime_src/tools/ci_build/build.py \ + --build_dir /build --cmake_generator Ninja \ + --config Release \ + --skip_submodule_sync \ + --build_shared_lib \ + --parallel \ + --build_wheel \ + --skip_tests \ + --cmake_extra_defines onnxruntime_ENABLE_ATEN=ON \ + --use_cache; \ + ccache -sv; \ + ccache -z" + workingDirectory: $(Build.SourcesDirectory) - task: CmdLine@2 displayName: 'install ort_torch_ext and launch test' diff --git a/tools/ci_build/github/azure-pipelines/linux-cpu-minimal-build-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-cpu-minimal-build-ci-pipeline.yml index 1f4f1a4980..9765deffea 100644 --- a/tools/ci_build/github/azure-pipelines/linux-cpu-minimal-build-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-cpu-minimal-build-ci-pipeline.yml @@ -36,9 +36,15 @@ jobs: clean: all pool: onnxruntime-Linux-CPU-2019 variables: + ORT_CACHE_DIR: $(Pipeline.Workspace)/ort_ccache + TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)] test_data_directory: $(Build.SourcesDirectory)/.test_data steps: + - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 + displayName: 'Clean Agent Directories' + condition: always() + - checkout: self clean: true submodules: none @@ -60,21 +66,34 @@ jobs: touch $(test_data_directory)/include_no_operators.config workingDirectory: $(Build.SourcesDirectory) - - task: CmdLine@2 - displayName: 1. Build full onnxruntime and generate ORT format test files - inputs: - script: | - docker run --rm \ - --volume $(Build.SourcesDirectory):/onnxruntime_src \ - --volume $(Build.BinariesDirectory):/build \ - --volume $(test_data_directory):/home/onnxruntimedev/.test_data \ - -e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \ - -e NIGHTLY_BUILD \ - -e BUILD_BUILDNUMBER \ - onnxruntimecpubuild \ - /bin/bash /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_full_ort_and_create_ort_files.sh \ - /build/1 - workingDirectory: $(Build.SourcesDirectory) + - template: templates/linux-build-step-with-cache.yml + parameters: + WithCache: true + Today: $(TODAY) + AdditionalKey: onnxruntime__full + CacheDir: $(ORT_CACHE_DIR) + ChangeEveryCommit: true + BuildStep: + - task: CmdLine@2 + displayName: 1. Build full onnxruntime and generate ORT format test files + inputs: + script: | + docker run --rm \ + --volume $(Build.SourcesDirectory):/onnxruntime_src \ + --volume $(Build.BinariesDirectory):/build \ + --volume $(test_data_directory):/home/onnxruntimedev/.test_data \ + --volume $(ORT_CACHE_DIR):/cache \ + -e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \ + -e NIGHTLY_BUILD \ + -e BUILD_BUILDNUMBER \ + -e CCACHE_DIR=/cache \ + -e ORT_BUILD_WITH_CACHE=1 \ + onnxruntimecpubuild \ + /bin/bash -c " + /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_full_ort_and_create_ort_files.sh /build/1; \ + ccache -sv; \ + ccache -z;" + workingDirectory: $(Build.SourcesDirectory) - task: CmdLine@2 displayName: 2. Build minimal onnxruntime [exceptions DISABLED, type reduction DISABLED, training ops ENABLED] @@ -300,4 +319,4 @@ jobs: --skip_tests --path_to_protoc_exe /usr/bin/protoc workingDirectory: $(Build.SourcesDirectory) - - template: templates/clean-agent-build-directory-step.yml + - template: templates/explicitly-defined-final-tasks.yml diff --git a/tools/ci_build/github/azure-pipelines/templates/linux-build-step-with-cache.yml b/tools/ci_build/github/azure-pipelines/templates/linux-build-step-with-cache.yml new file mode 100644 index 0000000000..4acaa0d027 --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/templates/linux-build-step-with-cache.yml @@ -0,0 +1,50 @@ +parameters: +- name: WithCache + displayName: Build with Cache + type: boolean + default: false + +- name: BuildStep + type: stepList + +- name: Today + type: string + default: "" + +# It'd be different in different jobs in one workflow. +- name: AdditionalKey + type: string + default: "" + +- name: ChangeEveryCommit + type: boolean + default: true + +- name: CacheDir + type: string + default: "" + +steps: + - ${{if eq(parameters.WithCache, true)}}: + - script: | + mkdir -p ${{ parameters.CacheDir }} + displayName: Create Cache Dir + + - ${{if eq(parameters.WithCache, true)}}: + - task: Cache@2 + inputs: + ${{ if eq(parameters.ChangeEveryCommit, true) }}: + ${{if eq(variables['Build.SourceBranchName'], 'merge')}}: + key: ' "${{parameters.Today}}" | merge | ${{parameters.AdditionalKey}} ' + ${{else}}: + key: ' "${{parameters.Today}}" | ${{parameters.AdditionalKey}} | "$(Build.SourceVersion)" ' + restoreKeys: | + "${{parameters.Today}}" | ${{parameters.AdditionalKey}} + ${{ else }}: + key: ' ${{parameters.AdditionalKey}} ' + restoreKeys: | + ${{parameters.AdditionalKey}} + path: ${{ parameters.CacheDir }} + displayName: ccache task + + - ${{ parameters.BuildStep }} diff --git a/tools/ci_build/github/azure-pipelines/templates/py-linux.yml b/tools/ci_build/github/azure-pipelines/templates/py-linux.yml index a42e005a3f..3645b0730f 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-linux.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-linux.yml @@ -21,6 +21,10 @@ parameters: type: string default: '-d CPU' +- name: with_cache + type: boolean + default: false + jobs: - job: Linux_py_Wheels_${{ parameters.arch }} timeoutInMinutes: 240 @@ -30,7 +34,13 @@ jobs: variables: # The build machine pool doesn't have dotnet, so it can't run CG. skipComponentGovernanceDetection: true + ORT_CACHE_DIR: $(Agent.TempDirectory)/ort_ccache + TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)] steps: + - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 + displayName: 'Clean Agent Directories' + condition: always() + - checkout: self clean: true submodules: none @@ -46,12 +56,23 @@ jobs: ${{ if eq(parameters.arch, 'aarch64') }}: UpdateDepsTxt: false - - task: Bash@3 - displayName: 'Build Python Wheel' - inputs: - targetType: filePath - filePath: tools/ci_build/github/linux/run_python_dockerbuild.sh - arguments: -i onnxruntimecpubuilpython${{ parameters.arch }} -x "${{ parameters.device }}" + - template: linux-build-step-with-cache.yml + parameters: + WithCache: ${{parameters.with_cache}} + Today: $(TODAY) + AdditionalKey: Linux_py_Wheels_${{ parameters.arch }} + CacheDir: $(ORT_CACHE_DIR) + ChangeEveryCommit: true + BuildStep: + - task: Bash@3 + displayName: 'Build Python Wheel' + inputs: + targetType: filePath + filePath: tools/ci_build/github/linux/run_python_dockerbuild.sh + arguments: -i onnxruntimecpubuilpython${{ parameters.arch }} -x "${{ parameters.device }}" + ${{ if eq(parameters.with_cache, 'true') }}: + env: + ADDITIONAL_DOCKER_PARAMETER: "--volume $(ORT_CACHE_DIR):/cache -e CCACHE_DIR=/cache -e ORT_BUILD_WITH_CACHE=1" - task: PublishBuildArtifacts@1 displayName: 'Publish Artifact: ONNXRuntime python wheel' @@ -64,8 +85,3 @@ jobs: inputs: artifactName: 'drop-linux-cpu-${{ parameters.arch }}' targetPath: '$(Build.BinariesDirectory)/Release' - - - - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 - displayName: 'Clean Agent Directories' - condition: always() diff --git a/tools/ci_build/github/linux/build_linux_arm64_python_package.sh b/tools/ci_build/github/linux/build_linux_arm64_python_package.sh index 14b72dcfc7..9af2f40a42 100755 --- a/tools/ci_build/github/linux/build_linux_arm64_python_package.sh +++ b/tools/ci_build/github/linux/build_linux_arm64_python_package.sh @@ -64,3 +64,5 @@ do cp /build/$BUILD_CONFIG/dist/*.whl /build/dist done + +which ccache && ccache -sv && ccache -z diff --git a/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_centos.sh b/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_centos.sh index ab262a614c..db4b617744 100755 --- a/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_centos.sh +++ b/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_centos.sh @@ -4,4 +4,4 @@ set -e os_major_version=$(cat /etc/redhat-release | tr -dc '0-9.'|cut -d \. -f1) echo "installing for os major version : $os_major_version" -yum install -y which gdb redhat-lsb-core expat-devel tar unzip zlib-devel make libunwind bzip2 bzip2-devel \ No newline at end of file +yum install -y which gdb redhat-lsb-core expat-devel tar unzip zlib-devel make libunwind bzip2 bzip2-devel diff --git a/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_deps.sh b/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_deps.sh index ec37598d39..7bf031ee78 100755 --- a/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_deps.sh +++ b/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_deps.sh @@ -15,4 +15,22 @@ do ${PYTHON_EXE} -m pip install -r requirements.txt done +# No release binary for ccache aarch64, so we need to build it from source. +if ! [ -x "$(command -v ccache)" ]; then + ccache_url="https://github.com/ccache/ccache/archive/refs/tags/v4.8.tar.gz" + pushd . + curl -sSL --retry 5 --retry-delay 10 --create-dirs --fail -L -o ccache_src.tar.gz $ccache_url + mkdir ccache_main + cd ccache_main + tar -zxf ../ccache_src.tar.gz --strip=1 + mkdir build + cd build + cmake -DCMAKE_INSTALL_PREFIX=/usr/local _DCMAKE_BUILD_TYPE=Release .. + make + make install + which ccache + popd + rm -f ccache_src.tar.gz + rm -rf ccache_src +fi diff --git a/tools/ci_build/github/linux/run_python_dockerbuild.sh b/tools/ci_build/github/linux/run_python_dockerbuild.sh index 14b86bf4e7..a9c406a551 100755 --- a/tools/ci_build/github/linux/run_python_dockerbuild.sh +++ b/tools/ci_build/github/linux/run_python_dockerbuild.sh @@ -18,6 +18,7 @@ docker run --rm \ -w /onnxruntime_src \ -e NIGHTLY_BUILD \ -e BUILD_BUILDNUMBER \ + $ADDITIONAL_DOCKER_PARAMETER \ $DOCKER_IMAGE tools/ci_build/github/linux/build_linux_arm64_python_package.sh $BUILD_EXTR_PAR sudo rm -rf $BUILD_BINARIESDIRECTORY/Release/onnxruntime $BUILD_BINARIESDIRECTORY/Release/pybind11 \