From aa9fbed3d4172f9d2c3c7c61ca1b8372b0cdaa86 Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Fri, 16 Dec 2022 16:38:12 +0800 Subject: [PATCH] Add compilation cache for Linux GPU (#13995) ### Description ### Motivation and Context --- tools/ci_build/build.py | 2 + .../azure-pipelines/linux-ci-pipeline.yml | 7 +--- .../azure-pipelines/linux-gpu-ci-pipeline.yml | 40 ++++++++++++++----- .../explicitly-defined-final-tasks.yml | 22 ++++++++++ 4 files changed, 55 insertions(+), 16 deletions(-) create mode 100644 tools/ci_build/github/azure-pipelines/templates/explicitly-defined-final-tasks.yml diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index e125479fbb..b021603727 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -965,6 +965,8 @@ def generate_build_tree( if args.use_cache: cmake_args.append("-DCMAKE_CXX_COMPILER_LAUNCHER=ccache") cmake_args.append("-DCMAKE_C_COMPILER_LAUNCHER=ccache") + if args.use_cuda: + cmake_args.append("-DCMAKE_C_COMPILER_LAUNCHER=ccache") # By default cmake does not check TLS/SSL certificates. Here we turn it on. # But, in some cases you may also need to supply a CA file. add_default_definition(cmake_extra_defines, "CMAKE_TLS_VERIFY", "ON") diff --git a/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml index 1f6d3b8605..1096b0e80e 100644 --- a/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml @@ -40,19 +40,14 @@ jobs: DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )" Repository: onnxruntimecpubuild - - script: | - sudo apt-get install ccache -y - echo "##vso[task.prependpath]/usr/lib/ccache" - displayName: Install ccache and update PATH to use linked versions of gcc, cc, etc - - task: Cache@2 inputs: key: '"ccache" | "$(Build.SourceBranch)" | "$(Build.SourceVersion)"' path: $(CCACHE_DIR) + cacheHitVar: CACHE_RESTORED restoreKeys: | "ccache" | "$(Build.SourceBranch)" "ccache" - cacheHitVar: CACHE_RESTORED displayName: Cach Task - script: | diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml index 04618b5881..6652eb6f40 100644 --- a/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml @@ -11,10 +11,15 @@ jobs: timeoutInMinutes: 120 variables: skipComponentGovernanceDetection: true + CCACHE_DIR: $(Pipeline.Workspace)/ccache workspace: clean: all pool: Linux-CPU-2019 steps: + - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 + displayName: 'Clean Agent Directories' + condition: always() + - checkout: self clean: true submodules: none @@ -26,6 +31,21 @@ jobs: DockerBuildArgs: "--network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64 --build-arg BASEIMAGE=nvidia/cuda:11.6.2-cudnn8-devel-centos7 --build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-11/root --build-arg PREPEND_PATH=/opt/rh/devtoolset-11/root/usr/bin: --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-11/root/usr/lib64:/opt/rh/devtoolset-11/root/usr/lib:/opt/rh/devtoolset-11/root/usr/lib64/dyninst:/opt/rh/devtoolset-11/root/usr/lib/dyninst:/usr/local/lib64 --build-arg BUILD_UID=$( id -u )" Repository: onnxruntimecuda11build + - task: Cache@2 + inputs: + key: '"ccache" | "$(Build.SourceBranch)" | "$(Build.SourceVersion)"' + path: $(CCACHE_DIR) + restoreKeys: | + "ccache" | "$(Build.SourceBranch)" + "ccache" + cacheHitVar: CACHE_RESTORED + displayName: Cach Task + + - script: | + sudo mkdir -p $(Pipeline.Workspace)/ccache + condition: ne(variables.CACHE_RESTORED, 'true') + displayName: Create Cache Dir + - task: CmdLine@2 inputs: script: | @@ -36,10 +56,16 @@ jobs: --volume $(Build.BinariesDirectory):/build \ --volume /data/models:/build/models:ro \ --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \ + --volume $(Pipeline.Workspace)/ccache:/cache \ -e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \ -e NIGHTLY_BUILD \ -e BUILD_BUILDNUMBER \ + -e CCACHE_DIR=/cache \ onnxruntimecuda11build \ + /bin/bash -c " + set -ex; \ + env; \ + ccache -s; \ /opt/python/cp38-cp38/bin/python3 /onnxruntime_src/tools/ci_build/build.py \ --build_dir /build --cmake_generator Ninja \ --config Release --update --build \ @@ -50,7 +76,9 @@ jobs: --enable_onnx_tests --use_cuda --cuda_version=11.6 --cuda_home=/usr/local/cuda-11.6 --cudnn_home=/usr/local/cuda-11.6 \ --enable_cuda_profiling \ --enable_pybind --build_java \ - --cmake_extra_defines CMAKE_CUDA_HOST_COMPILER=/opt/rh/devtoolset-11/root/usr/bin/cc CMAKE_CUDA_ARCHITECTURES=75 + --use_cache \ + --cmake_extra_defines CMAKE_CUDA_HOST_COMPILER=/opt/rh/devtoolset-11/root/usr/bin/cc CMAKE_CUDA_ARCHITECTURES=75; \ + ccache -s" workingDirectory: $(Build.SourcesDirectory) - task: CmdLine@2 @@ -68,15 +96,7 @@ jobs: artifactName: 'drop-linux' targetPath: '$(Build.BinariesDirectory)/Release' - - task: PublishTestResults@2 - displayName: 'Publish unit test results' - inputs: - testResultsFiles: '**/*.results.xml' - searchFolder: '$(Build.BinariesDirectory)' - testRunTitle: 'Unit Test Run' - condition: succeededOrFailed() - - - template: templates/clean-agent-build-directory-step.yml + - template: templates/explicitly-defined-final-tasks.yml - job: Linux_Test timeoutInMinutes: 180 diff --git a/tools/ci_build/github/azure-pipelines/templates/explicitly-defined-final-tasks.yml b/tools/ci_build/github/azure-pipelines/templates/explicitly-defined-final-tasks.yml new file mode 100644 index 0000000000..f2dc9492bf --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/templates/explicitly-defined-final-tasks.yml @@ -0,0 +1,22 @@ +# It's used to replace clean-agent-build-directory-step.yml +# mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 should be +# set as the first step of the job in case there's conflict with other task which +# defines Post-Job. + +steps: +- task: PublishTestResults@2 + displayName: 'Publish unit test results' + inputs: + testResultsFiles: '**/*.results.xml' + searchFolder: '$(Build.BinariesDirectory)' + testRunTitle: 'Unit Test Run' + condition: succeededOrFailed() + +- template: component-governance-component-detection-steps.yml + parameters : + condition : 'succeeded' + +- script: docker image prune -f + displayName: Clean docker images + condition: eq(variables['Agent.OS'], 'Linux') + continueOnError: true