Add compilation cache for Linux GPU (#13995)

### Description
<!-- Describe your changes. -->



### Motivation and Context
<!-- - Why is this change required? What problem does it solve?
- If it fixes an open issue, please link to the issue here. -->

tools/ci_build/build.py

@@ -965,6 +965,8 @@ def generate_build_tree(
     if args.use_cache:
         cmake_args.append("-DCMAKE_CXX_COMPILER_LAUNCHER=ccache")
         cmake_args.append("-DCMAKE_C_COMPILER_LAUNCHER=ccache")
+        if args.use_cuda:
+            cmake_args.append("-DCMAKE_C_COMPILER_LAUNCHER=ccache")
     # By default cmake does not check TLS/SSL certificates. Here we turn it on.
     # But, in some cases you may also need to supply a CA file.
     add_default_definition(cmake_extra_defines, "CMAKE_TLS_VERIFY", "ON")

tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml

@@ -40,19 +40,14 @@ jobs:
       DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )"
       Repository: onnxruntimecpubuild
 
-  - script: |
-      sudo apt-get install ccache -y
-      echo "##vso[task.prependpath]/usr/lib/ccache"
-    displayName: Install ccache and update PATH to use linked versions of gcc, cc, etc
-
   - task: Cache@2
     inputs:
       key: '"ccache" | "$(Build.SourceBranch)" | "$(Build.SourceVersion)"'
       path: $(CCACHE_DIR)
+      cacheHitVar: CACHE_RESTORED
       restoreKeys: |
         "ccache" | "$(Build.SourceBranch)"
         "ccache"
-      cacheHitVar: CACHE_RESTORED
     displayName: Cach Task
 
   - script: |

tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml

@@ -11,10 +11,15 @@ jobs:
   timeoutInMinutes: 120
   variables:
     skipComponentGovernanceDetection: true
+    CCACHE_DIR: $(Pipeline.Workspace)/ccache
   workspace:
     clean: all
   pool: Linux-CPU-2019
   steps:
+  - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
+    displayName: 'Clean Agent Directories'
+    condition: always()
+
   - checkout: self
     clean: true
     submodules: none
@@ -26,6 +31,21 @@ jobs:
       DockerBuildArgs: "--network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64 --build-arg BASEIMAGE=nvidia/cuda:11.6.2-cudnn8-devel-centos7 --build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-11/root --build-arg PREPEND_PATH=/opt/rh/devtoolset-11/root/usr/bin: --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-11/root/usr/lib64:/opt/rh/devtoolset-11/root/usr/lib:/opt/rh/devtoolset-11/root/usr/lib64/dyninst:/opt/rh/devtoolset-11/root/usr/lib/dyninst:/usr/local/lib64 --build-arg BUILD_UID=$( id -u )"
       Repository: onnxruntimecuda11build
 
+  - task: Cache@2
+    inputs:
+      key: '"ccache" | "$(Build.SourceBranch)" | "$(Build.SourceVersion)"'
+      path: $(CCACHE_DIR)
+      restoreKeys: |
+        "ccache" | "$(Build.SourceBranch)"
+        "ccache"
+      cacheHitVar: CACHE_RESTORED
+    displayName: Cach Task
+
+  - script: |
+      sudo mkdir -p $(Pipeline.Workspace)/ccache
+    condition: ne(variables.CACHE_RESTORED, 'true')
+    displayName: Create Cache Dir
+
   - task: CmdLine@2
     inputs:
       script: |
@@ -36,10 +56,16 @@ jobs:
           --volume $(Build.BinariesDirectory):/build \
           --volume /data/models:/build/models:ro \
           --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
+          --volume $(Pipeline.Workspace)/ccache:/cache \
           -e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \
           -e NIGHTLY_BUILD \
           -e BUILD_BUILDNUMBER \
+          -e CCACHE_DIR=/cache \
           onnxruntimecuda11build \
+          /bin/bash -c "
+            set -ex; \
+            env; \
+            ccache -s; \
             /opt/python/cp38-cp38/bin/python3 /onnxruntime_src/tools/ci_build/build.py \
               --build_dir /build --cmake_generator Ninja \
               --config Release --update --build \
@@ -50,7 +76,9 @@ jobs:
               --enable_onnx_tests --use_cuda --cuda_version=11.6 --cuda_home=/usr/local/cuda-11.6 --cudnn_home=/usr/local/cuda-11.6 \
               --enable_cuda_profiling \
               --enable_pybind --build_java \
-              --cmake_extra_defines CMAKE_CUDA_HOST_COMPILER=/opt/rh/devtoolset-11/root/usr/bin/cc  CMAKE_CUDA_ARCHITECTURES=75
+              --use_cache \
+              --cmake_extra_defines CMAKE_CUDA_HOST_COMPILER=/opt/rh/devtoolset-11/root/usr/bin/cc  CMAKE_CUDA_ARCHITECTURES=75; \
+            ccache -s"
       workingDirectory: $(Build.SourcesDirectory)
 
   - task: CmdLine@2
@@ -68,15 +96,7 @@ jobs:
       artifactName: 'drop-linux'
       targetPath: '$(Build.BinariesDirectory)/Release'
 
-  - task: PublishTestResults@2
-    displayName: 'Publish unit test results'
-    inputs:
-      testResultsFiles: '**/*.results.xml'
-      searchFolder: '$(Build.BinariesDirectory)'
-      testRunTitle: 'Unit Test Run'
-    condition: succeededOrFailed()
-
-  - template: templates/clean-agent-build-directory-step.yml
+  - template: templates/explicitly-defined-final-tasks.yml
 
 - job: Linux_Test
   timeoutInMinutes: 180

tools/ci_build/github/azure-pipelines/templates/explicitly-defined-final-tasks.yml

@@ -0,0 +1,22 @@
+# It's used to replace clean-agent-build-directory-step.yml
+# mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 should be
+# set as the first step of the job in case there's conflict with other task which
+# defines Post-Job.
+
+steps:
+- task: PublishTestResults@2
+  displayName: 'Publish unit test results'
+  inputs:
+    testResultsFiles: '**/*.results.xml'
+    searchFolder: '$(Build.BinariesDirectory)'
+    testRunTitle: 'Unit Test Run'
+  condition: succeededOrFailed()
+
+- template: component-governance-component-detection-steps.yml
+  parameters :
+    condition : 'succeeded'
+
+- script: docker image prune -f
+  displayName: Clean docker images
+  condition: eq(variables['Agent.OS'], 'Linux')
+  continueOnError: true