Use ADO cache to cache docker image instead of ACR (#14496)

### Description
Now, we only enable image cache in pipeline cache for Linux Aten
Pipeline.
It'll be enabled in other Linux pipelines gradually.

### Motivation and Context
<!-- - Why is this change required? What problem does it solve?
- If it fixes an open issue, please link to the issue here. -->

Fixed
[AB#13143](https://aiinfra.visualstudio.com/6a833879-cd9b-44a4-a9de-adc2d818f13c/_workitems/edit/13143)


### Verification
1. No Image Cache in Pipeline

https://dev.azure.com/onnxruntime/onnxruntime/_build/results?buildId=904531&view=results
2. Use Cached Image in Pipeline

https://dev.azure.com/onnxruntime/onnxruntime/_build/results?buildId=904533&view=results

tools/ci_build/get_docker_image.py

@@ -36,7 +36,9 @@ def parse_args():
         "--container-registry",
         help="The Azure container registry name. " "If not provided, no container registry will be used.",
     )
-    parser.add_argument("--repository", required=True, help="The image repository name.")
+    parser.add_argument("--repository", help="The image repository name.")
+
+    parser.add_argument("--use_imagecache", action="store_true", help="use cached image in pipeline cache")
 
     parser.add_argument("--docker-path", default="docker", help="Path to docker.")
 
@@ -93,6 +95,25 @@ def main():
             args.dockerfile,
             args.context,
         )
+    elif args.use_imagecache:
+        log.info("Building image with pipeline cache...")
+        run(
+            args.docker_path,
+            "--log-level",
+            "error",
+            "buildx",
+            "build",
+            "--tag",
+            full_image_name,
+            "--cache-from",
+            full_image_name,
+            "--build-arg",
+            "BUILDKIT_INLINE_CACHE=1",
+            *shlex.split(args.docker_build_args),
+            "-f",
+            args.dockerfile,
+            args.context,
+        )
     else:
         log.info("Building image...")
         run(

tools/ci_build/github/azure-pipelines/linux-cpu-aten-pipeline.yml

@@ -33,7 +33,12 @@ jobs:
       Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_aten_cpu
       Context: tools/ci_build/github/linux/docker
       DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )"
-      Repository: onnxruntimecpubuildaten
+      Repository: 'onnxruntimecpubuildaten'
+      UseImageCacheContainerRegistry: false
+      UsePipelineCache: true
+      DockerCacheKeys: 'tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_aten_cpu, tools/ci_build/github/linux/docker/manylinux*,
+        tools/ci_build/github/linux/docker/scripts/**/*, tools/ci_build/github/linux/docker/build_scripts/**/*,
+        !tools/ci_build/github/linux/docker/scripts/deps.txt'
 
   - task: Cache@2
     inputs:
@@ -113,3 +118,7 @@ jobs:
       workingDirectory: $(Build.SourcesDirectory)
 
   - template: templates/explicitly-defined-final-tasks.yml
+
+  - script: |
+      df -h
+    displayName: check disk space

tools/ci_build/github/azure-pipelines/templates/explicitly-defined-final-tasks.yml

@@ -16,7 +16,7 @@ steps:
   parameters :
     condition : 'succeeded'
 
-- script: docker image prune -f
+- script: docker system df && docker system prune -a -f && docker system df
   displayName: Clean docker images
   condition: eq(variables['Agent.OS'], 'Linux')
   continueOnError: true

tools/ci_build/github/azure-pipelines/templates/get-docker-image-steps.yml

@@ -13,12 +13,21 @@ parameters:
 - name: UseImageCacheContainerRegistry
   type: boolean
   default: true
+- name: UsePipelineCache
+  type: boolean
+  default: false
 - name: ScriptName
   type: string
   default: "tools/ci_build/get_docker_image.py"
 - name: UpdateDepsTxt
   type: boolean
   default: true
+- name: IMAGE_CACHE_DIR
+  type: string
+  default: $(Agent.TempDirectory)/docker
+- name: DockerCacheKeys
+  type: string
+  default: 'tools/ci_build/github/linux/docker/**/*, !tools/ci_build/github/linux/docker/scripts/deps.txt'
 
 steps:
 
@@ -43,6 +52,11 @@ steps:
         pythonInterpreter: /usr/bin/python3
       displayName: patch manylinux
 
+- script: |
+    docker image ls
+    docker system df
+  displayName: Check Docker Images
+
 - ${{ if eq(parameters.UseImageCacheContainerRegistry, true) }}:
   - template: with-container-registry-steps.yml
     parameters:
@@ -57,14 +71,52 @@ steps:
         displayName: "Get ${{ parameters.Repository }} image for ${{ parameters.Dockerfile }}"
       ContainerRegistry: onnxruntimebuildcache
 - ${{ if eq(parameters.UseImageCacheContainerRegistry, false) }}:
+  - task: Cache@2
+    displayName: Cache Docker Image Task
+    inputs:
+      key: ' "${{ parameters.Repository }}" | ${{ parameters.DockerCacheKeys }} '
+      path: ${{ parameters.IMAGE_CACHE_DIR }}
+      restoreKeys: |
+        "${{ parameters.Repository }}" | ${{ parameters.DockerCacheKeys }}
+      cacheHitVar: CACHE_RESTORED
+    condition: eq('${{ parameters.UsePipelineCache }}', 'true')
+
   - script: |
+      test -f ${{ parameters.IMAGE_CACHE_DIR }}/cache.tar && docker load -i ${{ parameters.IMAGE_CACHE_DIR }}/cache.tar
+      docker image ls
+    displayName: Docker restore
+    condition: eq('${{ parameters.UsePipelineCache }}', 'true')
+
+  - script: |
+      if [ ${{ parameters.UsePipelineCache}} ]
+      then
+        use_imagecache="--use_imagecache"
+      else
+        use_imagecache=""
+      fi
       ${{ parameters.ScriptName }} \
         --dockerfile "${{ parameters.Dockerfile }}" \
         --context "${{ parameters.Context }}" \
         --docker-build-args "${{ parameters.DockerBuildArgs }}" \
-        --repository "${{ parameters.Repository }}"
+        --repository "${{ parameters.Repository }}" \
+        $use_imagecache
     displayName: "Get ${{ parameters.Repository }} image for ${{ parameters.Dockerfile }}"
 
+  - script: |
+      set -ex
+      mkdir -p "${{ parameters.IMAGE_CACHE_DIR }}"
+      docker save -o "${{ parameters.IMAGE_CACHE_DIR }}/cache.tar" ${{ parameters.Repository }}
+      docker image ls
+      docker system df
+    displayName: Docker save
+    condition: eq('${{ parameters.UsePipelineCache }}', 'true')
+
+  - script: |
+      echo ${{ parameters.IMAGE_CACHE_DIR }}
+      ls -lah ${{ parameters.IMAGE_CACHE_DIR }}
+    displayName: Display docker dir
+    condition: eq('${{ parameters.UsePipelineCache }}', 'true')
+
 - ${{ if and(eq(parameters.UpdateDepsTxt, true), or(eq(variables['System.CollectionId'], 'f3ad12f2-e480-4533-baf2-635c95467d29'),eq(variables['System.CollectionId'], 'bc038106-a83b-4dab-9dd3-5a41bc58f34c'))) }}:
   - task: PythonScript@0
     displayName: 'Update deps.txt'