From 338e6672dd5f9a42f94bde96ec2fe970f142432f Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Fri, 24 Mar 2023 10:01:22 +0800 Subject: [PATCH] use build.sourceversion in cache image key (#15019) ### Description Use build.sourceversion in docker image cache key. ### Motivation and Context We used filpath as the cache key in #14496. In most cases, the docker base image tag is latest. So, the hash of the files couldn't be aware of the change of base image. As the result, the docker image restored, but the image will still be rebuilt . The maintenance cost would be huge if we pin image hash in docker file. For example, https://quay.io/repository/pypa/manylinux2014_x86_64?tab=tags&tag=latest, it's updated almost every week. So far, the build.sourceversion is the right way to keep cache is updated and valid. --- tools/ci_build/get_docker_image.py | 2 +- .../github/azure-pipelines/linux-cpu-aten-pipeline.yml | 3 --- .../azure-pipelines/templates/get-docker-image-steps.yml | 8 +++----- 3 files changed, 4 insertions(+), 9 deletions(-) diff --git a/tools/ci_build/get_docker_image.py b/tools/ci_build/get_docker_image.py index 3040b873c0..3639e58a1d 100755 --- a/tools/ci_build/get_docker_image.py +++ b/tools/ci_build/get_docker_image.py @@ -36,7 +36,7 @@ def parse_args(): "--container-registry", help="The Azure container registry name. " "If not provided, no container registry will be used.", ) - parser.add_argument("--repository", help="The image repository name.") + parser.add_argument("--repository", required=True, help="The image repository name.") parser.add_argument("--use_imagecache", action="store_true", help="use cached image in pipeline cache") diff --git a/tools/ci_build/github/azure-pipelines/linux-cpu-aten-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-cpu-aten-pipeline.yml index 1a345a8bf0..b4a92280dd 100644 --- a/tools/ci_build/github/azure-pipelines/linux-cpu-aten-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-cpu-aten-pipeline.yml @@ -36,9 +36,6 @@ jobs: Repository: 'onnxruntimecpubuildaten' UseImageCacheContainerRegistry: false UsePipelineCache: true - DockerCacheKeys: 'tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_aten_cpu, tools/ci_build/github/linux/docker/manylinux*, - tools/ci_build/github/linux/docker/scripts/**/*, tools/ci_build/github/linux/docker/build_scripts/**/*, - !tools/ci_build/github/linux/docker/scripts/deps.txt' - task: Cache@2 inputs: diff --git a/tools/ci_build/github/azure-pipelines/templates/get-docker-image-steps.yml b/tools/ci_build/github/azure-pipelines/templates/get-docker-image-steps.yml index f312af599d..e4f467da45 100644 --- a/tools/ci_build/github/azure-pipelines/templates/get-docker-image-steps.yml +++ b/tools/ci_build/github/azure-pipelines/templates/get-docker-image-steps.yml @@ -25,9 +25,6 @@ parameters: - name: IMAGE_CACHE_DIR type: string default: $(Agent.TempDirectory)/docker -- name: DockerCacheKeys - type: string - default: 'tools/ci_build/github/linux/docker/**/*, !tools/ci_build/github/linux/docker/scripts/deps.txt' steps: @@ -74,10 +71,11 @@ steps: - task: Cache@2 displayName: Cache Docker Image Task inputs: - key: ' "${{ parameters.Repository }}" | ${{ parameters.DockerCacheKeys }} ' + key: ' "${{ parameters.Repository }}" | "$(Build.SourceVersion)" ' path: ${{ parameters.IMAGE_CACHE_DIR }} restoreKeys: | - "${{ parameters.Repository }}" | ${{ parameters.DockerCacheKeys }} + "${{ parameters.Repository }}" | "$(Build.SourceVersion)" + "${{ parameters.Repository }}" cacheHitVar: CACHE_RESTORED condition: eq('${{ parameters.UsePipelineCache }}', 'true')