From ca315b9148274b8bfcd6721f147658d230fa280c Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Sat, 11 Mar 2023 10:32:02 +0800 Subject: [PATCH] Use ADO cache to cache docker image instead of ACR (#14496) ### Description Now, we only enable image cache in pipeline cache for Linux Aten Pipeline. It'll be enabled in other Linux pipelines gradually. ### Motivation and Context Fixed [AB#13143](https://aiinfra.visualstudio.com/6a833879-cd9b-44a4-a9de-adc2d818f13c/_workitems/edit/13143) ### Verification 1. No Image Cache in Pipeline https://dev.azure.com/onnxruntime/onnxruntime/_build/results?buildId=904531&view=results 2. Use Cached Image in Pipeline https://dev.azure.com/onnxruntime/onnxruntime/_build/results?buildId=904533&view=results --- tools/ci_build/get_docker_image.py | 23 +++++++- .../linux-cpu-aten-pipeline.yml | 11 +++- .../explicitly-defined-final-tasks.yml | 2 +- .../templates/get-docker-image-steps.yml | 54 ++++++++++++++++++- 4 files changed, 86 insertions(+), 4 deletions(-) diff --git a/tools/ci_build/get_docker_image.py b/tools/ci_build/get_docker_image.py index 7efeb92828..bbf4c26f5e 100755 --- a/tools/ci_build/get_docker_image.py +++ b/tools/ci_build/get_docker_image.py @@ -36,7 +36,9 @@ def parse_args(): "--container-registry", help="The Azure container registry name. " "If not provided, no container registry will be used.", ) - parser.add_argument("--repository", required=True, help="The image repository name.") + parser.add_argument("--repository", help="The image repository name.") + + parser.add_argument("--use_imagecache", action="store_true", help="use cached image in pipeline cache") parser.add_argument("--docker-path", default="docker", help="Path to docker.") @@ -93,6 +95,25 @@ def main(): args.dockerfile, args.context, ) + elif args.use_imagecache: + log.info("Building image with pipeline cache...") + run( + args.docker_path, + "--log-level", + "error", + "buildx", + "build", + "--tag", + full_image_name, + "--cache-from", + full_image_name, + "--build-arg", + "BUILDKIT_INLINE_CACHE=1", + *shlex.split(args.docker_build_args), + "-f", + args.dockerfile, + args.context, + ) else: log.info("Building image...") run( diff --git a/tools/ci_build/github/azure-pipelines/linux-cpu-aten-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-cpu-aten-pipeline.yml index 1a331e567c..1a345a8bf0 100644 --- a/tools/ci_build/github/azure-pipelines/linux-cpu-aten-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-cpu-aten-pipeline.yml @@ -33,7 +33,12 @@ jobs: Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_aten_cpu Context: tools/ci_build/github/linux/docker DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )" - Repository: onnxruntimecpubuildaten + Repository: 'onnxruntimecpubuildaten' + UseImageCacheContainerRegistry: false + UsePipelineCache: true + DockerCacheKeys: 'tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_aten_cpu, tools/ci_build/github/linux/docker/manylinux*, + tools/ci_build/github/linux/docker/scripts/**/*, tools/ci_build/github/linux/docker/build_scripts/**/*, + !tools/ci_build/github/linux/docker/scripts/deps.txt' - task: Cache@2 inputs: @@ -113,3 +118,7 @@ jobs: workingDirectory: $(Build.SourcesDirectory) - template: templates/explicitly-defined-final-tasks.yml + + - script: | + df -h + displayName: check disk space diff --git a/tools/ci_build/github/azure-pipelines/templates/explicitly-defined-final-tasks.yml b/tools/ci_build/github/azure-pipelines/templates/explicitly-defined-final-tasks.yml index f2dc9492bf..4f18d5e3ec 100644 --- a/tools/ci_build/github/azure-pipelines/templates/explicitly-defined-final-tasks.yml +++ b/tools/ci_build/github/azure-pipelines/templates/explicitly-defined-final-tasks.yml @@ -16,7 +16,7 @@ steps: parameters : condition : 'succeeded' -- script: docker image prune -f +- script: docker system df && docker system prune -a -f && docker system df displayName: Clean docker images condition: eq(variables['Agent.OS'], 'Linux') continueOnError: true diff --git a/tools/ci_build/github/azure-pipelines/templates/get-docker-image-steps.yml b/tools/ci_build/github/azure-pipelines/templates/get-docker-image-steps.yml index 5c5786a3f9..f312af599d 100644 --- a/tools/ci_build/github/azure-pipelines/templates/get-docker-image-steps.yml +++ b/tools/ci_build/github/azure-pipelines/templates/get-docker-image-steps.yml @@ -13,12 +13,21 @@ parameters: - name: UseImageCacheContainerRegistry type: boolean default: true +- name: UsePipelineCache + type: boolean + default: false - name: ScriptName type: string default: "tools/ci_build/get_docker_image.py" - name: UpdateDepsTxt type: boolean default: true +- name: IMAGE_CACHE_DIR + type: string + default: $(Agent.TempDirectory)/docker +- name: DockerCacheKeys + type: string + default: 'tools/ci_build/github/linux/docker/**/*, !tools/ci_build/github/linux/docker/scripts/deps.txt' steps: @@ -43,6 +52,11 @@ steps: pythonInterpreter: /usr/bin/python3 displayName: patch manylinux +- script: | + docker image ls + docker system df + displayName: Check Docker Images + - ${{ if eq(parameters.UseImageCacheContainerRegistry, true) }}: - template: with-container-registry-steps.yml parameters: @@ -57,14 +71,52 @@ steps: displayName: "Get ${{ parameters.Repository }} image for ${{ parameters.Dockerfile }}" ContainerRegistry: onnxruntimebuildcache - ${{ if eq(parameters.UseImageCacheContainerRegistry, false) }}: + - task: Cache@2 + displayName: Cache Docker Image Task + inputs: + key: ' "${{ parameters.Repository }}" | ${{ parameters.DockerCacheKeys }} ' + path: ${{ parameters.IMAGE_CACHE_DIR }} + restoreKeys: | + "${{ parameters.Repository }}" | ${{ parameters.DockerCacheKeys }} + cacheHitVar: CACHE_RESTORED + condition: eq('${{ parameters.UsePipelineCache }}', 'true') + - script: | + test -f ${{ parameters.IMAGE_CACHE_DIR }}/cache.tar && docker load -i ${{ parameters.IMAGE_CACHE_DIR }}/cache.tar + docker image ls + displayName: Docker restore + condition: eq('${{ parameters.UsePipelineCache }}', 'true') + + - script: | + if [ ${{ parameters.UsePipelineCache}} ] + then + use_imagecache="--use_imagecache" + else + use_imagecache="" + fi ${{ parameters.ScriptName }} \ --dockerfile "${{ parameters.Dockerfile }}" \ --context "${{ parameters.Context }}" \ --docker-build-args "${{ parameters.DockerBuildArgs }}" \ - --repository "${{ parameters.Repository }}" + --repository "${{ parameters.Repository }}" \ + $use_imagecache displayName: "Get ${{ parameters.Repository }} image for ${{ parameters.Dockerfile }}" + - script: | + set -ex + mkdir -p "${{ parameters.IMAGE_CACHE_DIR }}" + docker save -o "${{ parameters.IMAGE_CACHE_DIR }}/cache.tar" ${{ parameters.Repository }} + docker image ls + docker system df + displayName: Docker save + condition: eq('${{ parameters.UsePipelineCache }}', 'true') + + - script: | + echo ${{ parameters.IMAGE_CACHE_DIR }} + ls -lah ${{ parameters.IMAGE_CACHE_DIR }} + displayName: Display docker dir + condition: eq('${{ parameters.UsePipelineCache }}', 'true') + - ${{ if and(eq(parameters.UpdateDepsTxt, true), or(eq(variables['System.CollectionId'], 'f3ad12f2-e480-4533-baf2-635c95467d29'),eq(variables['System.CollectionId'], 'bc038106-a83b-4dab-9dd3-5a41bc58f34c'))) }}: - task: PythonScript@0 displayName: 'Update deps.txt'