mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-05-23 22:13:38 +00:00
338e6672dd
### Description Use build.sourceversion in docker image cache key. ### Motivation and Context We used filpath as the cache key in #14496. In most cases, the docker base image tag is latest. So, the hash of the files couldn't be aware of the change of base image. As the result, the docker image restored, but the image will still be rebuilt . The maintenance cost would be huge if we pin image hash in docker file. For example, https://quay.io/repository/pypa/manylinux2014_x86_64?tab=tags&tag=latest, it's updated almost every week. So far, the build.sourceversion is the right way to keep cache is updated and valid.
125 lines
4.3 KiB
YAML
125 lines
4.3 KiB
YAML
# calls tools/ci_build/get_docker_image.py
|
|
|
|
parameters:
|
|
- name: Dockerfile
|
|
type: string
|
|
- name: Context
|
|
type: string
|
|
- name: DockerBuildArgs
|
|
type: string
|
|
default: ""
|
|
- name: Repository
|
|
type: string
|
|
- name: UseImageCacheContainerRegistry
|
|
type: boolean
|
|
default: true
|
|
- name: UsePipelineCache
|
|
type: boolean
|
|
default: false
|
|
- name: ScriptName
|
|
type: string
|
|
default: "tools/ci_build/get_docker_image.py"
|
|
- name: UpdateDepsTxt
|
|
type: boolean
|
|
default: true
|
|
- name: IMAGE_CACHE_DIR
|
|
type: string
|
|
default: $(Agent.TempDirectory)/docker
|
|
|
|
steps:
|
|
|
|
- ${{ if eq(parameters.UpdateDepsTxt, true)}}:
|
|
- template: download-deps.yml
|
|
|
|
- ${{ if contains(parameters.Dockerfile, 'manylinux') }}:
|
|
- checkout: manylinux
|
|
- script: |
|
|
set -e -x
|
|
mv manylinux onnxruntime
|
|
mv onnxruntime ..
|
|
cd ..
|
|
rmdir $(Build.SourcesDirectory)
|
|
mv onnxruntime $(Build.SourcesDirectory)
|
|
displayName: "Move Manylinux source code to ORT folder"
|
|
# It makes the files in context keep consistent before and after docker build step.
|
|
- task: PythonScript@0
|
|
inputs:
|
|
scriptPath: $(Build.SourcesDirectory)/tools/ci_build/patch_manylinux.py
|
|
arguments: --dockerfile "${{ parameters.Dockerfile }}" --context "${{ parameters.Context }}"
|
|
pythonInterpreter: /usr/bin/python3
|
|
displayName: patch manylinux
|
|
|
|
- script: |
|
|
docker image ls
|
|
docker system df
|
|
displayName: Check Docker Images
|
|
|
|
- ${{ if eq(parameters.UseImageCacheContainerRegistry, true) }}:
|
|
- template: with-container-registry-steps.yml
|
|
parameters:
|
|
Steps:
|
|
- script: |
|
|
${{ parameters.ScriptName }} \
|
|
--dockerfile "${{ parameters.Dockerfile }}" \
|
|
--context "${{ parameters.Context }}" \
|
|
--docker-build-args "${{ parameters.DockerBuildArgs }}" \
|
|
--container-registry onnxruntimebuildcache \
|
|
--repository "${{ parameters.Repository }}"
|
|
displayName: "Get ${{ parameters.Repository }} image for ${{ parameters.Dockerfile }}"
|
|
ContainerRegistry: onnxruntimebuildcache
|
|
- ${{ if eq(parameters.UseImageCacheContainerRegistry, false) }}:
|
|
- task: Cache@2
|
|
displayName: Cache Docker Image Task
|
|
inputs:
|
|
key: ' "${{ parameters.Repository }}" | "$(Build.SourceVersion)" '
|
|
path: ${{ parameters.IMAGE_CACHE_DIR }}
|
|
restoreKeys: |
|
|
"${{ parameters.Repository }}" | "$(Build.SourceVersion)"
|
|
"${{ parameters.Repository }}"
|
|
cacheHitVar: CACHE_RESTORED
|
|
condition: eq('${{ parameters.UsePipelineCache }}', 'true')
|
|
|
|
- script: |
|
|
test -f ${{ parameters.IMAGE_CACHE_DIR }}/cache.tar && docker load -i ${{ parameters.IMAGE_CACHE_DIR }}/cache.tar
|
|
docker image ls
|
|
displayName: Docker restore
|
|
condition: eq('${{ parameters.UsePipelineCache }}', 'true')
|
|
|
|
- script: |
|
|
if [ ${{ parameters.UsePipelineCache}} ]
|
|
then
|
|
use_imagecache="--use_imagecache"
|
|
else
|
|
use_imagecache=""
|
|
fi
|
|
${{ parameters.ScriptName }} \
|
|
--dockerfile "${{ parameters.Dockerfile }}" \
|
|
--context "${{ parameters.Context }}" \
|
|
--docker-build-args "${{ parameters.DockerBuildArgs }}" \
|
|
--repository "${{ parameters.Repository }}" \
|
|
$use_imagecache
|
|
displayName: "Get ${{ parameters.Repository }} image for ${{ parameters.Dockerfile }}"
|
|
|
|
- script: |
|
|
set -ex
|
|
mkdir -p "${{ parameters.IMAGE_CACHE_DIR }}"
|
|
docker save -o "${{ parameters.IMAGE_CACHE_DIR }}/cache.tar" ${{ parameters.Repository }}
|
|
docker image ls
|
|
docker system df
|
|
displayName: Docker save
|
|
condition: eq('${{ parameters.UsePipelineCache }}', 'true')
|
|
|
|
- script: |
|
|
echo ${{ parameters.IMAGE_CACHE_DIR }}
|
|
ls -lah ${{ parameters.IMAGE_CACHE_DIR }}
|
|
displayName: Display docker dir
|
|
condition: eq('${{ parameters.UsePipelineCache }}', 'true')
|
|
|
|
- ${{ if and(eq(parameters.UpdateDepsTxt, true), or(eq(variables['System.CollectionId'], 'f3ad12f2-e480-4533-baf2-635c95467d29'),eq(variables['System.CollectionId'], 'bc038106-a83b-4dab-9dd3-5a41bc58f34c'))) }}:
|
|
- task: PythonScript@0
|
|
displayName: 'Update deps.txt'
|
|
inputs:
|
|
scriptPath: $(Build.SourcesDirectory)/tools/ci_build/replace_urls_in_deps.py
|
|
arguments: --new_dir /build/deps
|
|
workingDirectory: $(Build.BinariesDirectory)
|
|
pythonInterpreter: /usr/bin/python3
|