mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-05-15 20:50:42 +00:00
71e7c2b423
This PR adds infrastructure to automatically cache docker images used in CI builds in a container registry. Currently, build images are pulled from a container registry for some builds and built every time for others. The container registry requires maintenance to keep the images up to date and building images every time wastes build agent resources. With this change, a given build image can be looked up in a cache container registry and if present, pulled, and otherwise, built and pushed. The uniqueness of a build image is determined by a hash digest of the dockerfile, docker build context directory, and certain "docker build" options. This digest is part of the image tag in the cache container repository. The cache container registry will need to be cleaned up periodically. This is not automated yet.
84 lines
2.8 KiB
Python
84 lines
2.8 KiB
Python
# Copyright (c) Microsoft Corporation. All rights reserved.
|
|
# Licensed under the MIT License.
|
|
|
|
import contextlib
|
|
import logging
|
|
import os
|
|
import platform
|
|
import re
|
|
import shutil
|
|
import stat
|
|
import subprocess
|
|
import tempfile
|
|
import urllib.parse
|
|
import urllib.request
|
|
|
|
AZCOPY_VERSION = "10.4.3"
|
|
|
|
# See here for instructions on getting stable download links:
|
|
# https://docs.microsoft.com/en-us/azure/storage/common/storage-use-azcopy-v10#obtain-a-static-download-link
|
|
_AZCOPY_DOWNLOAD_URLS = {
|
|
"Linux": "https://azcopyvnext.azureedge.net/release20200501/azcopy_linux_amd64_10.4.3.tar.gz",
|
|
"Darwin": "https://azcopyvnext.azureedge.net/release20200501/azcopy_darwin_amd64_10.4.3.zip",
|
|
"Windows": "https://azcopyvnext.azureedge.net/release20200501/azcopy_windows_amd64_10.4.3.zip",
|
|
}
|
|
|
|
_log = logging.getLogger("util.get_azcopy")
|
|
|
|
|
|
def _check_version(azcopy_path):
|
|
proc = subprocess.run(
|
|
[azcopy_path, "--version"],
|
|
stdout=subprocess.PIPE, universal_newlines=True)
|
|
match = re.search(r"\d+(?:\.\d+)+", proc.stdout)
|
|
|
|
if not match:
|
|
raise RuntimeError("Failed to determine azcopy version.")
|
|
|
|
return match.group(0) == AZCOPY_VERSION
|
|
|
|
|
|
def _find_azcopy(start_dir):
|
|
for root, _, file_names in os.walk(start_dir):
|
|
for file_name in file_names:
|
|
if file_name == "azcopy" or file_name == "azcopy.exe":
|
|
return os.path.join(root, file_name)
|
|
raise RuntimeError("Failed to azcopy in '{}'.".format(start_dir))
|
|
|
|
|
|
@contextlib.contextmanager
|
|
def get_azcopy(local_azcopy_path="azcopy"):
|
|
"""
|
|
Creates a context manager that returns a path to a particular version of
|
|
azcopy (specified in AZCOPY_VERSION). Downloads a temporary copy if needed.
|
|
|
|
:param local_azcopy_path: Path to a local azcopy to try first.
|
|
|
|
Example usage:
|
|
with get_azcopy() as azcopy_path:
|
|
subprocess.run([azcopy_path, "--version"])
|
|
"""
|
|
with contextlib.ExitStack() as context_stack:
|
|
azcopy_path = shutil.which(local_azcopy_path)
|
|
|
|
if azcopy_path is None or not _check_version(azcopy_path):
|
|
temp_dir = context_stack.enter_context(
|
|
tempfile.TemporaryDirectory())
|
|
|
|
download_url = _AZCOPY_DOWNLOAD_URLS[platform.system()]
|
|
download_basename = urllib.parse.urlsplit(
|
|
download_url).path.rsplit("/", 1)[-1]
|
|
assert len(download_basename) > 0
|
|
downloaded_path = os.path.join(temp_dir, download_basename)
|
|
|
|
_log.info("Downloading azcopy from '{}'...".format(download_url))
|
|
urllib.request.urlretrieve(download_url, downloaded_path)
|
|
|
|
extracted_path = os.path.join(temp_dir, "azcopy")
|
|
shutil.unpack_archive(downloaded_path, extracted_path)
|
|
|
|
azcopy_path = _find_azcopy(extracted_path)
|
|
|
|
os.chmod(azcopy_path, stat.S_IXUSR)
|
|
|
|
yield azcopy_path
|