diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 5435d9593f..4d3e8161a2 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -983,6 +983,91 @@ def number_of_nvcc_threads(args): return nvcc_threads +# See https://learn.microsoft.com/en-us/vcpkg/commands/install +def generate_vcpkg_install_options(source_dir, args): + vcpkg_install_options = ["--x-feature=tests"] + if args.use_acl: + vcpkg_install_options.append("--x-feature=acl-ep") + if args.use_armnn: + vcpkg_install_options.append("--x-feature=armnn-ep") + if args.use_azure: + vcpkg_install_options.append("--x-feature=azure-ep") + if args.use_cann: + vcpkg_install_options.append("--x-feature=cann-ep") + if args.use_coreml: + vcpkg_install_options.append("--x-feature=coreml-ep") + if args.use_cuda: + vcpkg_install_options.append("--x-feature=cuda-ep") + if args.use_dml: + vcpkg_install_options.append("--x-feature=dml-ep") + if args.use_dnnl: + vcpkg_install_options.append("--x-feature=dnnl-ep") + if args.use_jsep: + vcpkg_install_options.append("--x-feature=js-ep") + if args.use_migraphx: + vcpkg_install_options.append("--x-feature=migraphx-ep") + if args.use_nnapi: + vcpkg_install_options.append("--x-feature=nnapi-ep") + if args.use_openvino: + vcpkg_install_options.append("--x-feature=openvino-ep") + if args.use_qnn: + vcpkg_install_options.append("--x-feature=qnn-ep") + if args.use_rknpu: + vcpkg_install_options.append("--x-feature=rknpu-ep") + if args.use_rocm: + vcpkg_install_options.append("--x-feature=rocm-ep") + if args.use_tensorrt: + vcpkg_install_options.append("--x-feature=tensorrt-ep") + if args.use_vitisai: + vcpkg_install_options.append("--x-feature=vitisai-ep") + if args.use_vsinpu: + vcpkg_install_options.append("--x-feature=vsinpu-ep") + if args.use_webgpu: + vcpkg_install_options.append("--x-feature=webgpu-ep") + if args.use_webnn: + vcpkg_install_options.append("--x-feature=webnn-ep") + if args.use_xnnpack: + vcpkg_install_options.append("--x-feature=xnnpack-ep") + + overlay_triplets_dir = None + + folder_name_parts = [] + if args.enable_address_sanitizer: + folder_name_parts.append("asan") + if args.use_binskim_compliant_compile_flags and not args.android: + folder_name_parts.append("binskim") + if args.disable_rtti: + folder_name_parts.append("nortti") + if args.disable_exceptions and not is_windows(): + folder_name_parts.append("noexception") + if len(folder_name_parts) == 0: + folder_name = "default" + else: + folder_name = "_".join(folder_name_parts) + overlay_triplets_dir = os.path.join(source_dir, "cmake", "vcpkg-triplets", folder_name) + + vcpkg_install_options.append(f"--overlay-triplets={overlay_triplets_dir}") + if "AGENT_TEMPDIRECTORY" in os.environ: + temp_dir = os.environ["AGENT_TEMPDIRECTORY"] + vcpkg_install_options.append(f"--x-buildtrees-root={temp_dir}") + terrapin_cmd_path = shutil.which("TerrapinRetrievalTool") + if terrapin_cmd_path is None: + terrapin_cmd_path = "C:\\local\\Terrapin\\TerrapinRetrievalTool.exe" + if not os.path.exists(terrapin_cmd_path): + terrapin_cmd_path = None + if terrapin_cmd_path is not None: + vcpkg_install_options.append( + "--x-asset-sources=x-script," + + terrapin_cmd_path + + " -b https://vcpkg.storage.devpackages.microsoft.io/artifacts/ -a true -u Environment -p {url} -s {sha512} -d {dst}\\;x-block-origin" + ) + else: + SYSTEM_COLLECTIONURI = os.getenv("SYSTEM_COLLECTIONURI") # noqa: N806 + if SYSTEM_COLLECTIONURI == "https://dev.azure.com/onnxruntime/" or SYSTEM_COLLECTIONURI == "https://dev.azure.com/aiinfra/" or SYSTEM_COLLECTIONURI == "https://aiinfra.visualstudio.com/": + vcpkg_install_options.append( + "--x-asset-sources=x-azurl,https://vcpkg.storage.devpackages.microsoft.io/artifacts/\\;x-block-origin" + ) + return vcpkg_install_options def generate_build_tree( cmake_path, @@ -1135,10 +1220,7 @@ def generate_build_tree( if args.use_vcpkg: # TODO: set VCPKG_PLATFORM_TOOLSET_VERSION # Setup CMake flags for vcpkg - vcpkg_install_options = ["--x-feature=tests"] - if args.use_xnnpack: - vcpkg_install_options.append("--x-feature=xnnpack-ep") - + # Find VCPKG's toolchain cmake file vcpkg_cmd_path = shutil.which("vcpkg") vcpkg_toolchain_path = None @@ -1160,42 +1242,8 @@ def generate_build_tree( run_subprocess(["git", "clone", "https://github.com/microsoft/vcpkg.git", "--recursive"], cwd=build_dir) vcpkg_toolchain_path = Path(vcpkg_installation_root) / "scripts" / "buildsystems" / "vcpkg.cmake" add_default_definition(cmake_extra_defines, "CMAKE_TOOLCHAIN_FILE", str(vcpkg_toolchain_path)) - overlay_triplets_dir = None - - folder_name_parts = [] - if args.enable_address_sanitizer: - folder_name_parts.append("asan") - if args.use_binskim_compliant_compile_flags and not args.android: - folder_name_parts.append("binskim") - if args.disable_rtti: - folder_name_parts.append("nortti") - if args.disable_exceptions and not is_windows(): - folder_name_parts.append("noexception") - if len(folder_name_parts) == 0: - folder_name = "default" - else: - folder_name = "_".join(folder_name_parts) - overlay_triplets_dir = os.path.join(source_dir, "cmake", "vcpkg-triplets", folder_name) - - vcpkg_install_options.append(f"--overlay-triplets={overlay_triplets_dir}") - if "AGENT_TEMPDIRECTORY" in os.environ: - temp_dir = os.environ["AGENT_TEMPDIRECTORY"] - vcpkg_install_options.append(f"--x-buildtrees-root={temp_dir}") - terrapin_cmd_path = shutil.which("TerrapinRetrievalTool") - if terrapin_cmd_path is None: - terrapin_cmd_path = "C:\\local\\Terrapin\\TerrapinRetrievalTool.exe" - if not os.path.exists(terrapin_cmd_path): - terrapin_cmd_path = None - if terrapin_cmd_path is not None: - vcpkg_install_options.append( - "--x-asset-sources=x-script," - + terrapin_cmd_path - + " -b https://vcpkg.storage.devpackages.microsoft.io/artifacts/ -a true -u Environment -p {url} -s {sha512} -d {dst}\\;x-block-origin" - ) - else: - vcpkg_install_options.append( - "--x-asset-sources=x-azurl,https://vcpkg.storage.devpackages.microsoft.io/artifacts/\\;x-block-origin" - ) + + vcpkg_install_options = generate_vcpkg_install_options(source_dir, args) # VCPKG_INSTALL_OPTIONS is a CMake list. It must be joined by semicolons # Therefore, if any of the option string contains a semicolon, it must be escaped add_default_definition(cmake_extra_defines, "VCPKG_INSTALL_OPTIONS", ";".join(vcpkg_install_options)) diff --git a/tools/ci_build/get_docker_image.py b/tools/ci_build/get_docker_image.py index e656cedae5..861a5081a3 100755 --- a/tools/ci_build/get_docker_image.py +++ b/tools/ci_build/get_docker_image.py @@ -42,14 +42,6 @@ def parse_args(): parser.add_argument("--docker-path", default="docker", help="Path to docker.") - parser.add_argument("--manylinux-src", default="manylinux", help="Path to manylinux src folder") - - parser.add_argument( - "--multiple_repos", - action="store_true", - help="used in packaging pipeline, which couldn't use get-docker-images-steps.yml", - ) - return parser.parse_args() @@ -78,24 +70,6 @@ def main(): log.info(f"Copy deps.txt to : {dst_deps_file}") shutil.copyfile(Path(REPO_DIR) / "cmake" / "deps.txt", str(dst_deps_file)) - if "manylinux" in args.dockerfile and args.multiple_repos: - manylinux_build_scripts_folder = Path(args.manylinux_src) / "docker" / "build_scripts" - dest = Path(args.context) / "build_scripts" - if dest.exists(): - log.info(f"Deleting: {dest!s}") - shutil.rmtree(str(dest)) - shutil.copytree(str(manylinux_build_scripts_folder), str(dest)) - src_entrypoint_file = str(Path(args.manylinux_src) / "docker" / "manylinux-entrypoint") - dst_entrypoint_file = str(Path(args.context) / "manylinux-entrypoint") - shutil.copyfile(src_entrypoint_file, dst_entrypoint_file) - shutil.copymode(src_entrypoint_file, dst_entrypoint_file) - run( - "patch", - "-p1", - "-i", - str((Path(SCRIPT_DIR) / "github" / "linux" / "docker" / "manylinux.patch").resolve()), - cwd=str(dest), - ) if use_container_registry: run( @@ -115,11 +89,15 @@ def main(): args.dockerfile, args.context, ) - run( - args.docker_path, - "push", - full_image_name, - ) + SYSTEM_COLLECTIONURI = os.getenv("SYSTEM_COLLECTIONURI") # noqa: N806 + BUILD_SOURCEBRANCH = os.getenv("BUILD_SOURCEBRANCH") # noqa: N806 + if BUILD_SOURCEBRANCH == "refs/heads/main" and (SYSTEM_COLLECTIONURI == "https://dev.azure.com/onnxruntime/" or SYSTEM_COLLECTIONURI == "https://dev.azure.com/aiinfra/" or + SYSTEM_COLLECTIONURI == "https://aiinfra.visualstudio.com/"): + run( + args.docker_path, + "push", + full_image_name, + ) else: log.info("Building image...") run( diff --git a/tools/ci_build/github/azure-pipelines/bigmodels-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/bigmodels-ci-pipeline.yml index 78e1c53bf8..f12d17fcc0 100644 --- a/tools/ci_build/github/azure-pipelines/bigmodels-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/bigmodels-ci-pipeline.yml @@ -81,7 +81,7 @@ stages: inputs: script: | mkdir -p $HOME/.onnx - docker run -e --rm \ + docker run -e SYSTEM_COLLECTIONURI -e --rm \ --volume /data/onnx:/data/onnx:ro \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ --volume $(Build.BinariesDirectory):/build \ @@ -178,7 +178,7 @@ stages: - script: | mkdir -p $(GenerateImage_DIR) - docker run --rm --gpus all -v $PWD:/workspace \ + docker run -e SYSTEM_COLLECTIONURI --rm --gpus all -v $PWD:/workspace \ -v $(Build.BinariesDirectory)/Release:/Release \ -v $(STABLE_DIFFUSION_MODEL_CACHE):/model_cache:rw \ -v $(GenerateImage_DIR):/images:rw \ @@ -219,7 +219,7 @@ stages: displayName: Cache clip model - script: | - docker run --rm --gpus all -v $PWD:/workspace \ + docker run -e SYSTEM_COLLECTIONURI --rm --gpus all -v $PWD:/workspace \ -v $(CLIP_MODEL_CACHE):/model_cache:rw \ onnxruntimeubuntupackagestest_cuda11 \ bash -c ' @@ -246,7 +246,7 @@ stages: # The step will execute if the gereneate image doesn't hit another test image - script: | - docker run --rm --gpus all -v $PWD:/workspace \ + docker run -e SYSTEM_COLLECTIONURI --rm --gpus all -v $PWD:/workspace \ -v $(CLIP_MODEL_CACHE):/model_cache:rw \ onnxruntimeubuntupackagestest_cuda11 \ bash -c ' @@ -319,7 +319,7 @@ stages: downloadPath: $(Agent.TempDirectory)/meta_llama2_7b_hf - script: | - docker run --rm --gpus all -v $(Build.SourcesDirectory):/workspace \ + docker run -e SYSTEM_COLLECTIONURI --rm --gpus all -v $(Build.SourcesDirectory):/workspace \ -v $(Build.BinariesDirectory)/ort-artifact/:/ort-artifact \ -v $(Agent.TempDirectory)/meta_llama2_7b_hf:/meta-llama2 \ onnxruntimeubi8packagestest_torch \ @@ -340,7 +340,7 @@ stages: workingDirectory: $(Build.SourcesDirectory) - script: | - docker run --rm --gpus all -v $(Build.SourcesDirectory):/workspace \ + docker run -e SYSTEM_COLLECTIONURI --rm --gpus all -v $(Build.SourcesDirectory):/workspace \ -v $(Build.BinariesDirectory)/ort-artifact/:/ort-artifact \ -v $(Agent.TempDirectory)/meta_llama2_7b_hf:/meta-llama2 \ onnxruntimeubi8packagestest_torch \ @@ -361,7 +361,7 @@ stages: workingDirectory: $(Build.SourcesDirectory) - script: | - docker run --rm --gpus all -v $(Build.SourcesDirectory):/workspace \ + docker run -e SYSTEM_COLLECTIONURI --rm --gpus all -v $(Build.SourcesDirectory):/workspace \ -v $(Build.BinariesDirectory)/ort-artifact/:/ort-artifact \ -v $(Agent.TempDirectory)/meta_llama2_7b_hf:/meta-llama2 \ onnxruntimeubi8packagestest_torch \ @@ -433,7 +433,7 @@ stages: downloadPath: $(Agent.TempDirectory)/whisper_large_v3 - script: | - docker run --rm --gpus all -v $(Build.SourcesDirectory):/workspace \ + docker run -e SYSTEM_COLLECTIONURI --rm --gpus all -v $(Build.SourcesDirectory):/workspace \ -v $(Build.BinariesDirectory)/ort-artifact/:/ort-artifact \ -v $(Agent.TempDirectory)/whisper_large_v3:/whisper_large_v3 \ onnxruntimepackagestest_ompffmpeg \ @@ -454,7 +454,7 @@ stages: workingDirectory: $(Build.SourcesDirectory) - script: | - docker run --rm --gpus all -v $(Build.SourcesDirectory):/workspace \ + docker run -e SYSTEM_COLLECTIONURI --rm --gpus all -v $(Build.SourcesDirectory):/workspace \ -v $(Build.BinariesDirectory)/ort-artifact/:/ort-artifact \ -v $(Agent.TempDirectory)/whisper_large_v3:/whisper_large_v3 \ onnxruntimepackagestest_ompffmpeg \ diff --git a/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml index 979d82de64..9e76481c34 100644 --- a/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml @@ -67,7 +67,7 @@ stages: inputs: script: | mkdir -p $HOME/.onnx - docker run --rm \ + docker run -e SYSTEM_COLLECTIONURI --rm \ --volume /data/onnx:/data/onnx:ro \ --volume /data/models:/data/models:ro \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ @@ -180,7 +180,7 @@ stages: inputs: script: | mkdir -p $HOME/.onnx - docker run --rm \ + docker run -e SYSTEM_COLLECTIONURI --rm \ --volume /data/onnx:/data/onnx:ro \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ --volume $(Build.BinariesDirectory):/build \ @@ -216,7 +216,7 @@ stages: - bash: | mkdir -p $HOME/.onnx - docker run --rm \ + docker run -e SYSTEM_COLLECTIONURI --rm \ --volume /data/onnx:/data/onnx:ro \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ --volume $(Build.BinariesDirectory):/build \ @@ -238,7 +238,7 @@ stages: - bash: | mkdir -p $HOME/.onnx - docker run --rm \ + docker run -e SYSTEM_COLLECTIONURI --rm \ --volume /data/onnx:/data/onnx:ro \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ --volume $(Build.BinariesDirectory):/build \ diff --git a/tools/ci_build/github/azure-pipelines/linux-cpu-minimal-build-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-cpu-minimal-build-ci-pipeline.yml index c323e51035..2451d39a1b 100644 --- a/tools/ci_build/github/azure-pipelines/linux-cpu-minimal-build-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-cpu-minimal-build-ci-pipeline.yml @@ -94,7 +94,7 @@ jobs: displayName: 1. Build full onnxruntime and generate ORT format test files inputs: script: | - docker run --rm \ + docker run -e SYSTEM_COLLECTIONURI --rm \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ --volume $(Build.BinariesDirectory):/build \ --volume $(test_data_directory):/home/onnxruntimedev/.test_data \ @@ -118,7 +118,7 @@ jobs: script: | # We will try to build minimal ORT with exceptions disabled and training ops enabled # Only the building process is verified here, no test will be performed - docker run --rm \ + docker run -e SYSTEM_COLLECTIONURI --rm \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ --volume $(Build.BinariesDirectory):/build \ -e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \ @@ -141,7 +141,7 @@ jobs: displayName: 3a. Build minimal onnxruntime [exceptions ENABLED, type reduction DISABLED, custom ops ENABLED] and run tests inputs: script: | - docker run --rm \ + docker run -e SYSTEM_COLLECTIONURI --rm \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ --volume $(Build.BinariesDirectory):/build \ --volume $(test_data_directory):/home/onnxruntimedev/.test_data \ @@ -159,7 +159,7 @@ jobs: displayName: 3b. Build minimal onnxruntime [exceptions ENABLED, type reduction ENABLED] and run tests inputs: script: | - docker run --rm \ + docker run -e SYSTEM_COLLECTIONURI --rm \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ --volume $(Build.BinariesDirectory):/build \ --volume $(test_data_directory):/home/onnxruntimedev/.test_data \ @@ -181,7 +181,7 @@ jobs: "!globally_allowed_types;bool,float,int8_t,uint8_t" \ "!no_ops_specified_means_all_ops_are_required" \ > $(test_data_directory)/globally_allowed_types.config && \ - docker run --rm \ + docker run -e SYSTEM_COLLECTIONURI --rm \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ --volume $(Build.BinariesDirectory):/build \ --volume $(test_data_directory):/home/onnxruntimedev/.test_data \ @@ -200,7 +200,7 @@ jobs: displayName: 5. Build extended minimal onnxruntime and run tests inputs: script: | - docker run --rm \ + docker run -e SYSTEM_COLLECTIONURI --rm \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ --volume $(Build.BinariesDirectory):/build \ -e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \ @@ -220,7 +220,7 @@ jobs: displayName: 6a. Regular build with python and all optional features disabled. inputs: script: | - docker run --rm \ + docker run -e SYSTEM_COLLECTIONURI --rm \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ --volume $(Build.BinariesDirectory):/build \ --volume $(test_data_directory):/home/onnxruntimedev/.test_data \ @@ -247,7 +247,7 @@ jobs: displayName: 6b. Minimal build with all optional features disabled. inputs: script: | - docker run --rm \ + docker run -e SYSTEM_COLLECTIONURI --rm \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ --volume $(Build.BinariesDirectory):/build \ --volume $(test_data_directory):/home/onnxruntimedev/.test_data \ @@ -276,7 +276,7 @@ jobs: displayName: 6c. Extended minimal build with all optional features disabled. inputs: script: | - docker run --rm \ + docker run -e SYSTEM_COLLECTIONURI --rm \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ --volume $(Build.BinariesDirectory):/build \ --volume $(test_data_directory):/home/onnxruntimedev/.test_data \ @@ -306,7 +306,7 @@ jobs: inputs: script: | NDK_HOME=$(realpath $ANDROID_NDK_HOME) - docker run --rm \ + docker run -e SYSTEM_COLLECTIONURI --rm \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ --volume $(Build.BinariesDirectory):/build \ --volume $ANDROID_HOME:/android_home \ diff --git a/tools/ci_build/github/azure-pipelines/linux-dnnl-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-dnnl-ci-pipeline.yml index 7391f99c2d..725cd35035 100644 --- a/tools/ci_build/github/azure-pipelines/linux-dnnl-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-dnnl-ci-pipeline.yml @@ -53,7 +53,7 @@ jobs: inputs: script: | mkdir -p $HOME/.onnx - docker run --rm \ + docker run -e SYSTEM_COLLECTIONURI --rm \ --volume /data/onnx:/data/onnx:ro \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ --volume $(Build.BinariesDirectory):/build \ diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml index 91c33820ac..1d375106a0 100644 --- a/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-ci-pipeline.yml @@ -102,7 +102,7 @@ stages: - script: | set -e -x mkdir -p $HOME/.onnx - docker run --rm \ + docker run -e SYSTEM_COLLECTIONURI --rm \ --volume /data/onnx:/data/onnx:ro \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ --volume $(Build.BinariesDirectory):/build \ @@ -163,7 +163,7 @@ stages: script: | set -e -x mkdir -p $HOME/.onnx - docker run --gpus all --rm \ + docker run -e SYSTEM_COLLECTIONURI --gpus all --rm \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ --volume $(Build.BinariesDirectory)/Release:/build/Release \ --volume /data/models:/build/models:ro \ @@ -191,7 +191,7 @@ stages: script: | set -e -x mkdir -p $HOME/.onnx - docker run --gpus all --shm-size=1g --ipc=host --ulimit memlock=-1 --ulimit stack=67108864 --rm \ + docker run -e SYSTEM_COLLECTIONURI --gpus all --shm-size=1g --ipc=host --ulimit memlock=-1 --ulimit stack=67108864 --rm \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ --volume $(Build.BinariesDirectory)/Release:/build/Release \ --volume /data/models:/build/models:ro \ diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml index c08eaaaa13..dd5288ab3a 100644 --- a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml @@ -88,7 +88,7 @@ jobs: - task: CmdLine@2 inputs: script: | - docker run --gpus all --rm \ + docker run -e SYSTEM_COLLECTIONURI --gpus all --rm \ --volume /data/onnx:/data/onnx:ro \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ --volume $(Build.BinariesDirectory):/build \ diff --git a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-cuda-minimal-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-cuda-minimal-ci-pipeline.yml index 4a86da167f..ad9d9bfc4b 100644 --- a/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-cuda-minimal-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-cuda-minimal-ci-pipeline.yml @@ -88,7 +88,7 @@ jobs: - task: CmdLine@2 inputs: script: | - docker run --gpus all --rm \ + docker run -e SYSTEM_COLLECTIONURI --gpus all --rm \ --volume /data/onnx:/data/onnx:ro \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ --volume $(Build.BinariesDirectory):/build \ diff --git a/tools/ci_build/github/azure-pipelines/linux-migraphx-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-migraphx-ci-pipeline.yml index c6efd306ed..40f29c6996 100644 --- a/tools/ci_build/github/azure-pipelines/linux-migraphx-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-migraphx-ci-pipeline.yml @@ -81,7 +81,7 @@ jobs: - task: CmdLine@2 inputs: script: | - docker run --rm \ + docker run -e SYSTEM_COLLECTIONURI --rm \ --security-opt seccomp=unconfined \ --shm-size=1024m \ --user $UID:$(id -g $USER) \ @@ -111,7 +111,7 @@ jobs: --update \ --build_dir /build \ --build \ - --parallel \ + --parallel --use_vcpkg \ --build_wheel \ --skip_submodule_sync \ --use_cache \ @@ -166,7 +166,7 @@ jobs: - task: CmdLine@2 inputs: script: | - docker run --rm \ + docker run -e SYSTEM_COLLECTIONURI --rm \ --security-opt seccomp=unconfined \ --shm-size=1024m \ --device=/dev/kfd \ diff --git a/tools/ci_build/github/azure-pipelines/linux-rocm-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-rocm-ci-pipeline.yml index af52d7b883..74576f6d0b 100644 --- a/tools/ci_build/github/azure-pipelines/linux-rocm-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-rocm-ci-pipeline.yml @@ -81,7 +81,7 @@ jobs: - task: CmdLine@2 inputs: script: | - docker run --rm \ + docker run -e SYSTEM_COLLECTIONURI --rm \ --security-opt seccomp=unconfined \ --shm-size=1024m \ --user $UID:$(id -g $USER) \ @@ -167,7 +167,7 @@ jobs: - task: CmdLine@2 inputs: script: | - docker run --rm \ + docker run -e SYSTEM_COLLECTIONURI --rm \ --security-opt seccomp=unconfined \ --shm-size=1024m \ --device=/dev/kfd \ @@ -208,7 +208,7 @@ jobs: - task: CmdLine@2 inputs: script: |- - docker run --rm \ + docker run -e SYSTEM_COLLECTIONURI --rm \ --security-opt seccomp=unconfined \ --shm-size=1024m \ --device=/dev/kfd \ diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml index f9ecfb7cf7..8c6efe2c55 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml @@ -71,7 +71,7 @@ stages: " Repository: onnxruntimepackagestest - bash: | - docker run --rm \ + docker run -e SYSTEM_COLLECTIONURI --rm \ --gpus all \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ --volume $(Build.BinariesDirectory):/build \ diff --git a/tools/ci_build/github/azure-pipelines/stages/java-cuda-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/stages/java-cuda-packaging-stage.yml index b458d3045b..82ac898906 100644 --- a/tools/ci_build/github/azure-pipelines/stages/java-cuda-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/java-cuda-packaging-stage.yml @@ -172,7 +172,7 @@ stages: Repository: onnxruntimeubi8packagestest - bash: | - docker run --rm \ + docker run -e SYSTEM_COLLECTIONURI --rm \ --gpus all \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ --volume $(Build.BinariesDirectory):/build \ diff --git a/tools/ci_build/github/azure-pipelines/stages/nuget-linux-cuda-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/stages/nuget-linux-cuda-packaging-stage.yml index 73728ecabe..6e9e3c80a2 100644 --- a/tools/ci_build/github/azure-pipelines/stages/nuget-linux-cuda-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/nuget-linux-cuda-packaging-stage.yml @@ -213,7 +213,7 @@ stages: displayName: 'Test C API application for GPU package' inputs: script: | - docker run --gpus all -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/models:/data/models --volume $(Build.SourcesDirectory):/src_dir \ + docker run -e SYSTEM_COLLECTIONURI --gpus all -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/models:/data/models --volume $(Build.SourcesDirectory):/src_dir \ --volume $(Build.ArtifactStagingDirectory):/artifact_src -e NIGHTLY_BUILD onnxruntimecuda${{ variables.CUDA_VERSION_MAJOR }}xtrt86build \ /src_dir/onnxruntime-inference-examples/c_cxx/squeezenet/run_capi_application.sh -o /src_dir/onnxruntime -p /artifact_src/onnxruntime-linux-x64-gpu-$(OnnxRuntimeVersion).tgz -w /src_dir/onnxruntime-inference-examples/c_cxx/squeezenet workingDirectory: '$(Build.ArtifactStagingDirectory)' diff --git a/tools/ci_build/github/azure-pipelines/templates/android-binary-size-check-stage.yml b/tools/ci_build/github/azure-pipelines/templates/android-binary-size-check-stage.yml index 3cccd3aee1..06075d3c30 100644 --- a/tools/ci_build/github/azure-pipelines/templates/android-binary-size-check-stage.yml +++ b/tools/ci_build/github/azure-pipelines/templates/android-binary-size-check-stage.yml @@ -77,7 +77,7 @@ stages: BINARY_SIZE_THRESHOLD_ARGS="--threshold_size_in_bytes ${{ parameters.BinarySizeThresholdInBytes }}" fi NDK_HOME=$(realpath $ANDROID_NDK_HOME) - docker run --rm \ + docker run -e SYSTEM_COLLECTIONURI --rm \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ --volume $(Build.BinariesDirectory):/build \ --volume $ANDROID_HOME:/android_home \ @@ -133,7 +133,7 @@ stages: script: | set -e -x NDK_HOME=$(realpath $ANDROID_NDK_HOME) - docker run --rm \ + docker run -e SYSTEM_COLLECTIONURI --rm \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ --volume $(Build.BinariesDirectory):/build \ --volume $ANDROID_HOME:/android_home \ diff --git a/tools/ci_build/github/azure-pipelines/templates/android-java-api-aar.yml b/tools/ci_build/github/azure-pipelines/templates/android-java-api-aar.yml index 15a984d688..cecd6fd912 100644 --- a/tools/ci_build/github/azure-pipelines/templates/android-java-api-aar.yml +++ b/tools/ci_build/github/azure-pipelines/templates/android-java-api-aar.yml @@ -111,7 +111,7 @@ jobs: QNN_VOLUME="" USE_QNN="0" fi - docker run --rm \ + docker run -e SYSTEM_COLLECTIONURI --rm \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ --volume $(Build.BinariesDirectory):/build \ --volume $ANDROID_HOME:/android_home \ diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml index 12d53c1a1b..042b80e582 100644 --- a/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml +++ b/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml @@ -63,7 +63,7 @@ jobs: script: | set -e -x mkdir -p $HOME/.onnx - docker run --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \ + docker run -e SYSTEM_COLLECTIONURI --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \ --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecpubuildcentos8${{parameters.OnnxruntimeArch}}_packaging /bin/bash -c "python3 \ /onnxruntime_src/tools/ci_build/build.py --enable_lto --build_java --build_nodejs --build_dir /build --config Release \ --skip_submodule_sync --parallel --use_vcpkg --use_binskim_compliant_compile_flags --use_vcpkg --build_shared_lib ${{ parameters.AdditionalBuildFlags }} && cd /build/Release && make install DESTDIR=/build/installed" diff --git a/tools/ci_build/github/azure-pipelines/templates/rocm.yml b/tools/ci_build/github/azure-pipelines/templates/rocm.yml index 25517ada74..fb5906ca0a 100644 --- a/tools/ci_build/github/azure-pipelines/templates/rocm.yml +++ b/tools/ci_build/github/azure-pipelines/templates/rocm.yml @@ -65,7 +65,7 @@ jobs: - task: CmdLine@2 inputs: script: | - docker run --rm \ + docker run -e SYSTEM_COLLECTIONURI --rm \ --privileged \ --ipc=host \ --network=host \ diff --git a/tools/ci_build/github/linux/build_cuda_c_api_package.sh b/tools/ci_build/github/linux/build_cuda_c_api_package.sh index cdda6307d8..8c7df02656 100755 --- a/tools/ci_build/github/linux/build_cuda_c_api_package.sh +++ b/tools/ci_build/github/linux/build_cuda_c_api_package.sh @@ -1,5 +1,5 @@ #!/bin/bash set -e -x -docker run --rm --volume \ +docker run -e SYSTEM_COLLECTIONURI --rm --volume \ $BUILD_SOURCESDIRECTORY:/onnxruntime_src --volume $BUILD_BINARIESDIRECTORY:/build -e NIGHTLY_BUILD onnxruntimecuda${CUDA_VERSION_MAJOR}build \ /bin/bash -c "/usr/bin/python3 /onnxruntime_src/tools/ci_build/build.py --enable_lto --build_java --build_nodejs --build_dir /build --config Release --skip_submodule_sync --parallel --use_binskim_compliant_compile_flags --build_shared_lib --use_cuda --cuda_version=$CUDA_VERSION --cuda_home=/usr/local/cuda-$CUDA_VERSION --cudnn_home=/usr/local/cuda-$CUDA_VERSION --skip_tests --use_vcpkg --cmake_extra_defines 'CMAKE_CUDA_ARCHITECTURES=75;80;90' && cd /build/Release && make install DESTDIR=/build/installed" diff --git a/tools/ci_build/github/linux/build_linux_python_package.sh b/tools/ci_build/github/linux/build_linux_python_package.sh index b5999da997..9764383309 100755 --- a/tools/ci_build/github/linux/build_linux_python_package.sh +++ b/tools/ci_build/github/linux/build_linux_python_package.sh @@ -40,7 +40,7 @@ done -BUILD_ARGS=("--build_dir" "/build" "--config" "$BUILD_CONFIG" "--update" "--build" "--skip_submodule_sync" "--parallel" "--use_binskim_compliant_compile_flags" "--build_wheel") +BUILD_ARGS=("--build_dir" "/build" "--config" "$BUILD_CONFIG" "--update" "--build" "--skip_submodule_sync" "--parallel" "--use_binskim_compliant_compile_flags" "--build_wheel" "--use_vcpkg") if [ "$BUILD_CONFIG" != "Debug" ]; then BUILD_ARGS+=("--enable_lto") diff --git a/tools/ci_build/github/linux/build_rocm_c_api_package.sh b/tools/ci_build/github/linux/build_rocm_c_api_package.sh index 9fee565170..3ea90c7334 100755 --- a/tools/ci_build/github/linux/build_rocm_c_api_package.sh +++ b/tools/ci_build/github/linux/build_rocm_c_api_package.sh @@ -20,7 +20,7 @@ done EXIT_CODE=1 -docker run --rm \ +docker run -e SYSTEM_COLLECTIONURI --rm \ --security-opt seccomp=unconfined \ --shm-size=1024m \ --user $UID:$(id -g $USER) \ diff --git a/tools/ci_build/github/linux/build_tensorrt_c_api_package.sh b/tools/ci_build/github/linux/build_tensorrt_c_api_package.sh index 68c782b1a3..b55bd3659c 100755 --- a/tools/ci_build/github/linux/build_tensorrt_c_api_package.sh +++ b/tools/ci_build/github/linux/build_tensorrt_c_api_package.sh @@ -1,6 +1,6 @@ #!/bin/bash set -e -x mkdir -p $HOME/.onnx -docker run --rm --volume /data/onnx:/data/onnx:ro --volume $BUILD_SOURCESDIRECTORY:/onnxruntime_src --volume $BUILD_BINARIESDIRECTORY:/build \ +docker run -e SYSTEM_COLLECTIONURI --rm --volume /data/onnx:/data/onnx:ro --volume $BUILD_SOURCESDIRECTORY:/onnxruntime_src --volume $BUILD_BINARIESDIRECTORY:/build \ --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecuda${CUDA_VERSION_MAJOR}xtrt86build \ /bin/bash -c "/usr/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_tests --skip_submodule_sync --parallel --use_binskim_compliant_compile_flags --build_shared_lib --build_java --build_nodejs --use_tensorrt --cuda_version=$CUDA_VERSION --cuda_home=/usr/local/cuda-$CUDA_VERSION --cudnn_home=/usr --tensorrt_home=/usr --cmake_extra_defines 'CMAKE_CUDA_ARCHITECTURES=75;80;90' --use_vcpkg && cd /build/Release && make install DESTDIR=/build/installed" diff --git a/tools/ci_build/github/linux/build_yocto.sh b/tools/ci_build/github/linux/build_yocto.sh index e948a105c0..f4949f651e 100755 --- a/tools/ci_build/github/linux/build_yocto.sh +++ b/tools/ci_build/github/linux/build_yocto.sh @@ -40,7 +40,7 @@ fi YOCTO_CONTAINER="arm_yocto" docker rm -f $YOCTO_CONTAINER || true -docker run --name $YOCTO_CONTAINER --volume $TARGET_FOLDER/fsl-arm-yocto-bsp:/fsl-arm-yocto-bsp --volume $SOURCE_ROOT:/onnxruntime_src $YOCTO_IMAGE /bin/bash /onnxruntime_src/tools/ci_build/github/linux/yocto_build_toolchain.sh -y $YOCTO_VERSION & +docker run -e SYSTEM_COLLECTIONURI --name $YOCTO_CONTAINER --volume $TARGET_FOLDER/fsl-arm-yocto-bsp:/fsl-arm-yocto-bsp --volume $SOURCE_ROOT:/onnxruntime_src $YOCTO_IMAGE /bin/bash /onnxruntime_src/tools/ci_build/github/linux/yocto_build_toolchain.sh -y $YOCTO_VERSION & wait $! diff --git a/tools/ci_build/github/linux/create_package.sh b/tools/ci_build/github/linux/create_package.sh index ed012a5abc..37ed6d772b 100755 --- a/tools/ci_build/github/linux/create_package.sh +++ b/tools/ci_build/github/linux/create_package.sh @@ -19,5 +19,5 @@ for version in '23'; do docker_image=fedora$version cd $SCRIPT_DIR/docker docker build --pull -t $docker_image --build-arg OS_VERSION=$version -f Dockerfile.fedora . - docker run --rm -e AZURESASKEY --volume "$HOME/.cache/onnxruntime:/root/.cache/onnxruntime" -v $BUILD_BINARIESDIRECTORY:/root/rpmbuild -v $BUILD_ARTIFACTSTAGINGDIRECTORY:/data/a -v $HOME/.ccache:/root/.ccache -v $TOP_SRC_DIR:/data/onnxruntime -w /data/b $docker_image /data/onnxruntime/tools/ci_build/github/linux/create_package_inside_docker.sh + docker run -e SYSTEM_COLLECTIONURI --rm -e AZURESASKEY --volume "$HOME/.cache/onnxruntime:/root/.cache/onnxruntime" -v $BUILD_BINARIESDIRECTORY:/root/rpmbuild -v $BUILD_ARTIFACTSTAGINGDIRECTORY:/data/a -v $HOME/.ccache:/root/.ccache -v $TOP_SRC_DIR:/data/onnxruntime -w /data/b $docker_image /data/onnxruntime/tools/ci_build/github/linux/create_package_inside_docker.sh done diff --git a/tools/ci_build/github/linux/run_python_dockerbuild.sh b/tools/ci_build/github/linux/run_python_dockerbuild.sh index 2fec98e569..7e6b8ddabd 100755 --- a/tools/ci_build/github/linux/run_python_dockerbuild.sh +++ b/tools/ci_build/github/linux/run_python_dockerbuild.sh @@ -26,7 +26,7 @@ if [ "${BUILD_EXTR_PAR}" != "" ] ; then DOCKER_SCRIPT_OPTIONS+=" -x ${BUILD_EXTR_PAR}" fi -docker run --rm \ +docker run -e SYSTEM_COLLECTIONURI --rm \ --volume /data/onnx:/data/onnx:ro \ --volume "${BUILD_SOURCESDIRECTORY}:/onnxruntime_src" \ --volume "${BUILD_BINARIESDIRECTORY}:/build" \ diff --git a/tools/ci_build/github/linux/run_python_dockertest.sh b/tools/ci_build/github/linux/run_python_dockertest.sh index 332dd9c728..1523824f8d 100755 --- a/tools/ci_build/github/linux/run_python_dockertest.sh +++ b/tools/ci_build/github/linux/run_python_dockertest.sh @@ -16,7 +16,7 @@ if [ $DEVICE = "GPU" ]; then fi mkdir -p $HOME/.onnx -docker run --rm \ +docker run -e SYSTEM_COLLECTIONURI --rm \ --volume /data/onnx:/data/onnx:ro \ --volume $BUILD_SOURCESDIRECTORY:/onnxruntime_src \ --volume $BUILD_BINARIESDIRECTORY:/build \