diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index e0ad8ed51a..fe343c5601 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -797,6 +797,23 @@ endif() # External dependencies list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/external) +include(external/helper_functions.cmake) + +file(STRINGS deps.txt ONNXRUNTIME_DEPS_LIST) +foreach(ONNXRUNTIME_DEP IN LISTS ONNXRUNTIME_DEPS_LIST) + # Lines start with "#" are comments + if(NOT ONNXRUNTIME_DEP MATCHES "^#") + # The first column is name + list(POP_FRONT ONNXRUNTIME_DEP ONNXRUNTIME_DEP_NAME) + # The second column is URL + # The URL below may be a local file path or an HTTPS URL + list(POP_FRONT ONNXRUNTIME_DEP ONNXRUNTIME_DEP_URL) + set(DEP_URL_${ONNXRUNTIME_DEP_NAME} ${ONNXRUNTIME_DEP_URL}) + # The third column is SHA1 hash value + set(DEP_SHA1_${ONNXRUNTIME_DEP_NAME} ${ONNXRUNTIME_DEP}) + endif() +endforeach() + #protobuf begin #Here we support two build mode: @@ -812,8 +829,8 @@ endif() FetchContent_Declare( Protobuf - URL https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.18.3.zip - URL_HASH SHA1=b95bf7e9de9c2249b6c1f2ca556ace49999e90bd + URL ${DEP_URL_protobuf} + URL_HASH SHA1=${DEP_SHA1_protobuf} SOURCE_SUBDIR cmake PATCH_COMMAND ${ONNXRUNTIME_PROTOBUF_PATCH_COMMAND} FIND_PACKAGE_ARGS NAMES Protobuf @@ -827,7 +844,7 @@ if (onnxruntime_DISABLE_RTTI) set(protobuf_DISABLE_RTTI ON CACHE BOOL "Remove runtime type information in the binaries" FORCE) endif() -FetchContent_MakeAvailable(Protobuf) +onnxruntime_fetchcontent_makeavailable(Protobuf) if(Protobuf_FOUND) message("Protobuf version: ${Protobuf_VERSION}") else() @@ -1587,13 +1604,13 @@ endif() FetchContent_Declare( onnx - URL https://github.com/onnx/onnx/archive/5a5f8a5935762397aa68429b5493084ff970f774.zip - URL_HASH SHA1=edc8e1338c02f3ab222f3d803a24e17608c13895 + URL ${DEP_URL_onnx} + URL_HASH SHA1=${DEP_SHA1_onnx} PATCH_COMMAND ${ONNXRUNTIME_ONNX_PATCH_COMMAND} ) if (NOT onnxruntime_MINIMAL_BUILD) - FetchContent_MakeAvailable(onnx) + onnxruntime_fetchcontent_makeavailable(onnx) else() include(onnx_minimal) endif() diff --git a/cmake/deps.txt b/cmake/deps.txt new file mode 100644 index 0000000000..7f900635f1 --- /dev/null +++ b/cmake/deps.txt @@ -0,0 +1,13 @@ +#Name;Url;SHA1 +#This is a CSV file. +#The columns are separated by ";" because a list in cmake is just a ";" separated group of strings. +#Names should be in lower case. They will be used as variable names in cmake. +#URLs can be either https URLs or local file paths in cmake-style(directory separator is a forward slash character). +#SHA1 hashes can be generated by running sha1sum command. +#If you need to change abseil's version to a different one, you may also want to update external\abseil-cpp.natvis +#since the file contains a version string: "lts_20211102". However, the file is for debugging purposes only and would +#not affect built binaries. +abseil_cpp;https://github.com/abseil/abseil-cpp/archive/refs/tags/20211102.0.zip;ce61532df974d00025b1220408ce1c900d81baf2 +microsoft_gsl;https://github.com/microsoft/GSL/archive/refs/tags/v4.0.0.zip;cf368104cd22a87b4dd0c80228919bb2df3e2a14 +onnx;https://github.com/onnx/onnx/archive/5a5f8a5935762397aa68429b5493084ff970f774.zip;edc8e1338c02f3ab222f3d803a24e17608c13895 +protobuf;https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.18.3.zip;b95bf7e9de9c2249b6c1f2ca556ace49999e90bd \ No newline at end of file diff --git a/cmake/external/abseil-cpp.cmake b/cmake/external/abseil-cpp.cmake index d1436fef88..6b3b709cfa 100644 --- a/cmake/external/abseil-cpp.cmake +++ b/cmake/external/abseil-cpp.cmake @@ -21,12 +21,12 @@ FetchContent_Declare( abseil_cpp PREFIX "${CMAKE_CURRENT_BINARY_DIR}/abseil-cpp" BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/external/abseil-cpp" - URL https://github.com/abseil/abseil-cpp/archive/refs/tags/20211102.0.zip - URL_HASH SHA1=ce61532df974d00025b1220408ce1c900d81baf2 + URL ${DEP_URL_abseil_cpp} + URL_HASH SHA1=${DEP_SHA1_abseil_cpp} PATCH_COMMAND ${ABSL_PATCH_COMMAND} ) -FetchContent_MakeAvailable(abseil_cpp) +onnxruntime_fetchcontent_makeavailable(abseil_cpp) FetchContent_GetProperties(abseil_cpp) set(ABSEIL_SOURCE_DIR ${abseil_cpp_SOURCE_DIR}) message(STATUS "Abseil source dir:" ${ABSEIL_SOURCE_DIR}) diff --git a/cmake/external/gsl.cmake b/cmake/external/gsl.cmake index b9ab273601..f4f9ce770d 100644 --- a/cmake/external/gsl.cmake +++ b/cmake/external/gsl.cmake @@ -6,20 +6,20 @@ include(FetchContent) if(onnxruntime_USE_CUDA) FetchContent_Declare( GSL - URL https://github.com/microsoft/GSL/archive/refs/tags/v4.0.0.zip - URL_HASH SHA1=cf368104cd22a87b4dd0c80228919bb2df3e2a14 + URL ${DEP_URL_microsoft_gsl} + URL_HASH SHA1=${DEP_SHA1_microsoft_gsl} PATCH_COMMAND ${Patch_EXECUTABLE} --binary --ignore-whitespace -p1 < ${PROJECT_SOURCE_DIR}/patches/gsl/1064.patch ) else() FetchContent_Declare( GSL - URL https://github.com/microsoft/GSL/archive/refs/tags/v4.0.0.zip - URL_HASH SHA1=cf368104cd22a87b4dd0c80228919bb2df3e2a14 + URL ${DEP_URL_microsoft_gsl} + URL_HASH SHA1=${DEP_SHA1_microsoft_gsl} ) endif() -FetchContent_MakeAvailable(GSL) +onnxruntime_fetchcontent_makeavailable(GSL) set(GSL_TARGET "Microsoft.GSL::GSL") set(GSL_INCLUDE_DIR "$") diff --git a/cmake/external/helper_functions.cmake b/cmake/external/helper_functions.cmake new file mode 100644 index 0000000000..a31e4b04f3 --- /dev/null +++ b/cmake/external/helper_functions.cmake @@ -0,0 +1,193 @@ +# Distributed under the OSI-approved BSD 3-Clause License. See accompanying +# file Copyright.txt or https://cmake.org/licensing for details. + +# This file was copied from cmake source with modifications +macro(onnxruntime_fetchcontent_makeavailable) + + # We must append an item, even if the variable is unset, so prefix its value. + # We will strip that prefix when we pop the value at the end of the macro. + list(APPEND __cmake_fcCurrentVarsStack + "__fcprefix__${CMAKE_VERIFY_INTERFACE_HEADER_SETS}" + ) + set(CMAKE_VERIFY_INTERFACE_HEADER_SETS FALSE) + + get_property(__cmake_providerCommand GLOBAL PROPERTY + __FETCHCONTENT_MAKEAVAILABLE_SERIAL_PROVIDER + ) + foreach(__cmake_contentName IN ITEMS ${ARGV}) + string(TOLOWER ${__cmake_contentName} __cmake_contentNameLower) + + # If user specified FETCHCONTENT_SOURCE_DIR_... for this dependency, that + # overrides everything else and we shouldn't try to use find_package() or + # a dependency provider. + string(TOUPPER ${__cmake_contentName} __cmake_contentNameUpper) + if("${FETCHCONTENT_SOURCE_DIR_${__cmake_contentNameUpper}}" STREQUAL "") + # Dependency provider gets first opportunity, but prevent infinite + # recursion if we are called again for the same thing + if(NOT "${__cmake_providerCommand}" STREQUAL "" AND + NOT DEFINED __cmake_fcProvider_${__cmake_contentNameLower}) + message(VERBOSE + "Trying FETCHCONTENT_MAKEAVAILABLE_SERIAL dependency provider for " + "${__cmake_contentName}" + ) + # It's still valid if there are no saved details. The project may have + # been written to assume a dependency provider is always set and will + # provide dependencies without having any declared details for them. + __FetchContent_getSavedDetails(${__cmake_contentName} __cmake_contentDetails) + set(__cmake_providerArgs + "FETCHCONTENT_MAKEAVAILABLE_SERIAL" + "${__cmake_contentName}" + ) + # Empty arguments must be preserved because of things like + # GIT_SUBMODULES (see CMP0097) + foreach(__cmake_item IN LISTS __cmake_contentDetails) + string(APPEND __cmake_providerArgs " [==[${__cmake_item}]==]") + endforeach() + + # This property might be defined but empty. As long as it is defined, + # find_package() can be called. + get_property(__cmake_addfpargs GLOBAL PROPERTY + _FetchContent_${contentNameLower}_find_package_args + DEFINED + ) + if(__cmake_addfpargs) + get_property(__cmake_fpargs GLOBAL PROPERTY + _FetchContent_${contentNameLower}_find_package_args + ) + string(APPEND __cmake_providerArgs " FIND_PACKAGE_ARGS") + foreach(__cmake_item IN LISTS __cmake_fpargs) + string(APPEND __cmake_providerArgs " [==[${__cmake_item}]==]") + endforeach() + endif() + + # Calling the provider could lead to onnxruntime_fetchcontent_makeavailable() being + # called for a nested dependency. That nested call may occur in the + # current variable scope. We have to save and restore the variables we + # need preserved. + list(APPEND __cmake_fcCurrentVarsStack + ${__cmake_contentName} + ${__cmake_contentNameLower} + ) + + set(__cmake_fcProvider_${__cmake_contentNameLower} YES) + cmake_language(EVAL CODE "${__cmake_providerCommand}(${__cmake_providerArgs})") + + list(POP_BACK __cmake_fcCurrentVarsStack + __cmake_contentNameLower + __cmake_contentName + ) + + unset(__cmake_fcProvider_${__cmake_contentNameLower}) + unset(__cmake_providerArgs) + unset(__cmake_addfpargs) + unset(__cmake_fpargs) + unset(__cmake_item) + unset(__cmake_contentDetails) + + FetchContent_GetProperties(${__cmake_contentName}) + if(${__cmake_contentNameLower}_POPULATED) + continue() + endif() + endif() + + # Check if we've been asked to try find_package() first, even if we + # have already populated this dependency. If we previously tried to + # use find_package() for this and it succeeded, those things might + # no longer be in scope, so we have to do it again. + get_property(__cmake_haveFpArgs GLOBAL PROPERTY + _FetchContent_${__cmake_contentNameLower}_find_package_args DEFINED + ) + if(__cmake_haveFpArgs) + unset(__cmake_haveFpArgs) + message(VERBOSE "Trying find_package(${__cmake_contentName} ...) before FetchContent") + get_property(__cmake_fpArgs GLOBAL PROPERTY + _FetchContent_${__cmake_contentNameLower}_find_package_args + ) + + # This call could lead to onnxruntime_fetchcontent_makeavailable() being called for + # a nested dependency and it may occur in the current variable scope. + # We have to save/restore the variables we need to preserve. + list(APPEND __cmake_fcCurrentNameStack + ${__cmake_contentName} + ${__cmake_contentNameLower} + ) + find_package(${__cmake_contentName} ${__cmake_fpArgs}) + list(POP_BACK __cmake_fcCurrentNameStack + __cmake_contentNameLower + __cmake_contentName + ) + unset(__cmake_fpArgs) + + if(${__cmake_contentName}_FOUND) + FetchContent_SetPopulated(${__cmake_contentName}) + FetchContent_GetProperties(${__cmake_contentName}) + continue() + endif() + endif() + else() + unset(__cmake_haveFpArgs) + endif() + + FetchContent_GetProperties(${__cmake_contentName}) + if(NOT ${__cmake_contentNameLower}_POPULATED) + FetchContent_Populate(${__cmake_contentName}) + __FetchContent_setupFindPackageRedirection(${__cmake_contentName}) + + # Only try to call add_subdirectory() if the populated content + # can be treated that way. Protecting the call with the check + # allows this function to be used for projects that just want + # to ensure the content exists, such as to provide content at + # a known location. We check the saved details for an optional + # SOURCE_SUBDIR which can be used in the same way as its meaning + # for ExternalProject. It won't matter if it was passed through + # to the ExternalProject sub-build, since it would have been + # ignored there. + set(__cmake_srcdir "${${__cmake_contentNameLower}_SOURCE_DIR}") + __FetchContent_getSavedDetails(${__cmake_contentName} __cmake_contentDetails) + if("${__cmake_contentDetails}" STREQUAL "") + message(FATAL_ERROR "No details have been set for content: ${__cmake_contentName}") + endif() + cmake_parse_arguments(__cmake_arg "SYSTEM" "SOURCE_SUBDIR" "" ${__cmake_contentDetails}) + if(NOT "${__cmake_arg_SOURCE_SUBDIR}" STREQUAL "") + string(APPEND __cmake_srcdir "/${__cmake_arg_SOURCE_SUBDIR}") + endif() + + if(EXISTS ${__cmake_srcdir}/CMakeLists.txt) + add_subdirectory(${__cmake_srcdir} ${${__cmake_contentNameLower}_BINARY_DIR} EXCLUDE_FROM_ALL) + get_property(subdir_import_targets DIRECTORY "${__cmake_srcdir}" PROPERTY BUILDSYSTEM_TARGETS) + foreach(subdir_target ${subdir_import_targets}) + if(TARGET ${subdir_target}) + get_target_property(subdir_target_type ${subdir_target} TYPE) + if(subdir_target_type STREQUAL "EXECUTABLE") + get_target_property(subdir_target_osx_arch ${subdir_target} OSX_ARCHITECTURES) + if (subdir_target_osx_arch) + if (NOT ${CMAKE_HOST_SYSTEM_PROCESSOR} IN_LIST subdir_target_osx_arch) + message("Added an executable target ${subdir_target} but it can not run natively on ${CMAKE_HOST_SYSTEM_PROCESSOR}, we will try to modify it") + endif() + endif() + endif() + set_target_properties(${subdir_target} PROPERTIES FOLDER "External") + set_target_properties(${subdir_target} PROPERTIES COMPILE_WARNING_AS_ERROR OFF) + endif() + endforeach() + endif() + + unset(__cmake_srcdir) + unset(__cmake_contentDetails) + unset(__cmake_arg_SOURCE_SUBDIR) + endif() + endforeach() + # Prefix will be "__fcprefix__" + list(POP_BACK __cmake_fcCurrentVarsStack __cmake_original_verify_setting) + string(SUBSTRING "${__cmake_original_verify_setting}" + 12 -1 __cmake_original_verify_setting + ) + set(CMAKE_VERIFY_INTERFACE_HEADER_SETS ${__cmake_original_verify_setting}) + + # clear local variables to prevent leaking into the caller's scope + unset(__cmake_contentName) + unset(__cmake_contentNameLower) + unset(__cmake_contentNameUpper) + unset(__cmake_providerCommand) + unset(__cmake_original_verify_setting) +endmacro() \ No newline at end of file diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml index ca1f1714af..d6f01d6a3c 100644 --- a/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml +++ b/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml @@ -53,6 +53,8 @@ jobs: Context: tools/ci_build/github/linux/docker/inference/${{parameters.OnnxruntimeArch}}/default/cpu DockerBuildArgs: "--build-arg BUILD_UID=$( id -u ) --build-arg BASEIMAGE=${{parameters.BaseImage}}" Repository: onnxruntimecpubuildcentos7${{parameters.OnnxruntimeArch}} + ${{ if eq(parameters.OnnxruntimeArch, 'aarch64') }}: + UpdateDepsTxt: false - task: CmdLine@2 inputs: diff --git a/tools/ci_build/github/azure-pipelines/templates/download-deps.yml b/tools/ci_build/github/azure-pipelines/templates/download-deps.yml new file mode 100644 index 0000000000..0579f047a8 --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/templates/download-deps.yml @@ -0,0 +1,28 @@ + +steps: +# You can the information of the System CollectionIds by visiting https://app.vssps.visualstudio.com/_apis/accounts?memberId=me, +# Then search the uuid below in the result. + +# The public ADO project +- ${{ if eq(variables['System.CollectionId'], 'f3ad12f2-e480-4533-baf2-635c95467d29') }}: + - task: DownloadPackage@1 + displayName: 'Download ONNX Runtime Build Time Deps' + inputs: + packageType: upack + feed: '/7424c8e4-5c62-490e-95c4-79446f31017c' + definition: '517c4f6f-5437-4392-a70d-4f15ec5be2f0' + version: 1.0.3 + downloadPath: $(Build.BinariesDirectory)/deps + +# The private ADO project +- ${{ if eq(variables['System.CollectionId'], 'bc038106-a83b-4dab-9dd3-5a41bc58f34c') }}: + - task: DownloadPackage@1 + displayName: 'Download ONNX Runtime Build Time Deps' + inputs: + packageType: upack + feed: '/4c7631f5-24c0-4307-8822-1aa8f180c325' + definition: 'fd9dd5ad-b73e-4678-890e-edcf680dbc1a' + version: 1.0.3 + downloadPath: $(Build.BinariesDirectory)/deps + +# You can add more ADO accounts at here. \ No newline at end of file diff --git a/tools/ci_build/github/azure-pipelines/templates/get-docker-image-steps.yml b/tools/ci_build/github/azure-pipelines/templates/get-docker-image-steps.yml index e8c5778929..e832d1098a 100644 --- a/tools/ci_build/github/azure-pipelines/templates/get-docker-image-steps.yml +++ b/tools/ci_build/github/azure-pipelines/templates/get-docker-image-steps.yml @@ -16,11 +16,19 @@ parameters: - name: ScriptName type: string default: "tools/ci_build/get_docker_image.py" - +- name: UpdateDepsTxt + type: boolean + default: true + steps: + +- ${{ if eq(parameters.UpdateDepsTxt, true)}}: + - template: download-deps.yml + - ${{ if contains(parameters.Dockerfile, 'manylinux') }}: - checkout: manylinux - script: | + set -e -x mv manylinux onnxruntime mv onnxruntime .. cd .. @@ -49,3 +57,12 @@ steps: --docker-build-args "${{ parameters.DockerBuildArgs }}" \ --repository "${{ parameters.Repository }}" displayName: "Get ${{ parameters.Repository }} image for ${{ parameters.Dockerfile }}" + +- ${{ if and(eq(parameters.UpdateDepsTxt, true), or(eq(variables['System.CollectionId'], 'f3ad12f2-e480-4533-baf2-635c95467d29'),eq(variables['System.CollectionId'], 'bc038106-a83b-4dab-9dd3-5a41bc58f34c'))) }}: + - task: PythonScript@0 + displayName: 'Update deps.txt' + inputs: + scriptPath: $(Build.SourcesDirectory)/tools/ci_build/replace_urls_in_deps.py + arguments: --new_dir /build/deps + workingDirectory: $(Build.BinariesDirectory) + pythonInterpreter: /usr/bin/python3 \ No newline at end of file diff --git a/tools/ci_build/github/azure-pipelines/templates/py-linux-ubuntu.yml b/tools/ci_build/github/azure-pipelines/templates/py-linux-ubuntu.yml index 76db6528fe..9026903e51 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-linux-ubuntu.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-linux-ubuntu.yml @@ -30,6 +30,8 @@ jobs: Context: tools/ci_build/github/linux/docker DockerBuildArgs: "--build-arg BUILD_UID=$( id -u ) --build-arg BASEIMAGE=${{ parameters.base_image }} --build-arg PLATFORM=${{ parameters.arch }}" Repository: onnxruntimecpubuildubuntu20python${{ parameters.arch }} + ${{ if eq(parameters.arch, 'aarch64') }}: + UpdateDepsTxt: false - task: CmdLine@2 displayName: 'Build Python Wheel' diff --git a/tools/ci_build/github/azure-pipelines/templates/py-linux.yml b/tools/ci_build/github/azure-pipelines/templates/py-linux.yml index 7c66c62a6d..6426c5a033 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-linux.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-linux.yml @@ -39,6 +39,8 @@ jobs: Context: tools/ci_build/github/linux/docker/inference/x64/python/cpu DockerBuildArgs: "--build-arg BUILD_UID=$( id -u ) --build-arg BASEIMAGE=${{ parameters.base_image }} --build-arg PLATFORM=${{ parameters.arch }} --build-arg PREPEND_PATH=${{ parameters.prepend_path }} --build-arg LD_LIBRARY_PATH_ARG=${{ parameters.ld_library_path_arg }} --build-arg DEVTOOLSET_ROOTPATH=${{ parameters.devtoolset_rootpath }}" Repository: onnxruntimecpubuilpython${{ parameters.arch }} + ${{ if eq(parameters.arch, 'aarch64') }}: + UpdateDepsTxt: false - task: CmdLine@2 displayName: 'Build Python Wheel' diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml index 0e58cbb39f..daae6acb19 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml @@ -131,6 +131,15 @@ stages: workingDirectory: '$(Build.BinariesDirectory)' displayName: 'Install python modules' + - template: download-deps.yml + + - task: PythonScript@0 + displayName: 'Update deps.txt' + inputs: + scriptPath: $(Build.SourcesDirectory)/tools/ci_build/replace_urls_in_deps.py + arguments: --new_dir $(Build.BinariesDirectory)/deps + workingDirectory: $(Build.BinariesDirectory) + - powershell: | $Env:USE_MSVC_STATIC_RUNTIME=1 $Env:ONNX_ML=1 diff --git a/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml b/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml index f6b82f9a42..a01675bf91 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml @@ -52,6 +52,15 @@ jobs: workingDirectory: '$(Build.BinariesDirectory)' displayName: 'Install python modules' + - template: download-deps.yml + + - task: PythonScript@0 + displayName: 'Update deps.txt' + inputs: + scriptPath: $(Build.SourcesDirectory)/tools/ci_build/replace_urls_in_deps.py + arguments: --new_dir $(Build.BinariesDirectory)/deps + workingDirectory: $(Build.BinariesDirectory) + - powershell: | $Env:USE_MSVC_STATIC_RUNTIME=1 $Env:ONNX_ML=1 diff --git a/tools/ci_build/github/azure-pipelines/templates/win-ci-vs-2019.yml b/tools/ci_build/github/azure-pipelines/templates/win-ci-vs-2019.yml index 1db620cc5a..ae58c5c951 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-ci-vs-2019.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-ci-vs-2019.yml @@ -71,6 +71,15 @@ jobs: addToPath: true architecture: ${{ parameters.buildArch }} + - template: download-deps.yml + + - task: PythonScript@0 + displayName: 'Update deps.txt' + inputs: + scriptPath: $(Build.SourcesDirectory)/tools/ci_build/replace_urls_in_deps.py + arguments: --new_dir $(Build.BinariesDirectory)/deps + workingDirectory: $(Build.BinariesDirectory) + - task: NodeTool@0 inputs: versionSpec: '16.x' diff --git a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml index 3abc75fe6c..dcb32a73b2 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml @@ -84,6 +84,16 @@ jobs: python -m pip install -q setuptools wheel numpy workingDirectory: '$(Build.BinariesDirectory)' displayName: 'Install python modules' + + - template: download-deps.yml + + - task: PythonScript@0 + displayName: 'Update deps.txt' + inputs: + scriptPath: $(Build.SourcesDirectory)/tools/ci_build/replace_urls_in_deps.py + arguments: --new_dir $(Build.BinariesDirectory)/deps + workingDirectory: $(Build.BinariesDirectory) + - powershell: | $Env:USE_MSVC_STATIC_RUNTIME=1 $Env:ONNX_ML=1 diff --git a/tools/ci_build/replace_urls_in_deps.py b/tools/ci_build/replace_urls_in_deps.py new file mode 100644 index 0000000000..839e12907c --- /dev/null +++ b/tools/ci_build/replace_urls_in_deps.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +# This file replaces https URLs in deps.txt to local file paths. It runs after we download the dependencies from Azure +# DevOps Artifacts + +import argparse +import csv +import os +from dataclasses import dataclass +from pathlib import Path + + +@dataclass(frozen=True) +class Dep: + name: str + url: str + sha1_hash: str + + +def parse_arguments(): + parser = argparse.ArgumentParser() + # The directory that contains downloaded zip files + parser.add_argument("--new_dir", required=False) + + return parser.parse_args() + + +def main(): + SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) + REPO_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, "..", "..")) + + args = parse_arguments() + new_dir = None + if args.new_dir: + new_dir = Path(args.new_dir) + else: + BUILD_BINARIESDIRECTORY = os.environ.get("BUILD_BINARIESDIRECTORY") + if BUILD_BINARIESDIRECTORY is None: + raise NameError("Please specify --new_dir or set the env var BUILD_BINARIESDIRECTORY") + new_dir = Path(BUILD_BINARIESDIRECTORY) / "deps" + + # Here we intentionally do not check if new_dir exists, because it might be used in a docker container instead. + + deps = [] + + csv_file_path = Path(REPO_DIR) / "cmake" / "deps.txt" + + # Read the whole file into memory first + with csv_file_path.open("r", encoding="utf-8") as f: + depfile_reader = csv.reader(f, delimiter=";") + for row in depfile_reader: + if len(row) != 3: + continue + # Lines start with "#" are comments + if row[0].startswith("#"): + continue + deps.append(Dep(row[0], row[1], row[2])) + + # Write updated content back + with csv_file_path.open("w", newline="", encoding="utf-8") as f: + depfile_writer = csv.writer(f, delimiter=";") + for dep in deps: + if dep.url.startswith("https://"): + new_url = new_dir / dep.url[8:] + depfile_writer.writerow([dep.name, new_url.as_posix(), dep.sha1_hash]) + else: + # Write the original thing back + depfile_writer.writerow([dep.name, dep.url, dep.sha1_hash]) + + +if __name__ == "__main__": + main()