Move C/C++ deps' URLs to deps.txt (#13769)

### Description 1. Move C/C++ deps' URLs to deps.txt, and download the dependencies from Azure Devops Artifacts instead of github. 2. Add "EXCLUDE_FROM_ALL" keyword to the cmake external projects, so that we only build the parts we need and avoid installing the 3rd-party dependencies when people run `make install` in ORT's build directory. However, at this moment cmake itself doesn't have the feature. So I copied their code to cmake/external/helper_functions.cmake and modified it. This PR is split from #13523, to make that one smaller. ### Motivation and Context 1. Secure the supply chain 2. Make it be possible to automatically detect if ORT has an old dependency that hasn't been updated from a long time.
2026-07-29 20:14:01 +00:00 · 2022-11-29 18:06:35 -08:00 · 2022-11-29 18:06:35 -08:00 · 29ed8811e5
commit 29ed8811e5
parent 571dc5a1f1
15 changed files with 400 additions and 15 deletions
--- a/cmake/CMakeLists.txt
+++ b/cmake/CMakeLists.txt
@ -797,6 +797,23 @@ endif()
 # External dependencies
 list(APPEND CMAKE_MODULE_PATH ${PROJECT_SOURCE_DIR}/external)

+include(external/helper_functions.cmake)
+
+file(STRINGS deps.txt ONNXRUNTIME_DEPS_LIST)
+foreach(ONNXRUNTIME_DEP IN LISTS ONNXRUNTIME_DEPS_LIST)
+  # Lines start with "#" are comments
+  if(NOT ONNXRUNTIME_DEP MATCHES "^#")
+    # The first column is name
+    list(POP_FRONT ONNXRUNTIME_DEP ONNXRUNTIME_DEP_NAME)
+    # The second column is URL
+    # The URL below may be a local file path or an HTTPS URL
+    list(POP_FRONT ONNXRUNTIME_DEP ONNXRUNTIME_DEP_URL)
+    set(DEP_URL_${ONNXRUNTIME_DEP_NAME} ${ONNXRUNTIME_DEP_URL})
+    # The third column is SHA1 hash value
+    set(DEP_SHA1_${ONNXRUNTIME_DEP_NAME} ${ONNXRUNTIME_DEP})
+  endif()
+endforeach()
+
 #protobuf begin

 #Here we support two build mode:
@ -812,8 +829,8 @@ endif()

 FetchContent_Declare(
  Protobuf
-  URL https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.18.3.zip
-  URL_HASH SHA1=b95bf7e9de9c2249b6c1f2ca556ace49999e90bd
+  URL ${DEP_URL_protobuf}
+  URL_HASH SHA1=${DEP_SHA1_protobuf}
  SOURCE_SUBDIR  cmake
  PATCH_COMMAND ${ONNXRUNTIME_PROTOBUF_PATCH_COMMAND}
  FIND_PACKAGE_ARGS NAMES Protobuf
@ -827,7 +844,7 @@ if (onnxruntime_DISABLE_RTTI)
  set(protobuf_DISABLE_RTTI ON CACHE BOOL "Remove runtime type information in the binaries" FORCE)
 endif()

-FetchContent_MakeAvailable(Protobuf)
+onnxruntime_fetchcontent_makeavailable(Protobuf)
 if(Protobuf_FOUND)
  message("Protobuf version: ${Protobuf_VERSION}")
 else()
@ -1587,13 +1604,13 @@ endif()

 FetchContent_Declare(
  onnx
-  URL https://github.com/onnx/onnx/archive/5a5f8a5935762397aa68429b5493084ff970f774.zip
-  URL_HASH SHA1=edc8e1338c02f3ab222f3d803a24e17608c13895
+  URL ${DEP_URL_onnx}
+  URL_HASH SHA1=${DEP_SHA1_onnx}
  PATCH_COMMAND ${ONNXRUNTIME_ONNX_PATCH_COMMAND}
 )

 if (NOT onnxruntime_MINIMAL_BUILD)
-  FetchContent_MakeAvailable(onnx)
+  onnxruntime_fetchcontent_makeavailable(onnx)
 else()
  include(onnx_minimal)
 endif()
--- a/cmake/deps.txt
+++ b/cmake/deps.txt
@ -0,0 +1,13 @@
+#Name;Url;SHA1
+#This is a CSV file.
+#The columns are separated by ";" because a list in cmake is just a ";" separated group of strings.
+#Names should be in lower case. They will be used as variable names in cmake.
+#URLs can be either https URLs or local file paths in cmake-style(directory separator is a forward slash character).
+#SHA1 hashes can be generated by running sha1sum command.
+#If you need to change abseil's version to a different one, you may also want to update external\abseil-cpp.natvis 
+#since the file contains a version string: "lts_20211102". However, the file is for debugging purposes only and would
+#not affect built binaries.
+abseil_cpp;https://github.com/abseil/abseil-cpp/archive/refs/tags/20211102.0.zip;ce61532df974d00025b1220408ce1c900d81baf2
+microsoft_gsl;https://github.com/microsoft/GSL/archive/refs/tags/v4.0.0.zip;cf368104cd22a87b4dd0c80228919bb2df3e2a14
+onnx;https://github.com/onnx/onnx/archive/5a5f8a5935762397aa68429b5493084ff970f774.zip;edc8e1338c02f3ab222f3d803a24e17608c13895
+protobuf;https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.18.3.zip;b95bf7e9de9c2249b6c1f2ca556ace49999e90bd
--- a/cmake/external/abseil-cpp.cmake
+++ b/cmake/external/abseil-cpp.cmake
@ -21,12 +21,12 @@ FetchContent_Declare(
    abseil_cpp
    PREFIX "${CMAKE_CURRENT_BINARY_DIR}/abseil-cpp"
    BINARY_DIR "${CMAKE_CURRENT_BINARY_DIR}/external/abseil-cpp"
-    URL https://github.com/abseil/abseil-cpp/archive/refs/tags/20211102.0.zip
-    URL_HASH SHA1=ce61532df974d00025b1220408ce1c900d81baf2
+    URL ${DEP_URL_abseil_cpp}
+    URL_HASH SHA1=${DEP_SHA1_abseil_cpp}
    PATCH_COMMAND ${ABSL_PATCH_COMMAND}
 )

-FetchContent_MakeAvailable(abseil_cpp)
+onnxruntime_fetchcontent_makeavailable(abseil_cpp)
 FetchContent_GetProperties(abseil_cpp)
 set(ABSEIL_SOURCE_DIR ${abseil_cpp_SOURCE_DIR})
 message(STATUS "Abseil source dir:" ${ABSEIL_SOURCE_DIR})
--- a/cmake/external/gsl.cmake
+++ b/cmake/external/gsl.cmake
@ -6,20 +6,20 @@ include(FetchContent)
 if(onnxruntime_USE_CUDA)
  FetchContent_Declare(
    GSL
-    URL https://github.com/microsoft/GSL/archive/refs/tags/v4.0.0.zip
-    URL_HASH SHA1=cf368104cd22a87b4dd0c80228919bb2df3e2a14
+    URL ${DEP_URL_microsoft_gsl}
+    URL_HASH SHA1=${DEP_SHA1_microsoft_gsl}
    PATCH_COMMAND ${Patch_EXECUTABLE} --binary --ignore-whitespace -p1 < ${PROJECT_SOURCE_DIR}/patches/gsl/1064.patch
  )
 else()
  FetchContent_Declare(
    GSL
-    URL https://github.com/microsoft/GSL/archive/refs/tags/v4.0.0.zip
-    URL_HASH SHA1=cf368104cd22a87b4dd0c80228919bb2df3e2a14
+    URL ${DEP_URL_microsoft_gsl}
+    URL_HASH SHA1=${DEP_SHA1_microsoft_gsl}
  )
 endif()


-FetchContent_MakeAvailable(GSL)
+onnxruntime_fetchcontent_makeavailable(GSL)

 set(GSL_TARGET "Microsoft.GSL::GSL")
 set(GSL_INCLUDE_DIR "$<TARGET_PROPERTY:${GSL_TARGET},INTERFACE_INCLUDE_DIRECTORIES>")
--- a/cmake/external/helper_functions.cmake
+++ b/cmake/external/helper_functions.cmake
@ -0,0 +1,193 @@
+# Distributed under the OSI-approved BSD 3-Clause License.  See accompanying
+# file Copyright.txt or https://cmake.org/licensing for details.
+
+# This file was copied from cmake source with modifications
+macro(onnxruntime_fetchcontent_makeavailable)
+
+  # We must append an item, even if the variable is unset, so prefix its value.
+  # We will strip that prefix when we pop the value at the end of the macro.
+  list(APPEND __cmake_fcCurrentVarsStack
+    "__fcprefix__${CMAKE_VERIFY_INTERFACE_HEADER_SETS}"
+  )
+  set(CMAKE_VERIFY_INTERFACE_HEADER_SETS FALSE)
+
+  get_property(__cmake_providerCommand GLOBAL PROPERTY
+    __FETCHCONTENT_MAKEAVAILABLE_SERIAL_PROVIDER
+  )
+  foreach(__cmake_contentName IN ITEMS ${ARGV})
+    string(TOLOWER ${__cmake_contentName} __cmake_contentNameLower)
+
+    # If user specified FETCHCONTENT_SOURCE_DIR_... for this dependency, that
+    # overrides everything else and we shouldn't try to use find_package() or
+    # a dependency provider.
+    string(TOUPPER ${__cmake_contentName} __cmake_contentNameUpper)
+    if("${FETCHCONTENT_SOURCE_DIR_${__cmake_contentNameUpper}}" STREQUAL "")
+      # Dependency provider gets first opportunity, but prevent infinite
+      # recursion if we are called again for the same thing
+      if(NOT "${__cmake_providerCommand}" STREQUAL "" AND
+        NOT DEFINED __cmake_fcProvider_${__cmake_contentNameLower})
+        message(VERBOSE
+          "Trying FETCHCONTENT_MAKEAVAILABLE_SERIAL dependency provider for "
+          "${__cmake_contentName}"
+        )
+        # It's still valid if there are no saved details. The project may have
+        # been written to assume a dependency provider is always set and will
+        # provide dependencies without having any declared details for them.
+        __FetchContent_getSavedDetails(${__cmake_contentName} __cmake_contentDetails)
+        set(__cmake_providerArgs
+          "FETCHCONTENT_MAKEAVAILABLE_SERIAL"
+          "${__cmake_contentName}"
+        )
+        # Empty arguments must be preserved because of things like
+        # GIT_SUBMODULES (see CMP0097)
+        foreach(__cmake_item IN LISTS __cmake_contentDetails)
+          string(APPEND __cmake_providerArgs " [==[${__cmake_item}]==]")
+        endforeach()
+
+        # This property might be defined but empty. As long as it is defined,
+        # find_package() can be called.
+        get_property(__cmake_addfpargs GLOBAL PROPERTY
+          _FetchContent_${contentNameLower}_find_package_args
+          DEFINED
+        )
+        if(__cmake_addfpargs)
+          get_property(__cmake_fpargs GLOBAL PROPERTY
+            _FetchContent_${contentNameLower}_find_package_args
+          )
+          string(APPEND __cmake_providerArgs " FIND_PACKAGE_ARGS")
+          foreach(__cmake_item IN LISTS __cmake_fpargs)
+            string(APPEND __cmake_providerArgs " [==[${__cmake_item}]==]")
+          endforeach()
+        endif()
+
+        # Calling the provider could lead to onnxruntime_fetchcontent_makeavailable() being
+        # called for a nested dependency. That nested call may occur in the
+        # current variable scope. We have to save and restore the variables we
+        # need preserved.
+        list(APPEND __cmake_fcCurrentVarsStack
+          ${__cmake_contentName}
+          ${__cmake_contentNameLower}
+        )
+
+        set(__cmake_fcProvider_${__cmake_contentNameLower} YES)
+        cmake_language(EVAL CODE "${__cmake_providerCommand}(${__cmake_providerArgs})")
+
+        list(POP_BACK __cmake_fcCurrentVarsStack
+          __cmake_contentNameLower
+          __cmake_contentName
+        )
+
+        unset(__cmake_fcProvider_${__cmake_contentNameLower})
+        unset(__cmake_providerArgs)
+        unset(__cmake_addfpargs)
+        unset(__cmake_fpargs)
+        unset(__cmake_item)
+        unset(__cmake_contentDetails)
+
+        FetchContent_GetProperties(${__cmake_contentName})
+        if(${__cmake_contentNameLower}_POPULATED)
+          continue()
+        endif()
+      endif()
+
+      # Check if we've been asked to try find_package() first, even if we
+      # have already populated this dependency. If we previously tried to
+      # use find_package() for this and it succeeded, those things might
+      # no longer be in scope, so we have to do it again.
+      get_property(__cmake_haveFpArgs GLOBAL PROPERTY
+        _FetchContent_${__cmake_contentNameLower}_find_package_args DEFINED
+      )
+      if(__cmake_haveFpArgs)
+        unset(__cmake_haveFpArgs)
+        message(VERBOSE "Trying find_package(${__cmake_contentName} ...) before FetchContent")
+        get_property(__cmake_fpArgs GLOBAL PROPERTY
+          _FetchContent_${__cmake_contentNameLower}_find_package_args
+        )
+
+        # This call could lead to onnxruntime_fetchcontent_makeavailable() being called for
+        # a nested dependency and it may occur in the current variable scope.
+        # We have to save/restore the variables we need to preserve.
+        list(APPEND __cmake_fcCurrentNameStack
+          ${__cmake_contentName}
+          ${__cmake_contentNameLower}
+        )
+        find_package(${__cmake_contentName} ${__cmake_fpArgs})
+        list(POP_BACK __cmake_fcCurrentNameStack
+          __cmake_contentNameLower
+          __cmake_contentName
+        )
+        unset(__cmake_fpArgs)
+
+        if(${__cmake_contentName}_FOUND)
+          FetchContent_SetPopulated(${__cmake_contentName})
+          FetchContent_GetProperties(${__cmake_contentName})
+          continue()
+        endif()
+      endif()
+    else()
+      unset(__cmake_haveFpArgs)
+    endif()
+
+    FetchContent_GetProperties(${__cmake_contentName})
+    if(NOT ${__cmake_contentNameLower}_POPULATED)
+      FetchContent_Populate(${__cmake_contentName})
+      __FetchContent_setupFindPackageRedirection(${__cmake_contentName})
+
+      # Only try to call add_subdirectory() if the populated content
+      # can be treated that way. Protecting the call with the check
+      # allows this function to be used for projects that just want
+      # to ensure the content exists, such as to provide content at
+      # a known location. We check the saved details for an optional
+      # SOURCE_SUBDIR which can be used in the same way as its meaning
+      # for ExternalProject. It won't matter if it was passed through
+      # to the ExternalProject sub-build, since it would have been
+      # ignored there.
+      set(__cmake_srcdir "${${__cmake_contentNameLower}_SOURCE_DIR}")
+      __FetchContent_getSavedDetails(${__cmake_contentName} __cmake_contentDetails)
+      if("${__cmake_contentDetails}" STREQUAL "")
+        message(FATAL_ERROR "No details have been set for content: ${__cmake_contentName}")
+      endif()
+      cmake_parse_arguments(__cmake_arg "SYSTEM" "SOURCE_SUBDIR" "" ${__cmake_contentDetails})
+      if(NOT "${__cmake_arg_SOURCE_SUBDIR}" STREQUAL "")
+        string(APPEND __cmake_srcdir "/${__cmake_arg_SOURCE_SUBDIR}")
+      endif()
+
+      if(EXISTS ${__cmake_srcdir}/CMakeLists.txt)
+          add_subdirectory(${__cmake_srcdir} ${${__cmake_contentNameLower}_BINARY_DIR} EXCLUDE_FROM_ALL)
+          get_property(subdir_import_targets DIRECTORY "${__cmake_srcdir}" PROPERTY BUILDSYSTEM_TARGETS)
+          foreach(subdir_target ${subdir_import_targets})
+            if(TARGET ${subdir_target})
+              get_target_property(subdir_target_type ${subdir_target} TYPE)            
+              if(subdir_target_type STREQUAL "EXECUTABLE")
+                get_target_property(subdir_target_osx_arch ${subdir_target} OSX_ARCHITECTURES)
+                if (subdir_target_osx_arch)
+                  if (NOT ${CMAKE_HOST_SYSTEM_PROCESSOR} IN_LIST subdir_target_osx_arch)                              
+                    message("Added an executable target ${subdir_target} but it can not run natively on ${CMAKE_HOST_SYSTEM_PROCESSOR}, we will try to modify it")
+                  endif()
+                endif()
+              endif()
+              set_target_properties(${subdir_target} PROPERTIES FOLDER "External")
+              set_target_properties(${subdir_target} PROPERTIES COMPILE_WARNING_AS_ERROR OFF)
+            endif()
+          endforeach()          
+      endif()
+
+      unset(__cmake_srcdir)
+      unset(__cmake_contentDetails)
+      unset(__cmake_arg_SOURCE_SUBDIR)
+    endif()
+  endforeach()
+  # Prefix will be "__fcprefix__"
+  list(POP_BACK __cmake_fcCurrentVarsStack __cmake_original_verify_setting)
+  string(SUBSTRING "${__cmake_original_verify_setting}"
+    12 -1 __cmake_original_verify_setting
+  )
+  set(CMAKE_VERIFY_INTERFACE_HEADER_SETS ${__cmake_original_verify_setting})
+
+  # clear local variables to prevent leaking into the caller's scope
+  unset(__cmake_contentName)
+  unset(__cmake_contentNameLower)
+  unset(__cmake_contentNameUpper)
+  unset(__cmake_providerCommand)
+  unset(__cmake_original_verify_setting)
+endmacro()
--- a/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/c-api-linux-cpu.yml
@ -53,6 +53,8 @@ jobs:
        Context: tools/ci_build/github/linux/docker/inference/${{parameters.OnnxruntimeArch}}/default/cpu
        DockerBuildArgs: "--build-arg BUILD_UID=$( id -u ) --build-arg BASEIMAGE=${{parameters.BaseImage}}"
        Repository: onnxruntimecpubuildcentos7${{parameters.OnnxruntimeArch}}
+        ${{ if eq(parameters.OnnxruntimeArch, 'aarch64') }}:
+            UpdateDepsTxt: false

    - task: CmdLine@2
      inputs:
--- a/tools/ci_build/github/azure-pipelines/templates/download-deps.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/download-deps.yml
@ -0,0 +1,28 @@
+ 
+steps:
+# You can the information of the System CollectionIds by visiting https://app.vssps.visualstudio.com/_apis/accounts?memberId=me, 
+# Then search the uuid below in the result.
+
+# The public ADO project
+- ${{ if eq(variables['System.CollectionId'], 'f3ad12f2-e480-4533-baf2-635c95467d29') }}:
+  - task: DownloadPackage@1
+    displayName: 'Download ONNX Runtime Build Time Deps'
+    inputs:
+      packageType: upack
+      feed: '/7424c8e4-5c62-490e-95c4-79446f31017c'
+      definition: '517c4f6f-5437-4392-a70d-4f15ec5be2f0'
+      version: 1.0.3
+      downloadPath: $(Build.BinariesDirectory)/deps
+
+# The private ADO project
+- ${{ if eq(variables['System.CollectionId'], 'bc038106-a83b-4dab-9dd3-5a41bc58f34c') }}:
+  - task: DownloadPackage@1
+    displayName: 'Download ONNX Runtime Build Time Deps'
+    inputs:
+      packageType: upack
+      feed: '/4c7631f5-24c0-4307-8822-1aa8f180c325'
+      definition: 'fd9dd5ad-b73e-4678-890e-edcf680dbc1a'
+      version: 1.0.3
+      downloadPath: $(Build.BinariesDirectory)/deps
+
+# You can add more ADO accounts at here.
--- a/tools/ci_build/github/azure-pipelines/templates/get-docker-image-steps.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/get-docker-image-steps.yml
@ -16,11 +16,19 @@ parameters:
 - name: ScriptName
  type: string
  default: "tools/ci_build/get_docker_image.py" 
-
+- name: UpdateDepsTxt
+  type: boolean
+  default: true
+  
 steps:
+
+- ${{ if eq(parameters.UpdateDepsTxt, true)}}:
+  - template: download-deps.yml  
+
 - ${{ if contains(parameters.Dockerfile, 'manylinux') }}:
    - checkout: manylinux
    - script: |
+        set -e -x 
        mv manylinux onnxruntime
        mv onnxruntime ..
        cd ..
@ -49,3 +57,12 @@ steps:
        --docker-build-args "${{ parameters.DockerBuildArgs }}" \
        --repository "${{ parameters.Repository }}"
    displayName: "Get ${{ parameters.Repository }} image for ${{ parameters.Dockerfile }}"
+
+- ${{ if and(eq(parameters.UpdateDepsTxt, true), or(eq(variables['System.CollectionId'], 'f3ad12f2-e480-4533-baf2-635c95467d29'),eq(variables['System.CollectionId'], 'bc038106-a83b-4dab-9dd3-5a41bc58f34c'))) }}:
+  - task: PythonScript@0
+    displayName: 'Update deps.txt'
+    inputs:
+      scriptPath: $(Build.SourcesDirectory)/tools/ci_build/replace_urls_in_deps.py
+      arguments: --new_dir /build/deps
+      workingDirectory: $(Build.BinariesDirectory)
+      pythonInterpreter: /usr/bin/python3
--- a/tools/ci_build/github/azure-pipelines/templates/py-linux-ubuntu.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/py-linux-ubuntu.yml
@ -30,6 +30,8 @@ jobs:
        Context: tools/ci_build/github/linux/docker
        DockerBuildArgs: "--build-arg BUILD_UID=$( id -u ) --build-arg BASEIMAGE=${{ parameters.base_image }} --build-arg PLATFORM=${{ parameters.arch }}"
        Repository: onnxruntimecpubuildubuntu20python${{ parameters.arch }}
+        ${{ if eq(parameters.arch, 'aarch64') }}:
+            UpdateDepsTxt: false

    - task: CmdLine@2
      displayName: 'Build Python Wheel'
--- a/tools/ci_build/github/azure-pipelines/templates/py-linux.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/py-linux.yml
@ -39,6 +39,8 @@ jobs:
        Context: tools/ci_build/github/linux/docker/inference/x64/python/cpu
        DockerBuildArgs: "--build-arg BUILD_UID=$( id -u ) --build-arg BASEIMAGE=${{ parameters.base_image }} --build-arg PLATFORM=${{ parameters.arch }} --build-arg PREPEND_PATH=${{ parameters.prepend_path }} --build-arg LD_LIBRARY_PATH_ARG=${{ parameters.ld_library_path_arg }} --build-arg DEVTOOLSET_ROOTPATH=${{ parameters.devtoolset_rootpath }}"
        Repository: onnxruntimecpubuilpython${{ parameters.arch }}
+        ${{ if eq(parameters.arch, 'aarch64') }}:
+            UpdateDepsTxt: false

    - task: CmdLine@2
      displayName: 'Build Python Wheel'
--- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml
@ -131,6 +131,15 @@ stages:
        workingDirectory: '$(Build.BinariesDirectory)'
        displayName: 'Install python modules'

+      - template: download-deps.yml
+      
+      - task: PythonScript@0
+        displayName: 'Update deps.txt'
+        inputs:
+          scriptPath: $(Build.SourcesDirectory)/tools/ci_build/replace_urls_in_deps.py
+          arguments: --new_dir $(Build.BinariesDirectory)/deps
+          workingDirectory: $(Build.BinariesDirectory)
+
      - powershell: |
          $Env:USE_MSVC_STATIC_RUNTIME=1
          $Env:ONNX_ML=1
--- a/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml
@ -52,6 +52,15 @@ jobs:
        workingDirectory: '$(Build.BinariesDirectory)'
        displayName: 'Install python modules'

+      - template: download-deps.yml
+      
+      - task: PythonScript@0
+        displayName: 'Update deps.txt'
+        inputs:
+          scriptPath: $(Build.SourcesDirectory)/tools/ci_build/replace_urls_in_deps.py
+          arguments: --new_dir $(Build.BinariesDirectory)/deps
+          workingDirectory: $(Build.BinariesDirectory)
+
      - powershell: |
          $Env:USE_MSVC_STATIC_RUNTIME=1
          $Env:ONNX_ML=1
--- a/tools/ci_build/github/azure-pipelines/templates/win-ci-vs-2019.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/win-ci-vs-2019.yml
@ -71,6 +71,15 @@ jobs:
      addToPath: true
      architecture: ${{ parameters.buildArch }}

+  - template: download-deps.yml
+      
+  - task: PythonScript@0
+    displayName: 'Update deps.txt'
+    inputs:
+      scriptPath: $(Build.SourcesDirectory)/tools/ci_build/replace_urls_in_deps.py
+      arguments: --new_dir $(Build.BinariesDirectory)/deps
+      workingDirectory: $(Build.BinariesDirectory)
+
  - task: NodeTool@0
    inputs:
      versionSpec: '16.x'
--- a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml
@ -84,6 +84,16 @@ jobs:
       python -m pip install -q setuptools wheel numpy
      workingDirectory: '$(Build.BinariesDirectory)'
      displayName: 'Install python modules' 
+
+    - template: download-deps.yml
+      
+    - task: PythonScript@0
+      displayName: 'Update deps.txt'
+      inputs:
+        scriptPath: $(Build.SourcesDirectory)/tools/ci_build/replace_urls_in_deps.py
+        arguments: --new_dir $(Build.BinariesDirectory)/deps
+        workingDirectory: $(Build.BinariesDirectory)
+
    - powershell: |
       $Env:USE_MSVC_STATIC_RUNTIME=1
       $Env:ONNX_ML=1
--- a/tools/ci_build/replace_urls_in_deps.py
+++ b/tools/ci_build/replace_urls_in_deps.py
@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+# This file replaces https URLs in deps.txt to local file paths. It runs after we download the dependencies from Azure
+# DevOps Artifacts
+
+import argparse
+import csv
+import os
+from dataclasses import dataclass
+from pathlib import Path
+
+
+@dataclass(frozen=True)
+class Dep:
+    name: str
+    url: str
+    sha1_hash: str
+
+
+def parse_arguments():
+    parser = argparse.ArgumentParser()
+    # The directory that contains downloaded zip files
+    parser.add_argument("--new_dir", required=False)
+
+    return parser.parse_args()
+
+
+def main():
+    SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
+    REPO_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, "..", ".."))
+
+    args = parse_arguments()
+    new_dir = None
+    if args.new_dir:
+        new_dir = Path(args.new_dir)
+    else:
+        BUILD_BINARIESDIRECTORY = os.environ.get("BUILD_BINARIESDIRECTORY")
+        if BUILD_BINARIESDIRECTORY is None:
+            raise NameError("Please specify --new_dir or set the env var BUILD_BINARIESDIRECTORY")
+        new_dir = Path(BUILD_BINARIESDIRECTORY) / "deps"
+
+    # Here we intentionally do not check if new_dir exists, because it might be used in a docker container instead.
+
+    deps = []
+
+    csv_file_path = Path(REPO_DIR) / "cmake" / "deps.txt"
+
+    # Read the whole file into memory first
+    with csv_file_path.open("r", encoding="utf-8") as f:
+        depfile_reader = csv.reader(f, delimiter=";")
+        for row in depfile_reader:
+            if len(row) != 3:
+                continue
+            # Lines start with "#" are comments
+            if row[0].startswith("#"):
+                continue
+            deps.append(Dep(row[0], row[1], row[2]))
+
+    # Write updated content back
+    with csv_file_path.open("w", newline="", encoding="utf-8") as f:
+        depfile_writer = csv.writer(f, delimiter=";")
+        for dep in deps:
+            if dep.url.startswith("https://"):
+                new_url = new_dir / dep.url[8:]
+                depfile_writer.writerow([dep.name, new_url.as_posix(), dep.sha1_hash])
+            else:
+                # Write the original thing back
+                depfile_writer.writerow([dep.name, dep.url, dep.sha1_hash])
+
+
+if __name__ == "__main__":
+    main()