Remove unused git submodules (#13830)

.gitmodules

@@ -1,30 +1,12 @@
 [submodule "cmake/external/protobuf"]
 	path = cmake/external/protobuf
 	url = https://github.com/protocolbuffers/protobuf.git
-[submodule "cmake/external/googletest"]
-	path = cmake/external/googletest
-	url = https://github.com/google/googletest.git
 [submodule "cmake/external/onnx"]
 	path = cmake/external/onnx
 	url = https://github.com/onnx/onnx.git
-[submodule "cmake/external/date"]
-	path = cmake/external/date
-	url = https://github.com/HowardHinnant/date.git
-[submodule "cmake/external/nsync"]
-	path = cmake/external/nsync
-	url = https://github.com/google/nsync.git
-[submodule "cmake/external/re2"]
-	path = cmake/external/re2
-	url = https://github.com/google/re2.git
 [submodule "cmake/external/eigen"]
 	path = cmake/external/eigen
 	url = https://gitlab.com/libeigen/eigen.git
-[submodule "cmake/external/tensorboard"]
-	path = cmake/external/tensorboard
-	url = https://github.com/tensorflow/tensorboard.git
-[submodule "cmake/external/wil"]
-	path = cmake/external/wil
-	url = https://github.com/microsoft/wil.git
 [submodule "cmake/external/libprotobuf-mutator"]
 	path = cmake/external/libprotobuf-mutator
 	url = https://github.com/google/libprotobuf-mutator.git
@@ -35,10 +17,3 @@
 [submodule "cmake/external/onnxruntime-extensions"]
 	path = cmake/external/onnxruntime-extensions
 	url = https://github.com/microsoft/onnxruntime-extensions.git
-[submodule "cmake/external/pytorch_cpuinfo"]
-	path = cmake/external/pytorch_cpuinfo
-	url = https://github.com/pytorch/cpuinfo.git
-[submodule "cmake/external/onnx-tensorrt"]
-	path = cmake/external/onnx-tensorrt
-	url = https://github.com/onnx/onnx-tensorrt.git
-	branch = 8.4-GA

cmake/CMakeLists.txt

@@ -584,15 +584,44 @@ if (onnxruntime_USE_CANN)
     list(APPEND ONNXRUNTIME_PROVIDER_NAMES cann)
 endif()
 
+if (onnxruntime_ENABLE_EAGER_MODE)
+  if (NOT onnxruntime_ENABLE_TRAINING OR NOT onnxruntime_ENABLE_PYTHON)
+    message(
+        FATAL_ERROR
+        "Option onnxruntime_ENABLE_EAGER_MODE can only be used when onnxruntime_ENABLE_TRAINING and onnxruntime_ENABLE_PYTHON are enabled")
+  endif()
+  set(ONNXRUNTIME_EAGER_CMAKE_FILE_NAME onnxruntime_eager)
+endif()
+
+if (onnxruntime_ENABLE_LAZY_TENSOR)
+  # To support LazyTensor, ORT needs to call Python function from C/C++.
+  # so onnxruntime_ENABLE_PYTHON is required.
+  if (NOT onnxruntime_ENABLE_TRAINING OR NOT onnxruntime_ENABLE_PYTHON)
+    message(
+        FATAL_ERROR
+        "Option onnxruntime_ENABLE_LAZY_TENSOR can only be set when onnxruntime_ENABLE_TRAINING and onnxruntime_ENABLE_PYTHON are enabled")
+  endif()
+  # TODO: In the future, we can compile LazyTensor into a standalone
+  # library target, onnxruntime_lazy_tensor, to make the buid
+  # cleaner.
+endif()
+
+
 function(onnxruntime_set_compile_flags target_name)
     if (CPUINFO_SUPPORTED)
       onnxruntime_add_include_to_target(${target_name} cpuinfo)
     endif()
+    if(onnxruntime_ENABLE_EAGER_MODE)
+      target_compile_definitions(${target_name} PRIVATE ENABLE_EAGER_MODE)
+    endif()
+    if(onnxruntime_ENABLE_LAZY_TENSOR)
+      target_compile_definitions(${target_name} PRIVATE ENABLE_LAZY_TENSOR)
+    endif()
     if (onnxruntime_ENABLE_CPU_FP16_OPS)
       target_compile_definitions(${target_name} PRIVATE ENABLE_CPU_FP16_TRAINING_OPS)
     endif()
-	if(onnxruntime_DISABLE_ABSEIL)
-	  target_compile_definitions(${target_name} PRIVATE DISABLE_ABSEIL)
+    if(onnxruntime_DISABLE_ABSEIL)
+      target_compile_definitions(${target_name} PRIVATE DISABLE_ABSEIL)
     endif()
     if(UNIX)
       target_compile_definitions(${target_name} PRIVATE PLATFORM_POSIX)
@@ -625,6 +654,16 @@ function(onnxruntime_set_compile_flags target_name)
       foreach(CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORY ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES})
         target_compile_options(${target_name} PRIVATE "$<$<NOT:$<COMPILE_LANGUAGE:CUDA>>:/external:I${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORY}>")
       endforeach()
+      foreach(onnxruntime_external_lib IN LISTS onnxruntime_EXTERNAL_LIBRARIES)
+        #TODO: the list contains cmake keywords like "debug". We should exclude them.
+        if(TARGET ${onnxruntime_external_lib})
+          get_target_property(onnxruntime_external_lib_include_dirs ${onnxruntime_external_lib} INTERFACE_INCLUDE_DIRECTORIES)
+          foreach(onnxruntime_external_lib_include_dir IN LISTS onnxruntime_external_lib_include_dirs)
+            # TODO: onnxruntime_external_lib_include_dir is a cmake generator expr. Very surprising it can fit in at here.
+            target_compile_options(${target_name} PRIVATE "$<$<NOT:$<COMPILE_LANGUAGE:CUDA>>:/external:I${onnxruntime_external_lib_include_dir}>")
+          endforeach()
+        endif()
+      endforeach()
       target_compile_definitions(${target_name} PUBLIC -DPLATFORM_WINDOWS -DNOGDI -DNOMINMAX -D_USE_MATH_DEFINES -D_SILENCE_ALL_CXX17_DEPRECATION_WARNINGS)
       if (onnxruntime_ENABLE_MEMLEAK_CHECKER)
         target_compile_definitions(${target_name} PUBLIC -DONNXRUNTIME_ENABLE_MEMLEAK_CHECK)
@@ -1356,7 +1395,7 @@ endif()
 
 #Now the 'onnxruntime_EXTERNAL_LIBRARIES' variable should be sealed. It will be used in onnxruntime.cmake which will be included in the next.
 #The order of the following targets matters. Right depends on left. If target A appears before target B. Then A.cmake can not use variables defined in B.cmake.
-set(ONNXRUNTIME_TARGETS onnxruntime_flatbuffers onnxruntime_common onnxruntime_mlas onnxruntime_graph onnxruntime_framework onnxruntime_util onnxruntime_providers onnxruntime_optimizer onnxruntime_session)
+set(ONNXRUNTIME_CMAKE_FILES onnxruntime_flatbuffers onnxruntime_common onnxruntime_mlas onnxruntime_graph onnxruntime_framework onnxruntime_util onnxruntime_providers onnxruntime_optimizer onnxruntime_session ${ONNXRUNTIME_EAGER_CMAKE_FILE_NAME})
 
 if (onnxruntime_USE_WINML)
   # WINML uses and depends on the shared lib.  Note:  You can build WINML without DML and you will get a
@@ -1366,92 +1405,67 @@ if (onnxruntime_USE_WINML)
         FATAL_ERROR
         "Option onnxruntime_USE_WINML can only be used when onnxruntime_BUILD_SHARED_LIB is also enabled")
   endif()
-  list(APPEND ONNXRUNTIME_TARGETS winml)
+  list(APPEND ONNXRUNTIME_CMAKE_FILES winml)
 endif() # if (onnxruntime_USE_WINML)
 
 if (onnxruntime_BUILD_SHARED_LIB OR onnxruntime_BUILD_APPLE_FRAMEWORK)
   if (onnxruntime_BUILD_APPLE_FRAMEWORK AND NOT ${CMAKE_SYSTEM_NAME} MATCHES "Darwin|iOS")
     message(FATAL_ERROR "onnxruntime_BUILD_APPLE_FRAMEWORK can only be enabled for macOS or iOS.")
   endif()
-  list(APPEND ONNXRUNTIME_TARGETS onnxruntime)
+  list(APPEND ONNXRUNTIME_CMAKE_FILES onnxruntime)
 endif()
 
 if (onnxruntime_BUILD_JAVA)
   message(STATUS "Java Build is enabled")
-  list(APPEND ONNXRUNTIME_TARGETS onnxruntime_java)
+  list(APPEND ONNXRUNTIME_CMAKE_FILES onnxruntime_java)
 endif()
 
 if (onnxruntime_BUILD_NODEJS)
   message(STATUS "Node.js Build is enabled")
-  list(APPEND ONNXRUNTIME_TARGETS onnxruntime_nodejs)
+  list(APPEND ONNXRUNTIME_CMAKE_FILES onnxruntime_nodejs)
 endif()
 
 if (onnxruntime_ENABLE_PYTHON)
   message(STATUS "Python Build is enabled")
-  list(APPEND ONNXRUNTIME_TARGETS onnxruntime_python)
+  list(APPEND ONNXRUNTIME_CMAKE_FILES onnxruntime_python)
 endif()
 
 if (onnxruntime_BUILD_OBJC)
   message(STATUS "Objective-C Build is enabled")
-  list(APPEND ONNXRUNTIME_TARGETS onnxruntime_objectivec)
+  list(APPEND ONNXRUNTIME_CMAKE_FILES onnxruntime_objectivec)
 endif()
 
 if (onnxruntime_BUILD_UNIT_TESTS)
-  list(APPEND ONNXRUNTIME_TARGETS onnxruntime_unittests)
+  list(APPEND ONNXRUNTIME_CMAKE_FILES onnxruntime_unittests)
 endif()
 
 if (onnxruntime_BUILD_WINML_TESTS)
-  list(APPEND ONNXRUNTIME_TARGETS winml_unittests)
+  list(APPEND ONNXRUNTIME_CMAKE_FILES winml_unittests)
 endif()
 
 # onnxruntime_training depends on onnxruntime_unittests since onnxruntime_training.cmake uses a variable `TEST_SRC_DIR`
 # that is defined in onnxruntime_unittests.cmake
 if (onnxruntime_ENABLE_TRAINING)
-  list(APPEND ONNXRUNTIME_TARGETS onnxruntime_training)
+  list(APPEND ONNXRUNTIME_CMAKE_FILES onnxruntime_training)
   if (onnxruntime_ENABLE_TRAINING_E2E_TESTS)
-    list(APPEND ONNXRUNTIME_TARGETS onnxruntime_training_e2e_tests)
+    list(APPEND ONNXRUNTIME_CMAKE_FILES onnxruntime_training_e2e_tests)
   endif()
 endif()
 
 if (onnxruntime_BUILD_CSHARP)
   message(STATUS "CSharp Build is enabled")
 #  set_property(GLOBAL PROPERTY VS_DOTNET_TARGET_FRAMEWORK_VERSION "netstandard2.0")
-  list(APPEND ONNXRUNTIME_TARGETS onnxruntime_csharp)
+  list(APPEND ONNXRUNTIME_CMAKE_FILES onnxruntime_csharp)
 endif()
 
 if (onnxruntime_BUILD_WEBASSEMBLY)
   message(STATUS "WebAssembly Build is enabled")
-  list(APPEND ONNXRUNTIME_TARGETS onnxruntime_webassembly)
+  list(APPEND ONNXRUNTIME_CMAKE_FILES onnxruntime_webassembly)
 endif()
 
 if(onnxruntime_BUILD_KERNEL_EXPLORER)
   message(STATUS "Kernel Explorer Build is enabled")
-  list(APPEND ONNXRUNTIME_TARGETS onnxruntime_kernel_explorer)
-endif()
-
-if (onnxruntime_ENABLE_EAGER_MODE)
-  if (NOT onnxruntime_ENABLE_TRAINING OR NOT onnxruntime_ENABLE_PYTHON)
-    message(
-        FATAL_ERROR
-        "Option onnxruntime_ENABLE_EAGER_MODE can only be used when onnxruntime_ENABLE_TRAINING and onnxruntime_ENABLE_PYTHON are enabled")
-  endif()
-  add_compile_definitions(ENABLE_EAGER_MODE)
-  list(APPEND ONNXRUNTIME_TARGETS onnxruntime_eager)
-endif()
-
-if (onnxruntime_ENABLE_LAZY_TENSOR)
-  # To support LazyTensor, ORT needs to call Python function from C/C++.
-  # so onnxruntime_ENABLE_PYTHON is required.
-  if (NOT onnxruntime_ENABLE_TRAINING OR NOT onnxruntime_ENABLE_PYTHON)
-    message(
-        FATAL_ERROR
-        "Option onnxruntime_ENABLE_LAZY_TENSOR can only be set when onnxruntime_ENABLE_TRAINING and onnxruntime_ENABLE_PYTHON are enabled")
-  endif()
-  add_compile_definitions(ENABLE_LAZY_TENSOR)
-  # TODO: In the future, we can compile LazyTensor into a standalone
-  # library target, onnxruntime_lazy_tensor, to make the buid
-  # cleaner.
-  #list(APPEND ONNXRUNTIME_TARGETS onnxruntime_lazy_tensor)
+  list(APPEND ONNXRUNTIME_CMAKE_FILES onnxruntime_kernel_explorer)
 endif()
 
 # When GDK_PLATFORM is set then WINAPI_FAMILY is defined in gdk_toolchain.cmake (along with other relevant flags/definitions).
@@ -1466,7 +1480,7 @@ if (WIN32 AND NOT GDK_PLATFORM)
   endif()
 endif()
 
-foreach(target_name ${ONNXRUNTIME_TARGETS})
+foreach(target_name ${ONNXRUNTIME_CMAKE_FILES})
   include(${target_name}.cmake)
 endforeach()
 if (UNIX)

cmake/deps.txt

@@ -24,6 +24,7 @@ microsoft_wil;https://github.com/microsoft/wil/archive/5f4caba4e7a9017816e47becd
 mimalloc;https://github.com/microsoft/mimalloc/archive/refs/tags/v2.0.3.zip;e4f37b93b2da78a5816c2495603a4188d316214b
 mp11;https://github.com/boostorg/mp11/archive/refs/tags/boost-1.79.0.zip;c8f04e378535ededbe5af52c8f969d2dedbe73d5
 onnx;https://github.com/onnx/onnx/archive/5a5f8a5935762397aa68429b5493084ff970f774.zip;edc8e1338c02f3ab222f3d803a24e17608c13895
+#Branch name: 8.4-GA
 onnx_tensorrt;https://github.com/onnx/onnx-tensorrt/archive/87c7a70688fd98fb355b8976f41425b40e4fe52f.zip;b97d112d9d6efa180c9b94e05268f2ff3294a534
 protobuf;https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.18.3.zip;b95bf7e9de9c2249b6c1f2ca556ace49999e90bd
 psimd;https://github.com/Maratyszcza/psimd/archive/072586a71b55b7f8c584153d223e95687148a900.zip;1f5454b01f06f9656b77e4a5e2e31d7422487013

cmake/external/date

@@ -1 +0,0 @@
-Subproject commit e7e1482087f58913b80a20b04d5c58d9d6d90155

cmake/external/googletest

@@ -1 +0,0 @@
-Subproject commit 58d77fa8070e8cec2dc1ed015d66b454c8d78850

cmake/external/nsync

@@ -1 +0,0 @@
-Subproject commit ac5489682760393fe21bd2a8e038b528442412a7

cmake/external/onnx-tensorrt

@@ -1 +0,0 @@
-Subproject commit 87c7a70688fd98fb355b8976f41425b40e4fe52f

cmake/external/onnxruntime_external_deps.cmake

@@ -135,9 +135,7 @@ FetchContent_Declare(
       URL ${DEP_URL_date}
       URL_HASH SHA1=${DEP_SHA1_date}
     )
-FetchContent_Populate(date)
-add_library(date_interface INTERFACE)
-target_include_directories(date_interface INTERFACE ${date_SOURCE_DIR}/include)
+onnxruntime_fetchcontent_makeavailable(date)
 
 
 
@@ -376,7 +374,7 @@ endif()
 #onnxruntime_EXTERNAL_LIBRARIES could contain onnx, onnx_proto,libprotobuf, cuda/cudnn,
 # dnnl/mklml, onnxruntime_codegen_tvm, tvm and pthread
 # pthread is always at the last
-set(onnxruntime_EXTERNAL_LIBRARIES ${onnxruntime_EXTERNAL_LIBRARIES_XNNPACK} WIL::WIL nlohmann_json::nlohmann_json onnx onnx_proto ${PROTOBUF_LIB} re2::re2 Boost::mp11 safeint_interface flatbuffers ${GSL_TARGET} ${ABSEIL_LIBS})
+set(onnxruntime_EXTERNAL_LIBRARIES ${onnxruntime_EXTERNAL_LIBRARIES_XNNPACK} WIL::WIL nlohmann_json::nlohmann_json onnx onnx_proto ${PROTOBUF_LIB} re2::re2 Boost::mp11 safeint_interface flatbuffers ${GSL_TARGET} ${ABSEIL_LIBS} date_interface)
 # The source code of onnx_proto is generated, we must build this lib first before starting to compile the other source code that uses ONNX protobuf types.
 # The other libs do not have the problem. All the sources are already there. We can compile them in any order.
 set(onnxruntime_EXTERNAL_DEPENDENCIES onnx_proto)

cmake/external/pytorch_cpuinfo

@@ -1 +0,0 @@
-Subproject commit 5916273f79a21551890fd3d56fc5375a78d1598d

cmake/external/re2

@@ -1 +0,0 @@
-Subproject commit 5723bb8950318135ed9cf4fc76bed988a087f536

cmake/external/tensorboard

@@ -1 +0,0 @@
-Subproject commit 373eb09e4c5d2b3cc2493f0949dc4be6b6a45e81

cmake/external/wil

@@ -1 +0,0 @@
-Subproject commit 5f4caba4e7a9017816e47becdd918fcc872039ba

cmake/onnxruntime_providers.cmake

@@ -1153,11 +1153,9 @@ if (onnxruntime_USE_MIGRAPHX)
   set(BUILD_LIBRARY_ONLY 1)
   add_definitions("-DONNX_ML=1")
   add_definitions("-DONNX_NAMESPACE=onnx")
-  # TODO: Remove the next line so that we can delete the git submodule
-  include_directories(${PROJECT_SOURCE_DIR}/external/protobuf ${PROJECT_SOURCE_DIR}/external/eigen)
+  include_directories(${protobuf_SOURCE_DIR} ${eigen_SOURCE_DIR})
   set(MIGRAPHX_ROOT ${onnxruntime_MIGRAPHX_HOME})
-  # TODO: Remove the next line so that we can delete the git submodule
-  include_directories(${PROJECT_SOURCE_DIR}/external/onnx)
+  include_directories(${onnx_SOURCE_DIR})
   set(OLD_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS})
   if ( CMAKE_COMPILER_IS_GNUCC )
     set(CMAKE_CXX_FLAGS  "${CMAKE_CXX_FLAGS} -Wno-unused-parameter -Wno-missing-field-initializers")

tools/ci_build/github/azure-pipelines/templates/web-browserstack-ci.yml

@@ -26,11 +26,7 @@ jobs:
     workingDirectory: '$(Build.SourcesDirectory)'
     displayName: 'Read commit SHA and checkout'
     condition: eq('${{ parameters.CommitOverride }}', 'true')
-  - script: |
-     git submodule sync -- cmake/external/onnx
-     git submodule update --init -- cmake/external/onnx
-    workingDirectory: '$(Build.SourcesDirectory)'
-    displayName: 'Checkout submodule onnx'
+
   - task: NodeTool@0
     inputs:
       versionSpec: '16.x'

tools/ci_build/github/linux/build_linux_arm64_python_package.sh

@@ -8,7 +8,7 @@ CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-st
 BUILD_DEVICE="CPU"
 BUILD_CONFIG="Release"
 PYTHON_EXES=("/opt/python/cp37-cp37m/bin/python3.7" "/opt/python/cp38-cp38/bin/python3.8" "/opt/python/cp39-cp39/bin/python3.9" "/opt/python/cp310-cp310/bin/python3.10")
-while getopts "d:" parameter_Option
+while getopts "d:p:" parameter_Option
 do case "${parameter_Option}"
 in
 #GPU or CPU.

tools/ci_build/github/linux/run_build.sh

@@ -50,12 +50,10 @@ else
             --cudnn_home /usr/local/cudnn-$_CUDNN_VERSION/cuda $BUILD_EXTR_PAR
     elif [[ $BUILD_DEVICE = "tensorrt"* ]]; then
         if [ $BUILD_DEVICE = "tensorrt-v7.1" ]; then
-            CUR_PWD=$(pwd)
-            cd $SCRIPT_DIR/../../../../cmake/external/onnx-tensorrt/
-            git remote update
-            git checkout 7.1
-            cd $CUR_PWD
-            COMMON_BUILD_ARGS=${COMMON_BUILD_ARGS/"--skip_submodule_sync"}
+            pushd .
+            cd $SCRIPT_DIR/../../../../cmake
+            sed -i "s/^onnx_tensorrt.*$/onnx_tensorrt;https:\/\/github.com\/onnx\/onnx-tensorrt\/archive\/refs\/tags\/release\/7.1.zip;e23bf76bbe4748c49951d6b401cf5e1006d86cce/g" deps.txt
+            popd
         fi
         _CUDNN_VERSION=$(echo $CUDNN_VERSION | cut -d. -f1-2)
         python3 $SCRIPT_DIR/../../build.py --build_dir /build \

tools/ci_build/github/windows/helpers.ps1

@@ -0,0 +1,461 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+enum CMakeBuildType{
+  Debug
+  Release
+  RelWithDebInfo
+  MinSizeRel
+}
+
+
+# The DownloadAndExtract function was copied from: https://github.com/dotnet/arcade/blob/main/eng/common/native/CommonLibrary.psm1
+<#
+.SYNOPSIS
+Get the name of a temporary folder under the native install directory
+#>
+function Get-TempDirectory {
+  #TODO: what if the env does not exist?
+  return $Env:AGENT_TEMPDIRECTORY
+}
+
+function Get-TempPathFilename {
+  [CmdletBinding(PositionalBinding=$false)]
+  Param (
+    [Parameter(Mandatory=$True)]
+    [string] $Path
+  )
+  $TempDir = Get-TempDirectory
+  $TempFilename = Split-Path $Path -leaf
+  $TempPath = Join-Path $TempDir $TempFilename
+  return $TempPath
+}
+
+<#
+.SYNOPSIS
+Unzip an archive
+.DESCRIPTION
+Powershell module to unzip an archive to a specified directory
+.PARAMETER ZipPath (Required)
+Path to archive to unzip
+.PARAMETER OutputDirectory (Required)
+Output directory for archive contents
+.PARAMETER Force
+Overwrite output directory contents if they already exist
+.NOTES
+- Returns True and does not perform an extraction if output directory already exists but Overwrite is not True.
+- Returns True if unzip operation is successful
+- Returns False if Overwrite is True and it is unable to remove contents of OutputDirectory
+- Returns False if unable to extract zip archive
+#>
+function Expand-Zip {
+  [CmdletBinding(PositionalBinding=$false)]
+  Param (
+    [Parameter(Mandatory=$True)]
+    [string] $ZipPath,
+    [Parameter(Mandatory=$True)]
+    [string] $OutputDirectory,
+    [switch] $Force
+  )
+
+  Write-Host "Extracting '$ZipPath' to '$OutputDirectory'"
+  try {
+    if ((Test-Path $OutputDirectory) -And (-Not $Force)) {
+      Write-Host "Directory '$OutputDirectory' already exists, skipping extract"
+      return $True
+    }
+    if (Test-Path $OutputDirectory) {
+      Write-Host "'Force' is 'True', but '$OutputDirectory' exists, removing directory"
+      Remove-Item -Path $OutputDirectory -Force -Recurse
+      if ($? -Eq $False) {
+        Write-Error "Unable to remove '$OutputDirectory'"
+        return $False
+      }
+    }
+
+    $TempOutputDirectory = Join-Path "$(Split-Path -Parent $OutputDirectory)" "$(Split-Path -Leaf $OutputDirectory).tmp"
+
+    if (Test-Path $TempOutputDirectory) {
+      Remove-Item $TempOutputDirectory -Force -Recurse
+    }
+    New-Item -Path $TempOutputDirectory -Force -ItemType "Directory" | Out-Null
+
+    Add-Type -assembly "system.io.compression.filesystem"
+    [io.compression.zipfile]::ExtractToDirectory("$ZipPath", "$TempOutputDirectory")
+    if ($? -Eq $False) {
+      Write-Error "Unable to extract '$ZipPath'"
+      return $False
+    }
+
+    Move-Item -Path $TempOutputDirectory -Destination $OutputDirectory
+  }
+  catch {
+    Write-Host $_
+    Write-Host $_.Exception
+
+    return $False
+  }
+  return $True
+}
+
+<#
+.SYNOPSIS
+Helper module to install an archive to a directory
+.DESCRIPTION
+Helper module to download and extract an archive to a specified directory
+.PARAMETER Uri
+Uri of artifact to download
+.PARAMETER InstallDirectory
+Directory to extract artifact contents to.
+.PARAMETER Force
+Force download / extraction if file or contents already exist. Default = False
+.PARAMETER DownloadRetries
+Total number of retry attempts. Default = 5
+.PARAMETER RetryWaitTimeInSeconds
+Wait time between retry attempts in seconds. Default = 30
+.NOTES
+Returns False if download or extraction fail, True otherwise
+#>
+function DownloadAndExtract {
+  [CmdletBinding(PositionalBinding=$false)]
+  Param (
+    [Parameter(Mandatory=$True)]
+    [string] $Uri,
+    [Parameter(Mandatory=$True)]
+    [string] $InstallDirectory,
+    [switch] $Force = $False,
+    [int] $DownloadRetries = 5,
+    [int] $RetryWaitTimeInSeconds = 30
+  )
+  # Define verbose switch if undefined
+  $Verbose = $VerbosePreference -Eq "Continue"
+
+  $TempToolPath = Get-TempPathFilename -Path $Uri
+  Write-Host "TempToolPath=$TempToolPath"
+  # Download native tool
+  $DownloadStatus = Get-File -Uri $Uri `
+                                           -Path $TempToolPath `
+                                           -DownloadRetries $DownloadRetries `
+                                           -RetryWaitTimeInSeconds $RetryWaitTimeInSeconds `
+                                           -Force:$Force `
+                                           -Verbose:$Verbose
+
+  if ($DownloadStatus -Eq $False) {
+    Write-Error "Download failed from $Uri"
+    return $False
+  }
+
+
+  # Extract native tool
+  $UnzipStatus = Expand-Zip -ZipPath $TempToolPath `
+                                          -OutputDirectory $InstallDirectory `
+                                          -Force:$Force `
+                                          -Verbose:$Verbose
+  if ($UnzipStatus -Eq $False) {
+    # Retry Download one more time with Force=true
+    $DownloadRetryStatus = Get-File -Uri $Uri `
+                                             -Path $TempToolPath `
+                                             -DownloadRetries 1 `
+                                             -RetryWaitTimeInSeconds $RetryWaitTimeInSeconds `
+                                             -Force:$True `
+                                             -Verbose:$Verbose
+
+    if ($DownloadRetryStatus -Eq $False) {
+      Write-Error "Last attempt of download failed as well"
+      return $False
+    }
+
+    # Retry unzip again one more time with Force=true
+    $UnzipRetryStatus = Expand-Zip -ZipPath $TempToolPath `
+                                            -OutputDirectory $InstallDirectory `
+                                            -Force:$True `
+                                            -Verbose:$Verbose
+    if ($UnzipRetryStatus -Eq $False)
+    {
+      Write-Error "Last attempt of unzip failed as well"
+      # Clean up partial zips and extracts
+      if (Test-Path $TempToolPath) {
+        Remove-Item $TempToolPath -Force
+      }
+      if (Test-Path $InstallDirectory) {
+        Remove-Item $InstallDirectory -Force -Recurse
+      }
+      return $False
+    }
+  }
+
+  return $True
+}
+
+<#
+.SYNOPSIS
+Download a file, retry on failure
+.DESCRIPTION
+Download specified file and retry if attempt fails
+.PARAMETER Uri
+Uri of file to download. If Uri is a local path, the file will be copied instead of downloaded
+.PARAMETER Path
+Path to download or copy uri file to
+.PARAMETER Force
+Overwrite existing file if present. Default = False
+.PARAMETER DownloadRetries
+Total number of retry attempts. Default = 5
+.PARAMETER RetryWaitTimeInSeconds
+Wait time between retry attempts in seconds Default = 30
+#>
+function Get-File {
+  [CmdletBinding(PositionalBinding=$false)]
+  Param (
+    [Parameter(Mandatory=$True)]
+    [string] $Uri,
+    [Parameter(Mandatory=$True)]
+    [string] $Path,
+    [int] $DownloadRetries = 5,
+    [int] $RetryWaitTimeInSeconds = 30,
+    [switch] $Force = $False
+  )
+  $Attempt = 0
+
+  if ($Force) {
+    if (Test-Path $Path) {
+      Remove-Item $Path -Force
+    }
+  }
+  if (Test-Path $Path) {
+    Write-Host "File '$Path' already exists, skipping download"
+    return $True
+  }
+
+  $DownloadDirectory = Split-Path -ErrorAction Ignore -Path "$Path" -Parent
+  if (-Not (Test-Path $DownloadDirectory)) {
+    New-Item -path $DownloadDirectory -force -itemType "Directory" | Out-Null
+  }
+
+  $TempPath = "$Path.tmp"
+  if (Test-Path -IsValid -Path $Uri) {
+    Write-Verbose "'$Uri' is a file path, copying temporarily to '$TempPath'"
+    Copy-Item -Path $Uri -Destination $TempPath
+    Write-Verbose "Moving temporary file to '$Path'"
+    Move-Item -Path $TempPath -Destination $Path
+    return $?
+  }
+  else {
+    # Don't display the console progress UI - it's a huge perf hit
+    $ProgressPreference = 'SilentlyContinue'
+    while($Attempt -Lt $DownloadRetries)
+    {
+      try {
+        Invoke-WebRequest -UseBasicParsing -Uri $Uri -OutFile $TempPath
+        Write-Verbose "Downloaded to temporary location '$TempPath'"
+        Move-Item -Path $TempPath -Destination $Path
+        Write-Verbose "Moved temporary file to '$Path'"
+        return $True
+      }
+      catch {
+        $Attempt++
+        if ($Attempt -Lt $DownloadRetries) {
+          $AttemptsLeft = $DownloadRetries - $Attempt
+          Write-Warning "Download failed, $AttemptsLeft attempts remaining, will retry in $RetryWaitTimeInSeconds seconds"
+          Start-Sleep -Seconds $RetryWaitTimeInSeconds
+        }
+        else {
+          Write-Error $_
+          Write-Error $_.Exception
+        }
+      }
+    }
+  }
+
+  return $False
+}
+
+<#
+    .Description
+    The Get-DownloadURL function returns the download URL of a external dependency. The URL might be local file path or
+     a remote HTTPS URL.
+
+    .PARAMETER name
+    The name of the dependency, should present in the first column of ONNX Runtime's deps.txt.
+
+    .PARAMETER src_root
+    The full path of ONNX Runtime's top level source diretory for locating deps.txt.
+#>
+
+function Get-DownloadURL {
+    param (
+        [Parameter(Mandatory)][string]$name,
+        [Parameter(Mandatory)][string]$src_root
+    )
+    $entry = Import-Csv -Path "$src_root\cmake\deps.txt"  -Delimiter ';'  -Header name,url,hash -Encoding UTF8 | Where-Object name -eq $name
+    return $entry.url
+}
+
+<#
+    .Description
+    The Install-Pybind function installs pybind11 headers, which are needed for building ONNX from source.
+
+    .PARAMETER cmake_path
+    The full path of cmake.exe
+
+    .PARAMETER src_root
+    The full path of ONNX Runtime's top level source diretory for locating deps.txt.
+
+    .PARAMETER build_config
+    The value of CMAKE_BUILD_TYPE, can be Debug, Release, RelWithDebInfo or MinSizeRel.
+#>
+
+function Install-Pybind {
+
+    param (
+        [Parameter(Mandatory)][string]$cmake_path,
+        [Parameter(Mandatory)][string]$src_root,
+        [Parameter(Mandatory)][CMakeBuildType]$build_config,
+        [Parameter(Mandatory)][string[]]$cmake_extra_args
+    )
+
+    pushd .
+
+    $url=Get-DownloadURL -name pybind11 -src_root $src_root
+    Write-Host "Downloading pybind11 from $url"
+    $temp_dir = Get-TempDirectory
+    $pybind_src_dir = Join-Path $temp_dir "pybind"
+    $download_finished = DownloadAndExtract -Uri $url -InstallDirectory $pybind_src_dir -Force
+    if(-Not $download_finished){
+        Write-Host -Object "Download failed"
+        exit 1
+    }
+    cd $pybind_src_dir
+    cd *
+    mkdir build
+    cd build
+    [string[]]$cmake_args = "..", "-DCMAKE_INSTALL_PREFIX=$install_prefix", "-DBUILD_TESTING=OFF"
+    $cmake_args += $cmake_extra_args
+    $p = Start-Process -FilePath $cmake_path -ArgumentList $cmake_args -NoNewWindow -Wait -PassThru
+    $exitCode = $p.ExitCode
+    if ($exitCode -ne 0) {
+        Write-Host -Object "CMake command failed. Exitcode: $exitCode"
+        exit $exitCode
+    }
+    $cmake_args = "--build", ".",  "--parallel", "--config", $build_config, "--target", "INSTALL"
+    $p = Start-Process -FilePath $cmake_path -ArgumentList $cmake_args -NoNewWindow -Wait -PassThru
+    $exitCode = $p.ExitCode
+    if ($exitCode -ne 0) {
+        Write-Host -Object "CMake command failed. Exitcode: $exitCode"
+        exit $exitCode
+    }
+    popd
+}
+
+<#
+    .Description
+    The Install-ONNX function installs ONNX python package from source and also the python packages that it depends on.
+    This script will build protobuf C/C++ lib/exe for the CPU arch of the current build machine, because we need to run
+    protoc.exe on this machine.
+
+    .PARAMETER cmake_path
+    The full path of cmake.exe
+
+    .PARAMETER src_root
+    The full path of ONNX Runtime's top level source diretory
+
+    .PARAMETER build_config
+    The value of CMAKE_BUILD_TYPE, can be Debug, Release, RelWithDebInfo or MinSizeRel.
+#>
+function Install-Protobuf {
+
+    param (
+        [Parameter(Mandatory)][string]$cmake_path,
+        [Parameter(Mandatory)][string]$src_root,
+        [Parameter(Mandatory)][CMakeBuildType]$build_config,
+        [Parameter(Mandatory)][string[]]$cmake_extra_args
+    )
+
+    pushd .
+    $url=Get-DownloadURL -name protobuf -src_root $src_root
+    $temp_dir = Get-TempDirectory
+    $protobuf_src_dir = Join-Path $temp_dir "protobuf"
+    $download_finished = DownloadAndExtract -Uri $url -InstallDirectory $protobuf_src_dir -Force
+    if(-Not $download_finished){
+        exit 1
+    }
+    cd $protobuf_src_dir
+    cd *
+    Get-Content $src_root\cmake\patches\protobuf\protobuf_cmake.patch | &'C:\Program Files\Git\usr\bin\patch.exe' --ignore-whitespace -p1
+
+    [string[]]$cmake_args = "cmake", "-DCMAKE_BUILD_TYPE=$build_config", "-Dprotobuf_BUILD_TESTS=OFF", "-DBUILD_SHARED_LIBS=OFF", "-DCMAKE_PREFIX_PATH=$install_prefix",  "-DCMAKE_INSTALL_PREFIX=$install_prefix", "-Dprotobuf_MSVC_STATIC_RUNTIME=OFF"
+    $cmake_args += $cmake_extra_args
+
+    $p = Start-Process -FilePath $cmake_path -ArgumentList $cmake_args -NoNewWindow -Wait -PassThru
+    $exitCode = $p.ExitCode
+    if ($exitCode -ne 0) {
+        Write-Host -Object "CMake command failed. Exitcode: $exitCode"
+        exit $exitCode
+    }
+    $cmake_args = "--build", ".",  "--parallel", "--config", $build_config, "--target", "INSTALL"
+    $p = Start-Process -FilePath $cmake_path -ArgumentList $cmake_args -NoNewWindow -Wait -PassThru
+    $exitCode = $p.ExitCode
+    if ($exitCode -ne 0) {
+        Write-Host -Object "CMake command failed. Exitcode: $exitCode"
+        exit $exitCode
+    }
+    popd
+}
+
+<#
+    .Description
+    The Install-ONNX function installs ONNX python package from source and also the python packages that it depends on.
+    protoc.exe must exist in the PATH.
+#>
+function Install-ONNX {
+
+    param (
+        [Parameter(Mandatory)][CMakeBuildType]$build_config,
+        [Parameter(Mandatory)][string]$src_root,
+        [Parameter(Mandatory)][string]$protobuf_version
+    )
+
+    pushd .
+
+    Write-Host "Installing python packages..."
+    $p = Start-Process -NoNewWindow -Wait -PassThru -FilePath "python.exe" -ArgumentList "-m", "pip", "install", "--disable-pip-version-check", "setuptools", "wheel", "numpy", "protobuf==$protobuf_version"
+    $exitCode = $p.ExitCode
+    if ($exitCode -ne 0) {
+        Write-Host -Object "Install dependent python wheels failed. Exitcode: $exitCode"
+        exit $exitCode
+    }
+
+    $url=Get-DownloadURL -name onnx -src_root $src_root
+    $temp_dir = Get-TempDirectory
+    $onnx_src_dir = Join-Path $temp_dir "onnx"
+    $download_finished = DownloadAndExtract -Uri $url -InstallDirectory $onnx_src_dir -Force
+    if(-Not $download_finished){
+        exit 1
+    }
+    cd $onnx_src_dir
+    cd *
+    $Env:ONNX_ML=1
+    if($build_config -eq 'Debug'){
+       $Env:DEBUG='1'
+    }
+    $Env:CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=OFF -DProtobuf_USE_STATIC_LIBS=ON -DONNX_USE_LITE_PROTO=OFF"
+
+    $p = Start-Process -NoNewWindow -Wait -PassThru -FilePath "python.exe" -ArgumentList "setup.py", "bdist_wheel"
+    $exitCode = $p.ExitCode
+    if ($exitCode -ne 0) {
+        Write-Host -Object "Generate wheel file failed. Exitcode: $exitCode"
+        exit $exitCode
+    }
+    Write-Host "Uninstalling onnx and ignore errors if there is any..."
+    python -m pip uninstall -y onnx -qq
+    Write-Host "Installing the newly built ONNX python package"
+    Get-ChildItem -Path dist/*.whl | foreach {
+        $p = Start-Process -NoNewWindow -Wait -PassThru -FilePath "python.exe" -ArgumentList "-m", "pip", "--disable-pip-version-check", "install", "--upgrade", $_.fullname
+        $exitCode = $p.ExitCode
+        if ($exitCode -ne 0) {
+          Write-Host -Object "Install wheel file failed. Exitcode: $exitCode"
+          exit $exitCode
+        }
+    }
+    Write-Host "Finished installing onnx"
+    popd
+}

tools/ci_build/github/windows/install_third_party_deps.ps1

@@ -1,60 +1,65 @@
 # Copyright (c) Microsoft Corporation. All rights reserved.
 # Licensed under the MIT License.
 
+# This script depends on python.exe, cmake.exe and Visual C++ spectre-mitigated libs.
+# Please setup AGENT_TEMPDIRECTORY env variable before running this script
+
  param (
     [string]$cpu_arch = "x64",
-	[string]$build_config = "RelWithDebInfo",
-	[string]$install_prefix
+    [string]$build_config = "RelWithDebInfo",
+    [string]$install_prefix = "."
  )
- 
+
+. "$PSScriptRoot\helpers.ps1"
+
+$ort_src_root = (Get-Item $PSScriptRoot).parent.parent.parent.parent.FullName
+
+Write-Host "ONNX Runtime src root: $ort_src_root"
+
 $ErrorActionPreference = "Stop"
 
-$env:Path = "$install_prefix\bin;C:\Program Files\7-Zip;" + $env:Path
+$Env:Path = "$install_prefix\bin;" + $env:Path
+$Env:MSBUILDDISABLENODEREUSE=1
 
 New-Item -Path "$install_prefix" -ItemType Directory -Force
 
-$cmake_extra_args=@()
+# Setup compile flags
+$compile_flags = '/MP /guard:cf /Qspectre /DWIN32 /D_WINDOWS /DWINVER=0x0601 /D_WIN32_WINNT=0x0601 /DNTDDI_VERSION=0x06010000 /W3 '
+$linker_flags=@('/guard:cf')
+
+if($build_config -eq 'Release'){
+  $compile_flags += "/O2", "/Ob2", "/DNDEBUG", "/Gw", "/GL"
+} elseif($build_config -eq 'RelWithDebInfo'){
+  $compile_flags += "/Zi", "/O2", "/Ob1", "/DNDEBUG", "/Gw", "/GL"
+} elseif($build_config -eq 'Debug'){
+  $compile_flags += "/Zi", "/Ob0", "/Od", "/RTC1"
+} elseif($build_config -eq 'MinSizeRel'){
+  $compile_flags += "/O1", "/Ob1", "/DNDEBUG", "/Gw", "/GL"
+}
+# cmake args that applies to every 3rd-party library
+[string[]]$cmake_extra_args="-DCMAKE_CXX_STANDARD=17 `"-DCMAKE_CXX_FLAGS=$compile_flags /EHsc`" ", "`"-DCMAKE_C_FLAGS=$compile_flags`"", "--compile-no-warning-as-error", "--fresh", "-Wno-dev"
 if($cpu_arch -eq 'x86'){
-  Write-Host "Build for x86"
-  $cmake_extra_args="-A", "Win32", "-T", "host=x64"
+  $cmake_extra_args +=  "-A", "Win32", "-T", "host=x64"
+  $linker_flags += '/machine:x86'
+} elseif($cpu_arch -eq 'x64') {
+  $linker_flags += '/machine:x64'
 } else {
-  Write-Host "Build for $cpu_arch"
+  throw "$cpu_arch is not supported"
 }
 
-$url='https://github.com/pybind/pybind11/archive/refs/tags/v2.10.1.zip'
-Write-Host "Downloading pybind11 from $url"
-Invoke-WebRequest -Uri $url -OutFile pybind11.zip
-7z x pybind11.zip
-cd pybind11-2.10.1
-mkdir build
-cd build
 
-cmake .. "-DCMAKE_INSTALL_PREFIX=$install_prefix" -DBUILD_TESTING=OFF $cmake_extra_args
-cmake --build .  --parallel --config $build_config --target INSTALL
-cd ../..
+$cmake_extra_args += "-DCMAKE_EXE_LINKER_FLAGS=`"$linker_flags`""
 
+# Find the full path of cmake.exe
+$cmake_command = Get-Command -CommandType Application cmake
+$cmake_path = $cmake_command.Path
+
+Install-Pybind -cmake_path $cmake_path -src_root $ort_src_root -build_config $build_config  -cmake_extra_args $cmake_extra_args
+
+Install-Protobuf -cmake_path $cmake_path -src_root $ort_src_root -build_config $build_config -cmake_extra_args $cmake_extra_args
+
+# TODO: parse it from deps.txt
 $protobuf_version="3.18.3"
-$url="https://github.com/protocolbuffers/protobuf/releases/download/v$protobuf_version/protobuf-cpp-$protobuf_version.zip"
-Write-Host "Downloading protobuf from $url"
-Invoke-WebRequest -Uri $url -OutFile protobuf_src.zip
-7z x protobuf_src.zip
-cd protobuf-$protobuf_version
-Get-Content $Env:BUILD_SOURCESDIRECTORY\cmake\patches\protobuf\protobuf_cmake.patch | &'C:\Program Files\Git\usr\bin\patch.exe' --binary --ignore-whitespace -p1
-cmake cmake  -DCMAKE_BUILD_TYPE=$build_config -Dprotobuf_BUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF "-DCMAKE_PREFIX_PATH=$install_prefix"  "-DCMAKE_INSTALL_PREFIX=$install_prefix" -Dprotobuf_MSVC_STATIC_RUNTIME=OFF $cmake_extra_args
-cmake --build .  --parallel --config $build_config --target INSTALL
-cd ..
-python -m pip install -q setuptools wheel numpy protobuf==$protobuf_version pybind11
-$onnx_commit_id="5a5f8a5935762397aa68429b5493084ff970f774"
-$url="https://github.com/onnx/onnx/archive/$onnx_commit_id.zip"
-Write-Host "Downloading onnx from $url"
-Invoke-WebRequest -Uri $url -OutFile onnx.zip
-7z x onnx.zip
-cd "onnx-$onnx_commit_id"
-$Env:ONNX_ML=1
-if($build_config -eq 'Debug'){
-  $Env:DEBUG='1'
-}
-$Env:CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=OFF -DProtobuf_USE_STATIC_LIBS=ON -DONNX_USE_LITE_PROTO=OFF"
-python setup.py bdist_wheel
-python -m pip uninstall -y onnx -qq
-Get-ChildItem -Path dist/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname}
+
+# ONNX doesn't allow us to specify CMake's path
+Install-ONNX -build_config $build_config -src_root $ort_src_root -protobuf_version $protobuf_version