diff --git a/.gitmodules b/.gitmodules index f8163e4be8..8e4217162b 100644 --- a/.gitmodules +++ b/.gitmodules @@ -1,30 +1,12 @@ [submodule "cmake/external/protobuf"] path = cmake/external/protobuf url = https://github.com/protocolbuffers/protobuf.git -[submodule "cmake/external/googletest"] - path = cmake/external/googletest - url = https://github.com/google/googletest.git [submodule "cmake/external/onnx"] path = cmake/external/onnx url = https://github.com/onnx/onnx.git -[submodule "cmake/external/date"] - path = cmake/external/date - url = https://github.com/HowardHinnant/date.git -[submodule "cmake/external/nsync"] - path = cmake/external/nsync - url = https://github.com/google/nsync.git -[submodule "cmake/external/re2"] - path = cmake/external/re2 - url = https://github.com/google/re2.git [submodule "cmake/external/eigen"] path = cmake/external/eigen url = https://gitlab.com/libeigen/eigen.git -[submodule "cmake/external/tensorboard"] - path = cmake/external/tensorboard - url = https://github.com/tensorflow/tensorboard.git -[submodule "cmake/external/wil"] - path = cmake/external/wil - url = https://github.com/microsoft/wil.git [submodule "cmake/external/libprotobuf-mutator"] path = cmake/external/libprotobuf-mutator url = https://github.com/google/libprotobuf-mutator.git @@ -35,10 +17,3 @@ [submodule "cmake/external/onnxruntime-extensions"] path = cmake/external/onnxruntime-extensions url = https://github.com/microsoft/onnxruntime-extensions.git -[submodule "cmake/external/pytorch_cpuinfo"] - path = cmake/external/pytorch_cpuinfo - url = https://github.com/pytorch/cpuinfo.git -[submodule "cmake/external/onnx-tensorrt"] - path = cmake/external/onnx-tensorrt - url = https://github.com/onnx/onnx-tensorrt.git - branch = 8.4-GA diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index 00964f7b1d..cabbf24daf 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -584,15 +584,44 @@ if (onnxruntime_USE_CANN) list(APPEND ONNXRUNTIME_PROVIDER_NAMES cann) endif() +if (onnxruntime_ENABLE_EAGER_MODE) + if (NOT onnxruntime_ENABLE_TRAINING OR NOT onnxruntime_ENABLE_PYTHON) + message( + FATAL_ERROR + "Option onnxruntime_ENABLE_EAGER_MODE can only be used when onnxruntime_ENABLE_TRAINING and onnxruntime_ENABLE_PYTHON are enabled") + endif() + set(ONNXRUNTIME_EAGER_CMAKE_FILE_NAME onnxruntime_eager) +endif() + +if (onnxruntime_ENABLE_LAZY_TENSOR) + # To support LazyTensor, ORT needs to call Python function from C/C++. + # so onnxruntime_ENABLE_PYTHON is required. + if (NOT onnxruntime_ENABLE_TRAINING OR NOT onnxruntime_ENABLE_PYTHON) + message( + FATAL_ERROR + "Option onnxruntime_ENABLE_LAZY_TENSOR can only be set when onnxruntime_ENABLE_TRAINING and onnxruntime_ENABLE_PYTHON are enabled") + endif() + # TODO: In the future, we can compile LazyTensor into a standalone + # library target, onnxruntime_lazy_tensor, to make the buid + # cleaner. +endif() + + function(onnxruntime_set_compile_flags target_name) if (CPUINFO_SUPPORTED) onnxruntime_add_include_to_target(${target_name} cpuinfo) endif() + if(onnxruntime_ENABLE_EAGER_MODE) + target_compile_definitions(${target_name} PRIVATE ENABLE_EAGER_MODE) + endif() + if(onnxruntime_ENABLE_LAZY_TENSOR) + target_compile_definitions(${target_name} PRIVATE ENABLE_LAZY_TENSOR) + endif() if (onnxruntime_ENABLE_CPU_FP16_OPS) target_compile_definitions(${target_name} PRIVATE ENABLE_CPU_FP16_TRAINING_OPS) endif() - if(onnxruntime_DISABLE_ABSEIL) - target_compile_definitions(${target_name} PRIVATE DISABLE_ABSEIL) + if(onnxruntime_DISABLE_ABSEIL) + target_compile_definitions(${target_name} PRIVATE DISABLE_ABSEIL) endif() if(UNIX) target_compile_definitions(${target_name} PRIVATE PLATFORM_POSIX) @@ -625,6 +654,16 @@ function(onnxruntime_set_compile_flags target_name) foreach(CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORY ${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORIES}) target_compile_options(${target_name} PRIVATE "$<$>:/external:I${CMAKE_CUDA_TOOLKIT_INCLUDE_DIRECTORY}>") endforeach() + foreach(onnxruntime_external_lib IN LISTS onnxruntime_EXTERNAL_LIBRARIES) + #TODO: the list contains cmake keywords like "debug". We should exclude them. + if(TARGET ${onnxruntime_external_lib}) + get_target_property(onnxruntime_external_lib_include_dirs ${onnxruntime_external_lib} INTERFACE_INCLUDE_DIRECTORIES) + foreach(onnxruntime_external_lib_include_dir IN LISTS onnxruntime_external_lib_include_dirs) + # TODO: onnxruntime_external_lib_include_dir is a cmake generator expr. Very surprising it can fit in at here. + target_compile_options(${target_name} PRIVATE "$<$>:/external:I${onnxruntime_external_lib_include_dir}>") + endforeach() + endif() + endforeach() target_compile_definitions(${target_name} PUBLIC -DPLATFORM_WINDOWS -DNOGDI -DNOMINMAX -D_USE_MATH_DEFINES -D_SILENCE_ALL_CXX17_DEPRECATION_WARNINGS) if (onnxruntime_ENABLE_MEMLEAK_CHECKER) target_compile_definitions(${target_name} PUBLIC -DONNXRUNTIME_ENABLE_MEMLEAK_CHECK) @@ -1356,7 +1395,7 @@ endif() #Now the 'onnxruntime_EXTERNAL_LIBRARIES' variable should be sealed. It will be used in onnxruntime.cmake which will be included in the next. #The order of the following targets matters. Right depends on left. If target A appears before target B. Then A.cmake can not use variables defined in B.cmake. -set(ONNXRUNTIME_TARGETS onnxruntime_flatbuffers onnxruntime_common onnxruntime_mlas onnxruntime_graph onnxruntime_framework onnxruntime_util onnxruntime_providers onnxruntime_optimizer onnxruntime_session) +set(ONNXRUNTIME_CMAKE_FILES onnxruntime_flatbuffers onnxruntime_common onnxruntime_mlas onnxruntime_graph onnxruntime_framework onnxruntime_util onnxruntime_providers onnxruntime_optimizer onnxruntime_session ${ONNXRUNTIME_EAGER_CMAKE_FILE_NAME}) if (onnxruntime_USE_WINML) # WINML uses and depends on the shared lib. Note: You can build WINML without DML and you will get a @@ -1366,92 +1405,67 @@ if (onnxruntime_USE_WINML) FATAL_ERROR "Option onnxruntime_USE_WINML can only be used when onnxruntime_BUILD_SHARED_LIB is also enabled") endif() - list(APPEND ONNXRUNTIME_TARGETS winml) + list(APPEND ONNXRUNTIME_CMAKE_FILES winml) endif() # if (onnxruntime_USE_WINML) if (onnxruntime_BUILD_SHARED_LIB OR onnxruntime_BUILD_APPLE_FRAMEWORK) if (onnxruntime_BUILD_APPLE_FRAMEWORK AND NOT ${CMAKE_SYSTEM_NAME} MATCHES "Darwin|iOS") message(FATAL_ERROR "onnxruntime_BUILD_APPLE_FRAMEWORK can only be enabled for macOS or iOS.") endif() - list(APPEND ONNXRUNTIME_TARGETS onnxruntime) + list(APPEND ONNXRUNTIME_CMAKE_FILES onnxruntime) endif() if (onnxruntime_BUILD_JAVA) message(STATUS "Java Build is enabled") - list(APPEND ONNXRUNTIME_TARGETS onnxruntime_java) + list(APPEND ONNXRUNTIME_CMAKE_FILES onnxruntime_java) endif() if (onnxruntime_BUILD_NODEJS) message(STATUS "Node.js Build is enabled") - list(APPEND ONNXRUNTIME_TARGETS onnxruntime_nodejs) + list(APPEND ONNXRUNTIME_CMAKE_FILES onnxruntime_nodejs) endif() if (onnxruntime_ENABLE_PYTHON) message(STATUS "Python Build is enabled") - list(APPEND ONNXRUNTIME_TARGETS onnxruntime_python) + list(APPEND ONNXRUNTIME_CMAKE_FILES onnxruntime_python) endif() if (onnxruntime_BUILD_OBJC) message(STATUS "Objective-C Build is enabled") - list(APPEND ONNXRUNTIME_TARGETS onnxruntime_objectivec) + list(APPEND ONNXRUNTIME_CMAKE_FILES onnxruntime_objectivec) endif() if (onnxruntime_BUILD_UNIT_TESTS) - list(APPEND ONNXRUNTIME_TARGETS onnxruntime_unittests) + list(APPEND ONNXRUNTIME_CMAKE_FILES onnxruntime_unittests) endif() if (onnxruntime_BUILD_WINML_TESTS) - list(APPEND ONNXRUNTIME_TARGETS winml_unittests) + list(APPEND ONNXRUNTIME_CMAKE_FILES winml_unittests) endif() # onnxruntime_training depends on onnxruntime_unittests since onnxruntime_training.cmake uses a variable `TEST_SRC_DIR` # that is defined in onnxruntime_unittests.cmake if (onnxruntime_ENABLE_TRAINING) - list(APPEND ONNXRUNTIME_TARGETS onnxruntime_training) + list(APPEND ONNXRUNTIME_CMAKE_FILES onnxruntime_training) if (onnxruntime_ENABLE_TRAINING_E2E_TESTS) - list(APPEND ONNXRUNTIME_TARGETS onnxruntime_training_e2e_tests) + list(APPEND ONNXRUNTIME_CMAKE_FILES onnxruntime_training_e2e_tests) endif() endif() if (onnxruntime_BUILD_CSHARP) message(STATUS "CSharp Build is enabled") # set_property(GLOBAL PROPERTY VS_DOTNET_TARGET_FRAMEWORK_VERSION "netstandard2.0") - list(APPEND ONNXRUNTIME_TARGETS onnxruntime_csharp) + list(APPEND ONNXRUNTIME_CMAKE_FILES onnxruntime_csharp) endif() if (onnxruntime_BUILD_WEBASSEMBLY) message(STATUS "WebAssembly Build is enabled") - list(APPEND ONNXRUNTIME_TARGETS onnxruntime_webassembly) + list(APPEND ONNXRUNTIME_CMAKE_FILES onnxruntime_webassembly) endif() if(onnxruntime_BUILD_KERNEL_EXPLORER) message(STATUS "Kernel Explorer Build is enabled") - list(APPEND ONNXRUNTIME_TARGETS onnxruntime_kernel_explorer) -endif() - -if (onnxruntime_ENABLE_EAGER_MODE) - if (NOT onnxruntime_ENABLE_TRAINING OR NOT onnxruntime_ENABLE_PYTHON) - message( - FATAL_ERROR - "Option onnxruntime_ENABLE_EAGER_MODE can only be used when onnxruntime_ENABLE_TRAINING and onnxruntime_ENABLE_PYTHON are enabled") - endif() - add_compile_definitions(ENABLE_EAGER_MODE) - list(APPEND ONNXRUNTIME_TARGETS onnxruntime_eager) -endif() - -if (onnxruntime_ENABLE_LAZY_TENSOR) - # To support LazyTensor, ORT needs to call Python function from C/C++. - # so onnxruntime_ENABLE_PYTHON is required. - if (NOT onnxruntime_ENABLE_TRAINING OR NOT onnxruntime_ENABLE_PYTHON) - message( - FATAL_ERROR - "Option onnxruntime_ENABLE_LAZY_TENSOR can only be set when onnxruntime_ENABLE_TRAINING and onnxruntime_ENABLE_PYTHON are enabled") - endif() - add_compile_definitions(ENABLE_LAZY_TENSOR) - # TODO: In the future, we can compile LazyTensor into a standalone - # library target, onnxruntime_lazy_tensor, to make the buid - # cleaner. - #list(APPEND ONNXRUNTIME_TARGETS onnxruntime_lazy_tensor) + list(APPEND ONNXRUNTIME_CMAKE_FILES onnxruntime_kernel_explorer) endif() # When GDK_PLATFORM is set then WINAPI_FAMILY is defined in gdk_toolchain.cmake (along with other relevant flags/definitions). @@ -1466,7 +1480,7 @@ if (WIN32 AND NOT GDK_PLATFORM) endif() endif() -foreach(target_name ${ONNXRUNTIME_TARGETS}) +foreach(target_name ${ONNXRUNTIME_CMAKE_FILES}) include(${target_name}.cmake) endforeach() if (UNIX) diff --git a/cmake/deps.txt b/cmake/deps.txt index 416d823918..1cc6ed26dc 100644 --- a/cmake/deps.txt +++ b/cmake/deps.txt @@ -24,6 +24,7 @@ microsoft_wil;https://github.com/microsoft/wil/archive/5f4caba4e7a9017816e47becd mimalloc;https://github.com/microsoft/mimalloc/archive/refs/tags/v2.0.3.zip;e4f37b93b2da78a5816c2495603a4188d316214b mp11;https://github.com/boostorg/mp11/archive/refs/tags/boost-1.79.0.zip;c8f04e378535ededbe5af52c8f969d2dedbe73d5 onnx;https://github.com/onnx/onnx/archive/5a5f8a5935762397aa68429b5493084ff970f774.zip;edc8e1338c02f3ab222f3d803a24e17608c13895 +#Branch name: 8.4-GA onnx_tensorrt;https://github.com/onnx/onnx-tensorrt/archive/87c7a70688fd98fb355b8976f41425b40e4fe52f.zip;b97d112d9d6efa180c9b94e05268f2ff3294a534 protobuf;https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.18.3.zip;b95bf7e9de9c2249b6c1f2ca556ace49999e90bd psimd;https://github.com/Maratyszcza/psimd/archive/072586a71b55b7f8c584153d223e95687148a900.zip;1f5454b01f06f9656b77e4a5e2e31d7422487013 diff --git a/cmake/external/date b/cmake/external/date deleted file mode 160000 index e7e1482087..0000000000 --- a/cmake/external/date +++ /dev/null @@ -1 +0,0 @@ -Subproject commit e7e1482087f58913b80a20b04d5c58d9d6d90155 diff --git a/cmake/external/googletest b/cmake/external/googletest deleted file mode 160000 index 58d77fa807..0000000000 --- a/cmake/external/googletest +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 58d77fa8070e8cec2dc1ed015d66b454c8d78850 diff --git a/cmake/external/nsync b/cmake/external/nsync deleted file mode 160000 index ac54896827..0000000000 --- a/cmake/external/nsync +++ /dev/null @@ -1 +0,0 @@ -Subproject commit ac5489682760393fe21bd2a8e038b528442412a7 diff --git a/cmake/external/onnx-tensorrt b/cmake/external/onnx-tensorrt deleted file mode 160000 index 87c7a70688..0000000000 --- a/cmake/external/onnx-tensorrt +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 87c7a70688fd98fb355b8976f41425b40e4fe52f diff --git a/cmake/external/onnxruntime_external_deps.cmake b/cmake/external/onnxruntime_external_deps.cmake index 5dfe2b13dc..6e59bc1f40 100644 --- a/cmake/external/onnxruntime_external_deps.cmake +++ b/cmake/external/onnxruntime_external_deps.cmake @@ -135,9 +135,7 @@ FetchContent_Declare( URL ${DEP_URL_date} URL_HASH SHA1=${DEP_SHA1_date} ) -FetchContent_Populate(date) -add_library(date_interface INTERFACE) -target_include_directories(date_interface INTERFACE ${date_SOURCE_DIR}/include) +onnxruntime_fetchcontent_makeavailable(date) @@ -376,7 +374,7 @@ endif() #onnxruntime_EXTERNAL_LIBRARIES could contain onnx, onnx_proto,libprotobuf, cuda/cudnn, # dnnl/mklml, onnxruntime_codegen_tvm, tvm and pthread # pthread is always at the last -set(onnxruntime_EXTERNAL_LIBRARIES ${onnxruntime_EXTERNAL_LIBRARIES_XNNPACK} WIL::WIL nlohmann_json::nlohmann_json onnx onnx_proto ${PROTOBUF_LIB} re2::re2 Boost::mp11 safeint_interface flatbuffers ${GSL_TARGET} ${ABSEIL_LIBS}) +set(onnxruntime_EXTERNAL_LIBRARIES ${onnxruntime_EXTERNAL_LIBRARIES_XNNPACK} WIL::WIL nlohmann_json::nlohmann_json onnx onnx_proto ${PROTOBUF_LIB} re2::re2 Boost::mp11 safeint_interface flatbuffers ${GSL_TARGET} ${ABSEIL_LIBS} date_interface) # The source code of onnx_proto is generated, we must build this lib first before starting to compile the other source code that uses ONNX protobuf types. # The other libs do not have the problem. All the sources are already there. We can compile them in any order. set(onnxruntime_EXTERNAL_DEPENDENCIES onnx_proto) diff --git a/cmake/external/pytorch_cpuinfo b/cmake/external/pytorch_cpuinfo deleted file mode 160000 index 5916273f79..0000000000 --- a/cmake/external/pytorch_cpuinfo +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 5916273f79a21551890fd3d56fc5375a78d1598d diff --git a/cmake/external/re2 b/cmake/external/re2 deleted file mode 160000 index 5723bb8950..0000000000 --- a/cmake/external/re2 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 5723bb8950318135ed9cf4fc76bed988a087f536 diff --git a/cmake/external/tensorboard b/cmake/external/tensorboard deleted file mode 160000 index 373eb09e4c..0000000000 --- a/cmake/external/tensorboard +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 373eb09e4c5d2b3cc2493f0949dc4be6b6a45e81 diff --git a/cmake/external/wil b/cmake/external/wil deleted file mode 160000 index 5f4caba4e7..0000000000 --- a/cmake/external/wil +++ /dev/null @@ -1 +0,0 @@ -Subproject commit 5f4caba4e7a9017816e47becdd918fcc872039ba diff --git a/cmake/onnxruntime_providers.cmake b/cmake/onnxruntime_providers.cmake index a8a195a597..99c985f9ee 100644 --- a/cmake/onnxruntime_providers.cmake +++ b/cmake/onnxruntime_providers.cmake @@ -1153,11 +1153,9 @@ if (onnxruntime_USE_MIGRAPHX) set(BUILD_LIBRARY_ONLY 1) add_definitions("-DONNX_ML=1") add_definitions("-DONNX_NAMESPACE=onnx") - # TODO: Remove the next line so that we can delete the git submodule - include_directories(${PROJECT_SOURCE_DIR}/external/protobuf ${PROJECT_SOURCE_DIR}/external/eigen) + include_directories(${protobuf_SOURCE_DIR} ${eigen_SOURCE_DIR}) set(MIGRAPHX_ROOT ${onnxruntime_MIGRAPHX_HOME}) - # TODO: Remove the next line so that we can delete the git submodule - include_directories(${PROJECT_SOURCE_DIR}/external/onnx) + include_directories(${onnx_SOURCE_DIR}) set(OLD_CMAKE_CXX_FLAGS ${CMAKE_CXX_FLAGS}) if ( CMAKE_COMPILER_IS_GNUCC ) set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-parameter -Wno-missing-field-initializers") diff --git a/tools/ci_build/github/azure-pipelines/templates/web-browserstack-ci.yml b/tools/ci_build/github/azure-pipelines/templates/web-browserstack-ci.yml index b926ad02c5..4494fd36b3 100644 --- a/tools/ci_build/github/azure-pipelines/templates/web-browserstack-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/web-browserstack-ci.yml @@ -26,11 +26,7 @@ jobs: workingDirectory: '$(Build.SourcesDirectory)' displayName: 'Read commit SHA and checkout' condition: eq('${{ parameters.CommitOverride }}', 'true') - - script: | - git submodule sync -- cmake/external/onnx - git submodule update --init -- cmake/external/onnx - workingDirectory: '$(Build.SourcesDirectory)' - displayName: 'Checkout submodule onnx' + - task: NodeTool@0 inputs: versionSpec: '16.x' diff --git a/tools/ci_build/github/linux/build_linux_arm64_python_package.sh b/tools/ci_build/github/linux/build_linux_arm64_python_package.sh index e272339a4d..283266977a 100755 --- a/tools/ci_build/github/linux/build_linux_arm64_python_package.sh +++ b/tools/ci_build/github/linux/build_linux_arm64_python_package.sh @@ -8,7 +8,7 @@ CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-st BUILD_DEVICE="CPU" BUILD_CONFIG="Release" PYTHON_EXES=("/opt/python/cp37-cp37m/bin/python3.7" "/opt/python/cp38-cp38/bin/python3.8" "/opt/python/cp39-cp39/bin/python3.9" "/opt/python/cp310-cp310/bin/python3.10") -while getopts "d:" parameter_Option +while getopts "d:p:" parameter_Option do case "${parameter_Option}" in #GPU or CPU. diff --git a/tools/ci_build/github/linux/run_build.sh b/tools/ci_build/github/linux/run_build.sh index c00dfa9e12..68fd68eb87 100755 --- a/tools/ci_build/github/linux/run_build.sh +++ b/tools/ci_build/github/linux/run_build.sh @@ -50,12 +50,10 @@ else --cudnn_home /usr/local/cudnn-$_CUDNN_VERSION/cuda $BUILD_EXTR_PAR elif [[ $BUILD_DEVICE = "tensorrt"* ]]; then if [ $BUILD_DEVICE = "tensorrt-v7.1" ]; then - CUR_PWD=$(pwd) - cd $SCRIPT_DIR/../../../../cmake/external/onnx-tensorrt/ - git remote update - git checkout 7.1 - cd $CUR_PWD - COMMON_BUILD_ARGS=${COMMON_BUILD_ARGS/"--skip_submodule_sync"} + pushd . + cd $SCRIPT_DIR/../../../../cmake + sed -i "s/^onnx_tensorrt.*$/onnx_tensorrt;https:\/\/github.com\/onnx\/onnx-tensorrt\/archive\/refs\/tags\/release\/7.1.zip;e23bf76bbe4748c49951d6b401cf5e1006d86cce/g" deps.txt + popd fi _CUDNN_VERSION=$(echo $CUDNN_VERSION | cut -d. -f1-2) python3 $SCRIPT_DIR/../../build.py --build_dir /build \ diff --git a/tools/ci_build/github/windows/helpers.ps1 b/tools/ci_build/github/windows/helpers.ps1 new file mode 100644 index 0000000000..95cc67f0e8 --- /dev/null +++ b/tools/ci_build/github/windows/helpers.ps1 @@ -0,0 +1,461 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +enum CMakeBuildType{ + Debug + Release + RelWithDebInfo + MinSizeRel +} + + +# The DownloadAndExtract function was copied from: https://github.com/dotnet/arcade/blob/main/eng/common/native/CommonLibrary.psm1 +<# +.SYNOPSIS +Get the name of a temporary folder under the native install directory +#> +function Get-TempDirectory { + #TODO: what if the env does not exist? + return $Env:AGENT_TEMPDIRECTORY +} + +function Get-TempPathFilename { + [CmdletBinding(PositionalBinding=$false)] + Param ( + [Parameter(Mandatory=$True)] + [string] $Path + ) + $TempDir = Get-TempDirectory + $TempFilename = Split-Path $Path -leaf + $TempPath = Join-Path $TempDir $TempFilename + return $TempPath +} + +<# +.SYNOPSIS +Unzip an archive +.DESCRIPTION +Powershell module to unzip an archive to a specified directory +.PARAMETER ZipPath (Required) +Path to archive to unzip +.PARAMETER OutputDirectory (Required) +Output directory for archive contents +.PARAMETER Force +Overwrite output directory contents if they already exist +.NOTES +- Returns True and does not perform an extraction if output directory already exists but Overwrite is not True. +- Returns True if unzip operation is successful +- Returns False if Overwrite is True and it is unable to remove contents of OutputDirectory +- Returns False if unable to extract zip archive +#> +function Expand-Zip { + [CmdletBinding(PositionalBinding=$false)] + Param ( + [Parameter(Mandatory=$True)] + [string] $ZipPath, + [Parameter(Mandatory=$True)] + [string] $OutputDirectory, + [switch] $Force + ) + + Write-Host "Extracting '$ZipPath' to '$OutputDirectory'" + try { + if ((Test-Path $OutputDirectory) -And (-Not $Force)) { + Write-Host "Directory '$OutputDirectory' already exists, skipping extract" + return $True + } + if (Test-Path $OutputDirectory) { + Write-Host "'Force' is 'True', but '$OutputDirectory' exists, removing directory" + Remove-Item -Path $OutputDirectory -Force -Recurse + if ($? -Eq $False) { + Write-Error "Unable to remove '$OutputDirectory'" + return $False + } + } + + $TempOutputDirectory = Join-Path "$(Split-Path -Parent $OutputDirectory)" "$(Split-Path -Leaf $OutputDirectory).tmp" + + if (Test-Path $TempOutputDirectory) { + Remove-Item $TempOutputDirectory -Force -Recurse + } + New-Item -Path $TempOutputDirectory -Force -ItemType "Directory" | Out-Null + + Add-Type -assembly "system.io.compression.filesystem" + [io.compression.zipfile]::ExtractToDirectory("$ZipPath", "$TempOutputDirectory") + if ($? -Eq $False) { + Write-Error "Unable to extract '$ZipPath'" + return $False + } + + Move-Item -Path $TempOutputDirectory -Destination $OutputDirectory + } + catch { + Write-Host $_ + Write-Host $_.Exception + + return $False + } + return $True +} + +<# +.SYNOPSIS +Helper module to install an archive to a directory +.DESCRIPTION +Helper module to download and extract an archive to a specified directory +.PARAMETER Uri +Uri of artifact to download +.PARAMETER InstallDirectory +Directory to extract artifact contents to. +.PARAMETER Force +Force download / extraction if file or contents already exist. Default = False +.PARAMETER DownloadRetries +Total number of retry attempts. Default = 5 +.PARAMETER RetryWaitTimeInSeconds +Wait time between retry attempts in seconds. Default = 30 +.NOTES +Returns False if download or extraction fail, True otherwise +#> +function DownloadAndExtract { + [CmdletBinding(PositionalBinding=$false)] + Param ( + [Parameter(Mandatory=$True)] + [string] $Uri, + [Parameter(Mandatory=$True)] + [string] $InstallDirectory, + [switch] $Force = $False, + [int] $DownloadRetries = 5, + [int] $RetryWaitTimeInSeconds = 30 + ) + # Define verbose switch if undefined + $Verbose = $VerbosePreference -Eq "Continue" + + $TempToolPath = Get-TempPathFilename -Path $Uri + Write-Host "TempToolPath=$TempToolPath" + # Download native tool + $DownloadStatus = Get-File -Uri $Uri ` + -Path $TempToolPath ` + -DownloadRetries $DownloadRetries ` + -RetryWaitTimeInSeconds $RetryWaitTimeInSeconds ` + -Force:$Force ` + -Verbose:$Verbose + + if ($DownloadStatus -Eq $False) { + Write-Error "Download failed from $Uri" + return $False + } + + + # Extract native tool + $UnzipStatus = Expand-Zip -ZipPath $TempToolPath ` + -OutputDirectory $InstallDirectory ` + -Force:$Force ` + -Verbose:$Verbose + if ($UnzipStatus -Eq $False) { + # Retry Download one more time with Force=true + $DownloadRetryStatus = Get-File -Uri $Uri ` + -Path $TempToolPath ` + -DownloadRetries 1 ` + -RetryWaitTimeInSeconds $RetryWaitTimeInSeconds ` + -Force:$True ` + -Verbose:$Verbose + + if ($DownloadRetryStatus -Eq $False) { + Write-Error "Last attempt of download failed as well" + return $False + } + + # Retry unzip again one more time with Force=true + $UnzipRetryStatus = Expand-Zip -ZipPath $TempToolPath ` + -OutputDirectory $InstallDirectory ` + -Force:$True ` + -Verbose:$Verbose + if ($UnzipRetryStatus -Eq $False) + { + Write-Error "Last attempt of unzip failed as well" + # Clean up partial zips and extracts + if (Test-Path $TempToolPath) { + Remove-Item $TempToolPath -Force + } + if (Test-Path $InstallDirectory) { + Remove-Item $InstallDirectory -Force -Recurse + } + return $False + } + } + + return $True +} + +<# +.SYNOPSIS +Download a file, retry on failure +.DESCRIPTION +Download specified file and retry if attempt fails +.PARAMETER Uri +Uri of file to download. If Uri is a local path, the file will be copied instead of downloaded +.PARAMETER Path +Path to download or copy uri file to +.PARAMETER Force +Overwrite existing file if present. Default = False +.PARAMETER DownloadRetries +Total number of retry attempts. Default = 5 +.PARAMETER RetryWaitTimeInSeconds +Wait time between retry attempts in seconds Default = 30 +#> +function Get-File { + [CmdletBinding(PositionalBinding=$false)] + Param ( + [Parameter(Mandatory=$True)] + [string] $Uri, + [Parameter(Mandatory=$True)] + [string] $Path, + [int] $DownloadRetries = 5, + [int] $RetryWaitTimeInSeconds = 30, + [switch] $Force = $False + ) + $Attempt = 0 + + if ($Force) { + if (Test-Path $Path) { + Remove-Item $Path -Force + } + } + if (Test-Path $Path) { + Write-Host "File '$Path' already exists, skipping download" + return $True + } + + $DownloadDirectory = Split-Path -ErrorAction Ignore -Path "$Path" -Parent + if (-Not (Test-Path $DownloadDirectory)) { + New-Item -path $DownloadDirectory -force -itemType "Directory" | Out-Null + } + + $TempPath = "$Path.tmp" + if (Test-Path -IsValid -Path $Uri) { + Write-Verbose "'$Uri' is a file path, copying temporarily to '$TempPath'" + Copy-Item -Path $Uri -Destination $TempPath + Write-Verbose "Moving temporary file to '$Path'" + Move-Item -Path $TempPath -Destination $Path + return $? + } + else { + # Don't display the console progress UI - it's a huge perf hit + $ProgressPreference = 'SilentlyContinue' + while($Attempt -Lt $DownloadRetries) + { + try { + Invoke-WebRequest -UseBasicParsing -Uri $Uri -OutFile $TempPath + Write-Verbose "Downloaded to temporary location '$TempPath'" + Move-Item -Path $TempPath -Destination $Path + Write-Verbose "Moved temporary file to '$Path'" + return $True + } + catch { + $Attempt++ + if ($Attempt -Lt $DownloadRetries) { + $AttemptsLeft = $DownloadRetries - $Attempt + Write-Warning "Download failed, $AttemptsLeft attempts remaining, will retry in $RetryWaitTimeInSeconds seconds" + Start-Sleep -Seconds $RetryWaitTimeInSeconds + } + else { + Write-Error $_ + Write-Error $_.Exception + } + } + } + } + + return $False +} + +<# + .Description + The Get-DownloadURL function returns the download URL of a external dependency. The URL might be local file path or + a remote HTTPS URL. + + .PARAMETER name + The name of the dependency, should present in the first column of ONNX Runtime's deps.txt. + + .PARAMETER src_root + The full path of ONNX Runtime's top level source diretory for locating deps.txt. +#> + +function Get-DownloadURL { + param ( + [Parameter(Mandatory)][string]$name, + [Parameter(Mandatory)][string]$src_root + ) + $entry = Import-Csv -Path "$src_root\cmake\deps.txt" -Delimiter ';' -Header name,url,hash -Encoding UTF8 | Where-Object name -eq $name + return $entry.url +} + +<# + .Description + The Install-Pybind function installs pybind11 headers, which are needed for building ONNX from source. + + .PARAMETER cmake_path + The full path of cmake.exe + + .PARAMETER src_root + The full path of ONNX Runtime's top level source diretory for locating deps.txt. + + .PARAMETER build_config + The value of CMAKE_BUILD_TYPE, can be Debug, Release, RelWithDebInfo or MinSizeRel. +#> + +function Install-Pybind { + + param ( + [Parameter(Mandatory)][string]$cmake_path, + [Parameter(Mandatory)][string]$src_root, + [Parameter(Mandatory)][CMakeBuildType]$build_config, + [Parameter(Mandatory)][string[]]$cmake_extra_args + ) + + pushd . + + $url=Get-DownloadURL -name pybind11 -src_root $src_root + Write-Host "Downloading pybind11 from $url" + $temp_dir = Get-TempDirectory + $pybind_src_dir = Join-Path $temp_dir "pybind" + $download_finished = DownloadAndExtract -Uri $url -InstallDirectory $pybind_src_dir -Force + if(-Not $download_finished){ + Write-Host -Object "Download failed" + exit 1 + } + cd $pybind_src_dir + cd * + mkdir build + cd build + [string[]]$cmake_args = "..", "-DCMAKE_INSTALL_PREFIX=$install_prefix", "-DBUILD_TESTING=OFF" + $cmake_args += $cmake_extra_args + $p = Start-Process -FilePath $cmake_path -ArgumentList $cmake_args -NoNewWindow -Wait -PassThru + $exitCode = $p.ExitCode + if ($exitCode -ne 0) { + Write-Host -Object "CMake command failed. Exitcode: $exitCode" + exit $exitCode + } + $cmake_args = "--build", ".", "--parallel", "--config", $build_config, "--target", "INSTALL" + $p = Start-Process -FilePath $cmake_path -ArgumentList $cmake_args -NoNewWindow -Wait -PassThru + $exitCode = $p.ExitCode + if ($exitCode -ne 0) { + Write-Host -Object "CMake command failed. Exitcode: $exitCode" + exit $exitCode + } + popd +} + +<# + .Description + The Install-ONNX function installs ONNX python package from source and also the python packages that it depends on. + This script will build protobuf C/C++ lib/exe for the CPU arch of the current build machine, because we need to run + protoc.exe on this machine. + + .PARAMETER cmake_path + The full path of cmake.exe + + .PARAMETER src_root + The full path of ONNX Runtime's top level source diretory + + .PARAMETER build_config + The value of CMAKE_BUILD_TYPE, can be Debug, Release, RelWithDebInfo or MinSizeRel. +#> +function Install-Protobuf { + + param ( + [Parameter(Mandatory)][string]$cmake_path, + [Parameter(Mandatory)][string]$src_root, + [Parameter(Mandatory)][CMakeBuildType]$build_config, + [Parameter(Mandatory)][string[]]$cmake_extra_args + ) + + pushd . + $url=Get-DownloadURL -name protobuf -src_root $src_root + $temp_dir = Get-TempDirectory + $protobuf_src_dir = Join-Path $temp_dir "protobuf" + $download_finished = DownloadAndExtract -Uri $url -InstallDirectory $protobuf_src_dir -Force + if(-Not $download_finished){ + exit 1 + } + cd $protobuf_src_dir + cd * + Get-Content $src_root\cmake\patches\protobuf\protobuf_cmake.patch | &'C:\Program Files\Git\usr\bin\patch.exe' --ignore-whitespace -p1 + + [string[]]$cmake_args = "cmake", "-DCMAKE_BUILD_TYPE=$build_config", "-Dprotobuf_BUILD_TESTS=OFF", "-DBUILD_SHARED_LIBS=OFF", "-DCMAKE_PREFIX_PATH=$install_prefix", "-DCMAKE_INSTALL_PREFIX=$install_prefix", "-Dprotobuf_MSVC_STATIC_RUNTIME=OFF" + $cmake_args += $cmake_extra_args + + $p = Start-Process -FilePath $cmake_path -ArgumentList $cmake_args -NoNewWindow -Wait -PassThru + $exitCode = $p.ExitCode + if ($exitCode -ne 0) { + Write-Host -Object "CMake command failed. Exitcode: $exitCode" + exit $exitCode + } + $cmake_args = "--build", ".", "--parallel", "--config", $build_config, "--target", "INSTALL" + $p = Start-Process -FilePath $cmake_path -ArgumentList $cmake_args -NoNewWindow -Wait -PassThru + $exitCode = $p.ExitCode + if ($exitCode -ne 0) { + Write-Host -Object "CMake command failed. Exitcode: $exitCode" + exit $exitCode + } + popd +} + +<# + .Description + The Install-ONNX function installs ONNX python package from source and also the python packages that it depends on. + protoc.exe must exist in the PATH. +#> +function Install-ONNX { + + param ( + [Parameter(Mandatory)][CMakeBuildType]$build_config, + [Parameter(Mandatory)][string]$src_root, + [Parameter(Mandatory)][string]$protobuf_version + ) + + pushd . + + Write-Host "Installing python packages..." + $p = Start-Process -NoNewWindow -Wait -PassThru -FilePath "python.exe" -ArgumentList "-m", "pip", "install", "--disable-pip-version-check", "setuptools", "wheel", "numpy", "protobuf==$protobuf_version" + $exitCode = $p.ExitCode + if ($exitCode -ne 0) { + Write-Host -Object "Install dependent python wheels failed. Exitcode: $exitCode" + exit $exitCode + } + + $url=Get-DownloadURL -name onnx -src_root $src_root + $temp_dir = Get-TempDirectory + $onnx_src_dir = Join-Path $temp_dir "onnx" + $download_finished = DownloadAndExtract -Uri $url -InstallDirectory $onnx_src_dir -Force + if(-Not $download_finished){ + exit 1 + } + cd $onnx_src_dir + cd * + $Env:ONNX_ML=1 + if($build_config -eq 'Debug'){ + $Env:DEBUG='1' + } + $Env:CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=OFF -DProtobuf_USE_STATIC_LIBS=ON -DONNX_USE_LITE_PROTO=OFF" + + $p = Start-Process -NoNewWindow -Wait -PassThru -FilePath "python.exe" -ArgumentList "setup.py", "bdist_wheel" + $exitCode = $p.ExitCode + if ($exitCode -ne 0) { + Write-Host -Object "Generate wheel file failed. Exitcode: $exitCode" + exit $exitCode + } + Write-Host "Uninstalling onnx and ignore errors if there is any..." + python -m pip uninstall -y onnx -qq + Write-Host "Installing the newly built ONNX python package" + Get-ChildItem -Path dist/*.whl | foreach { + $p = Start-Process -NoNewWindow -Wait -PassThru -FilePath "python.exe" -ArgumentList "-m", "pip", "--disable-pip-version-check", "install", "--upgrade", $_.fullname + $exitCode = $p.ExitCode + if ($exitCode -ne 0) { + Write-Host -Object "Install wheel file failed. Exitcode: $exitCode" + exit $exitCode + } + } + Write-Host "Finished installing onnx" + popd +} \ No newline at end of file diff --git a/tools/ci_build/github/windows/install_third_party_deps.ps1 b/tools/ci_build/github/windows/install_third_party_deps.ps1 index b2e3feafa0..d0177f1c68 100644 --- a/tools/ci_build/github/windows/install_third_party_deps.ps1 +++ b/tools/ci_build/github/windows/install_third_party_deps.ps1 @@ -1,60 +1,65 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. +# This script depends on python.exe, cmake.exe and Visual C++ spectre-mitigated libs. +# Please setup AGENT_TEMPDIRECTORY env variable before running this script + param ( [string]$cpu_arch = "x64", - [string]$build_config = "RelWithDebInfo", - [string]$install_prefix + [string]$build_config = "RelWithDebInfo", + [string]$install_prefix = "." ) - + +. "$PSScriptRoot\helpers.ps1" + +$ort_src_root = (Get-Item $PSScriptRoot).parent.parent.parent.parent.FullName + +Write-Host "ONNX Runtime src root: $ort_src_root" + $ErrorActionPreference = "Stop" -$env:Path = "$install_prefix\bin;C:\Program Files\7-Zip;" + $env:Path +$Env:Path = "$install_prefix\bin;" + $env:Path +$Env:MSBUILDDISABLENODEREUSE=1 New-Item -Path "$install_prefix" -ItemType Directory -Force -$cmake_extra_args=@() +# Setup compile flags +$compile_flags = '/MP /guard:cf /Qspectre /DWIN32 /D_WINDOWS /DWINVER=0x0601 /D_WIN32_WINNT=0x0601 /DNTDDI_VERSION=0x06010000 /W3 ' +$linker_flags=@('/guard:cf') + +if($build_config -eq 'Release'){ + $compile_flags += "/O2", "/Ob2", "/DNDEBUG", "/Gw", "/GL" +} elseif($build_config -eq 'RelWithDebInfo'){ + $compile_flags += "/Zi", "/O2", "/Ob1", "/DNDEBUG", "/Gw", "/GL" +} elseif($build_config -eq 'Debug'){ + $compile_flags += "/Zi", "/Ob0", "/Od", "/RTC1" +} elseif($build_config -eq 'MinSizeRel'){ + $compile_flags += "/O1", "/Ob1", "/DNDEBUG", "/Gw", "/GL" +} +# cmake args that applies to every 3rd-party library +[string[]]$cmake_extra_args="-DCMAKE_CXX_STANDARD=17 `"-DCMAKE_CXX_FLAGS=$compile_flags /EHsc`" ", "`"-DCMAKE_C_FLAGS=$compile_flags`"", "--compile-no-warning-as-error", "--fresh", "-Wno-dev" if($cpu_arch -eq 'x86'){ - Write-Host "Build for x86" - $cmake_extra_args="-A", "Win32", "-T", "host=x64" + $cmake_extra_args += "-A", "Win32", "-T", "host=x64" + $linker_flags += '/machine:x86' +} elseif($cpu_arch -eq 'x64') { + $linker_flags += '/machine:x64' } else { - Write-Host "Build for $cpu_arch" + throw "$cpu_arch is not supported" } -$url='https://github.com/pybind/pybind11/archive/refs/tags/v2.10.1.zip' -Write-Host "Downloading pybind11 from $url" -Invoke-WebRequest -Uri $url -OutFile pybind11.zip -7z x pybind11.zip -cd pybind11-2.10.1 -mkdir build -cd build -cmake .. "-DCMAKE_INSTALL_PREFIX=$install_prefix" -DBUILD_TESTING=OFF $cmake_extra_args -cmake --build . --parallel --config $build_config --target INSTALL -cd ../.. +$cmake_extra_args += "-DCMAKE_EXE_LINKER_FLAGS=`"$linker_flags`"" +# Find the full path of cmake.exe +$cmake_command = Get-Command -CommandType Application cmake +$cmake_path = $cmake_command.Path + +Install-Pybind -cmake_path $cmake_path -src_root $ort_src_root -build_config $build_config -cmake_extra_args $cmake_extra_args + +Install-Protobuf -cmake_path $cmake_path -src_root $ort_src_root -build_config $build_config -cmake_extra_args $cmake_extra_args + +# TODO: parse it from deps.txt $protobuf_version="3.18.3" -$url="https://github.com/protocolbuffers/protobuf/releases/download/v$protobuf_version/protobuf-cpp-$protobuf_version.zip" -Write-Host "Downloading protobuf from $url" -Invoke-WebRequest -Uri $url -OutFile protobuf_src.zip -7z x protobuf_src.zip -cd protobuf-$protobuf_version -Get-Content $Env:BUILD_SOURCESDIRECTORY\cmake\patches\protobuf\protobuf_cmake.patch | &'C:\Program Files\Git\usr\bin\patch.exe' --binary --ignore-whitespace -p1 -cmake cmake -DCMAKE_BUILD_TYPE=$build_config -Dprotobuf_BUILD_TESTS=OFF -DBUILD_SHARED_LIBS=OFF "-DCMAKE_PREFIX_PATH=$install_prefix" "-DCMAKE_INSTALL_PREFIX=$install_prefix" -Dprotobuf_MSVC_STATIC_RUNTIME=OFF $cmake_extra_args -cmake --build . --parallel --config $build_config --target INSTALL -cd .. -python -m pip install -q setuptools wheel numpy protobuf==$protobuf_version pybind11 -$onnx_commit_id="5a5f8a5935762397aa68429b5493084ff970f774" -$url="https://github.com/onnx/onnx/archive/$onnx_commit_id.zip" -Write-Host "Downloading onnx from $url" -Invoke-WebRequest -Uri $url -OutFile onnx.zip -7z x onnx.zip -cd "onnx-$onnx_commit_id" -$Env:ONNX_ML=1 -if($build_config -eq 'Debug'){ - $Env:DEBUG='1' -} -$Env:CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=OFF -DProtobuf_USE_STATIC_LIBS=ON -DONNX_USE_LITE_PROTO=OFF" -python setup.py bdist_wheel -python -m pip uninstall -y onnx -qq -Get-ChildItem -Path dist/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname} \ No newline at end of file + +# ONNX doesn't allow us to specify CMake's path +Install-ONNX -build_config $build_config -src_root $ort_src_root -protobuf_version $protobuf_version \ No newline at end of file