From 15f7dca9fb2aa6d107ad1c8bf9de7c6fa16256f1 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Wed, 29 Mar 2023 14:08:18 -0700 Subject: [PATCH] Update protobuf to 3.21.x (#15245) ### Description Fixed [AB#10092](https://aiinfra.visualstudio.com/6a833879-cd9b-44a4-a9de-adc2d818f13c/_workitems/edit/10092), [AB#11753](https://aiinfra.visualstudio.com/6a833879-cd9b-44a4-a9de-adc2d818f13c/_workitems/edit/11753), [AB#11759](https://aiinfra.visualstudio.com/6a833879-cd9b-44a4-a9de-adc2d818f13c/_workitems/edit/11759) ### Motivation and Context The one we use has a security issue in Java, though we don't use that version's protobuf java package. --- cgmanifests/generated/cgmanifest.json | 20 ++--- cmake/deps.txt | 2 +- .../external/onnxruntime_external_deps.cmake | 3 +- cmake/patches/protobuf/protobuf_cmake.patch | 28 +++--- .../templates/download-deps.yml | 4 +- .../win-qnn-arm64-ci-pipeline.yml | 9 +- .../python/cpu/scripts/install_protobuf.sh | 6 +- tools/ci_build/github/windows/helpers.ps1 | 90 ++++++++++++++++--- .../github/windows/install_protoc.ps1 | 35 ++++++-- .../windows/install_third_party_deps.ps1 | 38 ++++++-- 10 files changed, 178 insertions(+), 57 deletions(-) diff --git a/cgmanifests/generated/cgmanifest.json b/cgmanifests/generated/cgmanifest.json index ce8b9bf4b1..a4857ea3b6 100644 --- a/cgmanifests/generated/cgmanifest.json +++ b/cgmanifests/generated/cgmanifest.json @@ -128,16 +128,6 @@ "comments": "git submodule at cmake/external/onnxruntime-extensions" } }, - { - "component": { - "type": "git", - "git": { - "commitHash": "a20c65f2cd549445fda907f7b83894c8eb7427d6", - "repositoryUrl": "https://github.com/protocolbuffers/protobuf.git" - }, - "comments": "git submodule at cmake/external/protobuf" - } - }, { "component": { "type": "git", @@ -318,6 +308,16 @@ "comments": "onnx_tensorrt" } }, + { + "component": { + "type": "git", + "git": { + "commitHash": "f0dc78d7e6e331b8c6bb2d5283e06aa26883ca7c", + "repositoryUrl": "https://github.com/protocolbuffers/protobuf.git" + }, + "comments": "protobuf" + } + }, { "component": { "type": "git", diff --git a/cmake/deps.txt b/cmake/deps.txt index a97ce38c7e..08eeac82a6 100644 --- a/cmake/deps.txt +++ b/cmake/deps.txt @@ -26,7 +26,7 @@ mp11;https://github.com/boostorg/mp11/archive/refs/tags/boost-1.79.0.zip;c8f04e3 onnx;https://github.com/onnx/onnx/archive/refs/tags/v1.13.1.zip;7a2517d3e7442109d540de2c86f29cd76d34cf28 #use the commit where it's several commits after 8.5-GA branch (https://github.com/onnx/onnx-tensorrt/commit/369d6676423c2a6dbf4a5665c4b5010240d99d3c) onnx_tensorrt;https://github.com/onnx/onnx-tensorrt/archive/369d6676423c2a6dbf4a5665c4b5010240d99d3c.zip;62119892edfb78689061790140c439b111491275 -protobuf;https://github.com/protocolbuffers/protobuf/archive/refs/tags/v3.20.2.zip;9f71dad95fb83438e88822a9969fc93773fd8c48 +protobuf;https://github.com/protocolbuffers/protobuf/archive/refs/tags/v21.12.zip;7cf2733949036c7d52fda017badcab093fe73bfa psimd;https://github.com/Maratyszcza/psimd/archive/072586a71b55b7f8c584153d223e95687148a900.zip;1f5454b01f06f9656b77e4a5e2e31d7422487013 pthreadpool;https://github.com/Maratyszcza/pthreadpool/archive/1787867f6183f056420e532eec640cba25efafea.zip;e43e80781560c5ab404a4da20f34d846f5f5d101 pybind11;https://github.com/pybind/pybind11/archive/refs/tags/v2.10.1.zip;769b6aa67a77f17a770960f604b727645b6f6a13 diff --git a/cmake/external/onnxruntime_external_deps.cmake b/cmake/external/onnxruntime_external_deps.cmake index fb0c379522..b829e09173 100644 --- a/cmake/external/onnxruntime_external_deps.cmake +++ b/cmake/external/onnxruntime_external_deps.cmake @@ -113,9 +113,8 @@ FetchContent_Declare( Protobuf URL ${DEP_URL_protobuf} URL_HASH SHA1=${DEP_SHA1_protobuf} - SOURCE_SUBDIR cmake PATCH_COMMAND ${ONNXRUNTIME_PROTOBUF_PATCH_COMMAND} - FIND_PACKAGE_ARGS 3.20.2 NAMES Protobuf + FIND_PACKAGE_ARGS 3.21.12 NAMES Protobuf ) set(protobuf_BUILD_TESTS OFF CACHE BOOL "Build protobuf tests" FORCE) if (CMAKE_SYSTEM_NAME STREQUAL "Android") diff --git a/cmake/patches/protobuf/protobuf_cmake.patch b/cmake/patches/protobuf/protobuf_cmake.patch index 0b8887146f..fe8bd25ab1 100644 --- a/cmake/patches/protobuf/protobuf_cmake.patch +++ b/cmake/patches/protobuf/protobuf_cmake.patch @@ -1,8 +1,8 @@ -diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt -index ac92442a1..e930cbd2e 100644 ---- a/cmake/CMakeLists.txt -+++ b/cmake/CMakeLists.txt -@@ -240,9 +240,7 @@ if (MSVC) +diff --git a/CMakeLists.txt b/CMakeLists.txt +index 04cb3303a..4025805cf 100644 +--- a/CMakeLists.txt ++++ b/CMakeLists.txt +@@ -249,9 +249,7 @@ if (MSVC) # MSVC warning suppressions add_definitions( /wd4065 # switch statement contains 'default' but no 'case' labels @@ -12,7 +12,7 @@ index ac92442a1..e930cbd2e 100644 /wd4305 # 'identifier' : truncation from 'type1' to 'type2' /wd4307 # 'operator' : integral constant overflow /wd4309 # 'conversion' : truncation of constant value -@@ -250,7 +248,6 @@ if (MSVC) +@@ -259,7 +257,6 @@ if (MSVC) /wd4355 # 'this' : used in base member initializer list /wd4506 # no definition for inline function 'function' /wd4800 # 'type' : forcing value to bool 'true' or 'false' (performance warning) @@ -20,12 +20,12 @@ index ac92442a1..e930cbd2e 100644 ) # Allow big object add_definitions(/bigobj) -@@ -272,6 +269,8 @@ if (MSVC) - endif() - - configure_file(version.rc.in ${CMAKE_CURRENT_BINARY_DIR}/version.rc @ONLY) -+else (MSVC) +@@ -289,7 +286,7 @@ if (MSVC) + else (MSVC) + # No version.rc file. + set(protobuf_version_rc_file) +- + set(CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS} -Wno-unused-parameter") - endif (MSVC) - - + # When building with "make", "lib" prefix will be added automatically by + # the build tool. + set(LIB_PREFIX) diff --git a/tools/ci_build/github/azure-pipelines/templates/download-deps.yml b/tools/ci_build/github/azure-pipelines/templates/download-deps.yml index 4a55eaa33e..8e875d9449 100644 --- a/tools/ci_build/github/azure-pipelines/templates/download-deps.yml +++ b/tools/ci_build/github/azure-pipelines/templates/download-deps.yml @@ -11,7 +11,7 @@ steps: packageType: upack feed: '/7424c8e4-5c62-490e-95c4-79446f31017c' definition: '517c4f6f-5437-4392-a70d-4f15ec5be2f0' - version: 1.0.36 + version: 1.0.37 downloadPath: $(Build.BinariesDirectory)/deps # The private ADO project @@ -22,7 +22,7 @@ steps: packageType: upack feed: '/4c7631f5-24c0-4307-8822-1aa8f180c325' definition: 'fd9dd5ad-b73e-4678-890e-edcf680dbc1a' - version: 1.0.36 + version: 1.0.37 downloadPath: $(Build.BinariesDirectory)/deps # You can add more ADO accounts at here. diff --git a/tools/ci_build/github/azure-pipelines/win-qnn-arm64-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-qnn-arm64-ci-pipeline.yml index bce5ac573d..aca32ccbf6 100644 --- a/tools/ci_build/github/azure-pipelines/win-qnn-arm64-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-qnn-arm64-ci-pipeline.yml @@ -43,12 +43,19 @@ jobs: versionSpec: '3.x' addToPath: true architecture: $(buildArch) + + - task: PowerShell@2 + displayName: 'Install Protoc for ARM64' + inputs: + filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/windows/install_protoc.ps1' + workingDirectory: '$(Build.BinariesDirectory)' + arguments: -cpu_arch arm64 -install_prefix $(Build.BinariesDirectory)\$(BuildConfig)\installed -build_config $(BuildConfig) - task: PythonScript@0 displayName: 'Generate cmake config' inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: '--arm64 --config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --update --cmake_generator "Visual Studio 16 2019" --use_qnn --qnn_home $(QNN_SDK_ROOT) --parallel' + arguments: '--arm64 --config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --update --cmake_generator "Visual Studio 16 2019" --use_qnn --qnn_home $(QNN_SDK_ROOT) --parallel --path_to_protoc_exe $(Build.BinariesDirectory)\$(BuildConfig)\installed\bin\protoc' workingDirectory: '$(Build.BinariesDirectory)' - task: VSBuild@1 diff --git a/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_protobuf.sh b/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_protobuf.sh index 985c876795..b912b8e15b 100755 --- a/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_protobuf.sh +++ b/tools/ci_build/github/linux/docker/inference/x64/python/cpu/scripts/install_protobuf.sh @@ -35,8 +35,8 @@ case "$(uname -s)" in export CFLAGS export CXXFLAGS ;; - *) - exit -1 + *) + exit 1 esac mkdir -p $INSTALL_PREFIX echo "Installing protobuf ..." @@ -53,7 +53,7 @@ else cd protobuf-* fi -cmake ./cmake -DCMAKE_INSTALL_PREFIX=$INSTALL_PREFIX -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release -Dprotobuf_WITH_ZLIB_DEFAULT=OFF -Dprotobuf_BUILD_SHARED_LIBS=OFF $EXTRA_CMAKE_ARGS +cmake . -DCMAKE_INSTALL_PREFIX=$INSTALL_PREFIX -DCMAKE_POSITION_INDEPENDENT_CODE=ON -Dprotobuf_BUILD_TESTS=OFF -DCMAKE_BUILD_TYPE=Release -Dprotobuf_WITH_ZLIB_DEFAULT=OFF -Dprotobuf_BUILD_SHARED_LIBS=OFF $EXTRA_CMAKE_ARGS make -j$(getconf _NPROCESSORS_ONLN) make install cd .. \ No newline at end of file diff --git a/tools/ci_build/github/windows/helpers.ps1 b/tools/ci_build/github/windows/helpers.ps1 index d9384e6f77..30052ae93e 100644 --- a/tools/ci_build/github/windows/helpers.ps1 +++ b/tools/ci_build/github/windows/helpers.ps1 @@ -16,7 +16,11 @@ Get the name of a temporary folder under the native install directory #> function Get-TempDirectory { #TODO: what if the env does not exist? - return $Env:AGENT_TEMPDIRECTORY + if (-not [string]::IsNullOrWhitespace($Env:AGENT_TEMPDIRECTORY)){ + return $Env:AGENT_TEMPDIRECTORY + } else { + return $Env:TEMP + } } function Get-TempPathFilename { @@ -57,7 +61,9 @@ function Expand-Zip { [string] $OutputDirectory, [switch] $Force ) - + if ([string]::IsNullOrWhitespace($OutputDirectory)){ + Write-Error "OutputDirectory cannot be empty" + } Write-Host "Extracting '$ZipPath' to '$OutputDirectory'" try { if ((Test-Path $OutputDirectory) -And (-Not $Force)) { @@ -308,6 +314,7 @@ function Install-Pybind { param ( [Parameter(Mandatory)][string]$cmake_path, + [Parameter(Mandatory)][string]$msbuild_path, [Parameter(Mandatory)][string]$src_root, [Parameter(Mandatory)][CMakeBuildType]$build_config, [Parameter(Mandatory)][string[]]$cmake_extra_args @@ -336,13 +343,32 @@ function Install-Pybind { Write-Host -Object "CMake command failed. Exitcode: $exitCode" exit $exitCode } - $cmake_args = "--build", ".", "--parallel", "--config", $build_config, "--target", "INSTALL" - $p = Start-Process -FilePath $cmake_path -ArgumentList $cmake_args -NoNewWindow -Wait -PassThru + + $msbuild_args = "-nodeReuse:false", "-nologo", "-nr:false", "-maxcpucount", "-p:UseMultiToolTask=true", "-p:configuration=`"$build_config`"" + + if ($use_cache) { + $msbuild_args += "/p:CLToolExe=cl.exe /p:CLToolPath=C:\ProgramData\chocolatey\bin /p:TrackFileAccess=false /p:UseMultiToolTask=true" + } + + $final_args = $msbuild_args + "pybind11.sln" + Write-Host $final_args + + $p = Start-Process -FilePath $msbuild_path -ArgumentList $final_args -NoNewWindow -Wait -PassThru $exitCode = $p.ExitCode if ($exitCode -ne 0) { - Write-Host -Object "CMake command failed. Exitcode: $exitCode" + Write-Host -Object "Build pybind11.sln failed. Exitcode: $exitCode" exit $exitCode } + + $final_args = $msbuild_args + "INSTALL.vcxproj" + $p = Start-Process -FilePath $msbuild_path -ArgumentList $final_args -NoNewWindow -Wait -PassThru + $exitCode = $p.ExitCode + if ($exitCode -ne 0) { + Write-Host -Object "Install pybind failed. Exitcode: $exitCode" + exit $exitCode + } + Write-Host "Installing pybind finished." + popd } @@ -365,6 +391,7 @@ function Install-Protobuf { param ( [Parameter(Mandatory)][string]$cmake_path, + [Parameter(Mandatory)][string]$msbuild_path, [Parameter(Mandatory)][string]$src_root, [Parameter(Mandatory)][CMakeBuildType]$build_config, [Parameter(Mandatory)][string[]]$cmake_extra_args @@ -381,9 +408,23 @@ function Install-Protobuf { } cd $protobuf_src_dir cd * - Get-Content $src_root\cmake\patches\protobuf\protobuf_cmake.patch | &'C:\Program Files\Git\usr\bin\patch.exe' --ignore-whitespace -p1 + # Search patch.exe + $patch_path = 'C:\Program Files\Git\usr\bin\patch.exe' + if(-not (Test-Path $patch_path -PathType Leaf)){ + $git_command_path = (Get-Command -CommandType Application git)[0].Path + Write-Host "Git command path:$git_command_path" + $git_installation_folder = Split-Path -Path (Split-Path -Path $git_command_path) + $patch_path = Join-Path -Path $git_installation_folder "usr\bin\patch.exe" + } + if(Test-Path $patch_path -PathType Leaf){ + Write-Host "Patching protobuf ..." + Get-Content $src_root\cmake\patches\protobuf\protobuf_cmake.patch | &$patch_path --ignore-whitespace -p1 + } else { + Write-Host "Skip patching protobuf since we cannot find patch.exe at $patch_path" + } - [string[]]$cmake_args = "cmake", "-DCMAKE_BUILD_TYPE=$build_config", "-Dprotobuf_BUILD_TESTS=OFF", "-DBUILD_SHARED_LIBS=OFF", "-DCMAKE_PREFIX_PATH=$install_prefix", "-DCMAKE_INSTALL_PREFIX=$install_prefix", "-Dprotobuf_MSVC_STATIC_RUNTIME=OFF" + # Run cmake to generate Visual Studio sln file + [string[]]$cmake_args = ".", "-Dprotobuf_DISABLE_RTTI=ON", "-DCMAKE_BUILD_TYPE=$build_config", "-Dprotobuf_BUILD_TESTS=OFF", "-DBUILD_SHARED_LIBS=OFF", "-DCMAKE_PREFIX_PATH=$install_prefix", "-DCMAKE_INSTALL_PREFIX=$install_prefix", "-Dprotobuf_MSVC_STATIC_RUNTIME=OFF" $cmake_args += $cmake_extra_args $p = Start-Process -FilePath $cmake_path -ArgumentList $cmake_args -NoNewWindow -Wait -PassThru @@ -392,16 +433,31 @@ function Install-Protobuf { Write-Host -Object "CMake command failed. Exitcode: $exitCode" exit $exitCode } - $cmake_args = "--build", ".", "--parallel", "--config", $build_config, "--target", "INSTALL" + + $msbuild_args = "-nodeReuse:false", "-nologo", "-nr:false", "-maxcpucount", "-p:UseMultiToolTask=true", "-p:configuration=`"$build_config`"" + if ($use_cache) { - $cmake_args += "--", "/p:CLToolExe=cl.exe /p:CLToolPath=C:\ProgramData\chocolatey\bin /p:TrackFileAccess=false /p:UseMultiToolTask=true" + $msbuild_args += "/p:CLToolExe=cl.exe /p:CLToolPath=C:\ProgramData\chocolatey\bin /p:TrackFileAccess=false /p:UseMultiToolTask=true" } - $p = Start-Process -FilePath $cmake_path -ArgumentList $cmake_args -NoNewWindow -Wait -PassThru + + $final_args = $msbuild_args + "protobuf.sln" + Write-Host $final_args + + $p = Start-Process -FilePath $msbuild_path -ArgumentList $final_args -NoNewWindow -Wait -PassThru $exitCode = $p.ExitCode if ($exitCode -ne 0) { - Write-Host -Object "CMake command failed. Exitcode: $exitCode" + Write-Host -Object "Build protobuf.sln failed. Exitcode: $exitCode" exit $exitCode } + + $final_args = $msbuild_args + "INSTALL.vcxproj" + $p = Start-Process -FilePath $msbuild_path -ArgumentList $final_args -NoNewWindow -Wait -PassThru + $exitCode = $p.ExitCode + if ($exitCode -ne 0) { + Write-Host -Object "Install protobuf failed. Exitcode: $exitCode" + exit $exitCode + } + Write-Host "Installing protobuf finished." popd } @@ -437,6 +493,18 @@ function Install-ONNX { } cd $onnx_src_dir cd * + [String]$requirements_txt_content = "protobuf==$protobuf_version`n" + foreach($line in Get-Content '.\requirements.txt') { + if($line -match "^protobuf"){ + Write-Host "Replacing protobuf version to $protobuf_version" + } else{ + $requirements_txt_content += "$line`n" + } + } + + Set-Content -Path '.\requirements.txt' -Value $requirements_txt_content + + $Env:ONNX_ML=1 if($build_config -eq 'Debug'){ $Env:DEBUG='1' diff --git a/tools/ci_build/github/windows/install_protoc.ps1 b/tools/ci_build/github/windows/install_protoc.ps1 index 390fdb1737..ca506c49b7 100644 --- a/tools/ci_build/github/windows/install_protoc.ps1 +++ b/tools/ci_build/github/windows/install_protoc.ps1 @@ -27,7 +27,7 @@ $Env:MSBUILDDISABLENODEREUSE=1 New-Item -Path "$install_prefix" -ItemType Directory -Force # Setup compile flags -$compile_flags = '/MP /guard:cf /Qspectre /DWIN32 /D_WINDOWS /DWINVER=0x0601 /D_WIN32_WINNT=0x0601 /DNTDDI_VERSION=0x06010000 /W3 ' +$compile_flags = @('/MP', '/guard:cf', '/Qspectre', '/DWIN32', '/D_WINDOWS', '/DWINVER=0x0A00', '/D_WIN32_WINNT=0x0A00', '/DNTDDI_VERSION=0x0A000000', '/W3') $linker_flags=@('/guard:cf') if ($use_cache) { @@ -40,24 +40,39 @@ else { if($build_config -eq 'Release'){ $compile_flags += "/O2", "/Ob2", "/DNDEBUG", "/Gw", "/GL" } elseif($build_config -eq 'RelWithDebInfo'){ - $compile_flags += "$debug_info_format", "/O2", "/Ob1", "/DNDEBUG", "/Gw", "/GL" + $compile_flags += "$debug_info_format", "/O2", "/Ob1", "/DNDEBUG", "/Gw", "/GL" } elseif($build_config -eq 'Debug'){ $compile_flags += "$debug_info_format", "/Ob0", "/Od", "/RTC1" } elseif($build_config -eq 'MinSizeRel'){ $compile_flags += "/O1", "/Ob1", "/DNDEBUG", "/Gw", "/GL" } -Write-Host $compile_flags + + # cmake args that applies to every 3rd-party library -[string[]]$cmake_extra_args="-DCMAKE_CXX_STANDARD=17 `"-DCMAKE_CXX_FLAGS=$compile_flags /EHsc`" ", "`"-DCMAKE_C_FLAGS=$compile_flags`"", "--compile-no-warning-as-error", "--fresh", "-Wno-dev" +[string[]]$cmake_extra_args="`"-DCMAKE_C_FLAGS=$compile_flags`"", "--compile-no-warning-as-error", "--fresh", "-Wno-dev" + + if($cpu_arch -eq 'x86'){ $cmake_extra_args += "-A", "Win32", "-T", "host=x64" + $compile_flags += '/Qspectre' $linker_flags += '/machine:x86' } elseif($cpu_arch -eq 'x64') { $linker_flags += '/machine:x64' + $compile_flags += '/Qspectre' +} elseif($cpu_arch -eq 'arm') { + $linker_flags += '/machine:ARM' +} elseif($cpu_arch -eq 'arm64') { + $linker_flags += '/machine:ARM64' +} elseif($cpu_arch -eq 'arm64ec') { + $linker_flags += '/machine:ARM64EC' } else { throw "$cpu_arch is not supported" } +Write-Host $compile_flags + +$cmake_extra_args += "-DCMAKE_CXX_STANDARD=17", "`"-DCMAKE_CXX_FLAGS=$compile_flags /EHsc`"" + if ($use_cache) { if ($build_config -eq 'RelWithDebInfo') { $cmake_extra_args += "-DCMAKE_CXX_FLAGS_RELWITHDEBINFO=`"/MD /Z7 /O2 /Ob1 /DNDEBUG`"" @@ -72,5 +87,15 @@ $cmake_extra_args += "-DCMAKE_EXE_LINKER_FLAGS=`"$linker_flags`"" # Find the full path of cmake.exe $cmake_command = (Get-Command -CommandType Application cmake)[0] $cmake_path = $cmake_command.Path +$msbuild_path = "C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\msbuild.exe" +if(-not (Test-Path $msbuild_path)){ + $vshwere_path = Join-Path -Path ${env:ProgramFiles(x86)} "Microsoft Visual Studio\Installer\vswhere.exe" + if(-not (Test-Path $vshwere_path -PathType Leaf)){ + $vshwere_path = Join-Path -Path ${env:ProgramFiles} "Microsoft Visual Studio\Installer\vswhere.exe" + } + $msbuild_path = &$vshwere_path -latest -requires Microsoft.Component.MSBuild -find MSBuild\**\Bin\MSBuild.exe | select-object -first 1 +} -Install-Protobuf -cmake_path $cmake_path -src_root $ort_src_root -build_config $build_config -cmake_extra_args $cmake_extra_args \ No newline at end of file +Write-Host "$msbuild_path" + +Install-Protobuf -cmake_path $cmake_path -src_root $ort_src_root -build_config $build_config -cmake_extra_args $cmake_extra_args -msbuild_path $msbuild_path diff --git a/tools/ci_build/github/windows/install_third_party_deps.ps1 b/tools/ci_build/github/windows/install_third_party_deps.ps1 index e9fa26a80f..4de5e0c0cc 100644 --- a/tools/ci_build/github/windows/install_third_party_deps.ps1 +++ b/tools/ci_build/github/windows/install_third_party_deps.ps1 @@ -3,6 +3,8 @@ # This script depends on python.exe, cmake.exe and Visual C++ spectre-mitigated libs. # Please setup AGENT_TEMPDIRECTORY env variable before running this script +# Your PATH must contain a dir that contains python.exe. And cpu arch of the python.exe +# must match the $cpu_arch passed in to this script. param ( [string]$cpu_arch = "x64", @@ -25,7 +27,7 @@ $Env:MSBUILDDISABLENODEREUSE=1 New-Item -Path "$install_prefix" -ItemType Directory -Force # Setup compile flags -$compile_flags = '/MP /guard:cf /Qspectre /DWIN32 /D_WINDOWS /DWINVER=0x0A00 /D_WIN32_WINNT=0x0A00 /DNTDDI_VERSION=0x0A000000 /W3 ' +$compile_flags = @('/MP', '/guard:cf', '/DWIN32', '/D_WINDOWS', '/DWINVER=0x0A00', '/D_WIN32_WINNT=0x0A00', '/DNTDDI_VERSION=0x0A000000', '/W3') $linker_flags=@('/guard:cf') if ($use_cache) { @@ -38,24 +40,38 @@ else { if($build_config -eq 'Release'){ $compile_flags += "/O2", "/Ob2", "/DNDEBUG", "/Gw", "/GL" } elseif($build_config -eq 'RelWithDebInfo'){ - $compile_flags += "$debug_info_format", "/O2", "/Ob1", "/DNDEBUG", "/Gw", "/GL" + $compile_flags += "$debug_info_format", "/O2", "/Ob1", "/DNDEBUG", "/Gw", "/GL" } elseif($build_config -eq 'Debug'){ $compile_flags += "$debug_info_format", "/Ob0", "/Od", "/RTC1" } elseif($build_config -eq 'MinSizeRel'){ $compile_flags += "/O1", "/Ob1", "/DNDEBUG", "/Gw", "/GL" } -Write-Host $compile_flags + # cmake args that applies to every 3rd-party library -[string[]]$cmake_extra_args="-DCMAKE_CXX_STANDARD=17 `"-DCMAKE_CXX_FLAGS=$compile_flags /EHsc`" ", "`"-DCMAKE_C_FLAGS=$compile_flags`"", "--compile-no-warning-as-error", "--fresh", "-Wno-dev" +[string[]]$cmake_extra_args="`"-DCMAKE_C_FLAGS=$compile_flags`"", "--compile-no-warning-as-error", "--fresh", "-Wno-dev" + + if($cpu_arch -eq 'x86'){ $cmake_extra_args += "-A", "Win32", "-T", "host=x64" + $compile_flags += '/Qspectre' $linker_flags += '/machine:x86' } elseif($cpu_arch -eq 'x64') { $linker_flags += '/machine:x64' + $compile_flags += '/Qspectre' +} elseif($cpu_arch -eq 'arm') { + $linker_flags += '/machine:ARM' +} elseif($cpu_arch -eq 'arm64') { + $linker_flags += '/machine:ARM64' +} elseif($cpu_arch -eq 'arm64ec') { + $linker_flags += '/machine:ARM64EC' } else { throw "$cpu_arch is not supported" } +Write-Host $compile_flags + +$cmake_extra_args += "-DCMAKE_CXX_STANDARD=17", "`"-DCMAKE_CXX_FLAGS=$compile_flags /EHsc`"" + if ($use_cache) { if ($build_config -eq 'RelWithDebInfo') { $cmake_extra_args += "-DCMAKE_CXX_FLAGS_RELWITHDEBINFO=`"/MD /Z7 /O2 /Ob1 /DNDEBUG`"" @@ -70,13 +86,19 @@ $cmake_extra_args += "-DCMAKE_EXE_LINKER_FLAGS=`"$linker_flags`"" # Find the full path of cmake.exe $cmake_command = (Get-Command -CommandType Application cmake)[0] $cmake_path = $cmake_command.Path +$vshwere_path = Join-Path -Path ${env:ProgramFiles(x86)} "Microsoft Visual Studio\Installer\vswhere.exe" +if(-not (Test-Path $vshwere_path -PathType Leaf)){ + $vshwere_path = Join-Path -Path ${env:ProgramFiles} "Microsoft Visual Studio\Installer\vswhere.exe" +} -Install-Pybind -cmake_path $cmake_path -src_root $ort_src_root -build_config $build_config -cmake_extra_args $cmake_extra_args +$msbuild_path = &$vshwere_path -latest -requires Microsoft.Component.MSBuild -find MSBuild\**\Bin\MSBuild.exe | select-object -first 1 -Install-Protobuf -cmake_path $cmake_path -src_root $ort_src_root -build_config $build_config -cmake_extra_args $cmake_extra_args +Install-Pybind -cmake_path $cmake_path -src_root $ort_src_root -build_config $build_config -cmake_extra_args $cmake_extra_args -msbuild_path $msbuild_path -# TODO: parse it from deps.txt -$protobuf_version="3.20.2" +Install-Protobuf -cmake_path $cmake_path -src_root $ort_src_root -build_config $build_config -cmake_extra_args $cmake_extra_args -msbuild_path $msbuild_path + +# This is the python Protobuf version, which is different than the C++ version that is in deps.txt +$protobuf_version="4.21.12" # ONNX doesn't allow us to specify CMake's path Install-ONNX -build_config $build_config -src_root $ort_src_root -protobuf_version $protobuf_version