From e2b6827a590d8199017929d0c15e3e065058b23e Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Mon, 21 Aug 2023 13:04:36 -0700 Subject: [PATCH] Add a CUDA 12.x pipeline and improve install_third_party_deps.ps1 (#17231) ### Description 1. Add a CUDA 12.x pipeline 2. Improve install_third_party_deps.ps1: avoid using Start-process. Directly call the command instead. ### Motivation and Context Since our official packages and all CI pipelines still use CUDA 11.x, we need extra pipelines to validate our source code level compatibility with CUDA 12.x. BTW for sure the prebuilt binaries in our release page are not compatible with CUDA 12.x. Do not report bugs for that. AB#15152 --- .github/workflows/windows.yml | 32 +++ tools/ci_build/github/windows/helpers.ps1 | 251 ++++++++++++++---- .../windows/install_third_party_deps.ps1 | 3 +- 3 files changed, 230 insertions(+), 56 deletions(-) diff --git a/.github/workflows/windows.yml b/.github/workflows/windows.yml index 693fafbf82..8cd62db77b 100644 --- a/.github/workflows/windows.yml +++ b/.github/workflows/windows.yml @@ -10,7 +10,39 @@ concurrency: group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }} cancel-in-progress: true +env: + AZCOPY_AUTO_LOGIN_TYPE: MSI + AZCOPY_MSI_CLIENT_ID: 63b63039-6328-442f-954b-5a64d124e5b4 + jobs: + Windows-CUDA-12: + runs-on: ["self-hosted", "1ES.Pool=onnxruntime-github-vs2022-mms"] + steps: + - uses: actions/checkout@v3 + with: + submodules: false + - uses: actions/setup-python@v4 + with: + python-version: '3.11.x' + architecture: 'x64' + + - uses: actions/setup-node@v3 + with: + node-version: 18 + + - name: Download cuda + run: azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v12.2" cuda_sdk + + + - name: Delete build folder + run: | + if (Test-Path D:\b) { Remove-Item -Recurse -Force D:\b } + &tools\ci_build\github\windows\install_third_party_deps.ps1 -cpu_arch x64 -install_prefix D:\b\Debug\installed -build_config Debug + + # The build machine doesn't have a GPU. So the value of CMAKE_CUDA_ARCHITECTURES doesn't matter. + - name: Build code + run: python tools\ci_build\build.py --windows_sdk_version 10.0.22621.0 --enable_training --build_java --config Debug --build_dir D:\b --skip_submodule_sync --build_csharp --update --build --parallel --cmake_generator "Visual Studio 17 2022" --build_shared_lib --enable_pybind --use_cuda --cuda_home=${{ github.workspace }}\cuda_sdk\v12.2 --enable_cuda_profiling --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=75 + Onnxruntime-TVM: runs-on: windows-latest steps: diff --git a/tools/ci_build/github/windows/helpers.ps1 b/tools/ci_build/github/windows/helpers.ps1 index 30052ae93e..6e81f901a8 100644 --- a/tools/ci_build/github/windows/helpers.ps1 +++ b/tools/ci_build/github/windows/helpers.ps1 @@ -337,41 +337,176 @@ function Install-Pybind { cd build [string[]]$cmake_args = "..", "-DCMAKE_INSTALL_PREFIX=$install_prefix", "-DBUILD_TESTING=OFF" $cmake_args += $cmake_extra_args - $p = Start-Process -FilePath $cmake_path -ArgumentList $cmake_args -NoNewWindow -Wait -PassThru - $exitCode = $p.ExitCode - if ($exitCode -ne 0) { - Write-Host -Object "CMake command failed. Exitcode: $exitCode" - exit $exitCode + &$cmake_path $cmake_args + if ($lastExitCode -ne 0) { + Write-Host -Object "CMake command failed. Exitcode: $exitCode" + exit $lastExitCode } $msbuild_args = "-nodeReuse:false", "-nologo", "-nr:false", "-maxcpucount", "-p:UseMultiToolTask=true", "-p:configuration=`"$build_config`"" if ($use_cache) { - $msbuild_args += "/p:CLToolExe=cl.exe /p:CLToolPath=C:\ProgramData\chocolatey\bin /p:TrackFileAccess=false /p:UseMultiToolTask=true" + $msbuild_args += "/p:CLToolExe=cl.exe", "/p:CLToolPath=C:\ProgramData\chocolatey\bin", "/p:TrackFileAccess=false", "/p:UseMultiToolTask=true" } $final_args = $msbuild_args + "pybind11.sln" - Write-Host $final_args - - $p = Start-Process -FilePath $msbuild_path -ArgumentList $final_args -NoNewWindow -Wait -PassThru - $exitCode = $p.ExitCode - if ($exitCode -ne 0) { - Write-Host -Object "Build pybind11.sln failed. Exitcode: $exitCode" - exit $exitCode - } - + &$msbuild_path $final_args $final_args = $msbuild_args + "INSTALL.vcxproj" - $p = Start-Process -FilePath $msbuild_path -ArgumentList $final_args -NoNewWindow -Wait -PassThru - $exitCode = $p.ExitCode - if ($exitCode -ne 0) { - Write-Host -Object "Install pybind failed. Exitcode: $exitCode" - exit $exitCode - } + &$msbuild_path $final_args + Write-Host "Installing pybind finished." popd } +<# + .Description + The Install-Abseil function installs Google's abseil library from source. + + .PARAMETER cmake_path + The full path of cmake.exe + + .PARAMETER src_root + The full path of ONNX Runtime's top level source diretory + + .PARAMETER build_config + The value of CMAKE_BUILD_TYPE, can be Debug, Release, RelWithDebInfo or MinSizeRel. +#> +function Install-Abseil { + + param ( + [Parameter(Mandatory)][string]$cmake_path, + [Parameter(Mandatory)][string]$msbuild_path, + [Parameter(Mandatory)][string]$src_root, + [Parameter(Mandatory)][CMakeBuildType]$build_config, + [Parameter(Mandatory)][string[]]$cmake_extra_args + ) + + pushd . + $url=Get-DownloadURL -name abseil_cpp -src_root $src_root + Write-Host "Downloading abseil_cpp from $url" + $temp_dir = Get-TempDirectory + $absl_src_dir = Join-Path $temp_dir "abseil_cpp" + $download_finished = DownloadAndExtract -Uri $url -InstallDirectory $absl_src_dir -Force + if(-Not $download_finished){ + exit 1 + } + cd $absl_src_dir + cd * + + # Search patch.exe + $patch_path = 'C:\Program Files\Git\usr\bin\patch.exe' + if(-not (Test-Path $patch_path -PathType Leaf)){ + $git_command_path = (Get-Command -CommandType Application git)[0].Path + Write-Host "Git command path:$git_command_path" + $git_installation_folder = Split-Path -Path (Split-Path -Path $git_command_path) + $patch_path = Join-Path -Path $git_installation_folder "usr\bin\patch.exe" + } + if(Test-Path $patch_path -PathType Leaf){ + Write-Host "Patching abseil ..." + Get-Content $src_root\cmake\patches\abseil\absl_windows.patch | &$patch_path --ignore-whitespace -p1 + } else { + Write-Host "Skip patching abseil since we cannot find patch.exe at $patch_path" + } + + # Run cmake to generate Visual Studio sln file + [string[]]$cmake_args = ".", "-DABSL_PROPAGATE_CXX_STD=ON", "-DCMAKE_BUILD_TYPE=$build_config", "-DBUILD_TESTING=OFF", "-DABSL_USE_EXTERNAL_GOOGLETEST=ON", "-DCMAKE_PREFIX_PATH=$install_prefix", "-DCMAKE_INSTALL_PREFIX=$install_prefix" + $cmake_args += $cmake_extra_args + + &$cmake_path $cmake_args + if ($lastExitCode -ne 0) { + Write-Host -Object "CMake command failed. Exitcode: $exitCode" + exit $lastExitCode + } + + $msbuild_args = "-nodeReuse:false", "-nologo", "-nr:false", "-maxcpucount", "-p:UseMultiToolTask=true", "-p:configuration=`"$build_config`"" + + if ($use_cache) { + $msbuild_args += "/p:CLToolExe=cl.exe", "/p:CLToolPath=C:\ProgramData\chocolatey\bin", "/p:TrackFileAccess=false", "/p:UseMultiToolTask=true" + } + + + $final_args = $msbuild_args + "absl.sln" + &$msbuild_path $final_args + if ($lastExitCode -ne 0) { + exit $lastExitCode + } + $final_args = $msbuild_args + "INSTALL.vcxproj" + &$msbuild_path $final_args + if ($lastExitCode -ne 0) { + exit $lastExitCode + } + Write-Host "Installing absl finished." + popd +} + +<# + .Description + The Install-UTF8-Range function installs Google's utf8_range library from source. + utf8_range depends on Abseil. + + .PARAMETER cmake_path + The full path of cmake.exe + + .PARAMETER src_root + The full path of ONNX Runtime's top level source diretory + + .PARAMETER build_config + The value of CMAKE_BUILD_TYPE, can be Debug, Release, RelWithDebInfo or MinSizeRel. +#> +function Install-UTF8-Range { + + param ( + [Parameter(Mandatory)][string]$cmake_path, + [Parameter(Mandatory)][string]$msbuild_path, + [Parameter(Mandatory)][string]$src_root, + [Parameter(Mandatory)][CMakeBuildType]$build_config, + [Parameter(Mandatory)][string[]]$cmake_extra_args + ) + + pushd . + $url=Get-DownloadURL -name utf8_range -src_root $src_root + Write-Host "Downloading utf8_range from $url" + $temp_dir = Get-TempDirectory + $absl_src_dir = Join-Path $temp_dir "utf8_range" + $download_finished = DownloadAndExtract -Uri $url -InstallDirectory $absl_src_dir -Force + if(-Not $download_finished){ + exit 1 + } + cd $absl_src_dir + cd * + + # Run cmake to generate Visual Studio sln file + [string[]]$cmake_args = ".", "-Dutf8_range_ENABLE_TESTS=OFF", "-Dutf8_range_ENABLE_INSTALL=ON", "-DCMAKE_BUILD_TYPE=$build_config", "-DBUILD_TESTING=OFF", "-DCMAKE_PREFIX_PATH=$install_prefix", "-DCMAKE_INSTALL_PREFIX=$install_prefix" + $cmake_args += $cmake_extra_args + + &$cmake_path $cmake_args + if ($lastExitCode -ne 0) { + Write-Host -Object "CMake command failed. Exitcode: $exitCode" + exit $lastExitCode + } + + $msbuild_args = "-nodeReuse:false", "-nologo", "-nr:false", "-maxcpucount", "-p:UseMultiToolTask=true", "-p:configuration=`"$build_config`"" + + if ($use_cache) { + $msbuild_args += "/p:CLToolExe=cl.exe", "/p:CLToolPath=C:\ProgramData\chocolatey\bin", "/p:TrackFileAccess=false", "/p:UseMultiToolTask=true" + } + + + $final_args = $msbuild_args + "utf8_range.sln" + &$msbuild_path $final_args + if ($lastExitCode -ne 0) { + exit $lastExitCode + } + $final_args = $msbuild_args + "INSTALL.vcxproj" + &$msbuild_path $final_args + if ($lastExitCode -ne 0) { + exit $lastExitCode + } + Write-Host "Installing utf8_range finished." + popd +} + <# .Description The Install-ONNX function installs ONNX python package from source and also the python packages that it depends on. @@ -408,14 +543,14 @@ function Install-Protobuf { } cd $protobuf_src_dir cd * - # Search patch.exe - $patch_path = 'C:\Program Files\Git\usr\bin\patch.exe' - if(-not (Test-Path $patch_path -PathType Leaf)){ + # Search patch.exe + $patch_path = 'C:\Program Files\Git\usr\bin\patch.exe' + if(-not (Test-Path $patch_path -PathType Leaf)){ $git_command_path = (Get-Command -CommandType Application git)[0].Path Write-Host "Git command path:$git_command_path" $git_installation_folder = Split-Path -Path (Split-Path -Path $git_command_path) $patch_path = Join-Path -Path $git_installation_folder "usr\bin\patch.exe" - } + } if(Test-Path $patch_path -PathType Leaf){ Write-Host "Patching protobuf ..." Get-Content $src_root\cmake\patches\protobuf\protobuf_cmake.patch | &$patch_path --ignore-whitespace -p1 @@ -424,38 +559,30 @@ function Install-Protobuf { } # Run cmake to generate Visual Studio sln file - [string[]]$cmake_args = ".", "-Dprotobuf_DISABLE_RTTI=ON", "-DCMAKE_BUILD_TYPE=$build_config", "-Dprotobuf_BUILD_TESTS=OFF", "-DBUILD_SHARED_LIBS=OFF", "-DCMAKE_PREFIX_PATH=$install_prefix", "-DCMAKE_INSTALL_PREFIX=$install_prefix", "-Dprotobuf_MSVC_STATIC_RUNTIME=OFF" + [string[]]$cmake_args = ".", "-Dprotobuf_DISABLE_RTTI=ON", "-DCMAKE_BUILD_TYPE=$build_config", "-Dprotobuf_BUILD_TESTS=OFF", "-Dprotobuf_USE_EXTERNAL_GTEST=ON", "-DBUILD_SHARED_LIBS=OFF", "-DCMAKE_PREFIX_PATH=$install_prefix", "-DCMAKE_INSTALL_PREFIX=$install_prefix", "-Dprotobuf_MSVC_STATIC_RUNTIME=OFF", "-Dprotobuf_ABSL_PROVIDER=package" $cmake_args += $cmake_extra_args - $p = Start-Process -FilePath $cmake_path -ArgumentList $cmake_args -NoNewWindow -Wait -PassThru - $exitCode = $p.ExitCode - if ($exitCode -ne 0) { - Write-Host -Object "CMake command failed. Exitcode: $exitCode" - exit $exitCode + &$cmake_path $cmake_args + if ($lastExitCode -ne 0) { + Write-Host -Object "CMake command failed. Exitcode: $exitCode" + exit $lastExitCode } - + $msbuild_args = "-nodeReuse:false", "-nologo", "-nr:false", "-maxcpucount", "-p:UseMultiToolTask=true", "-p:configuration=`"$build_config`"" if ($use_cache) { - $msbuild_args += "/p:CLToolExe=cl.exe /p:CLToolPath=C:\ProgramData\chocolatey\bin /p:TrackFileAccess=false /p:UseMultiToolTask=true" + $msbuild_args += "/p:CLToolExe=cl.exe", "/p:CLToolPath=C:\ProgramData\chocolatey\bin", "/p:TrackFileAccess=false", "/p:UseMultiToolTask=true" } $final_args = $msbuild_args + "protobuf.sln" - Write-Host $final_args - - $p = Start-Process -FilePath $msbuild_path -ArgumentList $final_args -NoNewWindow -Wait -PassThru - $exitCode = $p.ExitCode - if ($exitCode -ne 0) { - Write-Host -Object "Build protobuf.sln failed. Exitcode: $exitCode" - exit $exitCode + &$msbuild_path $final_args + if ($lastExitCode -ne 0) { + exit $lastExitCode } - $final_args = $msbuild_args + "INSTALL.vcxproj" - $p = Start-Process -FilePath $msbuild_path -ArgumentList $final_args -NoNewWindow -Wait -PassThru - $exitCode = $p.ExitCode - if ($exitCode -ne 0) { - Write-Host -Object "Install protobuf failed. Exitcode: $exitCode" - exit $exitCode + &$msbuild_path $final_args + if ($lastExitCode -ne 0) { + exit $lastExitCode } Write-Host "Installing protobuf finished." popd @@ -476,6 +603,9 @@ function Install-ONNX { pushd . + Write-Host "Uninstalling onnx and ignore errors if there is any..." + python.exe -m pip uninstall -y onnx -qq + Write-Host "Installing python packages..." $p = Start-Process -NoNewWindow -Wait -PassThru -FilePath "python.exe" -ArgumentList "-m", "pip", "install", "--disable-pip-version-check", "setuptools", "wheel", "numpy", "protobuf==$protobuf_version" $exitCode = $p.ExitCode @@ -493,6 +623,22 @@ function Install-ONNX { } cd $onnx_src_dir cd * + + # Search patch.exe + $patch_path = 'C:\Program Files\Git\usr\bin\patch.exe' + if(-not (Test-Path $patch_path -PathType Leaf)){ + $git_command_path = (Get-Command -CommandType Application git)[0].Path + Write-Host "Git command path:$git_command_path" + $git_installation_folder = Split-Path -Path (Split-Path -Path $git_command_path) + $patch_path = Join-Path -Path $git_installation_folder "usr\bin\patch.exe" + } + if(Test-Path $patch_path -PathType Leaf){ + Write-Host "Patching onnx ..." + Get-Content $src_root\cmake\patches\onnx\onnx.patch | &$patch_path --ignore-whitespace -p1 + } else { + Write-Host "Skip patching onnx since we cannot find patch.exe at $patch_path" + } + [String]$requirements_txt_content = "protobuf==$protobuf_version`n" foreach($line in Get-Content '.\requirements.txt') { if($line -match "^protobuf"){ @@ -509,16 +655,11 @@ function Install-ONNX { if($build_config -eq 'Debug'){ $Env:DEBUG='1' } - $Env:CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=OFF -DProtobuf_USE_STATIC_LIBS=ON -DONNX_USE_LITE_PROTO=OFF" + $Env:CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=OFF -DProtobuf_USE_STATIC_LIBS=ON -DONNX_USE_LITE_PROTO=OFF -DCMAKE_PREFIX_PATH=$install_prefix" - $p = Start-Process -NoNewWindow -Wait -PassThru -FilePath "python.exe" -ArgumentList "setup.py", "bdist_wheel" - $exitCode = $p.ExitCode - if ($exitCode -ne 0) { - Write-Host -Object "Generate wheel file failed. Exitcode: $exitCode" - exit $exitCode - } - Write-Host "Uninstalling onnx and ignore errors if there is any..." - python -m pip uninstall -y onnx -qq + python.exe "setup.py" "bdist_wheel" + + Write-Host "Installing the newly built ONNX python package" Get-ChildItem -Path dist/*.whl | foreach { $p = Start-Process -NoNewWindow -Wait -PassThru -FilePath "python.exe" -ArgumentList "-m", "pip", "--disable-pip-version-check", "install", "--upgrade", $_.fullname diff --git a/tools/ci_build/github/windows/install_third_party_deps.ps1 b/tools/ci_build/github/windows/install_third_party_deps.ps1 index 4de5e0c0cc..c30b576953 100644 --- a/tools/ci_build/github/windows/install_third_party_deps.ps1 +++ b/tools/ci_build/github/windows/install_third_party_deps.ps1 @@ -95,9 +95,10 @@ $msbuild_path = &$vshwere_path -latest -requires Microsoft.Component.MSBuild -fi Install-Pybind -cmake_path $cmake_path -src_root $ort_src_root -build_config $build_config -cmake_extra_args $cmake_extra_args -msbuild_path $msbuild_path +Install-Abseil -cmake_path $cmake_path -src_root $ort_src_root -build_config $build_config -cmake_extra_args $cmake_extra_args -msbuild_path $msbuild_path + Install-Protobuf -cmake_path $cmake_path -src_root $ort_src_root -build_config $build_config -cmake_extra_args $cmake_extra_args -msbuild_path $msbuild_path -# This is the python Protobuf version, which is different than the C++ version that is in deps.txt $protobuf_version="4.21.12" # ONNX doesn't allow us to specify CMake's path