From e63ccd3cbb9e2479af94a69a0e2c9bb9b59a54e4 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Tue, 24 Oct 2023 10:47:23 -0700 Subject: [PATCH] Install CUDA 12.2 on Windows (#18044) ### Description ### Motivation and Context --- .../c-api-noopenmp-packaging-pipelines.yml | 2 +- .../azure-pipelines/post-merge-jobs.yml | 2 +- .../templates/jobs/set-winenv.yml | 40 +++++++++++-------- .../azure-pipelines/win-gpu-ci-pipeline.yml | 6 +-- .../win-gpu-reduce-op-ci-pipeline.yml | 2 +- ...tup_env_cuda_11.bat => setup_env_cuda.bat} | 6 +++ 6 files changed, 36 insertions(+), 22 deletions(-) rename tools/ci_build/github/windows/{setup_env_cuda_11.bat => setup_env_cuda.bat} (53%) diff --git a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml index fdd8c09333..b4edf088f3 100644 --- a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml +++ b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml @@ -193,7 +193,7 @@ stages: DoCompliance: ${{ parameters.DoCompliance }} DoEsrp: ${{ parameters.DoEsrp }} stage_name_suffix: gpu - EnvSetupScript: setup_env_cuda_11.bat + EnvSetupScript: setup_env_cuda.bat buildArch: x64 msbuildPlatform: x64 packageName: x64-cuda diff --git a/tools/ci_build/github/azure-pipelines/post-merge-jobs.yml b/tools/ci_build/github/azure-pipelines/post-merge-jobs.yml index d24b0e0539..2a94499c7a 100644 --- a/tools/ci_build/github/azure-pipelines/post-merge-jobs.yml +++ b/tools/ci_build/github/azure-pipelines/post-merge-jobs.yml @@ -67,7 +67,7 @@ stages: - template: templates/jobs/win-ci-vs-2022-job.yml parameters: BuildConfig: 'RelWithDebInfo' - EnvSetupScript: setup_env_cuda_11.bat + EnvSetupScript: setup_env_cuda.bat buildArch: x64 additionalBuildFlags: --enable_pybind --build_java --build_nodejs --use_cuda --cuda_home="$(Agent.TempDirectory)\v11.8" --enable_cuda_profiling --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=86 msbuildPlatform: x64 diff --git a/tools/ci_build/github/azure-pipelines/templates/jobs/set-winenv.yml b/tools/ci_build/github/azure-pipelines/templates/jobs/set-winenv.yml index ca5a52fa61..0c8fb91a24 100644 --- a/tools/ci_build/github/azure-pipelines/templates/jobs/set-winenv.yml +++ b/tools/ci_build/github/azure-pipelines/templates/jobs/set-winenv.yml @@ -1,19 +1,27 @@ parameters: -- name: EnvSetupScript - type: string - -- name: DownloadCUDA - type: boolean - default: false + - name: EnvSetupScript + type: string + - name: DownloadCUDA + type: boolean + default: false + - name: PrimaryCUDAVersion + type: string + default: '11.8' + - name: SecondaryCUDAVersion + type: string + default: '12.2' steps: -- ${{ if eq(parameters.DownloadCUDA, 'true') }}: - - powershell: | - azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v11.8" $(Agent.TempDirectory) - -- task: BatchScript@1 - displayName: 'setup env' - inputs: - filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\${{ parameters.EnvSetupScript }}' - modifyEnvironment: true - workingFolder: '$(Build.BinariesDirectory)' + - ${{ if eq(parameters.DownloadCUDA, 'true') }}: + - powershell: | + azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v${{ parameters.PrimaryCUDAVersion }}" $(Agent.TempDirectory) + displayName: 'Download Primary CUDA SDK v${{ parameters.PrimaryCUDAVersion }}' + - powershell: | + azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v${{ parameters.SecondaryCUDAVersion }}" $(Agent.TempDirectory) + displayName: 'Download Secondary CUDA SDK v${{ parameters.SecondaryCUDAVersion }}' + - task: BatchScript@1 + displayName: 'setup env' + inputs: + filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\${{ parameters.EnvSetupScript }}' + modifyEnvironment: true + workingFolder: '$(Build.BinariesDirectory)' diff --git a/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml index 07b5388ea5..ae2a4b4cea 100644 --- a/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml @@ -40,7 +40,7 @@ stages: - template: templates/jobs/win-ci-vs-2022-job.yml parameters: BuildConfig: 'RelWithDebInfo' - EnvSetupScript: setup_env_cuda_11.bat + EnvSetupScript: setup_env_cuda.bat buildArch: x64 additionalBuildFlags: --enable_pybind --build_java --build_nodejs --use_cuda --cuda_home="$(Agent.TempDirectory)\v11.8" --enable_cuda_profiling --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=86 msbuildPlatform: x64 @@ -57,7 +57,7 @@ stages: - template: templates/jobs/win-ci-vs-2022-job.yml parameters: BuildConfig: 'RelWithDebInfo' - EnvSetupScript: setup_env_cuda_11.bat + EnvSetupScript: setup_env_cuda.bat buildArch: x64 additionalBuildFlags: --enable_pybind --enable_training --use_cuda --cuda_home="$(Agent.TempDirectory)\v11.8" --skip_onnx_tests --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=75 msbuildPlatform: x64 @@ -76,7 +76,7 @@ stages: - template: templates/jobs/win-ci-vs-2022-job.yml parameters: BuildConfig: 'RelWithDebInfo' - EnvSetupScript: setup_env_cuda_11.bat + EnvSetupScript: setup_env_cuda.bat buildArch: x64 # note: need to specify `--gen_doc` when creating the build config so it has to be in additionalBuildFlags additionalBuildFlags: --gen_doc validate --skip_tests --enable_pybind --use_dml --use_cuda --cuda_home="$(Agent.TempDirectory)\v11.8" --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=86 --cmake_extra_defines onnxruntime_BUILD_UNIT_TESTS=OFF diff --git a/tools/ci_build/github/azure-pipelines/win-gpu-reduce-op-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-gpu-reduce-op-ci-pipeline.yml index b5db8a5201..d0f9772da7 100644 --- a/tools/ci_build/github/azure-pipelines/win-gpu-reduce-op-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-gpu-reduce-op-ci-pipeline.yml @@ -10,7 +10,7 @@ jobs: BuildConfig: 'MinSizeRel' variables: MsbuildArguments: '-detailedsummary -maxcpucount -consoleloggerparameters:PerformanceSummary' - EnvSetupScript: setup_env_cuda_11.bat + EnvSetupScript: setup_env_cuda.bat buildArch: x64 TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)] timeoutInMinutes: 120 diff --git a/tools/ci_build/github/windows/setup_env_cuda_11.bat b/tools/ci_build/github/windows/setup_env_cuda.bat similarity index 53% rename from tools/ci_build/github/windows/setup_env_cuda_11.bat rename to tools/ci_build/github/windows/setup_env_cuda.bat index 1308e43a4f..96569cbe0f 100644 --- a/tools/ci_build/github/windows/setup_env_cuda_11.bat +++ b/tools/ci_build/github/windows/setup_env_cuda.bat @@ -6,4 +6,10 @@ if exist PATH=%AGENT_TEMPDIRECTORY%\v11.8\ { } else { set PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8\bin;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8\extras\CUPTI\lib64;%PATH% } +@REM The default version is still cuda v11.8, because set cuda v12.2 after it +if exist PATH=%AGENT_TEMPDIRECTORY%\v12.2\ { + set PATH=%PATH%;%AGENT_TEMPDIRECTORY%\v12.2\bin;%AGENT_TEMPDIRECTORY%\v12.2\extras\CUPTI\lib64 +} else { + set PATH=%PATH%;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.2\bin;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v12.2\extras\CUPTI\lib64 +} set GRADLE_OPTS=-Dorg.gradle.daemon=false