diff --git a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml index 5af27815cb..2674c9f795 100644 --- a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml +++ b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml @@ -196,7 +196,7 @@ stages: buildArch: x64 msbuildPlatform: x64 packageName: x64-cuda - buildparameter: --use_cuda --cuda_version=11.8 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8" --enable_onnx_tests --enable_wcos --build_java --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80" + buildparameter: --use_cuda --cuda_version=11.8 --cuda_home=$(Agent.TempDirectory)\v11.8 --enable_onnx_tests --enable_wcos --build_java --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80" runTests: ${{ parameters.RunOnnxRuntimeTests }} buildJava: true java_artifact_id: onnxruntime_gpu diff --git a/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-vs-2022-job.yml b/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-vs-2022-job.yml index ea8cc6e1c4..8bfe234af1 100644 --- a/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-vs-2022-job.yml +++ b/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-vs-2022-job.yml @@ -130,6 +130,10 @@ jobs: workingDirectory: '$(Build.BinariesDirectory)' displayName: 'Install python modules' + - ${{ if contains(parameters.additionalBuildFlags, 'use_cuda') }}: + - powershell: | + azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v11.8" $(Agent.TempDirectory) + - ${{ if eq(parameters.WITH_CACHE, true) }}: - powershell: | if ([string]::IsNullOrEmpty((Get-Command ccache -errorAction SilentlyContinue))) diff --git a/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml b/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml index ae0241b766..07fb50b3be 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml @@ -60,6 +60,9 @@ jobs: - template: download-deps.yml + - ${{ if contains(parameters.EP_BUILD_FLAGS, 'use_cuda') }}: + - powershell: | + azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v11.8" $(Agent.TempDirectory) - task: PythonScript@0 displayName: 'Update deps.txt' inputs: diff --git a/tools/ci_build/github/azure-pipelines/templates/win-ci-vs-2019.yml b/tools/ci_build/github/azure-pipelines/templates/win-ci-vs-2019.yml index c85a4afd0b..1b20154f7e 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-ci-vs-2019.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-ci-vs-2019.yml @@ -131,6 +131,10 @@ jobs: workingDirectory: '$(Build.BinariesDirectory)' displayName: 'Install python modules' + - ${{ if contains(parameters.additionalBuildFlags, 'use_cuda') }}: + - powershell: | + azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v11.8" $(Agent.TempDirectory) + - ${{ if eq(parameters.WITH_CACHE, true) }}: - powershell: | if ([string]::IsNullOrEmpty((Get-Command ccache -errorAction SilentlyContinue))) diff --git a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml index f15bcb2030..2caaed98a1 100644 --- a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml +++ b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml @@ -118,6 +118,10 @@ stages: - template: set-version-number-variables-step.yml + - ${{ if contains(parameters.buildparameter, 'use_cuda') }}: + - powershell: | + azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v11.8" $(Agent.TempDirectory) + - task: PythonScript@0 displayName: 'Generate cmake config' inputs: diff --git a/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml index 8965dd849c..8bc9b79b38 100644 --- a/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-gpu-ci-pipeline.yml @@ -31,12 +31,12 @@ stages: - stage: cuda dependsOn: [] jobs: - - template: templates/win-ci-vs-2019.yml + - template: templates/jobs/win-ci-vs-2022-job.yml parameters: BuildConfig: 'RelWithDebInfo' EnvSetupScript: setup_env_cuda_11.bat buildArch: x64 - additionalBuildFlags: --enable_pybind --build_java --build_nodejs --use_cuda --cuda_version=11.8 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8" --enable_cuda_profiling --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=75 + additionalBuildFlags: --enable_pybind --build_java --build_nodejs --use_cuda --cuda_home="$(Agent.TempDirectory)\v11.8" --enable_cuda_profiling --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=75 msbuildPlatform: x64 isX86: false job_name_suffix: x64_RelWithDebInfo @@ -44,17 +44,17 @@ stages: RunStaticCodeAnalysis: false ORT_EP_NAME: CUDA WITH_CACHE: true - MachinePool: onnxruntime-Win2019-GPU-T4 + MachinePool: onnxruntime-Win2022-GPU-T4 - stage: training dependsOn: [] jobs: - - template: templates/win-ci-vs-2019.yml + - template: templates/jobs/win-ci-vs-2022-job.yml parameters: BuildConfig: 'RelWithDebInfo' EnvSetupScript: setup_env_cuda_11.bat buildArch: x64 - additionalBuildFlags: --enable_pybind --enable_training --use_cuda --cuda_version=11.8 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8" --skip_onnx_tests --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=75 + additionalBuildFlags: --enable_pybind --enable_training --use_cuda --cuda_home="$(Agent.TempDirectory)\v11.8" --skip_onnx_tests --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=75 msbuildPlatform: x64 isX86: false job_name_suffix: x64_RelWithDebInfo @@ -62,7 +62,7 @@ stages: RunStaticCodeAnalysis: false ORT_EP_NAME: CUDA WITH_CACHE: true - MachinePool: onnxruntime-Win2019-GPU-training-T4 + MachinePool: onnxruntime-Win2022-GPU-T4 isTraining: true - stage: dml @@ -87,13 +87,13 @@ stages: - stage: kernelDocumentation dependsOn: [] jobs: - - template: templates/win-ci-vs-2019.yml + - template: templates/jobs/win-ci-vs-2022-job.yml parameters: BuildConfig: 'RelWithDebInfo' EnvSetupScript: setup_env_cuda_11.bat buildArch: x64 # note: need to specify `--gen_doc` when creating the build config so it has to be in additionalBuildFlags - additionalBuildFlags: --gen_doc validate --skip_tests --enable_pybind --use_dml --use_cuda --cuda_version=11.8 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8" --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=75 --cmake_extra_defines onnxruntime_BUILD_UNIT_TESTS=OFF + additionalBuildFlags: --gen_doc validate --skip_tests --enable_pybind --use_dml --use_cuda --cuda_home="$(Agent.TempDirectory)\v11.8" --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=75 --cmake_extra_defines onnxruntime_BUILD_UNIT_TESTS=OFF msbuildPlatform: x64 isX86: false job_name_suffix: x64_RelWithDebInfo @@ -102,4 +102,4 @@ stages: GenerateDocumentation: true ORT_EP_NAME: CUDA # It doesn't really matter which EP is selected here since this stage is for documentation. WITH_CACHE: true - MachinePool: onnxruntime-Win2019-GPU-T4 + MachinePool: onnxruntime-Win2022-GPU-T4 diff --git a/tools/ci_build/github/azure-pipelines/win-gpu-reduce-op-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-gpu-reduce-op-ci-pipeline.yml index a95b100c2b..c40a0f82cc 100644 --- a/tools/ci_build/github/azure-pipelines/win-gpu-reduce-op-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-gpu-reduce-op-ci-pipeline.yml @@ -1,6 +1,6 @@ jobs: - job: 'build' - pool: 'Win-GPU-2019' + pool: 'onnxruntime-Win2022-GPU-T4' strategy: maxParallel: 2 matrix: @@ -38,6 +38,9 @@ jobs: workingDirectory: '$(Build.BinariesDirectory)' displayName: 'Install python modules' + - powershell: | + azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v11.8" $(Agent.TempDirectory) + - task: PowerShell@2 displayName: 'Install ONNX' inputs: @@ -49,7 +52,7 @@ jobs: displayName: 'Build and test' inputs: scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py' - arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_generator "Visual Studio 17 2022" --build_wheel --use_cuda --cuda_version=11.6 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.6" --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 "CMAKE_CUDA_ARCHITECTURES=75" --include_ops_by_config="$(Build.SourcesDirectory)\onnxruntime\test\testdata\required_ops.config"' + arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_generator "Visual Studio 17 2022" --build_wheel --use_cuda --cuda_home="$(Agent.TempDirectory)\v11.8" --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=75" --include_ops_by_config="$(Build.SourcesDirectory)\onnxruntime\test\testdata\required_ops.config"' workingDirectory: '$(Build.BinariesDirectory)' - template: templates/component-governance-component-detection-steps.yml diff --git a/tools/ci_build/github/windows/setup_env_cuda_11.bat b/tools/ci_build/github/windows/setup_env_cuda_11.bat index 489a9e297c..c50370f131 100644 --- a/tools/ci_build/github/windows/setup_env_cuda_11.bat +++ b/tools/ci_build/github/windows/setup_env_cuda_11.bat @@ -1,5 +1,5 @@ REM Copyright (c) Microsoft Corporation. All rights reserved. REM Licensed under the MIT License. -set PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8\bin;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8\extras\CUPTI\lib64;%PATH% +set PATH=%AGENT_TEMPDIRECTORY%\v11.8\bin;%AGENT_TEMPDIRECTORY%\v11.8\extras\CUPTI\lib64;%PATH% set GRADLE_OPTS=-Dorg.gradle.daemon=false