diff --git a/tools/ci_build/github/azure-pipelines/templates/jobs/set-winenv.yml b/tools/ci_build/github/azure-pipelines/templates/jobs/set-winenv.yml
new file mode 100644
index 0000000000..ca5a52fa61
--- /dev/null
+++ b/tools/ci_build/github/azure-pipelines/templates/jobs/set-winenv.yml
@@ -0,0 +1,19 @@
+parameters:
+- name: EnvSetupScript
+  type: string
+
+- name: DownloadCUDA
+  type: boolean
+  default: false
+
+steps:
+- ${{ if eq(parameters.DownloadCUDA, 'true') }}:
+  - powershell: |
+      azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v11.8" $(Agent.TempDirectory)
+
+- task: BatchScript@1
+  displayName: 'setup env'
+  inputs:
+    filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\${{ parameters.EnvSetupScript }}'
+    modifyEnvironment: true
+    workingFolder: '$(Build.BinariesDirectory)'
diff --git a/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-vs-2022-job.yml b/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-vs-2022-job.yml
index ca118fedfb..c4257888e1 100644
--- a/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-vs-2022-job.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-vs-2022-job.yml
@@ -58,7 +58,7 @@ parameters:
 - name: OnnxruntimeTestGpuDeviceId
   type: number
   default: 0
-  
+
 
 jobs:
 - job: build_${{ parameters.job_name_suffix }}
@@ -108,6 +108,7 @@ jobs:
       versionSpec: '16.x'
       force32bit: ${{ parameters.isX86 }}
 
+
   # Our build machine doesn't have java x86
   - ${{ if eq(parameters.buildArch, 'x64') }}:
       - task: JavaToolInstaller@0
@@ -116,12 +117,11 @@ jobs:
           jdkArchitectureOption: ${{ parameters.buildArch }}
           jdkSourceOption: 'PreInstalled'
 
-  - task: BatchScript@1
-    displayName: 'setup env'
-    inputs:
-      filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\${{ parameters.EnvSetupScript }}'
-      modifyEnvironment: true
-      workingFolder: '$(Build.BinariesDirectory)'
+  - template: set-winenv.yml
+    parameters:
+      EnvSetupScript: ${{ parameters.EnvSetupScript }}
+      ${{ if contains(parameters.additionalBuildFlags, 'use_cuda') }}:
+        DownloadCUDA: true
 
   - script: |
      set ORT_DOXY_SRC=$(Build.SourcesDirectory)
@@ -137,10 +137,6 @@ jobs:
     workingDirectory: '$(Build.BinariesDirectory)'
     displayName: 'Install python modules'
 
-  - ${{ if contains(parameters.additionalBuildFlags, 'use_cuda') }}:
-    - powershell: |
-       azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v11.8" $(Agent.TempDirectory)
-
   - ${{ if eq(parameters.WITH_CACHE, true) }}:
     - powershell: |
         if ([string]::IsNullOrEmpty((Get-Command ccache -errorAction SilentlyContinue)))
diff --git a/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml b/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml
index cde26f7767..b571e04f35 100644
--- a/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml
@@ -48,13 +48,6 @@ jobs:
           tsaConfigFilePath: '$(Build.SourcesDirectory)\.config\tsaoptions.json'
           appendSourceBranchName: false
 
-      - task: BatchScript@1
-        displayName: 'setup env'
-        inputs:
-          filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\${{ parameters.ENV_SETUP_SCRIPT }}'
-          modifyEnvironment: true
-          workingFolder: '$(Build.BinariesDirectory)'
-
       - task: PythonScript@0
         inputs:
           scriptSource: inline
@@ -68,8 +61,10 @@ jobs:
 
       - template: download-deps.yml
 
-      - powershell: |
-         azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v11.8" $(Agent.TempDirectory)
+      - template: jobs/set-winenv.yml
+        parameters:
+          EnvSetupScript: ${{ parameters.ENV_SETUP_SCRIPT }}
+          DownloadCUDA: true
 
       - task: PythonScript@0
         displayName: 'Update deps.txt'
diff --git a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml
index 00335e4672..721a28dd37 100644
--- a/tools/ci_build/github/azure-pipelines/templates/win-ci.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/win-ci.yml
@@ -96,12 +96,11 @@ stages:
         inputs:
           versionSpec: '16.x'
 
-      - task: BatchScript@1
-        displayName: 'setup env'
-        inputs:
-          filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\${{ parameters.EnvSetupScript }}'
-          modifyEnvironment: true
-          workingFolder: '$(Build.BinariesDirectory)'
+      - template: jobs/set-winenv.yml
+        parameters:
+          EnvSetupScript: ${{ parameters.EnvSetupScript }}
+          ${{ if contains(parameters.buildparameter, 'use_cuda') }}:
+            DownloadCUDA: true
 
       - template: download-deps.yml
 
@@ -121,10 +120,6 @@ stages:
 
       - template: set-version-number-variables-step.yml
 
-      - ${{ if contains(parameters.buildparameter, 'use_cuda') }}:
-        - powershell: |
-           azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v11.8" $(Agent.TempDirectory)
-
       - task: PythonScript@0
         displayName: 'Generate cmake config'
         inputs:
diff --git a/tools/ci_build/github/azure-pipelines/win-gpu-reduce-op-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-gpu-reduce-op-ci-pipeline.yml
index c40a0f82cc..3ac98f4d87 100644
--- a/tools/ci_build/github/azure-pipelines/win-gpu-reduce-op-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/win-gpu-reduce-op-ci-pipeline.yml
@@ -21,25 +21,20 @@ jobs:
     clean: all
   steps:
   - task: UsePythonVersion@0
-    inputs: 
+    inputs:
       versionSpec: '3.8'
       addToPath: true
       architecture: $(buildArch)
 
-  - task: BatchScript@1
-    displayName: 'setup env'
-    inputs:
-      filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)'
-      modifyEnvironment: true
-      workingFolder: '$(Build.BinariesDirectory)'
-
   - script: |
      python -m pip install -q setuptools wheel numpy flatbuffers
     workingDirectory: '$(Build.BinariesDirectory)'
     displayName: 'Install python modules'
 
-  - powershell: |
-      azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v11.8" $(Agent.TempDirectory)
+  - template: templates/jobs/set-winenv.yml
+    parameters:
+      EnvSetupScript: $(EnvSetupScript)
+      DownloadCUDA: true
 
   - task: PowerShell@2
     displayName: 'Install ONNX'
@@ -58,4 +53,3 @@ jobs:
   - template: templates/component-governance-component-detection-steps.yml
     parameters :
       condition : 'succeeded'
-
diff --git a/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml
index c2dd5a4f2b..a053dc4e56 100644
--- a/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml
@@ -22,13 +22,6 @@ jobs:
       addToPath: true
       architecture: $(buildArch)
 
-  - task: BatchScript@1
-    displayName: 'setup env'
-    inputs:
-      filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)'
-      modifyEnvironment: true
-      workingFolder: '$(Build.BinariesDirectory)'
-
   - script: |
      python -m pip install -q setuptools wheel numpy
     workingDirectory: '$(Build.BinariesDirectory)'
@@ -43,16 +36,10 @@ jobs:
       arguments: --new_dir $(Build.BinariesDirectory)/deps
       workingDirectory: $(Build.BinariesDirectory)
 
-  - powershell: |
-      azcopy.exe cp --recursive "https://lotusscus.blob.core.windows.net/models/cuda_sdk/v11.8" $(Agent.TempDirectory)
-    displayName: 'Download CUDA SDK'
-
-  - task: BatchScript@1
-    displayName: 'setup env'
-    inputs:
-      filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)'
-      modifyEnvironment: true
-      workingFolder: '$(Build.BinariesDirectory)'
+  - template: templates/jobs/set-winenv.yml
+    parameters:
+      EnvSetupScript: $(EnvSetupScript)
+      DownloadCUDA: true
 
   - task: PowerShell@2
     displayName: 'Install ONNX'
diff --git a/tools/ci_build/github/windows/setup_env_trt.bat b/tools/ci_build/github/windows/setup_env_trt.bat
index c65eb23ca5..1308e43a4f 100644
--- a/tools/ci_build/github/windows/setup_env_trt.bat
+++ b/tools/ci_build/github/windows/setup_env_trt.bat
@@ -1,2 +1,9 @@
-set PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8\bin;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8\extras\CUPTI\lib64;%PATH%
+REM Copyright (c) Microsoft Corporation. All rights reserved.
+REM Licensed under the MIT License.
+
+if exist PATH=%AGENT_TEMPDIRECTORY%\v11.8\ {
+    set PATH=%AGENT_TEMPDIRECTORY%\v11.8\bin;%AGENT_TEMPDIRECTORY%\v11.8\extras\CUPTI\lib64;%PATH%
+} else {
+    set PATH=C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8\bin;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.8\extras\CUPTI\lib64;%PATH%
+}
 set GRADLE_OPTS=-Dorg.gradle.daemon=false