diff --git a/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-prebuild-steps.yml b/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-prebuild-steps.yml
new file mode 100644
index 0000000000..e29d9d2cab
--- /dev/null
+++ b/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-prebuild-steps.yml
@@ -0,0 +1,94 @@
+parameters:
+- name: EnvSetupScript
+  type: string
+
+- name: BuildConfig
+  type: string
+
+- name: BuildArch
+  type: string
+
+- name: DownloadCUDA
+  type: boolean
+  default: false
+
+- name: WITHCACHE
+  type: boolean
+  default: false
+
+- name: MachinePool
+  type: string
+
+- name: Today
+  type: string
+
+steps:
+- task: UsePythonVersion@0
+  inputs:
+    versionSpec: '3.8'
+    addToPath: true
+    architecture: ${{parameters.BuildArch}}
+
+- script: |
+    python -m pip install -q setuptools wheel numpy flatbuffers
+  workingDirectory: '$(Build.BinariesDirectory)'
+  displayName: 'Install python modules'
+
+- template: ../download-deps.yml
+
+- task: PythonScript@0
+  displayName: 'Update deps.txt'
+  inputs:
+    scriptPath: $(Build.SourcesDirectory)/tools/ci_build/replace_urls_in_deps.py
+    arguments: --new_dir $(Build.BinariesDirectory)/deps
+    workingDirectory: $(Build.BinariesDirectory)
+
+- template: set-winenv.yml
+  parameters:
+    EnvSetupScript: ${{parameters.EnvSetupScript}}
+    DownloadCUDA: ${{parameters.DownloadCUDA}}
+
+- ${{ if eq(parameters.WITHCACHE, true) }}:
+  - powershell: |
+      if ([string]::IsNullOrEmpty((Get-Command ccache -errorAction SilentlyContinue)))
+      {
+        choco install ccache -y --version 4.7.4
+        $ccache_path = (Get-Command ccache).Source
+        $ccache_parent_dir = (Split-Path -parent $ccache_path)
+        Copy-Item "C:\ProgramData\chocolatey\lib\ccache\tools\ccache-4.7.4-windows-x86_64\ccache.exe" -Destination "C:\ProgramData\chocolatey\bin\cl.exe"
+        Get-ChildItem $ccache_parent_dir
+        ccache --version
+      }
+    displayName: Install ccache and update PATH to use linked versions of gcc, cc, etc
+
+  - ${{ if eq(parameters.WITHCACHE, true) }}:
+    - task: Cache@2
+      # machinepool is used to ensure the compiler is same
+      inputs:
+        key: '"$(TODAY)" | ${{ parameters.buildArch }} | ${{ parameters.BuildConfig }} | ${{ parameters.MachinePool }} | $(Build.SourcesDirectory)/cmake/deps.txt, $(Build.SourcesDirectory)/tools/ci_build/github/windows/install_third_party_deps.ps1, $(Build.SourcesDirectory)/tools/ci_build/github/windows/helpers.ps1'
+        path: $(Agent.TempDirectory)/deps_ccache
+        restoreKeys: |
+          "$(TODAY)" | ${{ parameters.buildArch }} | ${{ parameters.BuildConfig }} | ${{ parameters.MachinePool }}
+      displayName: Cache Task
+
+  - task: PowerShell@2
+    displayName: 'Install ONNX'
+    inputs:
+      filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/windows/install_third_party_deps.ps1'
+      workingDirectory: '$(Build.BinariesDirectory)'
+      ${{ if eq(parameters.WITHCACHE, true) }}:
+        arguments: -cpu_arch ${{ parameters.buildArch }} -install_prefix $(Build.BinariesDirectory)\${{ parameters.BuildConfig }}\installed -build_config ${{ parameters.BuildConfig }} -use_cache
+      ${{ else }}:
+        arguments: -cpu_arch ${{ parameters.buildArch }} -install_prefix $(Build.BinariesDirectory)\${{ parameters.BuildConfig }}\installed -build_config ${{ parameters.BuildConfig }}
+    ${{ if eq(parameters.WITHCACHE, true) }}:
+      env:
+        CCACHE_DIR: $(Agent.TempDirectory)/deps_ccache
+        CCACHE_COMPILERCHECK: content
+
+  - ${{ if eq(parameters.WITHCACHE, true) }}:
+    - powershell: |
+        ccache -sv
+        ccache -z
+      displayName: cache stat
+      env:
+        CCACHE_DIR: $(Agent.TempDirectory)/deps_ccache
diff --git a/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-vs-2022-job.yml b/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-vs-2022-job.yml
index 1cd21ea199..e49c522b61 100644
--- a/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-vs-2022-job.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-vs-2022-job.yml
@@ -88,27 +88,24 @@ jobs:
     clean: true
     submodules: none
 
-  - task: UsePythonVersion@0
-    inputs:
-      versionSpec: '3.8'
-      addToPath: true
-      architecture: ${{ parameters.buildArch }}
-
-  - template: ../download-deps.yml
-
-  - task: PythonScript@0
-    displayName: 'Update deps.txt'
-    inputs:
-      scriptPath: $(Build.SourcesDirectory)/tools/ci_build/replace_urls_in_deps.py
-      arguments: --new_dir $(Build.BinariesDirectory)/deps
-      workingDirectory: $(Build.BinariesDirectory)
+  - template: win-ci-prebuild-steps.yml
+    parameters:
+      EnvSetupScript: ${{parameters.EnvSetupScript}}
+      ${{ if contains(parameters.additionalBuildFlags, 'use_cuda') }}:
+        DownloadCUDA: true
+      ${{ else }}:
+        DownloadCUDA: false
+      BuildArch: ${{parameters.buildArch}}
+      BuildConfig: ${{parameters.BuildConfig}}
+      MachinePool: ${{parameters.MachinePool}}
+      WithCache: ${{parameters.WITH_CACHE}}
+      Today: $(Today)
 
   - task: NodeTool@0
     inputs:
       versionSpec: '16.x'
       force32bit: ${{ parameters.isX86 }}
 
-
   # Our build machine doesn't have java x86
   - ${{ if eq(parameters.buildArch, 'x64') }}:
       - task: JavaToolInstaller@0
@@ -117,12 +114,6 @@ jobs:
           jdkArchitectureOption: ${{ parameters.buildArch }}
           jdkSourceOption: 'PreInstalled'
 
-  - template: set-winenv.yml
-    parameters:
-      EnvSetupScript: ${{ parameters.EnvSetupScript }}
-      ${{ if contains(parameters.additionalBuildFlags, 'use_cuda') }}:
-        DownloadCUDA: true
-
   - script: |
      set ORT_DOXY_SRC=$(Build.SourcesDirectory)
      set ORT_DOXY_OUT=$(Build.BinariesDirectory)\${{ parameters.BuildConfig }}\${{ parameters.BuildConfig }}
@@ -132,54 +123,6 @@ jobs:
     workingDirectory: '$(Build.SourcesDirectory)'
     displayName: 'API Documentation Check and generate'
 
-  - script: |
-     python -m pip install -q setuptools wheel numpy
-    workingDirectory: '$(Build.BinariesDirectory)'
-    displayName: 'Install python modules'
-
-  - ${{ if eq(parameters.WITH_CACHE, true) }}:
-    - powershell: |
-        if ([string]::IsNullOrEmpty((Get-Command ccache -errorAction SilentlyContinue)))
-        {
-          choco install ccache -y --version 4.7.4
-          $ccache_path = (Get-Command ccache).Source
-          $ccache_parent_dir = (Split-Path -parent $ccache_path)
-          Copy-Item "C:\ProgramData\chocolatey\lib\ccache\tools\ccache-4.7.4-windows-x86_64\ccache.exe" -Destination "C:\ProgramData\chocolatey\bin\cl.exe"
-          Get-ChildItem $ccache_parent_dir
-          ccache --version
-        }
-      displayName: Install ccache and update PATH to use linked versions of gcc, cc, etc
-
-  - ${{ if or(eq(parameters.RunOnnxRuntimeTests, true), eq(parameters.GenerateDocumentation, true)) }}:
-      - ${{ if eq(parameters.WITH_CACHE, true) }}:
-        - task: Cache@2
-          # machinepool is used to ensure the compiler is same
-          inputs:
-            key: '"$(TODAY)" | ${{ parameters.buildArch }} | ${{ parameters.BuildConfig }} | ${{ parameters.MachinePool }} | $(Build.SourcesDirectory)/cmake/deps.txt, $(Build.SourcesDirectory)/tools/ci_build/github/windows/install_third_party_deps.ps1, $(Build.SourcesDirectory)/tools/ci_build/github/windows/helpers.ps1'
-            path: $(DEPS_CACHE_DIR)
-            restoreKeys: |
-              "$(TODAY)" | ${{ parameters.buildArch }} | ${{ parameters.BuildConfig }} | ${{ parameters.MachinePool }}
-          displayName: Cache Task
-
-      - task: PowerShell@2
-        displayName: 'Install ONNX'
-        inputs:
-          filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/windows/install_third_party_deps.ps1'
-          workingDirectory: '$(Build.BinariesDirectory)'
-          arguments: -cpu_arch ${{ parameters.buildArch }} -install_prefix $(Build.BinariesDirectory)\${{ parameters.BuildConfig }}\installed -build_config ${{ parameters.BuildConfig }} ${{ variables['PS_CACHE_ARG'] }}
-        ${{ if eq(parameters.WITH_CACHE, true) }}:
-          env:
-            CCACHE_DIR: $(DEPS_CACHE_DIR)
-            CCACHE_COMPILERCHECK: content
-
-      - ${{ if eq(parameters.WITH_CACHE, true) }}:
-        - powershell: |
-            ccache -sv
-            ccache -z
-          displayName: cache stat
-          env:
-            CCACHE_DIR: $(DEPS_CACHE_DIR)
-
   - task: NuGetToolInstaller@0
     displayName: Use Nuget 5.7.0
     inputs:
diff --git a/tools/ci_build/github/azure-pipelines/win-ci-fuzz-testing.yml b/tools/ci_build/github/azure-pipelines/win-ci-fuzz-testing.yml
index 0eb78412de..f3a5728d65 100644
--- a/tools/ci_build/github/azure-pipelines/win-ci-fuzz-testing.yml
+++ b/tools/ci_build/github/azure-pipelines/win-ci-fuzz-testing.yml
@@ -15,39 +15,25 @@ jobs:
     buildArch: x64
     setVcvars: true
     ALLOW_RELEASED_ONNX_OPSET_ONLY: '0'
+    TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)]
   timeoutInMinutes: 120
   workspace:
     clean: all
-  steps:    
-  - task: UsePythonVersion@0
-    inputs: 
-      versionSpec: '3.8' 
-      addToPath: true 
-      architecture: $(buildArch)
+  steps:
+  - template: win-ci-prebuild-steps.yml
+    parameters:
+      EnvSetupScript: $(EnvSetupScript)
+      DownloadCUDA: false
+      BuildArch: $(buildArch)
+      BuildConfig: $(BuildConfig)
+      MachinePool: 'onnxruntime-Win-CPU-2022'
+      WithCache: true
+      Today: $(Today)
 
   - task: NodeTool@0
     inputs:
       versionSpec: '16.x'
 
-  - task: BatchScript@1
-    displayName: 'setup env'
-    inputs:
-      filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)'
-      modifyEnvironment: true
-      workingFolder: '$(Build.BinariesDirectory)'
-
-  - script: |
-     python -m pip install -q setuptools wheel numpy
-    workingDirectory: '$(Build.BinariesDirectory)'
-    displayName: 'Install python modules'
-
-  - task: PowerShell@2
-    displayName: 'Install ONNX'
-    inputs:
-      filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/windows/install_third_party_deps.ps1'
-      workingDirectory: '$(Build.BinariesDirectory)'
-      arguments: -cpu_arch x64 -install_prefix $(Build.BinariesDirectory)\$(BuildConfig)\installed -build_config $(BuildConfig)
-
   - task: NuGetToolInstaller@0
     displayName: Use Nuget 5.7.0
     inputs:
diff --git a/tools/ci_build/github/azure-pipelines/win-gpu-reduce-op-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-gpu-reduce-op-ci-pipeline.yml
index 3ac98f4d87..aab8e5ca21 100644
--- a/tools/ci_build/github/azure-pipelines/win-gpu-reduce-op-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/win-gpu-reduce-op-ci-pipeline.yml
@@ -16,32 +16,20 @@ jobs:
     EnvSetupScript: setup_env_cuda_11.bat
     buildArch: x64
     setVcvars: true
+    TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)]
   timeoutInMinutes: 120
   workspace:
     clean: all
   steps:
-  - task: UsePythonVersion@0
-    inputs:
-      versionSpec: '3.8'
-      addToPath: true
-      architecture: $(buildArch)
-
-  - script: |
-     python -m pip install -q setuptools wheel numpy flatbuffers
-    workingDirectory: '$(Build.BinariesDirectory)'
-    displayName: 'Install python modules'
-
-  - template: templates/jobs/set-winenv.yml
+  - template: templates/jobs/win-ci-prebuild-steps.yml
     parameters:
       EnvSetupScript: $(EnvSetupScript)
       DownloadCUDA: true
-
-  - task: PowerShell@2
-    displayName: 'Install ONNX'
-    inputs:
-      filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/windows/install_third_party_deps.ps1'
-      workingDirectory: '$(Build.BinariesDirectory)'
-      arguments: -cpu_arch x64 -install_prefix $(Build.BinariesDirectory)\$(BuildConfig)\installed -build_config $(BuildConfig)
+      BuildArch: $(buildArch)
+      BuildConfig: $(BuildConfig)
+      MachinePool: 'onnxruntime-Win2022-GPU-T4'
+      WithCache: true
+      Today: $(Today)
 
   - task: PythonScript@0
     displayName: 'Build and test'
diff --git a/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml
index 973fdb9a2a..3008cc2d86 100644
--- a/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml
@@ -32,41 +32,20 @@ jobs:
     BuildConfig: 'RelWithDebInfo'
     ALLOW_RELEASED_ONNX_OPSET_ONLY: '1'
     skipComponentGovernanceDetection: true
+    TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)]
   timeoutInMinutes: 150
   workspace:
     clean: all
   steps:
-  - task: UsePythonVersion@0
-    inputs:
-      versionSpec: '3.8'
-      addToPath: true
-      architecture: $(buildArch)
-
-  - script: |
-     python -m pip install -q setuptools wheel numpy
-    workingDirectory: '$(Build.BinariesDirectory)'
-    displayName: 'Install python modules'
-
-  - template: templates/download-deps.yml
-
-  - task: PythonScript@0
-    displayName: 'Update deps.txt'
-    inputs:
-      scriptPath: $(Build.SourcesDirectory)/tools/ci_build/replace_urls_in_deps.py
-      arguments: --new_dir $(Build.BinariesDirectory)/deps
-      workingDirectory: $(Build.BinariesDirectory)
-
-  - template: templates/jobs/set-winenv.yml
+  - template: templates/jobs/win-ci-prebuild-steps.yml
     parameters:
       EnvSetupScript: $(EnvSetupScript)
       DownloadCUDA: true
-
-  - task: PowerShell@2
-    displayName: 'Install ONNX'
-    inputs:
-      filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/windows/install_third_party_deps.ps1'
-      workingDirectory: '$(Build.BinariesDirectory)'
-      arguments: -cpu_arch x64 -install_prefix $(Build.BinariesDirectory)\$(BuildConfig)\installed -build_config $(BuildConfig)
+      BuildArch: $(buildArch)
+      BuildConfig: $(BuildConfig)
+      MachinePool: 'onnxruntime-Win2022-GPU-T4'
+      WithCache: true
+      Today: $(Today)
 
   - task: PythonScript@0
     displayName: 'Generate cmake config'