diff --git a/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-build-steps.yml b/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-build-steps.yml
new file mode 100644
index 0000000000..6c9f036328
--- /dev/null
+++ b/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-build-steps.yml
@@ -0,0 +1,93 @@
+parameters:
+- name: WithCache
+  displayName: Build with Cache
+  type: boolean
+  default: false
+
+- name: Today
+  type: string
+  default: ""
+
+- name: CacheDir
+  type: string
+  default: "$(Agent.TempDirectory)/ort_ccache"
+
+- name: AdditionalKey
+  type: string
+  default: ""
+
+# it is used to pass additional arguments to build.py
+- name: BuildPyArguments
+  type: string
+  default: ""
+
+# it is used to pass msbuild arguments to VSBuild@1
+- name: MsbuildArguments
+  type: string
+  default: ""
+
+# it is used to pass platform arguments to VSBuild@1
+- name: Platform
+  type: string
+  default: "x64"
+
+# it is used to pass configuration arguments to VSBuild@1
+- name: BuildConfig
+  type: string
+
+# it is used to pass msbuildArchitecture arguments to VSBuild@1
+- name: BuildArch
+  type: string
+
+- name: CacheArg
+  type: string
+  default: '/p:CLToolExe=cl.exe /p:CLToolPath=C:\ProgramData\chocolatey\bin /p:TrackFileAccess=false /p:UseMultiToolTask=true /p:DebugInformationFormat=OldStyle'
+
+steps:
+  - ${{ if eq(parameters.WithCache, true) }}:
+    - task: Cache@2
+      inputs:
+        ${{if eq(variables['Build.SourceBranchName'], 'merge')}}:
+          key: ' "$(TODAY)" | ${{parameters.AdditionalKey}} | merge '
+        ${{else}}:
+          key: '"$(TODAY)" | onnxruntime | ${{parameters.AdditionalKey}} | $(Build.SourceVersion) '
+        path: ${{parameters.CacheDir}}
+        restoreKeys: |
+          "$(TODAY)" | onnxruntime | ${{parameters.AdditionalKey}}
+      displayName: Cache Task
+
+  - task: PythonScript@0
+    displayName: 'Generate cmake config'
+    inputs:
+      scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
+      ${{ if eq(parameters.WithCache, true) }}:
+        arguments: '${{parameters.BuildPyArguments}} --use_cache'
+      ${{ else }}:
+        arguments: '${{parameters.BuildPyArguments}}'
+      workingDirectory: '$(Build.BinariesDirectory)'
+
+  - task: VSBuild@1
+    displayName: 'Build'
+    inputs:
+      solution: '$(Build.BinariesDirectory)\${{parameters.BuildConfig}}\onnxruntime.sln'
+      platform: ${{parameters.Platform}}
+      configuration: ${{parameters.BuildConfig}}
+      ${{ if eq(parameters.WithCache, true) }}:
+        msbuildArgs: '${{parameters.MsbuildArguments}} ${{parameters.CacheArg}}'
+      ${{ else }}:
+        arguments: '${{parameters.CMakeArguments}}'
+      msbuildArchitecture: ${{parameters.BuildArch}}
+      maximumCpuCount: true
+      logProjectEvents: false
+      workingFolder: '$(Build.BinariesDirectory)\${{parameters.BuildConfig}}'
+      createLogFile: true
+    env:
+      CCACHE_DIR: ${{parameters.CacheDir}}
+
+  - ${{ if eq(parameters.WithCache, true) }}:
+    - powershell: |
+        ccache -sv
+        ccache -z
+      displayName: cache stat
+      env:
+        CCACHE_DIR: ${{parameters.CacheDir}}
diff --git a/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-vs-2022-job.yml b/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-vs-2022-job.yml
index e49c522b61..67a03beab9 100644
--- a/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-vs-2022-job.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/jobs/win-ci-vs-2022-job.yml
@@ -78,7 +78,6 @@ jobs:
     ${{ if eq(parameters.WITH_CACHE, true) }}:
       PS_CACHE_ARG: '-use_cache'
       PY_CACHE_ARG: '--use_cache'
-      MSBUILD_CACHE_ARG: '/p:CLToolExe=cl.exe /p:CLToolPath=C:\ProgramData\chocolatey\bin /p:TrackFileAccess=false /p:UseMultiToolTask=true /p:DebugInformationFormat=OldStyle'
   workspace:
     clean: all
   pool: ${{ parameters.MachinePool }}
@@ -137,48 +136,17 @@ jobs:
       nugetConfigPath: '$(Build.SourcesDirectory)\NuGet.config'
       restoreDirectory: '$(Build.BinariesDirectory)\${{ parameters.BuildConfig }}'
 
-  - ${{ if eq(parameters.WITH_CACHE, true) }}:
-    - task: Cache@2
-      inputs:
-        ${{if eq(variables['Build.SourceBranchName'], 'merge')}}:
-          key: ' "$(TODAY)" |  $(System.StageName) | ${{ parameters.BuildConfig }} | merge '
-        ${{else}}:
-          key: '"$(TODAY)" | onnxruntime | $(System.StageName) | ${{ parameters.BuildConfig }} | $(Build.SourceVersion) '
-        path: $(ORT_CACHE_DIR)
-        restoreKeys: |
-          "$(TODAY)" | onnxruntime | $(System.StageName) | ${{ parameters.BuildConfig }}
-      displayName: Cache Task
-
-  - task: PythonScript@0
-    displayName: 'Generate cmake config'
-    inputs:
-      scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
-      arguments: '--config ${{ parameters.BuildConfig }} --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_csharp --update --parallel --cmake_generator "Visual Studio 17 2022" --build_shared_lib --enable_onnx_tests ${{ variables.PY_CACHE_ARG }} ${{ parameters.additionalBuildFlags }}'
-      workingDirectory: '$(Build.BinariesDirectory)'
-
-  - task: VSBuild@1
-    displayName: 'Build'
-    inputs:
-      solution: '$(Build.BinariesDirectory)\${{ parameters.BuildConfig }}\onnxruntime.sln'
-      platform: ${{ parameters.msbuildPlatform }}
-      configuration: ${{ parameters.BuildConfig }}
-      msbuildArgs: '-maxcpucount ${{ variables.MSBUILD_CACHE_ARG }}'
-      msbuildArchitecture: ${{ parameters.buildArch }}
-      maximumCpuCount: true
-      logProjectEvents: false
-      workingFolder: '$(Build.BinariesDirectory)\${{ parameters.BuildConfig }}'
-      createLogFile: true
-    ${{ if eq(parameters.WITH_CACHE, true) }}:
-      env:
-        CCACHE_DIR: $(ORT_CACHE_DIR)
-
-  - ${{ if eq(parameters.WITH_CACHE, true) }}:
-    - powershell: |
-        ccache -sv
-        ccache -z
-      displayName: cache stat
-      env:
-        CCACHE_DIR: $(ORT_CACHE_DIR)
+  - template: win-ci-build-steps.yml
+    parameters:
+      WithCache: ${{ parameters.WITH_CACHE }}
+      Today: $(TODAY)
+      CacheDir: $(ORT_CACHE_DIR)
+      AdditionalKey: " $(System.StageName) | ${{ parameters.BuildConfig }} "
+      BuildPyArguments: '--config ${{ parameters.BuildConfig }} --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_csharp --update --parallel --cmake_generator "Visual Studio 17 2022" --build_shared_lib --enable_onnx_tests ${{ parameters.additionalBuildFlags }}'
+      MsbuildArguments: '-maxcpucount'
+      BuildArch: ${{ parameters.buildArch }}
+      Platform: ${{ parameters.msbuildPlatform }}
+      BuildConfig: ${{ parameters.BuildConfig }}
 
   - powershell: |
       Get-Volume $("$(Build.BinariesDirectory)")[0]
diff --git a/tools/ci_build/github/azure-pipelines/win-gpu-reduce-op-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-gpu-reduce-op-ci-pipeline.yml
index aab8e5ca21..70cc7e331d 100644
--- a/tools/ci_build/github/azure-pipelines/win-gpu-reduce-op-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/win-gpu-reduce-op-ci-pipeline.yml
@@ -9,13 +9,9 @@ jobs:
       minsizerel:
         BuildConfig: 'MinSizeRel'
   variables:
-    OrtPackageId: 'Microsoft.ML.OnnxRuntime'
     MsbuildArguments: '-detailedsummary -maxcpucount -consoleloggerparameters:PerformanceSummary'
-    OnnxRuntimeBuildDirectory: '$(Build.BinariesDirectory)'
-    DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
     EnvSetupScript: setup_env_cuda_11.bat
     buildArch: x64
-    setVcvars: true
     TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)]
   timeoutInMinutes: 120
   workspace:
@@ -31,12 +27,16 @@ jobs:
       WithCache: true
       Today: $(Today)
 
-  - task: PythonScript@0
-    displayName: 'Build and test'
-    inputs:
-      scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
-      arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_generator "Visual Studio 17 2022" --build_wheel --use_cuda --cuda_home="$(Agent.TempDirectory)\v11.8" --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=75" --include_ops_by_config="$(Build.SourcesDirectory)\onnxruntime\test\testdata\required_ops.config"'
-      workingDirectory: '$(Build.BinariesDirectory)'
+  - template: templates/jobs/win-ci-build-steps.yml
+    parameters:
+      WithCache: True
+      Today: $(TODAY)
+      AdditionalKey: "gpu-reduced-ops | $(BuildConfig)"
+      BuildPyArguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --update --skip_submodule_sync --cmake_generator "Visual Studio 17 2022" --build_wheel --use_cuda --cuda_home="$(Agent.TempDirectory)\v11.8" --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=75" --include_ops_by_config="$(Build.SourcesDirectory)\onnxruntime\test\testdata\required_ops.config"'
+      MsbuildArguments: $(MsbuildArguments)
+      BuildArch: $(buildArch)
+      Platform: 'x64'
+      BuildConfig: $(BuildConfig)
 
   - template: templates/component-governance-component-detection-steps.yml
     parameters :
diff --git a/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml
index 3008cc2d86..50c926fde7 100644
--- a/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/win-gpu-tensorrt-ci-pipeline.yml
@@ -22,15 +22,10 @@ jobs:
 - job: 'build'
   pool: 'onnxruntime-Win2022-GPU-T4'
   variables:
-    OrtPackageId: 'Microsoft.ML.OnnxRuntime'
     MsbuildArguments: '-detailedsummary -maxcpucount -consoleloggerparameters:PerformanceSummary'
-    OnnxRuntimeBuildDirectory: '$(Build.BinariesDirectory)'
-    DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
     EnvSetupScript: setup_env_trt.bat
     buildArch: x64
-    setVcvars: true
     BuildConfig: 'RelWithDebInfo'
-    ALLOW_RELEASED_ONNX_OPSET_ONLY: '1'
     skipComponentGovernanceDetection: true
     TODAY: $[format('{0:dd}{0:MM}{0:yyyy}', pipeline.startTime)]
   timeoutInMinutes: 150
@@ -47,25 +42,16 @@ jobs:
       WithCache: true
       Today: $(Today)
 
-  - task: PythonScript@0
-    displayName: 'Generate cmake config'
-    inputs:
-      scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
-      arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 17 2022" --build_wheel --enable_onnx_tests --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.6.1.6.Windows10.x86_64.cuda-11.8" --cuda_version=11.8 --cuda_home="$(Agent.TempDirectory)\v11.8" --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=75'
-      workingDirectory: '$(Build.BinariesDirectory)'
-
-  - task: VSBuild@1
-    displayName: 'Build'
-    inputs:
-      solution: '$(Build.BinariesDirectory)\$(BuildConfig)\onnxruntime.sln'
-      platform: 'x64'
-      configuration: $(BuildConfig)
-      msbuildArgs: $(MsbuildArguments)
-      msbuildArchitecture: $(buildArch)
-      maximumCpuCount: true
-      logProjectEvents: false
-      workingFolder: '$(Build.BinariesDirectory)\$(BuildConfig)'
-      createLogFile: true
+  - template: templates/jobs/win-ci-build-steps.yml
+    parameters:
+      WithCache: True
+      Today: $(TODAY)
+      AdditionalKey: "gpu-tensorrt | $(BuildConfig)"
+      BuildPyArguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 17 2022" --build_wheel --enable_onnx_tests --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.6.1.6.Windows10.x86_64.cuda-11.8" --cuda_version=11.8 --cuda_home="$(Agent.TempDirectory)\v11.8" --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=75'
+      MsbuildArguments: $(MsbuildArguments)
+      BuildArch: $(buildArch)
+      Platform: 'x64'
+      BuildConfig: $(BuildConfig)
 
   - task: PythonScript@0
     displayName: 'Build wheel'