diff --git a/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cuda116.yml b/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cuda116.yml
index 62a6f2d6a9..11e4e82a1d 100644
--- a/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cuda116.yml
+++ b/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cuda116.yml
@@ -1,11 +1,16 @@
 trigger: none
 
 parameters:
-- name: DebugBuild
-  displayName: Debug Build?
+- name: DebugLinuxBuild
+  displayName: Debug Linux Build?
   type: boolean
   default: false
 
+- name: WindowsBuild
+  displayName: Windows Build?
+  type: boolean
+  default: true
+
 resources:
   repositories:
   - repository: manylinux
@@ -26,4 +31,16 @@ stages:
     docker_file: Dockerfile.manylinux2014_training_cuda11_6
     agent_pool: Onnxruntime-Linux-GPU
     upload_wheel: 'yes'
-    debug_build: ${{ parameters.DebugBuild }}
+    debug_build: ${{ parameters.DebugLinuxBuild }}
+
+- template: templates/py-packaging-stage.yml
+  parameters:
+    build_py_parameters: --enable_training
+    enable_linux_gpu: false
+    enable_ubuntu_cpu: false
+    enable_linux_cpu: false
+    enable_windows_cpu: ${{ parameters.WindowsBuild }}
+    enable_windows_gpu: false
+    enable_mac_cpu: false
+    enable_mac_silicon: false
+    enable_linux_arm: false
diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml
index 4196a11838..7af26f95de 100644
--- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml
@@ -417,22 +417,25 @@ stages:
             PythonVersion: '3.10'
             MsbuildPlatform: x64
             buildArch: x64
-          Python37_x86:
-            PythonVersion: '3.7'
-            MsbuildPlatform: Win32
-            buildArch: x86
-          Python38_x86:
-            PythonVersion: '3.8'
-            MsbuildPlatform: Win32
-            buildArch: x86
-          Python39_x86:
-            PythonVersion: '3.9'
-            MsbuildPlatform: Win32
-            buildArch: x86
-          Python310_x86:
-            PythonVersion: '3.10'
-            MsbuildPlatform: Win32
-            buildArch: x86
+          # Training build cannot support Win32 for now because one or more of its python
+          # dependencies does not support Win32. So, don't build a training package for Win32
+          ${{ if not(contains(parameters.build_py_parameters, '--enable_training')) }}:
+            Python37_x86:
+              PythonVersion: '3.7'
+              MsbuildPlatform: Win32
+              buildArch: x86
+            Python38_x86:
+              PythonVersion: '3.8'
+              MsbuildPlatform: Win32
+              buildArch: x86
+            Python39_x86:
+              PythonVersion: '3.9'
+              MsbuildPlatform: Win32
+              buildArch: x86
+            Python310_x86:
+              PythonVersion: '3.10'
+              MsbuildPlatform: Win32
+              buildArch: x86
       variables:
         OnnxRuntimeBuildDirectory: '$(Build.BinariesDirectory)'
         EnvSetupScript: setup_env.bat
diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage.yml
index 00408cd950..eb17e7e7ac 100644
--- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage.yml
@@ -53,7 +53,7 @@ parameters:
   default: false
 
 stages:
-- stage: Python_Packaging
+- stage: Cuda_Python_Packaging
 
   variables:
     - name: isMain