diff --git a/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cuda116.yml b/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cuda116.yml
index 00bf2865cc..2befd8838f 100644
--- a/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cuda116.yml
+++ b/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cuda116.yml
@@ -1,16 +1,5 @@
 trigger: none
 
-parameters:
-- name: DebugLinuxBuild
-  displayName: Debug Linux Build?
-  type: boolean
-  default: false
-
-- name: WindowsBuild
-  displayName: Windows Build?
-  type: boolean
-  default: true
-
 resources:
   repositories:
   - repository: manylinux
@@ -31,7 +20,20 @@ stages:
     docker_file: Dockerfile.manylinux2014_training_cuda11_6
     agent_pool: Onnxruntime-Linux-GPU
     upload_wheel: 'yes'
-    debug_build: ${{ parameters.DebugLinuxBuild }}
+    debug_build: false
+
+- template: templates/py-packaging-training-cuda-stage.yml
+  parameters:
+    build_py_parameters: --enable_training --update --build
+    torch_version: '1.11.0'
+    opset_version: '15'
+    cuda_version: '11.6'
+    gcc_version: 11
+    cmake_cuda_architectures: 37;50;52;60;61;70;75;80;86;87
+    docker_file: Dockerfile.manylinux2014_training_cuda11_6
+    agent_pool: Onnxruntime-Linux-GPU
+    upload_wheel: 'no'
+    debug_build: true
 
 - template: templates/py-packaging-stage.yml
   parameters:
@@ -39,7 +41,7 @@ stages:
     enable_linux_gpu: false
     enable_ubuntu_cpu: false
     enable_linux_cpu: false
-    enable_windows_cpu: ${{ parameters.WindowsBuild }}
+    enable_windows_cpu: true
     enable_windows_gpu: false
     enable_mac_cpu: false
     enable_mac_silicon: false
diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage.yml
index c54d6925eb..162b6d0e1b 100644
--- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage.yml
@@ -53,7 +53,7 @@ parameters:
   default: false
 
 stages:
-- stage: Cuda_Python_Packaging
+- stage: "Cuda_Python_Packaging_debug_${{ parameters.debug_build }}"
 
   variables:
     - name: isMain
@@ -69,6 +69,8 @@ stages:
       ${{ else }}:
         value: 'Release'
 
+  dependsOn: []
+
   jobs:
     - job: Linux_py_Training_Cuda_Wheels
       timeoutInMinutes: 180
@@ -202,7 +204,7 @@ stages:
       - task: PublishBuildArtifacts@1
         displayName: 'Publish Artifact: ONNXRuntime python wheel and documentation'
         inputs:
-          ArtifactName: onnxruntime_gpu
+          ArtifactName: "onnxruntime_gpu_${{ variables['buildConfig'] }}"
 
       - task: CmdLine@2
         displayName: 'Upload wheel'