diff --git a/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cuda116.yml b/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cuda116.yml
index e6677a9567..62a6f2d6a9 100644
--- a/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cuda116.yml
+++ b/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cuda116.yml
@@ -1,5 +1,11 @@
 trigger: none
 
+parameters:
+- name: DebugBuild
+  displayName: Debug Build?
+  type: boolean
+  default: false
+
 resources:
   repositories:
   - repository: manylinux
@@ -20,3 +26,4 @@ stages:
     docker_file: Dockerfile.manylinux2014_training_cuda11_6
     agent_pool: Onnxruntime-Linux-GPU
     upload_wheel: 'yes'
+    debug_build: ${{ parameters.DebugBuild }}
diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage.yml
index 12e1f28673..00408cd950 100644
--- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage.yml
+++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage.yml
@@ -46,6 +46,12 @@ parameters:
   type: string
   default: ''
 
+- name: debug_build
+  displayName: >
+    debug_build.
+  type: boolean
+  default: false
+
 stages:
 - stage: Python_Packaging
 
@@ -57,6 +63,11 @@ stages:
         value: '--final_storage'
       ${{ else }}:
         value: ''
+    - name: buildConfig
+      ${{ if eq(parameters['debug_build'], 'true') }}:
+        value: 'Debug'
+      ${{ else }}:
+        value: 'Release'
 
   jobs:
     - job: Linux_py_Training_Cuda_Wheels
@@ -96,7 +107,7 @@ stages:
             echo "Branch is "${{ variables['Build.SourceBranch'] }} && \
             echo "isMain is "${{ variables['isMain'] }} && \
             echo "final_storage is "${{ variables['finalStorage'] }}
-     
+
       - checkout: self
         clean: true
         submodules: recursive
@@ -149,7 +160,7 @@ stages:
               onnxruntimetraininggpubuild \
                 $(PythonManylinuxDir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py \
                   --build_dir /build \
-                  --config Release \
+                  --config ${{ variables['buildConfig'] }} \
                   --skip_submodule_sync \
                   --parallel \
                   --build_wheel \
@@ -164,8 +175,8 @@ stages:
         displayName: 'test ortmodule'
         inputs:
           script: |
-            rm -rf $(Build.BinariesDirectory)/Release/onnxruntime/ && \
-            files=($(Build.BinariesDirectory)/Release/dist/*.whl) && \
+            rm -rf $(Build.BinariesDirectory)/${{ variables['buildConfig'] }}/onnxruntime/ && \
+            files=($(Build.BinariesDirectory)/${{ variables['buildConfig'] }}/dist/*.whl) && \
             echo ${files[0]} && \
             whlfilename=$(basename ${files[0]}) && \
             echo $whlfilename && \
@@ -177,14 +188,14 @@ stages:
               --volume /bert_data:/bert_data \
               --volume /hf_models_cache:/hf_models_cache \
               onnxruntimetraininggpubuild \
-                bash -c " $(PythonManylinuxDir)/bin/python3 -m pip install /build/Release/dist/$whlfilename && $(PythonManylinuxDir)/bin/python3 -m onnxruntime.training.ortmodule.torch_cpp_extensions.install " ;
+                bash -c " $(PythonManylinuxDir)/bin/python3 -m pip install /build/${{ variables['buildConfig'] }}/dist/$whlfilename && $(PythonManylinuxDir)/bin/python3 -m onnxruntime.training.ortmodule.torch_cpp_extensions.install " ;
           workingDirectory: $(Build.SourcesDirectory)
 
       - task: CopyFiles@2
         displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
         inputs:
           SourceFolder: '$(Build.BinariesDirectory)'
-          Contents: 'Release/dist/*.whl'
+          Contents: "${{ variables['buildConfig'] }}/dist/*.whl"
           TargetFolder: '$(Build.ArtifactStagingDirectory)'
 
       - task: CmdLine@2
@@ -204,7 +215,7 @@ stages:
               -e NIGHTLY_BUILD \
               -e BUILD_BUILDNUMBER \
               onnxruntimetraininggpubuild \
-                bash -c " $(PythonManylinuxDir)/bin/python3 -m pip install /build/Release/dist/*.whl && $(PythonManylinuxDir)/bin/python3 -m onnxruntime.training.ortmodule.torch_cpp_extensions.install ; /onnxruntime_src/tools/doc/builddoc.sh $(PythonManylinuxDir)/bin/ /onnxruntime_src /build Release " ;
+                bash -c " $(PythonManylinuxDir)/bin/python3 -m pip install /build/${{ variables['buildConfig'] }}/dist/*.whl && $(PythonManylinuxDir)/bin/python3 -m onnxruntime.training.ortmodule.torch_cpp_extensions.install ; /onnxruntime_src/tools/doc/builddoc.sh $(PythonManylinuxDir)/bin/ /onnxruntime_src /build ${{ variables['buildConfig'] }} " ;
           workingDirectory: $(Build.SourcesDirectory)
 
       - task: CopyFiles@2
@@ -226,7 +237,7 @@ stages:
         inputs:
           script: |
             set -e -x
-            files=($(Build.ArtifactStagingDirectory)/Release/dist/*.whl) && \
+            files=($(Build.ArtifactStagingDirectory)/${{ variables['buildConfig'] }}/dist/*.whl) && \
             echo ${files[0]} && \
             python3 tools/ci_build/upload_python_package_to_azure_storage.py \
                 --python_wheel_path ${files[0]} ${{ variables['finalStorage'] }}