diff --git a/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cuda116.yml b/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cuda116.yml
index 2befd8838f..253c4adc9c 100644
--- a/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cuda116.yml
+++ b/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cuda116.yml
@@ -11,7 +11,7 @@ resources:
 stages:
 - template: templates/py-packaging-training-cuda-stage.yml
   parameters:
-    build_py_parameters: --enable_training --update --build
+    build_py_parameters: --enable_training --update --build --enable_training_on_device
     torch_version: '1.11.0'
     opset_version: '15'
     cuda_version: '11.6'
@@ -24,7 +24,7 @@ stages:
 
 - template: templates/py-packaging-training-cuda-stage.yml
   parameters:
-    build_py_parameters: --enable_training --update --build
+    build_py_parameters: --enable_training --update --build --enable_training_on_device
     torch_version: '1.11.0'
     opset_version: '15'
     cuda_version: '11.6'
@@ -37,7 +37,7 @@ stages:
 
 - template: templates/py-packaging-stage.yml
   parameters:
-    build_py_parameters: --enable_training
+    build_py_parameters: --enable_training --enable_training_on_device
     enable_linux_gpu: false
     enable_ubuntu_cpu: false
     enable_linux_cpu: false