diff --git a/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-rocm.yml b/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-rocm.yml
index aefa3f7962..3e8089b904 100644
--- a/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-rocm.yml
+++ b/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-rocm.yml
@@ -50,15 +50,27 @@ stages:
   - template: templates/rocm.yml
     parameters:
       PythonVersion: '3.7'
-      RocmVersion: '5.4'
+      RocmVersion: '5.4.2'
+  - template: templates/rocm.yml
+    parameters:
+      PythonVersion: '3.8'
+      RocmVersion: '5.4.2'
+  - template: templates/rocm.yml
+    parameters:
+      PythonVersion: '3.9'
+      RocmVersion: '5.4.2'
+  - template: templates/rocm.yml
+    parameters:
+      PythonVersion: '3.7'
+      RocmVersion: '5.4.2'
       BuildConfig: 'RelWithDebInfo'
   - template: templates/rocm.yml
     parameters:
       PythonVersion: '3.8'
-      RocmVersion: '5.4'
+      RocmVersion: '5.4.2'
       BuildConfig: 'RelWithDebInfo'
   - template: templates/rocm.yml
     parameters:
       PythonVersion: '3.9'
-      RocmVersion: '5.4'
+      RocmVersion: '5.4.2'
       BuildConfig: 'RelWithDebInfo'