diff --git a/orttraining/orttraining/test/python/_test_commons.py b/orttraining/orttraining/test/python/_test_commons.py
index 325047f70f..24788448d3 100644
--- a/orttraining/orttraining/test/python/_test_commons.py
+++ b/orttraining/orttraining/test/python/_test_commons.py
@@ -21,7 +21,7 @@ def _single_run(execution_file, scenario, checkopint_dir = None):
 
 def _distributed_run(execution_file, scenario, checkopint_dir = None):
     ngpus = torch.cuda.device_count()
-    cmd = ['mpirun', '-n', str(ngpus), '-x', 'NCCL_DEBUG=INFO', '--tag-output', sys.executable, execution_file]
+    cmd = ['mpirun', '-n', str(ngpus), '--tag-output', sys.executable, execution_file]
     if scenario:
         cmd += ['--scenario', scenario]
     if checkopint_dir:
diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py
index 71509f5d1a..26b8f6a555 100644
--- a/tools/ci_build/build.py
+++ b/tools/ci_build/build.py
@@ -159,12 +159,6 @@ def parse_arguments():
         "--enable_memory_profile", action='store_true', help="Enable memory profile in ORT.")
     parser.add_argument(
         "--enable_training", action='store_true', help="Enable training in ORT.")
-    parser.add_argument(
-        "--enable_training_python_frontend_e2e_tests", action="store_true",
-        help="Enable the pytorch frontend training tests.")
-    parser.add_argument(
-        "--enable_training_pipeline_e2e_tests", action="store_true",
-        help="Enable the pipeline c++ e2e tests.")
     parser.add_argument(
         "--disable_nccl", action='store_true', help="Disable Nccl.")
     parser.add_argument(
diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-e2e-test-nightly-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-distributed-e2e-test-pipeline.yml
similarity index 95%
rename from tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-e2e-test-nightly-pipeline.yml
rename to tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-distributed-e2e-test-pipeline.yml
index 20e70cb60c..a6d2081ad5 100644
--- a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-e2e-test-nightly-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-distributed-e2e-test-pipeline.yml
@@ -1,9 +1,9 @@
 trigger: none
 
 jobs:
-- job: Orttraining_Linux_GPU_Training_E2E_Test_Nightly
+- job: Orttraining_Linux_GPU_Distributed_E2E_Test
 
-  timeoutInMinutes: 120
+  timeoutInMinutes: 180
   pool: 'Linux-Multi-GPU-V100-E2E3'
 
   steps:
@@ -21,8 +21,6 @@ jobs:
           --enable_training \
           --update --build \
           --build_wheel \
-          --enable_training_python_frontend_e2e_tests \
-          --enable_training_pipeline_e2e_tests \
           " \
         -m
       DisplayName: 'Build'
@@ -39,6 +37,21 @@ jobs:
     displayName: 'Map test data'
     condition: succeededOrFailed() # ensure all tests are run
 
+  - script: |
+      docker run \
+        --gpus all \
+        --shm-size=1024m \
+        --rm \
+        --volume $(Build.SourcesDirectory):/onnxruntime_src \
+        --volume $(Build.BinariesDirectory):/build \
+        onnxruntime_e2e_test_image \
+          /build/RelWithDebInfo/launch_test.py \
+            --cmd_line_with_args "python orttraining_distributed_tests.py" \
+            --cwd /build/RelWithDebInfo \
+    displayName: 'Run orttraining_distributed_tests.py'
+    condition: succeededOrFailed()
+    timeoutInMinutes: 120
+
   - script: |
       docker run \
         --gpus all \
@@ -300,7 +313,7 @@ jobs:
     displayName: 'Run orttraining_run_bert_pretrain.py ORTBertPretrainTest.test_pretrain_convergence'
     condition: succeededOrFailed() # ensure all tests are run
     timeoutInMinutes: 30
-
+  
   - template: templates/component-governance-component-detection-steps.yml
     parameters:
       condition: 'succeeded'
diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-frontend-test-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-frontend-test-ci-pipeline.yml
deleted file mode 100644
index bda20cfd82..0000000000
--- a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-frontend-test-ci-pipeline.yml
+++ /dev/null
@@ -1,44 +0,0 @@
-# 'trigger: none' to use UI to schedule the pipeline runs.
-# alternatively to schedule the pipeline run according to:
-# https://docs.microsoft.com/en-us/azure/devops/pipelines/process/scheduled-triggers?view=azure-devops&tabs=yaml
-trigger: none
-
-jobs:
-- job: Onnxruntime_Linux_GPU_Training_FrontEnd
-
-  timeoutInMinutes: 300
-
-  steps:
-  - checkout: self
-    clean: true
-    submodules: recursive
-
-  - template: templates/set-test-data-variables-step.yml
-
-  - task: CmdLine@2
-    displayName: 'Clean untagged docker images'
-    inputs:
-      script: |
-        docker rm $(docker ps -a | grep Exited | awk '{print $1;}') || true
-        docker container prune -f
-        docker image prune -f
-      workingDirectory: $(Build.BinariesDirectory)
-    continueOnError: true
-    condition: always()
-
-  - template: templates/run-docker-build-steps.yml
-    parameters:
-      RunDockerBuildArgs: >
-        -o ubuntu16.04 -d gpu -r $(Build.BinariesDirectory)
-        -x "
-        --enable_training
-        --config RelWithDebInfo
-        --skip_onnx_tests
-        --build_wheel
-        --enable_training_python_frontend_e2e_tests
-        --enable_training_pipeline_e2e_tests
-        "
-        -m
-      DisplayName: 'Build and run frontend tests'
-
-  - template: templates/clean-agent-build-directory-step.yml
\ No newline at end of file
diff --git a/tools/ci_build/github/linux/run_dockerbuild.sh b/tools/ci_build/github/linux/run_dockerbuild.sh
index 984a2e34d3..92063520f3 100755
--- a/tools/ci_build/github/linux/run_dockerbuild.sh
+++ b/tools/ci_build/github/linux/run_dockerbuild.sh
@@ -131,8 +131,6 @@ fi
 
 if [ $BUILD_DEVICE = "cpu" ] || [ $BUILD_DEVICE = "openvino" ] || [ $BUILD_DEVICE = "nnapi" ] || [ $BUILD_DEVICE = "arm" ]; then
     RUNTIME=
-elif [[ $BUILD_EXTR_PAR = *--enable_training_python_frontend_e2e_tests* ]]; then
-    RUNTIME="--gpus all --shm-size=1024m"
 else
     RUNTIME="--gpus all"
 fi
@@ -147,16 +145,6 @@ if [ $BUILD_DEVICE = "openvino" ] && [[ $BUILD_EXTR_PAR == *"--use_openvino GPU_
     DOCKER_RUN_PARAMETER="$DOCKER_RUN_PARAMETER --device /dev/dri:/dev/dri"
 fi
 
-if [[ $BUILD_EXTR_PAR = *--enable_training_python_frontend_e2e_tests* ]]; then
-    DOCKER_RUN_PARAMETER="$DOCKER_RUN_PARAMETER --volume /bert_data/hf_data:/bert_data/hf_data"
-    # DOCKER_RUN_PARAMETER="$DOCKER_RUN_PARAMETER -u0"
-fi
-
-if [[ $BUILD_EXTR_PAR = *--enable_training_pipeline_e2e_tests* ]]; then
-    DOCKER_RUN_PARAMETER="$DOCKER_RUN_PARAMETER --volume /bert_ort:/bert_ort \
-                                                --volume /bert_data:/bert_data"
-fi
-
 $DOCKER_CMD rm -f "onnxruntime-$BUILD_DEVICE" || true
 $DOCKER_CMD run $RUNTIME -h $HOSTNAME $DOCKER_RUN_PARAMETER \
     -e NIGHTLY_BUILD \