diff --git a/orttraining/tools/amdgpu/Dockerfile.rocm4.1.pytorch b/orttraining/tools/amdgpu/Dockerfile.rocm4.1.pytorch
index 0a10760274..f300c34083 100644
--- a/orttraining/tools/amdgpu/Dockerfile.rocm4.1.pytorch
+++ b/orttraining/tools/amdgpu/Dockerfile.rocm4.1.pytorch
@@ -132,7 +132,6 @@ RUN git clone --recursive https://github.com/microsoft/onnxruntime.git \
     --enable_training \
   && test -f $ORT_DIR/build/RelWithDebInfo/onnxruntime_training_bert \
   && pip install $ORT_DIR/build/RelWithDebInfo/dist/*.whl \
-  && rm -rf $ORT_DIR/build \
   && ldconfig
 
 # ONNX Runtime Training Examples
diff --git a/orttraining/tools/amdgpu/Dockerfile.rocm4.2.pytorch b/orttraining/tools/amdgpu/Dockerfile.rocm4.2.pytorch
index 243896ee89..076a524dd5 100644
--- a/orttraining/tools/amdgpu/Dockerfile.rocm4.2.pytorch
+++ b/orttraining/tools/amdgpu/Dockerfile.rocm4.2.pytorch
@@ -134,6 +134,42 @@ RUN git clone --recursive https://github.com/microsoft/onnxruntime.git \
   && pip install $ORT_DIR/build/Release/dist/*.whl \
   && ldconfig
 
+# ONNX Runtime Training Examples
+WORKDIR $GITHUB_DIR
+ARG GPT2_DATASET=wikitext-103
+RUN git clone -b wezhan/amdgpu https://github.com/microsoft/onnxruntime-training-examples.git \
+  && cd onnxruntime-training-examples \
+  # Nvidia BERT
+  && git clone --no-checkout https://github.com/NVIDIA/DeepLearningExamples.git \
+  && cd DeepLearningExamples \
+  && git checkout cf54b787 \
+  && cd .. \
+  && mv DeepLearningExamples/PyTorch/LanguageModeling/BERT ${WORKSPACE_DIR} \
+  && rm -rf DeepLearningExamples \
+  && cp -r ./nvidia-bert/ort_addon/* ${WORKSPACE_DIR}/BERT \
+  # GPT2 fine-tuning
+  && cd huggingface-gpt2 \
+  && git clone https://github.com/huggingface/transformers.git \
+  && cd transformers \
+  && git checkout 9a0a8c1c6f4f2f0c80ff07d36713a3ada785eec5 \
+  && cd .. \
+  && mkdir -p ${WORKSPACE_DIR}/GPT2 \
+  && cp -r transformers ${WORKSPACE_DIR}/GPT2 \
+  && cd ${WORKSPACE_DIR}/GPT2/transformers \
+  && git apply $GITHUB_DIR/onnxruntime-training-examples/huggingface-gpt2/ort_addon/src_changes.patch \
+  && cp -r $GITHUB_DIR/onnxruntime-training-examples/huggingface-gpt2/ort_addon/ort_supplement/* ./ \
+  && python3 -m pip install --no-cache-dir -e . \
+  && python3 -m pip install --no-cache-dir -r examples/requirements.txt \
+  && python3 -m pip install cerberus sympy \
+  && cd .. \
+  && wget https://s3.amazonaws.com/research.metamind.io/wikitext/${GPT2_DATASET}-v1.zip \
+  && unzip ${GPT2_DATASET}-v1.zip
+
+ENV BERT_DIR=${WORKSPACE_DIR}/BERT
+ENV GPT2_DIR=${WORKSPACE_DIR}/GPT2
+ENV TRAIN_FILE=${WORKSPACE_DIR}/GPT2/${GPT2_DATASET}/wiki.train.tokens
+ENV TEST_FILE=${WORKSPACE_DIR}/GPT2/${GPT2_DATASET}/wiki.test.tokens
+
 RUN pip3 install --no-cache-dir GPUtil azureml azureml-core datasets tokenizers ninja cerberus sympy sacremoses sacrebleu
 
 # Huggingface Examples