[ROCm] Update ROCm and MIGraphX CI pipeline to ROCm5.3 (#13257)

### Description
<!-- Describe your changes. -->

1. Update ROCm pipeline and MIGraphX pipeline to ROCm5.3
ROCm pipeline run ortmodule test one time and disable it :
https://dev.azure.com/onnxruntime/onnxruntime/_build/results?buildId=777794&view=logs&j=48b14a85-ff1a-5ca4-53fa-8ea420d27feb&t=9c199f35-fc50-565d-6c65-5162c9bb1b04
2. Add `workspace: clean: all `.


### Motivation and Context
<!-- - Why is this change required? What problem does it solve?
- If it fixes an open issue, please link to the issue here. -->

tools/ci_build/github/azure-pipelines/linux-migraphx-ci-pipeline.yml

@@ -3,6 +3,8 @@ trigger: none
 name: 'linux_ci_$(Date:yyyyMMdd)_$(Rev:r)'
 jobs:
 - job: AMDMIGraphX_CI
+  workspace:
+    clean: all
   pool: 'AMD-GPU'
   timeoutInMinutes: 180
 
@@ -13,7 +15,7 @@ jobs:
     - name: render
       value: 109
     - name: RocmVersion
-      value: 5.2.3
+      value: 5.3
 
   steps:
   - checkout: self
@@ -24,12 +26,7 @@ jobs:
     parameters:
       Dockerfile: tools/ci_build/github/pai/migraphx-ci-pipeline-env.Dockerfile
       Context: tools/ci_build/github/pai
-      Repository: onnxruntimetrainingrocm-cibuild-rocm$(RocmVersion)
-
-  - task: CmdLine@2
-    inputs:
-      script: rm -rf $(Build.BinariesDirectory)/*
-    displayName: 'Clean Build.BinariesDirectory'
+      Repository: onnxruntimetrainingmigraphx-cibuild-rocm$(RocmVersion)
 
   - task: CmdLine@2
     inputs:
@@ -47,14 +44,14 @@ jobs:
           --volume $(Build.SourcesDirectory):/onnxruntime_src \
           --volume $(Build.BinariesDirectory):/build \
           --workdir /onnxruntime_src \
-          onnxruntimetrainingrocm-cibuild-rocm$(RocmVersion) \
+          onnxruntimetrainingmigraphx-cibuild-rocm$(RocmVersion) \
             python tools/ci_build/build.py \
               --config RelWithDebInfo \
               --cmake_extra_defines \
                 CMAKE_HIP_COMPILER=/opt/rocm/llvm/bin/clang++ \
               --mpi_home /opt/ompi \
               --use_migraphx \
-              --rocm_version=5.2.3 \
+              --rocm_version=$(RocmVersion) \
               --rocm_home /opt/rocm \
               --nccl_home /opt/rocm \
               --update \
@@ -83,7 +80,7 @@ jobs:
           --volume $(Build.SourcesDirectory):/onnxruntime_src \
           --volume $(Build.BinariesDirectory):/build \
           --workdir /build/RelWithDebInfo \
-          onnxruntimetrainingrocm-cibuild-rocm$(RocmVersion) \
+          onnxruntimetrainingmigraphx-cibuild-rocm$(RocmVersion) \
             /onnxruntime_src/tools/ci_build/github/pai/migraphx_test_launcher.sh
       workingDirectory: $(Build.SourcesDirectory)
     displayName: 'Run onnxruntime unit tests'

tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml

@@ -3,6 +3,8 @@ trigger: none
 name: 'orttraining_ci_$(Date:yyyyMMdd)_$(Rev:r)'
 jobs:
 - job: AMD_CI
+  workspace:
+    clean: all
   pool: 'AMD-GPU'
   timeoutInMinutes: 150
 
@@ -14,10 +16,12 @@ jobs:
       value: 109
     - name: onnxruntimeBuildSucceeded
       value: false
+    - name: RocmVersion
+      value: 5.3
 
   # generated from tools/ci_build/github/pai/rocm-ci-pipeline-env.Dockerfile
   container:
-    image: onnxruntimecibuildenvironment.azurecr.io/rocm-ci-pipeline-env:rocm5.2.3
+    image: onnxruntimecibuildenvironment.azurecr.io/rocm-ci-pipeline-env:rocm$(RocmVersion)
     endpoint: onnxruntimecibuildenvironmentforamd
     options: --privileged -e HIP_VISIBLE_DEVICES --security-opt seccomp=unconfined --shm-size=1024m --device=/dev/kfd --device=/dev/dri  --group-add $(video) --group-add $(render)
 
@@ -54,7 +58,7 @@ jobs:
           --cmake_extra_defines \
               CMAKE_HIP_COMPILER=${ROCM_HOME}/llvm/bin/clang++ \
           --use_rocm \
-          --rocm_version=5.2.3 \
+          --rocm_version=$(RocmVersion) \
           --rocm_home ${ROCM_HOME} \
           --nccl_home ${ROCM_HOME}\
           --update \
@@ -90,7 +94,7 @@ jobs:
       script: |-
         cd ./build/RelWithDebInfo
         export PYTHONPATH=$PWD
-        export ORTMODULE_ONNX_OPSET_VERSION=14
+        export ORTMODULE_ONNX_OPSET_VERSION=15
         python \
           /stage/huggingface-transformers/examples/pytorch/language-modeling/run_mlm.py \
           --model_name_or_path bert-large-uncased \
@@ -108,7 +112,7 @@ jobs:
           --skip_memory_metrics
         python ../../orttraining/tools/ci_test/compare_huggingface.py \
           ci-pipeline-actual.json \
-          ../../orttraining/tools/ci_test/results/ci-mi100.huggingface.bert-large-rocm5.2.3.json
+          ../../orttraining/tools/ci_test/results/ci-mi100.huggingface.bert-large-rocm$(RocmVersion).json
     displayName: 'Run Python Hugging-Face BERT-L test'
     retryCountOnTaskFailure: 1
     condition: and(succeededOrFailed(), eq(variables.onnxruntimeBuildSucceeded, 'true')) # ensure all tests are run when the build successed
@@ -118,7 +122,7 @@ jobs:
       script: |-
         cd ./build/RelWithDebInfo
         export PYTHONPATH=$PWD
-        export ORTMODULE_ONNX_OPSET_VERSION=14
+        export ORTMODULE_ONNX_OPSET_VERSION=15
         python \
           /stage/huggingface-transformers/examples/pytorch/language-modeling/run_clm.py \
           --model_name_or_path gpt2 \
@@ -137,7 +141,7 @@ jobs:
           --skip_memory_metrics
         python ../../orttraining/tools/ci_test/compare_huggingface.py \
           ci-pipeline-actual.json \
-          ../../orttraining/tools/ci_test/results/ci-mi100.huggingface.gpt2-rocm5.2.3.json
+          ../../orttraining/tools/ci_test/results/ci-mi100.huggingface.gpt2-rocm$(RocmVersion).json
     displayName: 'Run Python Hugging-Face GPT2 test'
     retryCountOnTaskFailure: 1
     condition: and(succeededOrFailed(), eq(variables.onnxruntimeBuildSucceeded, 'true')) # ensure all tests are run when the build successed
@@ -202,7 +206,7 @@ jobs:
       script: |-
         cd ./build/RelWithDebInfo
         export PYTHONPATH=$PWD
-        export ORTMODULE_ONNX_OPSET_VERSION=14
+        export ORTMODULE_ONNX_OPSET_VERSION=15
         python \
           /stage/huggingface-transformers/examples/pytorch/language-modeling/run_mlm.py \
           --model_name_or_path distilbert-base-uncased \
@@ -220,7 +224,7 @@ jobs:
           --skip_memory_metrics
         python ../../orttraining/tools/ci_test/compare_huggingface.py \
           ci-pipeline-actual.json \
-          ../../orttraining/tools/ci_test/results/ci-mi100.huggingface.distilbert-base-rocm5.2.3.json
+          ../../orttraining/tools/ci_test/results/ci-mi100.huggingface.distilbert-base-rocm$(RocmVersion).json
     displayName: 'Run Python Hugging-Face DistilBERT test'
     retryCountOnTaskFailure: 1
     condition: and(succeededOrFailed(), eq(variables.onnxruntimeBuildSucceeded, 'true')) # ensure all tests are run when the build successed
@@ -326,18 +330,19 @@ jobs:
       sudo apt-get update
       sudo apt install -y cifs-utils
     displayName: 'Install filesystems util'
+    condition: and(succeededOrFailed(), eq(variables.onnxruntimeBuildSucceeded, 'true'))
 
   - bash: tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_test_data.sh -p $(orttrainingtestdatascus-storage-key) -s "//orttrainingtestdatascus.file.core.windows.net/mnist" -d "/mnist"
     displayName: 'Mount MNIST'
-    condition: succeededOrFailed()
+    condition: and(succeededOrFailed(), eq(variables.onnxruntimeBuildSucceeded, 'true'))
 
   - bash: tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_test_data.sh -p $(orttrainingtestdatascus-storage-key) -s "//orttrainingtestdatascus.file.core.windows.net/bert-data" -d "/bert_data"
     displayName: 'Mount bert-data'
-    condition: succeededOrFailed()
+    condition: and(succeededOrFailed(), eq(variables.onnxruntimeBuildSucceeded, 'true'))
 
   - bash: tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_test_data.sh -p $(orttrainingtestdatascus-storage-key) -s "//orttrainingtestdatascus.file.core.windows.net/hf-models-cache" -d "/hf_models_cache"
     displayName: 'Mount hf-models-cache'
-    condition: succeededOrFailed()
+    condition: and(succeededOrFailed(), eq(variables.onnxruntimeBuildSucceeded, 'true'))
 
 
   # Entry point for all ORTModule tests
@@ -362,4 +367,4 @@ jobs:
   #         --bert_data /bert_data/hf_data/glue_data/CoLA/original/raw \
   #         --transformers_cache /hf_models_cache/huggingface/transformers
   #   displayName: 'Run orttraining_ortmodule_tests.py'
-  #   condition: succeededOrFailed()
+  #   condition: and(succeededOrFailed(), eq(variables.onnxruntimeBuildSucceeded, 'true'))

tools/ci_build/github/pai/migraphx-ci-pipeline-env.Dockerfile

@@ -1,4 +1,4 @@
-FROM rocm/pytorch:rocm5.2.3_ubuntu20.04_py3.7_pytorch_1.12.1
+FROM rocm/pytorch:rocm5.3_ubuntu20.04_py3.7_pytorch_1.12.1
 
 ENV DEBIAN_FRONTEND noninteractive
 ENV MIGRAPHX_DISABLE_FAST_GELU=1

tools/ci_build/github/pai/rocm-ci-pipeline-env.Dockerfile

@@ -1,4 +1,4 @@
-FROM rocm/pytorch:rocm5.2.3_ubuntu20.04_py3.7_pytorch_1.12.1
+FROM rocm/pytorch:rocm5.3_ubuntu20.04_py3.7_pytorch_1.12.1
 
 WORKDIR /stage