change ci docker image to rocm6.1 (#21296)

### Description
<!-- Describe your changes. -->
There is a bug for kernel running on rocm6.0, so change ci docker image
to rocm6.1

For the torch installed in the docker image, change to rocm repo when it
is not 6.0 version.

### Motivation and Context
<!-- - Why is this change required? What problem does it solve?
- If it fixes an open issue, please link to the issue here. -->

orttraining/tools/ci_test/results/ci-mi200.huggingface.bert-large-rocm6.1.json

@@ -0,0 +1,57 @@
+{
+    "steps": [
+        {
+            "step": 20,
+            "loss": 2.0136
+        },
+        {
+            "step": 40,
+            "loss": 1.8466
+        },
+        {
+            "step": 60,
+            "loss": 1.7525
+        },
+        {
+            "step": 80,
+            "loss": 1.6682
+        },
+        {
+            "step": 100,
+            "loss": 1.658
+        },
+        {
+            "step": 120,
+            "loss": 1.6749
+        },
+        {
+            "step": 140,
+            "loss": 1.6263
+        },
+        {
+            "step": 160,
+            "loss": 1.6828
+        },
+        {
+            "step": 180,
+            "loss": 1.6145
+        },
+        {
+            "step": 200,
+            "loss": 1.6197
+        },
+        {
+            "step": 220,
+            "loss": 1.6353
+        },
+        {
+            "step": 240,
+            "loss": 1.5266
+        },
+        {
+            "step": 260,
+            "loss": 1.5441
+        }
+    ],
+    "samples_per_second": 34.561
+}

tools/ci_build/github/azure-pipelines/linux-migraphx-ci-pipeline.yml

@@ -36,7 +36,7 @@ variables:
   - name: render
     value: 109
   - name: RocmVersion
-    value: 6.0
+    value: 6.1
   - name: RocmVersionPatchSuffix
     value: ".3"
 

tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml

@@ -25,7 +25,7 @@ variables:
   - name: render
     value: 109
   - name: RocmVersion
-    value: 6.0
+    value: 6.1
   - name: RocmVersionPatchSuffix
     value: ".3"
   - name: BuildConfig

tools/ci_build/github/pai/rocm-ci-pipeline-env.Dockerfile

@@ -1,7 +1,7 @@
 # Refer to https://github.com/RadeonOpenCompute/ROCm-docker/blob/master/dev/Dockerfile-ubuntu-22.04-complete
 FROM ubuntu:22.04
 
-ARG ROCM_VERSION=6.0
+ARG ROCM_VERSION=6.1
 ARG AMDGPU_VERSION=${ROCM_VERSION}
 ARG APT_PREF='Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600'
 
@@ -77,11 +77,7 @@ RUN ln -sf /usr/lib/x86_64-linux-gnu/libstdc++.so.6 ${CONDA_ENVIRONMENT_PATH}/bi
 RUN export MAJOR=$(cut -d '.' -f 1 <<< "$ROCM_VERSION") && \
     export MINOR=$(cut -d '.' -f 2 <<< "$ROCM_VERSION") && \
     export PATCH=$(cut -d '.' -f 3 <<< "$ROCM_VERSION") && \
-    if (( MAJOR >= 6 )); then \
-        pip install torch torchvision torchaudio --index-url https://download.pytorch.org/whl/rocm${MAJOR}.${MINOR} ; \
-    else \
-        pip install torch==2.0.1 torchvision==0.15.2 -f https://repo.radeon.com/rocm/manylinux/rocm-rel-${MAJOR}.${MINOR}/ ; \
-    fi && \
+    pip install torch==2.1.2 torchvision==0.16.1 -f https://repo.radeon.com/rocm/manylinux/rocm-rel-${MAJOR}.${MINOR}/ && \
     pip install torch-ort --no-dependencies
 
 ##### Install Cupy to decrease CPU utilization