[ROCm] Update CI/Packaging pipeline to ROCm6.0 (#18985)

Update CI/Packaing pipeline to ROCm6.0
2026-05-14 20:48:00 +00:00 · 2024-01-03 17:25:15 +08:00 · 2024-01-03 17:25:15 +08:00 · 7a454acd61
commit 7a454acd61
parent c97e3f4821
8 changed files with 59 additions and 59 deletions
--- a/orttraining/tools/ci_test/results/ci-mi200.huggingface.bert-large-rocm6.0.json
+++ b/orttraining/tools/ci_test/results/ci-mi200.huggingface.bert-large-rocm6.0.json
@ -2,56 +2,56 @@
    "steps": [
        {
            "step": 20,
-            "loss": 2.0017
+            "loss": 2.0136
        },
        {
            "step": 40,
-            "loss": 1.8337
+            "loss": 1.8466
        },
        {
            "step": 60,
-            "loss": 1.7538
+            "loss": 1.7525
        },
        {
            "step": 80,
-            "loss": 1.6728
+            "loss": 1.6682
        },
        {
            "step": 100,
-            "loss": 1.6656
+            "loss": 1.658
        },
        {
            "step": 120,
-            "loss": 1.6752
+            "loss": 1.6749
        },
        {
            "step": 140,
-            "loss": 1.6335
+            "loss": 1.6263
        },
        {
            "step": 160,
-            "loss": 1.6815
+            "loss": 1.6828
        },
        {
            "step": 180,
-            "loss": 1.6155
+            "loss": 1.6145
        },
        {
            "step": 200,
-            "loss": 1.6177
+            "loss": 1.6197
        },
        {
            "step": 220,
-            "loss": 1.632
+            "loss": 1.6353
        },
        {
            "step": 240,
-            "loss": 1.5161
+            "loss": 1.5266
        },
        {
            "step": 260,
-            "loss": 1.5433
+            "loss": 1.5441
        }
    ],
-    "samples_per_second": 32.335
+    "samples_per_second": 34.561
 }
--- a/tools/ci_build/github/azure-pipelines/linux-migraphx-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/linux-migraphx-ci-pipeline.yml
@ -36,7 +36,7 @@ variables:
  - name: render
    value: 109
  - name: RocmVersion
-    value: 5.7
+    value: 6.0

 jobs:
 - job: Linux_Build
--- a/tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml
+++ b/tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml
@ -25,7 +25,7 @@ variables:
  - name: render
    value: 109
  - name: RocmVersion
-    value: 5.7
+    value: 6.0
  - name: BuildConfig
    value: Release

--- a/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-rocm.yml
+++ b/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-rocm.yml
@ -9,51 +9,51 @@ resources:
    ref: 5eda9aded5462201e6310105728d33016e637ea7

 stages:
+- stage: "Python_Packaging_ROCm60_Release"
+  jobs:
+  - template: templates/rocm.yml
+    parameters:
+      PythonVersion: '3.8'
+      RocmVersion: '6.0'
+  - template: templates/rocm.yml
+    parameters:
+      PythonVersion: '3.9'
+      RocmVersion: '6.0'
+  - template: templates/rocm.yml
+    parameters:
+      PythonVersion: '3.10'
+      RocmVersion: '6.0'
+
+- stage: "Python_Packaging_ROCm60_Debug"
+  jobs:
+  - template: templates/rocm.yml
+    parameters:
+      PythonVersion: '3.8'
+      RocmVersion: '6.0'
+      BuildConfig: 'Debug'
+  - template: templates/rocm.yml
+    parameters:
+      PythonVersion: '3.9'
+      RocmVersion: '6.0'
+      BuildConfig: 'Debug'
+  - template: templates/rocm.yml
+    parameters:
+      PythonVersion: '3.10'
+      RocmVersion: '6.0'
+      BuildConfig: 'Debug'
+
 - stage: "Python_Packaging_ROCm57_Release"
-  jobs:
-  - template: templates/rocm.yml
-    parameters:
-      PythonVersion: '3.8'
-      RocmVersion: '5.7'
-  - template: templates/rocm.yml
-    parameters:
-      PythonVersion: '3.9'
-      RocmVersion: '5.7'
-  - template: templates/rocm.yml
-    parameters:
-      PythonVersion: '3.10'
-      RocmVersion: '5.7'
-
- stage: "Python_Packaging_ROCm57_Debug"
-  jobs:
-  - template: templates/rocm.yml
-    parameters:
-      PythonVersion: '3.8'
-      RocmVersion: '5.7'
-      BuildConfig: 'Debug'
-  - template: templates/rocm.yml
-    parameters:
-      PythonVersion: '3.9'
-      RocmVersion: '5.7'
-      BuildConfig: 'Debug'
-  - template: templates/rocm.yml
-    parameters:
-      PythonVersion: '3.10'
-      RocmVersion: '5.7'
-      BuildConfig: 'Debug'
-
- stage: "Python_Packaging_ROCm56_Release"
  condition: ne(variables['ORT_DISABLE_PYTHON_PACKAGE_LOCAL_VERSION'], 'true')
  jobs:
  - template: templates/rocm.yml
    parameters:
      PythonVersion: '3.8'
-      RocmVersion: '5.6'
+      RocmVersion: '5.7'
  - template: templates/rocm.yml
    parameters:
      PythonVersion: '3.9'
-      RocmVersion: '5.6'
+      RocmVersion: '5.7'
  - template: templates/rocm.yml
    parameters:
      PythonVersion: '3.10'
-      RocmVersion: '5.6'
+      RocmVersion: '5.7'
--- a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_rocm
+++ b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_rocm
@ -31,7 +31,7 @@ RUN yum install -y hipify-clang
 RUN yum -y install wget

 # rocm lib
-RUN yum install -y miopen-hip-devel rocblas-devel rocrand-devel rccl-devel hipsparse-devel hipfft-devel hipcub-devel hipblas-devel rocthrust-devel migraphx-devel
+RUN yum install -y migraphx-devel

 ENV AUDITWHEEL_POLICY=${POLICY} AUDITWHEEL_ARCH=${PLATFORM} AUDITWHEEL_PLAT=${POLICY}_${PLATFORM}
 ENV LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 LANGUAGE=en_US.UTF-8
--- a/tools/ci_build/github/linux/docker/migraphx-ci-pipeline-env.Dockerfile
+++ b/tools/ci_build/github/linux/docker/migraphx-ci-pipeline-env.Dockerfile
@ -1,7 +1,7 @@
 # Refer to https://github.com/RadeonOpenCompute/ROCm-docker/blob/master/dev/Dockerfile-ubuntu-22.04-complete
 FROM ubuntu:22.04

-ARG ROCM_VERSION=5.7
+ARG ROCM_VERSION=6.0
 ARG AMDGPU_VERSION=${ROCM_VERSION}
 ARG APT_PREF='Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600'

--- a/tools/ci_build/github/linux/docker/scripts/setup_rocm_yum_repo.sh
+++ b/tools/ci_build/github/linux/docker/scripts/setup_rocm_yum_repo.sh
@ -2,7 +2,7 @@
 set -e -x

 # version
-ROCM_VERSION=5.6
+ROCM_VERSION=6.0

 while getopts "r:" parameter_Option
 do case "${parameter_Option}"
@ -14,7 +14,7 @@ done
 tee /etc/yum.repos.d/amdgpu.repo <<EOF
 [amdgpu]
 name=amdgpu
-baseurl=https://repo.radeon.com/amdgpu/$ROCM_VERSION/rhel/8.7/main/x86_64/
+baseurl=https://repo.radeon.com/amdgpu/$ROCM_VERSION/rhel/8.8/main/x86_64/
 enabled=1
 priority=50
 gpgcheck=1
@ -40,4 +40,4 @@ gpgcheck=1
 gpgkey=https://repo.radeon.com/rocm/rocm.gpg.key
 EOF

-dnf install -y rocm-dev
+dnf install -y rocm-dev rocm-libs
--- a/tools/ci_build/github/pai/rocm-ci-pipeline-env.Dockerfile
+++ b/tools/ci_build/github/pai/rocm-ci-pipeline-env.Dockerfile
@ -1,7 +1,7 @@
 # Refer to https://github.com/RadeonOpenCompute/ROCm-docker/blob/master/dev/Dockerfile-ubuntu-22.04-complete
 FROM ubuntu:22.04

-ARG ROCM_VERSION=5.7
+ARG ROCM_VERSION=6.0
 ARG AMDGPU_VERSION=${ROCM_VERSION}
 ARG APT_PREF='Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600'

@ -94,7 +94,7 @@ RUN wget https://download.open-mpi.org/release/open-mpi/v4.1/openmpi-4.1.5.tar.b

 # Install CuPy, No stable version is available
 RUN git clone https://github.com/ROCmSoftwarePlatform/cupy && cd cupy && \
-    git checkout fc251a808037f8a2270860c2a23a683bfc0de43e && \
+    git checkout 432a8683351d681e00903640489cb2f4055d2e09 && \
    export CUPY_INSTALL_USE_HIP=1 && \
    export ROCM_HOME=/opt/rocm && \
    export HCC_AMDGPU_TARGET=gfx906,gfx908,gfx90a && \