From 7a454acd6197f4ba1ffca13ec9948915ce82d20e Mon Sep 17 00:00:00 2001 From: PeixuanZuo <94887879+PeixuanZuo@users.noreply.github.com> Date: Wed, 3 Jan 2024 17:25:15 +0800 Subject: [PATCH] [ROCm] Update CI/Packaging pipeline to ROCm6.0 (#18985) Update CI/Packaing pipeline to ROCm6.0 --- ...mi200.huggingface.bert-large-rocm6.0.json} | 28 ++++---- .../linux-migraphx-ci-pipeline.yml | 2 +- .../orttraining-pai-ci-pipeline.yml | 2 +- ...orttraining-py-packaging-pipeline-rocm.yml | 72 +++++++++---------- .../docker/Dockerfile.manylinux2_28_rocm | 2 +- .../migraphx-ci-pipeline-env.Dockerfile | 2 +- .../docker/scripts/setup_rocm_yum_repo.sh | 6 +- .../pai/rocm-ci-pipeline-env.Dockerfile | 4 +- 8 files changed, 59 insertions(+), 59 deletions(-) rename orttraining/tools/ci_test/results/{ci-mi200.huggingface.bert-large-rocm5.7.json => ci-mi200.huggingface.bert-large-rocm6.0.json} (61%) diff --git a/orttraining/tools/ci_test/results/ci-mi200.huggingface.bert-large-rocm5.7.json b/orttraining/tools/ci_test/results/ci-mi200.huggingface.bert-large-rocm6.0.json similarity index 61% rename from orttraining/tools/ci_test/results/ci-mi200.huggingface.bert-large-rocm5.7.json rename to orttraining/tools/ci_test/results/ci-mi200.huggingface.bert-large-rocm6.0.json index a4ac02b566..05fcf08cd3 100644 --- a/orttraining/tools/ci_test/results/ci-mi200.huggingface.bert-large-rocm5.7.json +++ b/orttraining/tools/ci_test/results/ci-mi200.huggingface.bert-large-rocm6.0.json @@ -2,56 +2,56 @@ "steps": [ { "step": 20, - "loss": 2.0017 + "loss": 2.0136 }, { "step": 40, - "loss": 1.8337 + "loss": 1.8466 }, { "step": 60, - "loss": 1.7538 + "loss": 1.7525 }, { "step": 80, - "loss": 1.6728 + "loss": 1.6682 }, { "step": 100, - "loss": 1.6656 + "loss": 1.658 }, { "step": 120, - "loss": 1.6752 + "loss": 1.6749 }, { "step": 140, - "loss": 1.6335 + "loss": 1.6263 }, { "step": 160, - "loss": 1.6815 + "loss": 1.6828 }, { "step": 180, - "loss": 1.6155 + "loss": 1.6145 }, { "step": 200, - "loss": 1.6177 + "loss": 1.6197 }, { "step": 220, - "loss": 1.632 + "loss": 1.6353 }, { "step": 240, - "loss": 1.5161 + "loss": 1.5266 }, { "step": 260, - "loss": 1.5433 + "loss": 1.5441 } ], - "samples_per_second": 32.335 + "samples_per_second": 34.561 } diff --git a/tools/ci_build/github/azure-pipelines/linux-migraphx-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-migraphx-ci-pipeline.yml index 5dac8fc9cd..f7571a3b7e 100644 --- a/tools/ci_build/github/azure-pipelines/linux-migraphx-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-migraphx-ci-pipeline.yml @@ -36,7 +36,7 @@ variables: - name: render value: 109 - name: RocmVersion - value: 5.7 + value: 6.0 jobs: - job: Linux_Build diff --git a/tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml index 8d02a5e580..a53f91fb31 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml @@ -25,7 +25,7 @@ variables: - name: render value: 109 - name: RocmVersion - value: 5.7 + value: 6.0 - name: BuildConfig value: Release diff --git a/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-rocm.yml b/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-rocm.yml index f2ba99369c..bbdbe0fd8e 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-rocm.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-rocm.yml @@ -9,51 +9,51 @@ resources: ref: 5eda9aded5462201e6310105728d33016e637ea7 stages: +- stage: "Python_Packaging_ROCm60_Release" + jobs: + - template: templates/rocm.yml + parameters: + PythonVersion: '3.8' + RocmVersion: '6.0' + - template: templates/rocm.yml + parameters: + PythonVersion: '3.9' + RocmVersion: '6.0' + - template: templates/rocm.yml + parameters: + PythonVersion: '3.10' + RocmVersion: '6.0' + +- stage: "Python_Packaging_ROCm60_Debug" + jobs: + - template: templates/rocm.yml + parameters: + PythonVersion: '3.8' + RocmVersion: '6.0' + BuildConfig: 'Debug' + - template: templates/rocm.yml + parameters: + PythonVersion: '3.9' + RocmVersion: '6.0' + BuildConfig: 'Debug' + - template: templates/rocm.yml + parameters: + PythonVersion: '3.10' + RocmVersion: '6.0' + BuildConfig: 'Debug' + - stage: "Python_Packaging_ROCm57_Release" - jobs: - - template: templates/rocm.yml - parameters: - PythonVersion: '3.8' - RocmVersion: '5.7' - - template: templates/rocm.yml - parameters: - PythonVersion: '3.9' - RocmVersion: '5.7' - - template: templates/rocm.yml - parameters: - PythonVersion: '3.10' - RocmVersion: '5.7' - -- stage: "Python_Packaging_ROCm57_Debug" - jobs: - - template: templates/rocm.yml - parameters: - PythonVersion: '3.8' - RocmVersion: '5.7' - BuildConfig: 'Debug' - - template: templates/rocm.yml - parameters: - PythonVersion: '3.9' - RocmVersion: '5.7' - BuildConfig: 'Debug' - - template: templates/rocm.yml - parameters: - PythonVersion: '3.10' - RocmVersion: '5.7' - BuildConfig: 'Debug' - -- stage: "Python_Packaging_ROCm56_Release" condition: ne(variables['ORT_DISABLE_PYTHON_PACKAGE_LOCAL_VERSION'], 'true') jobs: - template: templates/rocm.yml parameters: PythonVersion: '3.8' - RocmVersion: '5.6' + RocmVersion: '5.7' - template: templates/rocm.yml parameters: PythonVersion: '3.9' - RocmVersion: '5.6' + RocmVersion: '5.7' - template: templates/rocm.yml parameters: PythonVersion: '3.10' - RocmVersion: '5.6' + RocmVersion: '5.7' diff --git a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_rocm b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_rocm index 9e12fe8c75..b9fd88083f 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_rocm +++ b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_rocm @@ -31,7 +31,7 @@ RUN yum install -y hipify-clang RUN yum -y install wget # rocm lib -RUN yum install -y miopen-hip-devel rocblas-devel rocrand-devel rccl-devel hipsparse-devel hipfft-devel hipcub-devel hipblas-devel rocthrust-devel migraphx-devel +RUN yum install -y migraphx-devel ENV AUDITWHEEL_POLICY=${POLICY} AUDITWHEEL_ARCH=${PLATFORM} AUDITWHEEL_PLAT=${POLICY}_${PLATFORM} ENV LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 LANGUAGE=en_US.UTF-8 diff --git a/tools/ci_build/github/linux/docker/migraphx-ci-pipeline-env.Dockerfile b/tools/ci_build/github/linux/docker/migraphx-ci-pipeline-env.Dockerfile index d02e7d8b91..85d738d216 100644 --- a/tools/ci_build/github/linux/docker/migraphx-ci-pipeline-env.Dockerfile +++ b/tools/ci_build/github/linux/docker/migraphx-ci-pipeline-env.Dockerfile @@ -1,7 +1,7 @@ # Refer to https://github.com/RadeonOpenCompute/ROCm-docker/blob/master/dev/Dockerfile-ubuntu-22.04-complete FROM ubuntu:22.04 -ARG ROCM_VERSION=5.7 +ARG ROCM_VERSION=6.0 ARG AMDGPU_VERSION=${ROCM_VERSION} ARG APT_PREF='Package: *\nPin: release o=repo.radeon.com\nPin-Priority: 600' diff --git a/tools/ci_build/github/linux/docker/scripts/setup_rocm_yum_repo.sh b/tools/ci_build/github/linux/docker/scripts/setup_rocm_yum_repo.sh index fcd9086061..269337bbba 100755 --- a/tools/ci_build/github/linux/docker/scripts/setup_rocm_yum_repo.sh +++ b/tools/ci_build/github/linux/docker/scripts/setup_rocm_yum_repo.sh @@ -2,7 +2,7 @@ set -e -x # version -ROCM_VERSION=5.6 +ROCM_VERSION=6.0 while getopts "r:" parameter_Option do case "${parameter_Option}" @@ -14,7 +14,7 @@ done tee /etc/yum.repos.d/amdgpu.repo <