From 1c39d22f4ec32f120b5c7ed4b79ac2b99d14db91 Mon Sep 17 00:00:00 2001
From: PeixuanZuo <94887879+PeixuanZuo@users.noreply.github.com>
Date: Mon, 11 Jul 2022 11:10:45 +0800
Subject: [PATCH] [ADD]  Rocm5.2 for Rocm python packaging pipeline (#12129)

[ADD] rocm5.2
---
 ...orttraining-py-packaging-pipeline-rocm.yml |  28 ++-
 .../docker/Dockerfile.manylinux2014_rocm5_1_1 |  14 --
 .../docker/Dockerfile.manylinux2014_rocm5_2   | 200 ++++++++++++++++++
 .../requirements.txt                          |  12 ++
 4 files changed, 234 insertions(+), 20 deletions(-)
 create mode 100644 tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm5_2
 create mode 100644 tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements_torch1.11.0_rocm5.2/requirements.txt

diff --git a/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-rocm.yml b/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-rocm.yml
index bcd36a2852..c6b8a4a339 100644
--- a/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-rocm.yml
+++ b/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-rocm.yml
@@ -36,6 +36,19 @@ stages:
             --build-arg PREPEND_PATH=/opt/rh/devtoolset-10/root/usr/bin:
             --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-10/root/usr/lib64:/opt/rh/devtoolset-10/root/usr/lib:/opt/rh/devtoolset-10/root/usr/lib64/dyninst:/opt/rh/devtoolset-10/root/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib
           Repository: onnxruntimetrainingrocmbuild-rocm5.1.1
+      - template: templates/ get-docker-image-steps.yml
+        parameters:
+          Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm5_2
+          Context: tools/ci_build/github/linux/docker
+          DockerBuildArgs: >-
+            --build-arg TORCH_VERSION=1.11.0
+            --build-arg INSTALL_DEPS_EXTRA_ARGS=-tmur
+            --build-arg BUILD_UID=$(id -u)
+            --network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64
+            --build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-10/root
+            --build-arg PREPEND_PATH=/opt/rh/devtoolset-10/root/usr/bin:
+            --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-10/root/usr/lib64:/opt/rh/devtoolset-10/root/usr/lib:/opt/rh/devtoolset-10/root/usr/lib64/dyninst:/opt/rh/devtoolset-10/root/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib
+          Repository: onnxruntimetrainingrocmbuild-rocm5.2
 
     - job: ROCM_training_wheels
       timeoutInMinutes: 180
@@ -49,28 +62,31 @@ stages:
         matrix:
           Python37 Torch1110 Rocm431:
             PythonVersion: '3.7'
-            TorchVersion: '1.11.0'
             RocmVersion: '4.3.1'
           Python38 Torch1110 Rocm431:
             PythonVersion: '3.8'
-            TorchVersion: '1.11.0'
             RocmVersion: '4.3.1'
           Python39 Torch1110 Rocm431:
             PythonVersion: '3.9'
-            TorchVersion: '1.11.0'
             RocmVersion: '4.3.1'
           Python37 Torch1110 Rocm511:
             PythonVersion: '3.7'
-            TorchVersion: '1.11.0'
             RocmVersion: '5.1.1'
           Python38 Torch1110 Rocm511:
             PythonVersion: '3.8'
-            TorchVersion: '1.11.0'
             RocmVersion: '5.1.1'
           Python39 Torch1110 Rocm511:
             PythonVersion: '3.9'
-            TorchVersion: '1.11.0'
             RocmVersion: '5.1.1'
+          Python37 Torch1110 Rocm52:
+            PythonVersion: '3.7'
+            RocmVersion: '5.2'
+          Python38 Torch1110 Rocm52:
+            PythonVersion: '3.8'
+            RocmVersion: '5.2'
+          Python39 Torch1110 Rocm52:
+            PythonVersion: '3.9'
+            RocmVersion: '5.2'
       steps:
 
       - checkout: self
diff --git a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm5_1_1 b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm5_1_1
index 65f5fa0590..b9670ccb98 100644
--- a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm5_1_1
+++ b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm5_1_1
@@ -20,13 +20,6 @@ RUN yum install -y hipify-clang
 
 RUN yum remove -y devtoolset\* git\* && conda remove -y cmake
 
-# remove protobuf 2.6.1 from rocm/pytorch:rocm5.1.1_centos7_py3.7_pytorch_staging
-# it's too old to compile onnx 1.10
-RUN rm -fr /usr/local/bin/protoc \
-           /usr/local/libproto* \
-           /usr/local/include/google \
-           /usr/local/lib/pkgconfig/protobuf*
-
 ENV AUDITWHEEL_POLICY=${POLICY} AUDITWHEEL_ARCH=${PLATFORM} AUDITWHEEL_PLAT=${POLICY}_${PLATFORM}
 ENV LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 LANGUAGE=en_US.UTF-8
 ENV DEVTOOLSET_ROOTPATH=${DEVTOOLSET_ROOTPATH}
@@ -88,13 +81,6 @@ RUN export LIBXCRYPT_VERSION=4.4.26 && \
     export PERL_DOWNLOAD_URL=https://www.cpan.org/src/5.0 && \
     manylinux-entrypoint /build_scripts/install-libxcrypt.sh
 
-COPY build_scripts/install-protobuf.sh /build_scripts/
-RUN export PROTOBUF_VERSION=3.17.3 && \
-    export PROTOBUF_ROOT=protobuf-all-${PROTOBUF_VERSION} && \
-    export PROTOBUF_HASH=77ad26d3f65222fd96ccc18b055632b0bfedf295cb748b712a98ba1ac0b704b2 && \
-    export PROTOBUF_DOWNLOAD_URL=https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION} && \
-    manylinux-entrypoint /build_scripts/install-protobuf.sh
-
 FROM runtime_base AS build_base
 COPY build_scripts/install-build-packages.sh /build_scripts/
 RUN manylinux-entrypoint /build_scripts/install-build-packages.sh
diff --git a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm5_2 b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm5_2
new file mode 100644
index 0000000000..9ba55f7443
--- /dev/null
+++ b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm5_2
@@ -0,0 +1,200 @@
+ARG BASEIMAGE=rocm/pytorch:rocm5.2_centos7_py3.7_pytorch_staging
+ARG POLICY=manylinux2014
+ARG PLATFORM=x86_64
+ARG DEVTOOLSET_ROOTPATH=
+ARG LD_LIBRARY_PATH_ARG=
+ARG PREPEND_PATH=
+
+#Build manylinux2014 docker image begin
+FROM $BASEIMAGE AS runtime_base
+ARG POLICY
+ARG PLATFORM
+ARG DEVTOOLSET_ROOTPATH
+ARG LD_LIBRARY_PATH_ARG
+ARG PREPEND_PATH
+LABEL maintainer="The ManyLinux project"
+
+# TODO: This should not be required if the hipify-perl binary is already a part
+# of the base docker image. Remove this line moving forward.
+RUN yum install -y hipify-clang
+
+RUN yum remove -y devtoolset\* git\* && conda remove -y cmake
+
+ENV AUDITWHEEL_POLICY=${POLICY} AUDITWHEEL_ARCH=${PLATFORM} AUDITWHEEL_PLAT=${POLICY}_${PLATFORM}
+ENV LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 LANGUAGE=en_US.UTF-8
+ENV DEVTOOLSET_ROOTPATH=${DEVTOOLSET_ROOTPATH}
+ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH_ARG}
+ENV PATH=${PREPEND_PATH}${PATH}
+ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig
+
+# first copy the fixup mirrors script, keep the script around
+COPY build_scripts/fixup-mirrors.sh /usr/local/sbin/fixup-mirrors
+
+# setup entrypoint, this will wrap commands with `linux32` with i686 images
+COPY build_scripts/install-entrypoint.sh \
+     build_scripts/update-system-packages.sh \
+     build_scripts/build_utils.sh \
+     /build_scripts/
+
+RUN /build_scripts/install-entrypoint.sh && rm -rf /build_scripts
+COPY manylinux-entrypoint /usr/local/bin/manylinux-entrypoint
+ENTRYPOINT ["manylinux-entrypoint"]
+
+COPY build_scripts/install-runtime-packages.sh \
+     build_scripts/update-system-packages.sh \
+     build_scripts/build_utils.sh \
+     /build_scripts/
+RUN manylinux-entrypoint /build_scripts/install-runtime-packages.sh && rm -rf /build_scripts/
+
+COPY build_scripts/build_utils.sh /build_scripts/
+
+COPY build_scripts/install-autoconf.sh /build_scripts/
+RUN export AUTOCONF_ROOT=autoconf-2.71 && \
+    export AUTOCONF_HASH=431075ad0bf529ef13cb41e9042c542381103e80015686222b8a9d4abef42a1c && \
+    export AUTOCONF_DOWNLOAD_URL=http://ftp.gnu.org/gnu/autoconf && \
+    manylinux-entrypoint /build_scripts/install-autoconf.sh
+
+COPY build_scripts/install-automake.sh /build_scripts/
+RUN export AUTOMAKE_ROOT=automake-1.16.5 && \
+    export AUTOMAKE_HASH=07bd24ad08a64bc17250ce09ec56e921d6343903943e99ccf63bbf0705e34605 && \
+    export AUTOMAKE_DOWNLOAD_URL=http://ftp.gnu.org/gnu/automake && \
+    manylinux-entrypoint /build_scripts/install-automake.sh
+
+COPY build_scripts/install-libtool.sh /build_scripts/
+RUN export LIBTOOL_ROOT=libtool-2.4.6 && \
+    export LIBTOOL_HASH=e3bd4d5d3d025a36c21dd6af7ea818a2afcd4dfc1ea5a17b39d7854bcd0c06e3 && \
+    export LIBTOOL_DOWNLOAD_URL=http://ftp.gnu.org/gnu/libtool && \
+    manylinux-entrypoint /build_scripts/install-libtool.sh
+
+COPY build_scripts/install-patchelf.sh /build_scripts/
+RUN export PATCHELF_VERSION=0.13 && \
+    export PATCHELF_HASH=60c6aeadb673de9cc1838b630c81f61e31c501de324ef7f1e8094a2431197d09 && \
+    export PATCHELF_DOWNLOAD_URL=https://github.com/NixOS/patchelf/archive && \
+    manylinux-entrypoint /build_scripts/install-patchelf.sh
+
+COPY build_scripts/install-libxcrypt.sh /build_scripts/
+RUN export LIBXCRYPT_VERSION=4.4.26 && \
+    export LIBXCRYPT_HASH=e8a544dd19171c1e6191a6044c96cc31496d781ba08b5a00f53310d001d58114 && \
+    export LIBXCRYPT_DOWNLOAD_URL=https://github.com/besser82/libxcrypt/archive && \
+    export PERL_ROOT=perl-5.34.0 && \
+    export PERL_HASH=551efc818b968b05216024fb0b727ef2ad4c100f8cb6b43fab615fa78ae5be9a && \
+    export PERL_DOWNLOAD_URL=https://www.cpan.org/src/5.0 && \
+    manylinux-entrypoint /build_scripts/install-libxcrypt.sh
+
+
+FROM runtime_base AS build_base
+COPY build_scripts/install-build-packages.sh /build_scripts/
+RUN manylinux-entrypoint /build_scripts/install-build-packages.sh
+
+
+FROM build_base AS build_git
+COPY build_scripts/build-git.sh /build_scripts/
+RUN export GIT_ROOT=git-2.33.1 && \
+    export GIT_HASH=02047f8dc8934d57ff5e02aadd8a2fe8e0bcf94a7158da375e48086cc46fce1d && \
+    export GIT_DOWNLOAD_URL=https://www.kernel.org/pub/software/scm/git && \
+    manylinux-entrypoint /build_scripts/build-git.sh
+
+
+FROM build_base AS build_swig
+COPY build_scripts/build-swig.sh /build_scripts/
+RUN export SWIG_ROOT=swig-4.0.2 && \
+    export SWIG_HASH=d53be9730d8d58a16bf0cbd1f8ac0c0c3e1090573168bfa151b01eb47fa906fc && \
+    export SWIG_DOWNLOAD_URL=https://sourceforge.net/projects/swig/files/swig/${SWIG_ROOT} && \
+    export PCRE_ROOT=pcre-8.45 && \
+    export PCRE_HASH=4e6ce03e0336e8b4a3d6c2b70b1c5e18590a5673a98186da90d4f33c23defc09 && \
+    export PCRE_DOWNLOAD_URL=https://sourceforge.net/projects/pcre/files/pcre/8.45 && \
+    manylinux-entrypoint /build_scripts/build-swig.sh
+
+
+FROM build_base AS build_cpython
+COPY build_scripts/build-sqlite3.sh /build_scripts/
+RUN export SQLITE_AUTOCONF_ROOT=sqlite-autoconf-3360000 && \
+    export SQLITE_AUTOCONF_HASH=bd90c3eb96bee996206b83be7065c9ce19aef38c3f4fb53073ada0d0b69bbce3 && \
+    export SQLITE_AUTOCONF_DOWNLOAD_URL=https://www.sqlite.org/2021 && \
+    manylinux-entrypoint /build_scripts/build-sqlite3.sh
+
+COPY build_scripts/build-openssl.sh /build_scripts/
+RUN export OPENSSL_ROOT=openssl-1.1.1l && \
+    export OPENSSL_HASH=0b7a3e5e59c34827fe0c3a74b7ec8baef302b98fa80088d7f9153aa16fa76bd1 && \
+    export OPENSSL_DOWNLOAD_URL=https://www.openssl.org/source && \
+    manylinux-entrypoint /build_scripts/build-openssl.sh
+
+COPY build_scripts/build-cpython.sh /build_scripts/
+
+
+FROM build_cpython AS build_cpython37
+COPY build_scripts/cpython-pubkeys.txt /build_scripts/cpython-pubkeys.txt
+RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.7.12
+
+
+FROM build_cpython AS build_cpython38
+COPY build_scripts/ambv-pubkey.txt /build_scripts/cpython-pubkeys.txt
+RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.8.12
+
+
+FROM build_cpython AS build_cpython39
+COPY build_scripts/ambv-pubkey.txt /build_scripts/cpython-pubkeys.txt
+RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.9.8
+
+
+FROM build_cpython AS all_python
+COPY build_scripts/install-pypy.sh /build_scripts/install-pypy.sh
+COPY build_scripts/pypy.sha256 /build_scripts/pypy.sha256
+RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.7 7.3.7
+RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.8 7.3.7
+COPY --from=build_cpython37 /opt/_internal /opt/_internal/
+COPY --from=build_cpython38 /opt/_internal /opt/_internal/
+COPY --from=build_cpython39 /opt/_internal /opt/_internal/
+RUN hardlink -cv /opt/_internal
+
+
+FROM runtime_base
+COPY --from=build_git /manylinux-rootfs /
+COPY --from=build_swig /manylinux-rootfs /
+COPY --from=build_cpython /manylinux-rootfs /
+COPY --from=all_python /opt/_internal /opt/_internal/
+COPY build_scripts/finalize.sh \
+     build_scripts/update-system-packages.sh \
+     build_scripts/python-tag-abi-tag.py \
+     build_scripts/requirements3.7.txt \
+     build_scripts/requirements3.8.txt \
+     build_scripts/requirements3.9.txt \
+     build_scripts/requirements-base-tools.txt \
+     /build_scripts/
+COPY build_scripts/requirements-tools/* /build_scripts/requirements-tools/
+RUN manylinux-entrypoint /build_scripts/finalize.sh && rm -rf /build_scripts
+
+ENV SSL_CERT_FILE=/opt/_internal/certs.pem
+
+CMD ["/bin/bash"]
+
+#Build manylinux2014 docker image end
+
+ARG PYTHON_VERSION=3.7
+ARG TORCH_VERSION=1.11.0
+ARG OPSET_VERSION=14
+ARG INSTALL_DEPS_EXTRA_ARGS
+
+#Add our own dependencies
+ADD scripts /tmp/scripts
+RUN cd /tmp/scripts && \
+    /tmp/scripts/manylinux/install_centos.sh && \
+    /tmp/scripts/install_os_deps.sh -d gpu $INSTALL_DEPS_EXTRA_ARGS && \
+    /tmp/scripts/install_python_deps.sh -d gpu -p 3.7 -h ${TORCH_VERSION} -o 5.2 $INSTALL_DEPS_EXTRA_ARGS && \
+    /tmp/scripts/install_python_deps.sh -d gpu -p 3.8 -h ${TORCH_VERSION} -o 5.2 $INSTALL_DEPS_EXTRA_ARGS && \
+    /tmp/scripts/install_python_deps.sh -d gpu -p 3.9 -h ${TORCH_VERSION} -o 5.2 $INSTALL_DEPS_EXTRA_ARGS && \
+     rm -rf /tmp/scripts
+
+# remove protobuf to prevent ambiguity which is used for onnxruntime build
+RUN rm -fr /usr/local/bin/protoc \
+           /usr/local/libproto* \
+           /usr/local/include/google \
+           /usr/local/lib/pkgconfig/protobuf*
+
+ARG BUILD_UID=1001
+ARG BUILD_USER=onnxruntimedev
+RUN adduser --uid $BUILD_UID $BUILD_USER
+WORKDIR /home/$BUILD_USER
+USER $BUILD_USER
+ENV PATH /usr/local/gradle/bin:/usr/local/dotnet:$PATH
+ENV ORTMODULE_ONNX_OPSET_VERSION=$OPSET_VERSION
diff --git a/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements_torch1.11.0_rocm5.2/requirements.txt b/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements_torch1.11.0_rocm5.2/requirements.txt
new file mode 100644
index 0000000000..9091b054b6
--- /dev/null
+++ b/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements_torch1.11.0_rocm5.2/requirements.txt
@@ -0,0 +1,12 @@
+# transformers requires sklearn
+--pre
+-f https://repo.radeon.com/rocm/manylinux/rocm-rel-5.2/
+torch==1.11.0
+pandas
+sklearn
+numpy==1.21.0
+transformers==v4.3.2
+tensorboard>=2.2.0,<2.5.0
+h5py
+wget
+setuptools>=41.4.0