From eec518780172038893eaccd8b745bfaa6573ca65 Mon Sep 17 00:00:00 2001 From: ytaous <4484531+ytaous@users.noreply.github.com> Date: Thu, 7 Apr 2022 11:42:09 -0700 Subject: [PATCH] Remove Rocm 4.2 from CI Build (#11130) * remove rocm42 CI * update torch to v1.11.0 Co-authored-by: Ethan Tao --- .../templates/py-packaging-stage.yml | 29 +-- .../docker/Dockerfile.manylinux2014_rocm4_2 | 216 ------------------ .../docker/Dockerfile.manylinux2014_rocm4_3_1 | 2 +- .../docker/Dockerfile.manylinux2014_rocm5_0_1 | 2 +- .../requirements.txt | 12 - .../requirements.txt | 2 +- 6 files changed, 5 insertions(+), 258 deletions(-) delete mode 100644 tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm4_2 delete mode 100644 tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements_torch1.10.0_rocm/requirements.txt diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml index 14030d241c..d626ecc4ac 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml @@ -296,25 +296,12 @@ stages: clean: all pool: AMD-GPU steps: - - template: get-docker-image-steps.yml - parameters: - Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm4_2 - Context: tools/ci_build/github/linux/docker - DockerBuildArgs: >- - --build-arg TORCH_VERSION=1.10.0 - --build-arg INSTALL_DEPS_EXTRA_ARGS=-tmur - --build-arg BUILD_UID=$(id -u) - --network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64 - --build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-10/root - --build-arg PREPEND_PATH=/opt/rh/devtoolset-10/root/usr/bin: - --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-10/root/usr/lib64:/opt/rh/devtoolset-10/root/usr/lib:/opt/rh/devtoolset-10/root/usr/lib64/dyninst:/opt/rh/devtoolset-10/root/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib - Repository: onnxruntimetrainingrocmbuild-torch1.10.0-rocm4.2 - template: get-docker-image-steps.yml parameters: Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm4_3_1 Context: tools/ci_build/github/linux/docker DockerBuildArgs: >- - --build-arg TORCH_VERSION=1.10.0 + --build-arg TORCH_VERSION=1.11.0 --build-arg INSTALL_DEPS_EXTRA_ARGS=-tmur --build-arg BUILD_UID=$(id -u) --network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64 @@ -327,7 +314,7 @@ stages: Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm5_0_1 Context: tools/ci_build/github/linux/docker DockerBuildArgs: >- - --build-arg TORCH_VERSION=1.10.0 + --build-arg TORCH_VERSION=1.11.0 --build-arg INSTALL_DEPS_EXTRA_ARGS=-tmur --build-arg BUILD_UID=$(id -u) --network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64 @@ -346,18 +333,6 @@ stages: - ROCm_build_environment strategy: matrix: - Python37 Torch1100 Rocm42: - PythonVersion: '3.7' - TorchVersion: '1.10.0' - RocmVersion: '4.2' - Python38 Torch1100 Rocm42: - PythonVersion: '3.8' - TorchVersion: '1.10.0' - RocmVersion: '4.2' - Python39 Torch1100 Rocm42: - PythonVersion: '3.9' - TorchVersion: '1.10.0' - RocmVersion: '4.2' Python37 Torch1100 Rocm431: PythonVersion: '3.7' TorchVersion: '1.10.0' diff --git a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm4_2 b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm4_2 deleted file mode 100644 index 9d5926911a..0000000000 --- a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm4_2 +++ /dev/null @@ -1,216 +0,0 @@ -ARG BASEIMAGE=rocm/pytorch:rocm4.2_centos7_py3.6_pytorch -ARG POLICY=manylinux2014 -ARG PLATFORM=x86_64 -ARG DEVTOOLSET_ROOTPATH= -ARG LD_LIBRARY_PATH_ARG= -ARG PREPEND_PATH= - -#Build manylinux2014 docker image begin -FROM $BASEIMAGE AS runtime_base -ARG POLICY -ARG PLATFORM -ARG DEVTOOLSET_ROOTPATH -ARG LD_LIBRARY_PATH_ARG -ARG PREPEND_PATH -LABEL maintainer="The ManyLinux project" - -RUN yum remove -y devtoolset\* git\* && conda remove -y cmake - -# remove protobuf 2.6.1 from rocm/pytorch:rocm4.2_centos7_py3.6_pytorch -# it's too old to compile onnx 1.10 -RUN rm -fr /usr/local/bin/protoc \ - /usr/local/libproto* \ - /usr/local/include/google \ - /usr/local/lib/pkgconfig/protobuf* - -ENV AUDITWHEEL_POLICY=${POLICY} AUDITWHEEL_ARCH=${PLATFORM} AUDITWHEEL_PLAT=${POLICY}_${PLATFORM} -ENV LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 LANGUAGE=en_US.UTF-8 -ENV DEVTOOLSET_ROOTPATH=${DEVTOOLSET_ROOTPATH} -ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH_ARG} -ENV PATH=${PREPEND_PATH}${PATH} -ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig - -# first copy the fixup mirrors script, keep the script around -COPY build_scripts/fixup-mirrors.sh /usr/local/sbin/fixup-mirrors - -# setup entrypoint, this will wrap commands with `linux32` with i686 images -COPY build_scripts/install-entrypoint.sh \ - build_scripts/update-system-packages.sh \ - build_scripts/build_utils.sh \ - /build_scripts/ - -RUN /build_scripts/install-entrypoint.sh && rm -rf /build_scripts -COPY manylinux-entrypoint /usr/local/bin/manylinux-entrypoint -ENTRYPOINT ["manylinux-entrypoint"] - -COPY build_scripts/install-runtime-packages.sh \ - build_scripts/update-system-packages.sh \ - build_scripts/build_utils.sh \ - /build_scripts/ -RUN manylinux-entrypoint /build_scripts/install-runtime-packages.sh && rm -rf /build_scripts/ - -COPY build_scripts/build_utils.sh /build_scripts/ - -COPY build_scripts/install-autoconf.sh /build_scripts/ -RUN export AUTOCONF_ROOT=autoconf-2.71 && \ - export AUTOCONF_HASH=431075ad0bf529ef13cb41e9042c542381103e80015686222b8a9d4abef42a1c && \ - export AUTOCONF_DOWNLOAD_URL=http://ftp.gnu.org/gnu/autoconf && \ - manylinux-entrypoint /build_scripts/install-autoconf.sh - -COPY build_scripts/install-automake.sh /build_scripts/ -RUN export AUTOMAKE_ROOT=automake-1.16.5 && \ - export AUTOMAKE_HASH=07bd24ad08a64bc17250ce09ec56e921d6343903943e99ccf63bbf0705e34605 && \ - export AUTOMAKE_DOWNLOAD_URL=http://ftp.gnu.org/gnu/automake && \ - manylinux-entrypoint /build_scripts/install-automake.sh - -COPY build_scripts/install-libtool.sh /build_scripts/ -RUN export LIBTOOL_ROOT=libtool-2.4.6 && \ - export LIBTOOL_HASH=e3bd4d5d3d025a36c21dd6af7ea818a2afcd4dfc1ea5a17b39d7854bcd0c06e3 && \ - export LIBTOOL_DOWNLOAD_URL=http://ftp.gnu.org/gnu/libtool && \ - manylinux-entrypoint /build_scripts/install-libtool.sh - -COPY build_scripts/install-patchelf.sh /build_scripts/ -RUN export PATCHELF_VERSION=0.13 && \ - export PATCHELF_HASH=60c6aeadb673de9cc1838b630c81f61e31c501de324ef7f1e8094a2431197d09 && \ - export PATCHELF_DOWNLOAD_URL=https://github.com/NixOS/patchelf/archive && \ - manylinux-entrypoint /build_scripts/install-patchelf.sh - -COPY build_scripts/install-libxcrypt.sh /build_scripts/ -RUN export LIBXCRYPT_VERSION=4.4.26 && \ - export LIBXCRYPT_HASH=e8a544dd19171c1e6191a6044c96cc31496d781ba08b5a00f53310d001d58114 && \ - export LIBXCRYPT_DOWNLOAD_URL=https://github.com/besser82/libxcrypt/archive && \ - export PERL_ROOT=perl-5.34.0 && \ - export PERL_HASH=551efc818b968b05216024fb0b727ef2ad4c100f8cb6b43fab615fa78ae5be9a && \ - export PERL_DOWNLOAD_URL=https://www.cpan.org/src/5.0 && \ - manylinux-entrypoint /build_scripts/install-libxcrypt.sh - -COPY build_scripts/install-protobuf.sh /build_scripts/ -RUN export PROTOBUF_VERSION=3.17.3 && \ - export PROTOBUF_ROOT=protobuf-all-${PROTOBUF_VERSION} && \ - export PROTOBUF_HASH=77ad26d3f65222fd96ccc18b055632b0bfedf295cb748b712a98ba1ac0b704b2 && \ - export PROTOBUF_DOWNLOAD_URL=https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION} && \ - manylinux-entrypoint /build_scripts/install-protobuf.sh - -FROM runtime_base AS build_base -COPY build_scripts/install-build-packages.sh /build_scripts/ -RUN manylinux-entrypoint /build_scripts/install-build-packages.sh - - -FROM build_base AS build_git -COPY build_scripts/build-git.sh /build_scripts/ -RUN export GIT_ROOT=git-2.33.1 && \ - export GIT_HASH=02047f8dc8934d57ff5e02aadd8a2fe8e0bcf94a7158da375e48086cc46fce1d && \ - export GIT_DOWNLOAD_URL=https://www.kernel.org/pub/software/scm/git && \ - manylinux-entrypoint /build_scripts/build-git.sh - - -FROM build_base AS build_swig -COPY build_scripts/build-swig.sh /build_scripts/ -RUN export SWIG_ROOT=swig-4.0.2 && \ - export SWIG_HASH=d53be9730d8d58a16bf0cbd1f8ac0c0c3e1090573168bfa151b01eb47fa906fc && \ - export SWIG_DOWNLOAD_URL=https://sourceforge.net/projects/swig/files/swig/${SWIG_ROOT} && \ - export PCRE_ROOT=pcre-8.45 && \ - export PCRE_HASH=4e6ce03e0336e8b4a3d6c2b70b1c5e18590a5673a98186da90d4f33c23defc09 && \ - export PCRE_DOWNLOAD_URL=https://sourceforge.net/projects/pcre/files/pcre/8.45 && \ - manylinux-entrypoint /build_scripts/build-swig.sh - - -FROM build_base AS build_cpython -COPY build_scripts/build-sqlite3.sh /build_scripts/ -RUN export SQLITE_AUTOCONF_ROOT=sqlite-autoconf-3360000 && \ - export SQLITE_AUTOCONF_HASH=bd90c3eb96bee996206b83be7065c9ce19aef38c3f4fb53073ada0d0b69bbce3 && \ - export SQLITE_AUTOCONF_DOWNLOAD_URL=https://www.sqlite.org/2021 && \ - manylinux-entrypoint /build_scripts/build-sqlite3.sh - -COPY build_scripts/build-openssl.sh /build_scripts/ -RUN export OPENSSL_ROOT=openssl-1.1.1l && \ - export OPENSSL_HASH=0b7a3e5e59c34827fe0c3a74b7ec8baef302b98fa80088d7f9153aa16fa76bd1 && \ - export OPENSSL_DOWNLOAD_URL=https://www.openssl.org/source && \ - manylinux-entrypoint /build_scripts/build-openssl.sh - -COPY build_scripts/build-cpython.sh /build_scripts/ - - -FROM build_cpython AS build_cpython36 -COPY build_scripts/cpython-pubkeys.txt /build_scripts/cpython-pubkeys.txt -RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.6.15 - - -FROM build_cpython AS build_cpython37 -COPY build_scripts/cpython-pubkeys.txt /build_scripts/cpython-pubkeys.txt -RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.7.12 - - -FROM build_cpython AS build_cpython38 -COPY build_scripts/ambv-pubkey.txt /build_scripts/cpython-pubkeys.txt -RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.8.12 - - -FROM build_cpython AS build_cpython39 -COPY build_scripts/ambv-pubkey.txt /build_scripts/cpython-pubkeys.txt -RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.9.8 - - -FROM build_cpython AS all_python -COPY build_scripts/install-pypy.sh /build_scripts/install-pypy.sh -COPY build_scripts/pypy.sha256 /build_scripts/pypy.sha256 -RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.7 7.3.7 -RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.8 7.3.7 -COPY --from=build_cpython36 /opt/_internal /opt/_internal/ -COPY --from=build_cpython37 /opt/_internal /opt/_internal/ -COPY --from=build_cpython38 /opt/_internal /opt/_internal/ -COPY --from=build_cpython39 /opt/_internal /opt/_internal/ -RUN hardlink -cv /opt/_internal - - -FROM runtime_base -COPY --from=build_git /manylinux-rootfs / -COPY --from=build_swig /manylinux-rootfs / -COPY --from=build_cpython /manylinux-rootfs / -COPY --from=all_python /opt/_internal /opt/_internal/ -COPY build_scripts/finalize.sh \ - build_scripts/update-system-packages.sh \ - build_scripts/python-tag-abi-tag.py \ - build_scripts/requirements3.6.txt \ - build_scripts/requirements3.7.txt \ - build_scripts/requirements3.8.txt \ - build_scripts/requirements3.9.txt \ - build_scripts/requirements-base-tools.txt \ - /build_scripts/ -COPY build_scripts/requirements-tools/* /build_scripts/requirements-tools/ -RUN manylinux-entrypoint /build_scripts/finalize.sh && rm -rf /build_scripts - -ENV SSL_CERT_FILE=/opt/_internal/certs.pem - -CMD ["/bin/bash"] - -#Build manylinux2014 docker image end - -ARG PYTHON_VERSION=3.6 -ARG TORCH_VERSION=1.9.0 -ARG OPSET_VERSION=12 -ARG INSTALL_DEPS_EXTRA_ARGS - -#Add our own dependencies -ADD scripts /tmp/scripts -RUN cd /tmp/scripts && \ - /tmp/scripts/manylinux/install_centos.sh && \ - /tmp/scripts/install_os_deps.sh -d gpu $INSTALL_DEPS_EXTRA_ARGS && \ - /tmp/scripts/install_python_deps.sh -d gpu -p 3.7 -h ${TORCH_VERSION} $INSTALL_DEPS_EXTRA_ARGS && \ - /tmp/scripts/install_python_deps.sh -d gpu -p 3.8 -h ${TORCH_VERSION} $INSTALL_DEPS_EXTRA_ARGS && \ - /tmp/scripts/install_python_deps.sh -d gpu -p 3.9 -h ${TORCH_VERSION} $INSTALL_DEPS_EXTRA_ARGS && \ - rm -rf /tmp/scripts - -# remove protobuf to prevent ambiguity which is used for onnxruntime build -RUN rm -fr /usr/local/bin/protoc \ - /usr/local/libproto* \ - /usr/local/include/google \ - /usr/local/lib/pkgconfig/protobuf* - -ARG BUILD_UID=1001 -ARG BUILD_USER=onnxruntimedev -RUN adduser --uid $BUILD_UID $BUILD_USER -WORKDIR /home/$BUILD_USER -USER $BUILD_USER -ENV PATH /usr/local/gradle/bin:/usr/local/dotnet:$PATH -ENV ORTMODULE_ONNX_OPSET_VERSION=$OPSET_VERSION diff --git a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm4_3_1 b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm4_3_1 index 0ea7ca5a93..1bba5aa3b7 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm4_3_1 +++ b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm4_3_1 @@ -187,7 +187,7 @@ CMD ["/bin/bash"] #Build manylinux2014 docker image end ARG PYTHON_VERSION=3.6 -ARG TORCH_VERSION=1.10.0 +ARG TORCH_VERSION=1.11.0 ARG OPSET_VERSION=14 ARG INSTALL_DEPS_EXTRA_ARGS diff --git a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm5_0_1 b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm5_0_1 index 9cfde07c2a..12cfca233f 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm5_0_1 +++ b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm5_0_1 @@ -191,7 +191,7 @@ CMD ["/bin/bash"] #Build manylinux2014 docker image end ARG PYTHON_VERSION=3.7 -ARG TORCH_VERSION=1.10.0 +ARG TORCH_VERSION=1.11.0 ARG OPSET_VERSION=14 ARG INSTALL_DEPS_EXTRA_ARGS diff --git a/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements_torch1.10.0_rocm/requirements.txt b/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements_torch1.10.0_rocm/requirements.txt deleted file mode 100644 index 65ccaad468..0000000000 --- a/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements_torch1.10.0_rocm/requirements.txt +++ /dev/null @@ -1,12 +0,0 @@ -# transformers requires sklearn ---pre --f https://download.pytorch.org/whl/rocm4.2/torch_stable.html -torch==1.10.0 -pandas -sklearn -numpy==1.21.0 -transformers==v4.3.2 -tensorboard>=2.2.0,<2.5.0 -h5py -wget -setuptools>=41.4.0 diff --git a/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements_torch1.11.0_rocm/requirements.txt b/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements_torch1.11.0_rocm/requirements.txt index eb7c5593db..648244f121 100644 --- a/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements_torch1.11.0_rocm/requirements.txt +++ b/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements_torch1.11.0_rocm/requirements.txt @@ -1,6 +1,6 @@ # transformers requires sklearn --pre --f https://download.pytorch.org/whl/rocm4.2/torch_stable.html +-f https://download.pytorch.org/whl/rocm4.3.1/torch_stable.html torch==1.11.0 pandas sklearn