From 7694b695a91170f9ccf30f14f184b05d3e2dce66 Mon Sep 17 00:00:00 2001 From: PeixuanZuo <94887879+PeixuanZuo@users.noreply.github.com> Date: Thu, 8 Dec 2022 09:18:27 +0800 Subject: [PATCH] [ROCm] Simplify ROCm manylinux dockerfile (#13873) ### Description 1. Remove ROCm5.3 pipeline because it has rocblas bug, we don't need it. 2. We removed the dependency on centos docker image provided by AMD(https://hub.docker.com/r/rocm/dev-centos-7) and build ROCm centos base image by ourselves. The reference dockerfile(https://github.com/RadeonOpenCompute/ROCm-docker/blob/master/dev/Dockerfile-centos-7) is very redundant for our need. We simplified the ROCm manylinux dockerfile. 3. Different versions of rocm use the same dockerfile `Dockerfile.manylinux2014_rocm`. ### Motivation and Context Co-authored-by: peixuanzuo --- ...orttraining-py-packaging-pipeline-rocm.yml | 12 - .../github/azure-pipelines/templates/rocm.yml | 3 +- ...ocm5.2.3 => Dockerfile.manylinux2014_rocm} | 28 +- .../docker/Dockerfile.manylinux2014_rocm5.3 | 199 ------------ .../docker/Dockerfile.manylinux2014_rocm5.3.2 | 295 ------------------ 5 files changed, 26 insertions(+), 511 deletions(-) rename tools/ci_build/github/linux/docker/{Dockerfile.manylinux2014_rocm5.2.3 => Dockerfile.manylinux2014_rocm} (89%) delete mode 100644 tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm5.3 delete mode 100644 tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm5.3.2 diff --git a/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-rocm.yml b/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-rocm.yml index 7b71876d27..df8ef8ce9f 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-rocm.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-rocm.yml @@ -23,18 +23,6 @@ stages: parameters: PythonVersion: '3.9' RocmVersion: '5.2.3' - - template: templates/rocm.yml - parameters: - PythonVersion: '3.7' - RocmVersion: '5.3' - - template: templates/rocm.yml - parameters: - PythonVersion: '3.8' - RocmVersion: '5.3' - - template: templates/rocm.yml - parameters: - PythonVersion: '3.9' - RocmVersion: '5.3' - template: templates/rocm.yml parameters: PythonVersion: '3.7' diff --git a/tools/ci_build/github/azure-pipelines/templates/rocm.yml b/tools/ci_build/github/azure-pipelines/templates/rocm.yml index 7020c4b683..443e7deb8a 100644 --- a/tools/ci_build/github/azure-pipelines/templates/rocm.yml +++ b/tools/ci_build/github/azure-pipelines/templates/rocm.yml @@ -45,12 +45,13 @@ jobs: - template: set-python-manylinux-variables-step.yml - template: get-docker-image-steps.yml parameters: - Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm${{ parameters.RocmVersion }} + Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm Context: tools/ci_build/github/linux/docker DockerBuildArgs: >- --build-arg INSTALL_DEPS_EXTRA_ARGS=-tmur --build-arg BUILD_UID=$(id -u) --network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64 + --build-arg ROCM_VERSION=${{ parameters.RocmVersion }} --build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-10/root --build-arg PREPEND_PATH=/opt/rh/devtoolset-10/root/usr/bin: --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-10/root/usr/lib64:/opt/rh/devtoolset-10/root/usr/lib:/opt/rh/devtoolset-10/root/usr/lib64/dyninst:/opt/rh/devtoolset-10/root/usr/lib/dyninst:/usr/local/lib64:/usr/local/lib diff --git a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm5.2.3 b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm similarity index 89% rename from tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm5.2.3 rename to tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm index 7208e0d46c..f07e982bc9 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm5.2.3 +++ b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm @@ -1,12 +1,33 @@ -ARG BASEIMAGE=rocm/dev-centos-7:5.2.3 +ARG BASEIMAGE=centos:7 ARG POLICY=manylinux2014 ARG PLATFORM=x86_64 ARG DEVTOOLSET_ROOTPATH= ARG LD_LIBRARY_PATH_ARG= ARG PREPEND_PATH= +FROM $BASEIMAGE AS base_image +ARG ROCM_VERSION=5.3.2 + +# Enable epel-release repositories +RUN yum --enablerepo=extras install -y epel-release + +# Install the ROCm rpms +RUN yum clean all +RUN echo -e "[ROCm]\nname=ROCm\nbaseurl=https://repo.radeon.com/rocm/yum/$ROCM_VERSION/main\nenabled=1\ngpgcheck=0" >> /etc/yum.repos.d/rocm.repo + +# After ROCm5.2.3, amdgpu_version use the same version as rocm_version +RUN if [ "$ROCM_VERSION" = "5.2.3" ] ; \ + then echo -e "[amdgpu]\nname=amdgpu\nbaseurl=https://repo.radeon.com/amdgpu/22.20.3/rhel/7.9/main/x86_64\nenabled=1\ngpgcheck=0" >> /etc/yum.repos.d/amdgpu.repo; \ + else echo -e "[amdgpu]\nname=amdgpu\nbaseurl=https://repo.radeon.com/amdgpu/$ROCM_VERSION/rhel/7.9/main/x86_64\nenabled=1\ngpgcheck=0" >> /etc/yum.repos.d/amdgpu.repo; \ + fi + +RUN yum install -y rocm-dev + +# Set ENV +ENV PATH=/opt/rocm/hcc/bin:/opt/rocm/hip/bin:/opt/rocm/bin${PATH:+:${PATH}} + #Build manylinux2014 docker image begin -FROM $BASEIMAGE AS runtime_base +FROM base_image AS runtime_base ARG POLICY ARG PLATFORM ARG DEVTOOLSET_ROOTPATH @@ -19,6 +40,7 @@ LABEL maintainer="The ManyLinux project" RUN yum install -y hipify-clang # CMake +RUN yum -y install wget ENV CMAKE_VERSION=3.24.3 RUN cd /usr/local && \ wget -q -O - https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz | tar zxf - @@ -27,8 +49,6 @@ ENV PATH=/usr/local/cmake-${CMAKE_VERSION}-linux-x86_64/bin:${PATH} # rocm lib RUN yum install -y miopen-hip-devel rocblas-devel rocrand-devel rccl-devel hipsparse-devel hipfft-devel hipcub-devel hipblas-devel rocthrust-devel -RUN yum remove -y devtoolset\* git\* - ENV AUDITWHEEL_POLICY=${POLICY} AUDITWHEEL_ARCH=${PLATFORM} AUDITWHEEL_PLAT=${POLICY}_${PLATFORM} ENV LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 LANGUAGE=en_US.UTF-8 ENV DEVTOOLSET_ROOTPATH=${DEVTOOLSET_ROOTPATH} diff --git a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm5.3 b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm5.3 deleted file mode 100644 index d499d6cd51..0000000000 --- a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm5.3 +++ /dev/null @@ -1,199 +0,0 @@ -ARG BASEIMAGE=rocm/dev-centos-7:5.3 -ARG POLICY=manylinux2014 -ARG PLATFORM=x86_64 -ARG DEVTOOLSET_ROOTPATH= -ARG LD_LIBRARY_PATH_ARG= -ARG PREPEND_PATH= - -#Build manylinux2014 docker image begin -FROM $BASEIMAGE AS runtime_base -ARG POLICY -ARG PLATFORM -ARG DEVTOOLSET_ROOTPATH -ARG LD_LIBRARY_PATH_ARG -ARG PREPEND_PATH -LABEL maintainer="The ManyLinux project" - -# TODO: This should not be required if the hipify-perl binary is already a part -# of the base docker image. Remove this line moving forward. -RUN yum install -y hipify-clang - -# CMake -ENV CMAKE_VERSION=3.24.3 -RUN cd /usr/local && \ - wget -q -O - https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz | tar zxf - -ENV PATH=/usr/local/cmake-${CMAKE_VERSION}-linux-x86_64/bin:${PATH} - -# rocm lib -RUN yum install -y miopen-hip-devel rocblas-devel rocrand-devel rccl-devel hipsparse-devel hipfft-devel hipcub-devel hipblas-devel rocthrust-devel - -RUN yum remove -y devtoolset\* git\* - -ENV AUDITWHEEL_POLICY=${POLICY} AUDITWHEEL_ARCH=${PLATFORM} AUDITWHEEL_PLAT=${POLICY}_${PLATFORM} -ENV LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 LANGUAGE=en_US.UTF-8 -ENV DEVTOOLSET_ROOTPATH=${DEVTOOLSET_ROOTPATH} -ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH_ARG} -ENV PATH=${PREPEND_PATH}${PATH} -ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig - -# first copy the fixup mirrors script, keep the script around -COPY build_scripts/fixup-mirrors.sh /usr/local/sbin/fixup-mirrors - -# setup entrypoint, this will wrap commands with `linux32` with i686 images -COPY build_scripts/install-entrypoint.sh \ - build_scripts/update-system-packages.sh \ - build_scripts/build_utils.sh \ - /build_scripts/ - -RUN /build_scripts/install-entrypoint.sh && rm -rf /build_scripts -COPY manylinux-entrypoint /usr/local/bin/manylinux-entrypoint -ENTRYPOINT ["manylinux-entrypoint"] - -COPY build_scripts/install-runtime-packages.sh \ - build_scripts/update-system-packages.sh \ - build_scripts/build_utils.sh \ - /build_scripts/ -RUN manylinux-entrypoint /build_scripts/install-runtime-packages.sh && rm -rf /build_scripts/ - -COPY build_scripts/build_utils.sh /build_scripts/ - -COPY build_scripts/install-autoconf.sh /build_scripts/ -RUN export AUTOCONF_ROOT=autoconf-2.71 && \ - export AUTOCONF_HASH=431075ad0bf529ef13cb41e9042c542381103e80015686222b8a9d4abef42a1c && \ - export AUTOCONF_DOWNLOAD_URL=http://ftp.gnu.org/gnu/autoconf && \ - manylinux-entrypoint /build_scripts/install-autoconf.sh - -COPY build_scripts/install-automake.sh /build_scripts/ -RUN export AUTOMAKE_ROOT=automake-1.16.5 && \ - export AUTOMAKE_HASH=07bd24ad08a64bc17250ce09ec56e921d6343903943e99ccf63bbf0705e34605 && \ - export AUTOMAKE_DOWNLOAD_URL=http://ftp.gnu.org/gnu/automake && \ - manylinux-entrypoint /build_scripts/install-automake.sh - -COPY build_scripts/install-libtool.sh /build_scripts/ -RUN export LIBTOOL_ROOT=libtool-2.4.7 && \ - export LIBTOOL_HASH=04e96c2404ea70c590c546eba4202a4e12722c640016c12b9b2f1ce3d481e9a8 && \ - export LIBTOOL_DOWNLOAD_URL=http://ftp.gnu.org/gnu/libtool && \ - manylinux-entrypoint /build_scripts/install-libtool.sh - -COPY build_scripts/install-libxcrypt.sh /build_scripts/ -RUN export LIBXCRYPT_VERSION=4.4.28 && \ - export LIBXCRYPT_HASH=db7e37901969cb1d1e8020cb73a991ef81e48e31ea5b76a101862c806426b457 && \ - export LIBXCRYPT_DOWNLOAD_URL=https://github.com/besser82/libxcrypt/archive && \ - export PERL_ROOT=perl-5.34.0 && \ - export PERL_HASH=551efc818b968b05216024fb0b727ef2ad4c100f8cb6b43fab615fa78ae5be9a && \ - export PERL_DOWNLOAD_URL=https://www.cpan.org/src/5.0 && \ - manylinux-entrypoint /build_scripts/install-libxcrypt.sh - -COPY scripts/install-protobuf.sh /build_scripts/ -RUN export PROTOBUF_VERSION=3.17.3 && \ - export PROTOBUF_ROOT=protobuf-all-${PROTOBUF_VERSION} && \ - export PROTOBUF_HASH=77ad26d3f65222fd96ccc18b055632b0bfedf295cb748b712a98ba1ac0b704b2 && \ - export PROTOBUF_DOWNLOAD_URL=https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION} && \ - manylinux-entrypoint /build_scripts/install-protobuf.sh - -FROM runtime_base AS build_base -COPY build_scripts/install-build-packages.sh /build_scripts/ -RUN manylinux-entrypoint /build_scripts/install-build-packages.sh - - -FROM build_base AS build_git -COPY build_scripts/build-git.sh /build_scripts/ -RUN export GIT_ROOT=git-2.36.2 && \ - export GIT_HASH=6dc2cdea5fb23d823ba4871cc23222c1db31dfbb6d6c6ff74c4128700df57c68 && \ - export GIT_DOWNLOAD_URL=https://www.kernel.org/pub/software/scm/git && \ - manylinux-entrypoint /build_scripts/build-git.sh - - -FROM build_base AS build_cpython -COPY build_scripts/build-sqlite3.sh /build_scripts/ -RUN export SQLITE_AUTOCONF_ROOT=sqlite-autoconf-3390200 && \ - export SQLITE_AUTOCONF_HASH=852be8a6183a17ba47cee0bbff7400b7aa5affd283bf3beefc34fcd088a239de && \ - export SQLITE_AUTOCONF_DOWNLOAD_URL=https://www.sqlite.org/2022 && \ - manylinux-entrypoint /build_scripts/build-sqlite3.sh - -COPY build_scripts/build-openssl.sh /build_scripts/ -RUN export OPENSSL_ROOT=openssl-1.1.1q && \ - export OPENSSL_HASH=d7939ce614029cdff0b6c20f0e2e5703158a489a72b2507b8bd51bf8c8fd10ca && \ - export OPENSSL_DOWNLOAD_URL=https://www.openssl.org/source && \ - manylinux-entrypoint /build_scripts/build-openssl.sh - -COPY build_scripts/build-cpython.sh /build_scripts/ - - -FROM build_cpython AS build_cpython37 -COPY build_scripts/cpython-pubkeys.txt /build_scripts/cpython-pubkeys.txt -RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.7.13 - - -FROM build_cpython AS build_cpython38 -COPY build_scripts/ambv-pubkey.txt /build_scripts/cpython-pubkeys.txt -RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.8.13 - - -FROM build_cpython AS build_cpython39 -COPY build_scripts/ambv-pubkey.txt /build_scripts/cpython-pubkeys.txt -RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.9.13 - - -FROM build_cpython AS all_python -COPY build_scripts/install-pypy.sh \ - build_scripts/pypy.sha256 \ - build_scripts/finalize-python.sh \ - /build_scripts/ -RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.7 7.3.9 -RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.8 7.3.9 -RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.9 7.3.9 -COPY --from=build_cpython37 /opt/_internal /opt/_internal/ -COPY --from=build_cpython38 /opt/_internal /opt/_internal/ -COPY --from=build_cpython39 /opt/_internal /opt/_internal/ -RUN manylinux-entrypoint /build_scripts/finalize-python.sh - - -FROM runtime_base -COPY --from=build_git /manylinux-rootfs / -COPY --from=build_cpython /manylinux-rootfs / -COPY --from=all_python /opt/_internal /opt/_internal/ -COPY build_scripts/finalize.sh \ - build_scripts/update-system-packages.sh \ - build_scripts/python-tag-abi-tag.py \ - build_scripts/requirements3.7.txt \ - build_scripts/requirements3.8.txt \ - build_scripts/requirements3.9.txt \ - build_scripts/requirements-base-tools.txt \ - /build_scripts/ -COPY build_scripts/requirements-tools/* /build_scripts/requirements-tools/ -RUN manylinux-entrypoint /build_scripts/finalize.sh && rm -rf /build_scripts - -ENV SSL_CERT_FILE=/opt/_internal/certs.pem - -CMD ["/bin/bash"] - -#Build manylinux2014 docker image end - -ARG PYTHON_VERSION=3.7 -ARG OPSET_VERSION=15 -ARG INSTALL_DEPS_EXTRA_ARGS - -#Add our own dependencies -ADD scripts /tmp/scripts -RUN cd /tmp/scripts && \ - /tmp/scripts/manylinux/install_centos.sh && \ - /tmp/scripts/install_os_deps.sh -d gpu $INSTALL_DEPS_EXTRA_ARGS && \ - /tmp/scripts/install_python_deps.sh -d gpu -p 3.7 $INSTALL_DEPS_EXTRA_ARGS && \ - /tmp/scripts/install_python_deps.sh -d gpu -p 3.8 $INSTALL_DEPS_EXTRA_ARGS && \ - /tmp/scripts/install_python_deps.sh -d gpu -p 3.9 $INSTALL_DEPS_EXTRA_ARGS && \ - rm -rf /tmp/scripts - -# remove protobuf to prevent ambiguity which is used for onnxruntime build -RUN rm -fr /usr/local/bin/protoc \ - /usr/local/libproto* \ - /usr/local/include/google \ - /usr/local/lib/pkgconfig/protobuf* - -ARG BUILD_UID=1001 -ARG BUILD_USER=onnxruntimedev -RUN adduser --uid $BUILD_UID $BUILD_USER -WORKDIR /home/$BUILD_USER -USER $BUILD_USER -ENV PATH /usr/local/gradle/bin:/usr/local/dotnet:$PATH -ENV ORTMODULE_ONNX_OPSET_VERSION=$OPSET_VERSION diff --git a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm5.3.2 b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm5.3.2 deleted file mode 100644 index 91c7591df4..0000000000 --- a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm5.3.2 +++ /dev/null @@ -1,295 +0,0 @@ -ARG BASEIMAGE=centos:7 -ARG POLICY=manylinux2014 -ARG PLATFORM=x86_64 -ARG DEVTOOLSET_ROOTPATH= -ARG LD_LIBRARY_PATH_ARG= -ARG PREPEND_PATH= - -# Same as rocm/dev-centos-7:5.3.2 -# We build ROCm5.3.2 dev centos docker image manually(stage base_image, line11~line102). -# It's same as https://github.com/RadeonOpenCompute/ROCm-docker/blob/master/dev/Dockerfile-centos-7 -FROM $BASEIMAGE AS base_image -ARG ROCM_VERSION=5.3.2 -ARG AMDGPU_VERSION=5.3.2 -# Base -RUN yum -y install git java-1.8.0-openjdk python; yum clean all - -# Enable epel-release repositories -RUN yum --enablerepo=extras install -y epel-release - -# Install required base build and packaging commands for ROCm -RUN yum -y install \ - ca-certificates \ - bc \ - bridge-utils \ - cmake \ - cmake3 \ - devscripts \ - dkms \ - doxygen \ - dpkg \ - dpkg-dev \ - dpkg-perl \ - elfutils-libelf-devel \ - expect \ - file \ - python3 \ - python3-pip \ - gettext \ - gcc-c++ \ - libgcc \ - glibc.i686 \ - libcxx-devel \ - ncurses \ - ncurses-base \ - ncurses-libs \ - numactl-devel \ - numactl-libs \ - libssh \ - libunwind-devel \ - libunwind \ - llvm \ - llvm-libs \ - make \ - openssl \ - openssl-libs \ - openssh \ - openssh-clients \ - pciutils \ - pciutils-devel \ - pciutils-libs \ - python \ - python-pip \ - python-devel \ - pkgconfig \ - pth \ - qemu-kvm \ - re2c \ - kmod \ - file \ - rpm \ - rpm-build \ - subversion \ - wget - -# Enable the epel repository for fakeroot -RUN yum --enablerepo=extras install -y fakeroot -RUN yum clean all - -# On CentOS, install package centos-release-scl available in CentOS repository: -RUN yum install -y centos-release-scl - -# Install the devtoolset-7 collection: -RUN yum install -y devtoolset-7 -RUN yum install -y devtoolset-7-libatomic-devel devtoolset-7-elfutils-libelf-devel - -# Install the ROCm rpms -RUN yum clean all -RUN echo -e "[ROCm]\nname=ROCm\nbaseurl=https://repo.radeon.com/rocm/yum/$ROCM_VERSION/main\nenabled=1\ngpgcheck=0" >> /etc/yum.repos.d/rocm.repo -RUN echo -e "[amdgpu]\nname=amdgpu\nbaseurl=https://repo.radeon.com/amdgpu/$AMDGPU_VERSION/rhel/7.9/main/x86_64\nenabled=1\ngpgcheck=0" >> /etc/yum.repos.d/amdgpu.repo - -RUN yum install -y rocm-dev - -# Set ENV to enable devtoolset7 by default -ENV PATH=/opt/rh/devtoolset-7/root/usr/bin:/opt/rocm/hcc/bin:/opt/rocm/hip/bin:/opt/rocm/bin:/opt/rocm/hcc/bin:${PATH:+:${PATH}} -ENV MANPATH=/opt/rh/devtoolset-7/root/usr/share/man:${MANPATH} -ENV INFOPATH=/opt/rh/devtoolset-7/root/usr/share/info${INFOPATH:+:${INFOPATH}} -ENV PCP_DIR=/opt/rh/devtoolset-7/root -ENV PERL5LIB=/opt/rh/devtoolset-7/root//usr/lib64/perl5/vendor_perl:/opt/rh/devtoolset-7/root/usr/lib/perl5:/opt/rh/devtoolset-7/root//usr/share/perl5/ -ENV LD_LIBRARY_PATH=/opt/rocm/lib:/usr/local/lib:/opt/rh/devtoolset-7/root$rpmlibdir$rpmlibdir32${LD_LIBRARY_PATH:+:${LD_LIBRARY_PATH}} -ENV PYTHONPATH=/opt/rh/devtoolset-7/root/usr/lib64/python$pythonvers/site-packages:/opt/rh/devtoolset-7/root/usr/lib/python$pythonvers/ -ENV LDFLAGS="-Wl,-rpath=/opt/rh/devtoolset-7/root/usr/lib64 -Wl,-rpath=/opt/rh/devtoolset-7/root/usr/lib" - - -#Build manylinux2014 docker image begin -FROM base_image AS runtime_base -ARG POLICY -ARG PLATFORM -ARG DEVTOOLSET_ROOTPATH -ARG LD_LIBRARY_PATH_ARG -ARG PREPEND_PATH -LABEL maintainer="The ManyLinux project" - -# TODO: This should not be required if the hipify-perl binary is already a part -# of the base docker image. Remove this line moving forward. -RUN yum install -y hipify-clang - -# CMake -ENV CMAKE_VERSION=3.24.3 -RUN cd /usr/local && \ - wget -q -O - https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz | tar zxf - -ENV PATH=/usr/local/cmake-${CMAKE_VERSION}-linux-x86_64/bin:${PATH} - -# rocm lib -RUN yum install -y miopen-hip-devel rocblas-devel rocrand-devel rccl-devel hipsparse-devel hipfft-devel hipcub-devel hipblas-devel rocthrust-devel - -RUN yum remove -y devtoolset\* git\* - -ENV AUDITWHEEL_POLICY=${POLICY} AUDITWHEEL_ARCH=${PLATFORM} AUDITWHEEL_PLAT=${POLICY}_${PLATFORM} -ENV LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 LANGUAGE=en_US.UTF-8 -ENV DEVTOOLSET_ROOTPATH=${DEVTOOLSET_ROOTPATH} -ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH_ARG} -ENV PATH=${PREPEND_PATH}${PATH} -ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig - -# first copy the fixup mirrors script, keep the script around -COPY build_scripts/fixup-mirrors.sh /usr/local/sbin/fixup-mirrors - -# setup entrypoint, this will wrap commands with `linux32` with i686 images -COPY build_scripts/install-entrypoint.sh \ - build_scripts/update-system-packages.sh \ - build_scripts/build_utils.sh \ - /build_scripts/ - -RUN /build_scripts/install-entrypoint.sh && rm -rf /build_scripts -COPY manylinux-entrypoint /usr/local/bin/manylinux-entrypoint -ENTRYPOINT ["manylinux-entrypoint"] - -COPY build_scripts/install-runtime-packages.sh \ - build_scripts/update-system-packages.sh \ - build_scripts/build_utils.sh \ - /build_scripts/ -RUN manylinux-entrypoint /build_scripts/install-runtime-packages.sh && rm -rf /build_scripts/ - -COPY build_scripts/build_utils.sh /build_scripts/ - -COPY build_scripts/install-autoconf.sh /build_scripts/ -RUN export AUTOCONF_ROOT=autoconf-2.71 && \ - export AUTOCONF_HASH=431075ad0bf529ef13cb41e9042c542381103e80015686222b8a9d4abef42a1c && \ - export AUTOCONF_DOWNLOAD_URL=http://ftp.gnu.org/gnu/autoconf && \ - manylinux-entrypoint /build_scripts/install-autoconf.sh - -COPY build_scripts/install-automake.sh /build_scripts/ -RUN export AUTOMAKE_ROOT=automake-1.16.5 && \ - export AUTOMAKE_HASH=07bd24ad08a64bc17250ce09ec56e921d6343903943e99ccf63bbf0705e34605 && \ - export AUTOMAKE_DOWNLOAD_URL=http://ftp.gnu.org/gnu/automake && \ - manylinux-entrypoint /build_scripts/install-automake.sh - -COPY build_scripts/install-libtool.sh /build_scripts/ -RUN export LIBTOOL_ROOT=libtool-2.4.7 && \ - export LIBTOOL_HASH=04e96c2404ea70c590c546eba4202a4e12722c640016c12b9b2f1ce3d481e9a8 && \ - export LIBTOOL_DOWNLOAD_URL=http://ftp.gnu.org/gnu/libtool && \ - manylinux-entrypoint /build_scripts/install-libtool.sh - -COPY build_scripts/install-libxcrypt.sh /build_scripts/ -RUN export LIBXCRYPT_VERSION=4.4.28 && \ - export LIBXCRYPT_HASH=db7e37901969cb1d1e8020cb73a991ef81e48e31ea5b76a101862c806426b457 && \ - export LIBXCRYPT_DOWNLOAD_URL=https://github.com/besser82/libxcrypt/archive && \ - export PERL_ROOT=perl-5.34.0 && \ - export PERL_HASH=551efc818b968b05216024fb0b727ef2ad4c100f8cb6b43fab615fa78ae5be9a && \ - export PERL_DOWNLOAD_URL=https://www.cpan.org/src/5.0 && \ - manylinux-entrypoint /build_scripts/install-libxcrypt.sh - -COPY scripts/install-protobuf.sh /build_scripts/ -RUN export PROTOBUF_VERSION=3.17.3 && \ - export PROTOBUF_ROOT=protobuf-all-${PROTOBUF_VERSION} && \ - export PROTOBUF_HASH=77ad26d3f65222fd96ccc18b055632b0bfedf295cb748b712a98ba1ac0b704b2 && \ - export PROTOBUF_DOWNLOAD_URL=https://github.com/protocolbuffers/protobuf/releases/download/v${PROTOBUF_VERSION} && \ - manylinux-entrypoint /build_scripts/install-protobuf.sh - -FROM runtime_base AS build_base -COPY build_scripts/install-build-packages.sh /build_scripts/ -RUN manylinux-entrypoint /build_scripts/install-build-packages.sh - - -FROM build_base AS build_git -COPY build_scripts/build-git.sh /build_scripts/ -RUN export GIT_ROOT=git-2.36.2 && \ - export GIT_HASH=6dc2cdea5fb23d823ba4871cc23222c1db31dfbb6d6c6ff74c4128700df57c68 && \ - export GIT_DOWNLOAD_URL=https://www.kernel.org/pub/software/scm/git && \ - manylinux-entrypoint /build_scripts/build-git.sh - - -FROM build_base AS build_cpython -COPY build_scripts/build-sqlite3.sh /build_scripts/ -RUN export SQLITE_AUTOCONF_ROOT=sqlite-autoconf-3390200 && \ - export SQLITE_AUTOCONF_HASH=852be8a6183a17ba47cee0bbff7400b7aa5affd283bf3beefc34fcd088a239de && \ - export SQLITE_AUTOCONF_DOWNLOAD_URL=https://www.sqlite.org/2022 && \ - manylinux-entrypoint /build_scripts/build-sqlite3.sh - -COPY build_scripts/build-openssl.sh /build_scripts/ -RUN export OPENSSL_ROOT=openssl-1.1.1q && \ - export OPENSSL_HASH=d7939ce614029cdff0b6c20f0e2e5703158a489a72b2507b8bd51bf8c8fd10ca && \ - export OPENSSL_DOWNLOAD_URL=https://www.openssl.org/source && \ - manylinux-entrypoint /build_scripts/build-openssl.sh - -COPY build_scripts/build-cpython.sh /build_scripts/ - - -FROM build_cpython AS build_cpython37 -COPY build_scripts/cpython-pubkeys.txt /build_scripts/cpython-pubkeys.txt -RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.7.13 - - -FROM build_cpython AS build_cpython38 -COPY build_scripts/ambv-pubkey.txt /build_scripts/cpython-pubkeys.txt -RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.8.13 - - -FROM build_cpython AS build_cpython39 -COPY build_scripts/ambv-pubkey.txt /build_scripts/cpython-pubkeys.txt -RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.9.13 - - -FROM build_cpython AS all_python -COPY build_scripts/install-pypy.sh \ - build_scripts/pypy.sha256 \ - build_scripts/finalize-python.sh \ - /build_scripts/ -RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.7 7.3.9 -RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.8 7.3.9 -RUN manylinux-entrypoint /build_scripts/install-pypy.sh 3.9 7.3.9 -COPY --from=build_cpython37 /opt/_internal /opt/_internal/ -COPY --from=build_cpython38 /opt/_internal /opt/_internal/ -COPY --from=build_cpython39 /opt/_internal /opt/_internal/ -RUN manylinux-entrypoint /build_scripts/finalize-python.sh - - -FROM runtime_base -COPY --from=build_git /manylinux-rootfs / -COPY --from=build_cpython /manylinux-rootfs / -COPY --from=all_python /opt/_internal /opt/_internal/ -COPY build_scripts/finalize.sh \ - build_scripts/update-system-packages.sh \ - build_scripts/python-tag-abi-tag.py \ - build_scripts/requirements3.7.txt \ - build_scripts/requirements3.8.txt \ - build_scripts/requirements3.9.txt \ - build_scripts/requirements-base-tools.txt \ - /build_scripts/ -COPY build_scripts/requirements-tools/* /build_scripts/requirements-tools/ -RUN manylinux-entrypoint /build_scripts/finalize.sh && rm -rf /build_scripts - -ENV SSL_CERT_FILE=/opt/_internal/certs.pem - -CMD ["/bin/bash"] - -#Build manylinux2014 docker image end - -ARG PYTHON_VERSION=3.7 -ARG OPSET_VERSION=15 -ARG INSTALL_DEPS_EXTRA_ARGS - -#Add our own dependencies -ADD scripts /tmp/scripts -RUN cd /tmp/scripts && \ - /tmp/scripts/manylinux/install_centos.sh && \ - /tmp/scripts/install_os_deps.sh -d gpu $INSTALL_DEPS_EXTRA_ARGS && \ - /tmp/scripts/install_python_deps.sh -d gpu -p 3.7 $INSTALL_DEPS_EXTRA_ARGS && \ - /tmp/scripts/install_python_deps.sh -d gpu -p 3.8 $INSTALL_DEPS_EXTRA_ARGS && \ - /tmp/scripts/install_python_deps.sh -d gpu -p 3.9 $INSTALL_DEPS_EXTRA_ARGS && \ - rm -rf /tmp/scripts - -# remove protobuf to prevent ambiguity which is used for onnxruntime build -RUN rm -fr /usr/local/bin/protoc \ - /usr/local/libproto* \ - /usr/local/include/google \ - /usr/local/lib/pkgconfig/protobuf* - -ARG BUILD_UID=1001 -ARG BUILD_USER=onnxruntimedev -RUN adduser --uid $BUILD_UID $BUILD_USER -WORKDIR /home/$BUILD_USER -USER $BUILD_USER -ENV PATH /usr/local/gradle/bin:/usr/local/dotnet:$PATH -ENV ORTMODULE_ONNX_OPSET_VERSION=$OPSET_VERSION