From bec24ca4c17d7228f3a6c00df44d1a40aa54d905 Mon Sep 17 00:00:00 2001 From: liqun Fu Date: Wed, 11 Aug 2021 17:44:57 -0700 Subject: [PATCH] create packaging pipeline to support cuda11.4 (#8663) --- ...py-packaging-pipeline-torch190-cuda114.yml | 12 ++ ...Dockerfile.manylinux2014_training_cuda11_4 | 193 ++++++++++++++++++ .../stage1/requirements_torch1.9.0_cu11.4.txt | 9 + 3 files changed, 214 insertions(+) create mode 100644 tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-torch190-cuda114.yml create mode 100644 tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_training_cuda11_4 create mode 100644 tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements_torch1.9.0_cu11.4.txt diff --git a/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-torch190-cuda114.yml b/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-torch190-cuda114.yml new file mode 100644 index 0000000000..72c3ffd942 --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-torch190-cuda114.yml @@ -0,0 +1,12 @@ +trigger: none + +stages: +- template: templates/py-packaging-training-cuda-stage.yml + parameters: + build_py_parameters: --enable_training --update --build + torch_version: '1.9.0' + cuda_version: '11.4' + gcc_version: 9 + cmake_cuda_architectures: 37;50;52;60;61;70;75;80;86 + docker_file: Dockerfile.manylinux2014_training_cuda11_4 + agent_pool: Onnxruntime-Linux-GPU diff --git a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_training_cuda11_4 b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_training_cuda11_4 new file mode 100644 index 0000000000..82d4382af9 --- /dev/null +++ b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_training_cuda11_4 @@ -0,0 +1,193 @@ +ARG BASEIMAGE=nvidia/cuda:11.4.0-cudnn8-devel-centos7 +ARG POLICY=manylinux2014 +ARG PLATFORM=x86_64 +ARG DEVTOOLSET_ROOTPATH= +ARG LD_LIBRARY_PATH_ARG= +ARG PREPEND_PATH= + +#We need both CUDA and manylinux. But the CUDA Toolkit End User License Agreement says NVIDIA CUDA Driver Libraries(libcuda.so, libnvidia-ptxjitcompiler.so) are only distributable in applications that meet this criteria: +#1. The application was developed starting from a NVIDIA CUDA container obtained from Docker Hub or the NVIDIA GPU Cloud, and +#2. The resulting application is packaged as a Docker container and distributed to users on Docker Hub or the NVIDIA GPU Cloud only. +#So we use CUDA as the base image then add manylinux on top of it. + +#Build manylinux2014 docker image begin +FROM $BASEIMAGE AS runtime_base +ARG POLICY +ARG PLATFORM +ARG DEVTOOLSET_ROOTPATH +ARG LD_LIBRARY_PATH_ARG +ARG PREPEND_PATH +LABEL maintainer="The ManyLinux project" + +ENV AUDITWHEEL_POLICY=${POLICY} AUDITWHEEL_ARCH=${PLATFORM} AUDITWHEEL_PLAT=${POLICY}_${PLATFORM} +ENV LC_ALL=en_US.UTF-8 LANG=en_US.UTF-8 LANGUAGE=en_US.UTF-8 +ENV DEVTOOLSET_ROOTPATH=${DEVTOOLSET_ROOTPATH} +ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH_ARG} +ENV PATH=${PREPEND_PATH}${PATH} +ENV PKG_CONFIG_PATH=/usr/local/lib/pkgconfig + +# first copy the fixup mirrors script, keep the script around +COPY build_scripts/fixup-mirrors.sh /usr/bin/fixup-mirrors + +# setup entrypoint, this will wrap commands with `linux32` with i686 images +COPY build_scripts/install-entrypoint.sh build_scripts/update-system-packages.sh /build_scripts/ +RUN bash /build_scripts/install-entrypoint.sh && rm -rf /build_scripts +COPY manylinux-entrypoint /usr/local/bin/manylinux-entrypoint +ENTRYPOINT ["manylinux-entrypoint"] + +COPY build_scripts/install-runtime-packages.sh build_scripts/update-system-packages.sh /build_scripts/ +RUN manylinux-entrypoint /build_scripts/install-runtime-packages.sh && rm -rf /build_scripts/ + +COPY build_scripts/build_utils.sh /build_scripts/ + +COPY build_scripts/install-autoconf.sh /build_scripts/ +RUN export AUTOCONF_ROOT=autoconf-2.71 && \ + export AUTOCONF_HASH=431075ad0bf529ef13cb41e9042c542381103e80015686222b8a9d4abef42a1c && \ + export AUTOCONF_DOWNLOAD_URL=http://ftp.gnu.org/gnu/autoconf && \ + manylinux-entrypoint /build_scripts/install-autoconf.sh + +COPY build_scripts/install-automake.sh /build_scripts/ +RUN export AUTOMAKE_ROOT=automake-1.16.3 && \ + export AUTOMAKE_HASH=ce010788b51f64511a1e9bb2a1ec626037c6d0e7ede32c1c103611b9d3cba65f && \ + export AUTOMAKE_DOWNLOAD_URL=http://ftp.gnu.org/gnu/automake && \ + manylinux-entrypoint /build_scripts/install-automake.sh + +COPY build_scripts/install-libtool.sh /build_scripts/ +RUN export LIBTOOL_ROOT=libtool-2.4.6 && \ + export LIBTOOL_HASH=e3bd4d5d3d025a36c21dd6af7ea818a2afcd4dfc1ea5a17b39d7854bcd0c06e3 && \ + export LIBTOOL_DOWNLOAD_URL=http://ftp.gnu.org/gnu/libtool && \ + manylinux-entrypoint /build_scripts/install-libtool.sh + +COPY build_scripts/install-patchelf.sh /build_scripts/ +RUN export PATCHELF_VERSION=0.12 && \ + export PATCHELF_HASH=3dca33fb862213b3541350e1da262249959595903f559eae0fbc68966e9c3f56 && \ + export PATCHELF_DOWNLOAD_URL=https://github.com/NixOS/patchelf/archive && \ + manylinux-entrypoint /build_scripts/install-patchelf.sh + +COPY build_scripts/install-libxcrypt.sh /build_scripts/ +RUN export LIBXCRYPT_VERSION=4.4.17 && \ + export LIBXCRYPT_HASH=7665168d0409574a03f7b484682e68334764c29c21ca5df438955a381384ca07 && \ + export LIBXCRYPT_DOWNLOAD_URL=https://github.com/besser82/libxcrypt/archive && \ + manylinux-entrypoint /build_scripts/install-libxcrypt.sh + + +FROM runtime_base AS build_base +COPY build_scripts/install-build-packages.sh /build_scripts/ +RUN manylinux-entrypoint /build_scripts/install-build-packages.sh + + +FROM build_base AS build_git +COPY build_scripts/build-git.sh /build_scripts/ +RUN export GIT_ROOT=git-2.31.1 && \ + export GIT_HASH=46d37c229e9d786510e0c53b60065704ce92d5aedc16f2c5111e3ed35093bfa7 && \ + export GIT_DOWNLOAD_URL=https://www.kernel.org/pub/software/scm/git && \ + manylinux-entrypoint /build_scripts/build-git.sh + + +FROM build_base AS build_cmake +COPY build_scripts/build-cmake.sh /build_scripts/ +RUN export CMAKE_VERSION=3.21.0 && \ + export CMAKE_HASH=4a42d56449a51f4d3809ab4d3b61fd4a96a469e56266e896ce1009b5768bd2ab && \ + export CMAKE_DOWNLOAD_URL=https://github.com/Kitware/CMake/releases/download && \ + manylinux-entrypoint /build_scripts/build-cmake.sh + + +FROM build_base AS build_swig +COPY build_scripts/build-swig.sh /build_scripts/ +RUN export SWIG_ROOT=swig-4.0.2 && \ + export SWIG_HASH=d53be9730d8d58a16bf0cbd1f8ac0c0c3e1090573168bfa151b01eb47fa906fc && \ + export SWIG_DOWNLOAD_URL=https://sourceforge.net/projects/swig/files/swig/${SWIG_ROOT} && \ + export PCRE_ROOT=pcre-8.44 && \ + export PCRE_HASH=aecafd4af3bd0f3935721af77b889d9024b2e01d96b58471bd91a3063fb47728 && \ + export PCRE_DOWNLOAD_URL=https://ftp.pcre.org/pub/pcre && \ + manylinux-entrypoint /build_scripts/build-swig.sh + + +FROM build_base AS build_cpython +COPY build_scripts/build-sqlite3.sh /build_scripts/ +RUN export SQLITE_AUTOCONF_ROOT=sqlite-autoconf-3350500 && \ + export SQLITE_AUTOCONF_HASH=f52b72a5c319c3e516ed7a92e123139a6e87af08a2dc43d7757724f6132e6db0 && \ + export SQLITE_AUTOCONF_DOWNLOAD_URL=https://www.sqlite.org/2021 && \ + manylinux-entrypoint /build_scripts/build-sqlite3.sh + +COPY build_scripts/build-openssl.sh /build_scripts/ +RUN export OPENSSL_ROOT=openssl-1.1.1k && \ + export OPENSSL_HASH=892a0875b9872acd04a9fde79b1f943075d5ea162415de3047c327df33fbaee5 && \ + export OPENSSL_DOWNLOAD_URL=https://www.openssl.org/source && \ + manylinux-entrypoint /build_scripts/build-openssl.sh + +COPY build_scripts/build-cpython.sh /build_scripts/ + + +FROM build_cpython AS build_cpython36 +COPY build_scripts/cpython-pubkeys.txt /build_scripts/cpython-pubkeys.txt + +RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.6.13 + + +FROM build_cpython AS build_cpython37 +COPY build_scripts/cpython-pubkeys.txt /build_scripts/cpython-pubkeys.txt + +RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.7.10 + + +FROM build_cpython AS build_cpython38 +COPY build_scripts/ambv-pubkey.txt /build_scripts/cpython-pubkeys.txt + +RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.8.10 + + +FROM build_cpython AS build_cpython39 +COPY build_scripts/ambv-pubkey.txt /build_scripts/cpython-pubkeys.txt + +RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.9.5 + + +FROM build_cpython AS build_cpython310 +COPY build_scripts/cpython-pubkey-310-311.txt /build_scripts/cpython-pubkeys.txt + +RUN manylinux-entrypoint /build_scripts/build-cpython.sh 3.10.0b1 + + +FROM build_cpython AS all_cpython +COPY --from=build_cpython36 /opt/_internal /opt/_internal/ +COPY --from=build_cpython37 /opt/_internal /opt/_internal/ +COPY --from=build_cpython38 /opt/_internal /opt/_internal/ +COPY --from=build_cpython39 /opt/_internal /opt/_internal/ +COPY --from=build_cpython310 /opt/_internal /opt/_internal/ +RUN hardlink -cv /opt/_internal + + +FROM runtime_base +COPY --from=build_git /manylinux-rootfs / +COPY --from=build_cmake /manylinux-rootfs / +COPY --from=build_swig /manylinux-rootfs / +COPY --from=build_cpython /manylinux-rootfs / +COPY --from=all_cpython /opt/_internal /opt/_internal/ +COPY build_scripts/finalize.sh build_scripts/update-system-packages.sh build_scripts/python-tag-abi-tag.py build_scripts/requirements.txt build_scripts/requirements-tools.txt /build_scripts/ +RUN manylinux-entrypoint /build_scripts/finalize.sh && rm -rf /build_scripts + +ENV SSL_CERT_FILE=/opt/_internal/certs.pem + +CMD ["/bin/bash"] + +#Build manylinux2014 docker image end +ARG PYTHON_VERSION=3.6 +ARG TORCH_VERSION=1.9.0 +ARG INSTALL_DEPS_EXTRA_ARGS + +#Add our own dependencies +ADD scripts /tmp/scripts +RUN cd /tmp/scripts && \ + /tmp/scripts/manylinux/install_centos.sh && \ + /tmp/scripts/install_os_deps.sh -d gpu $INSTALL_DEPS_EXTRA_ARGS && \ + /tmp/scripts/install_ninja.sh && \ + /tmp/scripts/install_python_deps.sh -d gpu -v 11.4 -p $PYTHON_VERSION -h $TORCH_VERSION $INSTALL_DEPS_EXTRA_ARGS && \ + rm -rf /tmp/scripts + +ARG BUILD_UID=1001 +ARG BUILD_USER=onnxruntimedev +RUN adduser --uid $BUILD_UID $BUILD_USER +WORKDIR /home/$BUILD_USER +USER $BUILD_USER +ENV PATH /usr/local/gradle/bin:/usr/local/dotnet:$PATH diff --git a/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements_torch1.9.0_cu11.4.txt b/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements_torch1.9.0_cu11.4.txt new file mode 100644 index 0000000000..fa3a86bce2 --- /dev/null +++ b/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements_torch1.9.0_cu11.4.txt @@ -0,0 +1,9 @@ +# TODO: Remove pinned version of Pillow of 8.2.0 when 8.3.0 release regression issue https://github.com/python-pillow/Pillow/issues/5571 is resolved +Pillow==8.2.0 +# TODO: set to cu114 once it is available from pytorch, and to match torchvision and torchtext versions +--pre +-f https://download.pytorch.org/whl/torch_stable.html +torch==1.9.0+cu111 +torchvision==0.10.0+cu111 +torchtext==0.10.0 +setuptools>=41.4.0