From 008c5f7640eb53c31fa063b1348cd5708746239e Mon Sep 17 00:00:00 2001 From: Suffian Khan Date: Tue, 6 Jul 2021 11:56:00 -0700 Subject: [PATCH] Use single builder image across Python versions for ROCm wheels (#8302) * first attempt share docker image across python and torch versons * set dependency between jobs * fix yaml grammer * remove python version from first stage * clean deepspeed directroy * split into two images according torch version * fix yaml syntax * invalidate cache * remove DS to prevent torch 1.9.0 upgrade --- .../templates/py-packaging-stage.yml | 68 ++++++++++++------- .../docker/Dockerfile.manylinux2014_rocm | 7 +- .../docker/scripts/install_python_deps.sh | 18 +++-- .../stage1/requirements-torch1.8.1_rocm.txt | 6 +- .../stage1/requirements-torch1.9.0_rocm.txt | 6 +- 5 files changed, 61 insertions(+), 44 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml index 0e3ae66f96..ddeec459dc 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml @@ -227,12 +227,48 @@ stages: - template: clean-agent-build-directory-step.yml - ${{ if eq(parameters.enable_linux_rocm_training, true) }}: - - job: Linux_py_ROCM_Wheels + - job: ROCm_build_environment + displayName: 'Construct ROCm wheels environment' timeoutInMinutes: 180 workspace: clean: all pool: AMD-GPU - # pool: Onnxruntime-Linux-GPU + steps: + - template: get-docker-image-steps.yml + parameters: + Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm + Context: tools/ci_build/github/linux/docker + DockerBuildArgs: >- + --build-arg TORCH_VERSION=1.8.1 + --build-arg INSTALL_DEPS_EXTRA_ARGS=-tmur + --build-arg BUILD_UID=$(id -u) + --network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64 + --build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-9/root + --build-arg PREPEND_PATH=/opt/rh/devtoolset-9/root/usr/bin: + --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-9/root/usr/lib64:/opt/rh/devtoolset-9/root/usr/lib:/opt/rh/devtoolset-9/root/usr/lib64/dyninst:/opt/rh/devtoolset-9/root/usr/lib/dyninst:/usr/local/lib64 + Repository: onnxruntimetrainingrocmbuild-torch1.8.1 + - template: get-docker-image-steps.yml + parameters: + Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm + Context: tools/ci_build/github/linux/docker + DockerBuildArgs: >- + --build-arg TORCH_VERSION=1.9.0 + --build-arg INSTALL_DEPS_EXTRA_ARGS=-tmur + --build-arg BUILD_UID=$(id -u) + --network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64 + --build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-9/root + --build-arg PREPEND_PATH=/opt/rh/devtoolset-9/root/usr/bin: + --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-9/root/usr/lib64:/opt/rh/devtoolset-9/root/usr/lib:/opt/rh/devtoolset-9/root/usr/lib64/dyninst:/opt/rh/devtoolset-9/root/usr/lib/dyninst:/usr/local/lib64 + Repository: onnxruntimetrainingrocmbuild-torch1.9.0 + + - job: ROCM_training_wheels + timeoutInMinutes: 180 + displayName: 'Build ROCm wheels (inside container)' + workspace: + clean: all + pool: AMD-GPU + dependsOn: + - ROCm_build_environment strategy: matrix: Python36 Torch181: @@ -267,21 +303,6 @@ stages: - template: set-python-manylinux-variables-step.yml - - template: get-docker-image-steps.yml - parameters: - Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm - Context: tools/ci_build/github/linux/docker - DockerBuildArgs: >- - --build-arg TORCH_VERSION=$(TorchVersion) - --build-arg PYTHON_VERSION=$(PythonVersion) - --build-arg INSTALL_DEPS_EXTRA_ARGS=-tmur - --build-arg BUILD_UID=$(id -u) - --network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64 - --build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-9/root - --build-arg PREPEND_PATH=/opt/rh/devtoolset-9/root/usr/bin: - --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-9/root/usr/lib64:/opt/rh/devtoolset-9/root/usr/lib:/opt/rh/devtoolset-9/root/usr/lib64/dyninst:/opt/rh/devtoolset-9/root/usr/lib/dyninst:/usr/local/lib64 - Repository: onnxruntimetrainingrocmbuild - - task: CmdLine@2 inputs: script: | @@ -296,11 +317,10 @@ stages: --volume $(Build.BinariesDirectory):/build \ --workdir /onnxruntime_src \ --entrypoint $(PythonManylinuxDir)/bin/python3 \ - -e NVIDIA_VISIBLE_DEVICES=all \ -e NIGHTLY_BUILD \ -e BUILD_BUILDNUMBER \ --user onnxruntimedev \ - onnxruntimetrainingrocmbuild \ + onnxruntimetrainingrocmbuild-torch$(TorchVersion) \ /onnxruntime_src/tools/ci_build/build.py \ --config Release \ --use_rocm \ @@ -355,7 +375,7 @@ stages: -e NIGHTLY_BUILD \ -e BUILD_BUILDNUMBER \ --user onnxruntimedev \ - onnxruntimetrainingrocmbuild \ + onnxruntimetrainingrocmbuild-torch$(TorchVersion) \ /onnxruntime_src/tools/ci_build/github/pai/pai_test_launcher.sh displayName: 'Run onnxruntime unit tests (in container)' @@ -378,7 +398,7 @@ stages: -e NIGHTLY_BUILD \ -e BUILD_BUILDNUMBER \ --user onnxruntimedev \ - onnxruntimetrainingrocmbuild \ + onnxruntimetrainingrocmbuild-torch$(TorchVersion) \ orttraining/tools/ci_test/run_batch_size_test.py \ --binary_dir /build/Release \ --model_root training_e2e_test_data/models \ @@ -405,7 +425,7 @@ stages: -e NIGHTLY_BUILD \ -e BUILD_BUILDNUMBER \ --user onnxruntimedev \ - onnxruntimetrainingrocmbuild \ + onnxruntimetrainingrocmbuild-torch$(TorchVersion) \ orttraining/tools/ci_test/run_bert_perf_test.py \ --binary_dir /build/Release \ --model_root training_e2e_test_data/models \ @@ -433,7 +453,7 @@ stages: -e NIGHTLY_BUILD \ -e BUILD_BUILDNUMBER \ --user onnxruntimedev \ - onnxruntimetrainingrocmbuild \ + onnxruntimetrainingrocmbuild-torch$(TorchVersion) \ orttraining/tools/ci_test/run_convergence_test.py \ --binary_dir /build/Release \ --model_root training_e2e_test_data/models \ @@ -472,7 +492,7 @@ stages: -e NIGHTLY_BUILD \ -e BUILD_BUILDNUMBER \ -e PythonManylinuxDir=$(PythonManylinuxdir) \ - onnxruntimetrainingrocmbuild \ + onnxruntimetrainingrocmbuild-torch$(TorchVersion) \ /onnxruntime_src/tools/ci_build/github/pai/wrap_rocm_python_doc_publisher.sh workingDirectory: $(Build.SourcesDirectory) diff --git a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm index 2bd6d9c4b6..a46c0a2add 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm +++ b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm @@ -179,8 +179,11 @@ ADD scripts /tmp/scripts RUN cd /tmp/scripts && \ /tmp/scripts/manylinux/install_centos.sh && \ /tmp/scripts/install_os_deps.sh -d gpu $INSTALL_DEPS_EXTRA_ARGS && \ - /tmp/scripts/install_python_deps.sh -d gpu -p $PYTHON_VERSION -h $TORCH_VERSION $INSTALL_DEPS_EXTRA_ARGS && \ - rm -rf /tmp/scripts + /tmp/scripts/install_python_deps.sh -d gpu -p 3.6 -h ${TORCH_VERSION} $INSTALL_DEPS_EXTRA_ARGS && \ + /tmp/scripts/install_python_deps.sh -d gpu -p 3.7 -h ${TORCH_VERSION} $INSTALL_DEPS_EXTRA_ARGS && \ + /tmp/scripts/install_python_deps.sh -d gpu -p 3.8 -h ${TORCH_VERSION} $INSTALL_DEPS_EXTRA_ARGS && \ + /tmp/scripts/install_python_deps.sh -d gpu -p 3.9 -h ${TORCH_VERSION} $INSTALL_DEPS_EXTRA_ARGS && \ + rm -rf /tmp/scripts ARG BUILD_UID=1001 ARG BUILD_USER=onnxruntimedev diff --git a/tools/ci_build/github/linux/docker/scripts/install_python_deps.sh b/tools/ci_build/github/linux/docker/scripts/install_python_deps.sh index 3301ac74ae..282efc402f 100755 --- a/tools/ci_build/github/linux/docker/scripts/install_python_deps.sh +++ b/tools/ci_build/github/linux/docker/scripts/install_python_deps.sh @@ -59,18 +59,16 @@ if [ $DEVICE_TYPE = "gpu" ]; then # Due to a [bug on DeepSpeed](https://github.com/microsoft/DeepSpeed/issues/663), we install it separately through ortmodule/stage2/requirements.txt ${PYTHON_EXE} -m pip install -r ${0/%install_python_deps.sh/training\/ortmodule\/stage2\/requirements.txt} else - ${PYTHON_EXE} -m pip install \ - --pre -f https://download.pytorch.org/whl/nightly/rocm4.2/torch_nightly.html \ - torch torchvision torchtext ${PYTHON_EXE} -m pip install -r ${0/%install_python_deps.sh/training\/ortmodule\/stage1\/requirements-torch${TORCH_VERSION}_rocm.txt} ${PYTHON_EXE} -m pip install fairscale - # remove triton requirement from getting triggered in requirements-sparse_attn.txt - git clone https://github.com/ROCmSoftwarePlatform/DeepSpeed - cd DeepSpeed &&\ - rm requirements/requirements-sparse_attn.txt &&\ - ${PYTHON_EXE} setup.py bdist_wheel &&\ - ${PYTHON_EXE} -m pip install dist/deepspeed*.whl &&\ - cd .. + # remove DeepSpeed until it's required for testing purposes + # remove triton requirement from getting triggered in requirements-sparse_attn.txt + # git clone https://github.com/ROCmSoftwarePlatform/DeepSpeed + # cd DeepSpeed &&\ + # rm requirements/requirements-sparse_attn.txt &&\ + # ${PYTHON_EXE} setup.py bdist_wheel &&\ + # ${PYTHON_EXE} -m pip install dist/deepspeed*.whl &&\ + # cd .. && rm -fr DeepSpeed fi fi fi diff --git a/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements-torch1.8.1_rocm.txt b/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements-torch1.8.1_rocm.txt index ac1524b451..5a1dc6fce4 100644 --- a/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements-torch1.8.1_rocm.txt +++ b/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements-torch1.8.1_rocm.txt @@ -1,9 +1,7 @@ # transformers requires sklearn --pre --f https://download.pytorch.org/whl/torch_stable.html -torch==1.8.1+cu111 -torchvision==0.9.1+cu111 -torchtext==0.9.1 +-f https://download.pytorch.org/whl/rocm4.2/torch_stable.html +torch==1.8.1 pandas sklearn numpy==1.19.5 diff --git a/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements-torch1.9.0_rocm.txt b/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements-torch1.9.0_rocm.txt index 3423e5fcf5..8c5422cddf 100644 --- a/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements-torch1.9.0_rocm.txt +++ b/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements-torch1.9.0_rocm.txt @@ -1,9 +1,7 @@ # transformers requires sklearn --pre --f https://download.pytorch.org/whl/torch_stable.html -torch==1.9.0+cu111 -torchvision==0.10.0+cu111 -torchtext==0.10.0 +-f https://download.pytorch.org/whl/rocm4.2/torch_stable.html +torch==1.9.0 pandas sklearn numpy==1.19.5