saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-10 17:37:14 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Use single builder image across Python versions for ROCm wheels (#8302)

Browse source 
* first attempt share docker image across python and torch versons

* set dependency between jobs

* fix yaml grammer

* remove python version from first stage

* clean deepspeed directroy

* split into two images according torch version

* fix yaml syntax

* invalidate cache

* remove DS to prevent torch 1.9.0 upgrade
This commit is contained in:
Suffian Khan 2021-07-06 11:56:00 -07:00 committed by GitHub
parent 56e4dd1d3e
commit 008c5f7640
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 61 additions and 44 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

68
tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml
View file

@ -227,12 +227,48 @@ stages:
- template: clean-agent-build-directory-step.yml
- ${{ if eq(parameters.enable_linux_rocm_training, true) }}:
- job: Linux_py_ROCM_Wheels
- job: ROCm_build_environment
displayName: 'Construct ROCm wheels environment'
timeoutInMinutes: 180
workspace:
clean: all
pool: AMD-GPU
# pool: Onnxruntime-Linux-GPU
steps:
- template: get-docker-image-steps.yml
parameters:
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm
Context: tools/ci_build/github/linux/docker
DockerBuildArgs: >-
--build-arg TORCH_VERSION=1.8.1
--build-arg INSTALL_DEPS_EXTRA_ARGS=-tmur
--build-arg BUILD_UID=$(id -u)
--network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64
--build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-9/root
--build-arg PREPEND_PATH=/opt/rh/devtoolset-9/root/usr/bin:
--build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-9/root/usr/lib64:/opt/rh/devtoolset-9/root/usr/lib:/opt/rh/devtoolset-9/root/usr/lib64/dyninst:/opt/rh/devtoolset-9/root/usr/lib/dyninst:/usr/local/lib64
Repository: onnxruntimetrainingrocmbuild-torch1.8.1
- template: get-docker-image-steps.yml
parameters:
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm
Context: tools/ci_build/github/linux/docker
DockerBuildArgs: >-
--build-arg TORCH_VERSION=1.9.0
--build-arg INSTALL_DEPS_EXTRA_ARGS=-tmur
--build-arg BUILD_UID=$(id -u)
--network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64
--build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-9/root
--build-arg PREPEND_PATH=/opt/rh/devtoolset-9/root/usr/bin:
--build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-9/root/usr/lib64:/opt/rh/devtoolset-9/root/usr/lib:/opt/rh/devtoolset-9/root/usr/lib64/dyninst:/opt/rh/devtoolset-9/root/usr/lib/dyninst:/usr/local/lib64
Repository: onnxruntimetrainingrocmbuild-torch1.9.0
- job: ROCM_training_wheels
timeoutInMinutes: 180
displayName: 'Build ROCm wheels (inside container)'
workspace:
clean: all
pool: AMD-GPU
dependsOn:
- ROCm_build_environment
strategy:
matrix:
Python36 Torch181:
@ -267,21 +303,6 @@ stages:
- template: set-python-manylinux-variables-step.yml
- template: get-docker-image-steps.yml
parameters:
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm
Context: tools/ci_build/github/linux/docker
DockerBuildArgs: >-
--build-arg TORCH_VERSION=$(TorchVersion)
--build-arg PYTHON_VERSION=$(PythonVersion)
--build-arg INSTALL_DEPS_EXTRA_ARGS=-tmur
--build-arg BUILD_UID=$(id -u)
--network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64
--build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-9/root
--build-arg PREPEND_PATH=/opt/rh/devtoolset-9/root/usr/bin:
--build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-9/root/usr/lib64:/opt/rh/devtoolset-9/root/usr/lib:/opt/rh/devtoolset-9/root/usr/lib64/dyninst:/opt/rh/devtoolset-9/root/usr/lib/dyninst:/usr/local/lib64
Repository: onnxruntimetrainingrocmbuild
- task: CmdLine@2
inputs:
script: |
@ -296,11 +317,10 @@ stages:
--volume $(Build.BinariesDirectory):/build \
--workdir /onnxruntime_src \
--entrypoint $(PythonManylinuxDir)/bin/python3 \
-e NVIDIA_VISIBLE_DEVICES=all \
-e NIGHTLY_BUILD \
-e BUILD_BUILDNUMBER \
--user onnxruntimedev \
onnxruntimetrainingrocmbuild \
onnxruntimetrainingrocmbuild-torch$(TorchVersion) \
/onnxruntime_src/tools/ci_build/build.py \
--config Release \
--use_rocm \
@ -355,7 +375,7 @@ stages:
-e NIGHTLY_BUILD \
-e BUILD_BUILDNUMBER \
--user onnxruntimedev \
onnxruntimetrainingrocmbuild \
onnxruntimetrainingrocmbuild-torch$(TorchVersion) \
/onnxruntime_src/tools/ci_build/github/pai/pai_test_launcher.sh
displayName: 'Run onnxruntime unit tests (in container)'
@ -378,7 +398,7 @@ stages:
-e NIGHTLY_BUILD \
-e BUILD_BUILDNUMBER \
--user onnxruntimedev \
onnxruntimetrainingrocmbuild \
onnxruntimetrainingrocmbuild-torch$(TorchVersion) \
orttraining/tools/ci_test/run_batch_size_test.py \
--binary_dir /build/Release \
--model_root training_e2e_test_data/models \
@ -405,7 +425,7 @@ stages:
-e NIGHTLY_BUILD \
-e BUILD_BUILDNUMBER \
--user onnxruntimedev \
onnxruntimetrainingrocmbuild \
onnxruntimetrainingrocmbuild-torch$(TorchVersion) \
orttraining/tools/ci_test/run_bert_perf_test.py \
--binary_dir /build/Release \
--model_root training_e2e_test_data/models \
@ -433,7 +453,7 @@ stages:
-e NIGHTLY_BUILD \
-e BUILD_BUILDNUMBER \
--user onnxruntimedev \
onnxruntimetrainingrocmbuild \
onnxruntimetrainingrocmbuild-torch$(TorchVersion) \
orttraining/tools/ci_test/run_convergence_test.py \
--binary_dir /build/Release \
--model_root training_e2e_test_data/models \
@ -472,7 +492,7 @@ stages:
-e NIGHTLY_BUILD \
-e BUILD_BUILDNUMBER \
-e PythonManylinuxDir=$(PythonManylinuxdir) \
onnxruntimetrainingrocmbuild \
onnxruntimetrainingrocmbuild-torch$(TorchVersion) \
/onnxruntime_src/tools/ci_build/github/pai/wrap_rocm_python_doc_publisher.sh
workingDirectory: $(Build.SourcesDirectory)

7
tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_rocm
View file

@ -179,8 +179,11 @@ ADD scripts /tmp/scripts
RUN cd /tmp/scripts && \
/tmp/scripts/manylinux/install_centos.sh && \
/tmp/scripts/install_os_deps.sh -d gpu $INSTALL_DEPS_EXTRA_ARGS && \
/tmp/scripts/install_python_deps.sh -d gpu -p $PYTHON_VERSION -h $TORCH_VERSION $INSTALL_DEPS_EXTRA_ARGS && \
rm -rf /tmp/scripts
/tmp/scripts/install_python_deps.sh -d gpu -p 3.6 -h ${TORCH_VERSION} $INSTALL_DEPS_EXTRA_ARGS && \
/tmp/scripts/install_python_deps.sh -d gpu -p 3.7 -h ${TORCH_VERSION} $INSTALL_DEPS_EXTRA_ARGS && \
/tmp/scripts/install_python_deps.sh -d gpu -p 3.8 -h ${TORCH_VERSION} $INSTALL_DEPS_EXTRA_ARGS && \
/tmp/scripts/install_python_deps.sh -d gpu -p 3.9 -h ${TORCH_VERSION} $INSTALL_DEPS_EXTRA_ARGS && \
rm -rf /tmp/scripts
ARG BUILD_UID=1001
ARG BUILD_USER=onnxruntimedev

18
tools/ci_build/github/linux/docker/scripts/install_python_deps.sh
View file

@ -59,18 +59,16 @@ if [ $DEVICE_TYPE = "gpu" ]; then
# Due to a [bug on DeepSpeed](https://github.com/microsoft/DeepSpeed/issues/663), we install it separately through ortmodule/stage2/requirements.txt
${PYTHON_EXE} -m pip install -r ${0/%install_python_deps.sh/training\/ortmodule\/stage2\/requirements.txt}
else
${PYTHON_EXE} -m pip install \
--pre -f https://download.pytorch.org/whl/nightly/rocm4.2/torch_nightly.html \
torch torchvision torchtext
${PYTHON_EXE} -m pip install -r ${0/%install_python_deps.sh/training\/ortmodule\/stage1\/requirements-torch${TORCH_VERSION}_rocm.txt}
${PYTHON_EXE} -m pip install fairscale
# remove triton requirement from getting triggered in requirements-sparse_attn.txt
git clone https://github.com/ROCmSoftwarePlatform/DeepSpeed
cd DeepSpeed &&\
rm requirements/requirements-sparse_attn.txt &&\
${PYTHON_EXE} setup.py bdist_wheel &&\
${PYTHON_EXE} -m pip install dist/deepspeed*.whl &&\
cd ..
# remove DeepSpeed until it's required for testing purposes
# remove triton requirement from getting triggered in requirements-sparse_attn.txt
# git clone https://github.com/ROCmSoftwarePlatform/DeepSpeed
# cd DeepSpeed &&\
# rm requirements/requirements-sparse_attn.txt &&\
# ${PYTHON_EXE} setup.py bdist_wheel &&\
# ${PYTHON_EXE} -m pip install dist/deepspeed*.whl &&\
# cd .. && rm -fr DeepSpeed
fi
fi
fi

6
tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements-torch1.8.1_rocm.txt
View file

@ -1,9 +1,7 @@
# transformers requires sklearn
--pre
-f https://download.pytorch.org/whl/torch_stable.html
torch==1.8.1+cu111
torchvision==0.9.1+cu111
torchtext==0.9.1
-f https://download.pytorch.org/whl/rocm4.2/torch_stable.html
torch==1.8.1
pandas
sklearn
numpy==1.19.5

6
tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements-torch1.9.0_rocm.txt
View file

@ -1,9 +1,7 @@
# transformers requires sklearn
--pre
-f https://download.pytorch.org/whl/torch_stable.html
torch==1.9.0+cu111
torchvision==0.10.0+cu111
torchtext==0.10.0
-f https://download.pytorch.org/whl/rocm4.2/torch_stable.html
torch==1.9.0
pandas
sklearn
numpy==1.19.5