saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-30 20:18:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add support for cuda 11.8 and python 3.11 for training (#15548)

Browse source
This commit is contained in:
Baiju Meswani 2023-04-20 12:56:45 -07:00 committed by GitHub
parent 1f7c2f724f
commit 11b0a18de6
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
17 changed files with 44 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
docs/ORTModule_Training_Guidelines.md
View file

@ -11,7 +11,7 @@ export CUDA_HOME=/usr/local/cuda
export CUDNN_HOME=/usr/local/cuda
export CUDACXX=$CUDA_HOME/bin/nvcc
./build.sh --config RelWithDebInfo --use_cuda --enable_training --build_wheel --skip_tests --cuda_version=11.6 --parallel 8 --use_mpi
./build.sh --config RelWithDebInfo --use_cuda --enable_training --build_wheel --skip_tests --cuda_version=11.8 --parallel 8 --use_mpi
```
Install the Python wheel.

11
orttraining/orttraining/test/python/orttraining_test_ortmodule_deepspeed_pipeline_parallel.py
View file

@ -2,12 +2,10 @@ import argparse
import deepspeed
import torch
import torch.distributed as dist
from deepspeed.pipe import LayerSpec, PipelineModule # noqa: F401
from deepspeed.utils import RepeatingLoader # noqa: F401
from torch import nn, optim # noqa: F401
from deepspeed.pipe import PipelineModule
from torch import nn
from onnxruntime.training.ortmodule import ORTModule, _utils # noqa: F401
from onnxruntime.training.ortmodule import ORTModule
# USAGE:
# pip install deepspeed
@ -56,7 +54,8 @@ else:
print("Running deepspeed pipeline parallel module with ORTModule")
dist.init_process_group(backend=args.backend)
# dist.init_process_group(backend=args.backend)
deepspeed.init_distributed(dist_backend=args.backend)
torch.manual_seed(args.seed)
# Model.

4
orttraining/orttraining/test/python/orttraining_test_ortmodule_fallback.py
View file

@ -562,6 +562,10 @@ def test_ortmodule_fallback_warn_message(is_training, persist_fallback):
del os.environ["ORTMODULE_SKIPCHECK_POLICY"]
# This test now results in a different error:
# torch.onnx.errors.UnsupportedOperatorError: Exporting the operator 'aten::unflatten' to ONNX opset version 15 is not supported.
# Skip this test for pytorch 2.0 until fix identified.
@pytest.mark.xfail(reason="This test now results in an export error.")
@pytest.mark.parametrize("is_training,persist_fallback", list(itertools.product([True, False], repeat=2)))
def test_ortmodule_fallback_non_contiguous_tensors(is_training, persist_fallback):
# is_training: True for torch.nn.Module training model, eval mode otherwise

2
tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ci-pipeline.yml
View file

@ -14,7 +14,7 @@ jobs:
-x "
--enable_training
--config $(buildConfig)
--use_cuda --cuda_version=11.6 --cuda_home=/usr/local/cuda-11.6 --cudnn_home=/usr/local/cuda-11.6
--use_cuda --cuda_version=11.8 --cuda_home=/usr/local/cuda-11.8 --cudnn_home=/usr/local/cuda-11.8
--build_wheel
--enable_nvtx_profile
--cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=70

4
tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ortmodule-distributed-test-ci-pipeline.yml
View file

@ -21,7 +21,7 @@ stages:
-t onnxruntime_ortmodule_distributed_tests_image \
-x " \
--config RelWithDebInfo \
--use_cuda --cuda_version=11.6 --cuda_home=/usr/local/cuda-11.6 --cudnn_home=/usr/local/cuda-11.6 \
--use_cuda --cuda_version=11.8 --cuda_home=/usr/local/cuda-11.8 --cudnn_home=/usr/local/cuda-11.8 \
--enable_training \
--update --build \
--build_wheel \
@ -77,7 +77,7 @@ stages:
-t onnxruntime_ortmodule_distributed_tests_image \
-x " \
--config RelWithDebInfo \
--use_cuda --cuda_version=11.6 --cuda_home=/usr/local/cuda-11.6 --cudnn_home=/usr/local/cuda-11.6 \
--use_cuda --cuda_version=11.8 --cuda_home=/usr/local/cuda-11.8 --cudnn_home=/usr/local/cuda-11.8 \
--update --build \
--build_wheel \
--use_mpi \

2
tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-training-apis.yml
View file

@ -20,7 +20,7 @@ jobs:
--config RelWithDebInfo \
--enable_training \
--enable_training_apis \
--use_cuda --cuda_version=11.6 --cuda_home=/usr/local/cuda-11.6 --cudnn_home=/usr/local/cuda-11.6 \
--use_cuda --cuda_version=11.8 --cuda_home=/usr/local/cuda-11.8 --cudnn_home=/usr/local/cuda-11.8 \
--build_wheel \
--build_java \
--skip_tests \

2
tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cpu.yml
View file

@ -35,6 +35,8 @@ stages:
PythonVersion: '3.9'
Python310:
PythonVersion: '3.10'
Python311:
PythonVersion: '3.11'
steps:
- task: CmdLine@2

12
tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cuda116.yml → tools/ci_build/github/azure-pipelines/orttraining-py-packaging-pipeline-cuda.yml
View file

@ -12,12 +12,12 @@ stages:
- template: templates/py-packaging-training-cuda-stage.yml
parameters:
build_py_parameters: --enable_training --update --build
torch_version: '1.13.1'
torch_version: '2.0.0'
opset_version: '15'
cuda_version: '11.6'
cuda_version: '11.8'
gcc_version: 11
cmake_cuda_architectures: 50;52;60;61;70;75;80;86;87
docker_file: Dockerfile.manylinux2014_training_cuda11_6
docker_file: Dockerfile.manylinux2014_training_cuda11_8
agent_pool: Onnxruntime-Linux-GPU
upload_wheel: 'yes'
debug_build: false
@ -25,12 +25,12 @@ stages:
- template: templates/py-packaging-training-cuda-stage.yml
parameters:
build_py_parameters: --enable_training --update --build
torch_version: '1.13.1'
torch_version: '2.0.0'
opset_version: '15'
cuda_version: '11.6'
cuda_version: '11.8'
gcc_version: 11
cmake_cuda_architectures: 50;52;60;61;70;75;80;86;87
docker_file: Dockerfile.manylinux2014_training_cuda11_6
docker_file: Dockerfile.manylinux2014_training_cuda11_8
agent_pool: Onnxruntime-Linux-GPU
upload_wheel: 'no'
debug_build: true

7
tools/ci_build/github/azure-pipelines/templates/py-packaging-training-cuda-stage.yml
View file

@ -100,6 +100,13 @@ stages:
CudaVersion: ${{ parameters.cuda_version }}
GccVersion: ${{ parameters.gcc_version }}
UploadWheel: ${{ parameters.upload_wheel }}
Python311:
PythonVersion: '3.11'
TorchVersion: ${{ parameters.torch_version }}
OpsetVersion: ${{ parameters.opset_version }}
CudaVersion: ${{ parameters.cuda_version }}
GccVersion: ${{ parameters.gcc_version }}
UploadWheel: ${{ parameters.upload_wheel }}
steps:
- task: CmdLine@2

6
tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_training_cuda11_6 → tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_training_cuda11_8
View file

@ -1,4 +1,4 @@
ARG BASEIMAGE=nvidia/cuda:11.6.2-cudnn8-devel-centos7
ARG BASEIMAGE=nvidia/cuda:11.8.0-cudnn8-devel-centos7
ARG POLICY=manylinux2014
ARG PLATFORM=x86_64
ARG DEVTOOLSET_ROOTPATH=
@ -157,7 +157,7 @@ CMD ["/bin/bash"]
#Build manylinux2014 docker image end
ARG PYTHON_VERSION=3.9
ARG TORCH_VERSION=1.13.1
ARG TORCH_VERSION=2.0.0
ARG OPSET_VERSION=15
ARG INSTALL_DEPS_EXTRA_ARGS
@ -171,7 +171,7 @@ RUN cd /tmp/scripts && \
ENV PATH="/root/.cargo/bin/:$PATH"
RUN /tmp/scripts/install_ninja.sh && \
/tmp/scripts/install_python_deps.sh -d gpu -v 11.6 -p $PYTHON_VERSION -h $TORCH_VERSION $INSTALL_DEPS_EXTRA_ARGS && \
/tmp/scripts/install_python_deps.sh -d gpu -v 11.8 -p $PYTHON_VERSION -h $TORCH_VERSION $INSTALL_DEPS_EXTRA_ARGS && \
rm -rf /tmp/scripts
ARG BUILD_UID=1001

4
tools/ci_build/github/linux/docker/Dockerfile.ubuntu_gpu_training
View file

@ -1,8 +1,8 @@
ARG BASEIMAGE=nvcr.io/nvidia/cuda:11.6.2-cudnn8-devel-ubuntu18.04
ARG BASEIMAGE=nvcr.io/nvidia/cuda:11.8.0-cudnn8-devel-ubuntu18.04
FROM $BASEIMAGE
ARG PYTHON_VERSION=3.6
ARG PYTHON_VERSION=3.9
ARG INSTALL_DEPS_EXTRA_ARGS
ARG USE_CONDA=false

4
tools/ci_build/github/linux/docker/scripts/install_python_deps.sh
View file

@ -5,8 +5,8 @@ INSTALL_DEPS_TRAINING=false
INSTALL_DEPS_DISTRIBUTED_SETUP=false
ORTMODULE_BUILD=false
TARGET_ROCM=false
CU_VER="11.6"
TORCH_VERSION='1.13.1'
CU_VER="11.8"
TORCH_VERSION='2.0.0'
USE_CONDA=false
while getopts p:h:d:v:tmurc parameter_Option

2
tools/ci_build/github/linux/docker/scripts/manylinux/install_deps_aten.sh
View file

@ -43,7 +43,7 @@ do
if ! [[ ${PYTHON_EXE} = "/opt/python/cp310-cp310/bin/python3.10" ]]; then
${PYTHON_EXE} -m pip install -r ${0/%install_deps_aten\.sh/..\/training\/ortmodule\/stage1\/requirements_torch_cpu\/requirements.txt}
else
${PYTHON_EXE} -m pip install torch==1.13.1+cpu -f https://download.pytorch.org/whl/torch_stable.html
${PYTHON_EXE} -m pip install torch==2.0.0+cpu -f https://download.pytorch.org/whl/torch_stable.html
fi
done

6
tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements_torch1.13.1_cu11.6/requirements.txt → tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements_torch2.0.0_cu11.8/requirements.txt
View file

@ -1,8 +1,8 @@
--pre
-f https://download.pytorch.org/whl/torch_stable.html
torch==1.13.1+cu116
torchvision==0.14.1+cu116
torchtext==0.14.1
torch==2.0.0+cu118
torchvision==0.15.1+cu118
torchtext==0.15.1
# TODO(bmeswani): packaging 22.0 removes support for LegacyVersion leading to errors because transformers 4.4.2 uses LegacyVersion
packaging==21.3
setuptools>=41.4.0

2
tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements_torch_cpu/requirements.txt
View file

@ -1,3 +1,3 @@
-f https://download.pytorch.org/whl/torch_stable.html
torch==1.13.1+cpu
torch==2.0.0+cpu
setuptools>=41.4.0

2
tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage2/requirements.txt
View file

@ -8,6 +8,6 @@ tensorboard>=2.2.0,<2.5.0
h5py
wget
pytorch-lightning
deepspeed==0.3.15
deepspeed==0.9.0
fairscale==0.4.6
parameterized>=0.8.1

4
tools/ci_build/github/linux/run_dockerbuild.sh
View file

@ -91,9 +91,9 @@ elif [ $BUILD_DEVICE = "gpu" ]; then
if [[ $ORTMODULE_BUILD = true ]]; then
INSTALL_DEPS_EXTRA_ARGS="${INSTALL_DEPS_EXTRA_ARGS} -u"
fi
INSTALL_DEPS_EXTRA_ARGS="${INSTALL_DEPS_EXTRA_ARGS} -v 11.6"
INSTALL_DEPS_EXTRA_ARGS="${INSTALL_DEPS_EXTRA_ARGS} -v 11.8"
$GET_DOCKER_IMAGE_CMD --repository "onnxruntime-$IMAGE" \
--docker-build-args="--build-arg BASEIMAGE=nvcr.io/nvidia/cuda:11.6.2-cudnn8-devel-${BUILD_OS} --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} --build-arg INSTALL_DEPS_EXTRA_ARGS=\"${INSTALL_DEPS_EXTRA_ARGS}\" --build-arg USE_CONDA=${USE_CONDA} --network=host" \
--docker-build-args="--build-arg BASEIMAGE=nvcr.io/nvidia/cuda:11.8.0-cudnn8-devel-${BUILD_OS} --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} --build-arg INSTALL_DEPS_EXTRA_ARGS=\"${INSTALL_DEPS_EXTRA_ARGS}\" --build-arg USE_CONDA=${USE_CONDA} --network=host" \
--dockerfile Dockerfile.ubuntu_gpu_training --context .
elif [[ $BUILD_DEVICE = "tensorrt"* ]]; then
IMAGE="$BUILD_OS-cuda11.8-cudnn8.7-tensorrt8.6"