saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-29 20:14:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fix training e2e pipeline (#7942)

Browse source 
1. Fix training e2e pipeline. The failure was caused by my recent change #7632. The fix is adding "--cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=70" to the build parameters because the machines are with V100 GPUs.
2. Simplify Nuphar pipeline. It doesn't need to install a separated ONNX version(1.5.0)
3. Fix a problem that run_dockerbuild.sh ignored OS version parameter. Now because it starts to take effect, I also set python version to the system default one(3.8 for ubuntu 20.04)
This commit is contained in:
Changming Sun 2021-06-04 09:37:09 -07:00 committed by GitHub
parent 2b3f953701
commit 5a7f65b831
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 44 additions and 43 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
tools/ci_build/build.py
View file

@ -1551,11 +1551,6 @@ def nuphar_run_python_tests(build_dir, configs):
if is_windows():
cwd = os.path.join(cwd, config)
dll_path = os.path.join(build_dir, config, "external", "tvm", config)
# install onnx for shape inference in testing Nuphar scripts
# this needs to happen after onnx_test_data preparation which
# uses onnx 1.3.0
run_subprocess(
[sys.executable, '-m', 'pip', 'install', '--user', 'onnx==1.5.0'])
run_subprocess(
[sys.executable, 'onnxruntime_test_python_nuphar.py'],
cwd=cwd, dll_path=dll_path)

2
tools/ci_build/github/azure-pipelines/linux-gpu-tensorrt-ci-pipeline.yml
View file

@ -39,7 +39,7 @@ jobs:
--enable_onnx_tests --use_cuda --cuda_version=11.1 --cuda_home=/usr/local/cuda-11.1 --cudnn_home=/usr/local/cuda-11.1 \
--enable_pybind --build_java --build_nodejs \
--use_tensorrt --tensorrt_home /usr \
--cmake_extra_defines CMAKE_CUDA_HOST_COMPILER=/opt/rh/devtoolset-9/root/usr/bin/cc PYTHON_INCLUDE_DIR=/opt/python/cp37-cp37m/include/python3.7m PYTHON_LIBRARY=/usr/lib64/librt.so CMAKE_CUDA_ARCHITECTURES=52
--cmake_extra_defines CMAKE_CUDA_HOST_COMPILER=/opt/rh/devtoolset-9/root/usr/bin/cc CMAKE_CUDA_ARCHITECTURES=52
workingDirectory: $(Build.SourcesDirectory)
- task: PublishTestResults@2
displayName: 'Publish unit test results'

2
tools/ci_build/github/azure-pipelines/linux-multi-gpu-tensorrt-ci-pipeline.yml
View file

@ -4,6 +4,6 @@ jobs:
AgentPool : 'Linux-Multi-GPU'
JobName: 'Linux_CI_Multi_GPU_TensorRT_Dev'
# The latest TensorRT container only supports ubuntu20.04 and python 3.8
RunDockerBuildArgs: '-o ubuntu20.04 -d tensorrt -r $(Build.BinariesDirectory) -p 3.8 -x "--enable_multi_device_test"'
RunDockerBuildArgs: '-o ubuntu20.04 -p 3.8 -d tensorrt -r $(Build.BinariesDirectory) -p 3.8 -x "--enable_multi_device_test"'
DoNugetPack: 'false'
ArtifactName: 'drop-linux'

2
tools/ci_build/github/azure-pipelines/linux-nuphar-ci-pipeline.yml
View file

@ -3,7 +3,7 @@ jobs:
parameters:
AgentPool : 'Linux-CPU-2019'
JobName: 'Linux_CI_Dev'
RunDockerBuildArgs: '-o ubuntu20.04 -d cpu -r $(Build.BinariesDirectory) -x "--enable_pybind --use_nuphar"'
RunDockerBuildArgs: '-o ubuntu20.04 -p 3.8 -d cpu -r $(Build.BinariesDirectory) -x "--enable_pybind --use_nuphar"'
DoNugetPack: 'false'
ArtifactName: 'drop-linux'
TimeoutInMinutes: 180

2
tools/ci_build/github/azure-pipelines/linux-openvino-ci-pipeline.yml
View file

@ -3,7 +3,7 @@ jobs:
parameters:
AgentPool : 'Linux-CPU-2019'
JobName: 'Linux_CI_Dev'
RunDockerBuildArgs: '-o ubuntu20.04 -d openvino -v 2021.3 -r $(Build.BinariesDirectory) -x "--use_openvino CPU_FP32 --build_wheel"'
RunDockerBuildArgs: '-o ubuntu20.04 -p 3.8 -d openvino -v 2021.3 -r $(Build.BinariesDirectory) -x "--use_openvino CPU_FP32 --build_wheel"'
DoNugetPack: 'false'
ArtifactName: 'drop-linux'
TimeoutInMinutes: 120

2
tools/ci_build/github/azure-pipelines/linux-openvino-nightly-pipeline.yml
View file

@ -23,7 +23,7 @@ jobs:
- template: templates/run-docker-build-steps.yml
parameters:
RunDockerBuildArgs: '-o ubuntu20.04 -d openvino -v 2020.2 -r $(Build.BinariesDirectory) -x "--use_openvino GPU_FP32 --build_wheel"'
RunDockerBuildArgs: '-o ubuntu20.04 -p 3.8 -d openvino -v 2020.2 -r $(Build.BinariesDirectory) -x "--use_openvino GPU_FP32 --build_wheel"'
- template: templates/component-governance-component-detection-steps.yml
parameters :

2
tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ci-pipeline.yml
View file

@ -7,7 +7,7 @@ jobs:
JobName: 'Onnxruntime_Linux_GPU_Training'
SubmoduleCheckoutMode: 'recursive'
RunDockerBuildArgs: >
-o ubuntu20.04 -d gpu -r $(Build.BinariesDirectory)
-o ubuntu20.04 -p 3.8 -d gpu -r $(Build.BinariesDirectory)
-x "
--enable_training
--config $(buildConfig)

4
tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-distributed-e2e-test-pipeline.yml
View file

@ -14,13 +14,13 @@ jobs:
- template: templates/run-docker-build-steps.yml
parameters:
RunDockerBuildArgs: |
-o ubuntu20.04 -d gpu -r $(Build.BinariesDirectory) \
-o ubuntu20.04 -p 3.8 -d gpu -r $(Build.BinariesDirectory) \
-t onnxruntime_e2e_test_image \
-x " \
--config RelWithDebInfo \
--enable_training \
--update --build \
--build_wheel \
--build_wheel --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=70 \
" \
-m
DisplayName: 'Build'

2
tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-distributed-test-ci-pipeline.yml
View file

@ -14,7 +14,7 @@ jobs:
- template: templates/run-docker-build-steps.yml
parameters:
RunDockerBuildArgs: |
-o ubuntu20.04 -d gpu -r $(Build.BinariesDirectory) \
-o ubuntu20.04 -p 3.8 -d gpu -r $(Build.BinariesDirectory) \
-t onnxruntime_distributed_tests_image \
-x " \
--config RelWithDebInfo \

2
tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ortmodule-distributed-test-ci-pipeline.yml
View file

@ -14,7 +14,7 @@ jobs:
- template: templates/run-docker-build-steps.yml
parameters:
RunDockerBuildArgs: |
-o ubuntu20.04 -d gpu -r $(Build.BinariesDirectory) \
-o ubuntu20.04 -p 3.8 -d gpu -r $(Build.BinariesDirectory) \
-t onnxruntime_ortmodule_distributed_tests_image \
-x " \
--config RelWithDebInfo \

2
tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-ortmodule-test-ci-pipeline.yml
View file

@ -14,7 +14,7 @@ jobs:
- template: templates/run-docker-build-steps.yml
parameters:
RunDockerBuildArgs: |
-o ubuntu20.04 -d gpu -r $(Build.BinariesDirectory) \
-o ubuntu20.04 -p 3.8 -p 3.8 -d gpu -r $(Build.BinariesDirectory) \
-t onnxruntime_ortmodule_tests_image \
-x " \
--config RelWithDebInfo \

4
tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-perf-test-ci-pipeline.yml
View file

@ -16,12 +16,12 @@ jobs:
- template: templates/run-docker-build-steps.yml
parameters:
RunDockerBuildArgs: >
-o ubuntu20.04 -d gpu -r $(Build.BinariesDirectory)
-o ubuntu20.04 -p 3.8 -d gpu -r $(Build.BinariesDirectory)
-t onnxruntime_perf_test_image
-x "
--config RelWithDebInfo
--enable_training
--update --build
--update --build --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=70
"
DisplayName: 'Build performance tests'

2
tools/ci_build/github/azure-pipelines/templates/linux-ci.yml
View file

@ -2,7 +2,7 @@ parameters:
AgentPool : 'Linux-CPU-2019'
JobName : 'Linux_CI_Dev'
SubmoduleCheckoutMode: ''
RunDockerBuildArgs: '-o ubuntu20.04 -d cpu -r $(Build.BinariesDirectory) -x "--build_wheel"'
RunDockerBuildArgs: '-o ubuntu20.04 -p 3.8 -d cpu -r $(Build.BinariesDirectory) -x "--build_wheel"'
DoNodejsPack: 'false'
DoNugetPack: 'false'
NuPackScript: ''

4
tools/ci_build/github/linux/docker/Dockerfile.ubuntu_gpu_training
View file

@ -1,4 +1,6 @@
FROM nvidia/cuda:11.1.1-cudnn8-devel-ubuntu18.04
ARG BASEIMAGE=nvcr.io/nvidia/cuda:11.1.1-cudnn8-devel-ubuntu18.04
FROM $BASEIMAGE
ARG PYTHON_VERSION=3.6
ARG INSTALL_DEPS_EXTRA_ARGS

17
tools/ci_build/github/linux/docker/scripts/install_ubuntu.sh
View file

@ -12,11 +12,12 @@ PYTHON_VER=${PYTHON_VER:=3.5}
# Some Edge devices only have limited disk space, use this option to exclude some package
DEVICE_TYPE=${DEVICE_TYPE:=Normal}
DEBIAN_FRONTEND=noninteractive
echo 'debconf debconf/frontend select Noninteractive' | debconf-set-selections
apt-get update && apt-get install -y software-properties-common lsb-release
OS_VERSION=$(lsb_release -r -s)
DEBIAN_FRONTEND=noninteractive
SYS_LONG_BIT=$(getconf LONG_BIT)
@ -144,11 +145,11 @@ rm -rf /var/lib/apt/lists/*
if [ "$SYS_LONG_BIT" = "64" ]; then
if [ "$DEVICE_TYPE" = "Normal" ]; then
if [ "$OS_VERSION" = "20.04" ]; then
aria2c -q -d /tmp -o llvm.tar.xz https://github.com/llvm/llvm-project/releases/download/llvmorg-11.0.0/llvm-project-11.0.0.tar.xz
else
aria2c -q -d /tmp -o llvm.tar.xz http://releases.llvm.org/9.0.0/clang+llvm-9.0.0-x86_64-linux-gnu-ubuntu-${OS_VERSION}.tar.xz
fi
tar --strip 1 -Jxf /tmp/llvm.tar.xz -C /usr
if [ "$OS_VERSION" = "20.04" ]; then
#llvm 9.0 doesn't have a release for 20.04, but the binaries for 18.04 should work well.
OS_VERSION="18.04"
fi
aria2c -q -d /tmp -o llvm.tar.xz http://releases.llvm.org/9.0.0/clang+llvm-9.0.0-x86_64-linux-gnu-ubuntu-${OS_VERSION}.tar.xz
tar --strip 1 -Jxf /tmp/llvm.tar.xz -C /usr
fi
fi

33
tools/ci_build/github/linux/run_dockerbuild.sh
View file

@ -59,12 +59,6 @@ if [ $BUILD_OS = "android" ]; then
$GET_DOCKER_IMAGE_CMD --repository "onnxruntime-$IMAGE" \
--docker-build-args="--build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER}" \
--dockerfile $DOCKER_FILE --context .
elif [ $BUILD_OS = "centos7" ]; then
IMAGE="centos7"
DOCKER_FILE=Dockerfile.centos
$GET_DOCKER_IMAGE_CMD --repository "onnxruntime-$IMAGE" \
--docker-build-args="--build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER}" \
--dockerfile $DOCKER_FILE --context .
elif [ $BUILD_OS = "yocto" ]; then
IMAGE="arm-yocto-$YOCTO_VERSION"
DOCKER_FILE=Dockerfile.ubuntu_for_arm
@ -76,10 +70,9 @@ elif [ $BUILD_OS = "yocto" ]; then
$GET_DOCKER_IMAGE_CMD --repository "onnxruntime-$IMAGE" \
--docker-build-args="--build-arg TOOL_CHAIN=$TOOL_CHAIN_SCRIPT --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER}" \
--dockerfile $DOCKER_FILE --context .
else
if [ $BUILD_DEVICE = "gpu" ]; then
elif [ $BUILD_DEVICE = "gpu" ]; then
#This code path is only for training. Inferecing pipeline uses CentOS
IMAGE="ubuntu_gpu_training"
IMAGE="$BUILD_OS-gpu_training"
INSTALL_DEPS_EXTRA_ARGS="${INSTALL_DEPS_EXTRA_ARGS} -t"
if [[ $INSTALL_DEPS_DISTRIBUTED_SETUP = true ]]; then
INSTALL_DEPS_EXTRA_ARGS="${INSTALL_DEPS_EXTRA_ARGS} -m"
@ -88,27 +81,37 @@ else
INSTALL_DEPS_EXTRA_ARGS="${INSTALL_DEPS_EXTRA_ARGS} -u"
fi
$GET_DOCKER_IMAGE_CMD --repository "onnxruntime-$IMAGE" \
--docker-build-args="--build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} --build-arg INSTALL_DEPS_EXTRA_ARGS=\"${INSTALL_DEPS_EXTRA_ARGS}\" --build-arg USE_CONDA=${USE_CONDA} --network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64 --build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-9/root --build-arg PREPEND_PATH=/opt/rh/devtoolset-9/root/usr/bin: --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-9/root/usr/lib64:/opt/rh/devtoolset-9/root/usr/lib:/opt/rh/devtoolset-9/root/usr/lib64/dyninst:/opt/rh/devtoolset-9/root/usr/lib/dyninst:/usr/local/lib64" \
--docker-build-args="--build-arg BASEIMAGE=nvcr.io/nvidia/cuda:11.1.1-cudnn8-devel-${BUILD_OS} --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} --build-arg INSTALL_DEPS_EXTRA_ARGS=\"${INSTALL_DEPS_EXTRA_ARGS}\" --build-arg USE_CONDA=${USE_CONDA} --network=host" \
--dockerfile Dockerfile.ubuntu_gpu_training --context .
elif [ $BUILD_DEVICE = "tensorrt" ]; then
elif [ $BUILD_DEVICE = "tensorrt" ]; then
# TensorRT container release 20.12
IMAGE="$BUILD_OS-cuda11.1-cudnn8.0-tensorrt7.2"
DOCKER_FILE=Dockerfile.ubuntu_tensorrt
$GET_DOCKER_IMAGE_CMD --repository "onnxruntime-$IMAGE" \
--docker-build-args="--build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} --network=host --build-arg POLICY=manylinux2014 --build-arg PLATFORM=x86_64 --build-arg DEVTOOLSET_ROOTPATH=/opt/rh/devtoolset-9/root --build-arg PREPEND_PATH=/opt/rh/devtoolset-9/root/usr/bin: --build-arg LD_LIBRARY_PATH_ARG=/opt/rh/devtoolset-9/root/usr/lib64:/opt/rh/devtoolset-9/root/usr/lib:/opt/rh/devtoolset-9/root/usr/lib64/dyninst:/opt/rh/devtoolset-9/root/usr/lib/dyninst:/usr/local/lib64" \
--dockerfile $DOCKER_FILE --context .
elif [ $BUILD_DEVICE = "openvino" ]; then
elif [ $BUILD_DEVICE = "openvino" ]; then
IMAGE="$BUILD_OS-openvino"
DOCKER_FILE=Dockerfile.ubuntu_openvino
$GET_DOCKER_IMAGE_CMD --repository "onnxruntime-$IMAGE" \
--docker-build-args="--build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} --build-arg OPENVINO_VERSION=${OPENVINO_VERSION}" \
--dockerfile $DOCKER_FILE --context .
else
else
IMAGE="$BUILD_OS"
IMAGE_OS_VERSION=""
if [ $BUILD_OS = "ubuntu18.04" ]; then
IMAGE_OS_VERSION="18.04"
PYTHON_VER="3.6"
elif [ $BUILD_OS = "ubuntu20.04" ]; then
IMAGE_OS_VERSION="20.04"
PYTHON_VER="3.8"
else
exit 1
fi
$GET_DOCKER_IMAGE_CMD --repository "onnxruntime-$IMAGE" \
--docker-build-args="--build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER}" \
--docker-build-args="--build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} --build-arg OS_VERSION=${IMAGE_OS_VERSION}" \
--dockerfile Dockerfile.ubuntu --context .
fi
fi
if [ -v EXTRA_IMAGE_TAG ]; then