mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-29 20:14:01 +00:00
Change training perf test build to use "docker" instead of "sudo docker" (#3995)
Change training perf test build to use "docker" instead of "sudo docker". The training perf test build runs in an environment that supports calling "docker" and not "sudo docker".
This commit is contained in:
edgchen1
GitHub
parent
354e571277
commit
989fe2498f
2 changed files with 36 additions and 29 deletions
|
|
@ -5,7 +5,7 @@ jobs:
|
|||
|
||||
timeoutInMinutes: 120
|
||||
|
||||
variables:
|
||||
variables:
|
||||
- group: 'ortperf' # variable group
|
||||
|
||||
steps:
|
||||
|
|
@ -24,36 +24,36 @@ jobs:
|
|||
displayName: 'Build performance tests'
|
||||
|
||||
- script: >
|
||||
sudo docker run --gpus all --rm --name onnxruntime-gpu-perf
|
||||
--volume $(Build.SourcesDirectory):/onnxruntime_src
|
||||
--volume $(Build.BinariesDirectory):/build
|
||||
--volume /bert_ort/bert_models:/build/bert_models:ro
|
||||
--volume /bert_data:/build/bert_data:ro
|
||||
docker run --gpus all --rm --name onnxruntime-gpu-perf
|
||||
--volume $(Build.SourcesDirectory):/onnxruntime_src
|
||||
--volume $(Build.BinariesDirectory):/build
|
||||
--volume /bert_ort/bert_models:/build/bert_models:ro
|
||||
--volume /bert_data:/build/bert_data:ro
|
||||
-e NIGHTLY_BUILD onnxruntime-ubuntu16.04-cuda10.1-cudnn7.6
|
||||
/usr/bin/python3.6 /onnxruntime_src/orttraining/tools/ci_test/run_bert_perf_test.py
|
||||
--binary_dir /build/RelWithDebInfo
|
||||
--training_data_root /build/bert_data
|
||||
--binary_dir /build/RelWithDebInfo
|
||||
--training_data_root /build/bert_data
|
||||
--model_root /build/bert_models
|
||||
displayName: 'Run bert performance tests'
|
||||
|
||||
- script: >
|
||||
docker run --gpus all --rm --name onnxruntime-gpu-perf
|
||||
--volume $(Build.SourcesDirectory):/onnxruntime_src
|
||||
--volume $(Build.BinariesDirectory):/build
|
||||
--volume /bert_ort/gpt2_models:/build/gpt2_models:ro
|
||||
--volume /bert_data/gpt2_data:/build/gpt2_data:ro
|
||||
docker run --gpus all --rm --name onnxruntime-gpu-perf
|
||||
--volume $(Build.SourcesDirectory):/onnxruntime_src
|
||||
--volume $(Build.BinariesDirectory):/build
|
||||
--volume /bert_ort/gpt2_models:/build/gpt2_models:ro
|
||||
--volume /bert_data/gpt2_data:/build/gpt2_data:ro
|
||||
-e NIGHTLY_BUILD onnxruntime-ubuntu16.04-cuda10.1-cudnn7.6
|
||||
/usr/bin/python3.6 /onnxruntime_src/orttraining/tools/ci_test/run_gpt2_perf_test.py
|
||||
--binary_dir /build/RelWithDebInfo
|
||||
--training_data_root /build/gpt2_data
|
||||
--binary_dir /build/RelWithDebInfo
|
||||
--training_data_root /build/gpt2_data
|
||||
--model_root /build/gpt2_models
|
||||
displayName: 'Run gpt-2 performance tests'
|
||||
|
||||
# generate jdbc.properties
|
||||
- script: >
|
||||
mkdir -p $(Build.SourcesDirectory)/tools/perf_util/src/main/resources &&
|
||||
printf "url=jdbc:mysql://onnxruntimedashboard.mysql.database.azure.com/onnxruntime?serverTimezone=UTC&useUnicode=true&characterEncoding=UTF-8\nuser=powerbi@onnxruntimedashboard\npassword_env=ORT_PERF_PASSWORD"
|
||||
> $(Build.SourcesDirectory)/tools/perf_util/src/main/resources/jdbc.properties
|
||||
printf "url=jdbc:mysql://onnxruntimedashboard.mysql.database.azure.com/onnxruntime?serverTimezone=UTC&useUnicode=true&characterEncoding=UTF-8\nuser=powerbi@onnxruntimedashboard\npassword_env=ORT_PERF_PASSWORD"
|
||||
> $(Build.SourcesDirectory)/tools/perf_util/src/main/resources/jdbc.properties
|
||||
displayName: 'Create resource file'
|
||||
|
||||
- script: >
|
||||
|
|
@ -65,7 +65,7 @@ jobs:
|
|||
- script: >
|
||||
java -cp target/send_perf_metrics-0.0.1-SNAPSHOT-jar-with-dependencies.jar com.msft.send_perf_metrics.App "$(Build.SourcesDirectory)/orttraining/tools/ci_test/results"
|
||||
env:
|
||||
ORT_PERF_PASSWORD: $(ortperf)
|
||||
ORT_PERF_PASSWORD: $(ortperf)
|
||||
displayName: 'Populate perf metrics'
|
||||
workingDirectory: $(Build.SourcesDirectory)/tools/perf_util
|
||||
|
||||
|
|
|
|||
|
|
@ -33,11 +33,18 @@ EXIT_CODE=1
|
|||
PYTHON_VER=${PYTHON_VER:=3.6}
|
||||
echo "bo=$BUILD_OS bd=$BUILD_DEVICE bdir=$BUILD_DIR pv=$PYTHON_VER bex=$BUILD_EXTR_PAR"
|
||||
|
||||
# If in docker group, call "docker". Otherwise, call "sudo docker".
|
||||
if id -Gnz | grep -zq "^docker$" ; then
|
||||
DOCKER_CMD=docker
|
||||
else
|
||||
DOCKER_CMD="sudo docker"
|
||||
fi
|
||||
|
||||
cd $SCRIPT_DIR/docker
|
||||
if [ $BUILD_OS = "android" ]; then
|
||||
IMAGE="android"
|
||||
DOCKER_FILE=Dockerfile.ubuntu_for_android
|
||||
sudo docker build --pull -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f $DOCKER_FILE .
|
||||
$DOCKER_CMD build --pull -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f $DOCKER_FILE .
|
||||
elif [ $BUILD_OS = "manylinux2010" ]; then
|
||||
if [ $BUILD_DEVICE = "gpu" ]; then
|
||||
IMAGE="manylinux2010-cuda10.1"
|
||||
|
|
@ -46,11 +53,11 @@ elif [ $BUILD_OS = "manylinux2010" ]; then
|
|||
IMAGE="manylinux2010"
|
||||
DOCKER_FILE=Dockerfile.manylinux2010
|
||||
fi
|
||||
sudo docker build --pull -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f $DOCKER_FILE .
|
||||
$DOCKER_CMD build --pull -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f $DOCKER_FILE .
|
||||
elif [ $BUILD_OS = "centos7" ]; then
|
||||
IMAGE="centos7"
|
||||
DOCKER_FILE=Dockerfile.centos
|
||||
sudo docker build --pull -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f $DOCKER_FILE .
|
||||
$DOCKER_CMD build --pull -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f $DOCKER_FILE .
|
||||
elif [ $BUILD_OS = "yocto" ]; then
|
||||
IMAGE="arm-yocto-$YOCTO_VERSION"
|
||||
DOCKER_FILE=Dockerfile.ubuntu_for_arm
|
||||
|
|
@ -59,7 +66,7 @@ elif [ $BUILD_OS = "yocto" ]; then
|
|||
if [ $YOCTO_VERSION = "4.14" ]; then
|
||||
TOOL_CHAIN_SCRIPT=fsl-imx-xwayland-glibc-x86_64-fsl-image-qt5-aarch64-toolchain-4.14-sumo.sh
|
||||
fi
|
||||
sudo docker build -t "onnxruntime-$IMAGE" --build-arg TOOL_CHAIN=$TOOL_CHAIN_SCRIPT --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f $DOCKER_FILE .
|
||||
$DOCKER_CMD build -t "onnxruntime-$IMAGE" --build-arg TOOL_CHAIN=$TOOL_CHAIN_SCRIPT --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f $DOCKER_FILE .
|
||||
else
|
||||
if [ $BUILD_DEVICE = "gpu" ]; then
|
||||
IMAGE="$BUILD_OS-$CUDA_VER"
|
||||
|
|
@ -67,23 +74,23 @@ else
|
|||
if [ $CUDA_VER = "cuda9.1-cudnn7.1" ]; then
|
||||
DOCKER_FILE=Dockerfile.ubuntu_gpu_cuda9
|
||||
fi
|
||||
sudo docker build --pull -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} --build-arg BUILD_EXTR_PAR="${BUILD_EXTR_PAR}" -f $DOCKER_FILE .
|
||||
$DOCKER_CMD build --pull -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} --build-arg BUILD_EXTR_PAR="${BUILD_EXTR_PAR}" -f $DOCKER_FILE .
|
||||
elif [ $BUILD_DEVICE = "tensorrt" ]; then
|
||||
# TensorRT container release 20.01
|
||||
IMAGE="$BUILD_OS-cuda10.2-cudnn7.6-tensorrt7.0"
|
||||
DOCKER_FILE=Dockerfile.ubuntu_tensorrt
|
||||
sudo docker build --pull -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f $DOCKER_FILE .
|
||||
$DOCKER_CMD build --pull -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f $DOCKER_FILE .
|
||||
elif [ $BUILD_DEVICE = "openvino" ]; then
|
||||
IMAGE="$BUILD_OS-openvino"
|
||||
DOCKER_FILE=Dockerfile.ubuntu_openvino
|
||||
sudo docker build --pull -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} --build-arg OPENVINO_VERSION=${OPENVINO_VERSION} -f $DOCKER_FILE .
|
||||
$DOCKER_CMD build --pull -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} --build-arg OPENVINO_VERSION=${OPENVINO_VERSION} -f $DOCKER_FILE .
|
||||
else
|
||||
IMAGE="$BUILD_OS"
|
||||
if [ $BUILD_ARCH = "x86" ]; then
|
||||
IMAGE="$IMAGE.x86"
|
||||
sudo docker build --pull -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f Dockerfile.ubuntu_x86 .
|
||||
$DOCKER_CMD build --pull -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f Dockerfile.ubuntu_x86 .
|
||||
else
|
||||
sudo docker build --pull -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f Dockerfile.ubuntu .
|
||||
$DOCKER_CMD build --pull -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f Dockerfile.ubuntu .
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
|
@ -112,8 +119,8 @@ if [ $BUILD_DEVICE = "openvino" ] && [[ $BUILD_EXTR_PAR == *"--use_openvino GPU_
|
|||
DOCKER_RUN_PARAMETER="$DOCKER_RUN_PARAMETER --device /dev/dri:/dev/dri"
|
||||
fi
|
||||
|
||||
sudo docker rm -f "onnxruntime-$BUILD_DEVICE" || true
|
||||
sudo docker run $RUNTIME -h $HOSTNAME $DOCKER_RUN_PARAMETER \
|
||||
$DOCKER_CMD rm -f "onnxruntime-$BUILD_DEVICE" || true
|
||||
$DOCKER_CMD run $RUNTIME -h $HOSTNAME $DOCKER_RUN_PARAMETER \
|
||||
-e NIGHTLY_BUILD \
|
||||
"onnxruntime-$IMAGE" \
|
||||
/bin/bash /onnxruntime_src/tools/ci_build/github/linux/run_build.sh \
|
||||
|
|
|
|||
Loading…
Reference in a new issue