From 989fe2498f4dd0beaf1b4324bf74a39abaf86a29 Mon Sep 17 00:00:00 2001 From: edgchen1 <18449977+edgchen1@users.noreply.github.com> Date: Tue, 19 May 2020 16:54:35 -0700 Subject: [PATCH] Change training perf test build to use "docker" instead of "sudo docker" (#3995) Change training perf test build to use "docker" instead of "sudo docker". The training perf test build runs in an environment that supports calling "docker" and not "sudo docker". --- ...aining-linux-gpu-perf-test-ci-pipeline.yml | 36 +++++++++---------- .../ci_build/github/linux/run_dockerbuild.sh | 29 +++++++++------ 2 files changed, 36 insertions(+), 29 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-perf-test-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-perf-test-ci-pipeline.yml index 644ea20a1b..e17de8350f 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-perf-test-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-perf-test-ci-pipeline.yml @@ -5,7 +5,7 @@ jobs: timeoutInMinutes: 120 - variables: + variables: - group: 'ortperf' # variable group steps: @@ -24,36 +24,36 @@ jobs: displayName: 'Build performance tests' - script: > - sudo docker run --gpus all --rm --name onnxruntime-gpu-perf - --volume $(Build.SourcesDirectory):/onnxruntime_src - --volume $(Build.BinariesDirectory):/build - --volume /bert_ort/bert_models:/build/bert_models:ro - --volume /bert_data:/build/bert_data:ro + docker run --gpus all --rm --name onnxruntime-gpu-perf + --volume $(Build.SourcesDirectory):/onnxruntime_src + --volume $(Build.BinariesDirectory):/build + --volume /bert_ort/bert_models:/build/bert_models:ro + --volume /bert_data:/build/bert_data:ro -e NIGHTLY_BUILD onnxruntime-ubuntu16.04-cuda10.1-cudnn7.6 /usr/bin/python3.6 /onnxruntime_src/orttraining/tools/ci_test/run_bert_perf_test.py - --binary_dir /build/RelWithDebInfo - --training_data_root /build/bert_data + --binary_dir /build/RelWithDebInfo + --training_data_root /build/bert_data --model_root /build/bert_models displayName: 'Run bert performance tests' - script: > - docker run --gpus all --rm --name onnxruntime-gpu-perf - --volume $(Build.SourcesDirectory):/onnxruntime_src - --volume $(Build.BinariesDirectory):/build - --volume /bert_ort/gpt2_models:/build/gpt2_models:ro - --volume /bert_data/gpt2_data:/build/gpt2_data:ro + docker run --gpus all --rm --name onnxruntime-gpu-perf + --volume $(Build.SourcesDirectory):/onnxruntime_src + --volume $(Build.BinariesDirectory):/build + --volume /bert_ort/gpt2_models:/build/gpt2_models:ro + --volume /bert_data/gpt2_data:/build/gpt2_data:ro -e NIGHTLY_BUILD onnxruntime-ubuntu16.04-cuda10.1-cudnn7.6 /usr/bin/python3.6 /onnxruntime_src/orttraining/tools/ci_test/run_gpt2_perf_test.py - --binary_dir /build/RelWithDebInfo - --training_data_root /build/gpt2_data + --binary_dir /build/RelWithDebInfo + --training_data_root /build/gpt2_data --model_root /build/gpt2_models displayName: 'Run gpt-2 performance tests' # generate jdbc.properties - script: > mkdir -p $(Build.SourcesDirectory)/tools/perf_util/src/main/resources && - printf "url=jdbc:mysql://onnxruntimedashboard.mysql.database.azure.com/onnxruntime?serverTimezone=UTC&useUnicode=true&characterEncoding=UTF-8\nuser=powerbi@onnxruntimedashboard\npassword_env=ORT_PERF_PASSWORD" - > $(Build.SourcesDirectory)/tools/perf_util/src/main/resources/jdbc.properties + printf "url=jdbc:mysql://onnxruntimedashboard.mysql.database.azure.com/onnxruntime?serverTimezone=UTC&useUnicode=true&characterEncoding=UTF-8\nuser=powerbi@onnxruntimedashboard\npassword_env=ORT_PERF_PASSWORD" + > $(Build.SourcesDirectory)/tools/perf_util/src/main/resources/jdbc.properties displayName: 'Create resource file' - script: > @@ -65,7 +65,7 @@ jobs: - script: > java -cp target/send_perf_metrics-0.0.1-SNAPSHOT-jar-with-dependencies.jar com.msft.send_perf_metrics.App "$(Build.SourcesDirectory)/orttraining/tools/ci_test/results" env: - ORT_PERF_PASSWORD: $(ortperf) + ORT_PERF_PASSWORD: $(ortperf) displayName: 'Populate perf metrics' workingDirectory: $(Build.SourcesDirectory)/tools/perf_util diff --git a/tools/ci_build/github/linux/run_dockerbuild.sh b/tools/ci_build/github/linux/run_dockerbuild.sh index 3b3a5264b9..d72a266b63 100755 --- a/tools/ci_build/github/linux/run_dockerbuild.sh +++ b/tools/ci_build/github/linux/run_dockerbuild.sh @@ -33,11 +33,18 @@ EXIT_CODE=1 PYTHON_VER=${PYTHON_VER:=3.6} echo "bo=$BUILD_OS bd=$BUILD_DEVICE bdir=$BUILD_DIR pv=$PYTHON_VER bex=$BUILD_EXTR_PAR" +# If in docker group, call "docker". Otherwise, call "sudo docker". +if id -Gnz | grep -zq "^docker$" ; then + DOCKER_CMD=docker +else + DOCKER_CMD="sudo docker" +fi + cd $SCRIPT_DIR/docker if [ $BUILD_OS = "android" ]; then IMAGE="android" DOCKER_FILE=Dockerfile.ubuntu_for_android - sudo docker build --pull -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f $DOCKER_FILE . + $DOCKER_CMD build --pull -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f $DOCKER_FILE . elif [ $BUILD_OS = "manylinux2010" ]; then if [ $BUILD_DEVICE = "gpu" ]; then IMAGE="manylinux2010-cuda10.1" @@ -46,11 +53,11 @@ elif [ $BUILD_OS = "manylinux2010" ]; then IMAGE="manylinux2010" DOCKER_FILE=Dockerfile.manylinux2010 fi - sudo docker build --pull -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f $DOCKER_FILE . + $DOCKER_CMD build --pull -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f $DOCKER_FILE . elif [ $BUILD_OS = "centos7" ]; then IMAGE="centos7" DOCKER_FILE=Dockerfile.centos - sudo docker build --pull -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f $DOCKER_FILE . + $DOCKER_CMD build --pull -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f $DOCKER_FILE . elif [ $BUILD_OS = "yocto" ]; then IMAGE="arm-yocto-$YOCTO_VERSION" DOCKER_FILE=Dockerfile.ubuntu_for_arm @@ -59,7 +66,7 @@ elif [ $BUILD_OS = "yocto" ]; then if [ $YOCTO_VERSION = "4.14" ]; then TOOL_CHAIN_SCRIPT=fsl-imx-xwayland-glibc-x86_64-fsl-image-qt5-aarch64-toolchain-4.14-sumo.sh fi - sudo docker build -t "onnxruntime-$IMAGE" --build-arg TOOL_CHAIN=$TOOL_CHAIN_SCRIPT --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f $DOCKER_FILE . + $DOCKER_CMD build -t "onnxruntime-$IMAGE" --build-arg TOOL_CHAIN=$TOOL_CHAIN_SCRIPT --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f $DOCKER_FILE . else if [ $BUILD_DEVICE = "gpu" ]; then IMAGE="$BUILD_OS-$CUDA_VER" @@ -67,23 +74,23 @@ else if [ $CUDA_VER = "cuda9.1-cudnn7.1" ]; then DOCKER_FILE=Dockerfile.ubuntu_gpu_cuda9 fi - sudo docker build --pull -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} --build-arg BUILD_EXTR_PAR="${BUILD_EXTR_PAR}" -f $DOCKER_FILE . + $DOCKER_CMD build --pull -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} --build-arg BUILD_EXTR_PAR="${BUILD_EXTR_PAR}" -f $DOCKER_FILE . elif [ $BUILD_DEVICE = "tensorrt" ]; then # TensorRT container release 20.01 IMAGE="$BUILD_OS-cuda10.2-cudnn7.6-tensorrt7.0" DOCKER_FILE=Dockerfile.ubuntu_tensorrt - sudo docker build --pull -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f $DOCKER_FILE . + $DOCKER_CMD build --pull -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f $DOCKER_FILE . elif [ $BUILD_DEVICE = "openvino" ]; then IMAGE="$BUILD_OS-openvino" DOCKER_FILE=Dockerfile.ubuntu_openvino - sudo docker build --pull -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} --build-arg OPENVINO_VERSION=${OPENVINO_VERSION} -f $DOCKER_FILE . + $DOCKER_CMD build --pull -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} --build-arg OPENVINO_VERSION=${OPENVINO_VERSION} -f $DOCKER_FILE . else IMAGE="$BUILD_OS" if [ $BUILD_ARCH = "x86" ]; then IMAGE="$IMAGE.x86" - sudo docker build --pull -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f Dockerfile.ubuntu_x86 . + $DOCKER_CMD build --pull -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f Dockerfile.ubuntu_x86 . else - sudo docker build --pull -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f Dockerfile.ubuntu . + $DOCKER_CMD build --pull -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f Dockerfile.ubuntu . fi fi fi @@ -112,8 +119,8 @@ if [ $BUILD_DEVICE = "openvino" ] && [[ $BUILD_EXTR_PAR == *"--use_openvino GPU_ DOCKER_RUN_PARAMETER="$DOCKER_RUN_PARAMETER --device /dev/dri:/dev/dri" fi -sudo docker rm -f "onnxruntime-$BUILD_DEVICE" || true -sudo docker run $RUNTIME -h $HOSTNAME $DOCKER_RUN_PARAMETER \ +$DOCKER_CMD rm -f "onnxruntime-$BUILD_DEVICE" || true +$DOCKER_CMD run $RUNTIME -h $HOSTNAME $DOCKER_RUN_PARAMETER \ -e NIGHTLY_BUILD \ "onnxruntime-$IMAGE" \ /bin/bash /onnxruntime_src/tools/ci_build/github/linux/run_build.sh \