From d604281a8604be362d57ef1483550a13272d7ed7 Mon Sep 17 00:00:00 2001 From: liqunfu Date: Sun, 16 May 2021 09:10:57 -0700 Subject: [PATCH] Liqun/training pkg to run tests (#7662) --- .../templates/py-packaging-stage.yml | 42 ++++++++++++++-- ...Dockerfile.manylinux2014_training_cuda10_2 | 1 + ...Dockerfile.manylinux2014_training_cuda11_1 | 1 + .../linux/docker/scripts/install_ninja.sh | 49 +++++++++++++++++++ 4 files changed, 89 insertions(+), 4 deletions(-) create mode 100755 tools/ci_build/github/linux/docker/scripts/install_ninja.sh diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml index 6ea9f6821e..e6670efd08 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml @@ -531,7 +531,20 @@ stages: --build-arg BUILD_UID=$(id -u) Repository: onnxruntimetraininggpubuild + - bash: tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_test_data.sh -p $(orttrainingtestdata-storage-key) -s "//orttrainingtestdata.file.core.windows.net/mnist" -d "/mnist" + displayName: 'Mount MNIST' + condition: succeededOrFailed() + + - bash: tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_test_data.sh -p $(orttrainingtestdata-storage-key) -s "//orttrainingtestdata.file.core.windows.net/bert-data" -d "/bert_data" + displayName: 'Mount bert-data' + condition: succeededOrFailed() + + - bash: tools/ci_build/github/linux/docker/scripts/training/azure_scale_set_vm_mount_test_data.sh -p $(orttrainingtestdata-storage-key) -s "//orttrainingtestdata.file.core.windows.net/hf-models-cache" -d "/hf_models_cache" + displayName: 'Mount hf-models-cache' + condition: succeededOrFailed() + - task: CmdLine@2 + displayName: 'build onnxruntime' inputs: script: | mkdir -p $HOME/.onnx @@ -554,12 +567,33 @@ stages: --enable_onnx_tests \ ${{ parameters.build_py_parameters }} \ --cmake_extra_defines CMAKE_CUDA_HOST_COMPILER=/opt/rh/devtoolset-8/root/usr/bin/cc PYTHON_INCLUDE_DIR=$(PythonManylinuxIncludeDir) PYTHON_LIBRARY=/usr/lib64/librt.so \ - --use_cuda --cuda_version=$(CudaVersion) --cuda_home=/usr/local/cuda-$(CudaVersion) --cudnn_home=/usr/local/cuda-$(CudaVersion) + --use_cuda --cuda_version=$(CudaVersion) --cuda_home=/usr/local/cuda-$(CudaVersion) --cudnn_home=/usr/local/cuda-$(CudaVersion) ; + workingDirectory: $(Build.SourcesDirectory) + + # with Cuda 11.1: + # test_bert_inputs_with_dynamic_shape: RuntimeError: CUDA error: CUBLAS_STATUS_EXECUTION_FAILED when calling `cublasSgemm( handle, opa, opb, m, n, k, &alpha, a, lda, b, ldb, &beta, c, ldc)` + # test_gpu_reserved_memory_with_torch_no_grad: RuntimeError: CUDA error: CUBLAS_STATUS_EXECUTION_FAILED when calling `cublasSgemmStridedBatched( handle, opa, opb, m, n, k, &alpha, a, lda, stridea, b, ldb, strideb, &beta, c, ldc, stridec, num_batches)` + - task: CmdLine@2 + displayName: 'test ortmodule' + condition: ne(variables['CudaVersion'], '11.1') + inputs: + script: | + rm -rf $(Build.BinariesDirectory)/Release/onnxruntime/ && \ + files=($(Build.BinariesDirectory)/Release/dist/*.whl) && \ + echo ${files[0]} && \ + whlfilename=$(basename ${files[0]}) && \ + echo $whlfilename && \ + docker run --rm \ + --gpus all \ + -e NVIDIA_VISIBLE_DEVICES=all \ + --volume $(Build.BinariesDirectory):/build \ + --volume /mnist:/mnist \ + --volume /bert_data:/bert_data \ + --volume /hf_models_cache:/hf_models_cache \ + onnxruntimetraininggpubuild \ + bash -c " $(PythonManylinuxDir)/bin/python3 -m pip install /build/Release/dist/$whlfilename ; $(PythonManylinuxDir)/bin/python3 /build/Release/launch_test.py --cmd_line_with_args 'python orttraining_ortmodule_tests.py --mnist /mnist --bert_data /bert_data/hf_data/glue_data/CoLA/original/raw --transformers_cache /hf_models_cache/huggingface/transformers' --cwd /build/Release " ; workingDirectory: $(Build.SourcesDirectory) - # TODO: run ortmodule test. - # without run_dockerbuild (build.py instead), /build/RelWithDebInfo/ is not available after build.py step. need to find a way to do module test within packaging pipeline - - task: CopyFiles@2 displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)' inputs: diff --git a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_training_cuda10_2 b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_training_cuda10_2 index ed853d7ffb..d0c9cd9a49 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_training_cuda10_2 +++ b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_training_cuda10_2 @@ -33,6 +33,7 @@ ADD scripts /tmp/scripts RUN cd /tmp/scripts && \ /tmp/scripts/install_centos.sh && \ /tmp/scripts/install_os_deps.sh -d gpu $INSTALL_DEPS_EXTRA_ARGS && \ + /tmp/scripts/install_ninja.sh && \ /tmp/scripts/install_python_deps.sh -d gpu -v 10.2 -p $PYTHON_VERSION $INSTALL_DEPS_EXTRA_ARGS && \ rm -rf /tmp/scripts diff --git a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_training_cuda11_1 b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_training_cuda11_1 index 60e02b0e59..20efc8cdd0 100644 --- a/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_training_cuda11_1 +++ b/tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_training_cuda11_1 @@ -33,6 +33,7 @@ ADD scripts /tmp/scripts RUN cd /tmp/scripts && \ /tmp/scripts/install_centos.sh && \ /tmp/scripts/install_os_deps.sh -d gpu $INSTALL_DEPS_EXTRA_ARGS && \ + /tmp/scripts/install_ninja.sh && \ /tmp/scripts/install_python_deps.sh -d gpu -v 11.1 -p $PYTHON_VERSION $INSTALL_DEPS_EXTRA_ARGS && \ rm -rf /tmp/scripts diff --git a/tools/ci_build/github/linux/docker/scripts/install_ninja.sh b/tools/ci_build/github/linux/docker/scripts/install_ninja.sh new file mode 100755 index 0000000000..0e815e46b5 --- /dev/null +++ b/tools/ci_build/github/linux/docker/scripts/install_ninja.sh @@ -0,0 +1,49 @@ +#!/bin/bash +set -e +#Download a file from internet +function GetFile { + local uri=$1 + local path=$2 + local force=${3:-false} + local download_retries=${4:-5} + local retry_wait_time_seconds=${5:-30} + + if [[ -f $path ]]; then + if [[ $force = false ]]; then + echo "File '$path' already exists. Skipping download" + return 0 + else + rm -rf $path + fi + fi + + if [[ -f $uri ]]; then + echo "'$uri' is a file path, copying file to '$path'" + cp $uri $path + return $? + fi + + echo "Downloading $uri" + # Use aria2c if available, otherwise use curl + if command -v aria2c > /dev/null; then + aria2c -q -d $(dirname $path) -o $(basename $path) "$uri" + else + curl "$uri" -sSL --retry $download_retries --retry-delay $retry_wait_time_seconds --create-dirs -o "$path" --fail + fi + + return $? +} + +mkdir -p /tmp/src +cd /tmp/src + +echo "Installing Ninja" +GetFile https://github.com/ninja-build/ninja/archive/v1.10.0.tar.gz /tmp/src/ninja-linux.tar.gz +tar -zxf ninja-linux.tar.gz +cd ninja-1.10.0 +cmake -Bbuild-cmake -H. +cmake --build build-cmake +mv ./build-cmake/ninja /usr/bin + +cd / +rm -rf /tmp/src