From c0a4fe777fcc1311bf1379651ca68dfde176d94d Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Wed, 13 Sep 2023 15:21:28 +0800 Subject: [PATCH] Move Linux python test into docker (#17479) ### Description supplement of #17417 ### Motivation and Context --- .../azure-pipelines/linux-ci-pipeline.yml | 103 ++++++------------ tools/scripts/python_test.sh | 28 +++++ tools/scripts/symbolic_shape_infer_test.sh | 13 +++ 3 files changed, 73 insertions(+), 71 deletions(-) create mode 100644 tools/scripts/python_test.sh create mode 100644 tools/scripts/symbolic_shape_infer_test.sh diff --git a/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml index eb6b274f87..21bc1c481b 100644 --- a/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-ci-pipeline.yml @@ -141,77 +141,39 @@ stages: " displayName: 'Dotnet build C# sln and Test' - - task: CmdLine@2 - displayName: 'Install python deps' - inputs: - script: | - set -e -x - python3 -m pip uninstall -y ort-nightly-gpu ort-nightly onnxruntime onnxruntime-gpu onnxruntime-training onnxruntime-directml ort-nightly-directml onnx -qq - cp $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/manylinux/requirements.txt $(Build.BinariesDirectory)/requirements.txt - # Test ORT with the latest ONNX release. - sed -i "s/git+http:\/\/github\.com\/onnx\/onnx.*/onnx/" $(Build.BinariesDirectory)/requirements.txt - python3 -m pip install -r $(Build.BinariesDirectory)/requirements.txt - mkdir $(Build.BinariesDirectory)/requirements_torch_cpu/ - cp $(Build.SourcesDirectory)/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements_torch_cpu/requirements.txt $(Build.BinariesDirectory)/requirements_torch_cpu/requirements.txt - python3 -m pip install -r $(Build.BinariesDirectory)/requirements_torch_cpu/requirements.txt + - bash: | + mkdir -p $HOME/.onnx + docker run --rm \ + --volume /data/onnx:/data/onnx:ro \ + --volume $(Build.SourcesDirectory):/onnxruntime_src \ + --volume $(Build.BinariesDirectory):/build \ + --volume /data/models:/build/models:ro \ + --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \ + -e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \ + -e NIGHTLY_BUILD \ + -e BUILD_BUILDNUMBER \ + onnxruntimecpubuild \ + /bin/bash -c " + set -ex; \ + /bin/bash /onnxruntime_src/tools/scripts/python_test.sh /onnxruntime_src /build Release && \ + /bin/bash /onnxruntime_src/tools/scripts/symbolic_shape_infer_test.sh /build + " + displayName: 'Run Release tests and symbolic shape infer test' - - task: CmdLine@2 - displayName: 'Install Release python package' - inputs: - script: | - rm -rf $(Build.BinariesDirectory)/Release/onnxruntime $(Build.BinariesDirectory)/Release/pybind11 - python3 -m pip install $(Build.BinariesDirectory)/Release/dist/*.whl - - - task: PythonScript@0 - displayName: 'Run Release unit tests' - inputs: - scriptPath: $(Build.SourcesDirectory)/tools/ci_build/build.py - workingDirectory: $(Build.BinariesDirectory)/Release - arguments: >- - --build_dir $(Build.BinariesDirectory) - --cmake_generator Ninja - --config Release - --test - --skip_submodule_sync - --build_shared_lib - --parallel - --build_wheel - --enable_onnx_tests - --enable_transformers_tool_test - --ctest_path "" - - - task: CmdLine@2 - displayName: 'Install Debug python package' - inputs: - script: | - set -e -x - rm -rf $(Build.BinariesDirectory)/Debug/onnxruntime $(Build.BinariesDirectory)/Debug/pybind11 - python3 -m pip uninstall -y ort-nightly-gpu ort-nightly onnxruntime onnxruntime-gpu onnxruntime-training onnxruntime-directml ort-nightly-directml -qq - python3 -m pip install $(Build.BinariesDirectory)/Debug/dist/*.whl - - - task: PythonScript@0 - displayName: 'Run Debug unit tests' - inputs: - scriptPath: $(Build.SourcesDirectory)/tools/ci_build/build.py - workingDirectory: $(Build.BinariesDirectory)/Debug - arguments: >- - --build_dir $(Build.BinariesDirectory) - --cmake_generator Ninja - --config Debug - --test - --skip_submodule_sync - --build_shared_lib - --parallel - --build_wheel - --enable_onnx_tests - --enable_transformers_tool_test - --ctest_path "" - - - task: PythonScript@0 - displayName: 'Symbolic shape infer' - inputs: - scriptPath: $(Build.BinariesDirectory)/Release/onnxruntime_test_python_symbolic_shape_infer.py - workingDirectory: $(Build.BinariesDirectory)/Release + - bash: | + mkdir -p $HOME/.onnx + docker run --rm \ + --volume /data/onnx:/data/onnx:ro \ + --volume $(Build.SourcesDirectory):/onnxruntime_src \ + --volume $(Build.BinariesDirectory):/build \ + --volume /data/models:/build/models:ro \ + --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \ + -e ALLOW_RELEASED_ONNX_OPSET_ONLY=0 \ + -e NIGHTLY_BUILD \ + -e BUILD_BUILDNUMBER \ + onnxruntimecpubuild \ + /bin/bash /onnxruntime_src/tools/scripts/python_test.sh /onnxruntime_src /build Debug + displayName: 'Run Debug tests' - task: PublishTestResults@2 displayName: 'Publish unit test results' @@ -221,7 +183,6 @@ stages: testRunTitle: 'Unit Test Run' condition: succeededOrFailed() - - stage: arm64_build dependsOn: [] jobs: diff --git a/tools/scripts/python_test.sh b/tools/scripts/python_test.sh new file mode 100644 index 0000000000..bfdd4663fe --- /dev/null +++ b/tools/scripts/python_test.sh @@ -0,0 +1,28 @@ +#!/bin/bash + +set -ex + +export src_dir=$1 +export build_dir=$2 +export config=$3 + +# it's for manylinux image +export PATH=/opt/python/cp38-cp38/bin:$PATH + +echo Install Python Deps +cp $src_dir/tools/ci_build/github/linux/docker/scripts/manylinux/requirements.txt $build_dir/requirements.txt + +python3 -m pip install -r $build_dir/requirements.txt +mkdir -p $build_dir/requirements_torch_cpu/ +cp $src_dir/tools/ci_build/github/linux/docker/scripts/training/ortmodule/stage1/requirements_torch_cpu/requirements.txt $build_dir/requirements_torch_cpu/requirements.txt +python3 -m pip install -r $build_dir/requirements_torch_cpu/requirements.txt +python3 -m pip list | grep onnx + +echo Install $config python package +rm -rf $build_dir/$config/onnxruntime $build_dir/$config/pybind11 +python3 -m pip install $build_dir/$config/dist/*.whl + +echo Run $config unit tests +pushd $build_dir/$config/ +python3 $src_dir/tools/ci_build/build.py --build_dir $build_dir --cmake_generator Ninja --config $config --test --skip_submodule_sync --build_shared_lib --parallel --build_wheel --enable_onnx_tests --enable_transformers_tool_test --ctest_path "" +popd diff --git a/tools/scripts/symbolic_shape_infer_test.sh b/tools/scripts/symbolic_shape_infer_test.sh new file mode 100644 index 0000000000..d8d50c5e3f --- /dev/null +++ b/tools/scripts/symbolic_shape_infer_test.sh @@ -0,0 +1,13 @@ +#!/bin/bash + +set -ex + +export build_dir=$1 + +# it's for manylinux image +export PATH=/opt/python/cp38-cp38/bin:$PATH + +echo Run symbolic shape infer test +pushd $build_dir/Release/ +python3 /build/Release/onnxruntime_test_python_symbolic_shape_infer.py +popd