From 382fa86af8703db4f5868d511632592d2058f568 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Thu, 2 Jan 2020 15:25:25 -0800 Subject: [PATCH] Pipeline changes for python 3.8 (#2753) 1. Pipeline changes for python 3.8 2. Fix a regression in setup.py which was just introduced in the previous commit. Please notice, we still haven't made python 3.8 + Windows + CUDA work. --- setup.py | 2 +- .../azure-pipelines-py-packaging.yml | 24 ++++++++++++++++--- .../linux/docker/scripts/install_deps.sh | 2 ++ .../linux/docker/scripts/install_onnx.sh | 8 +++---- 4 files changed, 28 insertions(+), 8 deletions(-) diff --git a/setup.py b/setup.py index c11c03f295..4f3ba6e535 100644 --- a/setup.py +++ b/setup.py @@ -214,7 +214,7 @@ setup( py_modules=python_modules_list, install_requires=[ 'onnx>=1.2.3', - 'numpy>=1.18.0,<2' + 'numpy>=1.18.0' ], entry_points= { 'console_scripts': [ diff --git a/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml b/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml index 742784dd8c..c2bafb57fe 100644 --- a/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml +++ b/tools/ci_build/github/azure-pipelines/azure-pipelines-py-packaging.yml @@ -8,12 +8,19 @@ jobs: Python35: python.version: '3.5' python.dir: '/opt/python/cp35-cp35m' + python.include.dir: '/opt/python/cp35-cp35m/include/python3.5m' Python36: python.version: '3.6' python.dir: '/opt/python/cp36-cp36m' + python.include.dir: '/opt/python/cp36-cp36m/include/python3.6m' Python37: python.version: '3.7' python.dir: '/opt/python/cp37-cp37m' + python.include.dir: '/opt/python/cp37-cp37m/include/python3.7m' + Python38: + python.version: '3.8' + python.dir: '/opt/python/cp38-cp38' + python.include.dir: '/opt/python/cp38-cp38/include/python3.8' steps: - template: templates/set-test-data-variables-step.yml @@ -40,7 +47,7 @@ jobs: - task: CmdLine@2 inputs: script: | - docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-manylinux-$(python.version) $(python.dir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_featurizers --build_wheel --enable_onnx_tests --cmake_extra_defines PYTHON_INCLUDE_DIR=$(python.dir)/include/python$(python.version)m PYTHON_LIBRARY=/usr/lib64/librt.so + docker run --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-manylinux-$(python.version) $(python.dir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --use_openmp --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_featurizers --build_wheel --enable_onnx_tests --cmake_extra_defines PYTHON_INCLUDE_DIR=$(python.include.dir) PYTHON_LIBRARY=/usr/lib64/librt.so workingDirectory: $(Build.SourcesDirectory) - task: CopyFiles@2 @@ -69,12 +76,19 @@ jobs: Python35: python.version: '3.5' python.dir: '/opt/python/cp35-cp35m' + python.include.dir: '/opt/python/cp35-cp35m/include/python3.5m' Python36: python.version: '3.6' python.dir: '/opt/python/cp36-cp36m' + python.include.dir: '/opt/python/cp36-cp36m/include/python3.6m' Python37: python.version: '3.7' python.dir: '/opt/python/cp37-cp37m' + python.include.dir: '/opt/python/cp37-cp37m/include/python3.7m' + Python38: + python.version: '3.8' + python.dir: '/opt/python/cp38-cp38' + python.include.dir: '/opt/python/cp38-cp38/include/python3.8' steps: - template: templates/set-test-data-variables-step.yml @@ -101,7 +115,7 @@ jobs: - task: CmdLine@2 inputs: script: | - docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-manylinux-gpu-$(python.version) $(python.dir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_featurizers --build_wheel --enable_onnx_tests --cmake_extra_defines PYTHON_INCLUDE_DIR=$(python.dir)/include/python$(python.version)m PYTHON_LIBRARY=/usr/lib64/librt.so --use_cuda --cuda_version=10.0 --cuda_home=/usr/local/cuda-10.0 --cudnn_home=/usr/local/cuda-10.0 + docker run --gpus all -e NVIDIA_VISIBLE_DEVICES=all --rm --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build -e NIGHTLY_BUILD onnxruntime-manylinux-gpu-$(python.version) $(python.dir)/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_dir /build --config Release --skip_submodule_sync --parallel --build_shared_lib --cmake_path /usr/bin/cmake --ctest_path /usr/bin/ctest --use_featurizers --build_wheel --enable_onnx_tests --cmake_extra_defines PYTHON_INCLUDE_DIR=$(python.include.dir) PYTHON_LIBRARY=/usr/lib64/librt.so --use_cuda --cuda_version=10.0 --cuda_home=/usr/local/cuda-10.0 --cudnn_home=/usr/local/cuda-10.0 workingDirectory: $(Build.SourcesDirectory) - task: CopyFiles@2 @@ -131,6 +145,8 @@ jobs: python.version: '3.6' Python37: python.version: '3.7' + Python38: + python.version: '3.8' variables: OrtPackageId: 'Microsoft.ML.OnnxRuntime' MsbuildArguments: '-maxcpucount' @@ -199,7 +215,7 @@ jobs: workspace: clean: all pool: 'Win-GPU-2019' - timeoutInMinutes: 120 + timeoutInMinutes: 60 variables: CUDA_VERSION: '10.0' EnvSetupScript: setup_env.bat @@ -276,6 +292,8 @@ jobs: python.version: '3.6' Python37: python.version: '3.7' + Python38: + python.version: '3.8' steps: - task: UsePythonVersion@0 displayName: 'Use Python' diff --git a/tools/ci_build/github/linux/docker/scripts/install_deps.sh b/tools/ci_build/github/linux/docker/scripts/install_deps.sh index 768c4f7852..bf39742be9 100755 --- a/tools/ci_build/github/linux/docker/scripts/install_deps.sh +++ b/tools/ci_build/github/linux/docker/scripts/install_deps.sh @@ -54,6 +54,8 @@ elif [[ "$PYTHON_VER" = "3.6" && -d "/opt/python/cp36-cp36m" ]]; then PYTHON_EXE="/opt/python/cp36-cp36m/bin/python3.6" elif [[ "$PYTHON_VER" = "3.7" && -d "/opt/python/cp37-cp37m" ]]; then PYTHON_EXE="/opt/python/cp37-cp37m/bin/python3.7" +elif [[ "$PYTHON_VER" = "3.8" && -d "/opt/python/cp38-cp38" ]]; then + PYTHON_EXE="/opt/python/cp38-cp38/bin/python3.8" else PYTHON_EXE="/usr/bin/python${PYTHON_VER}" fi diff --git a/tools/ci_build/github/linux/docker/scripts/install_onnx.sh b/tools/ci_build/github/linux/docker/scripts/install_onnx.sh index fc2cc11907..20b637f159 100755 --- a/tools/ci_build/github/linux/docker/scripts/install_onnx.sh +++ b/tools/ci_build/github/linux/docker/scripts/install_onnx.sh @@ -14,13 +14,12 @@ PYTHON_VER=$1 if [[ "$PYTHON_VER" = "3.5" && -d "/opt/python/cp35-cp35m" ]]; then PYTHON_EXE="/opt/python/cp35-cp35m/bin/python3.5" - export PATH=/opt/python/cp35-cp35m/bin:$PATH elif [[ "$PYTHON_VER" = "3.6" && -d "/opt/python/cp36-cp36m" ]]; then PYTHON_EXE="/opt/python/cp36-cp36m/bin/python3.6" - export PATH=/opt/python/cp36-cp36m/bin:$PATH elif [[ "$PYTHON_VER" = "3.7" && -d "/opt/python/cp37-cp37m" ]]; then PYTHON_EXE="/opt/python/cp37-cp37m/bin/python3.7" - export PATH=/opt/python/cp37-cp37m/bin:$PATH +elif [[ "$PYTHON_VER" = "3.8" && -d "/opt/python/cp38-cp38" ]]; then + PYTHON_EXE="/opt/python/cp38-cp38/bin/python3.8" else PYTHON_EXE="/usr/bin/python${PYTHON_VER}" fi @@ -57,5 +56,6 @@ for v2t in ${version2tag[*]}; do mv /tmp/src/onnx-$onnx_version/tools/repl_protoc-gen-mypy.py /tmp/src/onnx-$onnx_version/tools/protoc-gen-mypy.py mkdir -p /data/onnx/${onnx_tag} ${PYTHON_EXE} -m pip install . - backend-test-tools generate-data -o /data/onnx/$onnx_tag + cd /tmp + ${PYTHON_EXE} -m onnx.backend.test.cmd_tools generate-data -o /data/onnx/$onnx_tag done