From a6ea278502e757049312eb6b11865a6bd2e7d07b Mon Sep 17 00:00:00 2001 From: leqiao-1 <61653207+leqiao-1@users.noreply.github.com> Date: Mon, 21 Mar 2022 09:46:02 +0800 Subject: [PATCH] add python3.10 support (#10848) * add python3.10 support * upgrade numpy version in build pipeline * add python 3.10 path * upgrade torch version in build pipeline * update docker run arguments * change torch version * fix typo * fix permission issue * change python version * remove python3.10 for openvino build * remove python 3.10 for openvino build --- .../py-package-build-pipeline.yml | 4 +-- .../py-packaging-selectable-stage.yml | 8 ++--- .../templates/py-packaging-stage.yml | 36 +++++++++++++++++-- .../set-python-manylinux-variables-step.yml | 4 +++ .../docker/scripts/install_python_deps.sh | 2 ++ .../docker/scripts/manylinux/install_deps.sh | 8 +++-- 6 files changed, 51 insertions(+), 11 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/py-package-build-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-package-build-pipeline.yml index 1ed02e39dd..578ab5ab11 100644 --- a/tools/ci_build/github/azure-pipelines/py-package-build-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/py-package-build-pipeline.yml @@ -22,12 +22,12 @@ parameters: - name: python_version displayName: 'Python version used for build' type: object - default: ['3.7', '3.8', '3.9'] + default: ["'3.7'", "'3.8'", "'3.9'"] - name: cpu_build_py_parameters displayName: 'Extra parameters to pass to build.py for CPU package.' type: string - default: '--use_dnnl --use_openvino CPU_FP32' + default: '--use_openvino CPU_FP32' - name: gpu_build_py_parameters displayName: 'Extra parameters to pass to build.py for GPU package.' diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-selectable-stage.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-selectable-stage.yml index c6fd4c8d46..0a328d5cfc 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-selectable-stage.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-selectable-stage.yml @@ -22,12 +22,12 @@ parameters: - name: python_version displayName: 'Python version used for build' type: object - default: ['3.7', '3.8', '3.9'] + default: ["'3.7'", "'3.8'", "'3.9'"] - name: cpu_build_py_parameters displayName: 'Extra parameters to pass to build.py for CPU package.' type: string - default: '--use_dnnl --use_openvino CPU_FP32' + default: '--use_openvino CPU_FP32' - name: gpu_build_py_parameters displayName: 'Extra parameters to pass to build.py for GPU package.' @@ -153,7 +153,7 @@ stages: modifyEnvironment: true - script: | - python -m pip install -q setuptools wheel numpy==1.16.6 + python -m pip install -q setuptools wheel numpy==1.21.0 workingDirectory: '$(Build.BinariesDirectory)' displayName: 'Install python modules' @@ -416,7 +416,7 @@ stages: workingFolder: '$(Build.BinariesDirectory)' - script: | - python -m pip install -q setuptools wheel numpy==1.16.6 + python -m pip install -q setuptools wheel numpy==1.21.0 workingDirectory: '$(Build.BinariesDirectory)' displayName: 'Install python modules' diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml index 9fad69a341..0d80bf1bcc 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml @@ -68,6 +68,8 @@ stages: PythonVersion: '3.8' Python39: PythonVersion: '3.9' + Python310: + PythonVersion: '3.10' steps: - checkout: self clean: true @@ -197,6 +199,8 @@ stages: PythonVersion: '3.8' Python39: PythonVersion: '3.9' + Python310: + PythonVersion: '3.10' steps: - checkout: self clean: true @@ -627,6 +631,10 @@ stages: PythonVersion: '3.9' MsbuildPlatform: x64 buildArch: x64 + Python310_x64: + PythonVersion: '3.10' + MsbuildPlatform: x64 + buildArch: x64 Python37_x86: PythonVersion: '3.7' MsbuildPlatform: Win32 @@ -639,6 +647,10 @@ stages: PythonVersion: '3.9' MsbuildPlatform: Win32 buildArch: x86 + Python310_x86: + PythonVersion: '3.10' + MsbuildPlatform: Win32 + buildArch: x86 variables: OnnxRuntimeBuildDirectory: '$(Build.BinariesDirectory)' EnvSetupScript: setup_env.bat @@ -671,7 +683,7 @@ stages: workingFolder: '$(Build.BinariesDirectory)' - script: | - python -m pip install -q setuptools wheel numpy==1.16.6 + python -m pip install -q setuptools wheel numpy==1.21.0 workingDirectory: '$(Build.BinariesDirectory)' displayName: 'Install python modules' @@ -830,6 +842,11 @@ stages: EpBuildFlags: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.2.1.8.Windows10.x86_64.cuda-11.4.cudnn8.2" --cuda_version=$(CUDA_VERSION) --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v$(CUDA_VERSION)" --cudnn_home="C:\local\cudnn-$(CUDA_VERSION)-windows-x64-v8.2.2.26\cuda" --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=37;50;52;60;61;70;75;80" EnvSetupScript: setup_env_gpu.bat EP_NAME: gpu + Python310_GPU: + PythonVersion: '3.10' + EpBuildFlags: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.2.1.8.Windows10.x86_64.cuda-11.4.cudnn8.2" --cuda_version=$(CUDA_VERSION) --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v$(CUDA_VERSION)" --cudnn_home="C:\local\cudnn-$(CUDA_VERSION)-windows-x64-v8.2.2.26\cuda" --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=37;50;52;60;61;70;75;80" + EnvSetupScript: setup_env_gpu.bat + EP_NAME: gpu Python37_dml: PythonVersion: '3.7' EpBuildFlags: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos @@ -845,6 +862,11 @@ stages: EpBuildFlags: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos EP_NAME: directml EnvSetupScript: setup_env.bat + Python310_dml: + PythonVersion: '3.10' + EpBuildFlags: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos + EP_NAME: directml + EnvSetupScript: setup_env.bat steps: - checkout: self clean: true @@ -866,7 +888,7 @@ stages: workingFolder: '$(Build.BinariesDirectory)' - script: | - python -m pip install -q setuptools wheel numpy==1.16.6 + python -m pip install -q setuptools wheel numpy==1.21.0 workingDirectory: '$(Build.BinariesDirectory)' displayName: 'Install python modules' @@ -1016,6 +1038,8 @@ stages: PythonVersion: '3.8' Python39: PythonVersion: '3.9' + Python310: + PythonVersion: '3.10' steps: - checkout: self clean: true @@ -1067,6 +1091,9 @@ stages: pool: 'Linux-CPU' strategy: matrix: + ARM64_Py310: + PYTHON_EXE: '/opt/python/cp310-cp310/bin/python3' + Image: 'manylinux2014_aarch64' ARM64_Py39: PYTHON_EXE: '/opt/python/cp39-cp39/bin/python3' Image: 'manylinux2014_aarch64' @@ -1077,6 +1104,9 @@ stages: PYTHON_EXE: '/opt/python/cp37-cp37m/bin/python3' Image: 'manylinux2014_aarch64' # Uncomment the following lines to generate packages for PowerPC + # POWERPC_Py310: + # PYTHON_EXE: '/opt/python/cp310-cp310/bin/python3' + # Image: 'manylinux2014_ppc64le' # POWERPC_Py39: # PYTHON_EXE: '/opt/python/cp39-cp39/bin/python3' # Image: 'manylinux2014_ppc64le' @@ -1104,7 +1134,7 @@ stages: --volume $(Build.BinariesDirectory):/build \ -w /tmp/a \ quay.io/pypa/$(Image) \ - /usr/bin/bash -c "$(PYTHON_EXE) -m pip install numpy==1.19.5 && $(PYTHON_EXE) /onnxruntime_src/tools/ci_build/build.py \ + /usr/bin/bash -c "$(PYTHON_EXE) -m pip install numpy==1.21.0 && $(PYTHON_EXE) /onnxruntime_src/tools/ci_build/build.py \ --build_dir /build \ --config Release \ --skip_submodule_sync \ diff --git a/tools/ci_build/github/azure-pipelines/templates/set-python-manylinux-variables-step.yml b/tools/ci_build/github/azure-pipelines/templates/set-python-manylinux-variables-step.yml index 4d9e2cf4a8..e3ea2281bb 100644 --- a/tools/ci_build/github/azure-pipelines/templates/set-python-manylinux-variables-step.yml +++ b/tools/ci_build/github/azure-pipelines/templates/set-python-manylinux-variables-step.yml @@ -22,6 +22,10 @@ steps: variables = { "PythonManylinuxDir": "/opt/python/cp39-cp39" } + elif version == "3.10": + variables = { + "PythonManylinuxDir": "/opt/python/cp310-cp310" + } else: raise ValueError("Unsupported Python version: '{}'".format(version)) diff --git a/tools/ci_build/github/linux/docker/scripts/install_python_deps.sh b/tools/ci_build/github/linux/docker/scripts/install_python_deps.sh index d4f8aeae3b..345e4bee31 100755 --- a/tools/ci_build/github/linux/docker/scripts/install_python_deps.sh +++ b/tools/ci_build/github/linux/docker/scripts/install_python_deps.sh @@ -42,6 +42,8 @@ elif [[ "$PYTHON_VER" = "3.8" && -d "/opt/python/cp38-cp38" ]]; then PYTHON_EXE="/opt/python/cp38-cp38/bin/python3.8" elif [[ "$PYTHON_VER" = "3.9" && -d "/opt/python/cp39-cp39" ]]; then PYTHON_EXE="/opt/python/cp39-cp39/bin/python3.9" +elif [[ "$PYTHON_VER" = "3.10" && -d "/opt/python/cp310-cp310" ]]; then + PYTHON_EXE="/opt/python/cp310-cp310/bin/python3.10" else PYTHON_EXE="/usr/bin/python${PYTHON_VER}" fi diff --git a/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps.sh b/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps.sh index 687e5b43c0..1e4b52bde4 100755 --- a/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps.sh +++ b/tools/ci_build/github/linux/docker/scripts/manylinux/install_deps.sh @@ -40,7 +40,7 @@ function GetFile { } if [ ! -d "/opt/conda/bin" ]; then - PYTHON_EXES=("/opt/python/cp37-cp37m/bin/python3.7" "/opt/python/cp38-cp38/bin/python3.8" "/opt/python/cp39-cp39/bin/python3.9") + PYTHON_EXES=("/opt/python/cp37-cp37m/bin/python3.7" "/opt/python/cp38-cp38/bin/python3.8" "/opt/python/cp39-cp39/bin/python3.9" "/opt/python/cp310-cp310/bin/python3.10") else PYTHON_EXES=("/opt/conda/bin/python") fi @@ -93,7 +93,11 @@ export CMAKE_ARGS="-DONNX_GEN_PB_TYPE_STUBS=OFF -DONNX_WERROR=OFF" for PYTHON_EXE in "${PYTHON_EXES[@]}" do ${PYTHON_EXE} -m pip install -r ${0/%install_deps\.sh/requirements\.txt} - ${PYTHON_EXE} -m pip install -r ${0/%install_deps\.sh/..\/training\/ortmodule\/stage1\/requirements_torch_cpu.txt} + if ![[ ${PYTHON_EXE} = "/opt/python/cp310-cp310/bin/python3.10" ]]; then + ${PYTHON_EXE} -m pip install -r ${0/%install_deps\.sh/..\/training\/ortmodule\/stage1\/requirements_torch_cpu.txt} + else + ${PYTHON_EXE} -m pip install torch==1.11.0 + fi done cd /tmp/src