From f3b6664384aeed424692956eee0ff8c5a666c7ef Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Thu, 2 Mar 2023 19:44:49 -0800 Subject: [PATCH] Remove Python 3.7 from the python packaging pipeline (#14887) ### Description 1. Remove Python 3.7 from the python packaging pipeline. It is planned for the next release and approved by the PMs. Also we will add 3.11, but it will be addressed in another PR. 2. Stop generating python packages based on Ubuntu 18.04 which will reach EOL next month. We will either replace them with Ubuntu 20.04 or a CentOS 8 variant. --- .../templates/py-packaging-stage.yml | 37 +++---------------- .../azure-pipelines/templates/py-win-gpu.yml | 4 +- .../linux/build_linux_arm64_python_package.sh | 2 +- 3 files changed, 8 insertions(+), 35 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml index fac33f5d4e..bd5afa9065 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml @@ -55,10 +55,6 @@ stages: pool: 'Win-CPU-2021' strategy: matrix: - Python37_x64: - PythonVersion: '3.7' - MsbuildPlatform: x64 - buildArch: x64 Python38_x64: PythonVersion: '3.8' MsbuildPlatform: x64 @@ -74,10 +70,6 @@ stages: # Training build cannot support Win32 for now because one or more of its python # dependencies does not support Win32. So, don't build a training package for Win32 ${{ if not(contains(parameters.build_py_parameters, '--enable_training')) }}: - Python37_x86: - PythonVersion: '3.7' - MsbuildPlatform: Win32 - buildArch: x86 Python38_x86: PythonVersion: '3.8' MsbuildPlatform: Win32 @@ -237,7 +229,7 @@ stages: #in 32 bits Win32 build. I tried all the settings but they all don't work. - task: SDLNativeRules@3 displayName: 'Run the PREfast SDL Native Rules for MSBuild' - condition: and (succeeded(), and(eq(variables['buildArch'], 'x64'), eq(variables['PythonVersion'], '3.7'))) + condition: and (succeeded(), and(eq(variables['buildArch'], 'x64'), eq(variables['PythonVersion'], '3.8'))) inputs: msBuildArchitecture: amd64 setupCommandlines: 'python $(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_generator "Visual Studio 16 2019" --enable_pybind --enable_onnx_tests --parallel $(TelemetryOption) --update --cmake_extra_defines onnxruntime_ENABLE_STATIC_ANALYSIS=ON' @@ -247,7 +239,7 @@ stages: - task: TSAUpload@2 displayName: 'TSA upload' - condition: and(and (succeeded(), and(eq(variables['buildArch'], 'x64'), eq(variables['PythonVersion'], '3.7'))), eq(variables['Build.SourceBranch'], 'refs/heads/main')) + condition: and(and (succeeded(), and(eq(variables['buildArch'], 'x64'), eq(variables['PythonVersion'], '3.8'))), eq(variables['Build.SourceBranch'], 'refs/heads/main')) inputs: GdnPublishTsaOnboard: false GdnPublishTsaConfigFile: '$(Build.sourcesDirectory)\.gdn\.gdntsa' @@ -262,14 +254,6 @@ stages: condition: always() - ${{ if eq(parameters.enable_windows_gpu, true) }}: - - template: py-win-gpu.yml - parameters: - MACHINE_POOL: 'onnxruntime-gpu-winbuild-t4' - PYTHON_VERSION: '3.7' - EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.5.1.7.Windows10.x86_64.cuda-11.8.cudnn8.6" --cuda_version=11.6 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.6" --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=37;50;52;60;61;70;75;80" - ENV_SETUP_SCRIPT: setup_env_gpu.bat - EP_NAME: gpu - - template: py-win-gpu.yml parameters: MACHINE_POOL: 'onnxruntime-gpu-winbuild-t4' @@ -294,14 +278,6 @@ stages: ENV_SETUP_SCRIPT: setup_env_gpu.bat EP_NAME: gpu - - template: py-win-gpu.yml - parameters: - MACHINE_POOL: 'aiinfra-dml-winbuild' - PYTHON_VERSION: '3.7' - EP_BUILD_FLAGS: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos - ENV_SETUP_SCRIPT: setup_env.bat - EP_NAME: directml - - template: py-win-gpu.yml parameters: MACHINE_POOL: 'aiinfra-dml-winbuild' @@ -337,8 +313,6 @@ stages: MACOSX_DEPLOYMENT_TARGET: '10.15' strategy: matrix: - Python37: - PythonVersion: '3.7' Python38: PythonVersion: '3.8' Python39: @@ -490,16 +464,15 @@ stages: ${{ if contains(parameters.build_py_parameters, '--use_azure') }}: device: '-d AZURE' - - ${{ if eq(parameters.enable_ubuntu_cpu, true) }}: + - ${{ if and(eq(parameters.enable_ubuntu_cpu, true), contains(parameters.build_py_parameters, '--use_azure'))}}: - template: py-linux-ubuntu.yml parameters: arch: 'aarch64' machine_pool: 'aiinfra-linux-ARM64-CPU-2019' base_image: 'arm64v8/ubuntu:18.04' - ${{ if contains(parameters.build_py_parameters, '--use_azure') }}: - device: '-d AZURE' + device: '-d AZURE' - - ${{ if eq(parameters.enable_ubuntu_cpu, true) }}: + - ${{ if and(eq(parameters.enable_ubuntu_cpu, true), contains(parameters.build_py_parameters, '--use_azure'))}}: - template: py-linux-ubuntu.yml parameters: arch: 'x86_64' diff --git a/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml b/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml index 19d2a897fe..77fc2a617d 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml @@ -150,10 +150,10 @@ jobs: workingDirectory: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo' displayName: 'Run Python Tests' - - ${{ if eq(parameters.PYTHON_VERSION, '3.7') }}: + - ${{ if eq(parameters.PYTHON_VERSION, '3.8') }}: - task: DeleteFiles@1 displayName: 'Delete files from $(Build.BinariesDirectory)\RelWithDebInfo' - condition: and (succeeded(), eq(variables['$'], '3.7')) + condition: and (succeeded(), eq(variables['$'], '3.8')) inputs: SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo' Contents: | diff --git a/tools/ci_build/github/linux/build_linux_arm64_python_package.sh b/tools/ci_build/github/linux/build_linux_arm64_python_package.sh index 16b60985f8..20452509db 100755 --- a/tools/ci_build/github/linux/build_linux_arm64_python_package.sh +++ b/tools/ci_build/github/linux/build_linux_arm64_python_package.sh @@ -7,7 +7,7 @@ CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-st BUILD_DEVICE="CPU" BUILD_CONFIG="Release" -PYTHON_EXES=("/opt/python/cp37-cp37m/bin/python3.7" "/opt/python/cp38-cp38/bin/python3.8" "/opt/python/cp39-cp39/bin/python3.9" "/opt/python/cp310-cp310/bin/python3.10") +PYTHON_EXES=("/opt/python/cp38-cp38/bin/python3.8" "/opt/python/cp39-cp39/bin/python3.9" "/opt/python/cp310-cp310/bin/python3.10") while getopts "d:p:" parameter_Option do case "${parameter_Option}" in