From ffaddead0ad7014190def4dba2350dc4d935e9c9 Mon Sep 17 00:00:00 2001 From: Jian Chen Date: Wed, 23 Oct 2024 08:14:10 -0700 Subject: [PATCH] Refactor cuda packaging pipeline (#22542) ### Description ### Motivation and Context --- .../build-perf-test-binaries-pipeline.yml | 7 +- .../py-cuda-alt-packaging-pipeline.yml | 27 +++ .../py-cuda-packaging-pipeline.yml | 54 +++--- .../py-dml-packaging-pipeline.yml | 18 ++ .../azure-pipelines/py-packaging-pipeline.yml | 15 +- .../jobs/py-linux-cuda-package-test-job.yml | 8 +- .../py-cpu-packaging-stage.yml} | 170 ++---------------- ...g-stage.yml => py-gpu-packaging-stage.yml} | 53 +++--- .../py-linux-gpu-stage.yml} | 49 +++-- .../py-win-gpu-stage.yml} | 28 +-- .../py-packaging-linux-test-cuda.yml | 4 +- 11 files changed, 172 insertions(+), 261 deletions(-) create mode 100644 tools/ci_build/github/azure-pipelines/py-cuda-alt-packaging-pipeline.yml create mode 100644 tools/ci_build/github/azure-pipelines/py-dml-packaging-pipeline.yml rename tools/ci_build/github/azure-pipelines/{templates/py-packaging-stage.yml => stages/py-cpu-packaging-stage.yml} (64%) rename tools/ci_build/github/azure-pipelines/stages/{py-cuda-packaging-stage.yml => py-gpu-packaging-stage.yml} (68%) rename tools/ci_build/github/azure-pipelines/{templates/py-linux-gpu.yml => stages/py-linux-gpu-stage.yml} (63%) rename tools/ci_build/github/azure-pipelines/{templates/py-win-gpu.yml => stages/py-win-gpu-stage.yml} (93%) diff --git a/tools/ci_build/github/azure-pipelines/build-perf-test-binaries-pipeline.yml b/tools/ci_build/github/azure-pipelines/build-perf-test-binaries-pipeline.yml index 50d4d8a912..4e5d9a70be 100644 --- a/tools/ci_build/github/azure-pipelines/build-perf-test-binaries-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/build-perf-test-binaries-pipeline.yml @@ -34,11 +34,8 @@ stages: # build Python packages # Linux GPU only - ${{ if parameters.BuildPythonPackages }}: - - template: templates/py-packaging-stage.yml + - template: stages/py-gpu-packaging-stage.yml parameters: enable_linux_gpu: true - enable_linux_cpu: false - enable_windows_cpu: false enable_windows_gpu: false - enable_mac_cpu: false - enable_linux_arm: false + cuda_version: 12.2 diff --git a/tools/ci_build/github/azure-pipelines/py-cuda-alt-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-cuda-alt-packaging-pipeline.yml new file mode 100644 index 0000000000..cc2977721d --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/py-cuda-alt-packaging-pipeline.yml @@ -0,0 +1,27 @@ +trigger: none + +parameters: + - name: enable_linux_cuda + type: boolean + default: true + + - name: enable_windows_cuda + type: boolean + default: true + + - name: cmake_build_type + type: string + default: 'Release' + values: + - Debug + - Release + - RelWithDebInfo + - MinSizeRel + +stages: + - template: stages/py-gpu-packaging-stage.yml + parameters: + enable_linux_cuda: ${{ parameters.enable_linux_cuda }} + enable_windows_cuda: ${{ parameters.enable_windows_cuda }} + cmake_build_type: ${{ parameters.cmake_build_type }} + cuda_version: '11.8' diff --git a/tools/ci_build/github/azure-pipelines/py-cuda-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-cuda-packaging-pipeline.yml index 3503857a92..7e6b188968 100644 --- a/tools/ci_build/github/azure-pipelines/py-cuda-packaging-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/py-cuda-packaging-pipeline.yml @@ -1,12 +1,20 @@ trigger: none - +# The `resources` specify the location and version of the 1ES PT. +resources: + repositories: + - repository: 1esPipelines + type: git + name: 1ESPipelineTemplates/1ESPipelineTemplates + ref: refs/tags/release parameters: - - name: enable_linux_gpu + - name: enable_linux_cuda type: boolean default: true - - name: enable_windows_gpu + + - name: enable_windows_cuda type: boolean default: true + - name: cmake_build_type type: string default: 'Release' @@ -15,28 +23,22 @@ parameters: - Release - RelWithDebInfo - MinSizeRel - - name: cuda_version - type: string - default: '12.2' - values: - - 11.8 - - 12.2 - - name: SpecificArtifact - displayName: Use Specific Artifact - type: boolean - default: false - - name: BuildId - displayName: Specific Artifact's BuildId - type: string - default: '0' +extends: + # The pipeline extends the 1ES PT which will inject different SDL and compliance tasks. + # For non-production pipelines, use "Unofficial" as defined below. + # For productions pipelines, use "Official". + template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines + parameters: + # Update the pool with your team's 1ES hosted pool. + pool: + name: 'onnxruntime-Win-CPU-2022' # Name of your hosted pool + os: windows # OS of the image. This value cannot be a variable. Allowed values: windows, linux, macOS -stages: - - template: stages/py-cuda-packaging-stage.yml - parameters: - enable_linux_gpu: ${{ parameters.enable_linux_gpu }} - enable_windows_gpu: ${{ parameters.enable_windows_gpu }} - cmake_build_type: ${{ parameters.cmake_build_type }} - cuda_version: ${{ parameters.cuda_version }} - SpecificArtifact: ${{ parameters.SpecificArtifact }} - BuildId: ${{ parameters.BuildId }} + stages: + - template: stages/py-gpu-packaging-stage.yml + parameters: + enable_linux_cuda: ${{ parameters.enable_linux_cuda }} + enable_windows_cuda: ${{ parameters.enable_windows_cuda }} + cmake_build_type: ${{ parameters.cmake_build_type }} + cuda_version: '12.2' diff --git a/tools/ci_build/github/azure-pipelines/py-dml-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-dml-packaging-pipeline.yml new file mode 100644 index 0000000000..0c7c6abeb3 --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/py-dml-packaging-pipeline.yml @@ -0,0 +1,18 @@ +trigger: none + +parameters: + - name: cmake_build_type + type: string + default: 'Release' + values: + - Debug + - Release + - RelWithDebInfo + - MinSizeRel + +stages: + - template: stages/py-gpu-packaging-stage.yml + parameters: + enable_windows_dml: true + cmake_build_type: ${{ parameters.cmake_build_type }} + publish_symbols: true diff --git a/tools/ci_build/github/azure-pipelines/py-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/py-packaging-pipeline.yml index de17db216d..ed992be312 100644 --- a/tools/ci_build/github/azure-pipelines/py-packaging-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/py-packaging-pipeline.yml @@ -4,21 +4,11 @@ parameters: type: boolean default: true -- name: enable_linux_gpu - displayName: 'Whether Linux GPU package is built.' - type: boolean - default: true - - name: enable_windows_cpu displayName: 'Whether Windows CPU package is built.' type: boolean default: true -- name: enable_windows_gpu - displayName: 'Whether Windows GPU package is built.' - type: boolean - default: true - - name: enable_mac_cpu displayName: 'Whether Mac CPU package is built.' type: boolean @@ -74,12 +64,10 @@ parameters: trigger: none stages: -- template: templates/py-packaging-stage.yml +- template: stages/py-cpu-packaging-stage.yml parameters: - enable_linux_gpu: ${{ parameters.enable_linux_gpu }} enable_linux_cpu: ${{ parameters.enable_linux_cpu }} enable_windows_cpu: ${{ parameters.enable_windows_cpu }} - enable_windows_gpu: ${{ parameters.enable_windows_gpu }} enable_mac_cpu: ${{ parameters.enable_mac_cpu }} enable_linux_arm: ${{ parameters.enable_linux_arm }} enable_windows_arm64_qnn: ${{ parameters.enable_windows_arm64_qnn }} @@ -90,3 +78,4 @@ stages: cmake_build_type: ${{ parameters.cmake_build_type }} qnn_sdk_version: ${{ parameters.qnn_sdk_version }} publish_symbols: true + diff --git a/tools/ci_build/github/azure-pipelines/stages/jobs/py-linux-cuda-package-test-job.yml b/tools/ci_build/github/azure-pipelines/stages/jobs/py-linux-cuda-package-test-job.yml index 9289935b4e..a33f757c24 100644 --- a/tools/ci_build/github/azure-pipelines/stages/jobs/py-linux-cuda-package-test-job.yml +++ b/tools/ci_build/github/azure-pipelines/stages/jobs/py-linux-cuda-package-test-job.yml @@ -57,15 +57,15 @@ jobs: - checkout: self - task: DownloadPipelineArtifact@2 inputs: - artifact: 'drop-linux-gpu-x86_64' - targetPath: '$(Build.SourcesDirectory)/drop-linux-gpu-x86_64' + artifact: 'linux_gpu_wheel_x86_64' + targetPath: '$(Build.SourcesDirectory)/linux_gpu_wheel_x86_64' ${{ if ne(parameters.build_id, 'latest') }}: buildType: 'specific' project: '${{ parameters.project }}' pipeline: '${{ parameters.pipeline }}' buildVersionToDownload: 'specific' buildId: '${{ parameters.build_id }}' - displayName: 'Download Build Artifacts - drop-linux-gpu-x86_64' + displayName: 'Download Build Artifacts - linux_gpu_wheel_x86_64' - task: DownloadPipelineArtifact@2 inputs: @@ -82,7 +82,7 @@ jobs: - bash: | set -e -x ls $(Build.SourcesDirectory) - mv "$(Build.SourcesDirectory)/drop-linux-gpu-x86_64" $(Build.BinariesDirectory)/${{parameters.cmake_build_type}} + mv "$(Build.SourcesDirectory)/linux_gpu_wheel_x86_64" $(Build.BinariesDirectory)/${{parameters.cmake_build_type}} mv "$(Build.SourcesDirectory)/onnxruntime_gpu" "$(Build.BinariesDirectory)/whl" cp -r "$(Build.BinariesDirectory)/whl" $(Build.BinariesDirectory)/tmp find "$(Build.BinariesDirectory)/tmp" -name '*.whl' -exec bash -c 'unzip -d "${1%.*}" "$1"' _ {} \; diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/stages/py-cpu-packaging-stage.yml similarity index 64% rename from tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml rename to tools/ci_build/github/azure-pipelines/stages/py-cpu-packaging-stage.yml index 10d7ce0474..e92761e20d 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/py-cpu-packaging-stage.yml @@ -10,21 +10,11 @@ parameters: type: boolean default: true -- name: enable_linux_gpu - displayName: 'Whether Linux GPU package is built.' - type: boolean - default: true - - name: enable_windows_cpu displayName: 'Whether Windows CPU package is built.' type: boolean default: true -- name: enable_windows_gpu - displayName: 'Whether Windows GPU package is built.' - type: boolean - default: true - - name: enable_mac_cpu displayName: 'Whether Mac CPU package is built.' type: boolean @@ -65,10 +55,6 @@ parameters: - RelWithDebInfo - MinSizeRel -- name: publish_symbols - type: boolean - default: false - # Only applies to QNN packages. - name: qnn_sdk_version type: string @@ -128,7 +114,7 @@ stages: clean: true submodules: recursive - - template: telemetry-steps.yml + - template: ../templates/telemetry-steps.yml - task: UsePythonVersion@0 inputs: @@ -142,7 +128,7 @@ stages: tsaConfigFilePath: '$(Build.SourcesDirectory)\.config\tsaoptions.json' appendSourceBranchName: false - - template: set-nightly-build-option-variable-step.yml + - template: ../templates/set-nightly-build-option-variable-step.yml - task: BatchScript@1 displayName: 'setup env' @@ -151,7 +137,7 @@ stages: modifyEnvironment: true workingFolder: '$(Build.BinariesDirectory)' - - template: download-deps.yml + - template: ../templates/download-deps.yml - task: PythonScript@0 displayName: 'Update deps.txt' @@ -180,24 +166,12 @@ stages: --enable_pybind --enable_onnx_tests ${{ parameters.build_py_parameters }} - --parallel --use_binskim_compliant_compile_flags --update + --parallel --use_binskim_compliant_compile_flags --update --build $(TelemetryOption) workingDirectory: '$(Build.BinariesDirectory)' - - task: VSBuild@1 - displayName: 'Build' - inputs: - solution: '$(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}\onnxruntime.sln' - platform: $(MsbuildPlatform) - configuration: ${{ parameters.cmake_build_type }} - msbuildArchitecture: $(buildArch) - maximumCpuCount: true - logProjectEvents: true - workingFolder: '$(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}' - createLogFile: true - # Esrp signing - - template: win-esrp-dll.yml + - template: ../templates/win-esrp-dll.yml parameters: FolderPath: '$(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}\${{ parameters.cmake_build_type }}\onnxruntime\capi' DisplayName: 'ESRP - Sign Native dlls' @@ -251,29 +225,8 @@ stages: python onnx_backend_test_series.py workingDirectory: '$(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}\${{ parameters.cmake_build_type }}' displayName: 'Run Python Tests' - - ${{ if eq(parameters.publish_symbols, true) }}: - - task: PublishSymbols@2 - displayName: 'Publish symbols' - condition: and (succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/main'), startsWith(variables['Build.SourceBranch'], 'refs/heads/rel-'))) - inputs: - SymbolsFolder: '$(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}\${{ parameters.cmake_build_type }}' - SearchPattern: | - onnxruntime_pybind11_state.pdb - onnxruntime_providers_shared.pdb - IndexSources: true - SymbolServerType: TeamServices - SymbolExpirationInDays: 3650 - SymbolsArtifactName: 'win_cpu_$(PythonVersion)_$(buildArch)_$(Build.BuildNumber)' - - task: TSAUpload@2 - displayName: 'TSA upload' - condition: and(and (succeeded(), and(eq(variables['buildArch'], 'x64'), eq(variables['PythonVersion'], '3.8'))), eq(variables['Build.SourceBranch'], 'refs/heads/main')) - inputs: - GdnPublishTsaOnboard: false - GdnPublishTsaConfigFile: '$(Build.sourcesDirectory)\.gdn\.gdntsa' - continueOnError: true - - - template: component-governance-component-detection-steps.yml + - template: ../templates/component-governance-component-detection-steps.yml parameters: condition: 'succeeded' @@ -281,87 +234,6 @@ stages: displayName: 'Clean Agent Directories' condition: always() -- ${{ if eq(parameters.enable_windows_gpu, true) }}: - - template: py-win-gpu.yml - parameters: - MACHINE_POOL: 'onnxruntime-Win2022-GPU-A10' - PYTHON_VERSION: '3.10' - EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home="$(Agent.TempDirectory)\TensorRT-10.4.0.26.Windows10.x86_64.cuda-11.8" --cuda_home="$(Agent.TempDirectory)\v11.8" --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80" - ENV_SETUP_SCRIPT: setup_env_gpu.bat - EP_NAME: gpu - publish_symbols: ${{ parameters.publish_symbols }} - cmake_build_type: ${{ parameters.cmake_build_type }} - - - template: py-win-gpu.yml - parameters: - MACHINE_POOL: 'onnxruntime-Win2022-GPU-A10' - PYTHON_VERSION: '3.11' - EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home="$(Agent.TempDirectory)\TensorRT-10.4.0.26.Windows10.x86_64.cuda-11.8" --cuda_home="$(Agent.TempDirectory)\v11.8" --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80" - ENV_SETUP_SCRIPT: setup_env_gpu.bat - EP_NAME: gpu - publish_symbols: ${{ parameters.publish_symbols }} - cmake_build_type: ${{ parameters.cmake_build_type }} - - - template: py-win-gpu.yml - parameters: - MACHINE_POOL: 'onnxruntime-Win2022-GPU-A10' - PYTHON_VERSION: '3.12' - EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home="$(Agent.TempDirectory)\TensorRT-10.4.0.26.Windows10.x86_64.cuda-11.8" --cuda_home="$(Agent.TempDirectory)\v11.8" --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80" - ENV_SETUP_SCRIPT: setup_env_gpu.bat - EP_NAME: gpu - publish_symbols: ${{ parameters.publish_symbols }} - cmake_build_type: ${{ parameters.cmake_build_type }} - - - template: py-win-gpu.yml - parameters: - MACHINE_POOL: 'onnxruntime-Win2022-GPU-A10' - PYTHON_VERSION: '3.13' - EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home="$(Agent.TempDirectory)\TensorRT-10.4.0.26.Windows10.x86_64.cuda-11.8" --cuda_home="$(Agent.TempDirectory)\v11.8" --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80" - ENV_SETUP_SCRIPT: setup_env_gpu.bat - EP_NAME: gpu - publish_symbols: ${{ parameters.publish_symbols }} - cmake_build_type: ${{ parameters.cmake_build_type }} - - - template: py-win-gpu.yml - parameters: - MACHINE_POOL: 'onnxruntime-Win2022-GPU-dml-A10' - PYTHON_VERSION: '3.10' - EP_BUILD_FLAGS: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos - ENV_SETUP_SCRIPT: setup_env.bat - EP_NAME: directml - publish_symbols: ${{ parameters.publish_symbols }} - cmake_build_type: ${{ parameters.cmake_build_type }} - - - template: py-win-gpu.yml - parameters: - MACHINE_POOL: 'onnxruntime-Win2022-GPU-dml-A10' - PYTHON_VERSION: '3.11' - EP_BUILD_FLAGS: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos - ENV_SETUP_SCRIPT: setup_env.bat - EP_NAME: directml - publish_symbols: ${{ parameters.publish_symbols }} - cmake_build_type: ${{ parameters.cmake_build_type }} - - - template: py-win-gpu.yml - parameters: - MACHINE_POOL: 'onnxruntime-Win2022-GPU-dml-A10' - PYTHON_VERSION: '3.12' - EP_BUILD_FLAGS: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos - ENV_SETUP_SCRIPT: setup_env.bat - EP_NAME: directml - publish_symbols: ${{ parameters.publish_symbols }} - cmake_build_type: ${{ parameters.cmake_build_type }} - - - template: py-win-gpu.yml - parameters: - MACHINE_POOL: 'onnxruntime-Win2022-GPU-dml-A10' - PYTHON_VERSION: '3.13' - EP_BUILD_FLAGS: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos - ENV_SETUP_SCRIPT: setup_env.bat - EP_NAME: directml - publish_symbols: ${{ parameters.publish_symbols }} - cmake_build_type: ${{ parameters.cmake_build_type }} - - ${{ if eq(parameters.enable_mac_cpu, true) }}: - stage: Python_Packaging_MacOS dependsOn: [] @@ -395,9 +267,9 @@ stages: inputs: versionSpec: $(PythonVersion) - - template: use-xcode-version.yml + - template: ../templates/use-xcode-version.yml - - template: download-deps.yml + - template: ../templates/download-deps.yml - task: PythonScript@0 displayName: 'Update deps.txt' @@ -437,7 +309,7 @@ stages: inputs: ArtifactName: onnxruntime - - template: component-governance-component-detection-steps.yml + - template: ../templates/component-governance-component-detection-steps.yml parameters: condition: 'succeeded' @@ -446,7 +318,7 @@ stages: - stage: Python_Packaging_Linux_ARM dependsOn: [] jobs: - - template: py-linux.yml + - template: ../templates/py-linux.yml parameters: arch: 'aarch64' machine_pool: 'onnxruntime-linux-ARM64-CPU-2019' @@ -457,30 +329,18 @@ stages: - stage: Python_Packaging_Linux_CPU dependsOn: [] jobs: - - template: py-linux.yml + - template: ../templates/py-linux.yml parameters: arch: 'x86_64' machine_pool: 'onnxruntime-Ubuntu2204-AMD-CPU-Large' extra_build_arg: ${{ parameters.build_py_parameters }} cmake_build_type: ${{ parameters.cmake_build_type }} - - - ${{ if eq(parameters.enable_linux_gpu, true) }}: - - template: py-linux-gpu.yml - parameters: - arch: 'x86_64' - machine_pool: 'onnxruntime-Ubuntu2204-AMD-CPU-Large' - docker_base_image: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda11_x64_almalinux8_gcc11:20241020.1 - extra_build_arg: ${{ parameters.build_py_parameters }} - cmake_build_type: ${{ parameters.cmake_build_type }} - trt_version: '10.4.0.26-1.cuda11.8' - cuda_version: '11.8' - - ${{ if eq(parameters.enable_windows_arm64_qnn, true) }}: - stage: Python_Packaging_Windows_ARM64_QNN dependsOn: [] jobs: - - template: py-win-arm64-qnn.yml + - template: ../templates/py-win-arm64-qnn.yml parameters: MACHINE_POOL: 'onnxruntime-qnn-windows-vs-2022-arm64' QNN_SDK: ${{ parameters.qnn_sdk_version }} @@ -490,7 +350,7 @@ stages: - stage: Python_Packaging_Windows_arm64ec_QNN dependsOn: [] jobs: - - template: py-win-arm64ec-qnn.yml + - template: ../templates/py-win-arm64ec-qnn.yml parameters: MACHINE_POOL: 'Onnxruntime-QNNEP-Windows-2022-CPU' QNN_SDK: ${{ parameters.qnn_sdk_version }} @@ -500,7 +360,7 @@ stages: - stage: Python_Packaging_Windows_x64_QNN dependsOn: [] jobs: - - template: py-win-x64-qnn.yml + - template: ../templates/py-win-x64-qnn.yml parameters: MACHINE_POOL: 'Onnxruntime-QNNEP-Windows-2022-CPU' QNN_SDK: ${{ parameters.qnn_sdk_version }} @@ -510,7 +370,7 @@ stages: - stage: Python_Packaging_Linux_x64_QNN dependsOn: [] jobs: - - template: py-linux-qnn.yml + - template: ../templates/py-linux-qnn.yml parameters: machine_pool: 'onnxruntime-Ubuntu2204-AMD-CPU' extra_build_arg: ${{ parameters.build_py_parameters }} diff --git a/tools/ci_build/github/azure-pipelines/stages/py-cuda-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/stages/py-gpu-packaging-stage.yml similarity index 68% rename from tools/ci_build/github/azure-pipelines/stages/py-cuda-packaging-stage.yml rename to tools/ci_build/github/azure-pipelines/stages/py-gpu-packaging-stage.yml index ae18687cb9..1ae95a2961 100644 --- a/tools/ci_build/github/azure-pipelines/stages/py-cuda-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/stages/py-gpu-packaging-stage.yml @@ -5,15 +5,20 @@ parameters: type: string default: '' -- name: enable_linux_gpu - displayName: 'Whether Linux GPU package is built.' +- name: enable_linux_cuda + displayName: 'Whether Linux CUDA package is built.' type: boolean - default: true + default: false -- name: enable_windows_gpu - displayName: 'Whether Windows GPU package is built.' +- name: enable_windows_cuda + displayName: 'Whether Windows CUDA package is built.' type: boolean - default: true + default: false + +- name: enable_windows_dml + displayName: 'Whether Windows DML package is built.' + type: boolean + default: false # TODO: Now the Windows jobs use a different cmake build type. Consider to merge it. - name: cmake_build_type @@ -34,16 +39,6 @@ parameters: - 11.8 - 12.2 -- name: SpecificArtifact - displayName: Use Specific Artifact - type: boolean - default: false - -- name: BuildId - displayName: Specific Artifact's BuildId - type: string - default: '0' - - name: PythonVersions type: object displayName: 'Python versions to build' @@ -53,23 +48,25 @@ parameters: - '3.12' - '3.13' +- name: publish_symbols + type: boolean + default: false + stages: - - ${{ if eq(parameters.enable_windows_gpu, true) }}: + - ${{ if eq(parameters.enable_windows_cuda, true) }}: - ${{ each python_version in parameters.PythonVersions }}: - - template: ../templates/py-win-gpu.yml + - template: py-win-gpu-stage.yml parameters: PYTHON_VERSION: ${{ python_version }} EP_NAME: gpu CudaVersion: ${{ parameters.cuda_version }} - SpecificArtifact: ${{ parameters.SpecificArtifact }} - BuildId: ${{ parameters.BuildId }} ${{ if eq(parameters.cuda_version, '11.8') }}: EP_BUILD_FLAGS: --enable_lto --use_tensorrt --tensorrt_home=$(Agent.TempDirectory)\TensorRT-10.4.0.26.Windows10.x86_64.cuda-11.8 --cuda_home=$(Agent.TempDirectory)\v11.8 --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80" ${{ if eq(parameters.cuda_version, '12.2') }}: EP_BUILD_FLAGS: --enable_lto --use_tensorrt --tensorrt_home=$(Agent.TempDirectory)\TensorRT-10.4.0.26.Windows10.x86_64.cuda-12.6 --cuda_home=$(Agent.TempDirectory)\v12.2 --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80" - - ${{ if eq(parameters.enable_linux_gpu, true) }}: - - template: ../templates/py-linux-gpu.yml + - ${{ if eq(parameters.enable_linux_cuda, true) }}: + - template: py-linux-gpu-stage.yml parameters: arch: 'x86_64' machine_pool: 'onnxruntime-Ubuntu2204-AMD-CPU-Large' @@ -82,3 +79,15 @@ stages: ${{ if eq(parameters.cuda_version, '12.2') }}: docker_base_image: onnxruntimebuildcache.azurecr.io/internal/azureml/onnxruntime/build/cuda12_x64_ubi8_gcc12:20241020.1 trt_version: 10.4.0.26-1.cuda12.6 + + - ${{ if eq(parameters.enable_windows_dml, true) }}: + - ${{ each python_version in parameters.PythonVersions }}: + - template: py-win-gpu-stage.yml + parameters: + MACHINE_POOL: 'onnxruntime-Win2022-GPU-dml-A10' + PYTHON_VERSION: ${{ python_version }} + EP_BUILD_FLAGS: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos + ENV_SETUP_SCRIPT: setup_env.bat + EP_NAME: directml + publish_symbols: ${{ parameters.publish_symbols }} + cmake_build_type: ${{ parameters.cmake_build_type }} \ No newline at end of file diff --git a/tools/ci_build/github/azure-pipelines/templates/py-linux-gpu.yml b/tools/ci_build/github/azure-pipelines/stages/py-linux-gpu-stage.yml similarity index 63% rename from tools/ci_build/github/azure-pipelines/templates/py-linux-gpu.yml rename to tools/ci_build/github/azure-pipelines/stages/py-linux-gpu-stage.yml index d19472bcba..f9053cba56 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-linux-gpu.yml +++ b/tools/ci_build/github/azure-pipelines/stages/py-linux-gpu-stage.yml @@ -41,7 +41,27 @@ stages: timeoutInMinutes: 240 workspace: clean: all - pool: ${{ parameters.machine_pool }} + pool: + name: ${{ parameters.machine_pool }} + os: linux + templateContext: + codeSignValidation: + enabled: true + break: true + psscriptanalyzer: + enabled: true + sdl: + binskim: + enabled: true + scanOutputDirectoryOnly: true + targetPathPattern: '\".*.so\"' + outputs: + - output: pipelineArtifact + targetPath: $(Build.ArtifactStagingDirectory)/dist + artifactName: onnxruntime_gpu + - output: pipelineArtifact + targetPath: $(Build.ArtifactStagingDirectory)/${{ parameters.cmake_build_type }} + artifactName: linux_gpu_wheel_${{ parameters.arch }} variables: # The build machine pool doesn't have dotnet, so it can't run CG. - name: skipComponentGovernanceDetection @@ -56,9 +76,9 @@ stages: clean: true submodules: recursive - - template: set-nightly-build-option-variable-step.yml + - template: ../templates/set-nightly-build-option-variable-step.yml - - template: get-docker-image-steps.yml + - template: ../templates/get-docker-image-steps.yml parameters: Dockerfile: tools/ci_build/github/linux/docker/inference/${{ parameters.arch }}/python/cuda/Dockerfile Context: tools/ci_build/github/linux/docker/inference/${{ parameters.arch }}/python/cuda @@ -73,17 +93,18 @@ stages: filePath: tools/ci_build/github/linux/run_python_dockerbuild.sh arguments: -i onnxruntimecuda${{ replace(parameters.cuda_version, '.', '') }}xtrt86build${{ parameters.arch }} -d "GPU" -c ${{ parameters.cmake_build_type }} $(extra_build_args) - - task: PublishBuildArtifacts@1 - displayName: 'Publish Artifact: ONNXRuntime python wheel' - inputs: - PathtoPublish: '$(Build.BinariesDirectory)/dist' - ArtifactName: onnxruntime_gpu - - - task: PublishPipelineArtifact@0 - displayName: 'Publish Test Binaries' - inputs: - artifactName: 'drop-linux-gpu-${{ parameters.arch }}' - targetPath: '$(Build.BinariesDirectory)/Release' + - script: | + set -e -x + mv $(Build.BinariesDirectory)/${{ parameters.cmake_build_type }} ./${{ parameters.cmake_build_type }} + mv $(Build.BinariesDirectory)/dist ./dist + pushd dist + find . -name \*.whl -exec unzip -qq -o {} \; + popd + pushd ${{ parameters.cmake_build_type }} + find . -name \*.whl -exec unzip -qq -o {} \; + popd + workingDirectory: '$(Build.ArtifactStagingDirectory)' + displayName: 'Move files' - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 diff --git a/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml b/tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml similarity index 93% rename from tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml rename to tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml index 71500e4ef9..0cbcd2b743 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml +++ b/tools/ci_build/github/azure-pipelines/stages/py-win-gpu-stage.yml @@ -28,16 +28,6 @@ parameters: - 11.8 - 12.2 -- name: SpecificArtifact - displayName: Use Specific Artifact - type: boolean - default: false - -- name: BuildId - displayName: Specific Artifact's BuildId - type: string - default: '0' - - name: cmake_build_type type: string displayName: 'Linux packages cmake build type. Linux Only.' @@ -75,7 +65,7 @@ stages: clean: true submodules: recursive - - template: telemetry-steps.yml + - template: ../templates/telemetry-steps.yml - task: UsePythonVersion@0 inputs: @@ -89,10 +79,10 @@ stages: tsaConfigFilePath: '$(Build.SourcesDirectory)\.config\tsaoptions.json' appendSourceBranchName: false - - template: download-deps.yml + - template: ../templates/download-deps.yml - ${{ if ne(parameters.ENV_SETUP_SCRIPT, '') }}: - - template: jobs/set-winenv.yml + - template: ../templates/jobs/set-winenv.yml parameters: EnvSetupScript: ${{ parameters.ENV_SETUP_SCRIPT }} ${{ if or(contains(parameters.EP_BUILD_FLAGS, 'use_cuda'), contains(parameters.EP_BUILD_FLAGS, 'use_tensorrt')) }}: @@ -101,7 +91,7 @@ stages: DownloadTRT: true - ${{ if eq(parameters.ENV_SETUP_SCRIPT, '') }}: - - template: jobs/download_win_gpu_library.yml + - template: ../templates/jobs/download_win_gpu_library.yml parameters: CudaVersion: ${{ parameters.CudaVersion }} ${{ if or(contains(parameters.EP_BUILD_FLAGS, 'use_cuda'), contains(parameters.EP_BUILD_FLAGS, 'use_tensorrt')) }}: @@ -123,7 +113,7 @@ stages: workingDirectory: '$(Build.BinariesDirectory)' arguments: -cpu_arch x64 -install_prefix $(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}\installed -build_config ${{ parameters.cmake_build_type }} - - template: set-nightly-build-option-variable-step.yml + - template: ../templates/set-nightly-build-option-variable-step.yml - task: PythonScript@0 displayName: 'Generate cmake config' @@ -153,7 +143,7 @@ stages: workingDirectory: '$(Build.BinariesDirectory)' # Esrp signing - - template: win-esrp-dll.yml + - template: ../templates/win-esrp-dll.yml parameters: FolderPath: '$(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}\${{ parameters.cmake_build_type }}\onnxruntime\capi' DisplayName: 'ESRP - Sign Native dlls' @@ -216,7 +206,7 @@ stages: GdnPublishTsaOnboard: false GdnPublishTsaConfigFile: '$(Build.sourcesDirectory)\.gdn\.gdntsa' - - template: component-governance-component-detection-steps.yml + - template: ../templates/component-governance-component-detection-steps.yml parameters: condition: 'succeeded' @@ -243,13 +233,11 @@ stages: addToPath: true architecture: 'x64' - - template: flex-downloadPipelineArtifact.yml + - template: ../templates/flex-downloadPipelineArtifact.yml parameters: ArtifactName: onnxruntime_${{ parameters.EP_NAME }} StepName: 'Download Pipeline Artifact - Windows GPU Build' TargetPath: '$(Build.ArtifactStagingDirectory)' - SpecificArtifact: ${{ parameters.SpecificArtifact }} - BuildId: ${{ parameters.BuildId }} - task: PowerShell@2 displayName: 'Install ONNX' diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-linux-test-cuda.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-linux-test-cuda.yml index 4ca462bf96..6a74d0e7be 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-linux-test-cuda.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-linux-test-cuda.yml @@ -61,7 +61,7 @@ jobs: # The private ADO project - ${{ if eq(variables['System.CollectionId'], 'bc038106-a83b-4dab-9dd3-5a41bc58f34c') }}: - download: build # pipeline resource identifier. - artifact: 'drop-linux-gpu-${{ parameters.arch }}' + artifact: 'linux_gpu_wheel_${{ parameters.arch }}' - download: build # pipeline resource identifier. artifact: 'onnxruntime${{ parameters.python_wheel_suffix }}' @@ -69,7 +69,7 @@ jobs: - bash: | set -e -x ls $(Pipeline.Workspace)/build - mv "$(Pipeline.Workspace)/build/drop-linux-gpu-${{ parameters.arch }}" $(Build.BinariesDirectory)/${{parameters.cmake_build_type}} + mv "$(Pipeline.Workspace)/build/linux_gpu_wheel_${{ parameters.arch }}" $(Build.BinariesDirectory)/${{parameters.cmake_build_type}} mv "$(Pipeline.Workspace)/build/onnxruntime${{ parameters.python_wheel_suffix }}" "$(Build.BinariesDirectory)/whl" cp -r "$(Build.BinariesDirectory)/whl" $(Build.BinariesDirectory)/tmp find "$(Build.BinariesDirectory)/tmp" -name '*.whl' -exec bash -c 'unzip -d "${1%.*}" "$1"' _ {} \;