saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-28 20:11:22 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Refactor Python CUDA packaging pipeline to fix random hangs in building (#19989)

Browse source 
### Description
1. Move building on CPU machine.
2. Optimize the pipeline
3. Since there isn't official ONNX package for python 12, the python 12
test stage uses the packages built with ONNX source in build stage.


### Motivation and Context
1. Resolve the random hang in compilation
4. Save a lot of GPU resources.

---------
This commit is contained in:
Yi Zhang 2024-03-22 09:16:00 +08:00 committed by GitHub
parent dafbef3a21
commit cd6d3aea45
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 346 additions and 275 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

13
tools/ci_build/github/azure-pipelines/py-cuda-packaging-pipeline.yml
View file

@ -21,6 +21,15 @@ parameters:
values:
- 11.8
- 12.2
- name: SpecificArtifact
displayName: Use Specific Artifact
type: boolean
default: false
- name: BuildId
displayName: Specific Artifact's BuildId
type: string
default: '0'
resources:
repositories:
@ -36,4 +45,6 @@ stages:
enable_linux_gpu: ${{ parameters.enable_linux_gpu }}
enable_windows_gpu: ${{ parameters.enable_windows_gpu }}
cmake_build_type: ${{ parameters.cmake_build_type }}
cuda_version: ${{ parameters.cuda_version }}
cuda_version: ${{ parameters.cuda_version }}
SpecificArtifact: ${{ parameters.SpecificArtifact }}
BuildId: ${{ parameters.BuildId }}

96
tools/ci_build/github/azure-pipelines/stages/py-cuda-packaging-stage.yml
View file

@ -34,72 +34,40 @@ parameters:
- 11.8
- 12.2
- name: SpecificArtifact
displayName: Use Specific Artifact
type: boolean
default: false
- name: BuildId
displayName: Specific Artifact's BuildId
type: string
default: '0'
- name: PythonVersions
type: object
displayName: 'Python versions to build'
default:
- '3.8'
- '3.9'
- '3.10'
- '3.11'
- '3.12'
stages:
- stage: Python_Packaging
dependsOn: []
variables:
- name: docker_base_image
${{ if eq(parameters.cuda_version, '11.8') }}:
value: nvidia/cuda:11.8.0-cudnn8-devel-ubi8
${{ if eq(parameters.cuda_version, '12.2') }}:
value: nvidia/cuda:12.2.2-cudnn8-devel-ubi8
- name: linux_trt_version
${{ if eq(parameters.cuda_version, '11.8') }}:
value: 8.6.1.6-1.cuda11.8
${{ if eq(parameters.cuda_version, '12.2') }}:
value: 8.6.1.6-1.cuda12.0
- name: win_trt_home
${{ if eq(parameters.cuda_version, '11.8') }}:
value: $(Agent.TempDirectory)\TensorRT-8.6.1.6.Windows10.x86_64.cuda-11.8
${{ if eq(parameters.cuda_version, '12.2') }}:
value: $(Agent.TempDirectory)\TensorRT-8.6.1.6.Windows10.x86_64.cuda-12.0
- name: win_cuda_home
${{ if eq(parameters.cuda_version, '11.8') }}:
value: $(Agent.TempDirectory)\v11.8
${{ if eq(parameters.cuda_version, '12.2') }}:
value: $(Agent.TempDirectory)\v12.2
jobs:
- ${{ if eq(parameters.enable_windows_gpu, true) }}:
- ${{ each python_version in parameters.PythonVersions }}:
- template: ../templates/py-win-gpu.yml
parameters:
MACHINE_POOL: 'onnxruntime-Win2022-GPU-T4'
PYTHON_VERSION: '3.8'
EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home=${{ variables.win_trt_home }} --cuda_home=${{ variables.win_cuda_home }} --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80"
PYTHON_VERSION: ${{ python_version }}
EP_NAME: gpu
CudaVersion: ${{ parameters.cuda_version }}
- template: ../templates/py-win-gpu.yml
parameters:
MACHINE_POOL: 'onnxruntime-Win2022-GPU-T4'
PYTHON_VERSION: '3.9'
EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home=${{ variables.win_trt_home }} --cuda_home=${{ variables.win_cuda_home }} --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80"
EP_NAME: gpu
CudaVersion: ${{ parameters.cuda_version }}
- template: ../templates/py-win-gpu.yml
parameters:
MACHINE_POOL: 'onnxruntime-Win2022-GPU-T4'
PYTHON_VERSION: '3.10'
EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home=${{ variables.win_trt_home }} --cuda_home=${{ variables.win_cuda_home }} --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80"
EP_NAME: gpu
CudaVersion: ${{ parameters.cuda_version }}
- template: ../templates/py-win-gpu.yml
parameters:
MACHINE_POOL: 'onnxruntime-Win2022-GPU-T4'
PYTHON_VERSION: '3.11'
EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home=${{ variables.win_trt_home }} --cuda_home=${{ variables.win_cuda_home }} --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80"
EP_NAME: gpu
CudaVersion: ${{ parameters.cuda_version }}
- template: ../templates/py-win-gpu.yml
parameters:
MACHINE_POOL: 'onnxruntime-Win2022-GPU-T4'
PYTHON_VERSION: '3.12'
EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home=${{ variables.win_trt_home }} --cuda_home=${{ variables.win_cuda_home }} --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80"
EP_NAME: gpu
CudaVersion: ${{ parameters.cuda_version }}
SpecificArtifact: ${{ parameters.SpecificArtifact }}
BuildId: ${{ parameters.BuildId }}
${{ if eq(parameters.cuda_version, '11.8') }}:
EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home=$(Agent.TempDirectory)\TensorRT-8.6.1.6.Windows10.x86_64.cuda-11.8 --cuda_home=$(Agent.TempDirectory)\v11.8 --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80"
${{ if eq(parameters.cuda_version, '12.2') }}:
EP_BUILD_FLAGS: --use_tensorrt --tensorrt_home=$(Agent.TempDirectory)\TensorRT-8.6.1.6.Windows10.x86_64.cuda-12.0 --cuda_home=$(Agent.TempDirectory)\v12.2 --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52;60;61;70;75;80"
- ${{ if eq(parameters.enable_linux_gpu, true) }}:
- template: ../templates/py-linux-gpu.yml
@ -108,6 +76,10 @@ stages:
machine_pool: 'onnxruntime-Ubuntu2204-AMD-CPU'
extra_build_arg: ${{ parameters.build_py_parameters }}
cmake_build_type: ${{ parameters.cmake_build_type }}
docker_base_image: ${{ variables.docker_base_image }}
trt_version: ${{ variables.linux_trt_version }}
cuda_version: ${{ parameters.cuda_version }}
${{ if eq(parameters.cuda_version, '11.8') }}:
docker_base_image: nvidia/cuda:11.8.0-cudnn8-devel-ubi8
trt_version: 8.6.1.6-1.cuda11.8
${{ if eq(parameters.cuda_version, '12.2') }}:
docker_base_image: nvidia/cuda:12.2.2-cudnn8-devel-ubi8
trt_version: 8.6.1.6-1.cuda12.0

106
tools/ci_build/github/azure-pipelines/templates/py-linux-gpu.yml
View file

@ -35,62 +35,66 @@ parameters:
values:
- 11.8
- 12.2
jobs:
- job: Linux_py_GPU_Wheels_${{ parameters.arch }}
timeoutInMinutes: 240
workspace:
clean: all
pool: ${{ parameters.machine_pool }}
variables:
# The build machine pool doesn't have dotnet, so it can't run CG.
- name: skipComponentGovernanceDetection
value: true
- name: extra_build_args
${{ if ne(parameters.extra_build_arg, '') }}:
value: -x ${{ parameters.extra_build_arg }}
${{ if eq(parameters.extra_build_arg, '') }}:
value: ''
steps:
- checkout: self
clean: true
submodules: recursive
- template: set-nightly-build-option-variable-step.yml
stages:
- stage: Linux_py_GPU_Wheels_${{ parameters.arch }}
dependsOn: []
jobs:
- job: Linux_py_GPU_Wheels_${{ parameters.arch }}
timeoutInMinutes: 240
workspace:
clean: all
pool: ${{ parameters.machine_pool }}
variables:
# The build machine pool doesn't have dotnet, so it can't run CG.
- name: skipComponentGovernanceDetection
value: true
- name: extra_build_args
${{ if ne(parameters.extra_build_arg, '') }}:
value: -x ${{ parameters.extra_build_arg }}
${{ if eq(parameters.extra_build_arg, '') }}:
value: ''
steps:
- checkout: self
clean: true
submodules: recursive
- template: get-docker-image-steps.yml
parameters:
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cuda
Context: tools/ci_build/github/linux/docker
DockerBuildArgs: "
--network=host
--build-arg BASEIMAGE=${{ parameters.docker_base_image }}
--build-arg TRT_VERSION=${{ parameters.trt_version }}
--build-arg BUILD_UID=$( id -u )
--build-arg PLATFORM=${{ parameters.arch }}
"
Repository: onnxruntimecuda${{ replace(parameters.cuda_version, '.', '') }}xtrt86build${{ parameters.arch }}
- template: set-nightly-build-option-variable-step.yml
- template: get-docker-image-steps.yml
parameters:
Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2_28_cuda
Context: tools/ci_build/github/linux/docker
DockerBuildArgs: "
--network=host
--build-arg BASEIMAGE=${{ parameters.docker_base_image }}
--build-arg TRT_VERSION=${{ parameters.trt_version }}
--build-arg BUILD_UID=$( id -u )
--build-arg PLATFORM=${{ parameters.arch }}
"
Repository: onnxruntimecuda${{ replace(parameters.cuda_version, '.', '') }}xtrt86build${{ parameters.arch }}
- task: Bash@3
displayName: 'Build Python Wheel'
inputs:
targetType: filePath
filePath: tools/ci_build/github/linux/run_python_dockerbuild.sh
arguments: -i onnxruntimecuda${{ replace(parameters.cuda_version, '.', '') }}xtrt86build${{ parameters.arch }} -d "GPU" -c ${{ parameters.cmake_build_type }} $(extra_build_args)
- task: Bash@3
displayName: 'Build Python Wheel'
inputs:
targetType: filePath
filePath: tools/ci_build/github/linux/run_python_dockerbuild.sh
arguments: -i onnxruntimecuda${{ replace(parameters.cuda_version, '.', '') }}xtrt86build${{ parameters.arch }} -d "GPU" -c ${{ parameters.cmake_build_type }} $(extra_build_args)
- task: PublishBuildArtifacts@1
displayName: 'Publish Artifact: ONNXRuntime python wheel'
inputs:
PathtoPublish: '$(Build.BinariesDirectory)/dist'
ArtifactName: onnxruntime_gpu
- task: PublishBuildArtifacts@1
displayName: 'Publish Artifact: ONNXRuntime python wheel'
inputs:
PathtoPublish: '$(Build.BinariesDirectory)/dist'
ArtifactName: onnxruntime_gpu
- task: PublishPipelineArtifact@0
displayName: 'Publish Test Binaries'
inputs:
artifactName: 'drop-linux-gpu-${{ parameters.arch }}'
targetPath: '$(Build.BinariesDirectory)/Release'
- task: PublishPipelineArtifact@0
displayName: 'Publish Test Binaries'
inputs:
artifactName: 'drop-linux-gpu-${{ parameters.arch }}'
targetPath: '$(Build.BinariesDirectory)/Release'
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
displayName: 'Clean Agent Directories'
condition: always()
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
displayName: 'Clean Agent Directories'
condition: always()

406
tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml
View file

@ -1,8 +1,4 @@
parameters:
- name: MACHINE_POOL
type: string
- name: EP_NAME
type: string
@ -27,169 +23,257 @@ parameters:
values:
- 11.8
- 12.2
jobs:
- job: Win_py_${{ parameters.EP_NAME }}_Wheels_${{ replace(parameters.PYTHON_VERSION,'.','_') }}
timeoutInMinutes: 240
workspace:
clean: all
pool:
name: ${{ parameters.MACHINE_POOL }}
# demands:
# - ImageVersionOverride -equals 1.0.367516
variables:
GRADLE_OPTS: '-Dorg.gradle.daemon=false'
VSGenerator: 'Visual Studio 17 2022'
CUDA_MODULE_LOADING: 'LAZY'
steps:
- checkout: self
clean: true
submodules: recursive
- template: telemetry-steps.yml
- name: SpecificArtifact
displayName: Use Specific Artifact
type: boolean
default: false
- task: UsePythonVersion@0
inputs:
versionSpec: ${{ parameters.PYTHON_VERSION }}
addToPath: true
architecture: 'x64'
- name: BuildId
displayName: Specific Artifact's BuildId
type: string
default: '0'
- task: onebranch.pipeline.tsaoptions@1
displayName: 'OneBranch TSAOptions'
inputs:
tsaConfigFilePath: '$(Build.SourcesDirectory)\.config\tsaoptions.json'
appendSourceBranchName: false
stages:
- stage: Win_py_${{ parameters.EP_NAME }}_Wheels_${{ replace(parameters.PYTHON_VERSION,'.','_') }}_Build
dependsOn: []
jobs:
- job: Win_py_${{ parameters.EP_NAME }}_Wheels_${{ replace(parameters.PYTHON_VERSION,'.','_') }}_Build
timeoutInMinutes: 120
workspace:
clean: all
pool:
name: onnxruntime-Win-CPU-2022
# demands:
# - ImageVersionOverride -equals 1.0.367516
variables:
GRADLE_OPTS: '-Dorg.gradle.daemon=false'
VSGenerator: 'Visual Studio 17 2022'
CUDA_MODULE_LOADING: 'LAZY'
steps:
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
displayName: 'Clean Agent Directories'
condition: always()
- task: PythonScript@0
inputs:
scriptSource: inline
script: |
import sys
np_version = 'numpy==1.21.6' if sys.version_info < (3, 11) else 'numpy==1.24.2'
import subprocess
subprocess.call(['pip', 'install', '-q', 'setuptools', 'wheel', np_version])
workingDirectory: '$(Build.BinariesDirectory)'
displayName: 'Install python modules'
- checkout: self
clean: true
submodules: recursive
- template: download-deps.yml
- template: telemetry-steps.yml
- ${{ if ne(parameters.ENV_SETUP_SCRIPT, '') }}:
- template: jobs/set-winenv.yml
- task: UsePythonVersion@0
inputs:
versionSpec: ${{ parameters.PYTHON_VERSION }}
addToPath: true
architecture: 'x64'
- task: onebranch.pipeline.tsaoptions@1
displayName: 'OneBranch TSAOptions'
inputs:
tsaConfigFilePath: '$(Build.SourcesDirectory)\.config\tsaoptions.json'
appendSourceBranchName: false
- task: PythonScript@0
inputs:
scriptSource: inline
script: |
import sys
np_version = 'numpy==1.21.6' if sys.version_info < (3, 11) else 'numpy==1.26'
import subprocess
try:
subprocess.check_call(['pip', 'install', '-q', 'setuptools', 'wheel', np_version])
except subprocess.CalledProcessError:
sys.exit(1)
workingDirectory: '$(Build.BinariesDirectory)'
displayName: 'Install python modules'
- template: download-deps.yml
- ${{ if ne(parameters.ENV_SETUP_SCRIPT, '') }}:
- template: jobs/set-winenv.yml
parameters:
EnvSetupScript: ${{ parameters.ENV_SETUP_SCRIPT }}
${{ if or(contains(parameters.EP_BUILD_FLAGS, 'use_cuda'), contains(parameters.EP_BUILD_FLAGS, 'use_tensorrt')) }}:
DownloadCUDA: true
- ${{ if eq(parameters.ENV_SETUP_SCRIPT, '') }}:
- template: jobs/download_win_gpu_library.yml
parameters:
CudaVersion: ${{ parameters.CudaVersion }}
${{ if or(contains(parameters.EP_BUILD_FLAGS, 'use_cuda'), contains(parameters.EP_BUILD_FLAGS, 'use_tensorrt')) }}:
DownloadCUDA: true
${{ if contains(parameters.EP_BUILD_FLAGS, 'use_tensorrt') }}:
DownloadTRT: true
- task: PythonScript@0
displayName: 'Update deps.txt'
inputs:
scriptPath: $(Build.SourcesDirectory)/tools/ci_build/replace_urls_in_deps.py
arguments: --new_dir $(Build.BinariesDirectory)/deps
workingDirectory: $(Build.BinariesDirectory)
- task: PowerShell@2
displayName: 'Install ONNX'
inputs:
filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/windows/install_third_party_deps.ps1'
workingDirectory: '$(Build.BinariesDirectory)'
arguments: -cpu_arch x64 -install_prefix $(Build.BinariesDirectory)\RelWithDebInfo\installed -build_config RelWithDebInfo
# it could be removed once there's onnx wheel for python 3.12
- ${{ if eq(parameters.PYTHON_VERSION, '3.12') }}:
- task: PublishPipelineArtifact@1
displayName: 'Publish Artifact: ONNX python 12 wheel'
inputs:
targetPath: '$(Agent.TempDirectory)\onnx\onnx-1.15.0\dist\'
publishLocation: 'pipeline'
artifactName: onnx_py12_wheel
- template: set-nightly-build-option-variable-step.yml
- task: PythonScript@0
displayName: 'Generate cmake config'
inputs:
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
arguments: >
--config RelWithDebInfo
--build_dir $(Build.BinariesDirectory)
--skip_submodule_sync
--cmake_generator "$(VSGenerator)"
--enable_pybind
--enable_onnx_tests
--parallel --use_binskim_compliant_compile_flags --update
$(TelemetryOption) ${{ parameters.BUILD_PY_PARAMETERS }} ${{ parameters.EP_BUILD_FLAGS }}
workingDirectory: '$(Build.BinariesDirectory)'
# building with build.py so the parallelization parameters are added to the msbuild command
- task: PythonScript@0
displayName: 'Build'
inputs:
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
arguments: >
--config RelWithDebInfo
--build_dir $(Build.BinariesDirectory)
--parallel --build
$(TelemetryOption) ${{ parameters.BUILD_PY_PARAMETERS }} ${{ parameters.EP_BUILD_FLAGS }}
workingDirectory: '$(Build.BinariesDirectory)'
# Esrp signing
- template: win-esrp-dll.yml
parameters:
FolderPath: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\onnxruntime\capi'
DisplayName: 'ESRP - Sign Native dlls'
DoEsrp: true
Pattern: '*.pyd,*.dll'
- task: PythonScript@0
displayName: 'Build wheel'
inputs:
scriptPath: '$(Build.SourcesDirectory)\setup.py'
arguments: 'bdist_wheel ${{ parameters.BUILD_PY_PARAMETERS }} $(NightlyBuildOption) --wheel_name_suffix=${{ parameters.EP_NAME }}'
workingDirectory: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo'
- task: CopyFiles@2
displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
inputs:
SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\dist'
Contents: '*.whl'
TargetFolder: '$(Build.ArtifactStagingDirectory)'
- task: PublishBuildArtifacts@1
displayName: 'Publish Artifact: ONNXRuntime python wheel'
inputs:
ArtifactName: onnxruntime_${{ parameters.EP_NAME }}
- script: |
7z x *.whl
workingDirectory: '$(Build.ArtifactStagingDirectory)'
displayName: 'unzip the package'
- task: CredScan@3
displayName: 'Run CredScan'
inputs:
debugMode: false
continueOnError: true
- task: BinSkim@4
displayName: 'Run BinSkim'
inputs:
AnalyzeTargetGlob: '+:file|$(Build.ArtifactStagingDirectory)\**\*.dll;-:file|$(Build.ArtifactStagingDirectory)\**\DirectML.dll'
- stage: Win_py_${{ parameters.EP_NAME }}_Wheels_${{ replace(parameters.PYTHON_VERSION,'.','_') }}_Tests
dependsOn: Win_py_${{ parameters.EP_NAME }}_Wheels_${{ replace(parameters.PYTHON_VERSION,'.','_') }}_Build
jobs:
- job: Win_py_${{ parameters.EP_NAME }}_Wheels_${{ replace(parameters.PYTHON_VERSION,'.','_') }}_Tests
workspace:
clean: all
pool:
name: onnxruntime-Win2022-GPU-T4
steps:
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
displayName: 'Clean Agent Directories'
condition: always()
- checkout: self
clean: true
submodules: none
- task: UsePythonVersion@0
inputs:
versionSpec: ${{ parameters.PYTHON_VERSION }}
addToPath: true
architecture: 'x64'
- template: flex-downloadPipelineArtifact.yml
parameters:
EnvSetupScript: ${{ parameters.ENV_SETUP_SCRIPT }}
${{ if or(contains(parameters.EP_BUILD_FLAGS, 'use_cuda'), contains(parameters.EP_BUILD_FLAGS, 'use_tensorrt')) }}:
DownloadCUDA: true
ArtifactName: "onnxruntime_${{ parameters.EP_NAME }}"
StepName: 'Download Pipeline Artifact - Windows GPU Build'
TargetPath: '$(Build.ArtifactStagingDirectory)'
SpecificArtifact: ${{ parameters.SpecificArtifact }}
BuildId: ${{ parameters.BuildId }}
- ${{ if eq(parameters.ENV_SETUP_SCRIPT, '') }}:
- template: jobs/download_win_gpu_library.yml
# It could be remove once there's onnx wheel for python 3.12
- ${{ if eq(parameters.PYTHON_VERSION, '3.12') }}:
- template: flex-downloadPipelineArtifact.yml
parameters:
ArtifactName: "onnx_py12_wheel"
StepName: 'Download Pipeline Artifact - Onnx Python12 wheel'
TargetPath: '$(Agent.TempDirectory)\onnx\'
SpecificArtifact: ${{ parameters.SpecificArtifact }}
BuildId: ${{ parameters.BuildId }}
- powershell: |
python -m pip install upgrade pip
Get-ChildItem -Path $(Agent.TempDirectory)\onnx\*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname tabulate}
python -m pip install pytest
workingDirectory: '$(Build.SourcesDirectory)'
displayName: 'Install ONNX and pytest'
- ${{ else }}:
- powershell: |
pushd onnxruntime/test/python
python -m pip install --upgrade pip
python -m pip install -r requirements.txt
popd
workingDirectory: '$(Build.SourcesDirectory)'
displayName: 'Install ONNX'
- powershell: |
python -m pip uninstall -y ort-nightly-gpu ort-nightly onnxruntime onnxruntime-gpu -qq
Get-ChildItem -Path $(Build.ArtifactStagingDirectory)/*cp${{ replace(parameters.PYTHON_VERSION,'.','') }}*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname tabulate}
mkdir -p $(Agent.TempDirectory)\ort_test_data
Copy-Item -Path $(Build.sourcesDirectory)/onnxruntime/test/python/onnx_backend_test_series.py -Destination $(Agent.TempDirectory)\ort_test_data
Copy-Item -Recurse -Path $(Build.sourcesDirectory)/onnxruntime/test/testdata -Destination $(Agent.TempDirectory)\ort_test_data
cd $(Agent.TempDirectory)\ort_test_data
python onnx_backend_test_series.py
workingDirectory: '$(Build.sourcesDirectory)'
displayName: 'Run Python Tests'
- task: TSAUpload@2
displayName: 'TSA upload'
condition: and (succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main'))
inputs:
GdnPublishTsaOnboard: false
GdnPublishTsaConfigFile: '$(Build.sourcesDirectory)\.gdn\.gdntsa'
- template: component-governance-component-detection-steps.yml
parameters:
CudaVersion: ${{ parameters.CudaVersion }}
${{ if or(contains(parameters.EP_BUILD_FLAGS, 'use_cuda'), contains(parameters.EP_BUILD_FLAGS, 'use_tensorrt')) }}:
DownloadCUDA: true
${{ if contains(parameters.EP_BUILD_FLAGS, 'use_tensorrt') }}:
DownloadTRT: true
- task: PythonScript@0
displayName: 'Update deps.txt'
inputs:
scriptPath: $(Build.SourcesDirectory)/tools/ci_build/replace_urls_in_deps.py
arguments: --new_dir $(Build.BinariesDirectory)/deps
workingDirectory: $(Build.BinariesDirectory)
- task: PowerShell@2
displayName: 'Install ONNX'
inputs:
filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/windows/install_third_party_deps.ps1'
workingDirectory: '$(Build.BinariesDirectory)'
arguments: -cpu_arch x64 -install_prefix $(Build.BinariesDirectory)\RelWithDebInfo\installed -build_config RelWithDebInfo
- template: set-nightly-build-option-variable-step.yml
- task: PythonScript@0
displayName: 'Generate cmake config'
inputs:
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
arguments: >
--config RelWithDebInfo
--build_dir $(Build.BinariesDirectory)
--skip_submodule_sync
--cmake_generator "$(VSGenerator)"
--enable_pybind
--enable_onnx_tests
--parallel --use_binskim_compliant_compile_flags --update
$(TelemetryOption) ${{ parameters.BUILD_PY_PARAMETERS }} ${{ parameters.EP_BUILD_FLAGS }}
workingDirectory: '$(Build.BinariesDirectory)'
# building with build.py so the parallelization parameters are added to the msbuild command
- task: PythonScript@0
displayName: 'Build'
inputs:
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
arguments: >
--config RelWithDebInfo
--build_dir $(Build.BinariesDirectory)
--parallel --build
$(TelemetryOption) ${{ parameters.BUILD_PY_PARAMETERS }} ${{ parameters.EP_BUILD_FLAGS }}
workingDirectory: '$(Build.BinariesDirectory)'
# Esrp signing
- template: win-esrp-dll.yml
parameters:
FolderPath: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\onnxruntime\capi'
DisplayName: 'ESRP - Sign Native dlls'
DoEsrp: true
Pattern: '*.pyd,*.dll'
- task: PythonScript@0
displayName: 'Build wheel'
inputs:
scriptPath: '$(Build.SourcesDirectory)\setup.py'
arguments: 'bdist_wheel ${{ parameters.BUILD_PY_PARAMETERS }} $(NightlyBuildOption) --wheel_name_suffix=${{ parameters.EP_NAME }}'
workingDirectory: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo'
- task: CopyFiles@2
displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
inputs:
SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo\dist'
Contents: '*.whl'
TargetFolder: '$(Build.ArtifactStagingDirectory)'
- task: PublishBuildArtifacts@1
displayName: 'Publish Artifact: ONNXRuntime python wheel'
inputs:
ArtifactName: onnxruntime_${{ parameters.EP_NAME }}
- script: |
7z x *.whl
workingDirectory: '$(Build.ArtifactStagingDirectory)'
displayName: 'unzip the package'
- task: CredScan@3
displayName: 'Run CredScan'
inputs:
debugMode: false
continueOnError: true
- task: BinSkim@4
displayName: 'Run BinSkim'
inputs:
AnalyzeTargetGlob: '+:file|$(Build.ArtifactStagingDirectory)\**\*.dll;-:file|$(Build.ArtifactStagingDirectory)\**\DirectML.dll'
- powershell: |
python -m pip uninstall -y ort-nightly-gpu ort-nightly onnxruntime onnxruntime-gpu -qq
Get-ChildItem -Path $(Build.ArtifactStagingDirectory)/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname tabulate}
Remove-Item -Recurse -Force onnxruntime
python onnx_backend_test_series.py
workingDirectory: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo'
displayName: 'Run Python Tests'
- task: TSAUpload@2
displayName: 'TSA upload'
condition: and (succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main'))
inputs:
GdnPublishTsaOnboard: false
GdnPublishTsaConfigFile: '$(Build.sourcesDirectory)\.gdn\.gdntsa'
- template: component-governance-component-detection-steps.yml
parameters:
condition: 'succeeded'
condition: 'succeeded'