saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-06-08 00:23:03 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions
onnxruntime/tools/ci_build/github/azure-pipelines/templates/py-win-gpu.yml
Changming Sun 73271dd329
Move jobs in onnxruntime-Win2022-GPU-T4 machine pool to onnxruntime-Win2022-GPU-A10 (#21023) 
### Description
Move jobs in onnxruntime-Win2022-GPU-T4 machine pool to
onnxruntime-Win2022-GPU-A10

### Motivation and Context
To reduce the variants of VM images we need to maintain. Now we have 3:
1. Windows 2022 CPU
2. Windows 2022 GPU A10
3. Windows 2022 GPU T4

This change allows us removing the last one.
2024-06-12 22:04:40 -07:00

285 lines
11 KiB
YAML
Raw Blame History

parameters:
- name: MACHINE_POOL
type: string
default: 'onnxruntime-Win2022-GPU-A10'
- name: EP_NAME
type: string
- name: PYTHON_VERSION
type: string
- name: EP_BUILD_FLAGS
type: string
- name: ENV_SETUP_SCRIPT
type: string
default: ''
- name: BUILD_PY_PARAMETERS
displayName: >
Extra parameters to pass to build.py. Don't put newlines in here.
type: string
default: ''
- name: CudaVersion
type: string
default: '11.8'
values:
- 11.8
- 12.2
- name: SpecificArtifact
displayName: Use Specific Artifact
type: boolean
default: false
- name: BuildId
displayName: Specific Artifact's BuildId
type: string
default: '0'
- name: cmake_build_type
type: string
displayName: 'Linux packages cmake build type. Linux Only.'
default: 'Release'
values:
- Debug
- Release
- RelWithDebInfo
- MinSizeRel
- name: publish_symbols
type: boolean
default: false
stages:
- stage: Win_py_${{ parameters.EP_NAME }}_Wheels_${{ replace(parameters.PYTHON_VERSION,'.','_') }}_Build
dependsOn: []
jobs:
- job: Win_py_${{ parameters.EP_NAME }}_Wheels_${{ replace(parameters.PYTHON_VERSION,'.','_') }}_Build
timeoutInMinutes: 360
workspace:
clean: all
pool:
name: onnxruntime-Win-CPU-2022
variables:
GRADLE_OPTS: '-Dorg.gradle.daemon=false'
VSGenerator: 'Visual Studio 17 2022'
CUDA_MODULE_LOADING: 'LAZY'
steps:
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
displayName: 'Clean Agent Directories'
condition: always()
- checkout: self
clean: true
submodules: recursive
- template: telemetry-steps.yml
- task: UsePythonVersion@0
inputs:
versionSpec: ${{ parameters.PYTHON_VERSION }}
addToPath: true
architecture: 'x64'
- task: onebranch.pipeline.tsaoptions@1
displayName: 'OneBranch TSAOptions'
inputs:
tsaConfigFilePath: '$(Build.SourcesDirectory)\.config\tsaoptions.json'
appendSourceBranchName: false
- task: PythonScript@0
inputs:
scriptSource: inline
script: |
import sys
np_version = 'numpy==1.21.6' if sys.version_info < (3, 11) else 'numpy==1.26'
import subprocess
try:
subprocess.check_call(['pip', 'install', '-q', 'setuptools', 'wheel', np_version])
except subprocess.CalledProcessError:
sys.exit(1)
workingDirectory: '$(Build.BinariesDirectory)'
displayName: 'Install python modules'
- template: download-deps.yml
- ${{ if ne(parameters.ENV_SETUP_SCRIPT, '') }}:
- template: jobs/set-winenv.yml
parameters:
EnvSetupScript: ${{ parameters.ENV_SETUP_SCRIPT }}
${{ if or(contains(parameters.EP_BUILD_FLAGS, 'use_cuda'), contains(parameters.EP_BUILD_FLAGS, 'use_tensorrt')) }}:
DownloadCUDA: true
${{ if contains(parameters.EP_BUILD_FLAGS, 'use_tensorrt') }}:
DownloadTRT: true
- ${{ if eq(parameters.ENV_SETUP_SCRIPT, '') }}:
- template: jobs/download_win_gpu_library.yml
parameters:
CudaVersion: ${{ parameters.CudaVersion }}
${{ if or(contains(parameters.EP_BUILD_FLAGS, 'use_cuda'), contains(parameters.EP_BUILD_FLAGS, 'use_tensorrt')) }}:
DownloadCUDA: true
${{ if contains(parameters.EP_BUILD_FLAGS, 'use_tensorrt') }}:
DownloadTRT: true
- task: PythonScript@0
displayName: 'Update deps.txt'
inputs:
scriptPath: $(Build.SourcesDirectory)/tools/ci_build/replace_urls_in_deps.py
arguments: --new_dir $(Build.BinariesDirectory)/deps
workingDirectory: $(Build.BinariesDirectory)
- task: PowerShell@2
displayName: 'Install ONNX'
inputs:
filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/windows/install_third_party_deps.ps1'
workingDirectory: '$(Build.BinariesDirectory)'
arguments: -cpu_arch x64 -install_prefix $(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}\installed -build_config ${{ parameters.cmake_build_type }}
- template: set-nightly-build-option-variable-step.yml
- task: PythonScript@0
displayName: 'Generate cmake config'
inputs:
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
arguments: >
--config ${{ parameters.cmake_build_type }}
--build_dir $(Build.BinariesDirectory)
--skip_submodule_sync
--cmake_generator "$(VSGenerator)"
--enable_pybind
--enable_onnx_tests
--parallel --use_binskim_compliant_compile_flags --update
$(TelemetryOption) ${{ parameters.BUILD_PY_PARAMETERS }} ${{ parameters.EP_BUILD_FLAGS }}
workingDirectory: '$(Build.BinariesDirectory)'
# building with build.py so the parallelization parameters are added to the msbuild command
- task: PythonScript@0
displayName: 'Build'
inputs:
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
arguments: >
--config ${{ parameters.cmake_build_type }}
--build_dir $(Build.BinariesDirectory)
--parallel --build
$(TelemetryOption) ${{ parameters.BUILD_PY_PARAMETERS }} ${{ parameters.EP_BUILD_FLAGS }}
workingDirectory: '$(Build.BinariesDirectory)'
# Esrp signing
- template: win-esrp-dll.yml
parameters:
FolderPath: '$(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}\${{ parameters.cmake_build_type }}\onnxruntime\capi'
DisplayName: 'ESRP - Sign Native dlls'
DoEsrp: true
Pattern: '*.pyd,*.dll'
- task: PythonScript@0
displayName: 'Build wheel'
inputs:
scriptPath: '$(Build.SourcesDirectory)\setup.py'
arguments: 'bdist_wheel ${{ parameters.BUILD_PY_PARAMETERS }} $(NightlyBuildOption) --wheel_name_suffix=${{ parameters.EP_NAME }}'
workingDirectory: '$(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}\${{ parameters.cmake_build_type }}'
- task: CopyFiles@2
displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
inputs:
SourceFolder: '$(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}\${{ parameters.cmake_build_type }}\dist'
Contents: '*.whl'
TargetFolder: '$(Build.ArtifactStagingDirectory)'
- task: PublishBuildArtifacts@1
displayName: 'Publish Artifact: ONNXRuntime python wheel'
inputs:
ArtifactName: onnxruntime_${{ parameters.EP_NAME }}
- ${{ if eq(parameters.publish_symbols, true) }}:
- task: PublishSymbols@2
displayName: 'Publish symbols'
condition: and (succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/main'), startsWith(variables['Build.SourceBranch'], 'refs/heads/rel-')))
inputs:
SymbolsFolder: '$(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}\${{ parameters.cmake_build_type }}'
SearchPattern: |
onnxruntime_pybind11_state.pdb
onnxruntime_providers_shared.pdb
IndexSources: true
SymbolServerType: TeamServices
SymbolExpirationInDays: 3650
SymbolsArtifactName: 'win_${{ parameters.EP_NAME }}_${{ parameters.PYTHON_VERSION }}_$(Build.BuildNumber)'
- script: |
7z x *.whl
workingDirectory: '$(Build.ArtifactStagingDirectory)'
displayName: 'unzip the package'
- task: CredScan@3
displayName: 'Run CredScan'
inputs:
debugMode: false
continueOnError: true
- task: BinSkim@4
displayName: 'Run BinSkim'
inputs:
AnalyzeTargetGlob: '+:file|$(Build.ArtifactStagingDirectory)\**\*.dll;-:file|$(Build.ArtifactStagingDirectory)\**\DirectML.dll'
- task: TSAUpload@2
displayName: 'TSA upload'
condition: and (succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main'))
inputs:
GdnPublishTsaOnboard: false
GdnPublishTsaConfigFile: '$(Build.sourcesDirectory)\.gdn\.gdntsa'
- template: component-governance-component-detection-steps.yml
parameters:
condition: 'succeeded'
- stage: Win_py_${{ parameters.EP_NAME }}_Wheels_${{ replace(parameters.PYTHON_VERSION,'.','_') }}_Tests
dependsOn: Win_py_${{ parameters.EP_NAME }}_Wheels_${{ replace(parameters.PYTHON_VERSION,'.','_') }}_Build
jobs:
- job: Win_py_${{ parameters.EP_NAME }}_Wheels_${{ replace(parameters.PYTHON_VERSION,'.','_') }}_Tests
workspace:
clean: all
pool:
name: ${{parameters.MACHINE_POOL}}
steps:
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
displayName: 'Clean Agent Directories'
condition: always()
- checkout: self
clean: true
submodules: none
- task: UsePythonVersion@0
inputs:
versionSpec: ${{ parameters.PYTHON_VERSION }}
addToPath: true
architecture: 'x64'
- template: flex-downloadPipelineArtifact.yml
parameters:
ArtifactName: onnxruntime_${{ parameters.EP_NAME }}
StepName: 'Download Pipeline Artifact - Windows GPU Build'
TargetPath: '$(Build.ArtifactStagingDirectory)'
SpecificArtifact: ${{ parameters.SpecificArtifact }}
BuildId: ${{ parameters.BuildId }}
- powershell: |
pushd onnxruntime/test/python
python -m pip install --upgrade pip
python -m pip install -r requirements.txt
popd
workingDirectory: '$(Build.SourcesDirectory)'
displayName: 'Install ONNX'
- powershell: |
python -m pip uninstall -y ort-nightly-gpu ort-nightly onnxruntime onnxruntime-gpu -qq
Get-ChildItem -Path $(Build.ArtifactStagingDirectory)/*cp${{ replace(parameters.PYTHON_VERSION,'.','') }}*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname tabulate}
mkdir -p $(Agent.TempDirectory)\ort_test_data
Copy-Item -Path $(Build.sourcesDirectory)/onnxruntime/test/python/onnx_backend_test_series.py -Destination $(Agent.TempDirectory)\ort_test_data
Copy-Item -Recurse -Path $(Build.sourcesDirectory)/onnxruntime/test/testdata -Destination $(Agent.TempDirectory)\ort_test_data
cd $(Agent.TempDirectory)\ort_test_data
python onnx_backend_test_series.py
workingDirectory: '$(Build.sourcesDirectory)'
displayName: 'Run Python Tests'
Reference in a new issue View git blame Copy permalink