mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-05-25 22:26:24 +00:00
af00a20f8a
### Description Our nightly CPU python package's name is "ort-nightly" instead of "onnxruntime". It was because of some historical reasons. Tensorflow was like that. Now we would prefer to make them the same. Do this change for all nightly python packages, including CPU, GPU(CUDA), and maybe others. ### Motivation and Context <!-- - Why is this change required? What problem does it solve? - If it fixes an open issue, please link to the issue here. -->
270 lines
11 KiB
YAML
270 lines
11 KiB
YAML
parameters:
|
|
- name: MACHINE_POOL
|
|
type: string
|
|
default: 'onnxruntime-Win2022-GPU-A10'
|
|
|
|
- name: EP_NAME
|
|
type: string
|
|
|
|
- name: PYTHON_VERSION
|
|
type: string
|
|
|
|
- name: EP_BUILD_FLAGS
|
|
type: string
|
|
|
|
- name: ENV_SETUP_SCRIPT
|
|
type: string
|
|
default: ''
|
|
|
|
- name: BUILD_PY_PARAMETERS
|
|
displayName: >
|
|
Extra parameters to pass to build.py. Don't put newlines in here.
|
|
type: string
|
|
default: ''
|
|
- name: CudaVersion
|
|
type: string
|
|
default: '11.8'
|
|
values:
|
|
- 11.8
|
|
- 12.2
|
|
|
|
- name: SpecificArtifact
|
|
displayName: Use Specific Artifact
|
|
type: boolean
|
|
default: false
|
|
|
|
- name: BuildId
|
|
displayName: Specific Artifact's BuildId
|
|
type: string
|
|
default: '0'
|
|
|
|
- name: cmake_build_type
|
|
type: string
|
|
displayName: 'Linux packages cmake build type. Linux Only.'
|
|
default: 'Release'
|
|
values:
|
|
- Debug
|
|
- Release
|
|
- RelWithDebInfo
|
|
- MinSizeRel
|
|
|
|
- name: publish_symbols
|
|
type: boolean
|
|
default: false
|
|
|
|
stages:
|
|
- stage: Win_py_${{ parameters.EP_NAME }}_Wheels_${{ replace(parameters.PYTHON_VERSION,'.','_') }}_Build
|
|
dependsOn: []
|
|
jobs:
|
|
- job: Win_py_${{ parameters.EP_NAME }}_Wheels_${{ replace(parameters.PYTHON_VERSION,'.','_') }}_Build
|
|
timeoutInMinutes: 360
|
|
workspace:
|
|
clean: all
|
|
pool:
|
|
name: onnxruntime-Win-CPU-2022
|
|
variables:
|
|
GRADLE_OPTS: '-Dorg.gradle.daemon=false'
|
|
VSGenerator: 'Visual Studio 17 2022'
|
|
CUDA_MODULE_LOADING: 'LAZY'
|
|
steps:
|
|
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
|
|
displayName: 'Clean Agent Directories'
|
|
condition: always()
|
|
|
|
- checkout: self
|
|
clean: true
|
|
submodules: recursive
|
|
|
|
- template: telemetry-steps.yml
|
|
|
|
- task: UsePythonVersion@0
|
|
inputs:
|
|
versionSpec: ${{ parameters.PYTHON_VERSION }}
|
|
addToPath: true
|
|
architecture: 'x64'
|
|
|
|
- task: onebranch.pipeline.tsaoptions@1
|
|
displayName: 'OneBranch TSAOptions'
|
|
inputs:
|
|
tsaConfigFilePath: '$(Build.SourcesDirectory)\.config\tsaoptions.json'
|
|
appendSourceBranchName: false
|
|
|
|
- template: download-deps.yml
|
|
|
|
- ${{ if ne(parameters.ENV_SETUP_SCRIPT, '') }}:
|
|
- template: jobs/set-winenv.yml
|
|
parameters:
|
|
EnvSetupScript: ${{ parameters.ENV_SETUP_SCRIPT }}
|
|
${{ if or(contains(parameters.EP_BUILD_FLAGS, 'use_cuda'), contains(parameters.EP_BUILD_FLAGS, 'use_tensorrt')) }}:
|
|
DownloadCUDA: true
|
|
${{ if contains(parameters.EP_BUILD_FLAGS, 'use_tensorrt') }}:
|
|
DownloadTRT: true
|
|
|
|
- ${{ if eq(parameters.ENV_SETUP_SCRIPT, '') }}:
|
|
- template: jobs/download_win_gpu_library.yml
|
|
parameters:
|
|
CudaVersion: ${{ parameters.CudaVersion }}
|
|
${{ if or(contains(parameters.EP_BUILD_FLAGS, 'use_cuda'), contains(parameters.EP_BUILD_FLAGS, 'use_tensorrt')) }}:
|
|
DownloadCUDA: true
|
|
${{ if contains(parameters.EP_BUILD_FLAGS, 'use_tensorrt') }}:
|
|
DownloadTRT: true
|
|
|
|
- task: PythonScript@0
|
|
displayName: 'Update deps.txt'
|
|
inputs:
|
|
scriptPath: $(Build.SourcesDirectory)/tools/ci_build/replace_urls_in_deps.py
|
|
arguments: --new_dir $(Build.BinariesDirectory)/deps
|
|
workingDirectory: $(Build.BinariesDirectory)
|
|
|
|
- task: PowerShell@2
|
|
displayName: 'Install ONNX'
|
|
inputs:
|
|
filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/windows/install_third_party_deps.ps1'
|
|
workingDirectory: '$(Build.BinariesDirectory)'
|
|
arguments: -cpu_arch x64 -install_prefix $(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}\installed -build_config ${{ parameters.cmake_build_type }}
|
|
|
|
- template: set-nightly-build-option-variable-step.yml
|
|
|
|
- task: PythonScript@0
|
|
displayName: 'Generate cmake config'
|
|
inputs:
|
|
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
|
|
arguments: >
|
|
--config ${{ parameters.cmake_build_type }}
|
|
--build_dir $(Build.BinariesDirectory)
|
|
--skip_submodule_sync
|
|
--cmake_generator "$(VSGenerator)"
|
|
--enable_pybind
|
|
--enable_onnx_tests
|
|
--parallel --use_binskim_compliant_compile_flags --update
|
|
$(TelemetryOption) ${{ parameters.BUILD_PY_PARAMETERS }} ${{ parameters.EP_BUILD_FLAGS }}
|
|
workingDirectory: '$(Build.BinariesDirectory)'
|
|
|
|
# building with build.py so the parallelization parameters are added to the msbuild command
|
|
- task: PythonScript@0
|
|
displayName: 'Build'
|
|
inputs:
|
|
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
|
|
arguments: >
|
|
--config ${{ parameters.cmake_build_type }}
|
|
--build_dir $(Build.BinariesDirectory)
|
|
--parallel --build
|
|
$(TelemetryOption) ${{ parameters.BUILD_PY_PARAMETERS }} ${{ parameters.EP_BUILD_FLAGS }}
|
|
workingDirectory: '$(Build.BinariesDirectory)'
|
|
|
|
# Esrp signing
|
|
- template: win-esrp-dll.yml
|
|
parameters:
|
|
FolderPath: '$(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}\${{ parameters.cmake_build_type }}\onnxruntime\capi'
|
|
DisplayName: 'ESRP - Sign Native dlls'
|
|
DoEsrp: true
|
|
Pattern: '*.pyd,*.dll'
|
|
|
|
- task: PythonScript@0
|
|
displayName: 'Build wheel'
|
|
inputs:
|
|
scriptPath: '$(Build.SourcesDirectory)\setup.py'
|
|
arguments: 'bdist_wheel ${{ parameters.BUILD_PY_PARAMETERS }} $(NightlyBuildOption) --wheel_name_suffix=${{ parameters.EP_NAME }}'
|
|
workingDirectory: '$(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}\${{ parameters.cmake_build_type }}'
|
|
|
|
- task: CopyFiles@2
|
|
displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
|
|
inputs:
|
|
SourceFolder: '$(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}\${{ parameters.cmake_build_type }}\dist'
|
|
Contents: '*.whl'
|
|
TargetFolder: '$(Build.ArtifactStagingDirectory)'
|
|
|
|
- task: PublishBuildArtifacts@1
|
|
displayName: 'Publish Artifact: ONNXRuntime python wheel'
|
|
inputs:
|
|
ArtifactName: onnxruntime_${{ parameters.EP_NAME }}
|
|
|
|
- ${{ if eq(parameters.publish_symbols, true) }}:
|
|
- task: PublishSymbols@2
|
|
displayName: 'Publish symbols'
|
|
condition: and (succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/main'), startsWith(variables['Build.SourceBranch'], 'refs/heads/rel-')))
|
|
inputs:
|
|
SymbolsFolder: '$(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}\${{ parameters.cmake_build_type }}'
|
|
SearchPattern: |
|
|
onnxruntime_pybind11_state.pdb
|
|
onnxruntime_providers_shared.pdb
|
|
IndexSources: true
|
|
SymbolServerType: TeamServices
|
|
SymbolExpirationInDays: 3650
|
|
SymbolsArtifactName: 'win_${{ parameters.EP_NAME }}_${{ parameters.PYTHON_VERSION }}_$(Build.BuildNumber)'
|
|
|
|
- script: |
|
|
7z x *.whl
|
|
workingDirectory: '$(Build.ArtifactStagingDirectory)'
|
|
displayName: 'unzip the package'
|
|
|
|
- task: CredScan@3
|
|
displayName: 'Run CredScan'
|
|
inputs:
|
|
debugMode: false
|
|
continueOnError: true
|
|
|
|
- task: BinSkim@4
|
|
displayName: 'Run BinSkim'
|
|
inputs:
|
|
AnalyzeTargetGlob: '+:file|$(Build.ArtifactStagingDirectory)\**\*.dll;-:file|$(Build.ArtifactStagingDirectory)\**\DirectML.dll'
|
|
|
|
- task: TSAUpload@2
|
|
displayName: 'TSA upload'
|
|
condition: and (succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main'))
|
|
inputs:
|
|
GdnPublishTsaOnboard: false
|
|
GdnPublishTsaConfigFile: '$(Build.sourcesDirectory)\.gdn\.gdntsa'
|
|
|
|
- template: component-governance-component-detection-steps.yml
|
|
parameters:
|
|
condition: 'succeeded'
|
|
|
|
- stage: Win_py_${{ parameters.EP_NAME }}_Wheels_${{ replace(parameters.PYTHON_VERSION,'.','_') }}_Tests
|
|
dependsOn: Win_py_${{ parameters.EP_NAME }}_Wheels_${{ replace(parameters.PYTHON_VERSION,'.','_') }}_Build
|
|
jobs:
|
|
- job: Win_py_${{ parameters.EP_NAME }}_Wheels_${{ replace(parameters.PYTHON_VERSION,'.','_') }}_Tests
|
|
workspace:
|
|
clean: all
|
|
pool:
|
|
name: ${{parameters.MACHINE_POOL}}
|
|
steps:
|
|
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
|
|
displayName: 'Clean Agent Directories'
|
|
condition: always()
|
|
|
|
- checkout: self
|
|
clean: true
|
|
submodules: none
|
|
|
|
- task: UsePythonVersion@0
|
|
inputs:
|
|
versionSpec: ${{ parameters.PYTHON_VERSION }}
|
|
addToPath: true
|
|
architecture: 'x64'
|
|
|
|
- template: flex-downloadPipelineArtifact.yml
|
|
parameters:
|
|
ArtifactName: onnxruntime_${{ parameters.EP_NAME }}
|
|
StepName: 'Download Pipeline Artifact - Windows GPU Build'
|
|
TargetPath: '$(Build.ArtifactStagingDirectory)'
|
|
SpecificArtifact: ${{ parameters.SpecificArtifact }}
|
|
BuildId: ${{ parameters.BuildId }}
|
|
|
|
- task: PowerShell@2
|
|
displayName: 'Install ONNX'
|
|
inputs:
|
|
filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/windows/install_third_party_deps.ps1'
|
|
workingDirectory: '$(Build.BinariesDirectory)'
|
|
arguments: -cpu_arch x64 -install_prefix $(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}\installed -build_config ${{ parameters.cmake_build_type }}
|
|
|
|
- powershell: |
|
|
python -m pip uninstall -y onnxruntime onnxruntime-gpu -qq
|
|
Get-ChildItem -Path $(Build.ArtifactStagingDirectory)/*cp${{ replace(parameters.PYTHON_VERSION,'.','') }}*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname tabulate}
|
|
mkdir -p $(Agent.TempDirectory)\ort_test_data
|
|
Copy-Item -Path $(Build.sourcesDirectory)/onnxruntime/test/python/onnx_backend_test_series.py -Destination $(Agent.TempDirectory)\ort_test_data
|
|
Copy-Item -Recurse -Path $(Build.sourcesDirectory)/onnxruntime/test/testdata -Destination $(Agent.TempDirectory)\ort_test_data
|
|
cd $(Agent.TempDirectory)\ort_test_data
|
|
python onnx_backend_test_series.py
|
|
workingDirectory: '$(Build.sourcesDirectory)'
|
|
displayName: 'Run Python Tests'
|