mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-05-18 21:21:17 +00:00
d1ab94c2b0
### Description As suggested by SciPy's doc, we will `Build against NumPy 2.0.0, then it will work for all NumPy versions with the same major version number (NumPy does maintain backwards ABI compatibility), and as far back as NumPy 1.19 series at the time of writing` I think it works because in [numpyconfig.h#L64](https://github.com/numpy/numpy/blob/main/numpy/_core/include/numpy/numpyconfig.h#L64) there is a macro NPY_FEATURE_VERSION. By default it is set to NPY_1_19_API_VERSION. And the NPY_FEATURE_VERSION macro controls ABI. This PR only upgrade the build time dependency; When a user installs ONNX Runtime, they still can use numpy 1.x. ### Motivation and Context Recently numpy published a new version, 2.0.0, which is incompatible with the latest ONNX Runtime release.
271 lines
11 KiB
YAML
271 lines
11 KiB
YAML
parameters:
|
|
- name: MACHINE_POOL
|
|
type: string
|
|
default: 'onnxruntime-Win2022-GPU-A10'
|
|
|
|
- name: EP_NAME
|
|
type: string
|
|
|
|
- name: PYTHON_VERSION
|
|
type: string
|
|
|
|
- name: EP_BUILD_FLAGS
|
|
type: string
|
|
|
|
- name: ENV_SETUP_SCRIPT
|
|
type: string
|
|
default: ''
|
|
|
|
- name: BUILD_PY_PARAMETERS
|
|
displayName: >
|
|
Extra parameters to pass to build.py. Don't put newlines in here.
|
|
type: string
|
|
default: ''
|
|
- name: CudaVersion
|
|
type: string
|
|
default: '11.8'
|
|
values:
|
|
- 11.8
|
|
- 12.2
|
|
|
|
- name: SpecificArtifact
|
|
displayName: Use Specific Artifact
|
|
type: boolean
|
|
default: false
|
|
|
|
- name: BuildId
|
|
displayName: Specific Artifact's BuildId
|
|
type: string
|
|
default: '0'
|
|
|
|
- name: cmake_build_type
|
|
type: string
|
|
displayName: 'Linux packages cmake build type. Linux Only.'
|
|
default: 'Release'
|
|
values:
|
|
- Debug
|
|
- Release
|
|
- RelWithDebInfo
|
|
- MinSizeRel
|
|
|
|
- name: publish_symbols
|
|
type: boolean
|
|
default: false
|
|
|
|
stages:
|
|
- stage: Win_py_${{ parameters.EP_NAME }}_Wheels_${{ replace(parameters.PYTHON_VERSION,'.','_') }}_Build
|
|
dependsOn: []
|
|
jobs:
|
|
- job: Win_py_${{ parameters.EP_NAME }}_Wheels_${{ replace(parameters.PYTHON_VERSION,'.','_') }}_Build
|
|
timeoutInMinutes: 360
|
|
workspace:
|
|
clean: all
|
|
pool:
|
|
name: onnxruntime-Win-CPU-2022
|
|
variables:
|
|
GRADLE_OPTS: '-Dorg.gradle.daemon=false'
|
|
VSGenerator: 'Visual Studio 17 2022'
|
|
CUDA_MODULE_LOADING: 'LAZY'
|
|
steps:
|
|
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
|
|
displayName: 'Clean Agent Directories'
|
|
condition: always()
|
|
|
|
- checkout: self
|
|
clean: true
|
|
submodules: recursive
|
|
|
|
- template: telemetry-steps.yml
|
|
|
|
- task: UsePythonVersion@0
|
|
inputs:
|
|
versionSpec: ${{ parameters.PYTHON_VERSION }}
|
|
addToPath: true
|
|
architecture: 'x64'
|
|
|
|
- task: onebranch.pipeline.tsaoptions@1
|
|
displayName: 'OneBranch TSAOptions'
|
|
inputs:
|
|
tsaConfigFilePath: '$(Build.SourcesDirectory)\.config\tsaoptions.json'
|
|
appendSourceBranchName: false
|
|
|
|
- template: download-deps.yml
|
|
|
|
- ${{ if ne(parameters.ENV_SETUP_SCRIPT, '') }}:
|
|
- template: jobs/set-winenv.yml
|
|
parameters:
|
|
EnvSetupScript: ${{ parameters.ENV_SETUP_SCRIPT }}
|
|
${{ if or(contains(parameters.EP_BUILD_FLAGS, 'use_cuda'), contains(parameters.EP_BUILD_FLAGS, 'use_tensorrt')) }}:
|
|
DownloadCUDA: true
|
|
${{ if contains(parameters.EP_BUILD_FLAGS, 'use_tensorrt') }}:
|
|
DownloadTRT: true
|
|
|
|
- ${{ if eq(parameters.ENV_SETUP_SCRIPT, '') }}:
|
|
- template: jobs/download_win_gpu_library.yml
|
|
parameters:
|
|
CudaVersion: ${{ parameters.CudaVersion }}
|
|
${{ if or(contains(parameters.EP_BUILD_FLAGS, 'use_cuda'), contains(parameters.EP_BUILD_FLAGS, 'use_tensorrt')) }}:
|
|
DownloadCUDA: true
|
|
${{ if contains(parameters.EP_BUILD_FLAGS, 'use_tensorrt') }}:
|
|
DownloadTRT: true
|
|
|
|
- task: PythonScript@0
|
|
displayName: 'Update deps.txt'
|
|
inputs:
|
|
scriptPath: $(Build.SourcesDirectory)/tools/ci_build/replace_urls_in_deps.py
|
|
arguments: --new_dir $(Build.BinariesDirectory)/deps
|
|
workingDirectory: $(Build.BinariesDirectory)
|
|
|
|
- task: PowerShell@2
|
|
displayName: 'Install ONNX'
|
|
inputs:
|
|
filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/windows/install_third_party_deps.ps1'
|
|
workingDirectory: '$(Build.BinariesDirectory)'
|
|
arguments: -cpu_arch x64 -install_prefix $(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}\installed -build_config ${{ parameters.cmake_build_type }}
|
|
|
|
- template: set-nightly-build-option-variable-step.yml
|
|
|
|
- task: PythonScript@0
|
|
displayName: 'Generate cmake config'
|
|
inputs:
|
|
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
|
|
arguments: >
|
|
--config ${{ parameters.cmake_build_type }}
|
|
--build_dir $(Build.BinariesDirectory)
|
|
--skip_submodule_sync
|
|
--cmake_generator "$(VSGenerator)"
|
|
--enable_pybind
|
|
--enable_onnx_tests
|
|
--parallel --use_binskim_compliant_compile_flags --update
|
|
$(TelemetryOption) ${{ parameters.BUILD_PY_PARAMETERS }} ${{ parameters.EP_BUILD_FLAGS }}
|
|
workingDirectory: '$(Build.BinariesDirectory)'
|
|
|
|
# building with build.py so the parallelization parameters are added to the msbuild command
|
|
- task: PythonScript@0
|
|
displayName: 'Build'
|
|
inputs:
|
|
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
|
|
arguments: >
|
|
--config ${{ parameters.cmake_build_type }}
|
|
--build_dir $(Build.BinariesDirectory)
|
|
--parallel --build
|
|
$(TelemetryOption) ${{ parameters.BUILD_PY_PARAMETERS }} ${{ parameters.EP_BUILD_FLAGS }}
|
|
workingDirectory: '$(Build.BinariesDirectory)'
|
|
|
|
# Esrp signing
|
|
- template: win-esrp-dll.yml
|
|
parameters:
|
|
FolderPath: '$(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}\${{ parameters.cmake_build_type }}\onnxruntime\capi'
|
|
DisplayName: 'ESRP - Sign Native dlls'
|
|
DoEsrp: true
|
|
Pattern: '*.pyd,*.dll'
|
|
|
|
- task: PythonScript@0
|
|
displayName: 'Build wheel'
|
|
inputs:
|
|
scriptPath: '$(Build.SourcesDirectory)\setup.py'
|
|
arguments: 'bdist_wheel ${{ parameters.BUILD_PY_PARAMETERS }} $(NightlyBuildOption) --wheel_name_suffix=${{ parameters.EP_NAME }}'
|
|
workingDirectory: '$(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}\${{ parameters.cmake_build_type }}'
|
|
|
|
- task: CopyFiles@2
|
|
displayName: 'Copy Python Wheel to: $(Build.ArtifactStagingDirectory)'
|
|
inputs:
|
|
SourceFolder: '$(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}\${{ parameters.cmake_build_type }}\dist'
|
|
Contents: '*.whl'
|
|
TargetFolder: '$(Build.ArtifactStagingDirectory)'
|
|
|
|
- task: PublishBuildArtifacts@1
|
|
displayName: 'Publish Artifact: ONNXRuntime python wheel'
|
|
inputs:
|
|
ArtifactName: onnxruntime_${{ parameters.EP_NAME }}
|
|
|
|
- ${{ if eq(parameters.publish_symbols, true) }}:
|
|
- task: PublishSymbols@2
|
|
displayName: 'Publish symbols'
|
|
condition: and (succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/main'), startsWith(variables['Build.SourceBranch'], 'refs/heads/rel-')))
|
|
inputs:
|
|
SymbolsFolder: '$(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}\${{ parameters.cmake_build_type }}'
|
|
SearchPattern: |
|
|
onnxruntime_pybind11_state.pdb
|
|
onnxruntime_providers_shared.pdb
|
|
IndexSources: true
|
|
SymbolServerType: TeamServices
|
|
SymbolExpirationInDays: 3650
|
|
SymbolsArtifactName: 'win_${{ parameters.EP_NAME }}_${{ parameters.PYTHON_VERSION }}_$(Build.BuildNumber)'
|
|
|
|
- script: |
|
|
7z x *.whl
|
|
workingDirectory: '$(Build.ArtifactStagingDirectory)'
|
|
displayName: 'unzip the package'
|
|
|
|
- task: CredScan@3
|
|
displayName: 'Run CredScan'
|
|
inputs:
|
|
debugMode: false
|
|
continueOnError: true
|
|
|
|
- task: BinSkim@4
|
|
displayName: 'Run BinSkim'
|
|
inputs:
|
|
AnalyzeTargetGlob: '+:file|$(Build.ArtifactStagingDirectory)\**\*.dll;-:file|$(Build.ArtifactStagingDirectory)\**\DirectML.dll'
|
|
|
|
- task: TSAUpload@2
|
|
displayName: 'TSA upload'
|
|
condition: and (succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main'))
|
|
inputs:
|
|
GdnPublishTsaOnboard: false
|
|
GdnPublishTsaConfigFile: '$(Build.sourcesDirectory)\.gdn\.gdntsa'
|
|
|
|
- template: component-governance-component-detection-steps.yml
|
|
parameters:
|
|
condition: 'succeeded'
|
|
|
|
- stage: Win_py_${{ parameters.EP_NAME }}_Wheels_${{ replace(parameters.PYTHON_VERSION,'.','_') }}_Tests
|
|
dependsOn: Win_py_${{ parameters.EP_NAME }}_Wheels_${{ replace(parameters.PYTHON_VERSION,'.','_') }}_Build
|
|
jobs:
|
|
- job: Win_py_${{ parameters.EP_NAME }}_Wheels_${{ replace(parameters.PYTHON_VERSION,'.','_') }}_Tests
|
|
workspace:
|
|
clean: all
|
|
pool:
|
|
name: ${{parameters.MACHINE_POOL}}
|
|
steps:
|
|
- task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3
|
|
displayName: 'Clean Agent Directories'
|
|
condition: always()
|
|
|
|
- checkout: self
|
|
clean: true
|
|
submodules: none
|
|
|
|
- task: UsePythonVersion@0
|
|
inputs:
|
|
versionSpec: ${{ parameters.PYTHON_VERSION }}
|
|
addToPath: true
|
|
architecture: 'x64'
|
|
|
|
- template: flex-downloadPipelineArtifact.yml
|
|
parameters:
|
|
ArtifactName: onnxruntime_${{ parameters.EP_NAME }}
|
|
StepName: 'Download Pipeline Artifact - Windows GPU Build'
|
|
TargetPath: '$(Build.ArtifactStagingDirectory)'
|
|
SpecificArtifact: ${{ parameters.SpecificArtifact }}
|
|
BuildId: ${{ parameters.BuildId }}
|
|
|
|
- powershell: |
|
|
pushd onnxruntime/test/python
|
|
python -m pip install --upgrade pip
|
|
python -m pip install -r requirements.txt
|
|
popd
|
|
workingDirectory: '$(Build.SourcesDirectory)'
|
|
displayName: 'Install ONNX'
|
|
|
|
- powershell: |
|
|
python -m pip uninstall -y ort-nightly-gpu ort-nightly onnxruntime onnxruntime-gpu -qq
|
|
Get-ChildItem -Path $(Build.ArtifactStagingDirectory)/*cp${{ replace(parameters.PYTHON_VERSION,'.','') }}*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname tabulate}
|
|
mkdir -p $(Agent.TempDirectory)\ort_test_data
|
|
Copy-Item -Path $(Build.sourcesDirectory)/onnxruntime/test/python/onnx_backend_test_series.py -Destination $(Agent.TempDirectory)\ort_test_data
|
|
Copy-Item -Recurse -Path $(Build.sourcesDirectory)/onnxruntime/test/testdata -Destination $(Agent.TempDirectory)\ort_test_data
|
|
cd $(Agent.TempDirectory)\ort_test_data
|
|
python onnx_backend_test_series.py
|
|
workingDirectory: '$(Build.sourcesDirectory)'
|
|
displayName: 'Run Python Tests'
|