mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-05 04:17:53 +00:00
Enable 1ES on Python CUDA Package Pipelines (#22560)
### Description These 3 following CUDA packaging pipeline shoud be enabled with 1ES after this pull request. • [Python-CUDA-Packaging-Pipeline](https://dev.azure.com/aiinfra/Lotus/_build?definitionId=1299&view=runs) • [Python CUDA Alt Packaging Pipeline](https://dev.azure.com/aiinfra/Lotus/_build?definitionId=1626) • [Python DML Packaging Pipeline](https://dev.azure.com/aiinfra/Lotus/_build?definitionId=1625) This should also fix the issue where [Python packaging pipeline](https://aiinfra.visualstudio.com/Lotus/_build?definitionId=841&_a=summary) failed due to cannot find `publish_symbols` ### Motivation and Context <!-- - Why is this change required? What problem does it solve? - If it fixes an open issue, please link to the issue here. -->
This commit is contained in:
parent
cd60af0365
commit
3ae7c3c0a6
6 changed files with 63 additions and 90 deletions
|
|
@ -1,5 +1,10 @@
|
|||
trigger: none
|
||||
|
||||
resources:
|
||||
repositories:
|
||||
- repository: 1esPipelines
|
||||
type: git
|
||||
name: 1ESPipelineTemplates/1ESPipelineTemplates
|
||||
ref: refs/tags/release
|
||||
parameters:
|
||||
- name: enable_linux_cuda
|
||||
type: boolean
|
||||
|
|
@ -17,11 +22,21 @@ parameters:
|
|||
- Release
|
||||
- RelWithDebInfo
|
||||
- MinSizeRel
|
||||
extends:
|
||||
# The pipeline extends the 1ES PT which will inject different SDL and compliance tasks.
|
||||
# For non-production pipelines, use "Unofficial" as defined below.
|
||||
# For productions pipelines, use "Official".
|
||||
template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines
|
||||
parameters:
|
||||
# Update the pool with your team's 1ES hosted pool.
|
||||
pool:
|
||||
name: 'onnxruntime-Win-CPU-2022' # Name of your hosted pool
|
||||
os: windows # OS of the image. This value cannot be a variable. Allowed values: windows, linux, macOS
|
||||
|
||||
stages:
|
||||
- template: stages/py-gpu-packaging-stage.yml
|
||||
parameters:
|
||||
enable_linux_cuda: ${{ parameters.enable_linux_cuda }}
|
||||
enable_windows_cuda: ${{ parameters.enable_windows_cuda }}
|
||||
cmake_build_type: ${{ parameters.cmake_build_type }}
|
||||
cuda_version: '11.8'
|
||||
stages:
|
||||
- template: stages/py-gpu-packaging-stage.yml
|
||||
parameters:
|
||||
enable_linux_cuda: ${{ parameters.enable_linux_cuda }}
|
||||
enable_windows_cuda: ${{ parameters.enable_windows_cuda }}
|
||||
cmake_build_type: ${{ parameters.cmake_build_type }}
|
||||
cuda_version: '11.8'
|
||||
|
|
|
|||
|
|
@ -1,5 +1,10 @@
|
|||
trigger: none
|
||||
|
||||
resources:
|
||||
repositories:
|
||||
- repository: 1esPipelines
|
||||
type: git
|
||||
name: 1ESPipelineTemplates/1ESPipelineTemplates
|
||||
ref: refs/tags/release
|
||||
parameters:
|
||||
- name: cmake_build_type
|
||||
type: string
|
||||
|
|
@ -9,10 +14,19 @@ parameters:
|
|||
- Release
|
||||
- RelWithDebInfo
|
||||
- MinSizeRel
|
||||
extends:
|
||||
# The pipeline extends the 1ES PT which will inject different SDL and compliance tasks.
|
||||
# For non-production pipelines, use "Unofficial" as defined below.
|
||||
# For productions pipelines, use "Official".
|
||||
template: v1/1ES.Official.PipelineTemplate.yml@1esPipelines
|
||||
parameters:
|
||||
# Update the pool with your team's 1ES hosted pool.
|
||||
pool:
|
||||
name: 'onnxruntime-Win-CPU-2022' # Name of your hosted pool
|
||||
os: windows # OS of the image. This value cannot be a variable. Allowed values: windows, linux, macOS
|
||||
|
||||
stages:
|
||||
- template: stages/py-gpu-packaging-stage.yml
|
||||
parameters:
|
||||
enable_windows_dml: true
|
||||
cmake_build_type: ${{ parameters.cmake_build_type }}
|
||||
publish_symbols: true
|
||||
stages:
|
||||
- template: stages/py-gpu-packaging-stage.yml
|
||||
parameters:
|
||||
enable_windows_dml: true
|
||||
cmake_build_type: ${{ parameters.cmake_build_type }}
|
||||
|
|
|
|||
|
|
@ -77,5 +77,3 @@ stages:
|
|||
build_py_parameters: ${{ parameters.build_py_parameters }}
|
||||
cmake_build_type: ${{ parameters.cmake_build_type }}
|
||||
qnn_sdk_version: ${{ parameters.qnn_sdk_version }}
|
||||
publish_symbols: true
|
||||
|
||||
|
|
|
|||
|
|
@ -48,10 +48,6 @@ parameters:
|
|||
- '3.12'
|
||||
- '3.13'
|
||||
|
||||
- name: publish_symbols
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
stages:
|
||||
- ${{ if eq(parameters.enable_windows_cuda, true) }}:
|
||||
- ${{ each python_version in parameters.PythonVersions }}:
|
||||
|
|
@ -89,5 +85,4 @@ stages:
|
|||
EP_BUILD_FLAGS: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos
|
||||
ENV_SETUP_SCRIPT: setup_env.bat
|
||||
EP_NAME: directml
|
||||
publish_symbols: ${{ parameters.publish_symbols }}
|
||||
cmake_build_type: ${{ parameters.cmake_build_type }}
|
||||
|
|
@ -99,6 +99,7 @@ stages:
|
|||
mv $(Build.BinariesDirectory)/dist ./dist
|
||||
pushd dist
|
||||
find . -name \*.whl -exec unzip -qq -o {} \;
|
||||
rm -r onnxruntime
|
||||
popd
|
||||
pushd ${{ parameters.cmake_build_type }}
|
||||
find . -name \*.whl -exec unzip -qq -o {} \;
|
||||
|
|
|
|||
|
|
@ -38,10 +38,6 @@ parameters:
|
|||
- RelWithDebInfo
|
||||
- MinSizeRel
|
||||
|
||||
- name: publish_symbols
|
||||
type: boolean
|
||||
default: false
|
||||
|
||||
stages:
|
||||
- stage: Win_py_${{ parameters.EP_NAME }}_Wheels_${{ replace(parameters.PYTHON_VERSION,'.','_') }}_Build
|
||||
dependsOn: []
|
||||
|
|
@ -52,6 +48,22 @@ stages:
|
|||
clean: all
|
||||
pool:
|
||||
name: onnxruntime-Win-CPU-2022
|
||||
os: windows
|
||||
templateContext:
|
||||
codeSignValidation:
|
||||
enabled: true
|
||||
break: true
|
||||
psscriptanalyzer:
|
||||
enabled: true
|
||||
sdl:
|
||||
binskim:
|
||||
enabled: true
|
||||
scanOutputDirectoryOnly: true
|
||||
targetPathPattern: '+:file|*.dll;-:file|DirectML.dll'
|
||||
outputs:
|
||||
- output: pipelineArtifact
|
||||
targetPath: $(Build.ArtifactStagingDirectory)
|
||||
artifactName: win_${{ parameters.EP_NAME }}_wheel_${{ parameters.PYTHON_VERSION }}
|
||||
variables:
|
||||
GRADLE_OPTS: '-Dorg.gradle.daemon=false'
|
||||
VSGenerator: 'Visual Studio 17 2022'
|
||||
|
|
@ -73,12 +85,6 @@ stages:
|
|||
addToPath: true
|
||||
architecture: 'x64'
|
||||
|
||||
- task: onebranch.pipeline.tsaoptions@1
|
||||
displayName: 'OneBranch TSAOptions'
|
||||
inputs:
|
||||
tsaConfigFilePath: '$(Build.SourcesDirectory)\.config\tsaoptions.json'
|
||||
appendSourceBranchName: false
|
||||
|
||||
- template: ../templates/download-deps.yml
|
||||
|
||||
- ${{ if ne(parameters.ENV_SETUP_SCRIPT, '') }}:
|
||||
|
|
@ -106,13 +112,6 @@ stages:
|
|||
arguments: --new_dir $(Build.BinariesDirectory)/deps
|
||||
workingDirectory: $(Build.BinariesDirectory)
|
||||
|
||||
- task: PowerShell@2
|
||||
displayName: 'Install ONNX'
|
||||
inputs:
|
||||
filePath: '$(Build.SourcesDirectory)/tools/ci_build/github/windows/install_third_party_deps.ps1'
|
||||
workingDirectory: '$(Build.BinariesDirectory)'
|
||||
arguments: -cpu_arch x64 -install_prefix $(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}\installed -build_config ${{ parameters.cmake_build_type }}
|
||||
|
||||
- template: ../templates/set-nightly-build-option-variable-step.yml
|
||||
|
||||
- task: PythonScript@0
|
||||
|
|
@ -126,19 +125,7 @@ stages:
|
|||
--cmake_generator "$(VSGenerator)"
|
||||
--enable_pybind
|
||||
--enable_onnx_tests
|
||||
--parallel --use_binskim_compliant_compile_flags --update
|
||||
$(TelemetryOption) ${{ parameters.BUILD_PY_PARAMETERS }} ${{ parameters.EP_BUILD_FLAGS }}
|
||||
workingDirectory: '$(Build.BinariesDirectory)'
|
||||
|
||||
# building with build.py so the parallelization parameters are added to the msbuild command
|
||||
- task: PythonScript@0
|
||||
displayName: 'Build'
|
||||
inputs:
|
||||
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
|
||||
arguments: >
|
||||
--config ${{ parameters.cmake_build_type }}
|
||||
--build_dir $(Build.BinariesDirectory)
|
||||
--parallel --build
|
||||
--parallel --use_binskim_compliant_compile_flags --update --build
|
||||
$(TelemetryOption) ${{ parameters.BUILD_PY_PARAMETERS }} ${{ parameters.EP_BUILD_FLAGS }}
|
||||
workingDirectory: '$(Build.BinariesDirectory)'
|
||||
|
||||
|
|
@ -164,48 +151,11 @@ stages:
|
|||
Contents: '*.whl'
|
||||
TargetFolder: '$(Build.ArtifactStagingDirectory)'
|
||||
|
||||
- task: PublishBuildArtifacts@1
|
||||
displayName: 'Publish Artifact: ONNXRuntime python wheel'
|
||||
inputs:
|
||||
ArtifactName: onnxruntime_${{ parameters.EP_NAME }}
|
||||
|
||||
- ${{ if eq(parameters.publish_symbols, true) }}:
|
||||
- task: PublishSymbols@2
|
||||
displayName: 'Publish symbols'
|
||||
condition: and (succeeded(), or(eq(variables['Build.SourceBranch'], 'refs/heads/main'), startsWith(variables['Build.SourceBranch'], 'refs/heads/rel-')))
|
||||
inputs:
|
||||
SymbolsFolder: '$(Build.BinariesDirectory)\${{ parameters.cmake_build_type }}\${{ parameters.cmake_build_type }}'
|
||||
SearchPattern: |
|
||||
onnxruntime_pybind11_state.pdb
|
||||
onnxruntime_providers_shared.pdb
|
||||
IndexSources: true
|
||||
SymbolServerType: TeamServices
|
||||
SymbolExpirationInDays: 3650
|
||||
SymbolsArtifactName: 'win_${{ parameters.EP_NAME }}_${{ parameters.PYTHON_VERSION }}_$(Build.BuildNumber)'
|
||||
|
||||
- script: |
|
||||
7z x *.whl
|
||||
workingDirectory: '$(Build.ArtifactStagingDirectory)'
|
||||
displayName: 'unzip the package'
|
||||
|
||||
- task: CredScan@3
|
||||
displayName: 'Run CredScan'
|
||||
inputs:
|
||||
debugMode: false
|
||||
continueOnError: true
|
||||
|
||||
- task: BinSkim@4
|
||||
displayName: 'Run BinSkim'
|
||||
inputs:
|
||||
AnalyzeTargetGlob: '+:file|$(Build.ArtifactStagingDirectory)\**\*.dll;-:file|$(Build.ArtifactStagingDirectory)\**\DirectML.dll'
|
||||
|
||||
- task: TSAUpload@2
|
||||
displayName: 'TSA upload'
|
||||
condition: and (succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main'))
|
||||
inputs:
|
||||
GdnPublishTsaOnboard: false
|
||||
GdnPublishTsaConfigFile: '$(Build.sourcesDirectory)\.gdn\.gdntsa'
|
||||
|
||||
- template: ../templates/component-governance-component-detection-steps.yml
|
||||
parameters:
|
||||
condition: 'succeeded'
|
||||
|
|
@ -235,7 +185,7 @@ stages:
|
|||
|
||||
- template: ../templates/flex-downloadPipelineArtifact.yml
|
||||
parameters:
|
||||
ArtifactName: onnxruntime_${{ parameters.EP_NAME }}
|
||||
ArtifactName: win_${{ parameters.EP_NAME }}_wheel_${{ parameters.PYTHON_VERSION }}
|
||||
StepName: 'Download Pipeline Artifact - Windows GPU Build'
|
||||
TargetPath: '$(Build.ArtifactStagingDirectory)'
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue