mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-28 20:11:22 +00:00
Move DML build job's Prefast task to a CPU machine pool (#17192)
### Description Move DML build job's Prefast task to a CPU machine pool which has larger memory. The current one runs out of memory in every run. ### Motivation and Context To fix the broken python packaging pipeline.
This commit is contained in:
parent
e0022d061f
commit
0cccbcc47b
5 changed files with 83 additions and 36 deletions
|
|
@ -35,7 +35,9 @@ steps:
|
|||
msBuildArchitecture: x64
|
||||
msBuildVersion: 17.0
|
||||
msBuildCommandline: '"C:\Program Files\Microsoft Visual Studio\2022\Enterprise\MSBuild\Current\Bin\amd64\msbuild.exe" "$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln" /p:platform="${{parameters.msbuildPlatform}}" /p:configuration="RelWithDebInfo" /p:CAExcludePath="$(Build.BinariesDirectory);$(Build.SourcesDirectory)\cmake;C:\program files (x86)" /p:VisualStudioVersion="17.0" /m /p:PreferredToolArchitecture=x64'
|
||||
excludedPaths: '$(Build.SourcesDirectory)\b#$(Build.SourcesDirectory)\cmake#C:\program files'
|
||||
excludedPaths: '$(Build.SourcesDirectory)\b#$(Build.SourcesDirectory)\cmake#C:\program files#C:\program files (x86)#C:\program files'
|
||||
rulesetName: Custom
|
||||
customRuleset: $(Build.SourcesDirectory)\cmake\Sdl.ruleset
|
||||
|
||||
- task: SdtReport@2
|
||||
displayName: 'Create Security Analysis Report'
|
||||
|
|
|
|||
|
|
@ -6,6 +6,11 @@ parameters:
|
|||
|
||||
steps:
|
||||
- ${{ if eq(variables['System.TeamProject'], 'Lotus') }}:
|
||||
- task: DeleteFiles@1
|
||||
inputs:
|
||||
contents: $(Build.BinariesDirectory)/*
|
||||
displayName: 'Clean up build directory'
|
||||
|
||||
- task: ms.vss-governance-buildtask.governance-build-task-component-detection.ComponentGovernanceComponentDetection@0
|
||||
displayName: 'Component Detection'
|
||||
condition:
|
||||
|
|
|
|||
|
|
@ -328,7 +328,7 @@ jobs:
|
|||
displayName: 'Run the PREfast SDL Native Rules for MSBuild'
|
||||
inputs:
|
||||
msBuildArchitecture: amd64
|
||||
setupCommandlines: 'python $(Build.SourcesDirectory)\tools\ci_build\build.py --config RelWithDebInfo --build_dir $(Build.SourcesDirectory)\b --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 17 2022" --build_shared_lib --enable_onnx_tests ${{ parameters.additionalBuildFlags }} --cmake_extra_defines onnxruntime_ENABLE_STATIC_ANALYSIS=ON'
|
||||
setupCommandlines: 'python $(Build.SourcesDirectory)\tools\ci_build\build.py --config RelWithDebInfo --build_dir $(Build.SourcesDirectory)\b --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 17 2022" --build_shared_lib --enable_onnx_tests ${{ parameters.additionalBuildFlags }} --cmake_extra_defines onnxruntime_ENABLE_STATIC_ANALYSIS=ON onnxruntime_ENABLE_LTO=OFF'
|
||||
msBuildCommandline: '"C:\Program Files\Microsoft Visual Studio\2022\Enterprise\MSBuild\Current\Bin\amd64\msbuild.exe" "$(Build.SourcesDirectory)\b\RelWithDebInfo\onnxruntime.sln" /p:RunCodeAnalysis=true /p:platform=${{ parameters.msbuildPlatform }} /p:configuration=RelWithDebInfo /p:VisualStudioVersion="17.0" /m /p:PreferredToolArchitecture=x64'
|
||||
excludedPaths: '$(Build.SourcesDirectory)\b#$(Build.SourcesDirectory)\cmake#C:\program files#C:\program files (x86)#C:\program files'
|
||||
rulesetName: Custom
|
||||
|
|
|
|||
|
|
@ -247,10 +247,16 @@ stages:
|
|||
condition: and (succeeded(), and(eq(variables['buildArch'], 'x64'), eq(variables['PythonVersion'], '3.8')))
|
||||
inputs:
|
||||
msBuildArchitecture: amd64
|
||||
setupCommandlines: 'python $(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_generator "Visual Studio 17 2022" --enable_pybind --enable_onnx_tests --parallel $(TelemetryOption) --update --cmake_extra_defines onnxruntime_ENABLE_STATIC_ANALYSIS=ON'
|
||||
msBuildCommandline: '"C:\Program Files\Microsoft Visual Studio\2022\Enterprise\MSBuild\Current\Bin\amd64\msbuild.exe" "$(Build.BinariesDirectory)\Debug\onnxruntime.sln" /p:platform="$(MsbuildPlatform)" /p:configuration=Debug /p:VisualStudioVersion="17.0" /m /p:PreferredToolArchitecture=x64'
|
||||
excludedPaths: '$(Build.BinariesDirectory)#$(Build.SourcesDirectory)\cmake#C:\program files'
|
||||
setupCommandlines: 'python $(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.SourcesDirectory)\b --skip_submodule_sync --cmake_generator "Visual Studio 17 2022" --enable_pybind --enable_onnx_tests --parallel $(TelemetryOption) --update --cmake_extra_defines onnxruntime_ENABLE_STATIC_ANALYSIS=ON onnxruntime_ENABLE_LTO=OFF'
|
||||
msBuildCommandline: '"C:\Program Files\Microsoft Visual Studio\2022\Enterprise\MSBuild\Current\Bin\amd64\msbuild.exe" "$(Build.SourcesDirectory)\b\Debug\onnxruntime.sln" /p:RunCodeAnalysis=true /p:platform="$(MsbuildPlatform)" /p:configuration=Debug /p:VisualStudioVersion="17.0" /m /p:PreferredToolArchitecture=x64'
|
||||
excludedPaths: '$(Build.SourcesDirectory)\b#$(Build.SourcesDirectory)\cmake#C:\program files#C:\program files (x86)#C:\program files'
|
||||
rulesetName: Custom
|
||||
customRuleset: $(Build.SourcesDirectory)\cmake\Sdl.ruleset
|
||||
|
||||
- task: SdtReport@2
|
||||
displayName: 'Create Security Analysis Report'
|
||||
inputs:
|
||||
SDLNativeRules: true
|
||||
|
||||
- task: TSAUpload@2
|
||||
displayName: 'TSA upload'
|
||||
|
|
|
|||
|
|
@ -22,6 +22,65 @@ parameters:
|
|||
default: ''
|
||||
|
||||
jobs:
|
||||
- ${{ if eq(parameters.PYTHON_VERSION, '3.8') }}:
|
||||
- job: Win_py_${{ parameters.EP_NAME }}_Wheels_StaticAnalysis
|
||||
timeoutInMinutes: 240
|
||||
workspace:
|
||||
clean: all
|
||||
pool: onnxruntime-Win-CPU-2022
|
||||
steps:
|
||||
- checkout: self
|
||||
clean: true
|
||||
submodules: none
|
||||
- task: UsePythonVersion@0
|
||||
inputs:
|
||||
versionSpec: 3.8
|
||||
addToPath: true
|
||||
architecture: 'x64'
|
||||
- task: onebranch.pipeline.tsaoptions@1
|
||||
displayName: 'OneBranch TSAOptions'
|
||||
inputs:
|
||||
tsaConfigFilePath: '$(Build.SourcesDirectory)\.config\tsaoptions.json'
|
||||
appendSourceBranchName: false
|
||||
|
||||
- template: download-deps.yml
|
||||
|
||||
- template: jobs/set-winenv.yml
|
||||
parameters:
|
||||
EnvSetupScript: ${{ parameters.ENV_SETUP_SCRIPT }}
|
||||
DownloadCUDA: true
|
||||
|
||||
- task: PythonScript@0
|
||||
displayName: 'Update deps.txt'
|
||||
inputs:
|
||||
scriptPath: $(Build.SourcesDirectory)/tools/ci_build/replace_urls_in_deps.py
|
||||
arguments: --new_dir $(Build.BinariesDirectory)/deps
|
||||
workingDirectory: $(Build.BinariesDirectory)
|
||||
|
||||
- task: SDLNativeRules@3
|
||||
displayName: 'Run the PREfast SDL Native Rules for MSBuild'
|
||||
inputs:
|
||||
msBuildArchitecture: amd64
|
||||
setupCommandlines: 'python $(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.SourcesDirectory)\b --skip_submodule_sync --cmake_generator "Visual Studio 17 2022" --enable_pybind ${{ parameters.BUILD_PY_PARAMETERS }} ${{ parameters.EP_BUILD_FLAGS }} --update --cmake_extra_defines onnxruntime_ENABLE_STATIC_ANALYSIS=ON onnxruntime_ENABLE_LTO=OFF'
|
||||
msBuildCommandline: '"C:\Program Files\Microsoft Visual Studio\2022\Enterprise\MSBuild\Current\Bin\amd64\msbuild.exe" "$(Build.SourcesDirectory)\b\Debug\onnxruntime.sln" /p:RunCodeAnalysis=true /p:platform=x64 /p:configuration=Debug /p:VisualStudioVersion="17.0" /m /p:PreferredToolArchitecture=x64'
|
||||
excludedPaths: '$(Build.SourcesDirectory)\b#$(Build.SourcesDirectory)\cmake#C:\program files#C:\program files (x86)#C:\program files'
|
||||
rulesetName: Custom
|
||||
customRuleset: $(Build.SourcesDirectory)\cmake\Sdl.ruleset
|
||||
publishXML: true
|
||||
|
||||
- task: SdtReport@2
|
||||
displayName: 'Create Security Analysis Report'
|
||||
inputs:
|
||||
SDLNativeRules: true
|
||||
|
||||
- task: TSAUpload@2
|
||||
displayName: 'TSA upload'
|
||||
condition: and (succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main'))
|
||||
inputs:
|
||||
GdnPublishTsaOnboard: false
|
||||
GdnPublishTsaConfigFile: '$(Build.sourcesDirectory)\.gdn\.gdntsa'
|
||||
|
||||
|
||||
- job: Win_py_${{ parameters.EP_NAME }}_Wheels_${{ replace(parameters.PYTHON_VERSION,'.','_') }}
|
||||
timeoutInMinutes: 240
|
||||
workspace:
|
||||
|
|
@ -153,7 +212,6 @@ jobs:
|
|||
inputs:
|
||||
AnalyzeTargetGlob: '+:file|$(Build.ArtifactStagingDirectory)\**\*.dll;-:file|$(Build.ArtifactStagingDirectory)\**\DirectML.dll'
|
||||
|
||||
|
||||
- powershell: |
|
||||
python -m pip uninstall -y ort-nightly-gpu ort-nightly onnxruntime onnxruntime-gpu -qq
|
||||
Get-ChildItem -Path $(Build.ArtifactStagingDirectory)/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname tabulate}
|
||||
|
|
@ -162,36 +220,12 @@ jobs:
|
|||
workingDirectory: '$(Build.BinariesDirectory)\RelWithDebInfo\RelWithDebInfo'
|
||||
displayName: 'Run Python Tests'
|
||||
|
||||
- ${{ if eq(parameters.PYTHON_VERSION, '3.8') }}:
|
||||
- task: DeleteFiles@1
|
||||
displayName: 'Delete files from $(Build.BinariesDirectory)\RelWithDebInfo'
|
||||
condition: succeeded()
|
||||
inputs:
|
||||
SourceFolder: '$(Build.BinariesDirectory)\RelWithDebInfo'
|
||||
Contents: |
|
||||
**/*.obj
|
||||
**/*.pdb
|
||||
**/*.dll
|
||||
|
||||
#Manually set msBuildCommandline so that we can also set CAExcludePath
|
||||
- task: SDLNativeRules@3
|
||||
displayName: 'Run the PREfast SDL Native Rules for MSBuild'
|
||||
inputs:
|
||||
msBuildArchitecture: amd64
|
||||
setupCommandlines: 'python $(Build.SourcesDirectory)\tools\ci_build\build.py --config RelWithDebInfo --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_generator "$(VSGenerator)" --enable_pybind --enable_onnx_tests --parallel $(TelemetryOption) ${{ parameters.BUILD_PY_PARAMETERS }} ${{ parameters.EP_BUILD_FLAGS }} --update --cmake_extra_defines onnxruntime_ENABLE_STATIC_ANALYSIS=ON'
|
||||
${{if eq(variables.VSGenerator, 'Visual Studio 16 2019')}}:
|
||||
msBuildCommandline: '"C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\MSBuild\Current\Bin\amd64\msbuild.exe" "$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln" /p:platform=x64 /p:configuration="RelWithDebInfo" /p:VisualStudioVersion="16.0" /m /p:PreferredToolArchitecture=x64'
|
||||
${{else}}:
|
||||
msBuildCommandline: '"C:\Program Files\Microsoft Visual Studio\2022\Enterprise\MSBuild\Current\Bin\amd64\msbuild.exe" "$(Build.BinariesDirectory)\RelWithDebInfo\onnxruntime.sln" /p:platform=x64 /p:configuration="RelWithDebInfo" /p:VisualStudioVersion="17.0" /m /p:PreferredToolArchitecture=x64'
|
||||
excludedPaths: '$(Build.BinariesDirectory)#$(Build.SourcesDirectory)\cmake#C:\program files (x86)#C:\Program Files'
|
||||
|
||||
- task: TSAUpload@2
|
||||
displayName: 'TSA upload'
|
||||
condition: and (succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main'))
|
||||
inputs:
|
||||
GdnPublishTsaOnboard: false
|
||||
GdnPublishTsaConfigFile: '$(Build.sourcesDirectory)\.gdn\.gdntsa'
|
||||
continueOnError: true
|
||||
- task: TSAUpload@2
|
||||
displayName: 'TSA upload'
|
||||
condition: and (succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/main'))
|
||||
inputs:
|
||||
GdnPublishTsaOnboard: false
|
||||
GdnPublishTsaConfigFile: '$(Build.sourcesDirectory)\.gdn\.gdntsa'
|
||||
|
||||
- template: component-governance-component-detection-steps.yml
|
||||
parameters:
|
||||
|
|
|
|||
Loading…
Reference in a new issue