mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-29 20:14:01 +00:00
Refactor Windows CI pipeline yaml files (#9672)
This commit is contained in:
Changming Sun
GitHub
parent
24e35fba32
commit
53afaefe3b
6 changed files with 354 additions and 290 deletions
|
|
@ -49,6 +49,9 @@ endif()
|
|||
onnxruntime_add_shared_library_module(onnxruntime_pybind11_state ${onnxruntime_pybind_srcs})
|
||||
if(MSVC)
|
||||
target_compile_options(onnxruntime_pybind11_state PRIVATE "$<$<COMPILE_LANGUAGE:CUDA>:SHELL:--compiler-options /utf-8>" "$<$<NOT:$<COMPILE_LANGUAGE:CUDA>>:/utf-8>")
|
||||
if(onnxruntime_ENABLE_TRAINING)
|
||||
target_compile_options(onnxruntime_pybind11_state PRIVATE "/bigobj")
|
||||
endif()
|
||||
endif()
|
||||
if(HAS_CAST_FUNCTION_TYPE)
|
||||
target_compile_options(onnxruntime_pybind11_state PRIVATE "-Wno-cast-function-type")
|
||||
|
|
|
|||
|
|
@ -1,25 +0,0 @@
|
|||
trigger: none
|
||||
|
||||
jobs:
|
||||
- template: templates/win-ci-2019.yml
|
||||
parameters:
|
||||
AgentPool : 'Win-CPU-2019'
|
||||
JobName: 'Win_CPU_Training'
|
||||
# TODO fix test failures and remove --skip_onnx_tests
|
||||
BuildCommand: >-
|
||||
--build_dir $(Build.BinariesDirectory)
|
||||
--build_shared_lib
|
||||
--cmake_generator "Visual Studio 16 2019"
|
||||
--enable_onnx_tests
|
||||
--enable_training
|
||||
--skip_onnx_tests
|
||||
--skip_submodule_sync
|
||||
--use_openmp
|
||||
BuildArch: 'x64'
|
||||
EnvSetupScript: 'setup_env.bat'
|
||||
sln_platform: 'x64'
|
||||
BuildConfigurations: ['Debug', 'RelWithDebInfo']
|
||||
EnableLto: false
|
||||
# Enable unreleased onnx opsets in CI builds
|
||||
# This facilitates testing the implementation for the new opsets
|
||||
AllowReleasedOpsetOnly: '0'
|
||||
|
|
@ -1,31 +0,0 @@
|
|||
trigger: none
|
||||
|
||||
jobs:
|
||||
- template: templates/win-ci-2019.yml
|
||||
parameters:
|
||||
AgentPool : 'Win-GPU-2019'
|
||||
JobName: 'Win_GPU_Training'
|
||||
# TODO fix test failures and remove --skip_onnx_tests
|
||||
BuildCommand: >-
|
||||
--build_dir $(Build.BinariesDirectory)
|
||||
--build_shared_lib
|
||||
--cmake_generator "Visual Studio 16 2019"
|
||||
--enable_onnx_tests
|
||||
--enable_training
|
||||
--skip_onnx_tests
|
||||
--skip_submodule_sync
|
||||
--use_cuda
|
||||
--cuda_version 10.2
|
||||
--cuda_home "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.2"
|
||||
--cudnn_home "C:\local\cudnn-10.2-windows10-x64-v8.0.3.33\cuda"
|
||||
--cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52"
|
||||
BuildArch: 'x64'
|
||||
EnvSetupScript: 'setup_env_cuda.bat'
|
||||
sln_platform: 'x64'
|
||||
CudaVersion: '10.2'
|
||||
OrtPackageId: 'Microsoft.ML.OnnxRuntime.Gpu'
|
||||
BuildConfigurations: ['RelWithDebInfo']
|
||||
# Enable unreleased onnx opsets in CI builds
|
||||
# This facilitates testing the implementation for the new opsets
|
||||
AllowReleasedOpsetOnly: '0'
|
||||
DoCompliance: 'true'
|
||||
|
|
@ -24,11 +24,20 @@ parameters:
|
|||
type: boolean
|
||||
default: false
|
||||
|
||||
- name: EnablePython
|
||||
type: boolean
|
||||
default: true
|
||||
|
||||
- name: RunOnnxRuntimeTests
|
||||
displayName: Run Tests?
|
||||
type: boolean
|
||||
default: true
|
||||
|
||||
- name: RunStaticCodeAnalysis
|
||||
displayName: Run Static Code Analysis
|
||||
type: boolean
|
||||
default: true
|
||||
|
||||
jobs:
|
||||
- job: build_${{ parameters.job_name_suffix }}
|
||||
variables:
|
||||
|
|
@ -83,15 +92,16 @@ jobs:
|
|||
workingDirectory: '$(Build.BinariesDirectory)'
|
||||
displayName: 'Install python modules'
|
||||
|
||||
- powershell: |
|
||||
$Env:USE_MSVC_STATIC_RUNTIME=1
|
||||
$Env:ONNX_ML=1
|
||||
$Env:CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=OFF -DProtobuf_USE_STATIC_LIBS=ON -DONNX_USE_LITE_PROTO=ON -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=${{ parameters.buildArch }}-windows-static"
|
||||
python setup.py bdist_wheel
|
||||
python -m pip uninstall -y onnx -qq
|
||||
Get-ChildItem -Path dist/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname}
|
||||
workingDirectory: '$(Build.SourcesDirectory)\cmake\external\onnx'
|
||||
displayName: 'Install ONNX'
|
||||
- ${{ if eq(parameters.RunOnnxRuntimeTests, true) }}:
|
||||
- powershell: |
|
||||
$Env:USE_MSVC_STATIC_RUNTIME=1
|
||||
$Env:ONNX_ML=1
|
||||
$Env:CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=OFF -DProtobuf_USE_STATIC_LIBS=ON -DONNX_USE_LITE_PROTO=ON -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=${{ parameters.buildArch }}-windows-static"
|
||||
python setup.py bdist_wheel
|
||||
python -m pip uninstall -y onnx -qq
|
||||
Get-ChildItem -Path dist/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname}
|
||||
workingDirectory: '$(Build.SourcesDirectory)\cmake\external\onnx'
|
||||
displayName: 'Install ONNX'
|
||||
|
||||
- task: NuGetToolInstaller@0
|
||||
displayName: Use Nuget 5.7.0
|
||||
|
|
@ -111,7 +121,7 @@ jobs:
|
|||
displayName: 'Generate cmake config'
|
||||
inputs:
|
||||
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
|
||||
arguments: '--config ${{ parameters.BuildConfig }} --build_dir $(Build.BinariesDirectory) ${{ parameters.UseOmp }} --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_wheel --use_winml --build_shared_lib --enable_onnx_tests --enable_wcos ${{ parameters.additionalBuildFlags }}'
|
||||
arguments: '--config ${{ parameters.BuildConfig }} --build_dir $(Build.BinariesDirectory) ${{ parameters.UseOmp }} --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_shared_lib --enable_onnx_tests ${{ parameters.additionalBuildFlags }}'
|
||||
workingDirectory: '$(Build.BinariesDirectory)'
|
||||
|
||||
- task: VSBuild@1
|
||||
|
|
@ -127,12 +137,13 @@ jobs:
|
|||
workingFolder: '$(Build.BinariesDirectory)\${{ parameters.BuildConfig }}'
|
||||
createLogFile: true
|
||||
|
||||
- task: PythonScript@0
|
||||
displayName: 'Build wheel'
|
||||
inputs:
|
||||
scriptPath: '$(Build.SourcesDirectory)\setup.py'
|
||||
arguments: 'bdist_wheel'
|
||||
workingDirectory: '$(Build.BinariesDirectory)\${{ parameters.BuildConfig }}\${{ parameters.BuildConfig }}'
|
||||
- ${{ if eq(parameters.EnablePython, true) }}:
|
||||
- task: PythonScript@0
|
||||
displayName: 'Build wheel'
|
||||
inputs:
|
||||
scriptPath: '$(Build.SourcesDirectory)\setup.py'
|
||||
arguments: 'bdist_wheel'
|
||||
workingDirectory: '$(Build.BinariesDirectory)\${{ parameters.BuildConfig }}\${{ parameters.BuildConfig }}'
|
||||
|
||||
|
||||
- task: NuGetCommand@2
|
||||
|
|
@ -161,16 +172,22 @@ jobs:
|
|||
arguments: '--configuration ${{ parameters.BuildConfig }} -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId) --blame'
|
||||
workingDirectory: '$(Build.SourcesDirectory)\csharp'
|
||||
|
||||
- powershell: |
|
||||
Get-ChildItem -Path dist/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname}
|
||||
python $(Build.SourcesDirectory)\tools\ci_build\build.py --config ${{ parameters.BuildConfig }} --build_dir $(Build.BinariesDirectory) ${{ parameters.UseOmp }} --skip_submodule_sync --build_shared_lib --test --cmake_generator "Visual Studio 16 2019" --build_wheel --use_winml --build_shared_lib --enable_onnx_tests --enable_wcos ${{ parameters.additionalBuildFlags }}
|
||||
|
||||
workingDirectory: '$(Build.BinariesDirectory)\${{ parameters.BuildConfig }}\${{ parameters.BuildConfig }}'
|
||||
condition: and(succeeded(), eq('${{ parameters.RunOnnxRuntimeTests}}', true))
|
||||
displayName: 'Run tests'
|
||||
- ${{ if eq(parameters.EnablePython, true) }}:
|
||||
- powershell: |
|
||||
Get-ChildItem -Path dist/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname}
|
||||
|
||||
workingDirectory: '$(Build.BinariesDirectory)\${{ parameters.BuildConfig }}\${{ parameters.BuildConfig }}'
|
||||
displayName: 'Install onnxruntime wheel'
|
||||
|
||||
- ${{ if eq(parameters.RunOnnxRuntimeTests, true) }}:
|
||||
- powershell: |
|
||||
python $(Build.SourcesDirectory)\tools\ci_build\build.py --config ${{ parameters.BuildConfig }} --build_dir $(Build.BinariesDirectory) ${{ parameters.UseOmp }} --skip_submodule_sync --build_shared_lib --test --cmake_generator "Visual Studio 16 2019" --build_shared_lib --enable_onnx_tests ${{ parameters.additionalBuildFlags }}
|
||||
|
||||
workingDirectory: '$(Build.BinariesDirectory)\${{ parameters.BuildConfig }}\${{ parameters.BuildConfig }}'
|
||||
displayName: 'Run tests'
|
||||
|
||||
|
||||
- ${{ if eq(parameters.BuildConfig, 'RelWithDebInfo') }}:
|
||||
- ${{ if eq(parameters.RunStaticCodeAnalysis, true) }}:
|
||||
- task: DeleteFiles@1
|
||||
displayName: 'Delete binaries files from $(Build.BinariesDirectory)\RelWithDebInfo'
|
||||
inputs:
|
||||
|
|
@ -184,7 +201,7 @@ jobs:
|
|||
displayName: 'Generate cmake config'
|
||||
inputs:
|
||||
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
|
||||
arguments: '--config RelWithDebInfo --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_wheel --build_shared_lib --enable_onnx_tests ${{ parameters.additionalBuildFlags }} --cmake_extra_defines onnxruntime_ENABLE_STATIC_ANALYSIS=ON'
|
||||
arguments: '--config RelWithDebInfo --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_shared_lib --enable_onnx_tests ${{ parameters.additionalBuildFlags }} --cmake_extra_defines onnxruntime_ENABLE_STATIC_ANALYSIS=ON'
|
||||
workingDirectory: '$(Build.BinariesDirectory)'
|
||||
|
||||
|
||||
|
|
@ -202,13 +219,14 @@ jobs:
|
|||
createLogFile: true
|
||||
|
||||
|
||||
- task: PublishTestResults@2
|
||||
displayName: 'Publish unit test results'
|
||||
inputs:
|
||||
testResultsFiles: '**/*.results.xml'
|
||||
searchFolder: '$(Build.BinariesDirectory)/${{ parameters.BuildConfig }}'
|
||||
testRunTitle: 'Unit Test Run'
|
||||
condition: succeededOrFailed()
|
||||
- ${{ if eq(parameters.RunOnnxRuntimeTests, true) }}:
|
||||
- task: PublishTestResults@2
|
||||
displayName: 'Publish unit test results'
|
||||
inputs:
|
||||
testResultsFiles: '**/*.results.xml'
|
||||
searchFolder: '$(Build.BinariesDirectory)/${{ parameters.BuildConfig }}'
|
||||
testRunTitle: 'Unit Test Run'
|
||||
condition: succeededOrFailed()
|
||||
|
||||
- template: component-governance-component-detection-steps.yml
|
||||
parameters :
|
||||
|
|
|
|||
|
|
@ -4,39 +4,104 @@ parameters:
|
|||
type: boolean
|
||||
default: true
|
||||
|
||||
jobs:
|
||||
- template: templates/win-cpu-ci.yml
|
||||
parameters:
|
||||
BuildConfig: 'Debug'
|
||||
UseOmp: '--use_openmp'
|
||||
EnvSetupScript: setup_env.bat
|
||||
buildArch: x64
|
||||
additionalBuildFlags: --use_dnnl --build_java --build_nodejs
|
||||
msbuildPlatform: x64
|
||||
isX86: false
|
||||
job_name_suffix: x64_debug
|
||||
RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }}
|
||||
stages:
|
||||
- stage: x64_debug
|
||||
dependsOn: []
|
||||
jobs:
|
||||
- template: templates/win-cpu-ci.yml
|
||||
parameters:
|
||||
BuildConfig: 'Debug'
|
||||
UseOmp: '--use_openmp'
|
||||
EnvSetupScript: setup_env.bat
|
||||
buildArch: x64
|
||||
additionalBuildFlags: --use_dnnl --build_java --build_nodejs --build_wheel
|
||||
msbuildPlatform: x64
|
||||
isX86: false
|
||||
job_name_suffix: x64_debug
|
||||
RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }}
|
||||
RunStaticCodeAnalysis: false
|
||||
|
||||
- template: templates/win-cpu-ci.yml
|
||||
parameters:
|
||||
BuildConfig: 'RelWithDebInfo'
|
||||
UseOmp: ''
|
||||
EnvSetupScript: setup_env.bat
|
||||
buildArch: x64
|
||||
additionalBuildFlags: --use_dnnl --build_java --build_nodejs
|
||||
msbuildPlatform: x64
|
||||
isX86: false
|
||||
job_name_suffix: x64_release
|
||||
RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }}
|
||||
- stage: x64_release
|
||||
dependsOn: []
|
||||
jobs:
|
||||
- template: templates/win-cpu-ci.yml
|
||||
parameters:
|
||||
BuildConfig: 'RelWithDebInfo'
|
||||
UseOmp: ''
|
||||
EnvSetupScript: setup_env.bat
|
||||
buildArch: x64
|
||||
# Compare to our Nuget packaging pipeline, this job has "--build_wheel --use_dnnl" but doesn't have "--enable_lto --disable_rtti --use_telemetry --enable_wcos"
|
||||
# Python bindings use typeid so I can't disable RTTI here. So we may need to split it to two pipelines
|
||||
additionalBuildFlags: --build_wheel --use_dnnl --build_java --build_nodejs
|
||||
msbuildPlatform: x64
|
||||
isX86: false
|
||||
job_name_suffix: x64_release
|
||||
RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }}
|
||||
RunStaticCodeAnalysis: true
|
||||
|
||||
- template: templates/win-cpu-ci.yml
|
||||
parameters:
|
||||
BuildConfig: 'RelWithDebInfo'
|
||||
UseOmp: ''
|
||||
EnvSetupScript: setup_env_x86.bat
|
||||
buildArch: x86
|
||||
additionalBuildFlags:
|
||||
msbuildPlatform: Win32
|
||||
isX86: true
|
||||
job_name_suffix: x86_release
|
||||
RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }}
|
||||
- stage: x64_release_winml
|
||||
dependsOn: []
|
||||
jobs:
|
||||
- template: templates/win-cpu-ci.yml
|
||||
parameters:
|
||||
BuildConfig: 'RelWithDebInfo'
|
||||
UseOmp: ''
|
||||
EnvSetupScript: setup_env.bat
|
||||
buildArch: x64
|
||||
additionalBuildFlags: --use_winml --enable_wcos --disable_rtti
|
||||
msbuildPlatform: x64
|
||||
isX86: false
|
||||
job_name_suffix: x64_release_winml
|
||||
RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }}
|
||||
# WinML has many warnings
|
||||
RunStaticCodeAnalysis: false
|
||||
EnablePython: false
|
||||
|
||||
- stage: x86_release
|
||||
dependsOn: []
|
||||
jobs:
|
||||
- template: templates/win-cpu-ci.yml
|
||||
parameters:
|
||||
BuildConfig: 'RelWithDebInfo'
|
||||
UseOmp: ''
|
||||
EnvSetupScript: setup_env_x86.bat
|
||||
buildArch: x86
|
||||
additionalBuildFlags: --build_wheel
|
||||
msbuildPlatform: Win32
|
||||
isX86: true
|
||||
job_name_suffix: x86_release
|
||||
RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }}
|
||||
RunStaticCodeAnalysis: true
|
||||
|
||||
- stage: training_x64_debug
|
||||
dependsOn: []
|
||||
jobs:
|
||||
- template: templates/win-cpu-ci.yml
|
||||
parameters:
|
||||
BuildConfig: 'Debug'
|
||||
UseOmp: ''
|
||||
EnvSetupScript: setup_env.bat
|
||||
buildArch: x64
|
||||
additionalBuildFlags: --enable_training --build_wheel
|
||||
msbuildPlatform: x64
|
||||
isX86: false
|
||||
job_name_suffix: training_x64_debug
|
||||
RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }}
|
||||
RunStaticCodeAnalysis: false
|
||||
|
||||
- stage: training_x64_release
|
||||
dependsOn: []
|
||||
jobs:
|
||||
- template: templates/win-cpu-ci.yml
|
||||
parameters:
|
||||
BuildConfig: 'RelWithDebInfo'
|
||||
UseOmp: ''
|
||||
EnvSetupScript: setup_env.bat
|
||||
buildArch: x64
|
||||
additionalBuildFlags: --enable_training --build_wheel
|
||||
msbuildPlatform: x64
|
||||
isX86: false
|
||||
job_name_suffix: training_x64_release
|
||||
RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }}
|
||||
# Liqun is fixing them
|
||||
RunStaticCodeAnalysis: false
|
||||
|
|
|
|||
|
|
@ -4,193 +4,227 @@ parameters:
|
|||
type: boolean
|
||||
default: true
|
||||
|
||||
jobs:
|
||||
- job: 'build'
|
||||
pool: 'Win-GPU-2019'
|
||||
strategy:
|
||||
matrix:
|
||||
cuda:
|
||||
additionalBuildFlags: --build_java --build_nodejs --use_cuda --cuda_version=11.4 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.4" --cudnn_home="C:\local\cudnn-11.4-windows-x64-v8.2.2.26\cuda" --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=52 --gen_doc validate
|
||||
EnvSetupScript: setup_env_cuda_11.bat
|
||||
ORT_EP_NAME: CUDA
|
||||
dml:
|
||||
additionalBuildFlags: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos --use_winml
|
||||
EnvSetupScript: setup_env.bat
|
||||
ORT_EP_NAME: DML
|
||||
variables:
|
||||
OrtPackageId: 'Microsoft.ML.OnnxRuntime.Gpu'
|
||||
MsbuildArguments: '-maxcpucount'
|
||||
TESTONGPU: 'ON'
|
||||
OnnxRuntimeBuildDirectory: '$(Build.BinariesDirectory)'
|
||||
DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
|
||||
setVcvars: true
|
||||
BuildConfig: 'RelWithDebInfo'
|
||||
UseOmp: ''
|
||||
buildArch: x64
|
||||
msbuildPlatform: x64
|
||||
isX86: false
|
||||
ALLOW_RELEASED_ONNX_OPSET_ONLY: '0'
|
||||
DocUpdateNeeded: false
|
||||
timeoutInMinutes: 180
|
||||
workspace:
|
||||
clean: all
|
||||
steps:
|
||||
- task: UsePythonVersion@0
|
||||
inputs:
|
||||
versionSpec: '3.7'
|
||||
addToPath: true
|
||||
architecture: $(buildArch)
|
||||
stages:
|
||||
- stage: inference
|
||||
dependsOn: []
|
||||
jobs:
|
||||
- job: 'build'
|
||||
pool: 'Win-GPU-2019'
|
||||
strategy:
|
||||
matrix:
|
||||
cuda:
|
||||
additionalBuildFlags: --build_java --build_nodejs --use_cuda --cuda_version=11.4 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.4" --cudnn_home="C:\local\cudnn-11.4-windows-x64-v8.2.2.26\cuda" --cmake_extra_defines CMAKE_CUDA_ARCHITECTURES=52 --gen_doc validate
|
||||
EnvSetupScript: setup_env_cuda_11.bat
|
||||
ORT_EP_NAME: CUDA
|
||||
dml:
|
||||
additionalBuildFlags: --use_dml --cmake_extra_defines CMAKE_SYSTEM_VERSION=10.0.18362.0 --enable_wcos --use_winml
|
||||
EnvSetupScript: setup_env.bat
|
||||
ORT_EP_NAME: DML
|
||||
variables:
|
||||
OrtPackageId: 'Microsoft.ML.OnnxRuntime.Gpu'
|
||||
MsbuildArguments: '-maxcpucount'
|
||||
TESTONGPU: 'ON'
|
||||
OnnxRuntimeBuildDirectory: '$(Build.BinariesDirectory)'
|
||||
DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true
|
||||
setVcvars: true
|
||||
BuildConfig: 'RelWithDebInfo'
|
||||
UseOmp: ''
|
||||
buildArch: x64
|
||||
msbuildPlatform: x64
|
||||
isX86: false
|
||||
ALLOW_RELEASED_ONNX_OPSET_ONLY: '0'
|
||||
DocUpdateNeeded: false
|
||||
timeoutInMinutes: 180
|
||||
workspace:
|
||||
clean: all
|
||||
steps:
|
||||
- task: UsePythonVersion@0
|
||||
inputs:
|
||||
versionSpec: '3.7'
|
||||
addToPath: true
|
||||
architecture: $(buildArch)
|
||||
|
||||
- task: NodeTool@0
|
||||
inputs:
|
||||
versionSpec: '12.x'
|
||||
force32bit: $(isX86)
|
||||
- task: NodeTool@0
|
||||
inputs:
|
||||
versionSpec: '12.x'
|
||||
force32bit: $(isX86)
|
||||
|
||||
- task: JavaToolInstaller@0
|
||||
#Our build machine doesn't have java x86
|
||||
condition: and(succeeded(), eq(variables['buildArch'], 'x64'))
|
||||
inputs:
|
||||
versionSpec: '11'
|
||||
jdkArchitectureOption: $(buildArch)
|
||||
jdkSourceOption: 'PreInstalled'
|
||||
- task: JavaToolInstaller@0
|
||||
#Our build machine doesn't have java x86
|
||||
condition: and(succeeded(), eq(variables['buildArch'], 'x64'))
|
||||
inputs:
|
||||
versionSpec: '11'
|
||||
jdkArchitectureOption: $(buildArch)
|
||||
jdkSourceOption: 'PreInstalled'
|
||||
|
||||
- task: BatchScript@1
|
||||
displayName: 'setup env'
|
||||
inputs:
|
||||
filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)'
|
||||
modifyEnvironment: true
|
||||
workingFolder: '$(Build.BinariesDirectory)'
|
||||
- task: BatchScript@1
|
||||
displayName: 'setup env'
|
||||
inputs:
|
||||
filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)'
|
||||
modifyEnvironment: true
|
||||
workingFolder: '$(Build.BinariesDirectory)'
|
||||
|
||||
- script: |
|
||||
set ORT_DOXY_SRC=$(Build.SourcesDirectory)
|
||||
set ORT_DOXY_OUT=$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)
|
||||
mkdir %ORT_DOXY_SRC%
|
||||
mkdir %ORT_DOXY_OUT%
|
||||
"C:\Program Files\doxygen\bin\doxygen.exe" $(Build.SourcesDirectory)\tools\ci_build\github\Doxyfile_csharp.cfg
|
||||
- script: |
|
||||
set ORT_DOXY_SRC=$(Build.SourcesDirectory)
|
||||
set ORT_DOXY_OUT=$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)
|
||||
mkdir %ORT_DOXY_SRC%
|
||||
mkdir %ORT_DOXY_OUT%
|
||||
"C:\Program Files\doxygen\bin\doxygen.exe" $(Build.SourcesDirectory)\tools\ci_build\github\Doxyfile_csharp.cfg
|
||||
|
||||
workingDirectory: '$(Build.SourcesDirectory)'
|
||||
displayName: 'API Documentation Check and generate'
|
||||
workingDirectory: '$(Build.SourcesDirectory)'
|
||||
displayName: 'API Documentation Check and generate'
|
||||
|
||||
- script: |
|
||||
python -m pip install -q setuptools wheel numpy
|
||||
workingDirectory: '$(Build.BinariesDirectory)'
|
||||
displayName: 'Install python modules'
|
||||
- script: |
|
||||
python -m pip install -q setuptools wheel numpy
|
||||
workingDirectory: '$(Build.BinariesDirectory)'
|
||||
displayName: 'Install python modules'
|
||||
|
||||
- powershell: |
|
||||
$Env:USE_MSVC_STATIC_RUNTIME=1
|
||||
$Env:ONNX_ML=1
|
||||
$Env:CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=OFF -DProtobuf_USE_STATIC_LIBS=ON -DONNX_USE_LITE_PROTO=ON -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=$(buildArch)-windows-static"
|
||||
python setup.py bdist_wheel
|
||||
python -m pip uninstall -y onnx -qq
|
||||
Get-ChildItem -Path dist/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname}
|
||||
workingDirectory: '$(Build.SourcesDirectory)\cmake\external\onnx'
|
||||
displayName: 'Install ONNX'
|
||||
- powershell: |
|
||||
$Env:USE_MSVC_STATIC_RUNTIME=1
|
||||
$Env:ONNX_ML=1
|
||||
$Env:CMAKE_ARGS="-DONNX_USE_PROTOBUF_SHARED_LIBS=OFF -DProtobuf_USE_STATIC_LIBS=ON -DONNX_USE_LITE_PROTO=ON -DCMAKE_TOOLCHAIN_FILE=C:/vcpkg/scripts/buildsystems/vcpkg.cmake -DVCPKG_TARGET_TRIPLET=$(buildArch)-windows-static"
|
||||
python setup.py bdist_wheel
|
||||
python -m pip uninstall -y onnx -qq
|
||||
Get-ChildItem -Path dist/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname}
|
||||
workingDirectory: '$(Build.SourcesDirectory)\cmake\external\onnx'
|
||||
displayName: 'Install ONNX'
|
||||
|
||||
- task: NuGetToolInstaller@0
|
||||
displayName: Use Nuget 5.7.0
|
||||
inputs:
|
||||
versionSpec: 5.7.0
|
||||
- task: NuGetToolInstaller@0
|
||||
displayName: Use Nuget 5.7.0
|
||||
inputs:
|
||||
versionSpec: 5.7.0
|
||||
|
||||
- task: NuGetCommand@2
|
||||
displayName: 'NuGet restore'
|
||||
inputs:
|
||||
command: 'restore'
|
||||
feedsToUse: 'config'
|
||||
restoreSolution: '$(Build.SourcesDirectory)\packages.config'
|
||||
nugetConfigPath: '$(Build.SourcesDirectory)\NuGet.config'
|
||||
restoreDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)'
|
||||
- task: NuGetCommand@2
|
||||
displayName: 'NuGet restore'
|
||||
inputs:
|
||||
command: 'restore'
|
||||
feedsToUse: 'config'
|
||||
restoreSolution: '$(Build.SourcesDirectory)\packages.config'
|
||||
nugetConfigPath: '$(Build.SourcesDirectory)\NuGet.config'
|
||||
restoreDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)'
|
||||
|
||||
- task: PythonScript@0
|
||||
displayName: 'Generate cmake config'
|
||||
inputs:
|
||||
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
|
||||
arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) $(UseOmp) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_wheel --enable_onnx_tests $(additionalBuildFlags)'
|
||||
workingDirectory: '$(Build.BinariesDirectory)'
|
||||
- task: PythonScript@0
|
||||
displayName: 'Generate cmake config'
|
||||
inputs:
|
||||
scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\build.py'
|
||||
arguments: '--config $(BuildConfig) --build_dir $(Build.BinariesDirectory) $(UseOmp) --skip_submodule_sync --build_shared_lib --update --cmake_generator "Visual Studio 16 2019" --build_wheel --enable_onnx_tests $(additionalBuildFlags)'
|
||||
workingDirectory: '$(Build.BinariesDirectory)'
|
||||
|
||||
- task: VSBuild@1
|
||||
displayName: 'Build'
|
||||
inputs:
|
||||
solution: '$(Build.BinariesDirectory)\$(BuildConfig)\onnxruntime.sln'
|
||||
platform: $(msbuildPlatform)
|
||||
configuration: $(BuildConfig)
|
||||
msbuildArgs: $(MsbuildArguments)
|
||||
msbuildArchitecture: $(buildArch)
|
||||
maximumCpuCount: true
|
||||
logProjectEvents: false
|
||||
workingFolder: '$(Build.BinariesDirectory)\$(BuildConfig)'
|
||||
createLogFile: true
|
||||
- task: VSBuild@1
|
||||
displayName: 'Build'
|
||||
inputs:
|
||||
solution: '$(Build.BinariesDirectory)\$(BuildConfig)\onnxruntime.sln'
|
||||
platform: $(msbuildPlatform)
|
||||
configuration: $(BuildConfig)
|
||||
msbuildArgs: $(MsbuildArguments)
|
||||
msbuildArchitecture: $(buildArch)
|
||||
maximumCpuCount: true
|
||||
logProjectEvents: false
|
||||
workingFolder: '$(Build.BinariesDirectory)\$(BuildConfig)'
|
||||
createLogFile: true
|
||||
|
||||
- task: PythonScript@0
|
||||
displayName: 'Build wheel'
|
||||
inputs:
|
||||
scriptPath: '$(Build.SourcesDirectory)\setup.py'
|
||||
arguments: 'bdist_wheel'
|
||||
workingDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)'
|
||||
- task: PythonScript@0
|
||||
displayName: 'Build wheel'
|
||||
inputs:
|
||||
scriptPath: '$(Build.SourcesDirectory)\setup.py'
|
||||
arguments: 'bdist_wheel'
|
||||
workingDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)'
|
||||
|
||||
- task: NuGetCommand@2
|
||||
displayName: Restore NuGet Packages
|
||||
inputs:
|
||||
command: "restore"
|
||||
restoreSolution: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln'
|
||||
- task: NuGetCommand@2
|
||||
displayName: Restore NuGet Packages
|
||||
inputs:
|
||||
command: "restore"
|
||||
restoreSolution: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln'
|
||||
|
||||
- task: MSBuild@1
|
||||
displayName: 'Build C#'
|
||||
inputs:
|
||||
solution: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln'
|
||||
platform: 'Any CPU'
|
||||
configuration: '$(BuildConfig)'
|
||||
msbuildArguments: '-p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId) -p:DefineConstants=USE_$(ORT_EP_NAME)'
|
||||
workingDirectory: '$(Build.SourcesDirectory)\csharp'
|
||||
- task: MSBuild@1
|
||||
displayName: 'Build C#'
|
||||
inputs:
|
||||
solution: '$(Build.SourcesDirectory)\csharp\OnnxRuntime.CSharp.sln'
|
||||
platform: 'Any CPU'
|
||||
configuration: '$(BuildConfig)'
|
||||
msbuildArguments: '-p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId) -p:DefineConstants=USE_$(ORT_EP_NAME)'
|
||||
workingDirectory: '$(Build.SourcesDirectory)\csharp'
|
||||
|
||||
- task: DotNetCoreCLI@2
|
||||
displayName: 'Test C#'
|
||||
condition: and(and(succeeded(), eq(variables['BuildConfig'], 'RelWithDebInfo')),eq('${{ parameters.RunOnnxRuntimeTests}}', true))
|
||||
inputs:
|
||||
command: test
|
||||
projects: '$(Build.SourcesDirectory)\csharp\test\Microsoft.ML.OnnxRuntime.Tests.NetCoreApp\Microsoft.ML.OnnxRuntime.Tests.NetCoreApp.csproj'
|
||||
configuration: '$(BuildConfig)'
|
||||
arguments: '--configuration $(BuildConfig) -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId) -p:DefineConstants=USE_$(ORT_EP_NAME)'
|
||||
workingDirectory: '$(Build.SourcesDirectory)\csharp'
|
||||
- task: DotNetCoreCLI@2
|
||||
displayName: 'Test C#'
|
||||
condition: and(and(succeeded(), eq(variables['BuildConfig'], 'RelWithDebInfo')),eq('${{ parameters.RunOnnxRuntimeTests}}', true))
|
||||
inputs:
|
||||
command: test
|
||||
projects: '$(Build.SourcesDirectory)\csharp\test\Microsoft.ML.OnnxRuntime.Tests.NetCoreApp\Microsoft.ML.OnnxRuntime.Tests.NetCoreApp.csproj'
|
||||
configuration: '$(BuildConfig)'
|
||||
arguments: '--configuration $(BuildConfig) -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId) -p:DefineConstants=USE_$(ORT_EP_NAME)'
|
||||
workingDirectory: '$(Build.SourcesDirectory)\csharp'
|
||||
|
||||
- powershell: |
|
||||
Get-ChildItem -Path dist/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname}
|
||||
python $(Build.SourcesDirectory)\tools\ci_build\build.py --config $(BuildConfig) --build_dir $(Build.BinariesDirectory) $(UseOmp) --skip_submodule_sync --build_shared_lib --test --cmake_generator "Visual Studio 16 2019" --build_wheel --enable_onnx_tests $(additionalBuildFlags)
|
||||
workingDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)'
|
||||
condition: and(succeeded(), eq('${{ parameters.RunOnnxRuntimeTests}}', true))
|
||||
displayName: 'Run tests'
|
||||
- powershell: |
|
||||
Get-ChildItem -Path dist/*.whl | foreach {pip --disable-pip-version-check install --upgrade $_.fullname}
|
||||
python $(Build.SourcesDirectory)\tools\ci_build\build.py --config $(BuildConfig) --build_dir $(Build.BinariesDirectory) $(UseOmp) --skip_submodule_sync --build_shared_lib --test --cmake_generator "Visual Studio 16 2019" --build_wheel --enable_onnx_tests $(additionalBuildFlags)
|
||||
workingDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)'
|
||||
condition: and(succeeded(), eq('${{ parameters.RunOnnxRuntimeTests}}', true))
|
||||
displayName: 'Run tests'
|
||||
|
||||
# if the validation from --gen_doc failed it sets a variable so we can publish the latest version of the docs
|
||||
# as an artifact, allowing a developer to download this and replace the current version instead of having to build
|
||||
# and generate the docs locally themselves. handle each of the two md files separately - simpler than copying
|
||||
# them to another location and publishing from there in a single task.
|
||||
- task: PublishBuildArtifacts@1
|
||||
condition: and(failed(), eq(variables['DocUpdateNeeded'], 'true'))
|
||||
inputs:
|
||||
pathtoPublish: '$(Build.SourcesDirectory)/docs/OperatorKernels.md'
|
||||
artifactName: 'OperatorKernels.md'
|
||||
# if the validation from --gen_doc failed it sets a variable so we can publish the latest version of the docs
|
||||
# as an artifact, allowing a developer to download this and replace the current version instead of having to build
|
||||
# and generate the docs locally themselves. handle each of the two md files separately - simpler than copying
|
||||
# them to another location and publishing from there in a single task.
|
||||
- task: PublishBuildArtifacts@1
|
||||
condition: and(failed(), eq(variables['DocUpdateNeeded'], 'true'))
|
||||
inputs:
|
||||
pathtoPublish: '$(Build.SourcesDirectory)/docs/OperatorKernels.md'
|
||||
artifactName: 'OperatorKernels.md'
|
||||
|
||||
- task: PublishBuildArtifacts@1
|
||||
condition: and(failed(), eq(variables['DocUpdateNeeded'], 'true'))
|
||||
inputs:
|
||||
pathtoPublish: '$(Build.SourcesDirectory)/docs/ContribOperators.md'
|
||||
artifactName: 'ContribOperators.md'
|
||||
- task: PublishBuildArtifacts@1
|
||||
condition: and(failed(), eq(variables['DocUpdateNeeded'], 'true'))
|
||||
inputs:
|
||||
pathtoPublish: '$(Build.SourcesDirectory)/docs/ContribOperators.md'
|
||||
artifactName: 'ContribOperators.md'
|
||||
|
||||
|
||||
- task: PublishSecurityAnalysisLogs@3
|
||||
displayName: 'Publish Security Analysis Logs'
|
||||
condition: and(succeeded(), eq(variables['BuildConfig'], 'RelWithDebInfo'))
|
||||
inputs:
|
||||
ArtifactName: CodeAnalysisLogs
|
||||
- task: PublishSecurityAnalysisLogs@3
|
||||
displayName: 'Publish Security Analysis Logs'
|
||||
condition: and(succeeded(), eq(variables['BuildConfig'], 'RelWithDebInfo'))
|
||||
inputs:
|
||||
ArtifactName: CodeAnalysisLogs
|
||||
|
||||
- task: PublishTestResults@2
|
||||
displayName: 'Publish unit test results'
|
||||
inputs:
|
||||
testResultsFiles: '**/*.results.xml'
|
||||
searchFolder: '$(Build.BinariesDirectory)'
|
||||
testRunTitle: 'Unit Test Run'
|
||||
condition: succeededOrFailed()
|
||||
- task: PublishTestResults@2
|
||||
displayName: 'Publish unit test results'
|
||||
inputs:
|
||||
testResultsFiles: '**/*.results.xml'
|
||||
searchFolder: '$(Build.BinariesDirectory)'
|
||||
testRunTitle: 'Unit Test Run'
|
||||
condition: succeededOrFailed()
|
||||
|
||||
- template: templates/component-governance-component-detection-steps.yml
|
||||
parameters :
|
||||
condition : 'succeeded'
|
||||
|
||||
- template: templates/component-governance-component-detection-steps.yml
|
||||
parameters :
|
||||
condition : 'succeeded'
|
||||
|
||||
- stage: training
|
||||
dependsOn: []
|
||||
jobs:
|
||||
- template: templates/win-ci-2019.yml
|
||||
parameters:
|
||||
AgentPool : 'Win-GPU-2019'
|
||||
JobName: 'Win_GPU_Training'
|
||||
# TODO fix test failures and remove --skip_onnx_tests
|
||||
BuildCommand: >-
|
||||
--build_dir $(Build.BinariesDirectory)
|
||||
--build_shared_lib
|
||||
--cmake_generator "Visual Studio 16 2019"
|
||||
--enable_onnx_tests
|
||||
--enable_training
|
||||
--skip_onnx_tests
|
||||
--skip_submodule_sync
|
||||
--use_cuda
|
||||
--cuda_version 10.2
|
||||
--cuda_home "C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.2"
|
||||
--cudnn_home "C:\local\cudnn-10.2-windows10-x64-v8.0.3.33\cuda"
|
||||
--cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=52"
|
||||
BuildArch: 'x64'
|
||||
EnvSetupScript: 'setup_env_cuda.bat'
|
||||
sln_platform: 'x64'
|
||||
CudaVersion: '10.2'
|
||||
OrtPackageId: 'Microsoft.ML.OnnxRuntime.Gpu'
|
||||
BuildConfigurations: ['RelWithDebInfo']
|
||||
# Enable unreleased onnx opsets in CI builds
|
||||
# This facilitates testing the implementation for the new opsets
|
||||
AllowReleasedOpsetOnly: '0'
|
||||
DoCompliance: 'true'
|
||||
Loading…
Reference in a new issue