mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-12 17:57:38 +00:00
Add code coverage (#1192)
* added the runcoverage powershell script * updated the run coverage script. added installation to the windows CI for trying * exclude other parts of win ci * fix in the download script * fix in the download script * fix in the download script * fix in the download script * fix in the download script * fix in the download script * fix in the download script * fix in the download script * fix in the download script * added the runtestcoverage script to the pipeline * some typo fix * formatting * re-commenting previously commented block * cleaned up the powershell script * fix path in pipeline * fix path in pipeline * fixed model path * some fixes * excluded long running tests * add the publish job * uncomment other tasks * fixed excluded tests * some format correction * stopped running the test debug * try placing the tes-all at the beginning * try running the failing test only * edit run_coverage * some fix * skip onnx_model_test * Added memory size log in powershell script * try running the onnxruntime_test_all.exe separately from codecov * enable error reporting, and double memory size in powershell * corrected the set-item * remove memory resize, since we are already at max 2 GB * fixed the tvm.dll issue * added back the onnx tests in codecov. added back the regular test run * cleanup * remove * from the the module path * add junction target resolution for modules dir * remove junction-resolution * reduced tests * added target extraction for the junction paths in build machine * added the appropriate change in win ci pipeline to call the updated ps script * fix typo * added back all the tests that were disabled * try fixing the source root * cleanup and enable all tests * increase timeout for windows CPU CI due to codecoverage * templatized the code coverage steps. Conitnue on error with any codecoverage step * change quote marks
This commit is contained in:
shahasad
GitHub
parent
280ab9a2d0
commit
97dfd5ee21
4 changed files with 134 additions and 8 deletions
|
|
@ -16,14 +16,14 @@ steps:
|
|||
feedsToUse: config
|
||||
nugetConfigPath: '$(Build.SourcesDirectory)\csharp\Nuget.CSharp.config'
|
||||
restoreDirectory: '$(Build.SourcesDirectory)\csharp'
|
||||
#- task: UniversalPackages@0
|
||||
# displayName: 'Download python'
|
||||
# inputs:
|
||||
# command: download
|
||||
# vstsFeed: '$(System.TeamProject)'
|
||||
# vstsFeedPackage: 'miniconda3_win64'
|
||||
# vstsPackageVersion: '4.5.11'
|
||||
# downloadDirectory: '$(Build.BinariesDirectory)\python'
|
||||
# - task: UniversalPackages@0
|
||||
# displayName: 'Download python'
|
||||
# inputs:
|
||||
# command: download
|
||||
# vstsFeed: '$(System.TeamProject)'
|
||||
# vstsFeedPackage: 'miniconda3_win64'
|
||||
# vstsPackageVersion: '4.5.11'
|
||||
# downloadDirectory: '$(Build.BinariesDirectory)\python'
|
||||
|
||||
# Temporary bypass of artifacts permission issue
|
||||
- task: PowerShell@2
|
||||
|
|
@ -58,6 +58,26 @@ steps:
|
|||
filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
|
||||
arguments: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\download_cmake.py --build_dir $(Build.BinariesDirectory)'
|
||||
|
||||
- task: PowerShell@2
|
||||
displayName: 'Download OpenCppCoverage installer'
|
||||
continueOnError: true
|
||||
inputs:
|
||||
targetType: 'inline'
|
||||
script: '
|
||||
New-Item -Path "$(Build.BinariesDirectory)\installer" -ItemType "directory"
|
||||
|
||||
New-Item -Path "$(Build.BinariesDirectory)\installer\opencppcoverage" -ItemType "directory"
|
||||
|
||||
Invoke-WebRequest -OutFile $(Build.BinariesDirectory)\installer\opencppcoverage\installer.exe https://onnxruntimeinstaller.blob.core.windows.net/opencppcovergae-installer/OpenCppCoverageSetup-x64-0.9.7.0.exe
|
||||
'
|
||||
|
||||
- task: CmdLine@1
|
||||
continueOnError: true
|
||||
displayName: 'Run OpenCPPCoverage installer'
|
||||
inputs:
|
||||
filename: '$(Build.BinariesDirectory)\installer\opencppcoverage\installer.exe'
|
||||
arguments: '/SP- /VERYSILENT /SUPPRESSMSGBOXES /NORESTART /DIR="$(Build.BinariesDirectory)\OpenCppCoverage"'
|
||||
|
||||
- task: BatchScript@1
|
||||
displayName: 'Setup VS2017 env vars'
|
||||
inputs:
|
||||
|
|
|
|||
|
|
@ -0,0 +1,20 @@
|
|||
# sets variables $(TestDataUrl) and $(TestDataChecksum)
|
||||
|
||||
parameters:
|
||||
OpenCppCoverageExe: '$(Build.BinariesDirectory)\OpenCppCoverage\OpenCppCoverage.exe'
|
||||
|
||||
steps:
|
||||
|
||||
- task: PowerShell@2
|
||||
displayName: 'Run Test Coverage'
|
||||
inputs:
|
||||
filePath: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\run_OpenCppCoverage.ps1'
|
||||
arguments: '-OpenCppCoverageExe:"${{parameters.OpenCppCoverageExe}}" -SourceRoot:"$(Build.SourcesDirectory)" -BuildRoot:"$(Build.BinariesDirectory)" -LocalBuild:$false'
|
||||
continueOnError: true
|
||||
|
||||
- task: PublishCodeCoverageResults@1
|
||||
inputs:
|
||||
codeCoverageTool: 'Cobertura'
|
||||
summaryFileLocation: '$(Build.BinariesDirectory)/**/cobertura.xml'
|
||||
reportDirectory: '$(Build.BinariesDirectory)/**/OpenCppCoverageResults'
|
||||
continueOnError: true
|
||||
|
|
@ -2,6 +2,7 @@ jobs:
|
|||
- job: Windows_CI_Dev
|
||||
variables:
|
||||
buildDirectory: '$(Build.BinariesDirectory)'
|
||||
timeoutInMinutes: 90
|
||||
steps:
|
||||
- template: templates/set-test-data-variables-step.yml
|
||||
- template: templates/windows-build-tools-setup-steps.yml
|
||||
|
|
@ -26,12 +27,18 @@ jobs:
|
|||
msbuildArchitecture: 'x64'
|
||||
logProjectEvents: true
|
||||
workingFolder: '$(Build.BinariesDirectory)\Debug'
|
||||
|
||||
- task: BatchScript@1
|
||||
displayName: 'Test Debug'
|
||||
inputs:
|
||||
filename: '$(Build.BinariesDirectory)\packages\python\python.exe'
|
||||
arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_tvm --enable_pybind --use_mkldnn --use_mklml --use_openmp --build_shared_lib --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --gen_doc --test'
|
||||
workingFolder: '$(Build.BinariesDirectory)'
|
||||
|
||||
- template: templates/windows-code-coverage-steps.yml
|
||||
parameters:
|
||||
OpenCppCoverageExe: '$(Build.BinariesDirectory)\OpenCppCoverage\OpenCppCoverage.exe'
|
||||
|
||||
- task: VSBuild@1
|
||||
displayName: 'Build C# Debug'
|
||||
inputs:
|
||||
|
|
|
|||
79
tools/ci_build/github/windows/run_OpenCppCoverage.ps1
Normal file
79
tools/ci_build/github/windows/run_OpenCppCoverage.ps1
Normal file
|
|
@ -0,0 +1,79 @@
|
|||
# Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
# Licensed under the MIT License.
|
||||
|
||||
# Runs OpenCppCoverage for the Lotus unit tests and ONNX tests, and merges the coverage from all test runs.
|
||||
Param(
|
||||
[Parameter(Mandatory=$true, HelpMessage="OpenCppCoverage exe.")][string]$OpenCppCoverageExe,
|
||||
[Parameter(Mandatory=$true, HelpMessage="Lotus enlistment root.")][string]$SourceRoot,
|
||||
[Parameter(Mandatory=$true, HelpMessage="Build root.")][string]$BuildRoot,
|
||||
[Parameter(Mandatory=$false, HelpMessage="IsLocalBuild")][switch]$LocalBuild = $true
|
||||
)
|
||||
|
||||
if (-not $LocalBuild) {
|
||||
# This is a hack to get the target path of the junctions in the build machine, lacking a neater way to do this.
|
||||
# Assumes that the junction is 2 level upper from the SourceRoot/BuildRoot
|
||||
# This is needed, because apparently the OpenCppCoverage cannot load the PDB symbol files from a junction
|
||||
|
||||
$buildLeaf = Split-Path $BuildRoot -Leaf
|
||||
$buildParent = Split-Path $BuildRoot -Parent
|
||||
$buildParentLeaf = Split-Path $buildParent -Leaf
|
||||
$buildParentParent = Split-Path $buildParent -Parent
|
||||
$buildParentParentTarget = Get-Item $buildParentParent | Select-Object -ExpandProperty Target
|
||||
|
||||
$BuildRoot = Join-Path $buildParentParentTarget $buildParentLeaf
|
||||
$BuildRoot = Join-Path $BuildRoot $buildLeaf
|
||||
}
|
||||
|
||||
$coreSources = Join-Path $SourceRoot "onnxruntime\core"
|
||||
$headerSources = Join-Path $SourceRoot "include"
|
||||
$buildDir = Join-Path $BuildRoot "Debug\Debug"
|
||||
|
||||
function RunTest([string]$test_cmd, [string[]]$test_cmd_args, [string[]]$export_types, [string[]]$inputs)
|
||||
{
|
||||
$cmd = "$OpenCppCoverageExe"
|
||||
$cmdParams = @("--sources=$headerSources","--sources=$coreSources","--modules=$buildDir","--working_dir=$buildDir")
|
||||
|
||||
foreach($input in $inputs)
|
||||
{
|
||||
$inputPath = Join-Path $buildDir $input
|
||||
$cmdParams += "--input_coverage=$inputPath"
|
||||
}
|
||||
|
||||
foreach($export_type in $export_types)
|
||||
{
|
||||
$cmdParams += "--export_type=$export_type"
|
||||
}
|
||||
|
||||
$cmdParams += @("--","$test_cmd")
|
||||
$cmdParams += $test_cmd_args
|
||||
|
||||
& $cmd $cmdParams
|
||||
}
|
||||
|
||||
# generate cobertura xml output and html report
|
||||
$outputXml = Join-Path $buildDir "cobertura.xml"
|
||||
$outputDir = Join-Path $buildDir "OpenCppCoverageResults"
|
||||
$modelDir = Join-Path $BuildRoot "models"
|
||||
|
||||
|
||||
# ONNX test runner tests.
|
||||
$onnx_test_runner = Join-Path $buildDir "onnx_test_runner.exe"
|
||||
RunTest $onnx_test_runner ($modelDir) ("binary:" + (Join-Path $buildDir "onnx_test_runner.cov"))
|
||||
|
||||
|
||||
# C-API/Shared-lib test
|
||||
$shared_lib_test = Join-Path $buildDir "onnxruntime_shared_lib_test.exe"
|
||||
RunTest $shared_lib_test @() ("binary:" + (Join-Path $buildDir "onnxruntime_shared_lib_test.cov"))
|
||||
|
||||
|
||||
# MLAS test
|
||||
$mlas_test = Join-Path $buildDir "onnxruntime_mlas_test.exe"
|
||||
RunTest $mlas_test @() ("binary:" + (Join-Path $buildDir "onnxruntime_mlas_test.cov"))
|
||||
|
||||
# Lotus unit tests
|
||||
# need to copy the tvm.dll, since it is not in the buildDir path
|
||||
Copy-Item -Path $BuildRoot\Debug\external\tvm\Debug\tvm.dll -Destination $buildDir
|
||||
|
||||
$onnxruntime_test_all = Join-Path $buildDir "onnxruntime_test_all.exe"
|
||||
RunTest $onnxruntime_test_all @() ("cobertura:$outputXml","html:$outputDir") ("onnxruntime_shared_lib_test.cov","onnx_test_runner.cov","onnxruntime_mlas_test.cov")
|
||||
|
||||
Loading…
Reference in a new issue