diff --git a/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml b/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml index b3f1255137..d149775a78 100644 --- a/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml +++ b/tools/ci_build/github/azure-pipelines/templates/windows-build-tools-setup-steps.yml @@ -16,14 +16,14 @@ steps: feedsToUse: config nugetConfigPath: '$(Build.SourcesDirectory)\csharp\Nuget.CSharp.config' restoreDirectory: '$(Build.SourcesDirectory)\csharp' - #- task: UniversalPackages@0 - # displayName: 'Download python' - # inputs: - # command: download - # vstsFeed: '$(System.TeamProject)' - # vstsFeedPackage: 'miniconda3_win64' - # vstsPackageVersion: '4.5.11' - # downloadDirectory: '$(Build.BinariesDirectory)\python' + # - task: UniversalPackages@0 + # displayName: 'Download python' + # inputs: + # command: download + # vstsFeed: '$(System.TeamProject)' + # vstsFeedPackage: 'miniconda3_win64' + # vstsPackageVersion: '4.5.11' + # downloadDirectory: '$(Build.BinariesDirectory)\python' # Temporary bypass of artifacts permission issue - task: PowerShell@2 @@ -58,6 +58,26 @@ steps: filename: '$(Build.BinariesDirectory)\packages\python\python.exe' arguments: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\download_cmake.py --build_dir $(Build.BinariesDirectory)' + - task: PowerShell@2 + displayName: 'Download OpenCppCoverage installer' + continueOnError: true + inputs: + targetType: 'inline' + script: ' + New-Item -Path "$(Build.BinariesDirectory)\installer" -ItemType "directory" + + New-Item -Path "$(Build.BinariesDirectory)\installer\opencppcoverage" -ItemType "directory" + + Invoke-WebRequest -OutFile $(Build.BinariesDirectory)\installer\opencppcoverage\installer.exe https://onnxruntimeinstaller.blob.core.windows.net/opencppcovergae-installer/OpenCppCoverageSetup-x64-0.9.7.0.exe + ' + + - task: CmdLine@1 + continueOnError: true + displayName: 'Run OpenCPPCoverage installer' + inputs: + filename: '$(Build.BinariesDirectory)\installer\opencppcoverage\installer.exe' + arguments: '/SP- /VERYSILENT /SUPPRESSMSGBOXES /NORESTART /DIR="$(Build.BinariesDirectory)\OpenCppCoverage"' + - task: BatchScript@1 displayName: 'Setup VS2017 env vars' inputs: diff --git a/tools/ci_build/github/azure-pipelines/templates/windows-code-coverage-steps.yml b/tools/ci_build/github/azure-pipelines/templates/windows-code-coverage-steps.yml new file mode 100644 index 0000000000..66dd68cd06 --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/templates/windows-code-coverage-steps.yml @@ -0,0 +1,20 @@ +# sets variables $(TestDataUrl) and $(TestDataChecksum) + +parameters: + OpenCppCoverageExe: '$(Build.BinariesDirectory)\OpenCppCoverage\OpenCppCoverage.exe' + +steps: + +- task: PowerShell@2 + displayName: 'Run Test Coverage' + inputs: + filePath: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\run_OpenCppCoverage.ps1' + arguments: '-OpenCppCoverageExe:"${{parameters.OpenCppCoverageExe}}" -SourceRoot:"$(Build.SourcesDirectory)" -BuildRoot:"$(Build.BinariesDirectory)" -LocalBuild:$false' + continueOnError: true + +- task: PublishCodeCoverageResults@1 + inputs: + codeCoverageTool: 'Cobertura' + summaryFileLocation: '$(Build.BinariesDirectory)/**/cobertura.xml' + reportDirectory: '$(Build.BinariesDirectory)/**/OpenCppCoverageResults' + continueOnError: true diff --git a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml index 521184b484..682db714fb 100644 --- a/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-ci-pipeline.yml @@ -2,6 +2,7 @@ jobs: - job: Windows_CI_Dev variables: buildDirectory: '$(Build.BinariesDirectory)' + timeoutInMinutes: 90 steps: - template: templates/set-test-data-variables-step.yml - template: templates/windows-build-tools-setup-steps.yml @@ -26,12 +27,18 @@ jobs: msbuildArchitecture: 'x64' logProjectEvents: true workingFolder: '$(Build.BinariesDirectory)\Debug' + - task: BatchScript@1 displayName: 'Test Debug' inputs: filename: '$(Build.BinariesDirectory)\packages\python\python.exe' arguments: '$(Build.SourcesDirectory)\tools\ci_build\build.py --config Debug --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --cmake_path $(Build.BinariesDirectory)\cmake\bin\cmake.exe --ctest_path $(Build.BinariesDirectory)\cmake\bin\ctest.exe --use_tvm --enable_pybind --use_mkldnn --use_mklml --use_openmp --build_shared_lib --enable_onnx_tests --test_data_url $(TestDataUrl) --test_data_checksum $(TestDataChecksum) --gen_doc --test' workingFolder: '$(Build.BinariesDirectory)' + + - template: templates/windows-code-coverage-steps.yml + parameters: + OpenCppCoverageExe: '$(Build.BinariesDirectory)\OpenCppCoverage\OpenCppCoverage.exe' + - task: VSBuild@1 displayName: 'Build C# Debug' inputs: diff --git a/tools/ci_build/github/windows/run_OpenCppCoverage.ps1 b/tools/ci_build/github/windows/run_OpenCppCoverage.ps1 new file mode 100644 index 0000000000..bd7c3434df --- /dev/null +++ b/tools/ci_build/github/windows/run_OpenCppCoverage.ps1 @@ -0,0 +1,79 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +# Runs OpenCppCoverage for the Lotus unit tests and ONNX tests, and merges the coverage from all test runs. +Param( + [Parameter(Mandatory=$true, HelpMessage="OpenCppCoverage exe.")][string]$OpenCppCoverageExe, + [Parameter(Mandatory=$true, HelpMessage="Lotus enlistment root.")][string]$SourceRoot, + [Parameter(Mandatory=$true, HelpMessage="Build root.")][string]$BuildRoot, + [Parameter(Mandatory=$false, HelpMessage="IsLocalBuild")][switch]$LocalBuild = $true +) + +if (-not $LocalBuild) { +# This is a hack to get the target path of the junctions in the build machine, lacking a neater way to do this. +# Assumes that the junction is 2 level upper from the SourceRoot/BuildRoot +# This is needed, because apparently the OpenCppCoverage cannot load the PDB symbol files from a junction + + $buildLeaf = Split-Path $BuildRoot -Leaf + $buildParent = Split-Path $BuildRoot -Parent + $buildParentLeaf = Split-Path $buildParent -Leaf + $buildParentParent = Split-Path $buildParent -Parent + $buildParentParentTarget = Get-Item $buildParentParent | Select-Object -ExpandProperty Target + + $BuildRoot = Join-Path $buildParentParentTarget $buildParentLeaf + $BuildRoot = Join-Path $BuildRoot $buildLeaf +} + +$coreSources = Join-Path $SourceRoot "onnxruntime\core" +$headerSources = Join-Path $SourceRoot "include" +$buildDir = Join-Path $BuildRoot "Debug\Debug" + +function RunTest([string]$test_cmd, [string[]]$test_cmd_args, [string[]]$export_types, [string[]]$inputs) +{ + $cmd = "$OpenCppCoverageExe" + $cmdParams = @("--sources=$headerSources","--sources=$coreSources","--modules=$buildDir","--working_dir=$buildDir") + + foreach($input in $inputs) + { + $inputPath = Join-Path $buildDir $input + $cmdParams += "--input_coverage=$inputPath" + } + + foreach($export_type in $export_types) + { + $cmdParams += "--export_type=$export_type" + } + + $cmdParams += @("--","$test_cmd") + $cmdParams += $test_cmd_args + + & $cmd $cmdParams +} + +# generate cobertura xml output and html report +$outputXml = Join-Path $buildDir "cobertura.xml" +$outputDir = Join-Path $buildDir "OpenCppCoverageResults" +$modelDir = Join-Path $BuildRoot "models" + + +# ONNX test runner tests. +$onnx_test_runner = Join-Path $buildDir "onnx_test_runner.exe" +RunTest $onnx_test_runner ($modelDir) ("binary:" + (Join-Path $buildDir "onnx_test_runner.cov")) + + +# C-API/Shared-lib test +$shared_lib_test = Join-Path $buildDir "onnxruntime_shared_lib_test.exe" +RunTest $shared_lib_test @() ("binary:" + (Join-Path $buildDir "onnxruntime_shared_lib_test.cov")) + + +# MLAS test +$mlas_test = Join-Path $buildDir "onnxruntime_mlas_test.exe" +RunTest $mlas_test @() ("binary:" + (Join-Path $buildDir "onnxruntime_mlas_test.cov")) + +# Lotus unit tests +# need to copy the tvm.dll, since it is not in the buildDir path +Copy-Item -Path $BuildRoot\Debug\external\tvm\Debug\tvm.dll -Destination $buildDir + +$onnxruntime_test_all = Join-Path $buildDir "onnxruntime_test_all.exe" +RunTest $onnxruntime_test_all @() ("cobertura:$outputXml","html:$outputDir") ("onnxruntime_shared_lib_test.cov","onnx_test_runner.cov","onnxruntime_mlas_test.cov") +