From 93c8e29782138b71a9e0605745192e92c0613f2e Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Tue, 25 May 2021 08:26:01 -0700 Subject: [PATCH] Improve code coverage report (#7770) --- ...ndroid-x86_64-crosscompile-ci-pipeline.yml | 2 +- .../azure-pipelines/post-merge-jobs.yml | 16 +++++--- .../post_code_coverage_to_dashboard.py | 41 ++++++++++--------- 3 files changed, 33 insertions(+), 26 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/android-x86_64-crosscompile-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/android-x86_64-crosscompile-ci-pipeline.yml index e593931cd0..247c37b44c 100644 --- a/tools/ci_build/github/azure-pipelines/android-x86_64-crosscompile-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/android-x86_64-crosscompile-ci-pipeline.yml @@ -101,6 +101,6 @@ jobs: --commit_hash=$(Build.SourceVersion) --report_url="https://dev.azure.com/onnxruntime/onnxruntime/_build/results?buildId=$(Build.BuildId)" --report_file="$(Build.BinariesDirectory)/coverage_rpt.txt" - --build_config="{\"os\":\"android\", \"arch\":\"x86_64\", \"config\":\"nnapi\"}" + --branch $(Build.SourceBranchName) --arch $(Agent.OSArchitecture) --os "android" --build_config nnapi env: DASHBOARD_MYSQL_ORT_PASSWORD: $(dashboard-mysql-ort-password) diff --git a/tools/ci_build/github/azure-pipelines/post-merge-jobs.yml b/tools/ci_build/github/azure-pipelines/post-merge-jobs.yml index 3442e174c6..0c4c0aaee6 100644 --- a/tools/ci_build/github/azure-pipelines/post-merge-jobs.yml +++ b/tools/ci_build/github/azure-pipelines/post-merge-jobs.yml @@ -24,22 +24,26 @@ jobs: #Merge the mulitple prof data into a single indexed profile data file llvm-profdata merge -sparse -o ort.profdata *.profraw #Create coverage report, output the result to 'report.json' - llvm-cov export -summary-only -instr-profile=ort.profdata onnxruntime_test_all -object onnxruntime_mlas_test $(Build.SourcesDirectory)/include/onnxruntime $(Build.SourcesDirectory)/onnxruntime/core $(Build.SourcesDirectory)/onnxruntime/contrib_ops > $(Build.BinariesDirectory)/report.json - llvm-cov show -instr-profile=ort.profdata onnxruntime_test_all -object onnxruntime_mlas_test $(Build.SourcesDirectory)/include/onnxruntime $(Build.SourcesDirectory)/onnxruntime/core $(Build.SourcesDirectory)/onnxruntime/contrib_ops --format=html -output-dir=$(Build.ArtifactStagingDirectory) + llvm-cov export -summary-only -instr-profile=ort.profdata onnxruntime_test_all -object onnxruntime_mlas_test -object onnxruntime_api_tests_without_env -object onnx_test_runner -object onnxruntime_shared_lib_test -object onnxruntime_global_thread_pools_test -object onnxruntime_api_tests_without_env $(Build.SourcesDirectory)/include/onnxruntime $(Build.SourcesDirectory)/onnxruntime/core $(Build.SourcesDirectory)/onnxruntime/contrib_ops > $(Build.BinariesDirectory)/report.json + + llvm-cov show -instr-profile=ort.profdata onnxruntime_test_all -object onnxruntime_mlas_test -object onnxruntime_api_tests_without_env -object onnx_test_runner -object onnxruntime_shared_lib_test -object onnxruntime_global_thread_pools_test -object onnxruntime_api_tests_without_env $(Build.SourcesDirectory)/include/onnxruntime $(Build.SourcesDirectory)/onnxruntime/core $(Build.SourcesDirectory)/onnxruntime/contrib_ops --format=html -output-dir=$(Build.ArtifactStagingDirectory) workingDirectory: $(Build.BinariesDirectory) + + - task: PublishPipelineArtifact@1 displayName: 'Publish Pipeline Artifact' inputs: targetPath: '$(Build.ArtifactStagingDirectory)' artifact: html - - task: CmdLine@2 + - task: PythonScript@0 displayName: 'Post code coverage data to the Dashboard' inputs: - script: | - python3 $(Build.SourcesDirectory)/tools/ci_build/github/windows/post_code_coverage_to_dashboard.py --commit_hash=$(OnnxRuntimeGitCommitHash) --report_file="$(Build.BinariesDirectory)/report.json" --report_url="https://aiinfra.visualstudio.com/Lotus/_build/results?buildId=$(Build.BuildId)" + scriptPath: $(Build.SourcesDirectory)/tools/ci_build/github/windows/post_code_coverage_to_dashboard.py + arguments: --commit_hash=$(OnnxRuntimeGitCommitHash) --report_file="$(Build.BinariesDirectory)/report.json" --report_url="https://aiinfra.visualstudio.com/Lotus/_build/results?buildId=$(Build.BuildId)" --branch $(Build.SourceBranchName) --arch $(Agent.OSArchitecture) --os $(Agent.OS) --build_config default workingDirectory: $(Build.BinariesDirectory) + pythonInterpreter: '/usr/bin/python3' env: DASHBOARD_MYSQL_ORT_PASSWORD: $(dashboard-mysql-ort-password) @@ -69,4 +73,4 @@ jobs: azureSubscription: AIInfraBuildOnnxRuntimeOSS Destination: AzureBlob storage: onnxruntime - ContainerName: '$web' \ No newline at end of file + ContainerName: '$web' diff --git a/tools/ci_build/github/windows/post_code_coverage_to_dashboard.py b/tools/ci_build/github/windows/post_code_coverage_to_dashboard.py index 0c587dac1b..bd458dca33 100644 --- a/tools/ci_build/github/windows/post_code_coverage_to_dashboard.py +++ b/tools/ci_build/github/windows/post_code_coverage_to_dashboard.py @@ -7,7 +7,6 @@ # --report_url= # --report_file= # --commit_hash= -# --build_config= import argparse import mysql.connector @@ -19,11 +18,14 @@ import os def parse_arguments(): parser = argparse.ArgumentParser( description="ONNXRuntime test coverge report uploader for dashboard") - parser.add_argument("--report_url", help="URL to the LLVM json report") + parser.add_argument("--report_url", type=str, help="URL to the LLVM json report") parser.add_argument( - "--report_file", help="Path to the local JSON/TXT report", required=True) - parser.add_argument("--commit_hash", help="Full Git commit hash", required=True) - parser.add_argument("--build_config", help="Build configuration, os, arch and config, in JSON format") + "--report_file", type=str, help="Path to the local JSON/TXT report", required=True) + parser.add_argument("--commit_hash", type=str, help="Full Git commit hash", required=True) + parser.add_argument("--branch", type=str, help="Source code branch") + parser.add_argument("--os", type=str, help="Build configuration:os") + parser.add_argument("--arch", type=str, help="Build configuration:arch") + parser.add_argument("--build_config", type=str, help="Build configuration: build variants") return parser.parse_args() @@ -52,7 +54,7 @@ def parse_json_report(report_file): return result -def write_to_db(coverage_data, build_config, args): +def write_to_db(coverage_data, args): # connect to database cnx = mysql.connector.connect( @@ -74,26 +76,28 @@ def write_to_db(coverage_data, build_config, args): # insert current record insert_query = ('INSERT INTO onnxruntime.test_coverage ' '''(UploadTime, CommitId, Coverage, LinesCovered, TotalLines, OS, - Arch, BuildConfig, ReportURL) ''' - 'VALUES (Now(), "%s", %f, %d, %d, "%s", "%s", "%s", "%s") ' + Arch, BuildConfig, ReportURL, Branch) ''' + 'VALUES (Now(), "%s", %f, %d, %d, "%s", "%s", "%s", "%s", "%s") ' 'ON DUPLICATE KEY UPDATE ' '''UploadTime=Now(), Coverage=%f, LinesCovered=%d, TotalLines=%d, - OS="%s", Arch="%s", BuildConfig="%s", ReportURL="%s"; ''' + OS="%s", Arch="%s", BuildConfig="%s", ReportURL="%s", Branch="%s"; ''' ) % (args.commit_hash, coverage_data['coverage'], coverage_data['lines_covered'], coverage_data['lines_valid'], - build_config.get('os', 'win'), - build_config.get('arch', 'x64'), - build_config.get('config', 'default'), - args.report_url, + args.os.lower(), + args.arch.lower(), + args.build_config.lower(), + args.report_url.lower(), + args.branch.lower(), coverage_data['coverage'], coverage_data['lines_covered'], coverage_data['lines_valid'], - build_config.get('os', 'win'), - build_config.get('arch', 'x64'), - build_config.get('config', 'default'), - args.report_url + args.os.lower(), + args.arch.lower(), + args.build_config.lower(), + args.report_url.lower(), + args.branch.lower() ) cursor.execute(insert_query) cnx.commit() @@ -120,8 +124,7 @@ if __name__ == "__main__": else: raise ValueError("Only report extensions txt or json are accepted") - build_config = json.loads(args.build_config) if args.build_config else {} - write_to_db(coverage_data, build_config, args) + write_to_db(coverage_data, args) except BaseException as e: print(str(e)) sys.exit(1)