diff --git a/tools/ci_build/coverage.py b/tools/ci_build/coverage.py index 9d067dbc10..fd1ce44d23 100644 --- a/tools/ci_build/coverage.py +++ b/tools/ci_build/coverage.py @@ -39,12 +39,15 @@ def main(): script_dir = os.path.realpath(os.path.dirname(__file__)) source_dir = os.path.normpath(os.path.join(script_dir, "..", "..")) cwd = os.path.abspath(os.path.join(args.build_dir, args.config)) - adb_shell('cd /data/local/tmp && tar -zcvf gcda_files.tar.gz *.dir') + adb_shell('cd /data/local/tmp && tar -zcf gcda_files.tar.gz *.dir') adb_pull('/data/local/tmp/gcda_files.tar.gz', cwd) os.chdir(cwd) - run_subprocess("tar -zxvf gcda_files.tar.gz -C CMakeFiles".split(' ')) - run_subprocess("gcovr -s -r {} .".format(os.path.join(source_dir, "onnxruntime")).split(' '), - cwd=os.path.join(cwd, "CMakeFiles")) + run_subprocess("tar -zxf gcda_files.tar.gz -C CMakeFiles".split(' ')) + cmd = ["gcovr", "-s", "-r"] + cmd.append(os.path.join(source_dir, "onnxruntime")) + cmd.extend([".", "-o"]) + cmd.append(os.path.join(cwd, "coverage_rpt.txt")) + run_subprocess(cmd, cwd=os.path.join(cwd, "CMakeFiles")) if __name__ == "__main__": diff --git a/tools/ci_build/github/azure-pipelines/android-x86_64-crosscompile-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/android-x86_64-crosscompile-ci-pipeline.yml index b07d23063c..9f3b30feea 100644 --- a/tools/ci_build/github/azure-pipelines/android-x86_64-crosscompile-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/android-x86_64-crosscompile-ci-pipeline.yml @@ -17,6 +17,44 @@ jobs: displayName: CPU EP, Build and Test on Android Emulator - script: /bin/bash tools/ci_build/github/android/run_nnapi_code_coverage.sh $(pwd) displayName: NNAPI EP, Build, Test and Get Code Coverage on Android Emulator + - task: PublishPipelineArtifact@0 + displayName: 'Publish code coverage report' + inputs: + artifactName: "coverage_rpt.txt" + targetPath: '$(Build.SourcesDirectory)/build_nnapi/Debug/coverage_rpt.txt' + publishLocation: 'pipeline' - script: /bin/bash tools/ci_build/github/linux/ort_minimal/nnapi_minimal_build_minimal_ort_and_run_tests.sh $(pwd) # Build Minimal ORT with NNAPI and reduced Ops, run unit tests on Android Emulator displayName: Build Minimal ORT with NNAPI and run tests + +- job: Update_Dashboard + workspace: + clean: all + pool: + vmImage: 'ubuntu-latest' + dependsOn: Android_CI + condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI')) + steps: + - task: DownloadPipelineArtifact@0 + displayName: 'Download code coverage report' + inputs: + artifactName: 'coverage_rpt.txt' + targetPath: '$(Build.BinariesDirectory)' + - task: UsePythonVersion@0 + inputs: + versionSpec: '3.x' + - task: CmdLine@2 + inputs: + script: | + python3 -m pip install mysql-connector-python + - task: PythonScript@0 + displayName: Post Android Code Coverage To DashBoard + inputs: + scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/windows/post_code_coverage_to_dashboard.py' + arguments: > + --commit_hash=$(Build.SourceVersion) + --report_url="https://dev.azure.com/onnxruntime/onnxruntime/_build/results?buildId=$(Build.BuildId)" + --report_file="$(Build.BinariesDirectory)/coverage_rpt.txt" + --build_config="{\"os\":\"android\", \"arch\":\"x86_64\", \"config\":\"nnapi\"}" + env: + DASHBOARD_MYSQL_ORT_PASSWORD: $(dashboard-mysql-ort-password) diff --git a/tools/ci_build/github/windows/post_code_coverage_to_dashboard.py b/tools/ci_build/github/windows/post_code_coverage_to_dashboard.py index 9e467d8445..0c587dac1b 100644 --- a/tools/ci_build/github/windows/post_code_coverage_to_dashboard.py +++ b/tools/ci_build/github/windows/post_code_coverage_to_dashboard.py @@ -5,8 +5,9 @@ # command line arguments # --report_url= -# --report_file= +# --report_file= # --commit_hash= +# --build_config= import argparse import mysql.connector @@ -20,12 +21,26 @@ def parse_arguments(): description="ONNXRuntime test coverge report uploader for dashboard") parser.add_argument("--report_url", help="URL to the LLVM json report") parser.add_argument( - "--report_file", help="Path to the local cobertura XML report") - parser.add_argument("--commit_hash", help="Full Git commit hash") + "--report_file", help="Path to the local JSON/TXT report", required=True) + parser.add_argument("--commit_hash", help="Full Git commit hash", required=True) + parser.add_argument("--build_config", help="Build configuration, os, arch and config, in JSON format") return parser.parse_args() -def parse_xml_report(report_file): +def parse_txt_report(report_file): + data = {} + with open(report_file, 'r') as report: + for line in reversed(report.readlines()): + if 'TOTAL' in line: + fields = line.strip().split() + data['lines_valid'] = int(fields[1]) + data['lines_covered'] = int(fields[2]) + data['coverage'] = float(fields[3].strip('%'))/100 + break + return data + + +def parse_json_report(report_file): result = {} with open(report_file) as json_file: data = json.load(json_file) @@ -37,7 +52,7 @@ def parse_xml_report(report_file): return result -def write_to_db(coverage_data, args): +def write_to_db(coverage_data, build_config, args): # connect to database cnx = mysql.connector.connect( @@ -58,21 +73,28 @@ def write_to_db(coverage_data, args): # insert current record insert_query = ('INSERT INTO onnxruntime.test_coverage ' - '(UploadTime, CommitId, Coverage, LinesCovered, TotalLines, ReportURL) ' - 'VALUES (Now(), "%s", %f, %d, %d, "%s") ' + '''(UploadTime, CommitId, Coverage, LinesCovered, TotalLines, OS, + Arch, BuildConfig, ReportURL) ''' + 'VALUES (Now(), "%s", %f, %d, %d, "%s", "%s", "%s", "%s") ' 'ON DUPLICATE KEY UPDATE ' - 'UploadTime=Now(), Coverage=%f, LinesCovered=%d, TotalLines=%d, ReportURL="%s";' + '''UploadTime=Now(), Coverage=%f, LinesCovered=%d, TotalLines=%d, + OS="%s", Arch="%s", BuildConfig="%s", ReportURL="%s"; ''' ) % (args.commit_hash, coverage_data['coverage'], coverage_data['lines_covered'], coverage_data['lines_valid'], + build_config.get('os', 'win'), + build_config.get('arch', 'x64'), + build_config.get('config', 'default'), args.report_url, coverage_data['coverage'], coverage_data['lines_covered'], coverage_data['lines_valid'], + build_config.get('os', 'win'), + build_config.get('arch', 'x64'), + build_config.get('config', 'default'), args.report_url ) - cursor.execute(insert_query) cnx.commit() @@ -91,8 +113,15 @@ def write_to_db(coverage_data, args): if __name__ == "__main__": try: args = parse_arguments() - coverage_data = parse_xml_report(args.report_file) - write_to_db(coverage_data, args) + if args.report_file.endswith(".json"): + coverage_data = parse_json_report(args.report_file) + elif args.report_file.endswith(".txt"): + coverage_data = parse_txt_report(args.report_file) + else: + raise ValueError("Only report extensions txt or json are accepted") + + build_config = json.loads(args.build_config) if args.build_config else {} + write_to_db(coverage_data, build_config, args) except BaseException as e: print(str(e)) sys.exit(1)