Bring code coverage dashboard back (#8394)

tools/ci_build/github/azure-pipelines/android-x86_64-crosscompile-ci-pipeline.yml

@@ -73,12 +73,12 @@ jobs:
 - job: Update_Dashboard
   workspace:
     clean: all
-  pool:
-    vmImage: 'ubuntu-latest'
+  variables:
+  - name: skipComponentGovernanceDetection
+    value: true
+  pool: 'Linux-CPU-2019'
+  condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI'))
   dependsOn: Android_CI
-  # disable update dashboard of Android Code coverage until Azure CI machine cannot access database issue is resolved
-  # condition: and(succeeded(), in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI'))
-  condition: false
   steps:
   - task: DownloadPipelineArtifact@0
     displayName: 'Download code coverage report'
@@ -86,23 +86,11 @@ jobs:
       artifactName: 'coverage_rpt.txt'
       targetPath: '$(Build.BinariesDirectory)'
 
-  - task: UsePythonVersion@0
+  - task: AzureCLI@2
+    displayName: 'Post Android Code Coverage To DashBoard'
     inputs:
-      versionSpec: '3.x'
-
-  - task: CmdLine@2
-    inputs:
-      script: |
-        python3 -m pip install mysql-connector-python
-
-  - task: PythonScript@0
-    displayName: Post Android Code Coverage To DashBoard
-    inputs:
-      scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/windows/post_code_coverage_to_dashboard.py'
-      arguments: >
-        --commit_hash=$(Build.SourceVersion)
-        --report_url="https://dev.azure.com/onnxruntime/onnxruntime/_build/results?buildId=$(Build.BuildId)"
-        --report_file="$(Build.BinariesDirectory)/coverage_rpt.txt"
-        --branch $(Build.SourceBranchName) --arch $(Agent.OSArchitecture) --os "android" --build_config nnapi
-    env:
-      DASHBOARD_MYSQL_ORT_PASSWORD: $(dashboard-mysql-ort-password)
+      azureSubscription: AIInfraBuild
+      scriptType: bash
+      scriptPath: $(Build.SourcesDirectory)/tools/ci_build/github/linux/upload_code_coverage_data.sh
+      arguments: '"$(Build.BinariesDirectory)/coverage_rpt.txt" "https://dev.azure.com/onnxruntime/onnxruntime/_build/results?buildId=$(Build.BuildId)" arm android nnapi'
+      workingDirectory: '$(Build.BinariesDirectory)'

tools/ci_build/github/azure-pipelines/post-merge-jobs.yml

@@ -13,8 +13,6 @@ jobs:
       inputs:
         script: |
           set -e
-          sudo apt-get install -y python3-pip
-          sudo python3 -m pip install mysql-connector-python
           ln -s /data/models .
           #Build onnxruntime and run the instrumented program(unitests)
           LLVM_PROFILE_FILE="%p.profraw" CFLAGS="-g -fprofile-instr-generate -fcoverage-mapping" CXXFLAGS="-g -fprofile-instr-generate -fcoverage-mapping" CC=clang CXX=clang++  python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir=$(Build.BinariesDirectory) --config Debug --parallel --skip_submodule_sync --build_shared_lib --enable_onnx_tests --cmake_extra_defines RUN_MODELTEST_IN_DEBUG_MODE=ON
@@ -29,7 +27,14 @@ jobs:
           llvm-cov show -instr-profile=ort.profdata onnxruntime_test_all -object onnxruntime_mlas_test -object onnxruntime_api_tests_without_env -object onnx_test_runner -object onnxruntime_shared_lib_test -object onnxruntime_global_thread_pools_test -object onnxruntime_api_tests_without_env $(Build.SourcesDirectory)/include/onnxruntime $(Build.SourcesDirectory)/onnxruntime/core $(Build.SourcesDirectory)/onnxruntime/contrib_ops --format=html -output-dir=$(Build.ArtifactStagingDirectory)
         workingDirectory: $(Build.BinariesDirectory)
 
-
+    - task: AzureCLI@2
+      displayName: 'Azure CLI'
+      inputs:
+        azureSubscription: AIInfraBuildOnnxRuntimeOSS
+        scriptType: bash
+        scriptPath: $(Build.SourcesDirectory)/tools/ci_build/github/linux/upload_code_coverage_data.sh
+        arguments: '"$(Build.BinariesDirectory)/report.json" "https://aiinfra.visualstudio.com/Lotus/_build/results?buildId=$(Build.BuildId)" x64 linux default'
+        workingDirectory: '$(Build.BinariesDirectory)'
 
     - task: PublishPipelineArtifact@1
       displayName: 'Publish Pipeline Artifact'
@@ -37,24 +42,18 @@ jobs:
         targetPath: '$(Build.ArtifactStagingDirectory)'
         artifact: html
 
-    - task: PythonScript@0
-      displayName: 'Post code coverage data to the Dashboard'
-      inputs:
-        scriptPath: $(Build.SourcesDirectory)/tools/ci_build/github/windows/post_code_coverage_to_dashboard.py
-        arguments: --commit_hash=$(OnnxRuntimeGitCommitHash) --report_file="$(Build.BinariesDirectory)/report.json" --report_url="https://aiinfra.visualstudio.com/Lotus/_build/results?buildId=$(Build.BuildId)" --branch $(Build.SourceBranchName) --arch $(Agent.OSArchitecture) --os $(Agent.OS) --build_config default
-        workingDirectory: $(Build.BinariesDirectory)
-        pythonInterpreter: '/usr/bin/python3'
-      env: 
-        DASHBOARD_MYSQL_ORT_PASSWORD: $(dashboard-mysql-ort-password)
-
-
 - job: Upload
   workspace:
     clean: all
   pool: 'Win-CPU-2021'
+  variables:
+  - name: runCodesignValidationInjection
+    value: false
+  - name: skipComponentGovernanceDetection
+    value: true
   dependsOn:
   - CodeCoverage
-  condition: succeeded()
+  condition: and (succeeded(), eq(variables['Build.SourceBranch'], 'refs/heads/master'))
   steps:
   - checkout: self
     submodules: false

tools/ci_build/github/azure-pipelines/templates/windows-code-coverage-steps.yml

@@ -1,42 +0,0 @@
-# Runs steps to generate and publish code coverage data
-
-parameters:
-  OpenCppCoverageExe: '$(Build.BinariesDirectory)\OpenCppCoverage\OpenCppCoverage.exe'
-  GitCommitHash: $(OnnxRuntimeGitCommitHash)
-  PublishReport: true
-  PostToDashBoard: false
-  ContinueOnError: false
-steps:
-
-- task: PowerShell@2
-  displayName: 'Run Test Coverage'
-  inputs:
-    filePath: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\run_OpenCppCoverage.ps1'
-    arguments: '-OpenCppCoverageExe:"${{parameters.OpenCppCoverageExe}}" -SourceRoot:"$(Build.SourcesDirectory)" -BuildRoot:"$(Build.BinariesDirectory)" -LocalBuild:$true'
-  continueOnError: ${{parameters.ContinueOnError}}  
-
-- task: PublishCodeCoverageResults@1
-  inputs:
-    codeCoverageTool: 'Cobertura'
-    summaryFileLocation: '$(Build.BinariesDirectory)/**/cobertura.xml'
-    reportDirectory: '$(Build.BinariesDirectory)/**/OpenCppCoverageResults'
-  continueOnError: ${{parameters.ContinueOnError}}
-  condition: ${{parameters.PublishReport}}
-
-- task: CmdLine@1
-  displayName: 'Install mysql-connector-python conda module'
-  inputs:
-    filename: '$(Build.BinariesDirectory)\packages\python\scripts\conda.exe'
-    arguments: 'install -q --insecure -y mysql-connector-python'
-  timeoutInMinutes: 10
-
-- task: PythonScript@0
-  displayName: 'Generate cmake config'
-  inputs:
-    scriptPath: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\post_code_coverage_to_dashboard.py'
-    arguments: --commit_hash=${{parameters.GitCommitHash}} --report_file="$(Build.BinariesDirectory)/Debug/Debug/cobertura.xml" --report_url="https://dev.azure.com/onnxruntime/onnxruntime/_build/results?buildId=$(Build.BuildId)"
-    workingFolder: "$(Build.BinariesDirectory)"
-  continueOnError: ${{parameters.ContinueOnError}}
-  condition: ${{parameters.PostToDashBoard}}
-  env: 
-    DASHBOARD_MYSQL_ORT_PASSWORD: $(dashboard-mysql-ort-password)

tools/ci_build/github/linux/upload_code_coverage_data.sh

@@ -0,0 +1,5 @@
+#!/bin/bash
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+set -x
+$BUILD_SOURCESDIRECTORY/tools/ci_build/github/windows/post_code_coverage_to_dashboard.py --commit_hash=$BUILD_SOURCEVERSION --report_file $1 --report_url $2 --branch $BUILD_SOURCEBRANCHNAME --arch $3 --os $4 --build_config $5

tools/ci_build/github/windows/post_code_coverage_to_dashboard.py

@@ -9,10 +9,18 @@
 # --commit_hash=<string, full git commit hash>
 
 import argparse
-import mysql.connector
 import json
 import sys
-import os
+import datetime
+# ingest from dataframe
+import pandas
+from azure.kusto.data import KustoConnectionStringBuilder
+from azure.kusto.ingest import (
+    IngestionProperties,
+    DataFormat,
+    ReportLevel,
+    QueuedIngestClient,
+)
 
 
 def parse_arguments():
@@ -56,62 +64,28 @@ def parse_json_report(report_file):
 
 def write_to_db(coverage_data, args):
     # connect to database
-
-    cnx = mysql.connector.connect(
-        user='ort@onnxruntimedashboard',
-        password=os.environ.get('DASHBOARD_MYSQL_ORT_PASSWORD'),
-        host='onnxruntimedashboard.mysql.database.azure.com',
-        database='onnxruntime')
-
-    try:
-        cursor = cnx.cursor()
-
-        # delete old records
-        delete_query = ('DELETE FROM onnxruntime.test_coverage '
-                        'WHERE UploadTime < DATE_SUB(Now(), INTERVAL 30 DAY);'
-                        )
-
-        cursor.execute(delete_query)
-
-        # insert current record
-        insert_query = ('INSERT INTO onnxruntime.test_coverage '
-                        '''(UploadTime, CommitId, Coverage, LinesCovered, TotalLines, OS,
-                          Arch, BuildConfig, ReportURL, Branch) '''
-                        'VALUES (Now(), "%s", %f, %d, %d, "%s", "%s", "%s", "%s", "%s") '
-                        'ON DUPLICATE KEY UPDATE '
-                        '''UploadTime=Now(), Coverage=%f, LinesCovered=%d, TotalLines=%d,
-                          OS="%s", Arch="%s", BuildConfig="%s", ReportURL="%s", Branch="%s"; '''
-                        ) % (args.commit_hash,
-                             coverage_data['coverage'],
-                             coverage_data['lines_covered'],
-                             coverage_data['lines_valid'],
-                             args.os.lower(),
-                             args.arch.lower(),
-                             args.build_config.lower(),
-                             args.report_url.lower(),
-                             args.branch.lower(),
-                             coverage_data['coverage'],
-                             coverage_data['lines_covered'],
-                             coverage_data['lines_valid'],
-                             args.os.lower(),
-                             args.arch.lower(),
-                             args.build_config.lower(),
-                             args.report_url.lower(),
-                             args.branch.lower()
-                             )
-        cursor.execute(insert_query)
-        cnx.commit()
-
-        # # Use below for debugging:
-        # cursor.execute('select * from onnxruntime.test_coverage')
-        # for r in cursor:
-        #     print(r)
-
-        cursor.close()
-        cnx.close()
-    except BaseException as e:
-        cnx.close()
-        raise e
+    cluster = "https://ingest-onnxruntimedashboarddb.southcentralus.kusto.windows.net"
+    kcsb = KustoConnectionStringBuilder.with_az_cli_authentication(cluster)
+    # The authentication method will be taken from the chosen KustoConnectionStringBuilder.
+    client = QueuedIngestClient(kcsb)
+    fields = ["UploadTime", "CommitId", "Coverage", "LinesCovered", "TotalLines", "OS", "Arch", "BuildConfig",
+              "ReportURL", "Branch"]
+    now_str = datetime.datetime.now() .strftime("%Y-%m-%d %H:%M:%S")
+    rows = [[now_str,  args.commit_hash, coverage_data['coverage'],
+             coverage_data['lines_covered'],
+             coverage_data['lines_valid'], args.os.lower(),
+             args.arch.lower(),
+             args.build_config.lower(),
+             args.report_url.lower(),
+             args.branch.lower()]]
+    ingestion_props = IngestionProperties(
+      database="powerbi",
+      table="test_coverage",
+      data_format=DataFormat.CSV,
+      report_level=ReportLevel.FailuresAndSuccesses
+    )
+    df = pandas.DataFrame(data=rows, columns=fields)
+    client.ingest_from_dataframe(df, ingestion_properties=ingestion_props)
 
 
 if __name__ == "__main__":