From c6d9426ef2b36251cc03f220ec8a33f235706028 Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Mon, 30 Aug 2021 19:48:38 -0700 Subject: [PATCH] Add binary size reporting back (#8883) --- ...upload-binary-sizes-from-nuget-package.yml | 57 ------------- .../azure-pipelines/templates/c-api-cpu.yml | 35 ++++++++ .../windows/post_binary_sizes_to_dashboard.py | 81 +++++++++---------- 3 files changed, 71 insertions(+), 102 deletions(-) delete mode 100644 tools/ci_build/github/azure-pipelines/nuget/templates/upload-binary-sizes-from-nuget-package.yml diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/upload-binary-sizes-from-nuget-package.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/upload-binary-sizes-from-nuget-package.yml deleted file mode 100644 index a13f3c9c4b..0000000000 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/upload-binary-sizes-from-nuget-package.yml +++ /dev/null @@ -1,57 +0,0 @@ -parameters: -- name: gitCommitHash - type: string - default: '' - -- name: downloadPath - type: string - default: '$(Build.BinariesDirectory)/nuget-artifact/final-package' - -- name: buildVariant - type: string - default: 'CPU' - -steps: -- task: DownloadPipelineArtifact@0 - displayName: 'Download Pipeline Artifact - Signed NuGet Package' - inputs: - artifactName: 'drop-signed-nuget-${{ parameters.buildVariant }}' - targetPath: '${{ parameters.downloadPath }}' - -- task: UsePythonVersion@0 - inputs: - versionSpec: '3.7' - addToPath: true - architecture: 'x64' - -- script: | - python -m pip install -q mysql-connector-python - displayName: 'Install mysql-connector-python' - -- task: CmdLine@2 - displayName: 'Post binary sizes to the dashboard database using command line' - inputs: - script: | - echo changing directory to artifact download path - pushd "${{ parameters.downloadPath }}" - echo processing nupkg - SETLOCAL EnableDelayedExpansion - FOR /R %%i IN (*.nupkg) do ( - set filename=%%~ni - IF NOT "!filename:~25,7!"=="Managed" ( - echo processing %%~ni.nupkg - copy %%~ni.nupkg %%~ni.zip - echo copied to zip - echo listing lib files in the zip - REM use a single .csv file to put the data - echo os,arch,build_config,size > binary_size_data.txt - 7z.exe l -slt %%~ni.zip runtimes\linux-x64\native\libonnxruntime.so | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo linux,x64,${{ parameters.buildVariant }},%%a >> binary_size_data.txt - 7z.exe l -slt %%~ni.zip runtimes\osx-x64\native\libonnxruntime.dylib | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo osx,x64,${{ parameters.buildVariant }},%%a >> binary_size_data.txt - 7z.exe l -slt %%~ni.zip runtimes\win-x64\native\onnxruntime.dll | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo win,x64,${{ parameters.buildVariant }},%%a >> binary_size_data.txt - 7z.exe l -slt %%~ni.zip runtimes\win-x86\native\onnxruntime.dll | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo win,x86,${{ parameters.buildVariant }},%%a >> binary_size_data.txt - echo calling python script to post to database - python.exe $(Build.SourcesDirectory)\tools\ci_build\github\windows\post_binary_sizes_to_dashboard.py --commit_hash=${{ parameters.gitCommitHash }} --size_data_file=binary_size_data.txt --build_project=Lotus --build_id=$(Build.BuildId) - ) - ) - env: - DASHBOARD_MYSQL_ORT_PASSWORD: $(dashboard-mysql-ort-password) diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml index 6e125403c7..3c5def19d4 100644 --- a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml +++ b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml @@ -536,6 +536,41 @@ jobs: $date = $(Get-Date -Format "yyyy-MM-dd") Write-Host "##vso[task.setvariable variable=CurrentDate]$date" + - task: CmdLine@2 + displayName: 'Post binary sizes to the dashboard database using command line' + inputs: + script: | + python.exe -m pip install azure-kusto-data[pandas] azure-kusto-ingest[pandas] + echo changing directory to artifact download path + cd $(Build.BinariesDirectory)/nuget-artifact/final-package + echo processing nupkg + SETLOCAL EnableDelayedExpansion + FOR /R %%i IN (*.nupkg) do ( + set filename=%%~ni + IF NOT "!filename:~25,7!"=="Managed" ( + echo processing %%~ni.nupkg + copy %%~ni.nupkg %%~ni.zip + echo copied to zip + echo listing lib files in the zip + REM use a single .csv file to put the data + echo os,arch,build_config,size > $(Build.BinariesDirectory)\binary_size_data.txt + 7z.exe l -slt %%~ni.zip runtimes\linux-x64\native\libonnxruntime.so | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo linux,x64,${{ parameters.buildVariant }},%%a >> $(Build.BinariesDirectory)\binary_size_data.txt + 7z.exe l -slt %%~ni.zip runtimes\linux-aarch64\native\libonnxruntime.so | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo linux,aarch64,${{ parameters.buildVariant }},%%a >> $(Build.BinariesDirectory)\binary_size_data.txt + 7z.exe l -slt %%~ni.zip runtimes\osx.10.14-x64\native\libonnxruntime.dylib | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo osx,x64,${{ parameters.buildVariant }},%%a >> $(Build.BinariesDirectory)\binary_size_data.txt + 7z.exe l -slt %%~ni.zip runtimes\win-x64\native\onnxruntime.dll | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo win,x64,${{ parameters.buildVariant }},%%a >> $(Build.BinariesDirectory)\binary_size_data.txt + 7z.exe l -slt %%~ni.zip runtimes\win-x86\native\onnxruntime.dll | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo win,x86,${{ parameters.buildVariant }},%%a >> $(Build.BinariesDirectory)\binary_size_data.txt + ) + ) + - task: AzureCLI@2 + displayName: 'Azure CLI' + inputs: + azureSubscription: AIInfraBuildOnnxRuntimeOSS + scriptLocation: inlineScript + scriptType: batch + inlineScript: | + python.exe $(Build.SourcesDirectory)\tools\ci_build\github\windows\post_binary_sizes_to_dashboard.py --commit_hash=$(Build.SourceVersion) --size_data_file=binary_size_data.txt --build_project=Lotus --build_id=$(Build.BuildId) + workingDirectory: '$(Build.BinariesDirectory)' + - task: AzureFileCopy@3 displayName: 'Copy Signed Native NuGet Package to Blob Store' condition: ne(variables['IsReleaseBuild'], 'true') # release build has a different package naming scheme diff --git a/tools/ci_build/github/windows/post_binary_sizes_to_dashboard.py b/tools/ci_build/github/windows/post_binary_sizes_to_dashboard.py index 7081804b63..16cd8294ec 100644 --- a/tools/ci_build/github/windows/post_binary_sizes_to_dashboard.py +++ b/tools/ci_build/github/windows/post_binary_sizes_to_dashboard.py @@ -4,9 +4,18 @@ import argparse -import mysql.connector import sys import os +import datetime +# ingest from dataframe +import pandas +from azure.kusto.data import KustoConnectionStringBuilder +from azure.kusto.ingest import ( + IngestionProperties, + DataFormat, + ReportLevel, + QueuedIngestClient, +) def parse_arguments(): @@ -49,50 +58,32 @@ def get_binary_sizes(size_data_file): def write_to_db(binary_size_data, args): # connect to database - - cnx = mysql.connector.connect( - user='ort@onnxruntimedashboard', - password=os.environ.get('DASHBOARD_MYSQL_ORT_PASSWORD'), - host='onnxruntimedashboard.mysql.database.azure.com', - database='onnxruntime') - - try: - cursor = cnx.cursor() - - # insert current records - for row in binary_size_data: - insert_query = ('INSERT INTO onnxruntime.binary_size ' - '(build_time, build_project, build_id, commit_id, os, arch, build_config, size) ' - 'VALUES (Now(), "%s", "%s", "%s", "%s", "%s", "%s", %d) ' - 'ON DUPLICATE KEY UPDATE ' - 'build_time=Now(), build_project="%s", build_id="%s", size=%d;' - ) % ( - args.build_project, - args.build_id, - args.commit_hash, - row['os'], - row['arch'], - row['build_config'], - row['size'], - - args.build_project, - args.build_id, - row['size'] - ) - cursor.execute(insert_query) - - cnx.commit() - - # # Use below for debugging: - # cursor.execute('select * from onnxruntime.binary_size') - # for r in cursor: - # print(r) - - cursor.close() - cnx.close() - except BaseException as e: - cnx.close() - raise e + cluster = "https://ingest-onnxruntimedashboarddb.southcentralus.kusto.windows.net" + kcsb = KustoConnectionStringBuilder.with_az_cli_authentication(cluster) + # The authentication method will be taken from the chosen KustoConnectionStringBuilder. + client = QueuedIngestClient(kcsb) + fields = ["build_time", "build_id", "build_project", "commit_id", "os", "arch", "build_config", "size", "Branch"] + now_str = datetime.datetime.now().strftime("%Y-%m-%d %H:%M:%S") + branch_name = os.environ.get('BUILD_SOURCEBRANCHNAME', 'master') + rows = [] + for row in binary_size_data: + rows.append([now_str, + args.build_id, + args.build_project, + args.commit_hash, + row['os'], + row['arch'], + row['build_config'], + row['size'], + branch_name.lower()]) + ingestion_props = IngestionProperties( + database="powerbi", + table="binary_size", + data_format=DataFormat.CSV, + report_level=ReportLevel.FailuresAndSuccesses + ) + df = pandas.DataFrame(data=rows, columns=fields) + client.ingest_from_dataframe(df, ingestion_properties=ingestion_props) if __name__ == "__main__":