diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker-gpu.sh b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker-gpu.sh index 4782ba63d3..81dd88b9bf 100755 --- a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker-gpu.sh +++ b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker-gpu.sh @@ -4,7 +4,7 @@ # build docker image for CPU -set -x +set -x -e SOURCE_ROOT=$1 BUILD_DIR=$2 @@ -13,8 +13,6 @@ CurrentOnnxRuntimeVersion=$4 PackageName=${PACKAGENAME:-Microsoft.ML.OnnxRuntime.Gpu} RunTestCsharp=${RunTestCsharp:-true} RunTestNative=${RunTestNative:-true} -#CUDA_VER=cuda10.0-cudnn7.3, cuda9.1-cudnn7.1, cuda10.0-cudnn7.3 -CUDA_VER=${5:-cuda10.0-cudnn7.3} PYTHON_VER=3.5 IMAGE="ubuntu16.04-$CUDA_VER" @@ -22,22 +20,16 @@ OldDir=$(pwd) cd $SOURCE_ROOT/tools/ci_build/github/linux/docker -DOCKER_FILE=Dockerfile.ubuntu_gpu_cuda9 -if [ $CUDA_VER = "cuda10.0-cudnn7.3" ]; then -DOCKER_FILE=Dockerfile.ubuntu_gpu -fi +sudo docker build -t onnxruntime-ubuntu16.04-cuda10.1-cudnn7.6 --build-arg OS_VERSION=16.04 --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f Dockerfile.ubuntu_gpu . +sudo docker rm -f "onnxruntime-gpu-container" || true -docker build -t "onnxruntime-$IMAGE" --build-arg OS_VERSION=16.04 --build-arg PYTHON_VERSION=${PYTHON_VER} -f $DOCKER_FILE . -docker rm -f "onnxruntime-gpu-container" || true - -set +e - -docker run -h $HOSTNAME \ +sudo --preserve-env docker run -h $HOSTNAME \ --rm \ --name "onnxruntime-gpu-container" \ --volume "$SOURCE_ROOT:/onnxruntime_src" \ --volume "$BUILD_DIR:/home/onnxruntimedev" \ + --volume /data/models:/home/onnxruntimedev/models:ro \ -e "OnnxRuntimeBuildDirectory=/home/onnxruntimedev" \ -e "IsReleaseBuild=$ISRELEASEBUILD" \ -e "PackageName=$PackageName" \ @@ -45,12 +37,4 @@ docker run -h $HOSTNAME \ -e "RunTestNative=$RunTestNative" \ "onnxruntime-$IMAGE" \ /bin/bash /onnxruntime_src/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.sh \ - /home/onnxruntimedev/$NUGET_REPO_DIRNAME /onnxruntime_src /home/onnxruntimedev $CurrentOnnxRuntimeVersion & - -wait -n - -EXIT_CODE=$? - -set -e -cd $OldDir -exit $EXIT_CODE + /home/onnxruntimedev/$NUGET_REPO_DIRNAME /onnxruntime_src /home/onnxruntimedev $CurrentOnnxRuntimeVersion diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker.sh b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker.sh index 72ec50b5ea..e332d17b8f 100755 --- a/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker.sh +++ b/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest-docker.sh @@ -4,7 +4,7 @@ # build docker image for CPU -set -x +set -x -e SOURCE_ROOT=$1 BUILD_DIR=$2 @@ -25,27 +25,27 @@ cd $SOURCE_ROOT/tools/ci_build/github/linux/docker if [ $UseCentos7 = "false" ]; then echo "Image used for testing is onnxruntime-$IMAGE" if [ $Arch = "x86" ]; then - docker build -t "onnxruntime-$IMAGE" --build-arg OS_VERSION=16.04 --build-arg PYTHON_VERSION=${PYTHON_VER} -f Dockerfile.ubuntu_x86 . + sudo docker build -t "onnxruntime-$IMAGE" --build-arg OS_VERSION=16.04 --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f Dockerfile.ubuntu_x86 . else - docker build -t "onnxruntime-$IMAGE" --build-arg OS_VERSION=16.04 --build-arg PYTHON_VERSION=${PYTHON_VER} -f Dockerfile.ubuntu . + sudo docker build -t "onnxruntime-$IMAGE" --build-arg OS_VERSION=16.04 --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f Dockerfile.ubuntu . fi else IMAGE="centos7" PYTHON_VER=3.6 echo "Image used for testing is onnxruntime-$IMAGE" - docker build -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f Dockerfile.centos . + sudo docker build -t "onnxruntime-$IMAGE" --build-arg BUILD_USER=onnxruntimedev --build-arg BUILD_UID=$(id -u) --build-arg PYTHON_VERSION=${PYTHON_VER} -f Dockerfile.centos . fi -docker rm -f "onnxruntime-cpu" || true +sudo docker rm -f "onnxruntime-cpu" || true -set +e -docker run -h $HOSTNAME \ +sudo --preserve-env docker run -h $HOSTNAME \ --rm \ --name "onnxruntime-cpu" \ --volume "$SOURCE_ROOT:/onnxruntime_src" \ --volume "$BUILD_DIR:/home/onnxruntimedev" \ + --volume /data/models:/home/onnxruntimedev/models:ro \ -e "OnnxRuntimeBuildDirectory=/home/onnxruntimedev" \ -e "IsReleaseBuild=$ISRELEASEBUILD" \ -e "PackageName=$PackageName" \ @@ -54,12 +54,4 @@ docker run -h $HOSTNAME \ -e "RunTestNative=$RunTestNative" \ "onnxruntime-$IMAGE" \ /bin/bash /onnxruntime_src/csharp/test/Microsoft.ML.OnnxRuntime.EndToEndTests/runtest.sh \ - /home/onnxruntimedev/$NUGET_REPO_DIRNAME /onnxruntime_src /home/onnxruntimedev $CurrentOnnxRuntimeVersion & - -wait -n - -EXIT_CODE=$? - -set -e -exit $EXIT_CODE -cd $OldDir + /home/onnxruntimedev/$NUGET_REPO_DIRNAME /onnxruntime_src /home/onnxruntimedev $CurrentOnnxRuntimeVersion diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs index b0e30d12e9..80d87af996 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests/InferenceTest.cs @@ -555,6 +555,7 @@ namespace Microsoft.ML.OnnxRuntime.Tests { "tf_resnet_v1_50", "result mismatch when Conv BN Fusion is applied" }, { "tf_resnet_v1_101", "result mismatch when Conv BN Fusion is applied" }, { "tf_resnet_v1_152", "result mismatch when Conv BN Fusion is applied" }, + { "coreml_Imputer-LogisticRegression_sklearn_load_breast_cancer", "Can't determine model file name" }, { "mask_rcnn_keras", "Model should be edited to remove the extra outputs" }, }; diff --git a/onnxruntime/test/onnx/TestCase.cc b/onnxruntime/test/onnx/TestCase.cc index 7d6ed4a00a..1f93116d68 100644 --- a/onnxruntime/test/onnx/TestCase.cc +++ b/onnxruntime/test/onnx/TestCase.cc @@ -27,6 +27,8 @@ using namespace onnxruntime; using namespace onnxruntime::common; using google::protobuf::RepeatedPtrField; +static constexpr int protobuf_block_size_in_bytes = 1 << 20; + using ORT_VALUE_HOLDER = std::unique_ptr; const std::string TestModelInfo::unknown_version = "unknown version"; @@ -176,7 +178,7 @@ class OnnxModelInfo : public TestModelInfo { } ONNX_NAMESPACE::ModelProto model_pb; - ::google::protobuf::io::FileInputStream input(model_fd, 1 << 20); + ::google::protobuf::io::FileInputStream input(model_fd, protobuf_block_size_in_bytes); const bool parse_result = model_pb.ParseFromZeroCopyStream(&input) && input.GetErrno() == 0; if (!parse_result) { (void)Env::Default().FileClose(model_fd); @@ -277,7 +279,7 @@ OrtValue* TensorToOrtValue(const ONNX_NAMESPACE::TensorProto& t, onnxruntime::te void LoopDataFile(int test_data_pb_fd, bool is_input, const TestModelInfo* modelinfo, std::unordered_map& name_data_map, onnxruntime::test::HeapBuffer& b, std::ostringstream& oss) { - google::protobuf::io::FileInputStream f(test_data_pb_fd); + google::protobuf::io::FileInputStream f(test_data_pb_fd, protobuf_block_size_in_bytes); f.SetCloseOnDelete(true); google::protobuf::io::CodedInputStream coded_input(&f); bool clean_eof = false; @@ -502,7 +504,7 @@ static void LoadTensors(const std::vector& pb_files, if (!st.IsOK()) { ORT_THROW("open file '", ToMBString(pb_files.at(i)), "' failed:", st.ErrorMessage()); } - google::protobuf::io::FileInputStream f(tensor_fd); + google::protobuf::io::FileInputStream f(tensor_fd, protobuf_block_size_in_bytes); f.SetCloseOnDelete(true); ONNX_NAMESPACE::TensorProto tensor; if (!tensor.ParseFromZeroCopyStream(&f)) { diff --git a/onnxruntime/test/onnx/main.cc b/onnxruntime/test/onnx/main.cc index 2b16a2e601..f004698107 100644 --- a/onnxruntime/test/onnx/main.cc +++ b/onnxruntime/test/onnx/main.cc @@ -107,7 +107,7 @@ int real_main(int argc, char* argv[], Ort::Env& env) { bool user_graph_optimization_level_set = false; int verbosity_option_count = 0; - OrtLoggingLevel logging_level = ORT_LOGGING_LEVEL_WARNING; + OrtLoggingLevel logging_level = ORT_LOGGING_LEVEL_ERROR; { int ch; while ((ch = getopt(argc, argv, ORT_TSTR("Ac:hj:Mn:r:e:xvo:d:"))) != -1) { @@ -223,11 +223,9 @@ int real_main(int argc, char* argv[], Ort::Env& env) { } // set log level based on number of verbosity options - if (verbosity_option_count == 1) { - logging_level = ORT_LOGGING_LEVEL_INFO; - } else if (verbosity_option_count > 1) { - logging_level = ORT_LOGGING_LEVEL_VERBOSE; - } + logging_level = + static_cast(static_cast(ORT_LOGGING_LEVEL_ERROR) - + std::min(verbosity_option_count, static_cast(ORT_LOGGING_LEVEL_ERROR))); if (concurrent_session_runs > 1 && repeat_count > 1) { fprintf(stderr, "when you use '-r [repeat]', please set '-c' to 1\n"); diff --git a/onnxruntime/test/onnx/runner.cc b/onnxruntime/test/onnx/runner.cc index 4bde15b4f8..3dbcf26186 100644 --- a/onnxruntime/test/onnx/runner.cc +++ b/onnxruntime/test/onnx/runner.cc @@ -169,14 +169,7 @@ Status RunTests(TestEnv& env, int p_models, int concurrent_runs, size_t repeat_c ORT_EVENT ev; ORT_RETURN_IF_ERROR(CreateOnnxRuntimeEvent(&ev)); try { - RunSingleTestCase(env.tests[i], env.env, env.sf, concurrent_runs, repeat_count, tpool, nullptr, [repeat_count, &results, ev, concurrent_runs, test_case_name](std::shared_ptr result, ORT_CALLBACK_INSTANCE pci) { - //TODO:output this information to a xml - if (concurrent_runs == 1) { - TIME_SPEC ts = result->GetSpentTime(); - double spent = TimeSpecToSeconds(&ts); - double spent2 = spent / result->GetExcutionResult().size() / repeat_count; - LOGF_DEFAULT(ERROR, "Test %s finished in %.3g seconds, took %.3g for each input", test_case_name, spent, spent2); - } + RunSingleTestCase(env.tests[i], env.env, env.sf, concurrent_runs, repeat_count, tpool, nullptr, [&results, ev](std::shared_ptr result, ORT_CALLBACK_INSTANCE pci) { results.push_back(result); return OnnxRuntimeSetEventWhenCallbackReturns(pci, ev); }); diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml index bbfd073829..a78793f43a 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml @@ -74,6 +74,7 @@ jobs: - job: 'Linux_CI_GPU_Dev' workspace: clean: all + timeoutInMinutes: 120 pool: 'Linux-GPU-CUDA10' steps: - template: ../../templates/set-version-number-variables-step.yml diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/linux-set-variables-and-download.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/linux-set-variables-and-download.yml index da3540fae2..0138ee3d48 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/linux-set-variables-and-download.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/linux-set-variables-and-download.yml @@ -9,21 +9,3 @@ steps: workingDirectory: $(Build.BinariesDirectory) continueOnError: true condition: always() - -- task: CmdLine@2 - displayName: 'Download azcopy' - inputs: - script: | - curl -so azcopy.tar.gz -L 'https://aka.ms/downloadazcopy-v10-linux' - tar -zxvf azcopy.tar.gz --strip 1 - workingDirectory: $(Build.BinariesDirectory) - -- task: PythonScript@0 - displayName: 'Download test data' - inputs: - scriptPath: '$(Build.SourcesDirectory)/tools/ci_build/github/download_test_data.py' - arguments: --test_data_url $(TestDataUrl) --build_dir $(Build.BinariesDirectory) - pythonInterpreter: '/usr/bin/python3' - workingDirectory: $(Build.BinariesDirectory) - env: - Test_Data_Download_Key: $(ort-internal-test-data-sas) diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml index bc04efab26..d37efd2cb2 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/test_linux.yml @@ -6,6 +6,7 @@ jobs: - job: NuGet_Test_Linux workspace: clean: all + timeoutInMinutes: 120 pool: ${{ parameters.AgentPool }} dependsOn: - NuGet_Packaging diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml index 572823ba2d..c7606f68c9 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/test_win.yml @@ -36,9 +36,6 @@ jobs: modifyEnvironment: true workingFolder: '$(Build.BinariesDirectory)' - - template: set-test-data-variables-step.yml - - template: win-download-test-data.yml - - task: BatchScript@1 displayName: 'Setup VS2019 env vars' inputs: @@ -62,6 +59,11 @@ jobs: parameters: packageFolder: '$(Build.BinariesDirectory)\nuget-artifact' + - script: | + mklink /D /J models C:\local\models + workingDirectory: '$(Build.BinariesDirectory)' + displayName: 'Create models link' + # TODO: Add .Net Core AnyCPU test task - script: | diff --git a/tools/ci_build/github/azure-pipelines/post-merge-jobs.yml b/tools/ci_build/github/azure-pipelines/post-merge-jobs.yml index c24f1bda73..6322db53be 100644 --- a/tools/ci_build/github/azure-pipelines/post-merge-jobs.yml +++ b/tools/ci_build/github/azure-pipelines/post-merge-jobs.yml @@ -1,107 +1,39 @@ jobs: -- job: 'build' - pool: 'Win-CPU-2019' - variables: - OrtPackageId: 'Microsoft.ML.OnnxRuntime' - MsbuildArguments: '-maxcpucount' - OnnxRuntimeBuildDirectory: '$(Build.BinariesDirectory)' - DOTNET_SKIP_FIRST_TIME_EXPERIENCE: true - EnvSetupScript: setup_env.bat - buildArch: x64 - BuildConfig: 'Debug' - setVcvars: true - timeoutInMinutes: 90 +- job: CodeCoverage workspace: clean: all + timeoutInMinutes: 120 + pool: 'Linux-CPU' steps: - - task: UsePythonVersion@0 - inputs: - versionSpec: '3.7' - addToPath: true - architecture: $(buildArch) + - template: templates/set-version-number-variables-step.yml - - task: BatchScript@1 - displayName: 'setup env' - inputs: - filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\$(EnvSetupScript)' - modifyEnvironment: true - workingFolder: '$(Build.BinariesDirectory)' + - task: CmdLine@2 + inputs: + script: | + set -e + sudo apt-get install -y python3-pip + sudo python3 -m pip install mysql-connector-python + ln -s /data/models . + #Build onnxruntime and run the instrumented program(unitests) + LLVM_PROFILE_FILE="%p.profraw" CFLAGS="-g -fprofile-instr-generate -fcoverage-mapping" CXXFLAGS="-g -fprofile-instr-generate -fcoverage-mapping" CC=clang CXX=clang++ python3 $(Build.SourcesDirectory)/tools/ci_build/build.py --build_dir=$(Build.BinariesDirectory) --config Debug --parallel --skip_submodule_sync --build_shared_lib --enable_onnx_tests - - script: | - python -m pip install -q pyopenssl setuptools wheel numpy scipy - mkdir $(Build.BinariesDirectory)\$(BuildConfig) - echo "Creating hard link $(Build.BinariesDirectory)\$(BuildConfig)\models" - mklink /D /J $(Build.BinariesDirectory)\$(BuildConfig)\models C:\local\models - echo "$(Build.BinariesDirectory)\models" - mklink /D /J $(Build.BinariesDirectory)\models C:\local\models - workingDirectory: '$(Build.BinariesDirectory)' - displayName: 'Install python modules' + cd Debug + ./onnxruntime_mlas_test + #Run the model tests(It will take long because it is in debug mode) + LLVM_PROFILE_FILE="testrunner.profraw" ./onnx_test_runner -j 1 -c 2 /data/models + #Merge the mulitple prof data into a single indexed profile data file + llvm-profdata merge -sparse -o ort.profdata *.profraw + #Create coverage report, output the result to 'report.json' + llvm-cov export -summary-only -instr-profile=ort.profdata onnx_test_runner -object onnxruntime_test_all $(Build.SourcesDirectory)/include/onnxruntime $(Build.SourcesDirectory)/onnxruntime/core $(Build.SourcesDirectory)/onnxruntime/contrib_ops > $(Build.BinariesDirectory)/report.json + workingDirectory: $(Build.BinariesDirectory) - - task: NuGetToolInstaller@0 - displayName: Use Nuget 4.9 - inputs: - versionSpec: 4.9.4 - - - task: NuGetCommand@2 - displayName: 'NuGet restore' - inputs: - command: 'restore' - feedsToUse: 'config' - restoreSolution: '$(Build.SourcesDirectory)\packages.config' - nugetConfigPath: '$(Build.SourcesDirectory)\NuGet.config' - restoreDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)' - - - task: CMake@1 - inputs: - workingDirectory: '$(Build.BinariesDirectory)\$(BuildConfig)' - cmakeArgs: $(Build.SourcesDirectory)\cmake -DCMAKE_TOOLCHAIN_FILE=C:\vcpkg\scripts\buildsystems\vcpkg.cmake -DVCPKG_TARGET_TRIPLET=x64-windows -Donnxruntime_PREFER_SYSTEM_LIB=ON -Donnxruntime_RUN_ONNX_TESTS=ON -Donnxruntime_DEV_MODE=ON -Donnxruntime_USE_CUDA=OFF -Donnxruntime_USE_NSYNC=OFF -Donnxruntime_CUDNN_HOME= -Donnxruntime_USE_FEATURIZERS=OFF -Donnxruntime_CUDA_HOME= -Donnxruntime_USE_JEMALLOC=OFF -Donnxruntime_USE_MIMALLOC=OFF -Donnxruntime_ENABLE_PYTHON=OFF -Donnxruntime_BUILD_CSHARP=OFF -Donnxruntime_BUILD_JAVA=OFF -Donnxruntime_BUILD_SHARED_LIB=OFF -Donnxruntime_USE_EIGEN_FOR_BLAS=ON -Donnxruntime_USE_OPENBLAS=OFF -Donnxruntime_USE_DNNL=OFF -Donnxruntime_USE_MKLML=OFF -Donnxruntime_USE_OPENMP=OFF -Donnxruntime_USE_FULL_PROTOBUF=OFF -Donnxruntime_DISABLE_CONTRIB_OPS=OFF -Donnxruntime_MSVC_STATIC_RUNTIME=OFF -Donnxruntime_USE_DML=OFF -Donnxruntime_USE_TELEMETRY=OFF -A x64 -T host=x64 -G "Visual Studio 16 2019" -DCMAKE_BUILD_TYPE=$(BuildConfig) - - - task: VSBuild@1 - displayName: 'Build' - inputs: - solution: '$(Build.BinariesDirectory)\$(BuildConfig)\onnxruntime.sln' - platform: 'x64' - configuration: $(BuildConfig) - msbuildArgs: $(MsbuildArguments) - msbuildArchitecture: $(buildArch) - maximumCpuCount: true - logProjectEvents: false - workingFolder: '$(Build.BinariesDirectory)\$(BuildConfig)' - createLogFile: true - - - template: templates/set-test-data-variables-step.yml - - - - task: BatchScript@1 - displayName: 'Setup VS2019 env vars' - inputs: - filename: 'C:\Program Files (x86)\Microsoft Visual Studio\2019\Enterprise\VC\Auxiliary\Build\vcvarsall.bat' - arguments: $(buildArch) - modifyEnvironment: true - - - script: | - set PATH=$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig);%PATH% - - - task: VSTest@2 - displayName: 'VsTest - testAssemblies' - inputs: - testAssemblyVer2: 'onnxruntime_test_all.exe' - searchFolder: '$(Build.BinariesDirectory)\Debug\Debug' - runSettingsFile: 'cmake\\codeconv.runsettings' - pathtoCustomTestAdapters: '$(Build.BinariesDirectory)\Debug\googletestadapter.0.17.1\build\_common' - codeCoverageEnabled: true - platform: x64 - - - task: PublishTestResults@2 - inputs: - searchFolder: '$(Build.BinariesDirectory)\$(BuildConfig)\$(BuildConfig)\TestResults' - testResultsFormat: 'VSTest' - testResultsFiles: '**' - condition: succeededOrFailed() - - - template: templates/component-governance-component-detection-steps.yml - parameters : - condition : 'succeeded' - - - template: templates/clean-agent-build-directory-step.yml + - task: CmdLine@2 + displayName: 'Post code coverage data to the Dashboard' + inputs: + script: | + python3 $(Build.SourcesDirectory)/tools/ci_build/github/windows/post_code_coverage_to_dashboard.py --commit_hash=$(OnnxRuntimeGitCommitHash) --report_file="$(Build.BinariesDirectory)/report.json" --report_url="https://dev.azure.com/onnxruntime/onnxruntime/_build/results?buildId=$(Build.BuildId)" + workingDirectory: $(Build.BinariesDirectory) + env: + DASHBOARD_MYSQL_ORT_PASSWORD: $(dashboard-mysql-ort-password) + - template: templates/clean-agent-build-directory-step.yml diff --git a/tools/ci_build/github/windows/post_code_coverage_to_dashboard.py b/tools/ci_build/github/windows/post_code_coverage_to_dashboard.py index 53aa30835b..9e467d8445 100644 --- a/tools/ci_build/github/windows/post_code_coverage_to_dashboard.py +++ b/tools/ci_build/github/windows/post_code_coverage_to_dashboard.py @@ -10,27 +10,30 @@ import argparse import mysql.connector -import xml.etree.ElementTree as ET +import json import sys import os def parse_arguments(): - parser = argparse.ArgumentParser(description="ONNXRuntime test coverge report uploader for dashboard") - parser.add_argument("--report_url", help="URL to the Cobertura XML report") - parser.add_argument("--report_file", help="Path to the local cobertura XML report") + parser = argparse.ArgumentParser( + description="ONNXRuntime test coverge report uploader for dashboard") + parser.add_argument("--report_url", help="URL to the LLVM json report") + parser.add_argument( + "--report_file", help="Path to the local cobertura XML report") parser.add_argument("--commit_hash", help="Full Git commit hash") return parser.parse_args() def parse_xml_report(report_file): - tree = ET.parse(report_file) # may throw exception - root = tree.getroot() result = {} + with open(report_file) as json_file: + data = json.load(json_file) - result['coverage'] = float(root.get('line-rate')) - result['lines_covered'] = int(root.get('lines-covered')) - result['lines_valid'] = int(root.get('lines-valid')) + linestat = data['data'][0]['totals']['lines'] + result['coverage'] = float(linestat['percent']/100.0) + result['lines_covered'] = int(linestat['covered']) + result['lines_valid'] = int(linestat['count']) return result