From 9df07908568b385b9d1ba72c576f2cadeb0f6219 Mon Sep 17 00:00:00 2001 From: Guoyu Wang <62914304+gwang-msft@users.noreply.github.com> Date: Fri, 2 Oct 2020 17:35:23 -0700 Subject: [PATCH] Update linux minimal CI to report Android mininal baseline binary size (#5361) * Update linux minimal CI to report Android mininal baseline binary size * Fix some issues in the script --- tools/ci_build/build.py | 6 +++ ...ndroid-x86_64-crosscompile-ci-pipeline.yml | 8 ---- .../linux-cpu-minimal-build-ci-pipeline.yml | 24 +++++++++- .../build_minimal_ort_and_run_tests.sh | 6 +-- ...rt_android_baseline_and_report_bin_size.sh | 44 +++++++++++++++++++ .../windows/post_binary_sizes_to_dashboard.py | 11 +++-- 6 files changed, 84 insertions(+), 15 deletions(-) create mode 100644 tools/ci_build/github/linux/ort_minimal/build_minimal_ort_android_baseline_and_report_bin_size.sh diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index e6ab98b0c3..5b20f39b34 100755 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -1075,6 +1075,12 @@ def run_android_tests(args, source_dir, config, cwd): bin_size_threshold = 1100000 bin_actual_size = os.path.getsize(os.path.join(cwd, 'libonnxruntime.so')) log.info('Android arm64 minsizerel libonnxruntime.so size [' + str(bin_actual_size) + 'B]') + # Write the binary size to a file for uploading later + with open(os.path.join(cwd, 'binary_size_data.txt'), 'w') as file: + file.writelines([ + 'os,arch,build_config,size\n', + 'android,arm64-v8a,minimal-baseline,' + str(bin_actual_size) + '\n' + ]) if bin_actual_size > bin_size_threshold: raise BuildError('Android arm64 minsizerel libonnxruntime.so size [' + str(bin_actual_size) + 'B] is bigger than threshold [' + str(bin_size_threshold) + 'B]') diff --git a/tools/ci_build/github/azure-pipelines/android-x86_64-crosscompile-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/android-x86_64-crosscompile-ci-pipeline.yml index 51f184e740..2a11df956f 100644 --- a/tools/ci_build/github/azure-pipelines/android-x86_64-crosscompile-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/android-x86_64-crosscompile-ci-pipeline.yml @@ -8,11 +8,3 @@ jobs: displayName: Install coreutils and ninja - script: tools/ci_build/build.py --android --build_dir build --android_sdk_path $ANDROID_HOME --android_ndk_path $ANDROID_HOME/ndk-bundle --android_abi=x86_64 --android_api=29 --skip_submodule_sync --parallel --use_nnapi --cmake_generator=Ninja --build_java displayName: Build and Test on Android Emulator - - script: pip3 install -U onnx - # Since we want to build ORT with absolutely minimal size, we will need to exclude all the ops in build by running - # exclude_unused_ops.py, which will require onnx to be installed. - displayName: Install onnx for minimal build - - script: rm -rf build && mkdir -p $TMPDIR/include_ops && touch $TMPDIR/include_ops/include_no_ops.txt - displayName: Clear the previous build and setup for arm64 minimal build - - script: tools/ci_build/build.py --android --build_dir build --android_sdk_path $ANDROID_HOME --android_ndk_path $ANDROID_HOME/ndk-bundle --android_abi=arm64-v8a --android_api=29 --skip_submodule_sync --parallel --cmake_generator=Ninja --config MinSizeRel --minimal_build --disable_rtti --disable_ml_ops --disable_exceptions --build_java --test_binary_size --include_ops_by_config $TMPDIR/include_ops/include_no_ops.txt - displayName: Build arm64 absolutely minimal build and verify the library size diff --git a/tools/ci_build/github/azure-pipelines/linux-cpu-minimal-build-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/linux-cpu-minimal-build-ci-pipeline.yml index a7d0753429..cfb000cce2 100644 --- a/tools/ci_build/github/azure-pipelines/linux-cpu-minimal-build-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/linux-cpu-minimal-build-ci-pipeline.yml @@ -1,4 +1,4 @@ -# This CI has 3 major steps +# This CI has 4 major steps # 1. Build full ORT, install the full ORT python wheel and use it to generate ort format test models # and include ops config file for step 3 # 2. Build minimal ORT including all the kernels and disable exceptions. @@ -7,6 +7,8 @@ # and the models from /onnxruntime/test/testdata/, run UT, and use onnx_test_runner to # test the ort format models generated in step 1. # Exceptions are enabled in this step to help debugging in case of CI failure. +# 4. Build baseline minimal ORT for Android arm64-v8a including no kernels and disable exceptions +# This step is to report the baseline binary size for Android jobs: - job: Linux_CPU_Minimal_Build_E2E timeoutInMinutes: 120 @@ -77,6 +79,26 @@ jobs: onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecentoscpubuild:ch5g \ /bin/bash /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh workingDirectory: $(Build.SourcesDirectory) + - task: CmdLine@2 + displayName: Build onnxruntime minimal baseline for Android arm64-v8a and report binary size + inputs: + script: | + # Create a folder for all test data + mkdir -p $HOME/.test_data + docker run --rm \ + --volume $(Build.SourcesDirectory):/onnxruntime_src \ + --volume $(Build.BinariesDirectory):/build \ + --volume $ANDROID_HOME:/android_home \ + --volume $HOME/.test_data:/home/onnxruntimedev/.test_data \ + -e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \ + -e NIGHTLY_BUILD \ + -e BUILD_BUILDNUMBER \ + -e BUILD_SOURCEVERSION=$(Build.SourceVersion) \ + -e BUILD_ID=$(Build.BuildId) \ + -e DASHBOARD_MYSQL_ORT_PASSWORD=$(dashboard-mysql-ort-password) \ + onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecentoscpubuild:ch5g \ + /bin/bash /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_android_baseline_and_report_bin_size.sh + workingDirectory: $(Build.SourcesDirectory) - task: Docker@2 displayName: logout inputs: diff --git a/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh b/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh index 464ab04f92..78ee61d16e 100644 --- a/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh +++ b/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh @@ -14,8 +14,8 @@ rm -rf /build/Debug # We need copy the related test files to a separated folder since the --include_ops_by_model will search the testdata folder recursively # and include many unnecessary ops, minimal build UT currently uses .ort format models converted from the models we copied below, # which will be used as the input of --include_ops_by_model to have ops to be included for the minimal build UT. -mkdir -p /home/onnxruntimedev/.test_data/models_to_exclude -cp /onnxruntime_src/onnxruntime/test/testdata/ort_github_issue_4031.onnx /home/onnxruntimedev/.test_data/models_to_exclude +mkdir -p /home/onnxruntimedev/.test_data/models_to_include +cp /onnxruntime_src/onnxruntime/test/testdata/ort_github_issue_4031.onnx /home/onnxruntimedev/.test_data/models_to_include # Build a minimal build with included ops and models # then run ORT minimal UTs @@ -27,7 +27,7 @@ python3 /onnxruntime_src/tools/ci_build/build.py \ --parallel \ --minimal_build \ --disable_ml_ops \ - --include_ops_by_model /home/onnxruntimedev/.test_data/models_to_exclude/ \ + --include_ops_by_model /home/onnxruntimedev/.test_data/models_to_include/ \ --include_ops_by_config /home/onnxruntimedev/.test_data/ort_minimal_e2e_test_data/required_operators.config # Run the e2e test cases diff --git a/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_android_baseline_and_report_bin_size.sh b/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_android_baseline_and_report_bin_size.sh new file mode 100644 index 0000000000..f7a7787392 --- /dev/null +++ b/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_android_baseline_and_report_bin_size.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +# This script will run a baseline minimal ort build for android arm64-v8a ABI +# and report the binary size to the ort mysql DB + +set -e + +# Create an empty file to be used with build --include_ops_by_config, which will include no operators at all +echo -n > /home/onnxruntimedev/.test_data/include_no_operators.config + +# Run a baseline minimal build of ORT Android arm64-v8a +# Generate binary size as /build/MinSizeRel/binary_size_data.txt +python3 /onnxruntime_src/tools/ci_build/build.py \ + --build_dir /build --cmake_generator Ninja \ + --config MinSizeRel \ + --skip_submodule_sync \ + --parallel \ + --android \ + --android_sdk_path /android_home \ + --android_ndk_path /android_home/ndk-bundle \ + --android_abi=arm64-v8a \ + --android_api=29 \ + --minimal_build \ + --build_shared_lib \ + --build_java \ + --disable_ml_ops \ + --disable_exceptions \ + --test_binary_size \ + --include_ops_by_config /home/onnxruntimedev/.test_data/include_no_operators.config + +# Install the mysql connector +python3 -m pip install --user mysql-connector-python + +# Post the binary size info to ort mysql DB +# The report script's DB connection failure will not fail the pipeline +python3 /onnxruntime_src/tools/ci_build/github/windows/post_binary_sizes_to_dashboard.py \ + --ignore_db_error \ + --commit_hash=$BUILD_SOURCEVERSION \ + --size_data_file=/build/MinSizeRel/binary_size_data.txt \ + --build_project=onnxruntime \ + --build_id=$BUILD_ID + +# Clear the build +rm -rf /build/MinSizeRel diff --git a/tools/ci_build/github/windows/post_binary_sizes_to_dashboard.py b/tools/ci_build/github/windows/post_binary_sizes_to_dashboard.py index 0e4a344de1..7081804b63 100644 --- a/tools/ci_build/github/windows/post_binary_sizes_to_dashboard.py +++ b/tools/ci_build/github/windows/post_binary_sizes_to_dashboard.py @@ -16,6 +16,8 @@ def parse_arguments(): help="Lotus or onnxruntime build project, to construct the build URL") parser.add_argument("--build_id", help="Build Id") parser.add_argument("--size_data_file", help="Path to file that contains the binary size data") + parser.add_argument("--ignore_db_error", action='store_true', + help="Ignore database errors while executing this script") return parser.parse_args() @@ -94,10 +96,13 @@ def write_to_db(binary_size_data, args): if __name__ == "__main__": + args = parse_arguments() + binary_size_data = get_binary_sizes(args.size_data_file) try: - args = parse_arguments() - binary_size_data = get_binary_sizes(args.size_data_file) write_to_db(binary_size_data, args) except BaseException as e: print(str(e)) - sys.exit(1) + # If there is DB connection error, and we choose '--ignore_db_error' + # we can let the script exit clean in order not to fail the pipeline + if not args.ignore_db_error: + sys.exit(1)