Add build configuration to binary size checks pipeline. (#13208)

Add another build configuration to binary size checks pipeline. Enable additional configurations to be added more easily.

cmake/onnxruntime_unittests.cmake

@@ -223,12 +223,13 @@ else()  # minimal and/or reduced ops build
     "${TEST_SRC_DIR}/platform/*.cc"
     )
 
-  if (onnxruntime_MINIMAL_BUILD)
+  if (onnxruntime_MINIMAL_BUILD AND NOT onnxruntime_REDUCED_OPS_BUILD)
     list(APPEND onnxruntime_test_framework_src_patterns
       "${TEST_SRC_DIR}/framework/ort_model_only_test.cc"
     )
+  endif()
 
-  else() # reduced ops build
+  if (NOT onnxruntime_MINIMAL_BUILD)
     file(GLOB onnxruntime_test_ir_src CONFIGURE_DEPENDS
       "${TEST_SRC_DIR}/ir/*.cc"
       "${TEST_SRC_DIR}/ir/*.h"

tools/ci_build/github/azure-pipelines/binary-size-checks-pipeline.yml

@@ -1,6 +1,6 @@
 # steps in this build:
-# 1a. Android minimal baseline build (arm64-v8a, including no kernels, with exceptions disabled)
-# 1b. Android minimal baseline build with debug info
+# 1a. ORT build
+# 1b. ORT build with debug info
 
 parameters:
 - name: DoBuildWithDebugInfo
@@ -8,6 +8,11 @@ parameters:
   type: boolean
   default: false
 
+- name: BuildAllConfigurations
+  displayName: Build all binary size check configurations instead of just the required ones?
+  type: boolean
+  default: false
+
 resources:
   repositories:
   - repository: manylinux
@@ -23,6 +28,20 @@ jobs:
     clean: all
   pool: Linux-CPU-2019
 
+  strategy:
+    # Notes:
+    # - BuildConfigFile path is relative to repository root.
+    # - BinarySizeThresholdInBytes is optional - if unspecified it is not checked.
+    matrix:
+      # required configurations
+      AndroidMinimalBaseline:
+        BuildConfigFile: "tools/ci_build/github/linux/ort_minimal/build_check_binsize_config/android_minimal_baseline.config"
+        BinarySizeThresholdInBytes: 1306224
+      # additional configurations
+      ${{ if or(eq(parameters.BuildAllConfigurations, true), and(in(variables['Build.Reason'], 'IndividualCI', 'BatchedCI'), eq(variables['Build.SourceBranch'], 'refs/heads/main'))) }}:
+        AndroidMinimalWithMobilePackageOps:
+          BuildConfigFile: "tools/ci_build/github/linux/ort_minimal/build_check_binsize_config/android_minimal_with_mobile_package_ops.config"
+
   steps:
   - checkout: self
     clean: true
@@ -35,12 +54,37 @@ jobs:
       DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )"
       Repository: onnxruntimecpubuild
 
-  - task: CmdLine@2
-    displayName: 1a. Build onnxruntime minimal baseline for Android arm64-v8a
+  - task: PythonScript@0
+    displayName: 'Set variables from config file "$(BuildConfigFile)"'
     inputs:
+      scriptSource: inline
       script: |
+        import json
+
+        config_file_path = "$(BuildConfigFile)"
+        with open(config_file_path, mode="r") as config_file:
+            config = json.load(config_file)
+
+        def set_var(name, value):
+            print("Setting variable: {} = '{}'".format(name, value))
+            print("##vso[task.setvariable variable={}]{}".format(name, value))
+
+        set_var("BuildConfigType", config["type"])
+        set_var("BuildConfigOs", config["os"])
+      workingDirectory: $(Build.SourcesDirectory)
+
+  - task: Bash@3
+    displayName: 1a. Build onnxruntime
+    inputs:
+      targetType: inline
+      script: |
+        set -e -x
+        BINARY_SIZE_THRESHOLD_ARGS=""
+        if [[ ! -z "$(BinarySizeThresholdInBytes)" ]]; then
+          echo "Binary size threshold in bytes: $(BinarySizeThresholdInBytes)"
+          BINARY_SIZE_THRESHOLD_ARGS="--threshold_size_in_bytes $(BinarySizeThresholdInBytes)"
+        fi
         NDK_HOME=$(realpath $ANDROID_NDK_HOME)
-        BINARY_SIZE_THRESHOLD_IN_BYTES=1306224
         docker run --rm \
           --volume $(Build.SourcesDirectory):/onnxruntime_src \
           --volume $(Build.BinariesDirectory):/build \
@@ -54,9 +98,10 @@ jobs:
           -e BUILD_REASON=$(Build.Reason) \
           -e BUILD_BRANCH=$(Build.SourceBranch) \
           onnxruntimecpubuild \
-            /bin/bash /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_android_baseline_and_report_bin_size.sh \
-              -b /build/1a \
-              -t ${BINARY_SIZE_THRESHOLD_IN_BYTES}
+            /opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_ort_and_check_binary_size.py \
+              --build_dir /build/1a \
+              ${BINARY_SIZE_THRESHOLD_ARGS} \
+              "/onnxruntime_src/$(BuildConfigFile)"
       workingDirectory: $(Build.SourcesDirectory)
 
   - task: AzureCLI@2
@@ -68,6 +113,7 @@ jobs:
       scriptLocation: inlineScript
       scriptType: bash
       inlineScript: |
+        set -e -x
         BINARY_SIZE_DATA_FILE="$(Build.BinariesDirectory)/1a/MinSizeRel/binary_size_data.txt"
         if [[ ! -f "${BINARY_SIZE_DATA_FILE}" ]]; then
           echo "File not found: ${BINARY_SIZE_DATA_FILE}"
@@ -85,13 +131,15 @@ jobs:
     displayName: 1a. Publish binary artifact
     inputs:
       targetPath: $(Build.BinariesDirectory)/1a/MinSizeRel/libonnxruntime.so
-      artifactName: AndroidMinimalBaselineBinary
+      artifactName: $(BuildConfigOs)-$(BuildConfigType)-binary
 
   - ${{ if parameters.DoBuildWithDebugInfo }}:
-    - task: CmdLine@2
-      displayName: 1b. Build onnxruntime minimal baseline for Android arm64-v8a with debug info
+    - task: Bash@3
+      displayName: 1b. Build onnxruntime with debug info
       inputs:
+        targetType: inline
         script: |
+          set -e -x
           NDK_HOME=$(realpath $ANDROID_NDK_HOME)
           docker run --rm \
             --volume $(Build.SourcesDirectory):/onnxruntime_src \
@@ -106,15 +154,16 @@ jobs:
             -e BUILD_REASON=$(Build.Reason) \
             -e BUILD_BRANCH=$(Build.SourceBranch) \
             onnxruntimecpubuild \
-              /bin/bash /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_android_baseline_and_report_bin_size.sh \
-                -b /build/1b \
-                -d
+              /opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_ort_and_check_binary_size.py \
+                --build_dir /build/1b \
+                --with_debug_info \
+                "/onnxruntime_src/$(BuildConfigFile)"
         workingDirectory: $(Build.SourcesDirectory)
 
     - task: PublishPipelineArtifact@1
-      displayName: 1b. Publish binary artifact
+      displayName: 1b. Publish binary artifact with debug info
       inputs:
         targetPath: $(Build.BinariesDirectory)/1b/MinSizeRel/libonnxruntime.so
-        artifactName: AndroidMinimalBaselineDebugBinary
+        artifactName: $(BuildConfigOs)-$(BuildConfigType)-binary-with-debug-info
 
   - template: templates/clean-agent-build-directory-step.yml

tools/ci_build/github/linux/ort_minimal/build_check_binsize_config/android_minimal_baseline.config

@@ -0,0 +1,18 @@
+{
+    "type": "minimal-baseline",
+    "os": "android",
+    "arch": "arm64-v8a",
+    "build_params": [
+        "--android",
+        "--android_sdk_path=/android_home",
+        "--android_ndk_path=/ndk_home",
+        "--android_abi=arm64-v8a",
+        "--android_api=29",
+        "--minimal_build",
+        "--build_shared_lib",
+        "--build_java",
+        "--disable_ml_ops",
+        "--disable_exceptions",
+        "--include_ops_by_config=/onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_check_binsize_config/no_ops.config"
+    ]
+}

tools/ci_build/github/linux/ort_minimal/build_check_binsize_config/android_minimal_with_mobile_package_ops.config

@@ -0,0 +1,18 @@
+{
+    "type": "minimal-with-mobile-package-ops",
+    "os": "android",
+    "arch": "arm64-v8a",
+    "build_params": [
+        "--android",
+        "--android_sdk_path=/android_home",
+        "--android_ndk_path=/ndk_home",
+        "--android_abi=arm64-v8a",
+        "--android_api=29",
+        "--minimal_build",
+        "--build_shared_lib",
+        "--build_java",
+        "--disable_ml_ops",
+        "--disable_exceptions",
+        "--include_ops_by_config=/onnxruntime_src/tools/ci_build/github/android/mobile_package.required_operators.config"
+    ]
+}

tools/ci_build/github/linux/ort_minimal/build_check_binsize_config/no_ops.config

@@ -0,0 +1 @@
+# no operators

tools/ci_build/github/linux/ort_minimal/build_minimal_ort_android_baseline_and_report_bin_size.sh

@@ -1,57 +0,0 @@
-#!/bin/bash
-
-# This script will run a baseline minimal ort build for android arm64-v8a ABI
-# and write binary size data to a file
-
-set -e
-set -x
-
-while getopts b:t:d parameter
-do case "${parameter}"
-in
-b) BUILD_DIR="${OPTARG}";;
-t) THRESHOLD_SIZE="${OPTARG}";;
-d) INCLUDE_DEBUG_INFO=1;;
-esac
-done
-
-if [[ -z "${BUILD_DIR}" ]]; then
-    echo "Build directory must be specified with -b."
-    exit 1
-fi
-
-CHECK_THRESHOLD_SIZE_ARGS=${THRESHOLD_SIZE:+"--threshold ${THRESHOLD_SIZE}"}
-BUILD_WITH_DEBUG_INFO_ARGS=${INCLUDE_DEBUG_INFO:+"--cmake_extra_defines ADD_DEBUG_INFO_TO_MINIMAL_BUILD=ON"}
-
-export PATH=/opt/python/cp37-cp37m/bin:$PATH
-
-# Create an empty file to be used with build --include_ops_by_config, which will include no operators at all
-mkdir -p ${BUILD_DIR}
-echo -n > ${BUILD_DIR}/include_no_operators.config
-
-# Run a baseline minimal build of ORT Android arm64-v8a
-# Generate binary size as ${BUILD_DIR}/MinSizeRel/binary_size_data.txt
-python3 /onnxruntime_src/tools/ci_build/build.py \
-    --build_dir ${BUILD_DIR} --cmake_generator Ninja \
-    --config MinSizeRel \
-    --skip_submodule_sync \
-    --parallel \
-    --android \
-    --android_sdk_path /android_home \
-    --android_ndk_path /ndk_home \
-    --android_abi=arm64-v8a \
-    --android_api=29 \
-    --minimal_build \
-    --build_shared_lib \
-    --build_java \
-    --disable_ml_ops \
-    --disable_exceptions \
-    --include_ops_by_config ${BUILD_DIR}/include_no_operators.config \
-    ${BUILD_WITH_DEBUG_INFO_ARGS}
-
-python3 /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/check_build_binary_size.py \
-    ${CHECK_THRESHOLD_SIZE_ARGS} \
-    ${BUILD_DIR}/MinSizeRel/libonnxruntime.so
-
-echo "The content of binary_size_data.txt"
-cat ${BUILD_DIR}/MinSizeRel/binary_size_data.txt

tools/ci_build/github/linux/ort_minimal/build_ort_and_check_binary_size.py

@@ -0,0 +1,74 @@
+#!/usr/bin/env python3
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import argparse
+import json
+import pathlib
+import subprocess
+import sys
+
+REPO_ROOT = pathlib.Path(__file__).resolve().parents[5]
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="Builds ORT and checks the binary size.")
+
+    parser.add_argument("build_check_binsize_config", type=pathlib.Path, help="Path to configuration file.")
+    parser.add_argument("--build_dir", type=pathlib.Path, required=True, help="Path to build directory.")
+    parser.add_argument("--threshold_size_in_bytes", type=int, help="Binary size limit in bytes.")
+    parser.add_argument(
+        "--with_debug_info", action="store_true", help="Whether to include debug information in the build."
+    )
+
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+
+    with open(args.build_check_binsize_config, mode="r") as config_file:
+        config = json.load(config_file)
+
+    config_type = config["type"]
+    os = config["os"]
+    arch = config["arch"]
+    build_params = config["build_params"]
+    build_config = "MinSizeRel"  # could make this configurable if needed
+
+    # build ORT
+    build_command = (
+        [sys.executable, str(REPO_ROOT / "tools/ci_build/build.py")]
+        + build_params
+        + (["--cmake_extra_defines", "ADD_DEBUG_INFO_TO_MINIMAL_BUILD=ON"] if args.with_debug_info else [])
+        # put the following options last so they don't get overridden by build_params
+        + [
+            f"--build_dir={args.build_dir}",
+            f"--config={build_config}",
+            "--update",
+            "--build",
+            "--parallel",
+            "--test",
+        ]
+    )
+
+    subprocess.run(build_command, check=True)
+
+    # check binary size
+    check_binary_size_command = (
+        [
+            sys.executable,
+            str(REPO_ROOT / "tools/ci_build/github/linux/ort_minimal/check_build_binary_size.py"),
+            f"--os={os}",
+            f"--arch={arch}",
+            f"--build_config={config_type}",
+        ]
+        + ([f"--threshold={args.threshold_size_in_bytes}"] if args.threshold_size_in_bytes else [])
+        + [str(args.build_dir / build_config / "libonnxruntime.so")]
+    )
+
+    subprocess.run(check_binary_size_command, check=True)
+
+
+if __name__ == "__main__":
+    main()