Add CI build with type reduction enabled (#6622)

onnxruntime/test/testdata/required_ops_config.readme.txt

@@ -7,7 +7,7 @@ required_ops.config:
     List of all the required operators for .onnx models in the testdata directory.
       Note that add_opset_314159.onnx adds a spurious opset for 'Add' so the config file will contain an entry for that.
 
-required_operators_and_types.config
+required_ops_and_types.config
   To generate/update:
     python ../../../tools/python/create_reduced_build_config.py --format ORT --enable_type_reduction . required_ops_and_types.config
   Contains:

tools/ci_build/github/azure-pipelines/linux-cpu-minimal-build-ci-pipeline.yml

@@ -7,6 +7,7 @@
 #    and the models from <repo root>/onnxruntime/test/testdata/, run UT, and use onnx_test_runner to
 #    test the ort format models generated in step 1.
 #    Exceptions are enabled in this step to help debugging in case of CI failure.
+#    This step builds and tests ORT with and without type reduction enabled.
 # 4. Build baseline minimal ORT for Android arm64-v8a including no kernels and disable exceptions
 #    This step is to report the baseline binary size for Android
 jobs:
@@ -15,11 +16,23 @@ jobs:
   workspace:
     clean: all
   pool: Linux-CPU
+
+  variables:
+    test_data_directory: $(Build.SourcesDirectory)/.test_data
+
   steps:
   - checkout: self
     clean: true
     submodules: recursive
 
+  - task: CmdLine@2
+    displayName: Create test data directory
+    inputs:
+      script: |
+        # Create a folder for all test data
+        mkdir -p $(test_data_directory)
+      workingDirectory: $(Build.SourcesDirectory)
+
   - template: templates/get-docker-image-steps.yml
     parameters:
       Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.centos
@@ -31,23 +44,22 @@ jobs:
     displayName: Build full onnxruntime and generate ORT format test files
     inputs:
       script: |
-        # Create a folder for all test data
-        mkdir -p $HOME/.test_data
         docker run --rm \
           --volume $(Build.SourcesDirectory):/onnxruntime_src \
           --volume $(Build.BinariesDirectory):/build \
-          --volume $HOME/.test_data:/home/onnxruntimedev/.test_data \
+          --volume $(test_data_directory):/home/onnxruntimedev/.test_data \
           -e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \
           -e NIGHTLY_BUILD \
           -e BUILD_BUILDNUMBER \
           onnxruntimecentoscpubuild \
             /bin/bash /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_full_ort_and_create_ort_files.sh
       workingDirectory: $(Build.SourcesDirectory)
+
   - task: CmdLine@2
-    displayName: Build minimal onnxruntime with exceptions disabled
+    displayName: Build minimal onnxruntime [exceptions DISABLED, type reduction DISABLED]
     inputs:
       script: |
-        # We will try to build the ORT minimal with exception disabled
+        # We will try to build minimal ORT with exceptions disabled
         # Only the building process is verified here, no test will be performed
         docker run --rm \
           --volume $(Build.SourcesDirectory):/onnxruntime_src \
@@ -66,31 +78,59 @@ jobs:
               --minimal_build \
               --disable_exceptions
       workingDirectory: $(Build.SourcesDirectory)
+
   - task: CmdLine@2
-    displayName: Build minimal onnxruntime and run tests with exceptions enabled
+    displayName: Build minimal onnxruntime [exceptions ENABLED, type reduction DISABLED] and run tests
     inputs:
       script: |
         docker run --rm \
           --volume $(Build.SourcesDirectory):/onnxruntime_src \
           --volume $(Build.BinariesDirectory):/build \
-          --volume $HOME/.test_data:/home/onnxruntimedev/.test_data \
+          --volume $(test_data_directory):/home/onnxruntimedev/.test_data \
           -e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \
           -e NIGHTLY_BUILD \
           -e BUILD_BUILDNUMBER \
           onnxruntimecentoscpubuild \
-            /bin/bash /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh
+            /bin/bash /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh \
+              --build-directory /build/without_type_reduction \
+              --reduced-ops-config /home/onnxruntimedev/.test_data/required_ops.ort_models.config
       workingDirectory: $(Build.SourcesDirectory)
+
+  - script: git checkout -- .
+    displayName: Discard local changes to Git repository files
+    workingDirectory: $(Build.SourcesDirectory)
+
+  - task: CmdLine@2
+    displayName: Build minimal onnxruntime [exceptions ENABLED, type reduction ENABLED] and run tests
+    inputs:
+      script: |
+        docker run --rm \
+          --volume $(Build.SourcesDirectory):/onnxruntime_src \
+          --volume $(Build.BinariesDirectory):/build \
+          --volume $(test_data_directory):/home/onnxruntimedev/.test_data \
+          -e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \
+          -e NIGHTLY_BUILD \
+          -e BUILD_BUILDNUMBER \
+          onnxruntimecentoscpubuild \
+            /bin/bash /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh \
+              --build-directory /build/with_type_reduction \
+              --reduced-ops-config /home/onnxruntimedev/.test_data/required_ops_and_types.ort_models.config \
+              --enable-type-reduction
+      workingDirectory: $(Build.SourcesDirectory)
+
+  - script: git checkout -- .
+    displayName: Discard local changes to Git repository files
+    workingDirectory: $(Build.SourcesDirectory)
+
   - task: CmdLine@2
     displayName: Build onnxruntime minimal baseline for Android arm64-v8a and report binary size
     inputs:
       script: |
-        # Create a folder for all test data
-        mkdir -p $HOME/.test_data
         docker run --rm \
           --volume $(Build.SourcesDirectory):/onnxruntime_src \
           --volume $(Build.BinariesDirectory):/build \
           --volume $ANDROID_HOME:/android_home \
-          --volume $HOME/.test_data:/home/onnxruntimedev/.test_data \
+          --volume $(test_data_directory):/home/onnxruntimedev/.test_data \
           -e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \
           -e NIGHTLY_BUILD \
           -e BUILD_BUILDNUMBER \

tools/ci_build/github/linux/ort_minimal/build_full_ort_and_create_ort_files.sh

@@ -6,7 +6,7 @@
 set -e
 set -x
 
-# Validate the operator kernel registrations, as the ORT model uses hashes of the kernel registration details 
+# Validate the operator kernel registrations, as the ORT model uses hashes of the kernel registration details
 # to find kernels. If the hashes from the registration details are incorrect we will produce a model that will break
 # when the registration is fixed in the future.
 python3 /onnxruntime_src/tools/ci_build/op_registration_validator.py
@@ -26,25 +26,23 @@ python3 /onnxruntime_src/tools/ci_build/build.py \
 # Install the ORT python wheel
 python3 -m pip install --user /build/Debug/dist/*
 
-# Convert all the E2E ONNX models to ORT format 
+# Convert all the E2E ONNX models to ORT format
 python3 /onnxruntime_src/tools/python/convert_onnx_models_to_ort.py \
     /onnxruntime_src/onnxruntime/test/testdata/ort_minimal_e2e_test_data
 
-# Create a config with just the required ops for ORT format models in testdata
-# This is used by build_minimal_ort_and_run_tests.sh later in the linux-cpu-minimal-build-ci-pipeline CI
+# Create configs with just the required ops for ORT format models in testdata
+# These are used by build_minimal_ort_and_run_tests.sh later in the linux-cpu-minimal-build-ci-pipeline CI
 # and will include ops for the E2E models we just converted
+
+# Config without type reduction
 python3 /onnxruntime_src/tools/python/create_reduced_build_config.py --format ORT \
     /onnxruntime_src/onnxruntime/test/testdata \
-    /onnxruntime_src/onnxruntime/test/testdata/required_ops.ort_models.config
+    /home/onnxruntimedev/.test_data/required_ops.ort_models.config
 
-# Re-create testdata/required_ops_and_types.config.
-# This is meaningful when nnapi_minimal_build_minimal_ort_and_run_tests.sh runs later in the
-# linux-cpu-minimal-build-ci-pipeline CI, as recreating the configs checks that we are still creating
-# a valid config. We have a checked in version of the file for use in other CIs where we don't do a full ORT
-# build as part of the CI, such as the android-x86_64-crosscompile-ci-pipeline CI. 
+# Config with type reduction
 python3 /onnxruntime_src/tools/python/create_reduced_build_config.py --format ORT --enable_type_reduction \
     /onnxruntime_src/onnxruntime/test/testdata \
-    /onnxruntime_src/onnxruntime/test/testdata/required_ops_and_types.config
+    /home/onnxruntimedev/.test_data/required_ops_and_types.ort_models.config
 
 # Clear the build
 rm -rf /build/Debug

tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh

@@ -2,23 +2,75 @@
 
 # This script will create a minimal build with the required operators for all ORT format models
 # in the testdata directory. This includes E2E models generated by build_full_ort_and_create_ort_files.sh.
-# The build will run the unit tests for the minimal build, followed by running onnx_test_runner 
+# The build will run the unit tests for the minimal build, followed by running onnx_test_runner
 # for the E2E test cases.
 
-
 set -e
 set -x
 
+USAGE_TEXT="Usage:
+  -b|--build-directory <build directory>
+    Specifies the build directory. Required.
+  -c|--reduced-ops-config <reduced Ops config file>
+    Specifies the reduced Ops configuration file path. Required.
+  [--enable-type-reduction]
+    Builds with type reduction enabled."
+
+BUILD_DIR=
+REDUCED_OPS_CONFIG_FILE=
+ENABLE_TYPE_REDUCTION=
+
+while [[ $# -gt 0 ]]
+do
+    OPTION_KEY="$1"
+    case $OPTION_KEY in
+        -b|--build-directory)
+            BUILD_DIR="$2"
+            shift
+            shift
+            ;;
+        -c|--reduced-ops-config)
+            REDUCED_OPS_CONFIG_FILE="$2"
+            shift
+            shift
+            ;;
+        --enable-type-reduction)
+            ENABLE_TYPE_REDUCTION=1
+            shift
+            ;;
+        *)
+            echo "Invalid option: $1"
+            echo "$USAGE_TEXT"
+            exit 1
+            ;;
+    esac
+done
+
+if [[ -z "${BUILD_DIR}" || -z "${REDUCED_OPS_CONFIG_FILE}" ]]; then
+    echo "Required option was not provided."
+    echo "$USAGE_TEXT"
+    exit 1
+fi
+
 # Perform a minimal build with required ops and run ORT minimal build UTs
 python3 /onnxruntime_src/tools/ci_build/build.py \
-    --build_dir /build --cmake_generator Ninja \
+    --build_dir ${BUILD_DIR} --cmake_generator Ninja \
     --config Debug \
     --skip_submodule_sync \
     --build_shared_lib \
     --parallel \
     --minimal_build \
     --disable_ml_ops \
-    --include_ops_by_config /onnxruntime_src/onnxruntime/test/testdata/required_ops.ort_models.config
+    --include_ops_by_config ${REDUCED_OPS_CONFIG_FILE} \
+    ${ENABLE_TYPE_REDUCTION:+"--enable_reduced_operator_type_support"}
 
 # Run the e2e test cases
-/build/Debug/onnx_test_runner /onnxruntime_src/onnxruntime/test/testdata/ort_minimal_e2e_test_data
+${BUILD_DIR}/Debug/onnx_test_runner /onnxruntime_src/onnxruntime/test/testdata/ort_minimal_e2e_test_data
+
+# Print binary size info
+python3 /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/check_build_binary_size.py \
+    --arch "$(uname -m)" --os "$(uname -o)" --build_config "minimal-reduced" \
+    ${BUILD_DIR}/Debug/libonnxruntime.so
+
+echo "Binary size info:"
+cat ${BUILD_DIR}/Debug/binary_size_data.txt

tools/ci_build/github/linux/ort_minimal/check_build_binary_size.py

@@ -33,7 +33,7 @@ def _check_binary_size(path, readelf, threshold, os_str, arch, build_config):
             '{},{},{},{}\n'.format(os_str, arch, build_config, sections_total)
         ])
 
-    if sections_total > threshold:
+    if threshold is not None and sections_total > threshold:
         raise RuntimeError('Sections total size for {} of {} exceeds threshold of {} by {}. On-disk size={}'
                            .format(path, sections_total, threshold, sections_total - threshold, ondisk_size))
 
@@ -42,10 +42,9 @@ def main():
     argparser = argparse.ArgumentParser(description='Check the binary size for provided path and '
                                                     'create a text file for upload to the performance dashboard.')
 
-    # required
-    argparser.add_argument('-t', '--threshold', type=int, required=True,
-                           help='Return error if binary size exceeds this threshold.')
     # optional
+    argparser.add_argument('-t', '--threshold', type=int,
+                           help='Return error if binary size exceeds this threshold.')
     argparser.add_argument('-r', '--readelf_path', type=str, default='readelf', help='Path to readelf executable.')
     argparser.add_argument('--os', type=str, default='android',
                            help='OS value to include in binary_size_data.txt')