[ORT Mobile] ORT Minimal E2E CI (#5200)

* Modify the ort minimal CI to ort minimal e2e ci

onnxruntime/test/testdata/ort_minimal_e2e_test_data/test_voice_commands/test_data_set_0/input_0.pb

@@ -0,0 +1 @@
+

Binput_1JòÛ@*.?±©¼¿?ä€Á@£8Á,»}@

onnxruntime/test/testdata/ort_minimal_e2e_test_data/test_voice_commands/test_data_set_0/output_0.pb

@@ -0,0 +1 @@
+

Bdense_1Jh=§=Ý?=<3D>j=Æ"=Ùü#=Ð=J„=ý~=º` =Èè=#f=“B=î<>=W|=k?=BË=Xÿ=©‹=c=4_=ß)=é/"=êå=[@=«4=

tools/ci_build/exclude_unused_ops.py

@@ -345,7 +345,7 @@ def exclude_unused_ops(models_path, config_path, ort_root=None, use_cuda=True, o
         log.error('Please specify model_path and/or config_path.')
         sys.exit(-1)
 
-    if not ort_root:
+    if not ort_root and not output_config_path:
         log.info('ort_root was not specified. Inferring ONNX Runtime repository root from location of this script.')
 
     required_ops = _extract_ops_from_config(config_path, _extract_ops_from_model(models_path, {}))

tools/ci_build/github/azure-pipelines/linux-cpu-minimal-build-ci-pipeline.yml

@@ -1,5 +1,14 @@
+# This CI has 3 major steps
+# 1. Build full ORT, install the full ORT python wheel and use it to generate ort format test models
+#    and include ops config file for step 3
+# 2. Build minimal ORT including all the kernels and disable exceptions.
+#    This step is build only to safe-guard the --disable_exceptions option.
+# 3. Build minimal ORT include only the kernels using the include ops config file from step 1,
+#    and the models from <repo root>/onnxruntime/test/testdata/, run UT, and use onnx_test_runner to
+#    test the ort format models generated in step 1.
+#    Exceptions are enabled in this step to help debugging in case of CI failure.
 jobs:
-- job: Linux_CPU_Minimal_Build
+- job: Linux_CPU_Minimal_Build_E2E
   timeoutInMinutes: 120
   workspace:
     clean: all
@@ -16,21 +25,30 @@ jobs:
       command: login
       addPipelineData: false
   - task: CmdLine@2
+    displayName: Build full onnxruntime and generate ORT format test files
     inputs:
       script: |
-        # We need copy the related test files to a separated folder since the --include_ops_by_model will search the testdata folder recursively
-        # and include many unnecessary ops, minimal build UT currently uses .ort format models converted from the models we copied below,
-        # which will be used as the input of --include_ops_by_model to have ops to be included for the minimal build UT.
-        mkdir -p $HOME/.models
-        cp $(Build.SourcesDirectory)/onnxruntime/test/testdata/ort_github_issue_4031.onnx $HOME/.models/
-        mkdir -p $HOME/.onnx
+        # Create a folder for all test data
+        mkdir -p $HOME/.test_data
+        docker run --rm \
+          --volume $(Build.SourcesDirectory):/onnxruntime_src \
+          --volume $(Build.BinariesDirectory):/build \
+          --volume $HOME/.test_data:/home/onnxruntimedev/.test_data \
+          -e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \
+          -e NIGHTLY_BUILD \
+          -e BUILD_BUILDNUMBER \
+          onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:ch36 \
+            /bin/bash /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_full_ort_and_create_ort_files.sh
+      workingDirectory: $(Build.SourcesDirectory)
+  - task: CmdLine@2
+    displayName: Build minimal onnxruntime and disable exception
+    inputs:
+      script: |
+        # We will try to build the ORT minimal with exception disabled
+        # Only the building process is verified here, no test will be performed
         docker run --rm \
-          --volume /data/onnx:/data/onnx:ro \
           --volume $(Build.SourcesDirectory):/onnxruntime_src \
           --volume $(Build.BinariesDirectory):/build \
-          --volume /data/models:/build/models:ro \
-          --volume $HOME/.onnx:/home/onnxruntimedev/.onnx \
-          --volume $HOME/.models:/home/onnxruntimedev/.models \
           -e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \
           -e NIGHTLY_BUILD \
           -e BUILD_BUILDNUMBER \
@@ -41,10 +59,23 @@ jobs:
               --skip_submodule_sync \
               --build_shared_lib \
               --parallel \
+              --skip_tests \
               --minimal_build \
-              --disable_ml_ops \
-              --disable_exceptions \
-              --include_ops_by_model /home/onnxruntimedev/.models/
+              --disable_exceptions
+      workingDirectory: $(Build.SourcesDirectory)
+  - task: CmdLine@2
+    displayName: Build minimal onnxruntime and run tests, exception enabled
+    inputs:
+      script: |
+        docker run --rm \
+          --volume $(Build.SourcesDirectory):/onnxruntime_src \
+          --volume $(Build.BinariesDirectory):/build \
+          --volume $HOME/.test_data:/home/onnxruntimedev/.test_data \
+          -e ALLOW_RELEASED_ONNX_OPSET_ONLY=1 \
+          -e NIGHTLY_BUILD \
+          -e BUILD_BUILDNUMBER \
+          onnxruntimeregistry.azurecr.io/internal/azureml/onnxruntimecpubuild:ch36 \
+            /bin/bash /onnxruntime_src/tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh
       workingDirectory: $(Build.SourcesDirectory)
   - task: Docker@2
     displayName: logout

tools/ci_build/github/linux/ort_minimal/build_full_ort_and_create_ort_files.sh

@@ -0,0 +1,39 @@
+#!/bin/bash
+
+# This script will run a full ORT build and use the python package built to generate ort format test files,
+# and the exclude ops config file, which will be used in the build_minimal_ort_and_run_tests.sh
+
+set -e
+
+# Run a full build of ORT
+# Since we need the ORT python package to generate the ORT format files and the include ops config files
+# Will not run tests since those are covered by other CIs
+/opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py \
+    --build_dir /build --cmake_generator Ninja \
+    --config Debug \
+    --skip_submodule_sync \
+    --parallel \
+    --build_wheel \
+    --skip_tests \
+    --enable_pybind \
+    --cmake_extra_defines PYTHON_INCLUDE_DIR=/opt/python/cp37-cp37m/include/python3.7m PYTHON_LIBRARY=/usr/lib64/librt.so
+
+# Install the ORT python wheel
+/opt/python/cp37-cp37m/bin/python3 -m pip install -U /build/Debug/dist/*
+
+# Copy the test data to a separated folder
+cp -Rf /onnxruntime_src/onnxruntime/test/testdata/ort_minimal_e2e_test_data /home/onnxruntimedev/.test_data
+
+# Convert all the onnx models in the $HOME/.test_data/ort_minimal_e2e_test_data to ort model
+# and generate the included ops config file as $HOME/.test_data/ort_minimal_e2e_test_data/required_operators.config
+/opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/python/convert_onnx_models_to_ort.py \
+    /home/onnxruntimedev/.test_data/ort_minimal_e2e_test_data
+
+# Delete all the .onnx files, because the minimal build tests will not work on onnx files
+find /home/onnxruntimedev/.test_data/ort_minimal_e2e_test_data -type f -name "*.onnx" -delete
+
+# Uninstall the ORT python wheel
+/opt/python/cp37-cp37m/bin/python3 -m pip uninstall -y onnxruntime
+
+# Clear the build
+rm -rf /build/Debug

tools/ci_build/github/linux/ort_minimal/build_minimal_ort_and_run_tests.sh

@@ -0,0 +1,34 @@
+#!/bin/bash
+
+# This script will build a ORT minimal using
+# 1. The included ops config file generated by build_full_ort_and_create_ort_files.sh,
+# 2. The included models copied from <repo root>/onnxruntime/test/testdata/
+# The build will run the unit tests for the minimal build
+# Extra E2E test cases (converted by build_full_ort_and_create_ort_files.sh) will be run by onnx_test_runner
+
+set -e
+
+# Clear the previous build
+rm -rf /build/Debug
+
+# We need copy the related test files to a separated folder since the --include_ops_by_model will search the testdata folder recursively
+# and include many unnecessary ops, minimal build UT currently uses .ort format models converted from the models we copied below,
+# which will be used as the input of --include_ops_by_model to have ops to be included for the minimal build UT.
+mkdir -p /home/onnxruntimedev/.test_data/models_to_exclude
+cp /onnxruntime_src/onnxruntime/test/testdata/ort_github_issue_4031.onnx /home/onnxruntimedev/.test_data/models_to_exclude
+
+# Build a minimal build with included ops and models
+# then run ORT minimal UTs
+/opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py \
+    --build_dir /build --cmake_generator Ninja \
+    --config Debug \
+    --skip_submodule_sync \
+    --build_shared_lib \
+    --parallel \
+    --minimal_build \
+    --disable_ml_ops \
+    --include_ops_by_model /home/onnxruntimedev/.test_data/models_to_exclude/ \
+    --include_ops_by_config /home/onnxruntimedev/.test_data/ort_minimal_e2e_test_data/required_operators.config
+
+# Run the e2e test cases
+/build/Debug/onnx_test_runner /home/onnxruntimedev/.test_data/ort_minimal_e2e_test_data

tools/python/convert_onnx_models_to_ort.py

@@ -24,7 +24,7 @@ def create_config_file(optimized_model_path, config_file_path):
 
 
 def convert(model_path: str):
-    models = glob.glob(os.path.join(model_path, '*.onnx'))
+    models = glob.glob(os.path.join(model_path, '**', '*.onnx'), recursive=True)
 
     if len(models) == 0:
         raise ValueError("No .onnx files were found in " + model_path)