Build shared host protoc in iOS CI pipelines (#9087)

* iOS build, share host protoc build

* Change android ci

* ios packaging ci

* checkout submodule

* revert

* update package pipeline

* minor update

tools/ci_build/github/android/run_nnapi_code_coverage.sh

@@ -12,18 +12,19 @@ fi
 ORT_ROOT=$1
 # Build and run onnxruntime using NNAPI execution provider targeting android emulator
 python3 ${ORT_ROOT}/tools/ci_build/build.py \
-	--android \
-	--build_dir build_nnapi \
-	--android_sdk_path $ANDROID_HOME \
-	--android_ndk_path $ANDROID_NDK_HOME \
-	--android_abi=x86_64 \
-	--android_api=29 \
-	--skip_submodule_sync \
-	--parallel \
-	--use_nnapi \
-	--cmake_generator=Ninja \
-	--build_java \
-	--code_coverage
+    --android \
+    --build_dir build_nnapi \
+    --android_sdk_path $ANDROID_HOME \
+    --android_ndk_path $ANDROID_NDK_HOME \
+    --android_abi=x86_64 \
+    --android_api=29 \
+    --skip_submodule_sync \
+    --parallel \
+    --use_nnapi \
+    --cmake_generator=Ninja \
+    --build_java \
+    --path_to_protoc_exe $ORT_ROOT/protobuf_install/bin/protoc \
+    --code_coverage
 
 # Install gcovr
 python3 -m pip install gcovr

tools/ci_build/github/apple/build_host_protoc.sh

@@ -0,0 +1,33 @@
+#!/bin/bash
+
+# Note: This script is intended to be called from a macOS pipeline to build the host protoc
+# See tools/ci_build/github/azure-pipelines/mac-ios-ci-pipeline.yml
+# The host_protoc can be found as $PROTOC_INSTALL_PATH/bin/protoc
+
+set -e
+
+if [ $# -ne 3 ]
+then
+    echo "Usage: ${0} <repo_root_path> <host_protoc_build_path> <host_protoc_install_path>"
+    exit 1
+fi
+
+set -x
+
+ORT_REPO_ROOT=$1
+PROTOC_BUILD_PATH=$2
+PROTOC_INSTALL_PATH=$3
+
+pushd .
+mkdir -p $PROTOC_BUILD_PATH
+cd $PROTOC_BUILD_PATH
+cmake $ORT_REPO_ROOT/cmake/external/protobuf/cmake \
+    -DCMAKE_POSITION_INDEPENDENT_CODE=ON \
+    -Dprotobuf_BUILD_TESTS=OFF \
+    -Dprotobuf_WITH_ZLIB_DEFAULT=OFF \
+    -Dprotobuf_BUILD_SHARED_LIBS=OFF \
+    -DCMAKE_BUILD_TYPE=Release \
+    -DCMAKE_INSTALL_PREFIX=$PROTOC_INSTALL_PATH
+make -j $(getconf _NPROCESSORS_ONLN)
+make install
+popd

tools/ci_build/github/apple/build_ios_framework.py

@@ -105,9 +105,13 @@ def _build_package(args):
     intermediates_dir = os.path.join(build_dir, 'intermediates')
     build_config = args.config
     base_build_command = [sys.executable, BUILD_PY, '--config=' + build_config] + build_settings['build_params']
+
     if args.include_ops_by_config is not None:
         base_build_command += ['--include_ops_by_config=' + str(args.include_ops_by_config.resolve())]
 
+    if args.path_to_protoc_exe is not None:
+        base_build_command += ['--path_to_protoc_exe=' + str(args.path_to_protoc_exe.resolve())]
+
     # build framework for individual sysroot
     framework_dirs = []
     framework_info_path = ''
@@ -171,6 +175,8 @@ def parse_args():
     parser.add_argument('build_settings_file', type=pathlib.Path,
                         help='Provide the file contains settings for building iOS framework')
 
+    parser.add_argument("--path_to_protoc_exe", type=pathlib.Path, help="Path to protoc exe.")
+
     args = parser.parse_args()
 
     if not args.build_settings_file.resolve().is_file():

tools/ci_build/github/azure-pipelines/android-x86_64-crosscompile-ci-pipeline.yml

@@ -18,6 +18,14 @@ jobs:
   - script: /bin/bash tools/ci_build/github/android/setup_gradle_wrapper.sh $(pwd)
     displayName: Setup gradle wrapper to use gradle 6.8.3
 
+  # We build the host protoc to <ORT_ROOT>/protobuf_install
+  - script: |
+      /bin/bash $(Build.SourcesDirectory)/tools/ci_build/github/apple/build_host_protoc.sh \
+        $(Build.SourcesDirectory) \
+        $(Build.BinariesDirectory)/protobuf \
+        $(Build.SourcesDirectory)/protobuf_install
+    displayName: Build Host Protoc
+
   - script: |
       python3 tools/python/run_android_emulator.py \
         --android-sdk-root ${ANDROID_SDK_ROOT} \
@@ -45,6 +53,7 @@ jobs:
         --skip_submodule_sync \
         --parallel \
         --cmake_generator=Ninja \
+        --path_to_protoc_exe $(Build.SourcesDirectory)/protobuf_install/bin/protoc \
         --build_java
     displayName: CPU EP, Build and Test on Android Emulator
 

tools/ci_build/github/azure-pipelines/mac-ios-ci-pipeline.yml

@@ -6,9 +6,17 @@ jobs:
     MACOSX_DEPLOYMENT_TARGET: '10.14'
   timeoutInMinutes: 150
   steps:
+    - script: |
+        /bin/bash $(Build.SourcesDirectory)/tools/ci_build/github/apple/build_host_protoc.sh \
+          $(Build.SourcesDirectory) \
+          $(Build.BinariesDirectory)/protobuf \
+          $(Build.BinariesDirectory)/protobuf_install
+      displayName: Build Host Protoc
+
     - script: |
         python3 $(Build.SourcesDirectory)/tools/ci_build/build.py \
-          --skip_submodule_sync --build_dir build/iOS_cpu \
+          --skip_submodule_sync \
+          --build_dir $(Build.BinariesDirectory)/iOS_cpu \
           --ios \
           --ios_sysroot iphonesimulator  \
           --osx_arch x86_64 \
@@ -16,11 +24,14 @@ jobs:
           --use_xcode \
           --config RelWithDebInfo \
           --build_apple_framework \
+          --path_to_protoc_exe $(Build.BinariesDirectory)/protobuf_install/bin/protoc \
           --parallel
       displayName: (CPU EP) Build onnxruntime for iOS x86_64 and run tests using simulator
+
     - script: |
         python3 $(Build.SourcesDirectory)/tools/ci_build/build.py \
-          --skip_submodule_sync --build_dir build/iOS_coreml \
+          --skip_submodule_sync \
+          --build_dir $(Build.BinariesDirectory)/iOS_coreml \
           --use_coreml \
           --ios \
           --ios_sysroot iphonesimulator  \
@@ -29,5 +40,6 @@ jobs:
           --use_xcode \
           --config RelWithDebInfo \
           --build_apple_framework \
+          --path_to_protoc_exe $(Build.BinariesDirectory)/protobuf_install/bin/protoc \
           --parallel
       displayName: (CoreML EP) Build onnxruntime for iOS x86_64 and run tests using simulator

tools/ci_build/github/azure-pipelines/mac-ios-packaging-pipeline.yml

@@ -55,10 +55,18 @@ jobs:
       set_var "ORT_SHOULD_UPLOAD_ARCHIVES" "${SHOULD_UPLOAD_ARCHIVES}"
     displayName: "Set variables"
 
+  - script: |
+      /bin/bash $(Build.SourcesDirectory)/tools/ci_build/github/apple/build_host_protoc.sh \
+        $(Build.SourcesDirectory) \
+        $(Build.BinariesDirectory)/protobuf \
+        $(Build.BinariesDirectory)/protobuf_install
+    displayName: Build Host Protoc
+
   - script: |
       python tools/ci_build/github/apple/build_ios_framework.py \
         --build_dir "$(Build.BinariesDirectory)/ios_framework" \
         --include_ops_by_config tools/ci_build/github/android/mobile_package.required_operators.config \
+        --path_to_protoc_exe $(Build.BinariesDirectory)/protobuf_install/bin/protoc \
         tools/ci_build/github/apple/default_mobile_ios_framework_build_settings.json
     displayName: "Build iOS framework"
 

tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml

@@ -192,10 +192,19 @@ jobs:
   timeoutInMinutes:  180
   steps:
     - template: set-version-number-variables-step.yml
+
+    - script: |
+        /bin/bash $(Build.SourcesDirectory)/tools/ci_build/github/apple/build_host_protoc.sh \
+          $(Build.SourcesDirectory) \
+          $(Build.BinariesDirectory)/protobuf \
+          $(Build.BinariesDirectory)/protobuf_install
+      displayName: Build Host Protoc
+
     - script: |
         set -e -x
         python3 tools/ci_build/github/apple/build_ios_framework.py \
           --build_dir "$(Build.BinariesDirectory)/ios_framework" \
+          --path_to_protoc_exe $(Build.BinariesDirectory)/protobuf_install/bin/protoc \
           tools/ci_build/github/apple/default_full_ios_framework_build_settings.json
         mkdir $(Build.BinariesDirectory)/artifacts
         mkdir -p $(Build.BinariesDirectory)/artifacts_staging/onnxruntime-ios-xcframework-$(OnnxRuntimeVersion)

tools/ci_build/github/linux/ort_minimal/nnapi_minimal_build_minimal_ort_and_run_tests.sh

@@ -16,7 +16,7 @@ rm -rf $ORT_ROOT/build_nnapi
 python3 -m pip install --user flatbuffers
 
 # Build minimal package for Android x86_64 Emulator.
-# The unit tests in onnxruntime_test_all will be run on the Android simulator 
+# The unit tests in onnxruntime_test_all will be run on the Android simulator
 python3 $ORT_ROOT/tools/ci_build/build.py \
     --build_dir $MIN_BUILD_DIR \
     --config Debug \
@@ -34,6 +34,7 @@ python3 $ORT_ROOT/tools/ci_build/build.py \
     --disable_ml_ops \
     --disable_exceptions \
     --include_ops_by_config $ORT_ROOT/onnxruntime/test/testdata/required_ops_and_types.config \
+    --path_to_protoc_exe $ORT_ROOT/protobuf_install/bin/protoc \
     --skip_tests
 
 # Push onnxruntime_test_all and testdata to emulator