From 438175cb34d91ecc2f4d65c7c3092c3d734ecbd8 Mon Sep 17 00:00:00 2001 From: Guoyu Wang <62914304+gwang-msft@users.noreply.github.com> Date: Thu, 16 Sep 2021 17:16:50 -0700 Subject: [PATCH] Build shared host protoc in iOS CI pipelines (#9087) * iOS build, share host protoc build * Change android ci * ios packaging ci * checkout submodule * revert * update package pipeline * minor update --- .../github/android/run_nnapi_code_coverage.sh | 25 +++++++------- .../github/apple/build_host_protoc.sh | 33 +++++++++++++++++++ .../github/apple/build_ios_framework.py | 6 ++++ ...ndroid-x86_64-crosscompile-ci-pipeline.yml | 9 +++++ .../azure-pipelines/mac-ios-ci-pipeline.yml | 16 +++++++-- .../mac-ios-packaging-pipeline.yml | 8 +++++ .../azure-pipelines/templates/c-api-cpu.yml | 9 +++++ ...minimal_build_minimal_ort_and_run_tests.sh | 3 +- 8 files changed, 94 insertions(+), 15 deletions(-) create mode 100644 tools/ci_build/github/apple/build_host_protoc.sh diff --git a/tools/ci_build/github/android/run_nnapi_code_coverage.sh b/tools/ci_build/github/android/run_nnapi_code_coverage.sh index 657bd28b31..472e824eaa 100755 --- a/tools/ci_build/github/android/run_nnapi_code_coverage.sh +++ b/tools/ci_build/github/android/run_nnapi_code_coverage.sh @@ -12,18 +12,19 @@ fi ORT_ROOT=$1 # Build and run onnxruntime using NNAPI execution provider targeting android emulator python3 ${ORT_ROOT}/tools/ci_build/build.py \ - --android \ - --build_dir build_nnapi \ - --android_sdk_path $ANDROID_HOME \ - --android_ndk_path $ANDROID_NDK_HOME \ - --android_abi=x86_64 \ - --android_api=29 \ - --skip_submodule_sync \ - --parallel \ - --use_nnapi \ - --cmake_generator=Ninja \ - --build_java \ - --code_coverage + --android \ + --build_dir build_nnapi \ + --android_sdk_path $ANDROID_HOME \ + --android_ndk_path $ANDROID_NDK_HOME \ + --android_abi=x86_64 \ + --android_api=29 \ + --skip_submodule_sync \ + --parallel \ + --use_nnapi \ + --cmake_generator=Ninja \ + --build_java \ + --path_to_protoc_exe $ORT_ROOT/protobuf_install/bin/protoc \ + --code_coverage # Install gcovr python3 -m pip install gcovr diff --git a/tools/ci_build/github/apple/build_host_protoc.sh b/tools/ci_build/github/apple/build_host_protoc.sh new file mode 100644 index 0000000000..22b693ed25 --- /dev/null +++ b/tools/ci_build/github/apple/build_host_protoc.sh @@ -0,0 +1,33 @@ +#!/bin/bash + +# Note: This script is intended to be called from a macOS pipeline to build the host protoc +# See tools/ci_build/github/azure-pipelines/mac-ios-ci-pipeline.yml +# The host_protoc can be found as $PROTOC_INSTALL_PATH/bin/protoc + +set -e + +if [ $# -ne 3 ] +then + echo "Usage: ${0} " + exit 1 +fi + +set -x + +ORT_REPO_ROOT=$1 +PROTOC_BUILD_PATH=$2 +PROTOC_INSTALL_PATH=$3 + +pushd . +mkdir -p $PROTOC_BUILD_PATH +cd $PROTOC_BUILD_PATH +cmake $ORT_REPO_ROOT/cmake/external/protobuf/cmake \ + -DCMAKE_POSITION_INDEPENDENT_CODE=ON \ + -Dprotobuf_BUILD_TESTS=OFF \ + -Dprotobuf_WITH_ZLIB_DEFAULT=OFF \ + -Dprotobuf_BUILD_SHARED_LIBS=OFF \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_INSTALL_PREFIX=$PROTOC_INSTALL_PATH +make -j $(getconf _NPROCESSORS_ONLN) +make install +popd diff --git a/tools/ci_build/github/apple/build_ios_framework.py b/tools/ci_build/github/apple/build_ios_framework.py index 3d40a1c17c..92e2053ccc 100644 --- a/tools/ci_build/github/apple/build_ios_framework.py +++ b/tools/ci_build/github/apple/build_ios_framework.py @@ -105,9 +105,13 @@ def _build_package(args): intermediates_dir = os.path.join(build_dir, 'intermediates') build_config = args.config base_build_command = [sys.executable, BUILD_PY, '--config=' + build_config] + build_settings['build_params'] + if args.include_ops_by_config is not None: base_build_command += ['--include_ops_by_config=' + str(args.include_ops_by_config.resolve())] + if args.path_to_protoc_exe is not None: + base_build_command += ['--path_to_protoc_exe=' + str(args.path_to_protoc_exe.resolve())] + # build framework for individual sysroot framework_dirs = [] framework_info_path = '' @@ -171,6 +175,8 @@ def parse_args(): parser.add_argument('build_settings_file', type=pathlib.Path, help='Provide the file contains settings for building iOS framework') + parser.add_argument("--path_to_protoc_exe", type=pathlib.Path, help="Path to protoc exe.") + args = parser.parse_args() if not args.build_settings_file.resolve().is_file(): diff --git a/tools/ci_build/github/azure-pipelines/android-x86_64-crosscompile-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/android-x86_64-crosscompile-ci-pipeline.yml index 0713654c10..2f59a901fa 100644 --- a/tools/ci_build/github/azure-pipelines/android-x86_64-crosscompile-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/android-x86_64-crosscompile-ci-pipeline.yml @@ -18,6 +18,14 @@ jobs: - script: /bin/bash tools/ci_build/github/android/setup_gradle_wrapper.sh $(pwd) displayName: Setup gradle wrapper to use gradle 6.8.3 + # We build the host protoc to /protobuf_install + - script: | + /bin/bash $(Build.SourcesDirectory)/tools/ci_build/github/apple/build_host_protoc.sh \ + $(Build.SourcesDirectory) \ + $(Build.BinariesDirectory)/protobuf \ + $(Build.SourcesDirectory)/protobuf_install + displayName: Build Host Protoc + - script: | python3 tools/python/run_android_emulator.py \ --android-sdk-root ${ANDROID_SDK_ROOT} \ @@ -45,6 +53,7 @@ jobs: --skip_submodule_sync \ --parallel \ --cmake_generator=Ninja \ + --path_to_protoc_exe $(Build.SourcesDirectory)/protobuf_install/bin/protoc \ --build_java displayName: CPU EP, Build and Test on Android Emulator diff --git a/tools/ci_build/github/azure-pipelines/mac-ios-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/mac-ios-ci-pipeline.yml index cf719dfa4a..ba95ada12e 100644 --- a/tools/ci_build/github/azure-pipelines/mac-ios-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/mac-ios-ci-pipeline.yml @@ -6,9 +6,17 @@ jobs: MACOSX_DEPLOYMENT_TARGET: '10.14' timeoutInMinutes: 150 steps: + - script: | + /bin/bash $(Build.SourcesDirectory)/tools/ci_build/github/apple/build_host_protoc.sh \ + $(Build.SourcesDirectory) \ + $(Build.BinariesDirectory)/protobuf \ + $(Build.BinariesDirectory)/protobuf_install + displayName: Build Host Protoc + - script: | python3 $(Build.SourcesDirectory)/tools/ci_build/build.py \ - --skip_submodule_sync --build_dir build/iOS_cpu \ + --skip_submodule_sync \ + --build_dir $(Build.BinariesDirectory)/iOS_cpu \ --ios \ --ios_sysroot iphonesimulator \ --osx_arch x86_64 \ @@ -16,11 +24,14 @@ jobs: --use_xcode \ --config RelWithDebInfo \ --build_apple_framework \ + --path_to_protoc_exe $(Build.BinariesDirectory)/protobuf_install/bin/protoc \ --parallel displayName: (CPU EP) Build onnxruntime for iOS x86_64 and run tests using simulator + - script: | python3 $(Build.SourcesDirectory)/tools/ci_build/build.py \ - --skip_submodule_sync --build_dir build/iOS_coreml \ + --skip_submodule_sync \ + --build_dir $(Build.BinariesDirectory)/iOS_coreml \ --use_coreml \ --ios \ --ios_sysroot iphonesimulator \ @@ -29,5 +40,6 @@ jobs: --use_xcode \ --config RelWithDebInfo \ --build_apple_framework \ + --path_to_protoc_exe $(Build.BinariesDirectory)/protobuf_install/bin/protoc \ --parallel displayName: (CoreML EP) Build onnxruntime for iOS x86_64 and run tests using simulator diff --git a/tools/ci_build/github/azure-pipelines/mac-ios-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/mac-ios-packaging-pipeline.yml index 86d5bfa290..7d4c47afbd 100644 --- a/tools/ci_build/github/azure-pipelines/mac-ios-packaging-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/mac-ios-packaging-pipeline.yml @@ -55,10 +55,18 @@ jobs: set_var "ORT_SHOULD_UPLOAD_ARCHIVES" "${SHOULD_UPLOAD_ARCHIVES}" displayName: "Set variables" + - script: | + /bin/bash $(Build.SourcesDirectory)/tools/ci_build/github/apple/build_host_protoc.sh \ + $(Build.SourcesDirectory) \ + $(Build.BinariesDirectory)/protobuf \ + $(Build.BinariesDirectory)/protobuf_install + displayName: Build Host Protoc + - script: | python tools/ci_build/github/apple/build_ios_framework.py \ --build_dir "$(Build.BinariesDirectory)/ios_framework" \ --include_ops_by_config tools/ci_build/github/android/mobile_package.required_operators.config \ + --path_to_protoc_exe $(Build.BinariesDirectory)/protobuf_install/bin/protoc \ tools/ci_build/github/apple/default_mobile_ios_framework_build_settings.json displayName: "Build iOS framework" diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml index 0784003699..fa1f1e4e0f 100644 --- a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml +++ b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml @@ -192,10 +192,19 @@ jobs: timeoutInMinutes: 180 steps: - template: set-version-number-variables-step.yml + + - script: | + /bin/bash $(Build.SourcesDirectory)/tools/ci_build/github/apple/build_host_protoc.sh \ + $(Build.SourcesDirectory) \ + $(Build.BinariesDirectory)/protobuf \ + $(Build.BinariesDirectory)/protobuf_install + displayName: Build Host Protoc + - script: | set -e -x python3 tools/ci_build/github/apple/build_ios_framework.py \ --build_dir "$(Build.BinariesDirectory)/ios_framework" \ + --path_to_protoc_exe $(Build.BinariesDirectory)/protobuf_install/bin/protoc \ tools/ci_build/github/apple/default_full_ios_framework_build_settings.json mkdir $(Build.BinariesDirectory)/artifacts mkdir -p $(Build.BinariesDirectory)/artifacts_staging/onnxruntime-ios-xcframework-$(OnnxRuntimeVersion) diff --git a/tools/ci_build/github/linux/ort_minimal/nnapi_minimal_build_minimal_ort_and_run_tests.sh b/tools/ci_build/github/linux/ort_minimal/nnapi_minimal_build_minimal_ort_and_run_tests.sh index 80a7258ae1..1bf0ec1c05 100755 --- a/tools/ci_build/github/linux/ort_minimal/nnapi_minimal_build_minimal_ort_and_run_tests.sh +++ b/tools/ci_build/github/linux/ort_minimal/nnapi_minimal_build_minimal_ort_and_run_tests.sh @@ -16,7 +16,7 @@ rm -rf $ORT_ROOT/build_nnapi python3 -m pip install --user flatbuffers # Build minimal package for Android x86_64 Emulator. -# The unit tests in onnxruntime_test_all will be run on the Android simulator +# The unit tests in onnxruntime_test_all will be run on the Android simulator python3 $ORT_ROOT/tools/ci_build/build.py \ --build_dir $MIN_BUILD_DIR \ --config Debug \ @@ -34,6 +34,7 @@ python3 $ORT_ROOT/tools/ci_build/build.py \ --disable_ml_ops \ --disable_exceptions \ --include_ops_by_config $ORT_ROOT/onnxruntime/test/testdata/required_ops_and_types.config \ + --path_to_protoc_exe $ORT_ROOT/protobuf_install/bin/protoc \ --skip_tests # Push onnxruntime_test_all and testdata to emulator