From ffde44cd09e1fdc1ff84c41bcca96ed4469a2330 Mon Sep 17 00:00:00 2001 From: Edward Chen <18449977+edgchen1@users.noreply.github.com> Date: Mon, 28 Feb 2022 15:39:07 -0800 Subject: [PATCH] [iOS Packaging] Add full ORT build iOS package. (#10626) Add C/C++ and Objective-C packages with full ORT builds. --- .../{Podfile => Podfile.template} | 2 +- tools/ci_build/github/apple/__init__.py | 0 .../apple/assemble_ios_packaging_artifacts.sh | 5 +- .../apple/build_and_assemble_ios_pods.py | 85 +++++++++----- tools/ci_build/github/apple/c/__init__.py | 0 .../github/apple/c/assemble_c_pod_package.py | 111 +++++++++++++----- ...-c.podspec.template => c.podspec.template} | 16 ++- .../github/apple/c/onnxruntime-c.config.json | 5 + .../apple/c/onnxruntime-mobile-c.config.json | 5 + .../apple/c/onnxruntime-test-c.config.json | 5 + .../github/apple/objectivec/__init__.py | 0 .../objectivec/assemble_objc_pod_package.py | 99 +++++++++++----- ...podspec.template => objc.podspec.template} | 22 ++-- .../onnxruntime-mobile-objc.config.json | 5 + .../objectivec/onnxruntime-objc.config.json | 5 + .../github/apple/package_assembly_utils.py | 52 ++++++-- .../github/apple/test_ios_packages.py | 93 ++++++++------- .../mac-ios-packaging-pipeline.yml | 70 +++++++++-- .../azure-pipelines/templates/c-api-cpu.yml | 3 +- 19 files changed, 409 insertions(+), 174 deletions(-) rename onnxruntime/test/platform/ios/ios_package_test/{Podfile => Podfile.template} (86%) create mode 100644 tools/ci_build/github/apple/__init__.py create mode 100644 tools/ci_build/github/apple/c/__init__.py rename tools/ci_build/github/apple/c/{onnxruntime-mobile-c.podspec.template => c.podspec.template} (56%) create mode 100644 tools/ci_build/github/apple/c/onnxruntime-c.config.json create mode 100644 tools/ci_build/github/apple/c/onnxruntime-mobile-c.config.json create mode 100644 tools/ci_build/github/apple/c/onnxruntime-test-c.config.json create mode 100644 tools/ci_build/github/apple/objectivec/__init__.py rename tools/ci_build/github/apple/objectivec/{onnxruntime-mobile-objc.podspec.template => objc.podspec.template} (65%) create mode 100644 tools/ci_build/github/apple/objectivec/onnxruntime-mobile-objc.config.json create mode 100644 tools/ci_build/github/apple/objectivec/onnxruntime-objc.config.json diff --git a/onnxruntime/test/platform/ios/ios_package_test/Podfile b/onnxruntime/test/platform/ios/ios_package_test/Podfile.template similarity index 86% rename from onnxruntime/test/platform/ios/ios_package_test/Podfile rename to onnxruntime/test/platform/ios/ios_package_test/Podfile.template index 885e5eb18a..d2155660d7 100644 --- a/onnxruntime/test/platform/ios/ios_package_test/Podfile +++ b/onnxruntime/test/platform/ios/ios_package_test/Podfile.template @@ -6,7 +6,7 @@ target 'ios_package_test' do target 'ios_package_testUITests' do inherit! :search_paths - pod 'onnxruntime-mobile-c', :podspec => './onnxruntime-mobile-c.podspec' + pod '@C_POD_NAME@', :podspec => '@C_POD_PODSPEC@' end end diff --git a/tools/ci_build/github/apple/__init__.py b/tools/ci_build/github/apple/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tools/ci_build/github/apple/assemble_ios_packaging_artifacts.sh b/tools/ci_build/github/apple/assemble_ios_packaging_artifacts.sh index 3e9e114e79..9d89b44930 100755 --- a/tools/ci_build/github/apple/assemble_ios_packaging_artifacts.sh +++ b/tools/ci_build/github/apple/assemble_ios_packaging_artifacts.sh @@ -21,7 +21,7 @@ ORT_POD_VERSION=${3:?${USAGE_TEXT}} SHOULD_UPLOAD_ARCHIVES=${4:?${USAGE_TEXT}} STORAGE_ACCOUNT_NAME="onnxruntimepackages" -STORAGE_ACCOUNT_CONTAINER_NAME="ortmobilestore" +STORAGE_ACCOUNT_CONTAINER_NAME="ortmobilestore" # TODO look into moving to '$web' STORAGE_URL_PREFIX="https://${STORAGE_ACCOUNT_NAME}.blob.core.windows.net/${STORAGE_ACCOUNT_CONTAINER_NAME}" assemble_and_upload_pod() { @@ -50,8 +50,9 @@ assemble_and_upload_pod() { } assemble_and_upload_pod "onnxruntime-mobile-c" - assemble_and_upload_pod "onnxruntime-mobile-objc" +assemble_and_upload_pod "onnxruntime-c" +assemble_and_upload_pod "onnxruntime-objc" cd ${BINARIES_STAGING_DIR}/objc_api_docs zip -r ${ARTIFACTS_STAGING_DIR}/objc_api_docs.zip * diff --git a/tools/ci_build/github/apple/build_and_assemble_ios_pods.py b/tools/ci_build/github/apple/build_and_assemble_ios_pods.py index 5485748abc..7a3a0ee597 100755 --- a/tools/ci_build/github/apple/build_and_assemble_ios_pods.py +++ b/tools/ci_build/github/apple/build_and_assemble_ios_pods.py @@ -6,27 +6,31 @@ import argparse import logging import pathlib +import shutil import sys +import tempfile + + +from c.assemble_c_pod_package import assemble_c_pod_package +from objectivec.assemble_objc_pod_package import assemble_objc_pod_package +from package_assembly_utils import get_ort_version, PackageVariant SCRIPT_PATH = pathlib.Path(__file__).resolve() SCRIPT_DIR = SCRIPT_PATH.parent REPO_DIR = SCRIPT_PATH.parents[4] + logging.basicConfig( format="%(asctime)s %(name)s [%(levelname)s] - %(message)s", level=logging.DEBUG) log = logging.getLogger(SCRIPT_PATH.stem) -def ort_version(): - with open(REPO_DIR / "VERSION_NUMBER", mode="r") as version_file: - return version_file.read().strip() - - def parse_args(): parser = argparse.ArgumentParser( - description="Builds an iOS framework and uses it to assemble iOS pod package files.") + description="Builds an iOS framework and uses it to assemble iOS pod package files.", + formatter_class=argparse.ArgumentDefaultsHelpFormatter) parser.add_argument("--build-dir", type=pathlib.Path, default=REPO_DIR / "build" / "ios_framework", help="The build directory. This will contain the iOS framework build output.") @@ -34,9 +38,13 @@ def parse_args(): help="The staging directory. This will contain the iOS pod package files. " "The pod package files do not have dependencies on files in the build directory.") - parser.add_argument("--pod-version", default=f"{ort_version()}-local", + parser.add_argument("--pod-version", default=f"{get_ort_version()}-local", help="The version string of the pod. The same version is used for all pods.") + parser.add_argument("--variant", choices=PackageVariant.release_variant_names(), + default=PackageVariant.Mobile.name, + help="Pod package variant.") + parser.add_argument("--test", action="store_true", help="Run tests on the framework and pod package files.") @@ -74,6 +82,10 @@ def main(): build_dir = args.build_dir.resolve() staging_dir = args.staging_dir.resolve() + # build framework + package_variant = PackageVariant[args.variant] + framework_info_file = build_dir / "framework_info.json" + log.info("Building iOS framework.") build_ios_framework_args = \ @@ -90,41 +102,54 @@ def main(): if args.test: test_ios_packages_args = [sys.executable, str(SCRIPT_DIR / "test_ios_packages.py"), "--fail_if_cocoapods_missing", - "--framework_info_file", str(build_dir / "framework_info.json"), - "--c_framework_dir", str(build_dir / "framework_out")] + "--framework_info_file", str(framework_info_file), + "--c_framework_dir", str(build_dir / "framework_out"), + "--variant", package_variant.name] run(test_ios_packages_args) - log.info("Assembling onnxruntime-mobile-c pod.") + # assemble pods and then move them to their target locations (staging_dir/) + staging_dir.mkdir(parents=True, exist_ok=True) + with tempfile.TemporaryDirectory(dir=staging_dir) as pod_assembly_dir_name: + pod_assembly_dir = pathlib.Path(pod_assembly_dir_name) - assemble_c_pod_args = [sys.executable, str(SCRIPT_DIR / "c" / "assemble_c_pod_package.py"), - "--staging-dir", str(staging_dir / "onnxruntime-mobile-c"), - "--pod-version", args.pod_version, - "--framework-info-file", str(build_dir / "framework_info.json"), - "--framework-dir", str(build_dir / "framework_out" / "onnxruntime.xcframework"), - "--public-headers-dir", str(build_dir / "framework_out" / "Headers")] + log.info("Assembling C/C++ pod.") - run(assemble_c_pod_args) + c_pod_staging_dir = pod_assembly_dir / "c_pod" + c_pod_name, c_pod_podspec = assemble_c_pod_package( + staging_dir=c_pod_staging_dir, + pod_version=args.pod_version, + framework_info_file=framework_info_file, + framework_dir=build_dir / "framework_out" / "onnxruntime.xcframework", + public_headers_dir=build_dir / "framework_out" / "Headers", + package_variant=package_variant) - if args.test: - test_c_pod_args = ["pod", "lib", "lint", "--verbose"] + if args.test: + test_c_pod_args = ["pod", "lib", "lint", "--verbose"] - run(test_c_pod_args, cwd=staging_dir / "onnxruntime-mobile-c") + run(test_c_pod_args, cwd=c_pod_staging_dir) - log.info("Assembling onnxruntime-mobile-objc pod.") + log.info("Assembling Objective-C pod.") - assemble_objc_pod_args = [sys.executable, str(SCRIPT_DIR / "objectivec" / "assemble_objc_pod_package.py"), - "--staging-dir", str(staging_dir / "onnxruntime-mobile-objc"), - "--pod-version", args.pod_version, - "--framework-info-file", str(build_dir / "framework_info.json")] + objc_pod_staging_dir = pod_assembly_dir / "objc_pod" + objc_pod_name, objc_pod_podspec = assemble_objc_pod_package( + staging_dir=objc_pod_staging_dir, + pod_version=args.pod_version, + framework_info_file=framework_info_file, + package_variant=package_variant) - run(assemble_objc_pod_args) + if args.test: + test_objc_pod_args = ["pod", "lib", "lint", "--verbose", f"--include-podspecs={c_pod_podspec}"] - if args.test: - c_podspec_file = staging_dir / "onnxruntime-mobile-c" / "onnxruntime-mobile-c.podspec" - test_objc_pod_args = ["pod", "lib", "lint", "--verbose", f"--include-podspecs={c_podspec_file}"] + run(test_objc_pod_args, cwd=objc_pod_staging_dir) - run(test_objc_pod_args, cwd=staging_dir / "onnxruntime-mobile-objc") + def move_dir(src, dst): + if dst.is_dir(): + shutil.rmtree(dst) + shutil.move(src, dst) + + move_dir(c_pod_staging_dir, staging_dir / c_pod_name) + move_dir(objc_pod_staging_dir, staging_dir / objc_pod_name) log.info(f"Successfully assembled iOS pods at '{staging_dir}'.") diff --git a/tools/ci_build/github/apple/c/__init__.py b/tools/ci_build/github/apple/c/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tools/ci_build/github/apple/c/assemble_c_pod_package.py b/tools/ci_build/github/apple/c/assemble_c_pod_package.py index 261bc7eb6b..b1187c139f 100644 --- a/tools/ci_build/github/apple/c/assemble_c_pod_package.py +++ b/tools/ci_build/github/apple/c/assemble_c_pod_package.py @@ -14,7 +14,77 @@ sys.path.append(str(_script_dir.parent)) from package_assembly_utils import ( # noqa: E402 - copy_repo_relative_to_dir, gen_file_from_template, load_framework_info) + copy_repo_relative_to_dir, gen_file_from_template, load_json_config, + PackageVariant) + + +def get_pod_config_file(package_variant: PackageVariant): + ''' + Gets the pod configuration file path for the given package variant. + ''' + if package_variant == PackageVariant.Full: + return _script_dir / "onnxruntime-c.config.json" + elif package_variant == PackageVariant.Mobile: + return _script_dir / "onnxruntime-mobile-c.config.json" + elif package_variant == PackageVariant.Test: + return _script_dir / "onnxruntime-test-c.config.json" + else: + raise ValueError(f"Unhandled package variant: {package_variant}") + + +def assemble_c_pod_package(staging_dir: pathlib.Path, pod_version: str, + framework_info_file: pathlib.Path, + public_headers_dir: pathlib.Path, framework_dir: pathlib.Path, + package_variant: PackageVariant): + ''' + Assembles the files for the C/C++ pod package in a staging directory. + + :param staging_dir Path to the staging directory for the C/C++ pod files. + :param pod_version C/C++ pod version. + :param framework_info_file Path to the framework_info.json file containing additional values for the podspec. + :param public_headers_dir Path to the public headers directory to include in the pod. + :param framework_dir Path to the onnxruntime framework directory to include in the pod. + :param package_variant The pod package variant. + :return Tuple of (package name, path to the podspec file). + ''' + staging_dir = staging_dir.resolve() + framework_info_file = framework_info_file.resolve(strict=True) + public_headers_dir = public_headers_dir.resolve(strict=True) + framework_dir = framework_dir.resolve(strict=True) + + framework_info = load_json_config(framework_info_file) + pod_config = load_json_config(get_pod_config_file(package_variant)) + + pod_name = pod_config["name"] + + print(f"Assembling files in staging directory: {staging_dir}") + if staging_dir.exists(): + print("Warning: staging directory already exists", file=sys.stderr) + + # copy the necessary files to the staging directory + shutil.copytree(framework_dir, staging_dir / framework_dir.name, dirs_exist_ok=True) + shutil.copytree(public_headers_dir, staging_dir / public_headers_dir.name, dirs_exist_ok=True) + copy_repo_relative_to_dir(["LICENSE"], staging_dir) + + # generate the podspec file from the template + variable_substitutions = { + "DESCRIPTION": pod_config["description"], + "IOS_DEPLOYMENT_TARGET": framework_info["IOS_DEPLOYMENT_TARGET"], + "LICENSE_FILE": "LICENSE", + "NAME": pod_name, + "ORT_C_FRAMEWORK": framework_dir.name, + "ORT_C_HEADERS_DIR": public_headers_dir.name, + "SUMMARY": pod_config["summary"], + "VERSION": pod_version, + "WEAK_FRAMEWORK": framework_info["WEAK_FRAMEWORK"], + } + + podspec_template = _script_dir / "c.podspec.template" + podspec = staging_dir / f"{pod_name}.podspec" + + gen_file_from_template(podspec_template, podspec, variable_substitutions) + + return pod_name, podspec def parse_args(): @@ -24,7 +94,7 @@ def parse_args(): """) parser.add_argument("--staging-dir", type=pathlib.Path, - default=pathlib.Path("./onnxruntime-mobile-c-staging"), + default=pathlib.Path("./c-staging"), help="Path to the staging directory for the C/C++ pod files.") parser.add_argument("--pod-version", required=True, help="C/C++ pod version.") @@ -34,7 +104,9 @@ def parse_args(): parser.add_argument("--public-headers-dir", type=pathlib.Path, required=True, help="Path to the public headers directory to include in the pod.") parser.add_argument("--framework-dir", type=pathlib.Path, required=True, - help="Path to the onnxruntime.framework directory to include in the pod.") + help="Path to the onnxruntime framework directory to include in the pod.") + parser.add_argument("--variant", choices=PackageVariant.all_variant_names(), required=True, + help="Pod package variant.") return parser.parse_args() @@ -42,33 +114,12 @@ def parse_args(): def main(): args = parse_args() - framework_info = load_framework_info(args.framework_info_file.resolve()) - - staging_dir = args.staging_dir.resolve() - print(f"Assembling files in staging directory: {staging_dir}") - if staging_dir.exists(): - print("Warning: staging directory already exists", file=sys.stderr) - - # copy the necessary files to the staging directory - framework_dir = args.framework_dir.resolve() - shutil.copytree(framework_dir, staging_dir / framework_dir.name, dirs_exist_ok=True) - public_headers_dir = args.public_headers_dir.resolve() - shutil.copytree(public_headers_dir, staging_dir / public_headers_dir.name, dirs_exist_ok=True) - copy_repo_relative_to_dir(["LICENSE"], staging_dir) - - # generate the podspec file from the template - - variable_substitutions = { - "VERSION": args.pod_version, - "IOS_DEPLOYMENT_TARGET": framework_info["IOS_DEPLOYMENT_TARGET"], - "WEAK_FRAMEWORK": framework_info["WEAK_FRAMEWORK"], - "LICENSE_FILE": '"LICENSE"', - } - - podspec_template = _script_dir / "onnxruntime-mobile-c.podspec.template" - podspec = staging_dir / "onnxruntime-mobile-c.podspec" - - gen_file_from_template(podspec_template, podspec, variable_substitutions) + assemble_c_pod_package(staging_dir=args.staging_dir, + pod_version=args.pod_version, + framework_info_file=args.framework_info_file, + public_headers_dir=args.public_headers_dir, + framework_dir=args.framework_dir, + package_variant=PackageVariant[args.variant]) return 0 diff --git a/tools/ci_build/github/apple/c/onnxruntime-mobile-c.podspec.template b/tools/ci_build/github/apple/c/c.podspec.template similarity index 56% rename from tools/ci_build/github/apple/c/onnxruntime-mobile-c.podspec.template rename to tools/ci_build/github/apple/c/c.podspec.template index eb6264793c..e0cbfe2360 100644 --- a/tools/ci_build/github/apple/c/onnxruntime-mobile-c.podspec.template +++ b/tools/ci_build/github/apple/c/c.podspec.template @@ -1,20 +1,18 @@ Pod::Spec.new do |spec| - spec.name = "onnxruntime-mobile-c" + spec.name = "@NAME@" spec.version = "@VERSION@" spec.authors = { "ONNX Runtime" => "onnxruntime@microsoft.com" } - spec.license = { :type => "MIT", :file => @LICENSE_FILE@ } + spec.license = { :type => "MIT", :file => "@LICENSE_FILE@" } spec.homepage = "https://github.com/microsoft/onnxruntime" spec.source = { :http => "file:///http_source_placeholder" } - spec.summary = "ONNX Runtime Mobile C/C++ Pod" + spec.summary = "@SUMMARY@" spec.platform = :ios, "@IOS_DEPLOYMENT_TARGET@" - spec.vendored_frameworks = "onnxruntime.xcframework" + spec.vendored_frameworks = "@ORT_C_FRAMEWORK@" spec.static_framework = true spec.weak_framework = [ @WEAK_FRAMEWORK@ ] - spec.source_files = "Headers/*.h" - spec.preserve_paths = [ @LICENSE_FILE@ ] - spec.description = <<-DESC - A pod for the ONNX Runtime Mobile C/C++ library. - DESC + spec.source_files = "@ORT_C_HEADERS_DIR@/*.h" + spec.preserve_paths = [ "@LICENSE_FILE@" ] + spec.description = "@DESCRIPTION@" spec.library = "c++" spec.pod_target_xcconfig = { "OTHER_CPLUSPLUSFLAGS" => "-fvisibility=hidden -fvisibility-inlines-hidden", diff --git a/tools/ci_build/github/apple/c/onnxruntime-c.config.json b/tools/ci_build/github/apple/c/onnxruntime-c.config.json new file mode 100644 index 0000000000..d3eda57c17 --- /dev/null +++ b/tools/ci_build/github/apple/c/onnxruntime-c.config.json @@ -0,0 +1,5 @@ +{ + "name": "onnxruntime-c", + "summary": "ONNX Runtime C/C++ Pod", + "description": "A pod for the ONNX Runtime C/C++ library." +} diff --git a/tools/ci_build/github/apple/c/onnxruntime-mobile-c.config.json b/tools/ci_build/github/apple/c/onnxruntime-mobile-c.config.json new file mode 100644 index 0000000000..571d27db30 --- /dev/null +++ b/tools/ci_build/github/apple/c/onnxruntime-mobile-c.config.json @@ -0,0 +1,5 @@ +{ + "name": "onnxruntime-mobile-c", + "summary": "ONNX Runtime Mobile C/C++ Pod", + "description": "A pod for the ONNX Runtime Mobile C/C++ library. This library supports a reduced set of opsets, ops, and types and only supports ORT format models in order to reduce binary size." +} diff --git a/tools/ci_build/github/apple/c/onnxruntime-test-c.config.json b/tools/ci_build/github/apple/c/onnxruntime-test-c.config.json new file mode 100644 index 0000000000..d55dbc63e0 --- /dev/null +++ b/tools/ci_build/github/apple/c/onnxruntime-test-c.config.json @@ -0,0 +1,5 @@ +{ + "name": "onnxruntime-test-c", + "summary": "TEST POD", + "description": "Pod for testing. Not for actual release." +} diff --git a/tools/ci_build/github/apple/objectivec/__init__.py b/tools/ci_build/github/apple/objectivec/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/tools/ci_build/github/apple/objectivec/assemble_objc_pod_package.py b/tools/ci_build/github/apple/objectivec/assemble_objc_pod_package.py index 4b4ffcc638..5888c40925 100755 --- a/tools/ci_build/github/apple/objectivec/assemble_objc_pod_package.py +++ b/tools/ci_build/github/apple/objectivec/assemble_objc_pod_package.py @@ -12,8 +12,9 @@ _script_dir = pathlib.Path(__file__).parent.resolve(strict=True) sys.path.append(str(_script_dir.parent)) +from c.assemble_c_pod_package import get_pod_config_file as get_c_pod_config_file # noqa: E402 from package_assembly_utils import ( # noqa: E402 - copy_repo_relative_to_dir, gen_file_from_template, load_framework_info) + copy_repo_relative_to_dir, gen_file_from_template, load_json_config, PackageVariant) # these variables contain paths or path patterns that are relative to the repo root @@ -55,30 +56,39 @@ test_resource_files = [ ] -def parse_args(): - parser = argparse.ArgumentParser(description=""" - Assembles the files for the Objective-C pod package in a staging directory. - This directory can be validated (e.g., with `pod lib lint`) and then zipped to create a package for release. - """) - - parser.add_argument("--staging-dir", type=pathlib.Path, - default=pathlib.Path("./onnxruntime-mobile-objc-staging"), - help="Path to the staging directory for the Objective-C pod files.") - parser.add_argument("--pod-version", required=True, - help="Objective-C pod version.") - parser.add_argument("--framework-info-file", type=pathlib.Path, required=True, - help="Path to the framework_info.json file containing additional values for the podspec. " - "This file should be generated by CMake in the build directory.") - - return parser.parse_args() +def get_pod_config_file(package_variant: PackageVariant): + ''' + Gets the pod configuration file path for the given package variant. + ''' + if package_variant == PackageVariant.Full: + return _script_dir / "onnxruntime-objc.config.json" + elif package_variant == PackageVariant.Mobile: + return _script_dir / "onnxruntime-mobile-objc.config.json" + else: + raise ValueError(f"Unhandled package variant: {package_variant}") -def main(): - args = parse_args() +def assemble_objc_pod_package(staging_dir: pathlib.Path, pod_version: str, + framework_info_file: pathlib.Path, + package_variant: PackageVariant): + ''' + Assembles the files for the Objective-C pod package in a staging directory. - framework_info = load_framework_info(args.framework_info_file.resolve()) + :param staging_dir Path to the staging directory for the Objective-C pod files. + :param pod_version Objective-C pod version. + :param framework_info_file Path to the framework_info.json file containing additional values for the podspec. + :param package_variant The pod package variant. + :return Tuple of (package name, path to the podspec file). + ''' + staging_dir = staging_dir.resolve() + framework_info_file = framework_info_file.resolve(strict=True) + + framework_info = load_json_config(framework_info_file) + pod_config = load_json_config(get_pod_config_file(package_variant)) + c_pod_config = load_json_config(get_c_pod_config_file(package_variant)) + + pod_name = pod_config["name"] - staging_dir = args.staging_dir.resolve() print(f"Assembling files in staging directory: {staging_dir}") if staging_dir.exists(): print("Warning: staging directory already exists", file=sys.stderr) @@ -94,21 +104,56 @@ def main(): return ", ".join([f'"{pattern}"' for pattern in patterns]) variable_substitutions = { - "VERSION": args.pod_version, - "IOS_DEPLOYMENT_TARGET": framework_info["IOS_DEPLOYMENT_TARGET"], - "LICENSE_FILE": path_patterns_as_variable_value([license_file]), + "C_POD_NAME": c_pod_config["name"], + "DESCRIPTION": pod_config["description"], "INCLUDE_DIR_LIST": path_patterns_as_variable_value(include_dirs), + "IOS_DEPLOYMENT_TARGET": framework_info["IOS_DEPLOYMENT_TARGET"], + "LICENSE_FILE": license_file, + "NAME": pod_name, "PUBLIC_HEADER_FILE_LIST": path_patterns_as_variable_value(public_header_files), "SOURCE_FILE_LIST": path_patterns_as_variable_value(source_files), - "TEST_SOURCE_FILE_LIST": path_patterns_as_variable_value(test_source_files), + "SUMMARY": pod_config["summary"], "TEST_RESOURCE_FILE_LIST": path_patterns_as_variable_value(test_resource_files), + "TEST_SOURCE_FILE_LIST": path_patterns_as_variable_value(test_source_files), + "VERSION": pod_version, } - podspec_template = _script_dir / "onnxruntime-mobile-objc.podspec.template" - podspec = staging_dir / "onnxruntime-mobile-objc.podspec" + podspec_template = _script_dir / "objc.podspec.template" + podspec = staging_dir / f"{pod_name}.podspec" gen_file_from_template(podspec_template, podspec, variable_substitutions) + return pod_name, podspec + + +def parse_args(): + parser = argparse.ArgumentParser(description=""" + Assembles the files for the Objective-C pod package in a staging directory. + This directory can be validated (e.g., with `pod lib lint`) and then zipped to create a package for release. + """) + + parser.add_argument("--staging-dir", type=pathlib.Path, + default=pathlib.Path("./onnxruntime-mobile-objc-staging"), + help="Path to the staging directory for the Objective-C pod files.") + parser.add_argument("--pod-version", required=True, + help="Objective-C pod version.") + parser.add_argument("--framework-info-file", type=pathlib.Path, required=True, + help="Path to the framework_info.json file containing additional values for the podspec. " + "This file should be generated by CMake in the build directory.") + parser.add_argument("--variant", choices=PackageVariant.release_variant_names(), required=True, + help="Pod package variant.") + + return parser.parse_args() + + +def main(): + args = parse_args() + + assemble_objc_pod_package(staging_dir=args.staging_dir, + pod_version=args.pod_version, + framework_info_file=args.framework_info_file, + package_variant=PackageVariant[args.variant]) + return 0 diff --git a/tools/ci_build/github/apple/objectivec/onnxruntime-mobile-objc.podspec.template b/tools/ci_build/github/apple/objectivec/objc.podspec.template similarity index 65% rename from tools/ci_build/github/apple/objectivec/onnxruntime-mobile-objc.podspec.template rename to tools/ci_build/github/apple/objectivec/objc.podspec.template index cf3ca59174..8832b939f4 100644 --- a/tools/ci_build/github/apple/objectivec/onnxruntime-mobile-objc.podspec.template +++ b/tools/ci_build/github/apple/objectivec/objc.podspec.template @@ -1,23 +1,19 @@ Pod::Spec.new do |s| - s.name = 'onnxruntime-mobile-objc' - s.version = '@VERSION@' - s.summary = 'ONNX Runtime Mobile Objective-C Pod' - - s.description = <<-DESC - A pod for the ONNX Runtime Mobile Objective-C API. - DESC - - s.homepage = 'https://github.com/microsoft/onnxruntime' - s.license = { :type => 'MIT', :file => @LICENSE_FILE@ } + s.name = "@NAME@" + s.version = "@VERSION@" + s.summary = "@SUMMARY@" + s.description = "@DESCRIPTION@" + s.homepage = "https://github.com/microsoft/onnxruntime" + s.license = { :type => "MIT", :file => "@LICENSE_FILE@" } s.author = { "ONNX Runtime" => "onnxruntime@microsoft.com" } s.source = { :http => "file:///http_source_placeholder" } - s.ios.deployment_target = '@IOS_DEPLOYMENT_TARGET@' - s.preserve_paths = [ @LICENSE_FILE@ ] + s.ios.deployment_target = "@IOS_DEPLOYMENT_TARGET@" + s.preserve_paths = [ "@LICENSE_FILE@" ] s.default_subspec = "Core" s.static_framework = true s.subspec "Core" do |core| - core.dependency "onnxruntime-mobile-c", "#{s.version}" + core.dependency "@C_POD_NAME@", "#{s.version}" core.requires_arc = true core.compiler_flags = "-std=c++17", "-fobjc-arc-exceptions", "-Wall", "-Wextra", "-Werror" diff --git a/tools/ci_build/github/apple/objectivec/onnxruntime-mobile-objc.config.json b/tools/ci_build/github/apple/objectivec/onnxruntime-mobile-objc.config.json new file mode 100644 index 0000000000..dac83710f3 --- /dev/null +++ b/tools/ci_build/github/apple/objectivec/onnxruntime-mobile-objc.config.json @@ -0,0 +1,5 @@ +{ + "name": "onnxruntime-mobile-objc", + "summary": "ONNX Runtime Mobile Objective-C Pod", + "description": "A pod for the ONNX Runtime Mobile Objective-C API. The underlying ONNX Runtime library supports a reduced set of opsets, ops, and types and only supports ORT format models in order to reduce binary size." +} diff --git a/tools/ci_build/github/apple/objectivec/onnxruntime-objc.config.json b/tools/ci_build/github/apple/objectivec/onnxruntime-objc.config.json new file mode 100644 index 0000000000..5249918c12 --- /dev/null +++ b/tools/ci_build/github/apple/objectivec/onnxruntime-objc.config.json @@ -0,0 +1,5 @@ +{ + "name": "onnxruntime-objc", + "summary": "ONNX Runtime Objective-C Pod", + "description": "A pod for the ONNX Runtime Objective-C API." +} diff --git a/tools/ci_build/github/apple/package_assembly_utils.py b/tools/ci_build/github/apple/package_assembly_utils.py index 1478ae0958..a2a31ae112 100644 --- a/tools/ci_build/github/apple/package_assembly_utils.py +++ b/tools/ci_build/github/apple/package_assembly_utils.py @@ -1,6 +1,7 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. +import enum import json import os import pathlib @@ -12,11 +13,27 @@ from typing import Dict, List _script_dir = pathlib.Path(__file__).parent.resolve(strict=True) repo_root = _script_dir.parents[3] + +class PackageVariant(enum.Enum): + Full = 0 # full ORT build with all opsets, ops, and types + Mobile = 1 # minimal ORT build with reduced ops + Test = -1 # for testing purposes only + + @classmethod + def release_variant_names(cls): + return [v.name for v in cls if v.value >= 0] + + @classmethod + def all_variant_names(cls): + return [v.name for v in cls] + + _template_variable_pattern = re.compile(r"@(\w+)@") # match "@var@" def gen_file_from_template(template_file: pathlib.Path, output_file: pathlib.Path, - variable_substitutions: Dict[str, str]): + variable_substitutions: Dict[str, str], + strict: bool = True): ''' Generates a file from a template file. The template file may contain template variables that will be substituted @@ -27,16 +44,27 @@ def gen_file_from_template(template_file: pathlib.Path, output_file: pathlib.Pat :param template_file The template file path. :param output_file The generated output file path. :param variable_substitutions The mapping from template variable name to value. + :param strict Whether to require the set of template variable names in the file and the keys of + `variable_substitutions` to be equal. ''' with open(template_file, mode="r") as template: content = template.read() + variables_in_file = set() + def replace_template_variable(match): variable_name = match.group(1) + variables_in_file.add(variable_name) return variable_substitutions.get(variable_name, match.group(0)) content = _template_variable_pattern.sub(replace_template_variable, content) + if strict and variables_in_file != variable_substitutions.keys(): + variables_in_substitutions = set(variable_substitutions.keys()) + raise ValueError(f"Template file variables and substitution variables do not match. " + f"Only in template file: {sorted(variables_in_file - variables_in_substitutions)}. " + f"Only in substitutions: {sorted(variables_in_substitutions - variables_in_file)}.") + with open(output_file, mode="w") as output: output.write(content) @@ -58,12 +86,22 @@ def copy_repo_relative_to_dir(patterns: List[str], dest_dir: pathlib.Path): shutil.copy(path, dst_path) -def load_framework_info(framework_info_file: pathlib.Path): +def load_json_config(json_config_file: pathlib.Path): ''' - Loads framework info from a file. + Loads configuration info from a JSON file. - :param framework_info_file The framework info file path. - :return The framework info values. + :param json_config_file The JSON configuration file path. + :return The configuration info values. ''' - with open(framework_info_file, mode="r") as framework_info: - return json.load(framework_info) + with open(json_config_file, mode="r") as config: + return json.load(config) + + +def get_ort_version(): + ''' + Gets the ONNX Runtime version string from the repo. + + :return The ONNX Runtime version string. + ''' + with open(repo_root / "VERSION_NUMBER", mode="r") as version_file: + return version_file.read().strip() diff --git a/tools/ci_build/github/apple/test_ios_packages.py b/tools/ci_build/github/apple/test_ios_packages.py index 86e8736f3b..988dfce77e 100644 --- a/tools/ci_build/github/apple/test_ios_packages.py +++ b/tools/ci_build/github/apple/test_ios_packages.py @@ -10,12 +10,13 @@ import shutil import subprocess import tempfile -SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__)) -REPO_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, "..", "..", "..", "..")) + +from c.assemble_c_pod_package import assemble_c_pod_package +from package_assembly_utils import gen_file_from_template, get_ort_version, PackageVariant -from package_assembly_utils import ( # noqa: E402 - gen_file_from_template, load_framework_info) +SCRIPT_PATH = pathlib.Path(__file__).resolve(strict=True) +REPO_DIR = SCRIPT_PATH.parents[4] def _test_ios_packages(args): @@ -33,8 +34,8 @@ def _test_ios_packages(args): if not c_framework_dir.is_dir(): raise FileNotFoundError('c_framework_dir {} is not a folder.'.format(c_framework_dir)) - has_framework = pathlib.Path(os.path.join(c_framework_dir, 'onnxruntime.framework')).exists() - has_xcframework = pathlib.Path(os.path.join(c_framework_dir, 'onnxruntime.xcframework')).exists() + has_framework = (c_framework_dir / 'onnxruntime.framework').exists() + has_xcframework = (c_framework_dir / 'onnxruntime.xcframework').exists() if not has_framework and not has_xcframework: raise FileNotFoundError('{} does not have onnxruntime.framework/xcframework'.format(c_framework_dir)) @@ -49,60 +50,61 @@ def _test_ios_packages(args): with contextlib.ExitStack() as context_stack: if args.test_project_stage_dir is None: - stage_dir = context_stack.enter_context(tempfile.TemporaryDirectory()) + stage_dir = pathlib.Path(context_stack.enter_context(tempfile.TemporaryDirectory())).resolve() else: # If we specify the stage dir, then use it to create test project - stage_dir = args.test_project_stage_dir + stage_dir = args.test_project_stage_dir.resolve() if os.path.exists(stage_dir): shutil.rmtree(stage_dir) os.makedirs(stage_dir) - # create a zip file contains the framework - # TODO, move this into a util function - local_pods_dir = os.path.join(stage_dir, 'local_pods') - os.makedirs(local_pods_dir, exist_ok=True) - # shutil.make_archive require target file as full path without extension - zip_base_filename = os.path.join(local_pods_dir, 'onnxruntime-mobile-c') - zip_file_path = zip_base_filename + '.zip' - shutil.make_archive(zip_base_filename, 'zip', root_dir=c_framework_dir, base_dir=framework_name) + # assemble the test project here + target_proj_path = stage_dir / 'ios_package_test' - # copy the test project to the temp_dir - test_proj_path = os.path.join(REPO_DIR, 'onnxruntime', 'test', 'platform', 'ios', 'ios_package_test') - target_proj_path = os.path.join(stage_dir, 'ios_package_test') + # copy the test project source files to target_proj_path + test_proj_path = pathlib.Path(REPO_DIR, 'onnxruntime/test/platform/ios/ios_package_test') shutil.copytree(test_proj_path, target_proj_path) - # generate the podspec file from the template - framework_info = load_framework_info(args.framework_info_file.resolve()) - - with open(os.path.join(REPO_DIR, 'VERSION_NUMBER')) as version_file: - ORT_VERSION = version_file.readline().strip() - - variable_substitutions = { - "VERSION": ORT_VERSION, - "IOS_DEPLOYMENT_TARGET": framework_info["IOS_DEPLOYMENT_TARGET"], - "WEAK_FRAMEWORK": framework_info["WEAK_FRAMEWORK"], - "LICENSE_FILE": '"LICENSE"', - } - - podspec_template = os.path.join(SCRIPT_DIR, "c", "onnxruntime-mobile-c.podspec.template") - podspec = os.path.join(target_proj_path, "onnxruntime-mobile-c.podspec") - - gen_file_from_template(podspec_template, podspec, variable_substitutions) - - # update the podspec to point to the local framework zip file - with open(podspec, 'r') as file: - file_data = file.read() - file_data = file_data.replace('file:///http_source_placeholder', 'file:' + zip_file_path) + # assemble local pod files here + local_pods_dir = stage_dir / 'local_pods' # We will only publish xcframework, however, assembly of the xcframework is a post process # and it cannot be done by CMake for now. See, https://gitlab.kitware.com/cmake/cmake/-/issues/21752 # For a single sysroot and arch built by build.py or cmake, we can only generate framework - # We still need a way to test it, replace the xcframework with framework in the podspec - if has_framework: - file_data = file_data.replace('onnxruntime.xcframework', 'onnxruntime.framework') + # We still need a way to test it. framework_dir and public_headers_dir have different values when testing a + # framework and a xcframework. + framework_dir = args.c_framework_dir / framework_name + public_headers_dir = framework_dir / "Headers" if has_framework else args.c_framework_dir / "Headers" + + pod_name, podspec = assemble_c_pod_package(staging_dir=local_pods_dir, + pod_version=get_ort_version(), + framework_info_file=args.framework_info_file, + public_headers_dir=public_headers_dir, + framework_dir=framework_dir, + package_variant=PackageVariant[args.variant]) + + # move podspec out to target_proj_path first + podspec = shutil.move(podspec, target_proj_path / podspec.name) + + # create a zip file contains the framework + zip_file_path = local_pods_dir / f'{pod_name}.zip' + # shutil.make_archive require target file as full path without extension + shutil.make_archive(zip_file_path.with_suffix(''), 'zip', root_dir=local_pods_dir) + + # update the podspec to point to the local framework zip file + with open(podspec, 'r') as file: + file_data = file.read() + + file_data = file_data.replace('file:///http_source_placeholder', f'file:///{zip_file_path}') + with open(podspec, 'w') as file: file.write(file_data) + # generate Podfile to point to pod + gen_file_from_template(target_proj_path / "Podfile.template", target_proj_path / "Podfile", + {"C_POD_NAME": pod_name, + "C_POD_PODSPEC": f"./{podspec.name}"}) + # clean the Cocoapods cache first, in case the same pod was cached in previous runs subprocess.run(['pod', 'cache', 'clean', '--all'], shell=False, check=True, cwd=target_proj_path) @@ -135,6 +137,9 @@ def parse_args(): parser.add_argument('--c_framework_dir', type=pathlib.Path, required=True, help='Provide the parent directory for C/C++ framework') + parser.add_argument("--variant", choices=PackageVariant.all_variant_names(), default=PackageVariant.Test.name, + help="Pod package variant.") + parser.add_argument('--test_project_stage_dir', type=pathlib.Path, help='The stage dir for the test project, if not specified, will use a temporary path') diff --git a/tools/ci_build/github/azure-pipelines/mac-ios-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/mac-ios-packaging-pipeline.yml index cb7023eeff..7f024e97cc 100644 --- a/tools/ci_build/github/azure-pipelines/mac-ios-packaging-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/mac-ios-packaging-pipeline.yml @@ -18,7 +18,7 @@ jobs: pool: vmImage: "macOS-11" - timeoutInMinutes: 90 + timeoutInMinutes: 210 steps: - task: InstallAppleCertificate@2 @@ -76,39 +76,42 @@ jobs: $(Build.BinariesDirectory)/protobuf_install displayName: "Build Host Protoc" + # create and test mobile pods - script: | python tools/ci_build/github/apple/build_and_assemble_ios_pods.py \ --build-dir "$(Build.BinariesDirectory)/ios_framework" \ --staging-dir "$(Build.BinariesDirectory)/staging" \ --pod-version "${ORT_POD_VERSION}" \ --test \ + --variant Mobile \ --build-settings-file tools/ci_build/github/apple/default_mobile_ios_framework_build_settings.json \ --include-ops-by-config tools/ci_build/github/android/mobile_package.required_operators.config \ -b="--path_to_protoc_exe" -b "$(Build.BinariesDirectory)/protobuf_install/bin/protoc" - displayName: "Build iOS framework and assemble pod package files" + displayName: "[Mobile] Build iOS framework and assemble pod package files" - script: | python tools/ci_build/github/apple/test_ios_packages.py \ --fail_if_cocoapods_missing \ --framework_info_file "$(Build.BinariesDirectory)/ios_framework/framework_info.json" \ --c_framework_dir "$(Build.BinariesDirectory)/ios_framework/framework_out" \ - --test_project_stage_dir "$(Build.BinariesDirectory)/app_center_test" \ + --variant Mobile \ + --test_project_stage_dir "$(Build.BinariesDirectory)/app_center_test_mobile" \ --prepare_test_project_only - displayName: "Assemble test project for App Center" + displayName: "[Mobile] Assemble test project for App Center" - task: Xcode@5 inputs: actions: 'build-for-testing' configuration: 'Debug' - xcWorkspacePath: '$(Build.BinariesDirectory)/app_center_test/ios_package_test/ios_package_test.xcworkspace' + xcWorkspacePath: '$(Build.BinariesDirectory)/app_center_test_mobile/ios_package_test/ios_package_test.xcworkspace' sdk: 'iphoneos' scheme: 'ios_package_test' signingOption: 'manual' signingIdentity: '$(APPLE_CERTIFICATE_SIGNING_IDENTITY)' provisioningProfileName: 'iOS Team Provisioning Profile' - args: '-derivedDataPath $(Build.BinariesDirectory)/app_center_test/ios_package_test/DerivedData' - workingDirectory: $(Build.BinariesDirectory)/app_center_test/ios_package_test/ - displayName: 'Build iphone arm64 tests' + args: '-derivedDataPath $(Build.BinariesDirectory)/app_center_test_mobile/ios_package_test/DerivedData' + workingDirectory: $(Build.BinariesDirectory)/app_center_test_mobile/ios_package_test/ + displayName: '[Mobile] Build iphone arm64 tests' - script: | set -e -x @@ -117,9 +120,56 @@ jobs: --devices $(app_center_test_devices) \ --test-series "master" \ --locale "en_US" \ - --build-dir $(Build.BinariesDirectory)/app_center_test/ios_package_test/DerivedData/Build/Products/Debug-iphoneos \ + --build-dir $(Build.BinariesDirectory)/app_center_test_mobile/ios_package_test/DerivedData/Build/Products/Debug-iphoneos \ --token $(app_center_api_token) - displayName: Run E2E tests on App Center + displayName: "[Mobile] Run E2E tests on App Center" + + # create and test full pods + - script: | + python tools/ci_build/github/apple/build_and_assemble_ios_pods.py \ + --build-dir "$(Build.BinariesDirectory)/ios_framework" \ + --staging-dir "$(Build.BinariesDirectory)/staging" \ + --pod-version "${ORT_POD_VERSION}" \ + --test \ + --variant Full \ + --build-settings-file tools/ci_build/github/apple/default_full_ios_framework_build_settings.json \ + -b="--path_to_protoc_exe" -b "$(Build.BinariesDirectory)/protobuf_install/bin/protoc" + displayName: "[Full] Build iOS framework and assemble pod package files" + + - script: | + python tools/ci_build/github/apple/test_ios_packages.py \ + --fail_if_cocoapods_missing \ + --framework_info_file "$(Build.BinariesDirectory)/ios_framework/framework_info.json" \ + --c_framework_dir "$(Build.BinariesDirectory)/ios_framework/framework_out" \ + --variant Full \ + --test_project_stage_dir "$(Build.BinariesDirectory)/app_center_test_full" \ + --prepare_test_project_only + displayName: "[Full] Assemble test project for App Center" + + - task: Xcode@5 + inputs: + actions: 'build-for-testing' + configuration: 'Debug' + xcWorkspacePath: '$(Build.BinariesDirectory)/app_center_test_full/ios_package_test/ios_package_test.xcworkspace' + sdk: 'iphoneos' + scheme: 'ios_package_test' + signingOption: 'manual' + signingIdentity: '$(APPLE_CERTIFICATE_SIGNING_IDENTITY)' + provisioningProfileName: 'iOS Team Provisioning Profile' + args: '-derivedDataPath $(Build.BinariesDirectory)/app_center_test_full/ios_package_test/DerivedData' + workingDirectory: $(Build.BinariesDirectory)/app_center_test_full/ios_package_test/ + displayName: '[Full] Build iphone arm64 tests' + + - script: | + set -e -x + appcenter test run xcuitest \ + --app "AI-Frameworks/ORT-Mobile-iOS" \ + --devices $(app_center_test_devices) \ + --test-series "master" \ + --locale "en_US" \ + --build-dir $(Build.BinariesDirectory)/app_center_test_full/ios_package_test/DerivedData/Build/Products/Debug-iphoneos \ + --token $(app_center_api_token) + displayName: "[Full] Run E2E tests on App Center" - bash: | set -e diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml index 50b8e93648..efa51d8966 100644 --- a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml +++ b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml @@ -228,7 +228,8 @@ jobs: python3 tools/ci_build/github/apple/test_ios_packages.py \ --fail_if_cocoapods_missing \ --framework_info_file "$(Build.BinariesDirectory)/ios_framework/framework_info.json" \ - --c_framework_dir "$(Build.BinariesDirectory)/ios_framework/framework_out" + --c_framework_dir "$(Build.BinariesDirectory)/ios_framework/framework_out" \ + --variant Full displayName: "Test iOS framework" - task: PublishBuildArtifacts@1