[iOS Packaging] Add full ORT build iOS package. (#10626)

Add C/C++ and Objective-C packages with full ORT builds.
This commit is contained in:
Edward Chen 2022-02-28 15:39:07 -08:00 committed by GitHub
parent 1f6d8248da
commit ffde44cd09
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
19 changed files with 409 additions and 174 deletions

View file

@ -6,7 +6,7 @@ target 'ios_package_test' do
target 'ios_package_testUITests' do
inherit! :search_paths
pod 'onnxruntime-mobile-c', :podspec => './onnxruntime-mobile-c.podspec'
pod '@C_POD_NAME@', :podspec => '@C_POD_PODSPEC@'
end
end

View file

View file

@ -21,7 +21,7 @@ ORT_POD_VERSION=${3:?${USAGE_TEXT}}
SHOULD_UPLOAD_ARCHIVES=${4:?${USAGE_TEXT}}
STORAGE_ACCOUNT_NAME="onnxruntimepackages"
STORAGE_ACCOUNT_CONTAINER_NAME="ortmobilestore"
STORAGE_ACCOUNT_CONTAINER_NAME="ortmobilestore" # TODO look into moving to '$web'
STORAGE_URL_PREFIX="https://${STORAGE_ACCOUNT_NAME}.blob.core.windows.net/${STORAGE_ACCOUNT_CONTAINER_NAME}"
assemble_and_upload_pod() {
@ -50,8 +50,9 @@ assemble_and_upload_pod() {
}
assemble_and_upload_pod "onnxruntime-mobile-c"
assemble_and_upload_pod "onnxruntime-mobile-objc"
assemble_and_upload_pod "onnxruntime-c"
assemble_and_upload_pod "onnxruntime-objc"
cd ${BINARIES_STAGING_DIR}/objc_api_docs
zip -r ${ARTIFACTS_STAGING_DIR}/objc_api_docs.zip *

View file

@ -6,27 +6,31 @@
import argparse
import logging
import pathlib
import shutil
import sys
import tempfile
from c.assemble_c_pod_package import assemble_c_pod_package
from objectivec.assemble_objc_pod_package import assemble_objc_pod_package
from package_assembly_utils import get_ort_version, PackageVariant
SCRIPT_PATH = pathlib.Path(__file__).resolve()
SCRIPT_DIR = SCRIPT_PATH.parent
REPO_DIR = SCRIPT_PATH.parents[4]
logging.basicConfig(
format="%(asctime)s %(name)s [%(levelname)s] - %(message)s",
level=logging.DEBUG)
log = logging.getLogger(SCRIPT_PATH.stem)
def ort_version():
with open(REPO_DIR / "VERSION_NUMBER", mode="r") as version_file:
return version_file.read().strip()
def parse_args():
parser = argparse.ArgumentParser(
description="Builds an iOS framework and uses it to assemble iOS pod package files.")
description="Builds an iOS framework and uses it to assemble iOS pod package files.",
formatter_class=argparse.ArgumentDefaultsHelpFormatter)
parser.add_argument("--build-dir", type=pathlib.Path, default=REPO_DIR / "build" / "ios_framework",
help="The build directory. This will contain the iOS framework build output.")
@ -34,9 +38,13 @@ def parse_args():
help="The staging directory. This will contain the iOS pod package files. "
"The pod package files do not have dependencies on files in the build directory.")
parser.add_argument("--pod-version", default=f"{ort_version()}-local",
parser.add_argument("--pod-version", default=f"{get_ort_version()}-local",
help="The version string of the pod. The same version is used for all pods.")
parser.add_argument("--variant", choices=PackageVariant.release_variant_names(),
default=PackageVariant.Mobile.name,
help="Pod package variant.")
parser.add_argument("--test", action="store_true",
help="Run tests on the framework and pod package files.")
@ -74,6 +82,10 @@ def main():
build_dir = args.build_dir.resolve()
staging_dir = args.staging_dir.resolve()
# build framework
package_variant = PackageVariant[args.variant]
framework_info_file = build_dir / "framework_info.json"
log.info("Building iOS framework.")
build_ios_framework_args = \
@ -90,41 +102,54 @@ def main():
if args.test:
test_ios_packages_args = [sys.executable, str(SCRIPT_DIR / "test_ios_packages.py"),
"--fail_if_cocoapods_missing",
"--framework_info_file", str(build_dir / "framework_info.json"),
"--c_framework_dir", str(build_dir / "framework_out")]
"--framework_info_file", str(framework_info_file),
"--c_framework_dir", str(build_dir / "framework_out"),
"--variant", package_variant.name]
run(test_ios_packages_args)
log.info("Assembling onnxruntime-mobile-c pod.")
# assemble pods and then move them to their target locations (staging_dir/<pod_name>)
staging_dir.mkdir(parents=True, exist_ok=True)
with tempfile.TemporaryDirectory(dir=staging_dir) as pod_assembly_dir_name:
pod_assembly_dir = pathlib.Path(pod_assembly_dir_name)
assemble_c_pod_args = [sys.executable, str(SCRIPT_DIR / "c" / "assemble_c_pod_package.py"),
"--staging-dir", str(staging_dir / "onnxruntime-mobile-c"),
"--pod-version", args.pod_version,
"--framework-info-file", str(build_dir / "framework_info.json"),
"--framework-dir", str(build_dir / "framework_out" / "onnxruntime.xcframework"),
"--public-headers-dir", str(build_dir / "framework_out" / "Headers")]
log.info("Assembling C/C++ pod.")
run(assemble_c_pod_args)
c_pod_staging_dir = pod_assembly_dir / "c_pod"
c_pod_name, c_pod_podspec = assemble_c_pod_package(
staging_dir=c_pod_staging_dir,
pod_version=args.pod_version,
framework_info_file=framework_info_file,
framework_dir=build_dir / "framework_out" / "onnxruntime.xcframework",
public_headers_dir=build_dir / "framework_out" / "Headers",
package_variant=package_variant)
if args.test:
test_c_pod_args = ["pod", "lib", "lint", "--verbose"]
if args.test:
test_c_pod_args = ["pod", "lib", "lint", "--verbose"]
run(test_c_pod_args, cwd=staging_dir / "onnxruntime-mobile-c")
run(test_c_pod_args, cwd=c_pod_staging_dir)
log.info("Assembling onnxruntime-mobile-objc pod.")
log.info("Assembling Objective-C pod.")
assemble_objc_pod_args = [sys.executable, str(SCRIPT_DIR / "objectivec" / "assemble_objc_pod_package.py"),
"--staging-dir", str(staging_dir / "onnxruntime-mobile-objc"),
"--pod-version", args.pod_version,
"--framework-info-file", str(build_dir / "framework_info.json")]
objc_pod_staging_dir = pod_assembly_dir / "objc_pod"
objc_pod_name, objc_pod_podspec = assemble_objc_pod_package(
staging_dir=objc_pod_staging_dir,
pod_version=args.pod_version,
framework_info_file=framework_info_file,
package_variant=package_variant)
run(assemble_objc_pod_args)
if args.test:
test_objc_pod_args = ["pod", "lib", "lint", "--verbose", f"--include-podspecs={c_pod_podspec}"]
if args.test:
c_podspec_file = staging_dir / "onnxruntime-mobile-c" / "onnxruntime-mobile-c.podspec"
test_objc_pod_args = ["pod", "lib", "lint", "--verbose", f"--include-podspecs={c_podspec_file}"]
run(test_objc_pod_args, cwd=objc_pod_staging_dir)
run(test_objc_pod_args, cwd=staging_dir / "onnxruntime-mobile-objc")
def move_dir(src, dst):
if dst.is_dir():
shutil.rmtree(dst)
shutil.move(src, dst)
move_dir(c_pod_staging_dir, staging_dir / c_pod_name)
move_dir(objc_pod_staging_dir, staging_dir / objc_pod_name)
log.info(f"Successfully assembled iOS pods at '{staging_dir}'.")

View file

@ -14,7 +14,77 @@ sys.path.append(str(_script_dir.parent))
from package_assembly_utils import ( # noqa: E402
copy_repo_relative_to_dir, gen_file_from_template, load_framework_info)
copy_repo_relative_to_dir, gen_file_from_template, load_json_config,
PackageVariant)
def get_pod_config_file(package_variant: PackageVariant):
'''
Gets the pod configuration file path for the given package variant.
'''
if package_variant == PackageVariant.Full:
return _script_dir / "onnxruntime-c.config.json"
elif package_variant == PackageVariant.Mobile:
return _script_dir / "onnxruntime-mobile-c.config.json"
elif package_variant == PackageVariant.Test:
return _script_dir / "onnxruntime-test-c.config.json"
else:
raise ValueError(f"Unhandled package variant: {package_variant}")
def assemble_c_pod_package(staging_dir: pathlib.Path, pod_version: str,
framework_info_file: pathlib.Path,
public_headers_dir: pathlib.Path, framework_dir: pathlib.Path,
package_variant: PackageVariant):
'''
Assembles the files for the C/C++ pod package in a staging directory.
:param staging_dir Path to the staging directory for the C/C++ pod files.
:param pod_version C/C++ pod version.
:param framework_info_file Path to the framework_info.json file containing additional values for the podspec.
:param public_headers_dir Path to the public headers directory to include in the pod.
:param framework_dir Path to the onnxruntime framework directory to include in the pod.
:param package_variant The pod package variant.
:return Tuple of (package name, path to the podspec file).
'''
staging_dir = staging_dir.resolve()
framework_info_file = framework_info_file.resolve(strict=True)
public_headers_dir = public_headers_dir.resolve(strict=True)
framework_dir = framework_dir.resolve(strict=True)
framework_info = load_json_config(framework_info_file)
pod_config = load_json_config(get_pod_config_file(package_variant))
pod_name = pod_config["name"]
print(f"Assembling files in staging directory: {staging_dir}")
if staging_dir.exists():
print("Warning: staging directory already exists", file=sys.stderr)
# copy the necessary files to the staging directory
shutil.copytree(framework_dir, staging_dir / framework_dir.name, dirs_exist_ok=True)
shutil.copytree(public_headers_dir, staging_dir / public_headers_dir.name, dirs_exist_ok=True)
copy_repo_relative_to_dir(["LICENSE"], staging_dir)
# generate the podspec file from the template
variable_substitutions = {
"DESCRIPTION": pod_config["description"],
"IOS_DEPLOYMENT_TARGET": framework_info["IOS_DEPLOYMENT_TARGET"],
"LICENSE_FILE": "LICENSE",
"NAME": pod_name,
"ORT_C_FRAMEWORK": framework_dir.name,
"ORT_C_HEADERS_DIR": public_headers_dir.name,
"SUMMARY": pod_config["summary"],
"VERSION": pod_version,
"WEAK_FRAMEWORK": framework_info["WEAK_FRAMEWORK"],
}
podspec_template = _script_dir / "c.podspec.template"
podspec = staging_dir / f"{pod_name}.podspec"
gen_file_from_template(podspec_template, podspec, variable_substitutions)
return pod_name, podspec
def parse_args():
@ -24,7 +94,7 @@ def parse_args():
""")
parser.add_argument("--staging-dir", type=pathlib.Path,
default=pathlib.Path("./onnxruntime-mobile-c-staging"),
default=pathlib.Path("./c-staging"),
help="Path to the staging directory for the C/C++ pod files.")
parser.add_argument("--pod-version", required=True,
help="C/C++ pod version.")
@ -34,7 +104,9 @@ def parse_args():
parser.add_argument("--public-headers-dir", type=pathlib.Path, required=True,
help="Path to the public headers directory to include in the pod.")
parser.add_argument("--framework-dir", type=pathlib.Path, required=True,
help="Path to the onnxruntime.framework directory to include in the pod.")
help="Path to the onnxruntime framework directory to include in the pod.")
parser.add_argument("--variant", choices=PackageVariant.all_variant_names(), required=True,
help="Pod package variant.")
return parser.parse_args()
@ -42,33 +114,12 @@ def parse_args():
def main():
args = parse_args()
framework_info = load_framework_info(args.framework_info_file.resolve())
staging_dir = args.staging_dir.resolve()
print(f"Assembling files in staging directory: {staging_dir}")
if staging_dir.exists():
print("Warning: staging directory already exists", file=sys.stderr)
# copy the necessary files to the staging directory
framework_dir = args.framework_dir.resolve()
shutil.copytree(framework_dir, staging_dir / framework_dir.name, dirs_exist_ok=True)
public_headers_dir = args.public_headers_dir.resolve()
shutil.copytree(public_headers_dir, staging_dir / public_headers_dir.name, dirs_exist_ok=True)
copy_repo_relative_to_dir(["LICENSE"], staging_dir)
# generate the podspec file from the template
variable_substitutions = {
"VERSION": args.pod_version,
"IOS_DEPLOYMENT_TARGET": framework_info["IOS_DEPLOYMENT_TARGET"],
"WEAK_FRAMEWORK": framework_info["WEAK_FRAMEWORK"],
"LICENSE_FILE": '"LICENSE"',
}
podspec_template = _script_dir / "onnxruntime-mobile-c.podspec.template"
podspec = staging_dir / "onnxruntime-mobile-c.podspec"
gen_file_from_template(podspec_template, podspec, variable_substitutions)
assemble_c_pod_package(staging_dir=args.staging_dir,
pod_version=args.pod_version,
framework_info_file=args.framework_info_file,
public_headers_dir=args.public_headers_dir,
framework_dir=args.framework_dir,
package_variant=PackageVariant[args.variant])
return 0

View file

@ -1,20 +1,18 @@
Pod::Spec.new do |spec|
spec.name = "onnxruntime-mobile-c"
spec.name = "@NAME@"
spec.version = "@VERSION@"
spec.authors = { "ONNX Runtime" => "onnxruntime@microsoft.com" }
spec.license = { :type => "MIT", :file => @LICENSE_FILE@ }
spec.license = { :type => "MIT", :file => "@LICENSE_FILE@" }
spec.homepage = "https://github.com/microsoft/onnxruntime"
spec.source = { :http => "file:///http_source_placeholder" }
spec.summary = "ONNX Runtime Mobile C/C++ Pod"
spec.summary = "@SUMMARY@"
spec.platform = :ios, "@IOS_DEPLOYMENT_TARGET@"
spec.vendored_frameworks = "onnxruntime.xcframework"
spec.vendored_frameworks = "@ORT_C_FRAMEWORK@"
spec.static_framework = true
spec.weak_framework = [ @WEAK_FRAMEWORK@ ]
spec.source_files = "Headers/*.h"
spec.preserve_paths = [ @LICENSE_FILE@ ]
spec.description = <<-DESC
A pod for the ONNX Runtime Mobile C/C++ library.
DESC
spec.source_files = "@ORT_C_HEADERS_DIR@/*.h"
spec.preserve_paths = [ "@LICENSE_FILE@" ]
spec.description = "@DESCRIPTION@"
spec.library = "c++"
spec.pod_target_xcconfig = {
"OTHER_CPLUSPLUSFLAGS" => "-fvisibility=hidden -fvisibility-inlines-hidden",

View file

@ -0,0 +1,5 @@
{
"name": "onnxruntime-c",
"summary": "ONNX Runtime C/C++ Pod",
"description": "A pod for the ONNX Runtime C/C++ library."
}

View file

@ -0,0 +1,5 @@
{
"name": "onnxruntime-mobile-c",
"summary": "ONNX Runtime Mobile C/C++ Pod",
"description": "A pod for the ONNX Runtime Mobile C/C++ library. This library supports a reduced set of opsets, ops, and types and only supports ORT format models in order to reduce binary size."
}

View file

@ -0,0 +1,5 @@
{
"name": "onnxruntime-test-c",
"summary": "TEST POD",
"description": "Pod for testing. Not for actual release."
}

View file

@ -12,8 +12,9 @@ _script_dir = pathlib.Path(__file__).parent.resolve(strict=True)
sys.path.append(str(_script_dir.parent))
from c.assemble_c_pod_package import get_pod_config_file as get_c_pod_config_file # noqa: E402
from package_assembly_utils import ( # noqa: E402
copy_repo_relative_to_dir, gen_file_from_template, load_framework_info)
copy_repo_relative_to_dir, gen_file_from_template, load_json_config, PackageVariant)
# these variables contain paths or path patterns that are relative to the repo root
@ -55,30 +56,39 @@ test_resource_files = [
]
def parse_args():
parser = argparse.ArgumentParser(description="""
Assembles the files for the Objective-C pod package in a staging directory.
This directory can be validated (e.g., with `pod lib lint`) and then zipped to create a package for release.
""")
parser.add_argument("--staging-dir", type=pathlib.Path,
default=pathlib.Path("./onnxruntime-mobile-objc-staging"),
help="Path to the staging directory for the Objective-C pod files.")
parser.add_argument("--pod-version", required=True,
help="Objective-C pod version.")
parser.add_argument("--framework-info-file", type=pathlib.Path, required=True,
help="Path to the framework_info.json file containing additional values for the podspec. "
"This file should be generated by CMake in the build directory.")
return parser.parse_args()
def get_pod_config_file(package_variant: PackageVariant):
'''
Gets the pod configuration file path for the given package variant.
'''
if package_variant == PackageVariant.Full:
return _script_dir / "onnxruntime-objc.config.json"
elif package_variant == PackageVariant.Mobile:
return _script_dir / "onnxruntime-mobile-objc.config.json"
else:
raise ValueError(f"Unhandled package variant: {package_variant}")
def main():
args = parse_args()
def assemble_objc_pod_package(staging_dir: pathlib.Path, pod_version: str,
framework_info_file: pathlib.Path,
package_variant: PackageVariant):
'''
Assembles the files for the Objective-C pod package in a staging directory.
framework_info = load_framework_info(args.framework_info_file.resolve())
:param staging_dir Path to the staging directory for the Objective-C pod files.
:param pod_version Objective-C pod version.
:param framework_info_file Path to the framework_info.json file containing additional values for the podspec.
:param package_variant The pod package variant.
:return Tuple of (package name, path to the podspec file).
'''
staging_dir = staging_dir.resolve()
framework_info_file = framework_info_file.resolve(strict=True)
framework_info = load_json_config(framework_info_file)
pod_config = load_json_config(get_pod_config_file(package_variant))
c_pod_config = load_json_config(get_c_pod_config_file(package_variant))
pod_name = pod_config["name"]
staging_dir = args.staging_dir.resolve()
print(f"Assembling files in staging directory: {staging_dir}")
if staging_dir.exists():
print("Warning: staging directory already exists", file=sys.stderr)
@ -94,21 +104,56 @@ def main():
return ", ".join([f'"{pattern}"' for pattern in patterns])
variable_substitutions = {
"VERSION": args.pod_version,
"IOS_DEPLOYMENT_TARGET": framework_info["IOS_DEPLOYMENT_TARGET"],
"LICENSE_FILE": path_patterns_as_variable_value([license_file]),
"C_POD_NAME": c_pod_config["name"],
"DESCRIPTION": pod_config["description"],
"INCLUDE_DIR_LIST": path_patterns_as_variable_value(include_dirs),
"IOS_DEPLOYMENT_TARGET": framework_info["IOS_DEPLOYMENT_TARGET"],
"LICENSE_FILE": license_file,
"NAME": pod_name,
"PUBLIC_HEADER_FILE_LIST": path_patterns_as_variable_value(public_header_files),
"SOURCE_FILE_LIST": path_patterns_as_variable_value(source_files),
"TEST_SOURCE_FILE_LIST": path_patterns_as_variable_value(test_source_files),
"SUMMARY": pod_config["summary"],
"TEST_RESOURCE_FILE_LIST": path_patterns_as_variable_value(test_resource_files),
"TEST_SOURCE_FILE_LIST": path_patterns_as_variable_value(test_source_files),
"VERSION": pod_version,
}
podspec_template = _script_dir / "onnxruntime-mobile-objc.podspec.template"
podspec = staging_dir / "onnxruntime-mobile-objc.podspec"
podspec_template = _script_dir / "objc.podspec.template"
podspec = staging_dir / f"{pod_name}.podspec"
gen_file_from_template(podspec_template, podspec, variable_substitutions)
return pod_name, podspec
def parse_args():
parser = argparse.ArgumentParser(description="""
Assembles the files for the Objective-C pod package in a staging directory.
This directory can be validated (e.g., with `pod lib lint`) and then zipped to create a package for release.
""")
parser.add_argument("--staging-dir", type=pathlib.Path,
default=pathlib.Path("./onnxruntime-mobile-objc-staging"),
help="Path to the staging directory for the Objective-C pod files.")
parser.add_argument("--pod-version", required=True,
help="Objective-C pod version.")
parser.add_argument("--framework-info-file", type=pathlib.Path, required=True,
help="Path to the framework_info.json file containing additional values for the podspec. "
"This file should be generated by CMake in the build directory.")
parser.add_argument("--variant", choices=PackageVariant.release_variant_names(), required=True,
help="Pod package variant.")
return parser.parse_args()
def main():
args = parse_args()
assemble_objc_pod_package(staging_dir=args.staging_dir,
pod_version=args.pod_version,
framework_info_file=args.framework_info_file,
package_variant=PackageVariant[args.variant])
return 0

View file

@ -1,23 +1,19 @@
Pod::Spec.new do |s|
s.name = 'onnxruntime-mobile-objc'
s.version = '@VERSION@'
s.summary = 'ONNX Runtime Mobile Objective-C Pod'
s.description = <<-DESC
A pod for the ONNX Runtime Mobile Objective-C API.
DESC
s.homepage = 'https://github.com/microsoft/onnxruntime'
s.license = { :type => 'MIT', :file => @LICENSE_FILE@ }
s.name = "@NAME@"
s.version = "@VERSION@"
s.summary = "@SUMMARY@"
s.description = "@DESCRIPTION@"
s.homepage = "https://github.com/microsoft/onnxruntime"
s.license = { :type => "MIT", :file => "@LICENSE_FILE@" }
s.author = { "ONNX Runtime" => "onnxruntime@microsoft.com" }
s.source = { :http => "file:///http_source_placeholder" }
s.ios.deployment_target = '@IOS_DEPLOYMENT_TARGET@'
s.preserve_paths = [ @LICENSE_FILE@ ]
s.ios.deployment_target = "@IOS_DEPLOYMENT_TARGET@"
s.preserve_paths = [ "@LICENSE_FILE@" ]
s.default_subspec = "Core"
s.static_framework = true
s.subspec "Core" do |core|
core.dependency "onnxruntime-mobile-c", "#{s.version}"
core.dependency "@C_POD_NAME@", "#{s.version}"
core.requires_arc = true
core.compiler_flags = "-std=c++17", "-fobjc-arc-exceptions", "-Wall", "-Wextra", "-Werror"

View file

@ -0,0 +1,5 @@
{
"name": "onnxruntime-mobile-objc",
"summary": "ONNX Runtime Mobile Objective-C Pod",
"description": "A pod for the ONNX Runtime Mobile Objective-C API. The underlying ONNX Runtime library supports a reduced set of opsets, ops, and types and only supports ORT format models in order to reduce binary size."
}

View file

@ -0,0 +1,5 @@
{
"name": "onnxruntime-objc",
"summary": "ONNX Runtime Objective-C Pod",
"description": "A pod for the ONNX Runtime Objective-C API."
}

View file

@ -1,6 +1,7 @@
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License.
import enum
import json
import os
import pathlib
@ -12,11 +13,27 @@ from typing import Dict, List
_script_dir = pathlib.Path(__file__).parent.resolve(strict=True)
repo_root = _script_dir.parents[3]
class PackageVariant(enum.Enum):
Full = 0 # full ORT build with all opsets, ops, and types
Mobile = 1 # minimal ORT build with reduced ops
Test = -1 # for testing purposes only
@classmethod
def release_variant_names(cls):
return [v.name for v in cls if v.value >= 0]
@classmethod
def all_variant_names(cls):
return [v.name for v in cls]
_template_variable_pattern = re.compile(r"@(\w+)@") # match "@var@"
def gen_file_from_template(template_file: pathlib.Path, output_file: pathlib.Path,
variable_substitutions: Dict[str, str]):
variable_substitutions: Dict[str, str],
strict: bool = True):
'''
Generates a file from a template file.
The template file may contain template variables that will be substituted
@ -27,16 +44,27 @@ def gen_file_from_template(template_file: pathlib.Path, output_file: pathlib.Pat
:param template_file The template file path.
:param output_file The generated output file path.
:param variable_substitutions The mapping from template variable name to value.
:param strict Whether to require the set of template variable names in the file and the keys of
`variable_substitutions` to be equal.
'''
with open(template_file, mode="r") as template:
content = template.read()
variables_in_file = set()
def replace_template_variable(match):
variable_name = match.group(1)
variables_in_file.add(variable_name)
return variable_substitutions.get(variable_name, match.group(0))
content = _template_variable_pattern.sub(replace_template_variable, content)
if strict and variables_in_file != variable_substitutions.keys():
variables_in_substitutions = set(variable_substitutions.keys())
raise ValueError(f"Template file variables and substitution variables do not match. "
f"Only in template file: {sorted(variables_in_file - variables_in_substitutions)}. "
f"Only in substitutions: {sorted(variables_in_substitutions - variables_in_file)}.")
with open(output_file, mode="w") as output:
output.write(content)
@ -58,12 +86,22 @@ def copy_repo_relative_to_dir(patterns: List[str], dest_dir: pathlib.Path):
shutil.copy(path, dst_path)
def load_framework_info(framework_info_file: pathlib.Path):
def load_json_config(json_config_file: pathlib.Path):
'''
Loads framework info from a file.
Loads configuration info from a JSON file.
:param framework_info_file The framework info file path.
:return The framework info values.
:param json_config_file The JSON configuration file path.
:return The configuration info values.
'''
with open(framework_info_file, mode="r") as framework_info:
return json.load(framework_info)
with open(json_config_file, mode="r") as config:
return json.load(config)
def get_ort_version():
'''
Gets the ONNX Runtime version string from the repo.
:return The ONNX Runtime version string.
'''
with open(repo_root / "VERSION_NUMBER", mode="r") as version_file:
return version_file.read().strip()

View file

@ -10,12 +10,13 @@ import shutil
import subprocess
import tempfile
SCRIPT_DIR = os.path.dirname(os.path.realpath(__file__))
REPO_DIR = os.path.normpath(os.path.join(SCRIPT_DIR, "..", "..", "..", ".."))
from c.assemble_c_pod_package import assemble_c_pod_package
from package_assembly_utils import gen_file_from_template, get_ort_version, PackageVariant
from package_assembly_utils import ( # noqa: E402
gen_file_from_template, load_framework_info)
SCRIPT_PATH = pathlib.Path(__file__).resolve(strict=True)
REPO_DIR = SCRIPT_PATH.parents[4]
def _test_ios_packages(args):
@ -33,8 +34,8 @@ def _test_ios_packages(args):
if not c_framework_dir.is_dir():
raise FileNotFoundError('c_framework_dir {} is not a folder.'.format(c_framework_dir))
has_framework = pathlib.Path(os.path.join(c_framework_dir, 'onnxruntime.framework')).exists()
has_xcframework = pathlib.Path(os.path.join(c_framework_dir, 'onnxruntime.xcframework')).exists()
has_framework = (c_framework_dir / 'onnxruntime.framework').exists()
has_xcframework = (c_framework_dir / 'onnxruntime.xcframework').exists()
if not has_framework and not has_xcframework:
raise FileNotFoundError('{} does not have onnxruntime.framework/xcframework'.format(c_framework_dir))
@ -49,60 +50,61 @@ def _test_ios_packages(args):
with contextlib.ExitStack() as context_stack:
if args.test_project_stage_dir is None:
stage_dir = context_stack.enter_context(tempfile.TemporaryDirectory())
stage_dir = pathlib.Path(context_stack.enter_context(tempfile.TemporaryDirectory())).resolve()
else:
# If we specify the stage dir, then use it to create test project
stage_dir = args.test_project_stage_dir
stage_dir = args.test_project_stage_dir.resolve()
if os.path.exists(stage_dir):
shutil.rmtree(stage_dir)
os.makedirs(stage_dir)
# create a zip file contains the framework
# TODO, move this into a util function
local_pods_dir = os.path.join(stage_dir, 'local_pods')
os.makedirs(local_pods_dir, exist_ok=True)
# shutil.make_archive require target file as full path without extension
zip_base_filename = os.path.join(local_pods_dir, 'onnxruntime-mobile-c')
zip_file_path = zip_base_filename + '.zip'
shutil.make_archive(zip_base_filename, 'zip', root_dir=c_framework_dir, base_dir=framework_name)
# assemble the test project here
target_proj_path = stage_dir / 'ios_package_test'
# copy the test project to the temp_dir
test_proj_path = os.path.join(REPO_DIR, 'onnxruntime', 'test', 'platform', 'ios', 'ios_package_test')
target_proj_path = os.path.join(stage_dir, 'ios_package_test')
# copy the test project source files to target_proj_path
test_proj_path = pathlib.Path(REPO_DIR, 'onnxruntime/test/platform/ios/ios_package_test')
shutil.copytree(test_proj_path, target_proj_path)
# generate the podspec file from the template
framework_info = load_framework_info(args.framework_info_file.resolve())
with open(os.path.join(REPO_DIR, 'VERSION_NUMBER')) as version_file:
ORT_VERSION = version_file.readline().strip()
variable_substitutions = {
"VERSION": ORT_VERSION,
"IOS_DEPLOYMENT_TARGET": framework_info["IOS_DEPLOYMENT_TARGET"],
"WEAK_FRAMEWORK": framework_info["WEAK_FRAMEWORK"],
"LICENSE_FILE": '"LICENSE"',
}
podspec_template = os.path.join(SCRIPT_DIR, "c", "onnxruntime-mobile-c.podspec.template")
podspec = os.path.join(target_proj_path, "onnxruntime-mobile-c.podspec")
gen_file_from_template(podspec_template, podspec, variable_substitutions)
# update the podspec to point to the local framework zip file
with open(podspec, 'r') as file:
file_data = file.read()
file_data = file_data.replace('file:///http_source_placeholder', 'file:' + zip_file_path)
# assemble local pod files here
local_pods_dir = stage_dir / 'local_pods'
# We will only publish xcframework, however, assembly of the xcframework is a post process
# and it cannot be done by CMake for now. See, https://gitlab.kitware.com/cmake/cmake/-/issues/21752
# For a single sysroot and arch built by build.py or cmake, we can only generate framework
# We still need a way to test it, replace the xcframework with framework in the podspec
if has_framework:
file_data = file_data.replace('onnxruntime.xcframework', 'onnxruntime.framework')
# We still need a way to test it. framework_dir and public_headers_dir have different values when testing a
# framework and a xcframework.
framework_dir = args.c_framework_dir / framework_name
public_headers_dir = framework_dir / "Headers" if has_framework else args.c_framework_dir / "Headers"
pod_name, podspec = assemble_c_pod_package(staging_dir=local_pods_dir,
pod_version=get_ort_version(),
framework_info_file=args.framework_info_file,
public_headers_dir=public_headers_dir,
framework_dir=framework_dir,
package_variant=PackageVariant[args.variant])
# move podspec out to target_proj_path first
podspec = shutil.move(podspec, target_proj_path / podspec.name)
# create a zip file contains the framework
zip_file_path = local_pods_dir / f'{pod_name}.zip'
# shutil.make_archive require target file as full path without extension
shutil.make_archive(zip_file_path.with_suffix(''), 'zip', root_dir=local_pods_dir)
# update the podspec to point to the local framework zip file
with open(podspec, 'r') as file:
file_data = file.read()
file_data = file_data.replace('file:///http_source_placeholder', f'file:///{zip_file_path}')
with open(podspec, 'w') as file:
file.write(file_data)
# generate Podfile to point to pod
gen_file_from_template(target_proj_path / "Podfile.template", target_proj_path / "Podfile",
{"C_POD_NAME": pod_name,
"C_POD_PODSPEC": f"./{podspec.name}"})
# clean the Cocoapods cache first, in case the same pod was cached in previous runs
subprocess.run(['pod', 'cache', 'clean', '--all'], shell=False, check=True, cwd=target_proj_path)
@ -135,6 +137,9 @@ def parse_args():
parser.add_argument('--c_framework_dir', type=pathlib.Path, required=True,
help='Provide the parent directory for C/C++ framework')
parser.add_argument("--variant", choices=PackageVariant.all_variant_names(), default=PackageVariant.Test.name,
help="Pod package variant.")
parser.add_argument('--test_project_stage_dir', type=pathlib.Path,
help='The stage dir for the test project, if not specified, will use a temporary path')

View file

@ -18,7 +18,7 @@ jobs:
pool:
vmImage: "macOS-11"
timeoutInMinutes: 90
timeoutInMinutes: 210
steps:
- task: InstallAppleCertificate@2
@ -76,39 +76,42 @@ jobs:
$(Build.BinariesDirectory)/protobuf_install
displayName: "Build Host Protoc"
# create and test mobile pods
- script: |
python tools/ci_build/github/apple/build_and_assemble_ios_pods.py \
--build-dir "$(Build.BinariesDirectory)/ios_framework" \
--staging-dir "$(Build.BinariesDirectory)/staging" \
--pod-version "${ORT_POD_VERSION}" \
--test \
--variant Mobile \
--build-settings-file tools/ci_build/github/apple/default_mobile_ios_framework_build_settings.json \
--include-ops-by-config tools/ci_build/github/android/mobile_package.required_operators.config \
-b="--path_to_protoc_exe" -b "$(Build.BinariesDirectory)/protobuf_install/bin/protoc"
displayName: "Build iOS framework and assemble pod package files"
displayName: "[Mobile] Build iOS framework and assemble pod package files"
- script: |
python tools/ci_build/github/apple/test_ios_packages.py \
--fail_if_cocoapods_missing \
--framework_info_file "$(Build.BinariesDirectory)/ios_framework/framework_info.json" \
--c_framework_dir "$(Build.BinariesDirectory)/ios_framework/framework_out" \
--test_project_stage_dir "$(Build.BinariesDirectory)/app_center_test" \
--variant Mobile \
--test_project_stage_dir "$(Build.BinariesDirectory)/app_center_test_mobile" \
--prepare_test_project_only
displayName: "Assemble test project for App Center"
displayName: "[Mobile] Assemble test project for App Center"
- task: Xcode@5
inputs:
actions: 'build-for-testing'
configuration: 'Debug'
xcWorkspacePath: '$(Build.BinariesDirectory)/app_center_test/ios_package_test/ios_package_test.xcworkspace'
xcWorkspacePath: '$(Build.BinariesDirectory)/app_center_test_mobile/ios_package_test/ios_package_test.xcworkspace'
sdk: 'iphoneos'
scheme: 'ios_package_test'
signingOption: 'manual'
signingIdentity: '$(APPLE_CERTIFICATE_SIGNING_IDENTITY)'
provisioningProfileName: 'iOS Team Provisioning Profile'
args: '-derivedDataPath $(Build.BinariesDirectory)/app_center_test/ios_package_test/DerivedData'
workingDirectory: $(Build.BinariesDirectory)/app_center_test/ios_package_test/
displayName: 'Build iphone arm64 tests'
args: '-derivedDataPath $(Build.BinariesDirectory)/app_center_test_mobile/ios_package_test/DerivedData'
workingDirectory: $(Build.BinariesDirectory)/app_center_test_mobile/ios_package_test/
displayName: '[Mobile] Build iphone arm64 tests'
- script: |
set -e -x
@ -117,9 +120,56 @@ jobs:
--devices $(app_center_test_devices) \
--test-series "master" \
--locale "en_US" \
--build-dir $(Build.BinariesDirectory)/app_center_test/ios_package_test/DerivedData/Build/Products/Debug-iphoneos \
--build-dir $(Build.BinariesDirectory)/app_center_test_mobile/ios_package_test/DerivedData/Build/Products/Debug-iphoneos \
--token $(app_center_api_token)
displayName: Run E2E tests on App Center
displayName: "[Mobile] Run E2E tests on App Center"
# create and test full pods
- script: |
python tools/ci_build/github/apple/build_and_assemble_ios_pods.py \
--build-dir "$(Build.BinariesDirectory)/ios_framework" \
--staging-dir "$(Build.BinariesDirectory)/staging" \
--pod-version "${ORT_POD_VERSION}" \
--test \
--variant Full \
--build-settings-file tools/ci_build/github/apple/default_full_ios_framework_build_settings.json \
-b="--path_to_protoc_exe" -b "$(Build.BinariesDirectory)/protobuf_install/bin/protoc"
displayName: "[Full] Build iOS framework and assemble pod package files"
- script: |
python tools/ci_build/github/apple/test_ios_packages.py \
--fail_if_cocoapods_missing \
--framework_info_file "$(Build.BinariesDirectory)/ios_framework/framework_info.json" \
--c_framework_dir "$(Build.BinariesDirectory)/ios_framework/framework_out" \
--variant Full \
--test_project_stage_dir "$(Build.BinariesDirectory)/app_center_test_full" \
--prepare_test_project_only
displayName: "[Full] Assemble test project for App Center"
- task: Xcode@5
inputs:
actions: 'build-for-testing'
configuration: 'Debug'
xcWorkspacePath: '$(Build.BinariesDirectory)/app_center_test_full/ios_package_test/ios_package_test.xcworkspace'
sdk: 'iphoneos'
scheme: 'ios_package_test'
signingOption: 'manual'
signingIdentity: '$(APPLE_CERTIFICATE_SIGNING_IDENTITY)'
provisioningProfileName: 'iOS Team Provisioning Profile'
args: '-derivedDataPath $(Build.BinariesDirectory)/app_center_test_full/ios_package_test/DerivedData'
workingDirectory: $(Build.BinariesDirectory)/app_center_test_full/ios_package_test/
displayName: '[Full] Build iphone arm64 tests'
- script: |
set -e -x
appcenter test run xcuitest \
--app "AI-Frameworks/ORT-Mobile-iOS" \
--devices $(app_center_test_devices) \
--test-series "master" \
--locale "en_US" \
--build-dir $(Build.BinariesDirectory)/app_center_test_full/ios_package_test/DerivedData/Build/Products/Debug-iphoneos \
--token $(app_center_api_token)
displayName: "[Full] Run E2E tests on App Center"
- bash: |
set -e

View file

@ -228,7 +228,8 @@ jobs:
python3 tools/ci_build/github/apple/test_ios_packages.py \
--fail_if_cocoapods_missing \
--framework_info_file "$(Build.BinariesDirectory)/ios_framework/framework_info.json" \
--c_framework_dir "$(Build.BinariesDirectory)/ios_framework/framework_out"
--c_framework_dir "$(Build.BinariesDirectory)/ios_framework/framework_out" \
--variant Full
displayName: "Test iOS framework"
- task: PublishBuildArtifacts@1