diff --git a/cmake/onnxruntime.cmake b/cmake/onnxruntime.cmake index 86fbf37d0b..f16035b39b 100644 --- a/cmake/onnxruntime.cmake +++ b/cmake/onnxruntime.cmake @@ -71,9 +71,8 @@ elseif(onnxruntime_BUILD_APPLE_FRAMEWORK) set(MACOSX_FRAMEWORK_IDENTIFIER "com.microsoft.onnxruntime") configure_file(${REPO_ROOT}/cmake/Info.plist.in ${CMAKE_CURRENT_BINARY_DIR}/Info.plist) configure_file( - ${REPO_ROOT}/tools/ci_build/github/apple/onnxruntime-mobile-c.podspec.template - ${CMAKE_CURRENT_BINARY_DIR}/onnxruntime-mobile-c.podspec - ) + ${REPO_ROOT}/tools/ci_build/github/apple/framework_info.json.template + ${CMAKE_CURRENT_BINARY_DIR}/framework_info.json) set_target_properties(onnxruntime PROPERTIES FRAMEWORK TRUE FRAMEWORK_VERSION A diff --git a/tools/ci_build/github/apple/assemble_ios_packaging_artifacts.sh b/tools/ci_build/github/apple/assemble_ios_packaging_artifacts.sh new file mode 100755 index 0000000000..c43a9d89d3 --- /dev/null +++ b/tools/ci_build/github/apple/assemble_ios_packaging_artifacts.sh @@ -0,0 +1,59 @@ +#!/bin/bash + +# Note: This script is intended to be called from the iOS packaging build or a similar context +# See tools/ci_build/github/azure-pipelines/mac-ios-packaging-pipeline.yml + +set -e +set -x + +USAGE_TEXT="Usage: ${0} " + +abspath() { + local INPUT_PATH=${1:?"Expected path as the first argument."} + echo "$(cd "$(dirname "${INPUT_PATH}")" && pwd)/$(basename "${INPUT_PATH}")" +} + +# staging directory for binaries (source) +BINARIES_STAGING_DIR=$(abspath "${1:?${USAGE_TEXT}}") +# staging directory for build artifacts (destination) +ARTIFACTS_STAGING_DIR=$(abspath "${2:?${USAGE_TEXT}}") +ORT_POD_VERSION=${3:?${USAGE_TEXT}} + +STORAGE_ACCOUNT_NAME="onnxruntimepackages" +STORAGE_ACCOUNT_CONTAINER_NAME="ortmobilestore" +STORAGE_URL_PREFIX="https://${STORAGE_ACCOUNT_NAME}.blob.core.windows.net/${STORAGE_ACCOUNT_CONTAINER_NAME}" + +assemble_and_upload_pod() { + local POD_NAME=${1:?"Expected pod name as first argument."} + local POD_ARCHIVE_BASENAME="${POD_NAME}-${ORT_POD_VERSION}.zip" + local PODSPEC_BASENAME="${POD_NAME}.podspec" + + pushd ${BINARIES_STAGING_DIR}/${POD_NAME} + + # assemble the files in the artifacts staging directory + zip -r ${ARTIFACTS_STAGING_DIR}/${POD_ARCHIVE_BASENAME} * --exclude ${PODSPEC_BASENAME} + cp ${PODSPEC_BASENAME} ${ARTIFACTS_STAGING_DIR}/${PODSPEC_BASENAME} + + # upload the pod archive and set the podspec source to the pod archive URL + az storage blob upload \ + --account-name ${STORAGE_ACCOUNT_NAME} --container-name ${STORAGE_ACCOUNT_CONTAINER_NAME} \ + --file ${ARTIFACTS_STAGING_DIR}/${POD_ARCHIVE_BASENAME} --name ${POD_ARCHIVE_BASENAME} + + sed -i "" -e "s|file:///http_source_placeholder|${STORAGE_URL_PREFIX}/${POD_ARCHIVE_BASENAME}|" \ + ${ARTIFACTS_STAGING_DIR}/${PODSPEC_BASENAME} + + popd +} + +assemble_and_upload_pod "onnxruntime-mobile-c" + +assemble_and_upload_pod "onnxruntime-mobile-objc" + +cd ${BINARIES_STAGING_DIR}/objc_api_docs +zip -r ${ARTIFACTS_STAGING_DIR}/objc_api_docs.zip * + +cat > ${ARTIFACTS_STAGING_DIR}/readme.txt <<'EOM' +Release TODO: +- publish the podspecs +- publish the Objective-C API documentation +EOM diff --git a/tools/ci_build/github/apple/build_ios_framework.py b/tools/ci_build/github/apple/build_ios_framework.py index cf0c8a1196..433fa1b734 100644 --- a/tools/ci_build/github/apple/build_ios_framework.py +++ b/tools/ci_build/github/apple/build_ios_framework.py @@ -75,17 +75,19 @@ def _build_package(args): build_dir_current_arch, build_config, build_config + "-" + sysroot, 'onnxruntime.framework') ort_libs.append(os.path.join(framework_dir, 'onnxruntime')) - # We actually only need to define the Info.plist and headers once since they are all the same + # We only need to copy Info.plist, framework_info.json, and headers once since they are the same if not info_plist_path: info_plist_path = os.path.join(build_dir_current_arch, build_config, 'Info.plist') + framework_info_path = os.path.join(build_dir_current_arch, build_config, 'framework_info.json') headers = glob.glob(os.path.join(framework_dir, 'Headers', '*.h')) # manually create the fat framework framework_dir = os.path.join(build_dir, 'framework_out', 'onnxruntime.framework') pathlib.Path(framework_dir).mkdir(parents=True, exist_ok=True) - # copy the header files and Info.plist + # copy the Info.plist, framework_info.json, and header files shutil.copy(info_plist_path, framework_dir) + shutil.copy(framework_info_path, build_dir) header_dir = os.path.join(framework_dir, 'Headers') pathlib.Path(header_dir).mkdir(parents=True, exist_ok=True) for _header in headers: diff --git a/tools/ci_build/github/apple/c/assemble_c_pod_package.py b/tools/ci_build/github/apple/c/assemble_c_pod_package.py new file mode 100644 index 0000000000..094e7cbd4c --- /dev/null +++ b/tools/ci_build/github/apple/c/assemble_c_pod_package.py @@ -0,0 +1,72 @@ +#!/usr/bin/env python3 + +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import argparse +import pathlib +import shutil +import sys + + +_script_dir = pathlib.Path(__file__).parent.resolve(strict=True) +sys.path.append(str(_script_dir.parent)) + + +from package_assembly_utils import ( # noqa: E402 + copy_repo_relative_to_dir, gen_file_from_template, load_framework_info) + + +def parse_args(): + parser = argparse.ArgumentParser(description=""" + Assembles the files for the C/C++ pod package in a staging directory. + This directory can be validated (e.g., with `pod lib lint`) and then zipped to create a package for release. + """) + + parser.add_argument("--staging-dir", type=pathlib.Path, + default=pathlib.Path("./onnxruntime-mobile-c-staging"), + help="Path to the staging directory for the C/C++ pod files.") + parser.add_argument("--pod-version", required=True, + help="C/C++ pod version.") + parser.add_argument("--framework-info-file", type=pathlib.Path, required=True, + help="Path to the framework_info.json file containing additional values for the podspec. " + "This file should be generated by CMake in the build directory.") + parser.add_argument("--framework-dir", type=pathlib.Path, required=True, + help="Path to the onnxruntime.framework directory to include in the pod.") + + return parser.parse_args() + + +def main(): + args = parse_args() + + framework_info = load_framework_info(args.framework_info_file.resolve()) + + staging_dir = args.staging_dir.resolve() + print(f"Assembling files in staging directory: {staging_dir}") + if staging_dir.exists(): + print("Warning: staging directory already exists", file=sys.stderr) + + # copy the necessary files to the staging directory + framework_dir = args.framework_dir.resolve() + shutil.copytree(framework_dir, staging_dir / framework_dir.name, dirs_exist_ok=True) + copy_repo_relative_to_dir(["LICENSE"], staging_dir) + + # generate the podspec file from the template + + variable_substitutions = { + "VERSION": args.pod_version, + "IOS_DEPLOYMENT_TARGET": framework_info["IOS_DEPLOYMENT_TARGET"], + "LICENSE_FILE": '"LICENSE"', + } + + podspec_template = _script_dir / "onnxruntime-mobile-c.podspec.template" + podspec = staging_dir / "onnxruntime-mobile-c.podspec" + + gen_file_from_template(podspec_template, podspec, variable_substitutions) + + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/ci_build/github/apple/onnxruntime-mobile-c.podspec.template b/tools/ci_build/github/apple/c/onnxruntime-mobile-c.podspec.template similarity index 51% rename from tools/ci_build/github/apple/onnxruntime-mobile-c.podspec.template rename to tools/ci_build/github/apple/c/onnxruntime-mobile-c.podspec.template index 4cde006ab4..c2f7870bcf 100644 --- a/tools/ci_build/github/apple/onnxruntime-mobile-c.podspec.template +++ b/tools/ci_build/github/apple/c/onnxruntime-mobile-c.podspec.template @@ -1,16 +1,19 @@ Pod::Spec.new do |spec| spec.name = "onnxruntime-mobile-c" - spec.version = "${ORT_VERSION}" + spec.version = "@VERSION@" spec.authors = { "ONNX Runtime" => "onnxruntime@microsoft.com" } - spec.license = { :type => "MIT" } + spec.license = { :type => "MIT", :file => @LICENSE_FILE@ } spec.homepage = "https://github.com/microsoft/onnxruntime" - spec.source = { :http => "_ORT_DOWNLOAD_URL_" } + spec.source = { :http => "file:///http_source_placeholder" } spec.summary = "ONNX Runtime Mobile C/C++ Pod" - spec.platform = :ios, "${CMAKE_OSX_DEPLOYMENT_TARGET}" + spec.platform = :ios, "@IOS_DEPLOYMENT_TARGET@" spec.vendored_frameworks = "onnxruntime.framework" spec.weak_framework = 'CoreML' spec.source_files = 'onnxruntime.framework/Headers/*.h' + spec.preserve_paths = [ @LICENSE_FILE@ ] spec.description = <<-DESC - A preview pod for ONNX Runtime Mobile C/C++ library. Pods for Objective-C and Swift coming soon. + A pod for the ONNX Runtime Mobile C/C++ library. DESC + # TODO workaround - support arm64 iphonesimulator later + spec.user_target_xcconfig = { "EXCLUDED_ARCHS[sdk=iphonesimulator*]" => "arm64" } end diff --git a/tools/ci_build/github/apple/framework_info.json.template b/tools/ci_build/github/apple/framework_info.json.template new file mode 100644 index 0000000000..d8de4115ef --- /dev/null +++ b/tools/ci_build/github/apple/framework_info.json.template @@ -0,0 +1,3 @@ +{ + "IOS_DEPLOYMENT_TARGET": "@CMAKE_OSX_DEPLOYMENT_TARGET@" +} \ No newline at end of file diff --git a/tools/ci_build/github/apple/ios_packaging.requirements.txt b/tools/ci_build/github/apple/ios_packaging.requirements.txt new file mode 100644 index 0000000000..adf11d699f --- /dev/null +++ b/tools/ci_build/github/apple/ios_packaging.requirements.txt @@ -0,0 +1 @@ +flatbuffers diff --git a/tools/ci_build/github/apple/objectivec/assemble_pod_package.py b/tools/ci_build/github/apple/objectivec/assemble_objc_pod_package.py similarity index 58% rename from tools/ci_build/github/apple/objectivec/assemble_pod_package.py rename to tools/ci_build/github/apple/objectivec/assemble_objc_pod_package.py index 7543df0b82..4b4ffcc638 100755 --- a/tools/ci_build/github/apple/objectivec/assemble_pod_package.py +++ b/tools/ci_build/github/apple/objectivec/assemble_objc_pod_package.py @@ -4,15 +4,17 @@ # Licensed under the MIT License. import argparse -import os import pathlib -import re -import shutil import sys _script_dir = pathlib.Path(__file__).parent.resolve(strict=True) -_repo_root = _script_dir.parents[4] +sys.path.append(str(_script_dir.parent)) + + +from package_assembly_utils import ( # noqa: E402 + copy_repo_relative_to_dir, gen_file_from_template, load_framework_info) + # these variables contain paths or path patterns that are relative to the repo root @@ -61,71 +63,28 @@ def parse_args(): parser.add_argument("--staging-dir", type=pathlib.Path, default=pathlib.Path("./onnxruntime-mobile-objc-staging"), - help="Staging directory for the Objective-C pod files.") + help="Path to the staging directory for the Objective-C pod files.") parser.add_argument("--pod-version", required=True, help="Objective-C pod version.") - parser.add_argument("--pod-source-http", required=True, - help="Objective-C pod source http value (e.g., the package download URL).") + parser.add_argument("--framework-info-file", type=pathlib.Path, required=True, + help="Path to the framework_info.json file containing additional values for the podspec. " + "This file should be generated by CMake in the build directory.") return parser.parse_args() -_template_variable_pattern = re.compile(r"@(\w+)@") # match "@var@" - - -def gen_file_from_template(template_file: pathlib.Path, output_file: pathlib.Path, - variable_substitutions: dict[str, str]): - ''' - Generates a file from a template file. - The template file may contain template variables that will be substituted - with the provided values in the generated output file. - In the template file, template variable names are delimited by "@"'s, - e.g., "@var@". - - :param template_file The template file path. - :param output_file The generated output file path. - :variable_substitutions The mapping from template variable name to value. - ''' - with open(template_file, mode="r") as template: - content = template.read() - - def replace_template_variable(match): - variable_name = match.group(1) - return variable_substitutions.get(variable_name, match.group(0)) - - content = _template_variable_pattern.sub(replace_template_variable, content) - - with open(output_file, mode="w") as output: - output.write(content) - - -def copy_to_staging_dir(patterns: list[str], staging_dir: pathlib.Path): - ''' - Copies files to a staging directory. - The files are relative to the repo root, and the repo root-relative - intermediate directory structure is maintained. - - :param patterns The paths or path patterns relative to the repo root. - :param staging_dir The staging directory. - ''' - paths = [path for pattern in patterns for path in _repo_root.glob(pattern)] - for path in paths: - repo_relative_path = path.relative_to(_repo_root) - dst_path = staging_dir / repo_relative_path - os.makedirs(dst_path.parent, exist_ok=True) - shutil.copy(path, dst_path) - - def main(): args = parse_args() + framework_info = load_framework_info(args.framework_info_file.resolve()) + staging_dir = args.staging_dir.resolve() print(f"Assembling files in staging directory: {staging_dir}") if staging_dir.exists(): print("Warning: staging directory already exists", file=sys.stderr) # copy the necessary files to the staging directory - copy_to_staging_dir( + copy_repo_relative_to_dir( [license_file] + source_files + test_source_files + test_resource_files, staging_dir) @@ -136,7 +95,7 @@ def main(): variable_substitutions = { "VERSION": args.pod_version, - "SOURCE_HTTP": args.pod_source_http, + "IOS_DEPLOYMENT_TARGET": framework_info["IOS_DEPLOYMENT_TARGET"], "LICENSE_FILE": path_patterns_as_variable_value([license_file]), "INCLUDE_DIR_LIST": path_patterns_as_variable_value(include_dirs), "PUBLIC_HEADER_FILE_LIST": path_patterns_as_variable_value(public_header_files), diff --git a/tools/ci_build/github/apple/objectivec/onnxruntime-mobile-objc.podspec.template b/tools/ci_build/github/apple/objectivec/onnxruntime-mobile-objc.podspec.template index a022129253..c784662197 100644 --- a/tools/ci_build/github/apple/objectivec/onnxruntime-mobile-objc.podspec.template +++ b/tools/ci_build/github/apple/objectivec/onnxruntime-mobile-objc.podspec.template @@ -1,17 +1,17 @@ Pod::Spec.new do |s| s.name = 'onnxruntime-mobile-objc' s.version = '@VERSION@' - s.summary = 'ONNX Runtime Objective-C Pod' + s.summary = 'ONNX Runtime Mobile Objective-C Pod' s.description = <<-DESC - Preview pod for the ONNX Runtime Objective-C API. + A pod for the ONNX Runtime Mobile Objective-C API. DESC s.homepage = 'https://github.com/microsoft/onnxruntime' s.license = { :type => 'MIT', :file => @LICENSE_FILE@ } s.author = { "ONNX Runtime" => "onnxruntime@microsoft.com" } - s.source = { :http => "@SOURCE_HTTP@" } - s.ios.deployment_target = '11.0' + s.source = { :http => "file:///http_source_placeholder" } + s.ios.deployment_target = '@IOS_DEPLOYMENT_TARGET@' s.preserve_paths = [ @LICENSE_FILE@ ] s.default_subspec = "Core" diff --git a/tools/ci_build/github/apple/package_assembly_utils.py b/tools/ci_build/github/apple/package_assembly_utils.py new file mode 100644 index 0000000000..c6a0c97be7 --- /dev/null +++ b/tools/ci_build/github/apple/package_assembly_utils.py @@ -0,0 +1,68 @@ +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import json +import os +import pathlib +import re +import shutil + + +_script_dir = pathlib.Path(__file__).parent.resolve(strict=True) +repo_root = _script_dir.parents[3] + +_template_variable_pattern = re.compile(r"@(\w+)@") # match "@var@" + + +def gen_file_from_template(template_file: pathlib.Path, output_file: pathlib.Path, + variable_substitutions: dict[str, str]): + ''' + Generates a file from a template file. + The template file may contain template variables that will be substituted + with the provided values in the generated output file. + In the template file, template variable names are delimited by "@"'s, + e.g., "@var@". + + :param template_file The template file path. + :param output_file The generated output file path. + :param variable_substitutions The mapping from template variable name to value. + ''' + with open(template_file, mode="r") as template: + content = template.read() + + def replace_template_variable(match): + variable_name = match.group(1) + return variable_substitutions.get(variable_name, match.group(0)) + + content = _template_variable_pattern.sub(replace_template_variable, content) + + with open(output_file, mode="w") as output: + output.write(content) + + +def copy_repo_relative_to_dir(patterns: list[str], dest_dir: pathlib.Path): + ''' + Copies file paths relative to the repo root to a directory. + The given paths or path patterns are relative to the repo root, and the + repo root-relative intermediate directory structure is maintained. + + :param patterns The paths or path patterns relative to the repo root. + :param dest_dir The destination directory. + ''' + paths = [path for pattern in patterns for path in repo_root.glob(pattern)] + for path in paths: + repo_relative_path = path.relative_to(repo_root) + dst_path = dest_dir / repo_relative_path + os.makedirs(dst_path.parent, exist_ok=True) + shutil.copy(path, dst_path) + + +def load_framework_info(framework_info_file: pathlib.Path): + ''' + Loads framework info from a file. + + :param framework_info_file The framework info file path. + :return The framework info values. + ''' + with open(framework_info_file, mode="r") as framework_info: + return json.load(framework_info) diff --git a/tools/ci_build/github/azure-pipelines/mac-ios-packaging-pipeline.yml b/tools/ci_build/github/azure-pipelines/mac-ios-packaging-pipeline.yml new file mode 100644 index 0000000000..b2086976d9 --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/mac-ios-packaging-pipeline.yml @@ -0,0 +1,97 @@ +parameters: +- name: IsReleaseBuild + displayName: Is this a release build? Set this parameter to true if you are doing an Onnx Runtime release. + type: boolean + default: false + +jobs: +- job: IosPackaging + displayName: "iOS Packaging" + + pool: + vmImage: "macOS-10.15" + + steps: + - task: UsePythonVersion@0 + inputs: + versionSpec: "3.9" + addToPath: true + architecture: "x64" + + - script: | + pip install -r tools/ci_build/github/apple/ios_packaging.requirements.txt + displayName: "Install Python requirements" + + - bash: | + BASE_VERSION=$(cat VERSION_NUMBER) + IS_RELEASE_BUILD=$(echo "${{ parameters.IsReleaseBuild }}" | tr "[:upper:]" "[:lower:]") + + if [[ "${IS_RELEASE_BUILD}" == "true" ]]; then + VERSION=${BASE_VERSION} + else + VERSION="${BASE_VERSION}-dev" + fi + + echo "##vso[task.setvariable variable=ORT_POD_VERSION]${VERSION}" + echo "Version number: ${VERSION}" + displayName: "Set version number" + + - script: | + python tools/ci_build/github/apple/build_ios_framework.py \ + --build_dir $(Build.BinariesDirectory)/ios_framework \ + --include_ops_by_config tools/ci_build/github/android/mobile_package.required_operators.config \ + tools/ci_build/github/apple/default_mobile_ios_framework_build_settings.json + displayName: "Build iOS framework" + + - script: | + python tools/ci_build/github/apple/test_ios_packages.py \ + --fail_if_cocoapods_missing \ + --c_framework_dir $(Build.BinariesDirectory)/ios_framework/framework_out + displayName: "Test iOS framework" + + - script: | + python tools/ci_build/github/apple/c/assemble_c_pod_package.py \ + --staging-dir $(Build.BinariesDirectory)/staging/onnxruntime-mobile-c \ + --pod-version ${ORT_POD_VERSION} \ + --framework-info-file $(Build.BinariesDirectory)/ios_framework/framework_info.json \ + --framework-dir $(Build.BinariesDirectory)/ios_framework/framework_out/onnxruntime.framework + displayName: "Assemble onnxruntime-mobile-c pod files" + + - script: | + pod lib lint --verbose + workingDirectory: "$(Build.BinariesDirectory)/staging/onnxruntime-mobile-c" + displayName: "Check onnxruntime-mobile-c pod" + + - script: | + python tools/ci_build/github/apple/objectivec/assemble_objc_pod_package.py \ + --staging-dir $(Build.BinariesDirectory)/staging/onnxruntime-mobile-objc \ + --pod-version ${ORT_POD_VERSION} \ + --framework-info-file $(Build.BinariesDirectory)/ios_framework/framework_info.json + displayName: "Assemble onnxruntime-mobile-objc pod files" + + - script: | + pod lib lint --verbose \ + --include-podspecs=$(Build.BinariesDirectory)/staging/onnxruntime-mobile-c/onnxruntime-mobile-c.podspec + workingDirectory: "$(Build.BinariesDirectory)/staging/onnxruntime-mobile-objc" + displayName: "Check onnxruntime-mobile-objc pod" + + - bash: | + set -e + gem install jazzy + jazzy --config objectivec/docs/jazzy_config.yaml \ + --output $(Build.BinariesDirectory)/staging/objc_api_docs \ + --module-version ${ORT_POD_VERSION} + displayName: "Generate Objective-C API docs" + + - task: AzureCLI@2 + inputs: + azureSubscription: 'AIInfraBuildOnnxRuntimeOSS' + scriptType: 'bash' + scriptLocation: 'scriptPath' + scriptPath: 'tools/ci_build/github/apple/assemble_ios_packaging_artifacts.sh' + arguments: '"$(Build.BinariesDirectory)/staging" "$(Build.ArtifactStagingDirectory)" "$(ORT_POD_VERSION)"' + displayName: "Assemble artifacts" + + - publish: $(Build.ArtifactStagingDirectory) + artifact: ios_packaging_artifacts + displayName: "Publish artifacts"