Add iOS packaging pipeline (#8264)

Create a pipeline to produce the iOS package artifacts.

cmake/onnxruntime.cmake

@@ -71,9 +71,8 @@ elseif(onnxruntime_BUILD_APPLE_FRAMEWORK)
   set(MACOSX_FRAMEWORK_IDENTIFIER "com.microsoft.onnxruntime")
   configure_file(${REPO_ROOT}/cmake/Info.plist.in ${CMAKE_CURRENT_BINARY_DIR}/Info.plist)
   configure_file(
-    ${REPO_ROOT}/tools/ci_build/github/apple/onnxruntime-mobile-c.podspec.template
-    ${CMAKE_CURRENT_BINARY_DIR}/onnxruntime-mobile-c.podspec
-  )
+    ${REPO_ROOT}/tools/ci_build/github/apple/framework_info.json.template
+    ${CMAKE_CURRENT_BINARY_DIR}/framework_info.json)
   set_target_properties(onnxruntime PROPERTIES
     FRAMEWORK TRUE
     FRAMEWORK_VERSION A

tools/ci_build/github/apple/assemble_ios_packaging_artifacts.sh

@@ -0,0 +1,59 @@
+#!/bin/bash
+
+# Note: This script is intended to be called from the iOS packaging build or a similar context
+# See tools/ci_build/github/azure-pipelines/mac-ios-packaging-pipeline.yml
+
+set -e
+set -x
+
+USAGE_TEXT="Usage: ${0} <binaries staging directory> <artifacts staging directory> <ORT pod version>"
+
+abspath() {
+  local INPUT_PATH=${1:?"Expected path as the first argument."}
+  echo "$(cd "$(dirname "${INPUT_PATH}")" && pwd)/$(basename "${INPUT_PATH}")"
+}
+
+# staging directory for binaries (source)
+BINARIES_STAGING_DIR=$(abspath "${1:?${USAGE_TEXT}}")
+# staging directory for build artifacts (destination)
+ARTIFACTS_STAGING_DIR=$(abspath "${2:?${USAGE_TEXT}}")
+ORT_POD_VERSION=${3:?${USAGE_TEXT}}
+
+STORAGE_ACCOUNT_NAME="onnxruntimepackages"
+STORAGE_ACCOUNT_CONTAINER_NAME="ortmobilestore"
+STORAGE_URL_PREFIX="https://${STORAGE_ACCOUNT_NAME}.blob.core.windows.net/${STORAGE_ACCOUNT_CONTAINER_NAME}"
+
+assemble_and_upload_pod() {
+  local POD_NAME=${1:?"Expected pod name as first argument."}
+  local POD_ARCHIVE_BASENAME="${POD_NAME}-${ORT_POD_VERSION}.zip"
+  local PODSPEC_BASENAME="${POD_NAME}.podspec"
+
+  pushd ${BINARIES_STAGING_DIR}/${POD_NAME}
+
+  # assemble the files in the artifacts staging directory
+  zip -r ${ARTIFACTS_STAGING_DIR}/${POD_ARCHIVE_BASENAME} * --exclude ${PODSPEC_BASENAME}
+  cp ${PODSPEC_BASENAME} ${ARTIFACTS_STAGING_DIR}/${PODSPEC_BASENAME}
+
+  # upload the pod archive and set the podspec source to the pod archive URL
+  az storage blob upload \
+    --account-name ${STORAGE_ACCOUNT_NAME} --container-name ${STORAGE_ACCOUNT_CONTAINER_NAME} \
+    --file ${ARTIFACTS_STAGING_DIR}/${POD_ARCHIVE_BASENAME} --name ${POD_ARCHIVE_BASENAME}
+
+  sed -i "" -e "s|file:///http_source_placeholder|${STORAGE_URL_PREFIX}/${POD_ARCHIVE_BASENAME}|" \
+    ${ARTIFACTS_STAGING_DIR}/${PODSPEC_BASENAME}
+
+  popd
+}
+
+assemble_and_upload_pod "onnxruntime-mobile-c"
+
+assemble_and_upload_pod "onnxruntime-mobile-objc"
+
+cd ${BINARIES_STAGING_DIR}/objc_api_docs
+zip -r ${ARTIFACTS_STAGING_DIR}/objc_api_docs.zip *
+
+cat > ${ARTIFACTS_STAGING_DIR}/readme.txt <<'EOM'
+Release TODO:
+- publish the podspecs
+- publish the Objective-C API documentation
+EOM

tools/ci_build/github/apple/build_ios_framework.py

@@ -75,17 +75,19 @@ def _build_package(args):
             build_dir_current_arch, build_config, build_config + "-" + sysroot, 'onnxruntime.framework')
         ort_libs.append(os.path.join(framework_dir, 'onnxruntime'))
 
-        # We actually only need to define the Info.plist and headers once since they are all the same
+        # We only need to copy Info.plist, framework_info.json, and headers once since they are the same
         if not info_plist_path:
             info_plist_path = os.path.join(build_dir_current_arch, build_config, 'Info.plist')
+            framework_info_path = os.path.join(build_dir_current_arch, build_config, 'framework_info.json')
             headers = glob.glob(os.path.join(framework_dir, 'Headers', '*.h'))
 
     # manually create the fat framework
     framework_dir = os.path.join(build_dir, 'framework_out', 'onnxruntime.framework')
     pathlib.Path(framework_dir).mkdir(parents=True, exist_ok=True)
 
-    # copy the header files and Info.plist
+    # copy the Info.plist, framework_info.json, and header files
     shutil.copy(info_plist_path, framework_dir)
+    shutil.copy(framework_info_path, build_dir)
     header_dir = os.path.join(framework_dir, 'Headers')
     pathlib.Path(header_dir).mkdir(parents=True, exist_ok=True)
     for _header in headers:

tools/ci_build/github/apple/c/assemble_c_pod_package.py

@@ -0,0 +1,72 @@
+#!/usr/bin/env python3
+
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import argparse
+import pathlib
+import shutil
+import sys
+
+
+_script_dir = pathlib.Path(__file__).parent.resolve(strict=True)
+sys.path.append(str(_script_dir.parent))
+
+
+from package_assembly_utils import (  # noqa: E402
+    copy_repo_relative_to_dir, gen_file_from_template, load_framework_info)
+
+
+def parse_args():
+    parser = argparse.ArgumentParser(description="""
+        Assembles the files for the C/C++ pod package in a staging directory.
+        This directory can be validated (e.g., with `pod lib lint`) and then zipped to create a package for release.
+    """)
+
+    parser.add_argument("--staging-dir", type=pathlib.Path,
+                        default=pathlib.Path("./onnxruntime-mobile-c-staging"),
+                        help="Path to the staging directory for the C/C++ pod files.")
+    parser.add_argument("--pod-version", required=True,
+                        help="C/C++ pod version.")
+    parser.add_argument("--framework-info-file", type=pathlib.Path, required=True,
+                        help="Path to the framework_info.json file containing additional values for the podspec. "
+                             "This file should be generated by CMake in the build directory.")
+    parser.add_argument("--framework-dir", type=pathlib.Path, required=True,
+                        help="Path to the onnxruntime.framework directory to include in the pod.")
+
+    return parser.parse_args()
+
+
+def main():
+    args = parse_args()
+
+    framework_info = load_framework_info(args.framework_info_file.resolve())
+
+    staging_dir = args.staging_dir.resolve()
+    print(f"Assembling files in staging directory: {staging_dir}")
+    if staging_dir.exists():
+        print("Warning: staging directory already exists", file=sys.stderr)
+
+    # copy the necessary files to the staging directory
+    framework_dir = args.framework_dir.resolve()
+    shutil.copytree(framework_dir, staging_dir / framework_dir.name, dirs_exist_ok=True)
+    copy_repo_relative_to_dir(["LICENSE"], staging_dir)
+
+    # generate the podspec file from the template
+
+    variable_substitutions = {
+        "VERSION": args.pod_version,
+        "IOS_DEPLOYMENT_TARGET": framework_info["IOS_DEPLOYMENT_TARGET"],
+        "LICENSE_FILE": '"LICENSE"',
+    }
+
+    podspec_template = _script_dir / "onnxruntime-mobile-c.podspec.template"
+    podspec = staging_dir / "onnxruntime-mobile-c.podspec"
+
+    gen_file_from_template(podspec_template, podspec, variable_substitutions)
+
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())

tools/ci_build/github/apple/c/onnxruntime-mobile-c.podspec.template

@@ -1,16 +1,19 @@
 Pod::Spec.new do |spec|
     spec.name                   = "onnxruntime-mobile-c"
-    spec.version                = "${ORT_VERSION}"
+    spec.version                = "@VERSION@"
     spec.authors                = { "ONNX Runtime" => "onnxruntime@microsoft.com" }
-    spec.license                = { :type => "MIT" }
+    spec.license                = { :type => "MIT", :file => @LICENSE_FILE@ }
     spec.homepage               = "https://github.com/microsoft/onnxruntime"
-    spec.source                 = { :http => "_ORT_DOWNLOAD_URL_" }
+    spec.source                 = { :http => "file:///http_source_placeholder" }
     spec.summary                = "ONNX Runtime Mobile C/C++ Pod"
-    spec.platform               = :ios, "${CMAKE_OSX_DEPLOYMENT_TARGET}"
+    spec.platform               = :ios, "@IOS_DEPLOYMENT_TARGET@"
     spec.vendored_frameworks    = "onnxruntime.framework"
     spec.weak_framework         = 'CoreML'
     spec.source_files           = 'onnxruntime.framework/Headers/*.h'
+    spec.preserve_paths         = [ @LICENSE_FILE@ ]
     spec.description            = <<-DESC
-    A preview pod for ONNX Runtime Mobile C/C++ library. Pods for Objective-C and Swift coming soon.
+    A pod for the ONNX Runtime Mobile C/C++ library.
                                     DESC
+    # TODO workaround - support arm64 iphonesimulator later
+    spec.user_target_xcconfig    = { "EXCLUDED_ARCHS[sdk=iphonesimulator*]" => "arm64" }
 end

tools/ci_build/github/apple/framework_info.json.template

@@ -0,0 +1,3 @@
+{
+    "IOS_DEPLOYMENT_TARGET": "@CMAKE_OSX_DEPLOYMENT_TARGET@"
+}

tools/ci_build/github/apple/ios_packaging.requirements.txt

@@ -0,0 +1 @@
+flatbuffers

tools/ci_build/github/apple/objectivec/assemble_objc_pod_package.py

@@ -4,15 +4,17 @@
 # Licensed under the MIT License.
 
 import argparse
-import os
 import pathlib
-import re
-import shutil
 import sys
 
 
 _script_dir = pathlib.Path(__file__).parent.resolve(strict=True)
-_repo_root = _script_dir.parents[4]
+sys.path.append(str(_script_dir.parent))
+
+
+from package_assembly_utils import (  # noqa: E402
+    copy_repo_relative_to_dir, gen_file_from_template, load_framework_info)
+
 
 # these variables contain paths or path patterns that are relative to the repo root
 
@@ -61,71 +63,28 @@ def parse_args():
 
     parser.add_argument("--staging-dir", type=pathlib.Path,
                         default=pathlib.Path("./onnxruntime-mobile-objc-staging"),
-                        help="Staging directory for the Objective-C pod files.")
+                        help="Path to the staging directory for the Objective-C pod files.")
     parser.add_argument("--pod-version", required=True,
                         help="Objective-C pod version.")
-    parser.add_argument("--pod-source-http", required=True,
-                        help="Objective-C pod source http value (e.g., the package download URL).")
+    parser.add_argument("--framework-info-file", type=pathlib.Path, required=True,
+                        help="Path to the framework_info.json file containing additional values for the podspec. "
+                             "This file should be generated by CMake in the build directory.")
 
     return parser.parse_args()
 
 
-_template_variable_pattern = re.compile(r"@(\w+)@")  # match "@var@"
-
-
-def gen_file_from_template(template_file: pathlib.Path, output_file: pathlib.Path,
-                           variable_substitutions: dict[str, str]):
-    '''
-    Generates a file from a template file.
-    The template file may contain template variables that will be substituted
-    with the provided values in the generated output file.
-    In the template file, template variable names are delimited by "@"'s,
-    e.g., "@var@".
-
-    :param template_file The template file path.
-    :param output_file The generated output file path.
-    :variable_substitutions The mapping from template variable name to value.
-    '''
-    with open(template_file, mode="r") as template:
-        content = template.read()
-
-    def replace_template_variable(match):
-        variable_name = match.group(1)
-        return variable_substitutions.get(variable_name, match.group(0))
-
-    content = _template_variable_pattern.sub(replace_template_variable, content)
-
-    with open(output_file, mode="w") as output:
-        output.write(content)
-
-
-def copy_to_staging_dir(patterns: list[str], staging_dir: pathlib.Path):
-    '''
-    Copies files to a staging directory.
-    The files are relative to the repo root, and the repo root-relative
-    intermediate directory structure is maintained.
-
-    :param patterns The paths or path patterns relative to the repo root.
-    :param staging_dir The staging directory.
-    '''
-    paths = [path for pattern in patterns for path in _repo_root.glob(pattern)]
-    for path in paths:
-        repo_relative_path = path.relative_to(_repo_root)
-        dst_path = staging_dir / repo_relative_path
-        os.makedirs(dst_path.parent, exist_ok=True)
-        shutil.copy(path, dst_path)
-
-
 def main():
     args = parse_args()
 
+    framework_info = load_framework_info(args.framework_info_file.resolve())
+
     staging_dir = args.staging_dir.resolve()
     print(f"Assembling files in staging directory: {staging_dir}")
     if staging_dir.exists():
         print("Warning: staging directory already exists", file=sys.stderr)
 
     # copy the necessary files to the staging directory
-    copy_to_staging_dir(
+    copy_repo_relative_to_dir(
         [license_file] + source_files + test_source_files + test_resource_files,
         staging_dir)
 
@@ -136,7 +95,7 @@ def main():
 
     variable_substitutions = {
         "VERSION": args.pod_version,
-        "SOURCE_HTTP": args.pod_source_http,
+        "IOS_DEPLOYMENT_TARGET": framework_info["IOS_DEPLOYMENT_TARGET"],
         "LICENSE_FILE": path_patterns_as_variable_value([license_file]),
         "INCLUDE_DIR_LIST": path_patterns_as_variable_value(include_dirs),
         "PUBLIC_HEADER_FILE_LIST": path_patterns_as_variable_value(public_header_files),

tools/ci_build/github/apple/objectivec/onnxruntime-mobile-objc.podspec.template

@@ -1,17 +1,17 @@
 Pod::Spec.new do |s|
   s.name             = 'onnxruntime-mobile-objc'
   s.version          = '@VERSION@'
-  s.summary          = 'ONNX Runtime Objective-C Pod'
+  s.summary          = 'ONNX Runtime Mobile Objective-C Pod'
 
   s.description      = <<-DESC
-    Preview pod for the ONNX Runtime Objective-C API.
+    A pod for the ONNX Runtime Mobile Objective-C API.
                        DESC
 
   s.homepage         = 'https://github.com/microsoft/onnxruntime'
   s.license          = { :type => 'MIT', :file => @LICENSE_FILE@ }
   s.author           = { "ONNX Runtime" => "onnxruntime@microsoft.com" }
-  s.source           = { :http => "@SOURCE_HTTP@" }
-  s.ios.deployment_target = '11.0'
+  s.source           = { :http => "file:///http_source_placeholder" }
+  s.ios.deployment_target = '@IOS_DEPLOYMENT_TARGET@'
   s.preserve_paths = [ @LICENSE_FILE@ ]
   s.default_subspec = "Core"
 

tools/ci_build/github/apple/package_assembly_utils.py

@@ -0,0 +1,68 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+import json
+import os
+import pathlib
+import re
+import shutil
+
+
+_script_dir = pathlib.Path(__file__).parent.resolve(strict=True)
+repo_root = _script_dir.parents[3]
+
+_template_variable_pattern = re.compile(r"@(\w+)@")  # match "@var@"
+
+
+def gen_file_from_template(template_file: pathlib.Path, output_file: pathlib.Path,
+                           variable_substitutions: dict[str, str]):
+    '''
+    Generates a file from a template file.
+    The template file may contain template variables that will be substituted
+    with the provided values in the generated output file.
+    In the template file, template variable names are delimited by "@"'s,
+    e.g., "@var@".
+
+    :param template_file The template file path.
+    :param output_file The generated output file path.
+    :param variable_substitutions The mapping from template variable name to value.
+    '''
+    with open(template_file, mode="r") as template:
+        content = template.read()
+
+    def replace_template_variable(match):
+        variable_name = match.group(1)
+        return variable_substitutions.get(variable_name, match.group(0))
+
+    content = _template_variable_pattern.sub(replace_template_variable, content)
+
+    with open(output_file, mode="w") as output:
+        output.write(content)
+
+
+def copy_repo_relative_to_dir(patterns: list[str], dest_dir: pathlib.Path):
+    '''
+    Copies file paths relative to the repo root to a directory.
+    The given paths or path patterns are relative to the repo root, and the
+    repo root-relative intermediate directory structure is maintained.
+
+    :param patterns The paths or path patterns relative to the repo root.
+    :param dest_dir The destination directory.
+    '''
+    paths = [path for pattern in patterns for path in repo_root.glob(pattern)]
+    for path in paths:
+        repo_relative_path = path.relative_to(repo_root)
+        dst_path = dest_dir / repo_relative_path
+        os.makedirs(dst_path.parent, exist_ok=True)
+        shutil.copy(path, dst_path)
+
+
+def load_framework_info(framework_info_file: pathlib.Path):
+    '''
+    Loads framework info from a file.
+
+    :param framework_info_file The framework info file path.
+    :return The framework info values.
+    '''
+    with open(framework_info_file, mode="r") as framework_info:
+        return json.load(framework_info)

tools/ci_build/github/azure-pipelines/mac-ios-packaging-pipeline.yml

@@ -0,0 +1,97 @@
+parameters:
+- name: IsReleaseBuild
+  displayName: Is this a release build? Set this parameter to true if you are doing an Onnx Runtime release.
+  type: boolean
+  default: false
+
+jobs:
+- job: IosPackaging
+  displayName: "iOS Packaging"
+
+  pool:
+    vmImage: "macOS-10.15"
+
+  steps:
+  - task: UsePythonVersion@0
+    inputs:
+      versionSpec: "3.9"
+      addToPath: true
+      architecture: "x64"
+
+  - script: |
+      pip install -r tools/ci_build/github/apple/ios_packaging.requirements.txt
+    displayName: "Install Python requirements"
+
+  - bash: |
+      BASE_VERSION=$(cat VERSION_NUMBER)
+      IS_RELEASE_BUILD=$(echo "${{ parameters.IsReleaseBuild }}" | tr "[:upper:]" "[:lower:]")
+
+      if [[ "${IS_RELEASE_BUILD}" == "true" ]]; then
+        VERSION=${BASE_VERSION}
+      else
+        VERSION="${BASE_VERSION}-dev"
+      fi
+
+      echo "##vso[task.setvariable variable=ORT_POD_VERSION]${VERSION}"
+      echo "Version number: ${VERSION}"
+    displayName: "Set version number"
+
+  - script: |
+      python tools/ci_build/github/apple/build_ios_framework.py \
+        --build_dir $(Build.BinariesDirectory)/ios_framework \
+        --include_ops_by_config tools/ci_build/github/android/mobile_package.required_operators.config \
+        tools/ci_build/github/apple/default_mobile_ios_framework_build_settings.json
+    displayName: "Build iOS framework"
+
+  - script: |
+      python tools/ci_build/github/apple/test_ios_packages.py \
+        --fail_if_cocoapods_missing \
+        --c_framework_dir $(Build.BinariesDirectory)/ios_framework/framework_out
+    displayName: "Test iOS framework"
+
+  - script: |
+      python tools/ci_build/github/apple/c/assemble_c_pod_package.py \
+        --staging-dir $(Build.BinariesDirectory)/staging/onnxruntime-mobile-c \
+        --pod-version ${ORT_POD_VERSION} \
+        --framework-info-file $(Build.BinariesDirectory)/ios_framework/framework_info.json \
+        --framework-dir $(Build.BinariesDirectory)/ios_framework/framework_out/onnxruntime.framework
+    displayName: "Assemble onnxruntime-mobile-c pod files"
+
+  - script: |
+      pod lib lint --verbose
+    workingDirectory: "$(Build.BinariesDirectory)/staging/onnxruntime-mobile-c"
+    displayName: "Check onnxruntime-mobile-c pod"
+
+  - script: |
+      python tools/ci_build/github/apple/objectivec/assemble_objc_pod_package.py \
+        --staging-dir $(Build.BinariesDirectory)/staging/onnxruntime-mobile-objc \
+        --pod-version ${ORT_POD_VERSION} \
+        --framework-info-file $(Build.BinariesDirectory)/ios_framework/framework_info.json
+    displayName: "Assemble onnxruntime-mobile-objc pod files"
+
+  - script: |
+      pod lib lint --verbose \
+        --include-podspecs=$(Build.BinariesDirectory)/staging/onnxruntime-mobile-c/onnxruntime-mobile-c.podspec
+    workingDirectory: "$(Build.BinariesDirectory)/staging/onnxruntime-mobile-objc"
+    displayName: "Check onnxruntime-mobile-objc pod"
+
+  - bash: |
+      set -e
+      gem install jazzy
+      jazzy --config objectivec/docs/jazzy_config.yaml \
+        --output $(Build.BinariesDirectory)/staging/objc_api_docs \
+        --module-version ${ORT_POD_VERSION}
+    displayName: "Generate Objective-C API docs"
+  
+  - task: AzureCLI@2
+    inputs:
+      azureSubscription: 'AIInfraBuildOnnxRuntimeOSS'
+      scriptType: 'bash'
+      scriptLocation: 'scriptPath'
+      scriptPath: 'tools/ci_build/github/apple/assemble_ios_packaging_artifacts.sh'
+      arguments: '"$(Build.BinariesDirectory)/staging" "$(Build.ArtifactStagingDirectory)" "$(ORT_POD_VERSION)"'
+    displayName: "Assemble artifacts"
+
+  - publish: $(Build.ArtifactStagingDirectory)
+    artifact: ios_packaging_artifacts
+    displayName: "Publish artifacts"