diff --git a/tools/android_custom_build/.dockerignore b/tools/android_custom_build/.dockerignore new file mode 100644 index 0000000000..38b8b7b75d --- /dev/null +++ b/tools/android_custom_build/.dockerignore @@ -0,0 +1,2 @@ +# ignore all files +* diff --git a/tools/android_custom_build/Dockerfile b/tools/android_custom_build/Dockerfile new file mode 100644 index 0000000000..d119514c92 --- /dev/null +++ b/tools/android_custom_build/Dockerfile @@ -0,0 +1,73 @@ +# -------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +# -------------------------------------------------------------- +# Dockerfile for ONNX Runtime Android package build environment + +FROM ubuntu:18.04 + +ENV DEBIAN_FRONTEND=noninteractive + +# install utilities +RUN apt-get update && apt-get install --yes --no-install-recommends \ + aria2 \ + unzip + +# install Java +RUN apt-get install --yes --no-install-recommends openjdk-8-jdk-headless + +ENV ANDROID_HOME=/opt/android-sdk +ENV NDK_VERSION=24.0.8215888 +ENV ANDROID_NDK_HOME=${ANDROID_HOME}/ndk/${NDK_VERSION} + +# install Android command line tools +RUN aria2c -q -d /tmp -o cmdline-tools.zip \ + --checksum=sha-256=d71f75333d79c9c6ef5c39d3456c6c58c613de30e6a751ea0dbd433e8f8b9cbf \ + https://dl.google.com/android/repository/commandlinetools-linux-8092744_latest.zip && \ + unzip /tmp/cmdline-tools.zip -d /tmp/cmdline-tools && \ + mkdir -p ${ANDROID_HOME}/cmdline-tools && \ + mv /tmp/cmdline-tools/cmdline-tools ${ANDROID_HOME}/cmdline-tools/latest + +RUN yes | ${ANDROID_HOME}/cmdline-tools/latest/bin/sdkmanager --licenses +RUN ${ANDROID_HOME}/cmdline-tools/latest/bin/sdkmanager --install \ + "platforms;android-32" \ + "ndk;${NDK_VERSION}" + +# install ORT dependencies +RUN apt-get install --yes --no-install-recommends \ + ca-certificates \ + build-essential \ + git \ + ninja-build \ + python3-dev \ + python3-numpy \ + python3-pip \ + python3-setuptools \ + python3-wheel + +# cmake +RUN CMAKE_VERSION=3.21.0 && \ + aria2c -q -d /tmp -o cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz \ + --checksum=sha-256=d54ef6909f519740bc85cec07ff54574cd1e061f9f17357d9ace69f61c6291ce \ + https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz && \ + tar -zxf /tmp/cmake-${CMAKE_VERSION}-linux-x86_64.tar.gz --strip=1 -C /usr + +# gradle +RUN GRADLE_VERSION=6.8.3 && \ + aria2c -q -d /tmp -o gradle-${GRADLE_VERSION}-bin.zip \ + --checksum=sha-256=7faa7198769f872826c8ef4f1450f839ec27f0b4d5d1e51bade63667cbccd205 \ + https://services.gradle.org/distributions/gradle-${GRADLE_VERSION}-bin.zip && \ + mkdir /opt/gradle && \ + unzip -d /opt/gradle /tmp/gradle-${GRADLE_VERSION}-bin.zip && \ + ln -s /opt/gradle/gradle-${GRADLE_VERSION}/bin/gradle /usr/bin + +# flatbuffers +RUN python3 -m pip install flatbuffers==2.0 + +WORKDIR /workspace + +# get ORT repo +ARG ONNXRUNTIME_REPO=https://github.com/microsoft/onnxruntime.git +ARG ONNXRUNTIME_BRANCH_OR_TAG=master +RUN git clone --single-branch --branch=${ONNXRUNTIME_BRANCH_OR_TAG} --recurse-submodules ${ONNXRUNTIME_REPO} \ + /workspace/onnxruntime diff --git a/tools/android_custom_build/build_custom_android_package.py b/tools/android_custom_build/build_custom_android_package.py new file mode 100755 index 0000000000..2c99bdfcc2 --- /dev/null +++ b/tools/android_custom_build/build_custom_android_package.py @@ -0,0 +1,117 @@ +#!/usr/bin/env python3 +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. + +import argparse +import pathlib +import shutil +import subprocess + +SCRIPT_DIR = pathlib.Path(__file__).parent.resolve() +DEFAULT_OPS_CONFIG_RELATIVE_PATH = "tools/ci_build/github/android/mobile_package.required_operators.config" +DEFAULT_BUILD_SETTINGS_RELATIVE_PATH = "tools/ci_build/github/android/default_mobile_aar_build_settings.json" + + +def parse_args(): + parser = argparse.ArgumentParser( + description="""Builds a custom ONNX Runtime Android package. + This script first builds a Docker image with the ONNX Runtime Android build environment + dependencies. Then, from a Docker container with that image, it calls the ONNX Runtime build + scripts to build a custom Android package. The resulting package will be under + /output/aar_out. See https://onnxruntime.ai/docs/build/custom.html for more + information about custom builds.""") + + parser.add_argument("working_dir", type=pathlib.Path, + help="The directory used to store intermediate and output files.") + + parser.add_argument("--onnxruntime_branch_or_tag", + help="The ONNX Runtime branch or tag to build. " + "Supports branches and tags starting from 1.11 (branch rel-1.11.0 or tag v1.11.0). " + "If unspecified, builds the latest.") + + parser.add_argument("--include_ops_by_config", type=pathlib.Path, + help="The configuration file specifying which ops to include. " + "Such a configuration file is generated during ONNX to ORT format model conversion. " + f"The default is {DEFAULT_OPS_CONFIG_RELATIVE_PATH} in the ONNX Runtime repo.") + + parser.add_argument("--build_settings", type=pathlib.Path, + help="The configuration file specifying the build.py options. " + f"The default is {DEFAULT_BUILD_SETTINGS_RELATIVE_PATH} in the ONNX Runtime repo.") + + default_config = "Release" + parser.add_argument("--config", choices=["Debug", "MinSizeRel", "Release", "RelWithDebInfo"], + default=default_config, + help="The build configuration. " + f"The default is {default_config}.") + + default_docker_image_tag = "onnxruntime-android-custom-build:latest" + parser.add_argument("--docker_image_tag", default=default_docker_image_tag, + help="The tag for the Docker image. " + f"The default is {default_docker_image_tag}.") + + parser.add_argument("--docker_path", default=shutil.which("docker"), + help="The path to docker. If unspecified, docker should be in PATH.") + + args = parser.parse_args() + + if args.docker_path is None: + raise ValueError("Unable to determine docker path. Please provide it with --docker_path.") + + return args + + +def main(): + args = parse_args() + + docker_build_args = ["--build-arg", f"ONNXRUNTIME_BRANCH_OR_TAG={args.onnxruntime_branch_or_tag}"] \ + if args.onnxruntime_branch_or_tag else [] + + docker_build_cmd = [args.docker_path, "build", + "--tag", args.docker_image_tag, + "--file", str(SCRIPT_DIR / "Dockerfile"), + ] + docker_build_args + [str(SCRIPT_DIR)] + + subprocess.run(docker_build_cmd, check=True) + + working_dir = args.working_dir + working_dir.mkdir(parents=True, exist_ok=True) + working_dir = working_dir.resolve() + + # copy over any custom build configuration files + config_files = [f for f in [args.include_ops_by_config, args.build_settings] if f] + if config_files: + input_dir = working_dir / "input" + input_dir.mkdir(exist_ok=True) + for config_file in config_files: + shutil.copy(config_file, input_dir) + + output_dir = working_dir / "output" + output_dir.mkdir(exist_ok=True) + + container_ops_config_file = \ + f"/workspace/shared/input/{args.include_ops_by_config.name}" if args.include_ops_by_config \ + else f"/workspace/onnxruntime/{DEFAULT_OPS_CONFIG_RELATIVE_PATH}" + + container_build_settings_file =\ + f"/workspace/shared/input/{args.build_settings.name}" if args.build_settings \ + else f"/workspace/onnxruntime/{DEFAULT_BUILD_SETTINGS_RELATIVE_PATH}" + + docker_run_cmd = [args.docker_path, "run", + "--rm", "-it", + f"--volume={str(working_dir)}:/workspace/shared", + args.docker_image_tag, + "/usr/bin/env", "python3", + "/workspace/onnxruntime/tools/ci_build/github/android/build_aar_package.py", + "--build_dir=/workspace/shared/output", + f"--config={args.config}", + f"--include_ops_by_config={container_ops_config_file}", + container_build_settings_file, + ] + + subprocess.run(docker_run_cmd, check=True) + + print("Finished building Android package at '{}'.".format(output_dir / "aar_out")) + + +if __name__ == "__main__": + main() diff --git a/tools/android_custom_build/readme.md b/tools/android_custom_build/readme.md new file mode 100644 index 0000000000..4cfc7b6282 --- /dev/null +++ b/tools/android_custom_build/readme.md @@ -0,0 +1,9 @@ +This directory contains helper files for building a custom ONNX Runtime Android package. It can be copied outside of the ONNX Runtime repo and used independently. + +Run the ./build_custom_android_package.py script. Use the --help option for more information. + +Prerequisites: +- Python 3.6+ +- Docker + +See https://onnxruntime.ai/docs/build/custom.html for more information about creating and using custom builds. diff --git a/tools/ci_build/github/android/build_aar_package.py b/tools/ci_build/github/android/build_aar_package.py index 44580555a5..efaf4579a9 100644 --- a/tools/ci_build/github/android/build_aar_package.py +++ b/tools/ci_build/github/android/build_aar_package.py @@ -79,6 +79,7 @@ def _parse_build_settings(args): def _build_aar(args): build_settings = _parse_build_settings(args) build_dir = os.path.abspath(args.build_dir) + ops_config_path = os.path.abspath(args.include_ops_by_config) if args.include_ops_by_config else None # Setup temp environment for building temp_env = os.environ.copy() @@ -91,9 +92,7 @@ def _build_aar(args): aar_dir = os.path.join(intermediates_dir, 'aar', build_config) jnilibs_dir = os.path.join(intermediates_dir, 'jnilibs', build_config) exe_dir = os.path.join(intermediates_dir, 'executables', build_config) - base_build_command = [ - sys.executable, BUILD_PY, '--config=' + build_config - ] + build_settings['build_params'] + base_build_command = [sys.executable, BUILD_PY] + build_settings['build_params'] + ['--config=' + build_config] header_files_path = '' # Build binary for each ABI, one by one @@ -104,8 +103,8 @@ def _build_aar(args): '--build_dir=' + abi_build_dir ] - if args.include_ops_by_config is not None: - abi_build_command += ['--include_ops_by_config=' + args.include_ops_by_config] + if ops_config_path is not None: + abi_build_command += ['--include_ops_by_config=' + ops_config_path] subprocess.run(abi_build_command, env=temp_env, shell=False, check=True, cwd=REPO_DIR) diff --git a/tools/ci_build/github/apple/build_ios_framework.py b/tools/ci_build/github/apple/build_ios_framework.py index 92e2053ccc..4447f2f3b8 100644 --- a/tools/ci_build/github/apple/build_ios_framework.py +++ b/tools/ci_build/github/apple/build_ios_framework.py @@ -104,7 +104,7 @@ def _build_package(args): # Temp dirs to hold building results intermediates_dir = os.path.join(build_dir, 'intermediates') build_config = args.config - base_build_command = [sys.executable, BUILD_PY, '--config=' + build_config] + build_settings['build_params'] + base_build_command = [sys.executable, BUILD_PY] + build_settings['build_params'] + ['--config=' + build_config] if args.include_ops_by_config is not None: base_build_command += ['--include_ops_by_config=' + str(args.include_ops_by_config.resolve())]