From fce67e2b9be965e0b5dc98012f1cb477aa15f002 Mon Sep 17 00:00:00 2001 From: Guoyu Wang <62914304+gwang-msft@users.noreply.github.com> Date: Mon, 12 Apr 2021 17:56:25 -0700 Subject: [PATCH] Create Android Package pipeline (#7295) * Create Android Package pipeline * adress CR comments * Switch to jdk11 --- java/build-android.gradle | 4 +- .../android/build_aar_and_copy_artifacts.sh | 26 ++++++++ .../github/android/build_aar_package.py | 28 +++++---- .../default_mobile_aar_build_settings.json | 3 +- .../c-api-noopenmp-packaging-pipelines.yml | 18 +++--- .../templates/android-java-api-aar.yml | 60 +++++++++++++++++++ .../scripts/manylinux/install_centos.sh | 6 +- 7 files changed, 119 insertions(+), 26 deletions(-) create mode 100644 tools/ci_build/github/android/build_aar_and_copy_artifacts.sh create mode 100644 tools/ci_build/github/azure-pipelines/templates/android-java-api-aar.yml diff --git a/java/build-android.gradle b/java/build-android.gradle index 8e6762602d..ddef02ac8b 100644 --- a/java/build-android.gradle +++ b/java/build-android.gradle @@ -125,7 +125,9 @@ publishing { pom { name = 'onnx-runtime' - description = 'ONNX Runtime is a performance-focused inference engine for ONNX (Open Neural Network Exchange) models.' + description = 'ONNX Runtime Mobile package is a size optimized inference library for executing ' + + 'ONNX (Open Neural Network Exchange) models. This package is built from the open source ' + + 'inference engine targeting reduced disk footprint for mobile platforms.' url = 'https://microsoft.github.io/onnxruntime/' licenses { license { diff --git a/tools/ci_build/github/android/build_aar_and_copy_artifacts.sh b/tools/ci_build/github/android/build_aar_and_copy_artifacts.sh new file mode 100644 index 0000000000..a067649366 --- /dev/null +++ b/tools/ci_build/github/android/build_aar_and_copy_artifacts.sh @@ -0,0 +1,26 @@ +#!/bin/bash + +# This script will run build_aar_package.py to build Android AAR and copy all the artifacts +# to a given folder for publishing to Maven Central +# This script is intended to be used in CI build only + +set -e +set -x +export PATH=/opt/python/cp37-cp37m/bin:$PATH + +# build the AAR package, using the build settings specified in +# /tools/ci_build/github/android/default_mobile_aar_build_settings.json +python3 /onnxruntime_src/tools/ci_build/github/android/build_aar_package.py \ + --build_dir /build \ + --config $BUILD_CONFIG \ + --android_sdk_path /android_home \ + --android_ndk_path /ndk_home \ + /onnxruntime_src/tools/ci_build/github/android/default_mobile_aar_build_settings.json + +# Copy the built artifacts to give folder for publishing +PACKAGE_NAME=onnxruntime-mobile +BASE_PATH=/build/aar_out/${BUILD_CONFIG}/com/microsoft/onnxruntime/${PACKAGE_NAME}/${ORT_VERSION} +cp ${BASE_PATH}/${PACKAGE_NAME}-${ORT_VERSION}-javadoc.jar /home/onnxruntimedev/.artifacts +cp ${BASE_PATH}/${PACKAGE_NAME}-${ORT_VERSION}-sources.jar /home/onnxruntimedev/.artifacts +cp ${BASE_PATH}/${PACKAGE_NAME}-${ORT_VERSION}.aar /home/onnxruntimedev/.artifacts +cp ${BASE_PATH}/${PACKAGE_NAME}-${ORT_VERSION}.pom /home/onnxruntimedev/.artifacts diff --git a/tools/ci_build/github/android/build_aar_package.py b/tools/ci_build/github/android/build_aar_package.py index 6c4ff7fcfe..f903e19382 100644 --- a/tools/ci_build/github/android/build_aar_package.py +++ b/tools/ci_build/github/android/build_aar_package.py @@ -19,8 +19,11 @@ from util import is_windows # noqa: E402 # We by default will build all 4 ABIs DEFAULT_BUILD_ABIS = ["armeabi-v7a", "arm64-v8a", "x86", "x86_64"] -# Android API 21 is the lowest API version we support -DEFAULT_ANDROID_MIN_SDK_VER = 21 + +# Android API levle 24 is the lowest API version officially supported, based on Microsoft 1CS +# It is possible to build from source using API level 21 and higher as the minimal SDK version +DEFAULT_ANDROID_MIN_SDK_VER = 24 + # Android API 28 is the default target API version for Android builds DEFAULT_ANDROID_TARGET_SDK_VER = 28 @@ -36,11 +39,6 @@ def _parse_build_settings(args): build_settings = {} - if 'build_flavor' in _build_settings_data: - build_settings['build_flavor'] = _build_settings_data['build_flavor'] - else: - raise ValueError('build_flavor is required in the build config file') - if 'build_abis' in _build_settings_data: build_settings['build_abis'] = _build_settings_data['build_abis'] else: @@ -84,11 +82,11 @@ def _build_aar(args): # Temp dirs to hold building results _intermediates_dir = os.path.join(build_dir, 'intermediates') - _build_flavor = build_settings['build_flavor'] - _aar_dir = os.path.join(_intermediates_dir, 'aar', _build_flavor) - _jnilibs_dir = os.path.join(_intermediates_dir, 'jnilibs', _build_flavor) + _build_config = args.config + _aar_dir = os.path.join(_intermediates_dir, 'aar', _build_config) + _jnilibs_dir = os.path.join(_intermediates_dir, 'jnilibs', _build_config) _base_build_command = [ - 'python3', BUILD_PY, '--config=' + _build_flavor + sys.executable, BUILD_PY, '--config=' + _build_config ] + build_settings['build_params'] # Build binary for each ABI, one by one @@ -115,10 +113,10 @@ def _build_aar(args): # add double check with os.path.islink if os.path.exists(_target_lib_name) or os.path.islink(_target_lib_name): os.remove(_target_lib_name) - os.symlink(os.path.join(_build_dir, _build_flavor, lib_name), _target_lib_name) + os.symlink(os.path.join(_build_dir, _build_config, lib_name), _target_lib_name) # The directory to publish final AAR - _aar_publish_dir = os.path.join(build_dir, 'aar_out', _build_flavor) + _aar_publish_dir = os.path.join(build_dir, 'aar_out', _build_config) os.makedirs(_aar_publish_dir, exist_ok=True) # get the common gradle command args @@ -166,6 +164,10 @@ def parse_args(): "--include_ops_by_config", type=str, help="Include ops from config file. See /docs/Reduced_Operator_Kernel_build.md for more information.") + parser.add_argument("--config", type=str, default="Release", + choices=["Debug", "MinSizeRel", "Release", "RelWithDebInfo"], + help="Configuration to build.") + parser.add_argument('build_settings_file', type=pathlib.Path, help='Provide the file contains settings for building AAR') diff --git a/tools/ci_build/github/android/default_mobile_aar_build_settings.json b/tools/ci_build/github/android/default_mobile_aar_build_settings.json index 250fd91a49..02cea4024e 100644 --- a/tools/ci_build/github/android/default_mobile_aar_build_settings.json +++ b/tools/ci_build/github/android/default_mobile_aar_build_settings.json @@ -5,8 +5,7 @@ "x86", "x86_64" ], - "build_flavor": "Release", - "android_min_sdk_version": 21, + "android_min_sdk_version": 24, "android_target_sdk_version": 28, "build_params": [ "--android", diff --git a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml index 8d2ccd2ee6..16ed2e333f 100644 --- a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml +++ b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml @@ -39,10 +39,10 @@ jobs: clean: all timeoutInMinutes: 120 pool: 'Onnxruntime-Linux-GPU' - variables: + variables: CUDA_VERSION: '11.0' steps: - - template: templates/set-version-number-variables-step.yml + - template: templates/set-version-number-variables-step.yml - template: templates/get-docker-image-steps.yml parameters: Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11 @@ -99,6 +99,10 @@ jobs: buildJava: true java_artifact_id: onnxruntime_gpu +- template: templates/android-java-api-aar.yml + parameters: + buildConfig: 'Release' + - job: Jar_Packaging_GPU workspace: clean: all @@ -110,7 +114,7 @@ jobs: steps: - checkout: self submodules: false - - template: templates/set-version-number-variables-step.yml + - template: templates/set-version-number-variables-step.yml - task: DownloadPipelineArtifact@2 displayName: 'Download Pipeline Artifact - Win x64' @@ -155,7 +159,7 @@ jobs: - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 displayName: 'Clean Agent Directories' condition: always() - + - job: Final_Jar_Testing_Windows_GPU workspace: @@ -168,7 +172,7 @@ jobs: dependsOn: Jar_Packaging_GPU steps: - - template: templates/set-version-number-variables-step.yml + - template: templates/set-version-number-variables-step.yml - task: BatchScript@1 displayName: 'setup env' @@ -187,7 +191,7 @@ jobs: - task: CmdLine@2 inputs: script: | - mkdir test + mkdir test pushd test jar xf $(Build.BinariesDirectory)\final-jar\testing.jar popd @@ -216,7 +220,7 @@ jobs: steps: - checkout: self submodules: false - - template: templates/set-version-number-variables-step.yml + - template: templates/set-version-number-variables-step.yml - task: DownloadPipelineArtifact@2 displayName: 'Download Final Jar' inputs: diff --git a/tools/ci_build/github/azure-pipelines/templates/android-java-api-aar.yml b/tools/ci_build/github/azure-pipelines/templates/android-java-api-aar.yml new file mode 100644 index 0000000000..91e6380f01 --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/templates/android-java-api-aar.yml @@ -0,0 +1,60 @@ +parameters: +- name: buildConfig + displayName: Configuration to build + type: string + default: 'Release' + +jobs: +- job: Android_Java_API_AAR_Packaging_Pipeline + timeoutInMinutes: 120 + workspace: + clean: all + pool: Linux-CPU + + variables: + artifacts_directory: $(Build.BinariesDirectory)/.artifacts + + steps: + - checkout: self + clean: true + submodules: recursive + + - task: CmdLine@2 + displayName: Create artifacts directory + inputs: + script: | + # Create a folder for artifacts + mkdir -p $(artifacts_directory) + workingDirectory: $(Build.BinariesDirectory) + + - template: set-version-number-variables-step.yml + - template: get-docker-image-steps.yml + parameters: + Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cpu + Context: tools/ci_build/github/linux/docker + DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )" + Repository: onnxruntimecpubuild + + - task: CmdLine@2 + displayName: Build Android AAR Packages + inputs: + script: | + NDK_HOME=$(realpath $ANDROID_NDK_HOME) + docker run --rm \ + --volume $(Build.SourcesDirectory):/onnxruntime_src \ + --volume $(Build.BinariesDirectory):/build \ + --volume $ANDROID_HOME:/android_home \ + --volume $NDK_HOME:/ndk_home \ + --volume $(artifacts_directory):/home/onnxruntimedev/.artifacts \ + -e NIGHTLY_BUILD \ + -e BUILD_BUILDNUMBER \ + -e BUILD_CONFIG=${{parameters.buildConfig}} \ + -e ORT_VERSION=$(OnnxRuntimeVersion) \ + onnxruntimecpubuild \ + /bin/bash /onnxruntime_src/tools/ci_build/github/android/build_aar_and_copy_artifacts.sh + workingDirectory: $(Build.SourcesDirectory) + + - task: PublishBuildArtifacts@1 + inputs: + pathtoPublish: '$(artifacts_directory)' + artifactName: 'onnxruntime-android-aar' diff --git a/tools/ci_build/github/linux/docker/scripts/manylinux/install_centos.sh b/tools/ci_build/github/linux/docker/scripts/manylinux/install_centos.sh index 28ddc78cd1..b7e14a6098 100755 --- a/tools/ci_build/github/linux/docker/scripts/manylinux/install_centos.sh +++ b/tools/ci_build/github/linux/docker/scripts/manylinux/install_centos.sh @@ -19,9 +19,9 @@ if [ "$os_major_version" == "7" ]; then yum install -y dotnet-sdk-2.1 fi -yum install -y java-1.8.0-openjdk-devel +yum install -y java-11-openjdk-devel -#If the /opt/python folder exists, we assume this is the manylinux docker image +#If the /opt/python folder exists, we assume this is the manylinux docker image if [ ! -d "/opt/python/cp37-cp37m" ]; then - yum install -y ccache gcc gcc-c++ python3 python3-devel python3-pip + yum install -y ccache gcc gcc-c++ python3 python3-devel python3-pip fi