diff --git a/tools/ci_build/github/android/build_aar_and_copy_artifacts.sh b/tools/ci_build/github/android/build_aar_and_copy_artifacts.sh index dd2037faa1..e1efd1471a 100755 --- a/tools/ci_build/github/android/build_aar_and_copy_artifacts.sh +++ b/tools/ci_build/github/android/build_aar_and_copy_artifacts.sh @@ -1,22 +1,31 @@ #!/bin/bash # This script will run build_aar_package.py to build Android AAR and copy all the artifacts -# to a given folder for publishing to Maven Central +# to a given folder for publishing to Maven Central, or building nuget package # This script is intended to be used in CI build only set -e set -x export PATH=/opt/python/cp37-cp37m/bin:$PATH -# build the AAR package, using the build settings specified in -# /tools/ci_build/github/android/default_mobile_aar_build_settings.json -python3 /onnxruntime_src/tools/ci_build/github/android/build_aar_package.py \ - --build_dir /build \ - --config $BUILD_CONFIG \ - --android_sdk_path /android_home \ - --android_ndk_path /ndk_home \ - --include_ops_by_config /onnxruntime_src/tools/ci_build/github/android/mobile_package.required_operators.config \ - /onnxruntime_src/tools/ci_build/github/android/default_mobile_aar_build_settings.json +# build the AAR package, using the build settings under /home/onnxruntimedev/.build_settings/ +# if there is also include_ops_and_types.config exists in the same folder, use it to build with included ops/types +if [ -f "/home/onnxruntimedev/.build_settings/include_ops_and_types.config" ]; then + python3 /onnxruntime_src/tools/ci_build/github/android/build_aar_package.py \ + --build_dir /build \ + --config $BUILD_CONFIG \ + --android_sdk_path /android_home \ + --android_ndk_path /ndk_home \ + --include_ops_by_config /home/onnxruntimedev/.build_settings/include_ops_and_types.config \ + /home/onnxruntimedev/.build_settings/build_settings.json +else + python3 /onnxruntime_src/tools/ci_build/github/android/build_aar_package.py \ + --build_dir /build \ + --config $BUILD_CONFIG \ + --android_sdk_path /android_home \ + --android_ndk_path /ndk_home \ + /home/onnxruntimedev/.build_settings/build_settings.json +fi # Copy the built artifacts to give folder for publishing PACKAGE_NAME=onnxruntime-mobile diff --git a/tools/ci_build/github/android/default_full_aar_build_settings.json b/tools/ci_build/github/android/default_full_aar_build_settings.json new file mode 100644 index 0000000000..337f5e6ba9 --- /dev/null +++ b/tools/ci_build/github/android/default_full_aar_build_settings.json @@ -0,0 +1,19 @@ +{ + "build_abis": [ + "armeabi-v7a", + "arm64-v8a", + "x86", + "x86_64" + ], + "android_min_sdk_version": 21, + "android_target_sdk_version": 24, + "build_params": [ + "--android", + "--parallel", + "--cmake_generator=Ninja", + "--build_java", + "--build_shared_lib", + "--use_nnapi", + "--skip_tests" + ] +} \ No newline at end of file diff --git a/tools/ci_build/github/azure-pipelines/templates/android-java-api-aar-test.yml b/tools/ci_build/github/azure-pipelines/templates/android-java-api-aar-test.yml new file mode 100644 index 0000000000..5d156c0f9a --- /dev/null +++ b/tools/ci_build/github/azure-pipelines/templates/android-java-api-aar-test.yml @@ -0,0 +1,71 @@ +parameters: +- name: artifactName + displayName: Artifact Name + type: string + default: '' + +- name: job_name_suffix + displayName: job name + type: string + default: '' + +jobs: +- job: Final_AAR_Testing_Android_${{ parameters.job_name_suffix }} + workspace: + clean: all + pool: + vmImage: 'macOS-10.15' + variables: + - name: runCodesignValidationInjection + value: false + timeoutInMinutes: 60 + dependsOn: + - Android_Java_API_AAR_Packaging_${{ parameters.job_name_suffix }} + steps: + - template: set-version-number-variables-step.yml + + - task: DownloadPipelineArtifact@2 + displayName: 'Download Final Android AAR' + inputs: + buildType: 'current' + artifactName: '${{parameters.artifactName}}' + targetPath: '$(Build.BinariesDirectory)/final-android-aar' + + - script: /bin/bash tools/ci_build/github/android/setup_gradle_wrapper.sh $(Build.SourcesDirectory) + displayName: Setup gradle wrapper to use gradle 6.8.3 + + - script: | + python3 $(Build.SourcesDirectory)/tools/python/run_android_emulator.py \ + --android-sdk-root ${ANDROID_SDK_ROOT} \ + --create-avd --system-image "system-images;android-30;google_apis;x86_64" \ + --start --emulator-extra-args="-partition-size 4096" \ + --emulator-pid-file $(Build.BinariesDirectory)/emulator.pid + displayName: Start Android emulator + + - script: | + set -e -x + mkdir android_test + cd android_test + cp -av $(Build.SourcesDirectory)/java/src/test/android ./ + cd ./android + mkdir -p app/libs + cp $(Build.BinariesDirectory)/final-android-aar/onnxruntime-mobile-$(OnnxRuntimeVersion).aar app/libs/onnxruntime-mobile.aar + $(Build.SourcesDirectory)/java/gradlew --no-daemon clean connectedDebugAndroidTest + displayName: Run E2E test for Android + workingDirectory: $(Build.BinariesDirectory) + + - script: | + python3 $(Build.SourcesDirectory)/tools/python/run_android_emulator.py \ + --android-sdk-root ${ANDROID_SDK_ROOT} \ + --stop \ + --emulator-pid-file $(Build.BinariesDirectory)/emulator.pid + displayName: Stop Android emulator + condition: always() + + - template: component-governance-component-detection-steps.yml + parameters : + condition : 'succeeded' + + - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 + displayName: 'Clean Agent Directories' + condition: always() diff --git a/tools/ci_build/github/azure-pipelines/templates/android-java-api-aar.yml b/tools/ci_build/github/azure-pipelines/templates/android-java-api-aar.yml index 91e6380f01..e245ae278d 100644 --- a/tools/ci_build/github/azure-pipelines/templates/android-java-api-aar.yml +++ b/tools/ci_build/github/azure-pipelines/templates/android-java-api-aar.yml @@ -4,8 +4,28 @@ parameters: type: string default: 'Release' +- name: buildSettings + displayName: Path to the build setting file + type: string + default: '' + +- name: includedOpsConfig + displayName: Path to the included ops config file + type: string + default: '' + +- name: artifactName + displayName: Artifact Name + type: string + default: '' + +- name: job_name_suffix + displayName: job name + type: string + default: '' + jobs: -- job: Android_Java_API_AAR_Packaging_Pipeline +- job: Android_Java_API_AAR_Packaging_${{ parameters.job_name_suffix }} timeoutInMinutes: 120 workspace: clean: all @@ -39,13 +59,19 @@ jobs: displayName: Build Android AAR Packages inputs: script: | + set -e -x NDK_HOME=$(realpath $ANDROID_NDK_HOME) + mkdir $(Build.BinariesDirectory)/.build_settings + cp ${{parameters.buildSettings}} $(Build.BinariesDirectory)/.build_settings/build_settings.json + [ -f "${{parameters.includedOpsConfig}}" ] && \ + cp ${{parameters.includedOpsConfig}} $(Build.BinariesDirectory)/.build_settings/include_ops_and_types.config docker run --rm \ --volume $(Build.SourcesDirectory):/onnxruntime_src \ --volume $(Build.BinariesDirectory):/build \ --volume $ANDROID_HOME:/android_home \ --volume $NDK_HOME:/ndk_home \ --volume $(artifacts_directory):/home/onnxruntimedev/.artifacts \ + --volume $(Build.BinariesDirectory)/.build_settings:/home/onnxruntimedev/.build_settings \ -e NIGHTLY_BUILD \ -e BUILD_BUILDNUMBER \ -e BUILD_CONFIG=${{parameters.buildConfig}} \ @@ -57,4 +83,4 @@ jobs: - task: PublishBuildArtifacts@1 inputs: pathtoPublish: '$(artifacts_directory)' - artifactName: 'onnxruntime-android-aar' + artifactName: '${{parameters.artifactName}}' diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml index f3e2facb74..2394ea800f 100644 --- a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml +++ b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml @@ -171,6 +171,17 @@ jobs: - template: android-java-api-aar.yml parameters: buildConfig: 'Release' + buildSettings: '$(Build.SourcesDirectory)/tools/ci_build/github/android/default_mobile_aar_build_settings.json' + includedOpsConfig: '$(Build.SourcesDirectory)/tools/ci_build/github/android/mobile_package.required_operators.config' + artifactName: 'onnxruntime-android-mobile-aar' + job_name_suffix: 'Mobile' + +- template: android-java-api-aar.yml + parameters: + buildConfig: 'Release' + buildSettings: '$(Build.SourcesDirectory)/tools/ci_build/github/android/default_full_aar_build_settings.json' + artifactName: 'onnxruntime-android-full-aar' + job_name_suffix: 'Full' - template: win-ci.yml parameters: @@ -405,7 +416,7 @@ jobs: configuration: RelWithDebInfo arguments: '--configuration RelWithDebInfo -t:CreatePackage -p:Platform="Any CPU" -p:OnnxRuntimeBuildDirectory="$(Build.BinariesDirectory)" -p:OrtPackageId=$(OrtPackageId) -p:IsReleaseBuild=${{ parameters.IsReleaseBuild }}' workingDirectory: '$(Build.SourcesDirectory)\csharp' - + - task: CopyFiles@2 displayName: 'Copy nuget packages to: $(Build.ArtifactStagingDirectory)' inputs: @@ -535,23 +546,23 @@ jobs: echo changing directory to artifact download path cd $(Build.BinariesDirectory)/nuget-artifact/final-package echo processing nupkg - SETLOCAL EnableDelayedExpansion + SETLOCAL EnableDelayedExpansion FOR /R %%i IN (*.nupkg) do ( set filename=%%~ni IF NOT "!filename:~25,7!"=="Managed" ( echo processing %%~ni.nupkg copy %%~ni.nupkg %%~ni.zip echo copied to zip - echo listing lib files in the zip - REM use a single .csv file to put the data + echo listing lib files in the zip + REM use a single .csv file to put the data echo os,arch,build_config,size > $(Build.BinariesDirectory)\binary_size_data.txt 7z.exe l -slt %%~ni.zip runtimes\linux-x64\native\libonnxruntime.so | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo linux,x64,${{ parameters.buildVariant }},%%a >> $(Build.BinariesDirectory)\binary_size_data.txt 7z.exe l -slt %%~ni.zip runtimes\linux-aarch64\native\libonnxruntime.so | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo linux,aarch64,${{ parameters.buildVariant }},%%a >> $(Build.BinariesDirectory)\binary_size_data.txt 7z.exe l -slt %%~ni.zip runtimes\osx.10.14-x64\native\libonnxruntime.dylib | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo osx,x64,${{ parameters.buildVariant }},%%a >> $(Build.BinariesDirectory)\binary_size_data.txt 7z.exe l -slt %%~ni.zip runtimes\win-x64\native\onnxruntime.dll | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo win,x64,${{ parameters.buildVariant }},%%a >> $(Build.BinariesDirectory)\binary_size_data.txt - 7z.exe l -slt %%~ni.zip runtimes\win-x86\native\onnxruntime.dll | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo win,x86,${{ parameters.buildVariant }},%%a >> $(Build.BinariesDirectory)\binary_size_data.txt + 7z.exe l -slt %%~ni.zip runtimes\win-x86\native\onnxruntime.dll | findstr /R /C:"^Size = [0-9]*" | for /F "tokens=3" %%a in ('more') do if not "%%a" == "" echo win,x86,${{ parameters.buildVariant }},%%a >> $(Build.BinariesDirectory)\binary_size_data.txt ) - ) + ) - task: AzureCLI@2 displayName: 'Azure CLI' inputs: @@ -749,61 +760,9 @@ jobs: displayName: 'Clean Agent Directories' condition: always() -- job: Final_AAR_Testing_Android - workspace: - clean: all - pool: - vmImage: 'macOS-10.15' - variables: - - name: runCodesignValidationInjection - value: false - timeoutInMinutes: 60 - dependsOn: - - Android_Java_API_AAR_Packaging_Pipeline - steps: - - template: set-version-number-variables-step.yml +- template: android-java-api-aar-test.yml + parameters: + artifactName: 'onnxruntime-android-mobile-aar' + job_name_suffix: 'Mobile' - - task: DownloadPipelineArtifact@2 - displayName: 'Download Final Android AAR' - inputs: - buildType: 'current' - artifactName: 'onnxruntime-android-aar' - targetPath: '$(Build.BinariesDirectory)/final-android-aar' - - script: /bin/bash tools/ci_build/github/android/setup_gradle_wrapper.sh $(Build.SourcesDirectory) - displayName: Setup gradle wrapper to use gradle 6.8.3 - - - script: | - python3 $(Build.SourcesDirectory)/tools/python/run_android_emulator.py \ - --android-sdk-root ${ANDROID_SDK_ROOT} \ - --create-avd --system-image "system-images;android-30;google_apis;x86_64" \ - --start --emulator-extra-args="-partition-size 4096" \ - --emulator-pid-file $(Build.BinariesDirectory)/emulator.pid - displayName: Start Android emulator - - - script: | - set -e -x - mkdir android_test - cd android_test - cp -av $(Build.SourcesDirectory)/java/src/test/android ./ - cd ./android - mkdir -p app/libs - cp $(Build.BinariesDirectory)/final-android-aar/onnxruntime-mobile-$(OnnxRuntimeVersion).aar app/libs/onnxruntime-mobile.aar - $(Build.SourcesDirectory)/java/gradlew --no-daemon clean connectedDebugAndroidTest - displayName: Run E2E test for Android - workingDirectory: $(Build.BinariesDirectory) - - - script: | - python3 $(Build.SourcesDirectory)/tools/python/run_android_emulator.py \ - --android-sdk-root ${ANDROID_SDK_ROOT} \ - --stop \ - --emulator-pid-file $(Build.BinariesDirectory)/emulator.pid - displayName: Stop Android emulator - condition: always() - - - template: component-governance-component-detection-steps.yml - parameters : - condition : 'succeeded' - - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 - displayName: 'Clean Agent Directories' - condition: always()