From 6a1939252fad1ca2917b29a3530c7fdcd28ca368 Mon Sep 17 00:00:00 2001 From: Guoyu Wang <62914304+gwang-msft@users.noreply.github.com> Date: Fri, 27 Aug 2021 15:58:56 -0700 Subject: [PATCH] Fix Android java API failure (#8865) * Fix Android Package break * Without java fix -- pipeline should fail * With java fix, should pass now * address CR comments --- cmake/onnxruntime_java.cmake | 2 +- java/build-android.gradle | 4 +- .../main/java/ai/onnxruntime/OnnxRuntime.java | 4 + java/src/test/android/app/build.gradle | 2 +- java/src/test/android/build.gradle | 4 +- tools/ci_build/build.py | 26 +++--- .../c-api-noopenmp-packaging-pipelines.yml | 14 ++-- .../azure-pipelines/templates/c-api-cpu.yml | 80 ++++++++++++++++--- 8 files changed, 99 insertions(+), 37 deletions(-) diff --git a/cmake/onnxruntime_java.cmake b/cmake/onnxruntime_java.cmake index 4a05a3af06..afc8ed1bf1 100644 --- a/cmake/onnxruntime_java.cmake +++ b/cmake/onnxruntime_java.cmake @@ -206,7 +206,7 @@ if (CMAKE_SYSTEM_NAME STREQUAL "Android") set(ANDROID_TEST_PACKAGE_LIB_DIR ${ANDROID_TEST_PACKAGE_DIR}/app/libs) file(MAKE_DIRECTORY ${ANDROID_TEST_PACKAGE_LIB_DIR}) # Copy the built Android AAR package to libs folder of our test app - add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different ${ANDROID_PACKAGE_OUTPUT_DIR}/outputs/aar/onnxruntime-debug.aar ${ANDROID_TEST_PACKAGE_LIB_DIR}/onnxruntime-debug.aar) + add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${CMAKE_COMMAND} -E copy_if_different ${ANDROID_PACKAGE_OUTPUT_DIR}/outputs/aar/onnxruntime-debug.aar ${ANDROID_TEST_PACKAGE_LIB_DIR}/onnxruntime-mobile.aar) # Build Android test apk for java package add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${GRADLE_EXECUTABLE} clean WORKING_DIRECTORY ${ANDROID_TEST_PACKAGE_DIR}) add_custom_command(TARGET onnxruntime4j_jni POST_BUILD COMMAND ${GRADLE_EXECUTABLE} assembleDebug assembleDebugAndroidTest -DminSdkVer=${ANDROID_MIN_SDK} WORKING_DIRECTORY ${ANDROID_TEST_PACKAGE_DIR}) diff --git a/java/build-android.gradle b/java/build-android.gradle index bde9fa5327..1d1d874db6 100644 --- a/java/build-android.gradle +++ b/java/build-android.gradle @@ -29,7 +29,7 @@ def mavenArtifactId = project.name + '-mobile' buildscript { repositories { google() - jcenter() + mavenCentral() } dependencies { classpath 'com.android.tools.build:gradle:3.5.3' @@ -42,7 +42,7 @@ buildscript { allprojects { repositories { google() - jcenter() + mavenCentral() } } diff --git a/java/src/main/java/ai/onnxruntime/OnnxRuntime.java b/java/src/main/java/ai/onnxruntime/OnnxRuntime.java index e48038ad18..4a2f9269fd 100644 --- a/java/src/main/java/ai/onnxruntime/OnnxRuntime.java +++ b/java/src/main/java/ai/onnxruntime/OnnxRuntime.java @@ -207,6 +207,10 @@ final class OnnxRuntime { * @return True if the library is ready for loading by ORT's native code, false otherwise. */ static synchronized boolean extractProviderLibrary(String libraryName) { + // Android does not need to extract library and it has no shared provider library + if (isAndroid()) { + return false; + } // Check if we've already extracted or check this provider, and it's ready if (extractedSharedProviders.contains(libraryName)) { return true; diff --git a/java/src/test/android/app/build.gradle b/java/src/test/android/app/build.gradle index bdf9a5d02b..416d8c96ed 100644 --- a/java/src/test/android/app/build.gradle +++ b/java/src/test/android/app/build.gradle @@ -44,5 +44,5 @@ dependencies { testImplementation 'junit:junit:4.+' androidTestImplementation 'androidx.test.ext:junit:1.1.2' androidTestImplementation 'androidx.test.espresso:espresso-core:3.3.0' - implementation(name: "onnxruntime-debug", ext: "aar") + implementation(name: "onnxruntime-mobile", ext: "aar") } diff --git a/java/src/test/android/build.gradle b/java/src/test/android/build.gradle index a79b514cea..0f448b3f1e 100644 --- a/java/src/test/android/build.gradle +++ b/java/src/test/android/build.gradle @@ -4,7 +4,7 @@ buildscript { repositories { google() - jcenter() + mavenCentral() } dependencies { classpath "com.android.tools.build:gradle:3.5.3" @@ -17,7 +17,7 @@ buildscript { allprojects { repositories { google() - jcenter() + mavenCentral() flatDir{dirs 'libs'} } } diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index d964545a35..ee8f60b365 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -1295,20 +1295,20 @@ def run_android_tests(args, source_dir, build_dir, config, cwd): adb_push('onnx_test_runner', device_dir, cwd=cwd) adb_shell('chmod +x {}/onnx_test_runner'.format(device_dir)) run_adb_shell('{0}/onnxruntime_test_all'.format(device_dir)) + if args.build_java: - adb_install( - os.path.join( - get_config_build_dir(build_dir, config), - "java", "androidtest", "android", "app", "build", "outputs", "apk", - "debug", "app-debug.apk")) - adb_install( - os.path.join( - get_config_build_dir(build_dir, config), - "java", "androidtest", "android", "app", "build", "outputs", "apk", - "androidTest", "debug", "app-debug-androidTest.apk")) - adb_shell( - 'am instrument -w ai.onnxruntime.example.javavalidator.test/androidx.test.runner.AndroidJUnitRunner' - ) + gradle_executable = 'gradle' + # use the gradle wrapper if it exists, the gradlew should be setup under /java + gradlew_path = os.path.join(source_dir, 'java', + 'gradlew.bat' if is_windows() else 'gradlew') + if os.path.exists(gradlew_path): + gradle_executable = gradlew_path + android_test_path = os.path.join(cwd, "java", "androidtest", "android") + run_subprocess([gradle_executable, '--no-daemon', + '-DminSdkVer={}'.format(args.android_api), + 'clean', 'connectedDebugAndroidTest'], + cwd=android_test_path) + if args.use_nnapi: adb_shell('cd {0} && {0}/onnx_test_runner -e nnapi {0}/test'.format(device_dir)) else: diff --git a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml index 901f2f32c0..f154cb4513 100644 --- a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml +++ b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml @@ -105,20 +105,16 @@ jobs: ort_build_pool_name: 'onnxruntime-gpu-tensorrt8-winbuild' DoCompliance: ${{ parameters.DoCompliance }} DoEsrp: ${{ parameters.DoEsrp }} - job_name_suffix: tensorrt + job_name_suffix: tensorrt EnvSetupScript: setup_env_gpu.bat buildArch: x64 msbuildPlatform: x64 packageName: x64-tensorrt buildparameter: --use_tensorrt --tensorrt_home="C:\local\TensorRT-8.0.1.6.Windows10.x86_64.cuda-11.3.cudnn8.2" --cuda_version=11.1 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.1" --cudnn_home="C:\local\cudnn-11.4-windows-x64-v8.2.2.26\cuda" --enable_onnx_tests --enable_wcos --build_java --cmake_extra_defines "CMAKE_CUDA_ARCHITECTURES=37;52;60;61;70;75;80" runTests: ${{ parameters.RunOnnxRuntimeTests }} - buildJava: true + buildJava: true java_artifact_id: onnxruntime_gpu -- template: templates/android-java-api-aar.yml - parameters: - buildConfig: 'Release' - - job: Jar_Packaging_GPU workspace: clean: all @@ -297,7 +293,7 @@ jobs: - task: ArchiveFiles@2 inputs: rootFolderOrFile: '$(Build.BinariesDirectory)/tgz-artifacts/onnxruntime-linux-x64-gpu' - includeRootFolder: false + includeRootFolder: false archiveType: 'tar' # Options: zip, 7z, tar, wim tarCompression: 'gz' archiveFile: '$(Build.ArtifactStagingDirectory)/onnxruntime-linux-x64-gpu-$(OnnxRuntimeVersion).tgz' @@ -535,14 +531,14 @@ jobs: - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 displayName: 'Clean Agent Directories' condition: always() - + - template: nuget/templates/test_win.yml parameters: AgentPool : 'onnxruntime-gpu-tensorrt8-winbuild' NugetPackageName : 'Microsoft.ML.OnnxRuntime.Gpu' ArtifactSuffix: 'GPU' Skipx86Tests: 'true' - + - template: nuget/templates/test_linux.yml parameters: AgentPool : Onnxruntime-Linux-GPU diff --git a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml index f0f2a74130..6e125403c7 100644 --- a/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml +++ b/tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml @@ -69,9 +69,9 @@ jobs: workingDirectory: $(Build.SourcesDirectory) - script: | - set -e -x + set -e -x mkdir $(Build.ArtifactStagingDirectory)/testdata - cp $(Build.BinariesDirectory)/Release/libcustom_op_library.so* $(Build.ArtifactStagingDirectory)/testdata + cp $(Build.BinariesDirectory)/Release/libcustom_op_library.so* $(Build.ArtifactStagingDirectory)/testdata ls -al $(Build.ArtifactStagingDirectory) displayName: 'Create Artifacts' @@ -110,7 +110,7 @@ jobs: clean: all variables: MACOSX_DEPLOYMENT_TARGET: '10.13' - pool: + pool: vmImage: 'macOS-10.15' timeoutInMinutes: 120 steps: @@ -141,7 +141,7 @@ jobs: commitId: $(OnnxRuntimeGitCommitHash) - script: | - set -e -x + set -e -x mkdir $(Build.ArtifactStagingDirectory)/testdata cp $(Build.BinariesDirectory)/Release/libcustom_op_library.dylib $(Build.ArtifactStagingDirectory)/testdata displayName: 'Create Artifacts' @@ -168,6 +168,9 @@ jobs: displayName: 'Clean Agent Directories' condition: always() +- template: android-java-api-aar.yml + parameters: + buildConfig: 'Release' - template: win-ci.yml parameters: @@ -259,7 +262,7 @@ jobs: buildType: 'current' artifactName: 'drop-onnxruntime-java-osx-x64' targetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-osx-x64' - + - task: PowerShell@2 displayName: 'PowerShell Script' inputs: @@ -319,7 +322,7 @@ jobs: inputs: artifactName: 'onnxruntime-win-x86' targetPath: '$(Build.BinariesDirectory)/nuget-artifact' - + - task: DownloadPipelineArtifact@0 displayName: 'Download Pipeline Artifact - NuGet' inputs: @@ -619,7 +622,7 @@ jobs: - task: CmdLine@2 inputs: script: | - mkdir test + mkdir test pushd test jar xf $(Build.BinariesDirectory)\final-jar\testing.jar popd @@ -679,7 +682,7 @@ jobs: - job: Final_Jar_Testing_MacOs workspace: clean: all - pool: + pool: vmImage: 'macOS-10.15' variables: - name: runCodesignValidationInjection @@ -718,4 +721,63 @@ jobs: condition : 'succeeded' - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 displayName: 'Clean Agent Directories' - condition: always() \ No newline at end of file + condition: always() + +- job: Final_AAR_Testing_Android + workspace: + clean: all + pool: + vmImage: 'macOS-10.15' + variables: + - name: runCodesignValidationInjection + value: false + timeoutInMinutes: 60 + dependsOn: + - Android_Java_API_AAR_Packaging_Pipeline + steps: + - template: set-version-number-variables-step.yml + + - task: DownloadPipelineArtifact@2 + displayName: 'Download Final Android AAR' + inputs: + buildType: 'current' + artifactName: 'onnxruntime-android-aar' + targetPath: '$(Build.BinariesDirectory)/final-android-aar' + + - script: /bin/bash tools/ci_build/github/android/setup_gradle_wrapper.sh $(Build.SourcesDirectory) + displayName: Setup gradle wrapper to use gradle 6.8.3 + + - script: | + python3 $(Build.SourcesDirectory)/tools/python/run_android_emulator.py \ + --android-sdk-root ${ANDROID_SDK_ROOT} \ + --create-avd --system-image "system-images;android-30;google_apis;x86_64" \ + --start --emulator-extra-args="-partition-size 4096" \ + --emulator-pid-file $(Build.BinariesDirectory)/emulator.pid + displayName: Start Android emulator + + - script: | + set -e -x + mkdir android_test + cd android_test + cp -av $(Build.SourcesDirectory)/java/src/test/android ./ + cd ./android + mkdir -p app/libs + cp $(Build.BinariesDirectory)/final-android-aar/onnxruntime-mobile-$(OnnxRuntimeVersion).aar app/libs/onnxruntime-mobile.aar + $(Build.SourcesDirectory)/java/gradlew --no-daemon clean connectedDebugAndroidTest + displayName: Run E2E test for Android + workingDirectory: $(Build.BinariesDirectory) + + - script: | + python3 $(Build.SourcesDirectory)/tools/python/run_android_emulator.py \ + --android-sdk-root ${ANDROID_SDK_ROOT} \ + --stop \ + --emulator-pid-file $(Build.BinariesDirectory)/emulator.pid + displayName: Stop Android emulator + condition: always() + + - template: component-governance-component-detection-steps.yml + parameters : + condition : 'succeeded' + - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 + displayName: 'Clean Agent Directories' + condition: always()