Add Android QNN Browserstack test (#22434)

Add Android QNN Browserstack test



### Motivation and Context
Real device test in CI

java/src/test/android/README.md

@@ -29,6 +29,11 @@ Use the android's [build instructions](https://onnxruntime.ai/docs/build/android
 
 Please note that you may need to set the `--android_abi=x86_64` (the default option is `arm64-v8a`). This is because android instrumentation test is run on an android emulator which requires an abi of `x86_64`.
 
+#### QNN Builds
+We use two AndroidManifest.xml files to manage different runtime requirements for QNN support. In the [build configuration](app/build.gradle), we specify which manifest file to use based on the qnnVersion.
+In the [QNN manifest](app/src/main/AndroidManifestQnn.xml), we include the <uses-native-library> declaration for libcdsprpc.so, which is required for devices using QNN and Qualcomm DSP capabilities.
+For QNN builds, it is also necessary to set the `ADSP_LIBRARY_PATH` environment variable to the [native library directory](https://developer.android.com/reference/android/content/pm/ApplicationInfo#nativeLibraryDir) depending on the device. This ensures that any native libraries downloaded as dependencies such as QNN libraries are found by the application. This is conditionally added by using the BuildConfig field IS_QNN_BUILD set in the build.gradle file.
+
 #### Build Output
 
 The build will generate two apks which is required to run the test application in `$YOUR_BUILD_DIR/java/androidtest/android/app/build/outputs/apk`:

java/src/test/android/app/build.gradle

@@ -4,6 +4,7 @@ plugins {
 }
 
 def minSdkVer = System.properties.get("minSdkVer")?:24
+def qnnVersion = System.properties['qnnVersion']
 
 android {
 	compileSdkVersion 32
@@ -16,6 +17,14 @@ android {
 		versionName "1.0"
 
 		testInstrumentationRunner "androidx.test.runner.AndroidJUnitRunner"
+
+		// Add BuildConfig field for qnnVersion
+		if (qnnVersion != null) {
+			buildConfigField "boolean", "IS_QNN_BUILD", "true"
+		}
+		else {
+			buildConfigField "boolean", "IS_QNN_BUILD", "false"
+		}
 	}
 
 	buildTypes {
@@ -31,6 +40,24 @@ android {
 	kotlinOptions {
 		jvmTarget = '1.8'
 	}
+	// Conditional packagingOptions for QNN builds only
+	if (qnnVersion != null)	{
+		packagingOptions  {
+			jniLibs {
+				useLegacyPackaging = true
+			}
+			// Dsp is used in older QC devices and not supported by ORT
+			// Gpu support isn't the target, we just want Npu support (Htp)
+			exclude 'lib/arm64-v8a/libQnnGpu.so'
+			exclude 'lib/arm64-v8a/libQnnDsp*.so'
+		}
+
+		sourceSets {
+			main {
+				manifest.srcFile 'src/main/AndroidManifestQnn.xml'  // Use QNN manifest
+				}
+			}
+	}
 	namespace 'ai.onnxruntime.example.javavalidator'
 }
 
@@ -44,9 +71,18 @@ dependencies {
 	testImplementation 'junit:junit:4.+'
 	androidTestImplementation 'androidx.test.ext:junit:1.1.3'
 	androidTestImplementation 'androidx.test.espresso:espresso-core:3.4.0'
-	implementation(name: "onnxruntime-android", ext: "aar")
 
 	androidTestImplementation 'androidx.test:runner:1.4.0'
 	androidTestImplementation 'androidx.test:rules:1.4.0'
 	androidTestImplementation 'com.microsoft.appcenter:espresso-test-extension:1.4'
+
+	// dependencies for onnxruntime-android-qnn
+	if (qnnVersion != null) {
+		implementation(name: "onnxruntime-android-qnn", ext: "aar")
+		implementation "com.qualcomm.qti:qnn-runtime:$qnnVersion"
+	}
+	else {
+		implementation(name: "onnxruntime-android", ext: "aar")
+	}
+
 }

java/src/test/android/app/src/androidTest/java/ai/onnxruntime/example/javavalidator/SimpleTest.kt

@@ -38,13 +38,18 @@ class SimpleTest {
     @Test
     fun runSigmoidModelTest() {
         for (intraOpNumThreads in 1..4) {
-            runSigmoidModelTestImpl(intraOpNumThreads)
+            runSigmoidModelTestImpl(intraOpNumThreads, OrtProvider.CPU)
         }
     }
 
     @Test
     fun runSigmoidModelTestNNAPI() {
-        runSigmoidModelTestImpl(1, true)
+        runSigmoidModelTestImpl(1, OrtProvider.NNAPI)
+    }
+
+    @Test
+    fun runSigmoidModelTestQNN() {
+        runSigmoidModelTestImpl(1, OrtProvider.QNN)
     }
 
     @Throws(IOException::class)
@@ -54,22 +59,49 @@ class SimpleTest {
     }
 
     @Throws(OrtException::class, IOException::class)
-    fun runSigmoidModelTestImpl(intraOpNumThreads: Int, useNNAPI: Boolean = false) {
-        reportHelper.label("Start Running Test with intraOpNumThreads=$intraOpNumThreads, useNNAPI=$useNNAPI")
+    fun runSigmoidModelTestImpl(intraOpNumThreads: Int, executionProvider: OrtProvider) {
+        reportHelper.label("Start Running Test with intraOpNumThreads=$intraOpNumThreads, executionProvider=$executionProvider")
         Log.println(Log.INFO, TAG, "Testing with intraOpNumThreads=$intraOpNumThreads")
-        Log.println(Log.INFO, TAG, "Testing with useNNAPI=$useNNAPI")
+        Log.println(Log.INFO, TAG, "Testing with executionProvider=$executionProvider")
+
         val env = OrtEnvironment.getEnvironment(OrtLoggingLevel.ORT_LOGGING_LEVEL_VERBOSE)
         env.use {
             val opts = SessionOptions()
             opts.setIntraOpNumThreads(intraOpNumThreads)
-            if (useNNAPI) {
-                if (OrtEnvironment.getAvailableProviders().contains(OrtProvider.NNAPI)) {
-                    opts.addNnapi()
-                } else {
-                    Log.println(Log.INFO, TAG, "NO NNAPI EP available, skip the test")
-                    return
+
+            when (executionProvider) {
+
+                OrtProvider.NNAPI -> {
+                    if (OrtEnvironment.getAvailableProviders().contains(OrtProvider.NNAPI)) {
+                        opts.addNnapi()
+                    } else {
+                        Log.println(Log.INFO, TAG, "NO NNAPI EP available, skip the test")
+                        return
+                    }
+                }
+
+                OrtProvider.QNN -> {
+                    if (OrtEnvironment.getAvailableProviders().contains(OrtProvider.QNN)) {
+                        // Since this is running in an Android environment, we use the .so library
+                        val qnnLibrary = "libQnnHtp.so"
+                        val providerOptions = Collections.singletonMap("backend_path", qnnLibrary)
+                        opts.addQnn(providerOptions)
+                    } else {
+                        Log.println(Log.INFO, TAG, "NO QNN EP available, skip the test")
+                        return
+                    }
+                }
+
+                OrtProvider.CPU -> {
+                    // No additional configuration is needed for CPU
+                }
+
+                else -> {
+                    //  Non exhaustive when statements on enum will be prohibited in future Gradle versions
+                    Log.println(Log.INFO, TAG, "Skipping test as OrtProvider is not implemented")
                 }
             }
+
             opts.use {
                 val session = env.createSession(readModel("sigmoid.ort"), opts)
                 session.use {
@@ -92,13 +124,15 @@ class SimpleTest {
                         output.use {
                             @Suppress("UNCHECKED_CAST")
                             val rawOutput = output[0].value as Array<Array<FloatArray>>
+                            // QNN EP will run the Sigmoid float32 op with fp16 precision
+                            val precision = if (executionProvider == OrtProvider.QNN) 1e-3 else 1e-6
                             for (i in 0..2) {
                                 for (j in 0..3) {
                                     for (k in 0..4) {
                                         Assert.assertEquals(
                                             rawOutput[i][j][k],
                                             expected[i][j][k],
-                                            1e-6.toFloat()
+                                            precision.toFloat()
                                         )
                                     }
                                 }

java/src/test/android/app/src/main/AndroidManifest.xml

@@ -17,4 +17,4 @@
         </activity>
     </application>
 
-</manifest>
+</manifest>

java/src/test/android/app/src/main/AndroidManifestQnn.xml

@@ -0,0 +1,23 @@
+<?xml version="1.0" encoding="utf-8"?>
+<manifest xmlns:android="http://schemas.android.com/apk/res/android">
+
+    <application
+        android:allowBackup="true"
+        android:icon="@mipmap/ic_launcher"
+        android:label="@string/app_name"
+        android:roundIcon="@mipmap/ic_launcher_round"
+        android:supportsRtl="true"
+        android:theme="@style/Theme.JavaValidator">
+        <activity android:name=".MainActivity" android:exported="true">
+            <intent-filter>
+                <action android:name="android.intent.action.MAIN" />
+
+                <category android:name="android.intent.category.LAUNCHER" />
+            </intent-filter>
+        </activity>
+        <uses-native-library
+            android:name="libcdsprpc.so"
+            android:required="false" />
+    </application>
+
+</manifest>

java/src/test/android/app/src/main/java/ai/onnxruntime/example/javavalidator/MainActivity.kt

@@ -1,11 +1,19 @@
 package ai.onnxruntime.example.javavalidator
 
 import android.os.Bundle
+import android.system.Os
 import androidx.appcompat.app.AppCompatActivity
 
 /*Empty activity app mainly used for testing*/
 class MainActivity : AppCompatActivity() {
     override fun onCreate(savedInstanceState: Bundle?) {
+        if (BuildConfig.IS_QNN_BUILD) {
+            val adspLibraryPath = applicationContext.applicationInfo.nativeLibraryDir
+            // set the path variable to the native library directory
+            // so that any native libraries downloaded as dependencies
+            // (like qnn libs) are found
+            Os.setenv("ADSP_LIBRARY_PATH", adspLibraryPath, true)
+        }
         super.onCreate(savedInstanceState)
     }
-}
+}

tools/ci_build/github/azure-pipelines/templates/android-java-api-aar-test.yml

@@ -19,6 +19,11 @@ parameters:
   type: string
   default: ''
 
+- name: QnnSDKVersion
+  displayName: QNN SDK Version
+  type: string
+  default: '2.28.0.241029'
+
 jobs:
 - job: Final_AAR_Testing_Android_${{ parameters.job_name_suffix }}
   workspace:
@@ -50,36 +55,61 @@ jobs:
 
   - template: use-android-ndk.yml
 
-  - template: use-android-emulator.yml
-    parameters:
-      create: true
-      start: true
-
   - script: |
-      set -e -x
-      mkdir android_test
-      cd android_test
-      cp -av $(Build.SourcesDirectory)/java/src/test/android ./
-      cd ./android
-      mkdir -p app/libs
-      cp $(Build.BinariesDirectory)/final-android-aar/${{parameters.packageName}}-$(OnnxRuntimeVersion)${{parameters.ReleaseVersionSuffix}}.aar app/libs/onnxruntime-android.aar
-      $(Build.SourcesDirectory)/java/gradlew --no-daemon clean connectedDebugAndroidTest --stacktrace
-    displayName: Run E2E test using Emulator
+        set -e -x
+        mkdir -p android_test/android/app/libs
+        cd android_test/android
+        cp -av $(Build.SourcesDirectory)/java/src/test/android/* ./
+        cp $(Build.BinariesDirectory)/final-android-aar/${{parameters.packageName}}-$(OnnxRuntimeVersion)${{parameters.ReleaseVersionSuffix}}.aar app/libs/${{parameters.packageName}}.aar
+    displayName: Copy Android test files and AAR to android_test directory
     workingDirectory: $(Build.BinariesDirectory)
 
-  - template: use-android-emulator.yml
-    parameters:
-      stop: true
+  # skip emulator tests for qnn package as there are no arm64-v8a emulators and no qnn libraries for x86
+  - ${{ if not(contains(parameters.packageName, 'qnn')) }}:
+    - template: use-android-emulator.yml
+      parameters:
+        create: true
+        start: true
+
+    - script: |
+        set -e -x
+        cd android_test/android
+        $(Build.SourcesDirectory)/java/gradlew --no-daemon clean connectedDebugAndroidTest --stacktrace
+      displayName: Run E2E test using Emulator
+      workingDirectory: $(Build.BinariesDirectory)
+
+    - template: use-android-emulator.yml
+      parameters:
+        stop: true
+
+  - ${{ else }}:
+    - script: |
+        # QNN SDK version string, expected format: 2.28.0.241029
+        # Extract the first three parts of the version string to get the Maven package version (e.g., 2.28.0)
+        QnnMavenPackageVersion=$(echo ${{ parameters.QnnSDKVersion }} | cut -d'.' -f1-3)
+        echo "QnnMavenPackageVersion: $QnnMavenPackageVersion"
+        echo "##vso[task.setvariable variable=QnnMavenPackageVersion]$QnnMavenPackageVersion"
+      displayName: Trim QNN SDK version to major.minor.patch
+
+    - script: |
+          set -e -x
+          # build apks for qnn package as they are not built in the emulator test step
+          $(Build.SourcesDirectory)/java/gradlew  --no-daemon clean assembleDebug assembleAndroidTest -DqnnVersion=$(QnnMavenPackageVersion) --stacktrace
+      displayName: Build QNN APK
+      workingDirectory: $(Build.BinariesDirectory)/android_test/android
 
   # we run e2e tests on one older device (Pixel 3) and one newer device (Galaxy 23)
   - script: |
       set -e -x
       pip install requests
+
       python $(Build.SourcesDirectory)/tools/python/upload_and_run_browserstack_tests.py \
         --test_platform espresso \
         --app_path "debug/app-debug.apk" \
         --test_path "androidTest/debug/app-debug-androidTest.apk" \
-        --devices "Samsung Galaxy S23-13.0" "Google Pixel 3-9.0"
+        --devices "Samsung Galaxy S23-13.0" "Google Pixel 3-9.0" \
+        --build_tag "${{ parameters.packageName }}"
+
     displayName: Run E2E tests using Browserstack
     workingDirectory: $(Build.BinariesDirectory)/android_test/android/app/build/outputs/apk
     timeoutInMinutes: 15

tools/ci_build/github/azure-pipelines/templates/c-api-cpu.yml

@@ -98,7 +98,12 @@ stages:
       enable_code_sign: ${{ parameters.DoEsrp }}
       packageName: 'onnxruntime-android-qnn'
       ReleaseVersionSuffix: $(ReleaseVersionSuffix)
-  #TODO: Add test job for QNN Android AAR
+
+  - template: android-java-api-aar-test.yml
+    parameters:
+      artifactName: 'onnxruntime-android-qnn-aar'
+      job_name_suffix: 'QNN'
+      packageName: 'onnxruntime-android-qnn'
 
 - stage: iOS_Full_xcframework
   dependsOn: []

tools/ci_build/github/azure-pipelines/templates/jobs/download_linux_qnn_sdk.yml

@@ -16,6 +16,29 @@ steps:
       echo $(QnnSDKRootDir)
     displayName: 'Print QnnSDKRootDir after downloading QNN SDK'
 
+  - script: |
+      set -x
+      sdk_file="$(QnnSDKRootDir)/sdk.yaml"
+      # Parse the sdk.yaml file to get the QNN SDK version downloaded
+      downloaded_qnn_sdk_version=$(grep '^version:' "$sdk_file" | head -n 1 | cut -d':' -f2 | xargs | cut -d'.' -f1-3 | tr -d '\r')
+
+      # Extract major.minor.patch part from QnnSDKVersion passed as parameter
+      expected_qnn_sdk_version=$(echo ${{ parameters.QnnSDKVersion }} | cut -d'.' -f1-3)
+
+      if [[ -z "$downloaded_qnn_sdk_version" ]]; then
+        echo "QNN version not found in sdk.yaml."
+        exit 1
+      fi
+
+      # Compare provided version with version from sdk.yaml
+      if [[ "$downloaded_qnn_sdk_version" == "$expected_qnn_sdk_version" ]]; then
+        echo "Success: QnnSDKVersion matches sdk.yaml version ($downloaded_qnn_sdk_version)."
+      else
+        echo "Error: QnnSDKVersion ($expected_qnn_sdk_version) does not match sdk.yaml version ($downloaded_qnn_sdk_version) in the QNN SDK directory"
+        exit 1
+      fi
+    displayName: "Sanity Check: QnnSDKVersion vs sdk.yaml version"
+
   - script: |
       azcopy cp --recursive 'https://lotusscus.blob.core.windows.net/models/qnnsdk/Qualcomm AI Hub Proprietary License.pdf' $(QnnSDKRootDir)
     displayName: 'Download Qualcomm AI Hub license'

tools/python/upload_and_run_browserstack_tests.py

@@ -29,13 +29,16 @@ def upload_apk_parse_json(post_url, apk_path, id, token):
     return response_to_json(response)
 
 
-def browserstack_build_request(devices, app_url, test_suite_url, test_platform, id, token):
+def browserstack_build_request(devices, app_url, test_suite_url, test_platform, id, token, project, build_tag):
     headers = {}
 
     json_data = {
         "devices": devices,
         "app": app_url,
         "testSuite": test_suite_url,
+        "project": project,
+        "buildTag": build_tag,
+        "deviceLogs": True,
     }
 
     build_response = requests.post(
@@ -108,6 +111,13 @@ if __name__ == "__main__":
         required=True,
     )
 
+    parser.add_argument(
+        "--project",
+        type=str,
+        help="Identifier to logically group multiple builds together",
+        default="ONNXRuntime tests",
+    )
+    parser.add_argument("--build_tag", type=str, help="Identifier to tag the build with a unique name", default="")
     args = parser.parse_args()
 
     try:
@@ -142,6 +152,8 @@ if __name__ == "__main__":
         args.test_platform,
         browserstack_id,
         browserstack_token,
+        args.project,
+        args.build_tag,
     )
 
     # Get build status until the tests are no longer running