mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-05-14 20:48:00 +00:00
Migrate Android Java E2E tests from App Center to Browserstack (#22117)
### Description - removed installing AppCenter + pipeline step that runs AppCenter Espresso tests - added script for running AppCenter tests ### Motivation and Context App Center is getting deprecated in the next year + we have upcoming Android work that depends on working E2E testing. --------- Co-authored-by: Edward Chen <18449977+edgchen1@users.noreply.github.com>
This commit is contained in:
parent
224f0651d0
commit
c73e6afa6c
2 changed files with 171 additions and 13 deletions
|
|
@ -45,8 +45,6 @@ jobs:
|
|||
|
||||
- template: use-android-ndk.yml
|
||||
|
||||
- template: install-appcenter.yml
|
||||
|
||||
- template: use-android-emulator.yml
|
||||
parameters:
|
||||
create: true
|
||||
|
|
@ -68,19 +66,21 @@ jobs:
|
|||
parameters:
|
||||
stop: true
|
||||
|
||||
# we run e2e tests on one older device (Pixel 3) and one newer device (Galaxy 23)
|
||||
- script: |
|
||||
set -e -x
|
||||
cd android_test/android
|
||||
appcenter test run espresso \
|
||||
--app "AI-Frameworks/ORT-Mobile-Android" \
|
||||
--devices $(app_center_android_test_devices) \
|
||||
--app-path ./app/build/outputs/apk/debug/app-debug.apk \
|
||||
--test-series "master" \
|
||||
--locale "en_US" \
|
||||
--build-dir ./app/build/outputs/apk/androidTest/debug \
|
||||
--token $(app_center_api_token)
|
||||
displayName: Run E2E tests using App Center
|
||||
workingDirectory: $(Build.BinariesDirectory)
|
||||
pip install requests
|
||||
python $(Build.SourcesDirectory)/tools/python/upload_and_run_browserstack_tests.py \
|
||||
--test_platform espresso \
|
||||
--app_apk_path "debug/app-debug.apk" \
|
||||
--test_apk_path "androidTest/debug/app-debug-androidTest.apk" \
|
||||
--devices "Samsung Galaxy S23-13.0" "Google Pixel 3-9.0"
|
||||
displayName: Run E2E tests using Browserstack
|
||||
workingDirectory: $(Build.BinariesDirectory)/android_test/android/app/build/outputs/apk
|
||||
timeoutInMinutes: 15
|
||||
env:
|
||||
BROWSERSTACK_ID: $(browserstack_username)
|
||||
BROWSERSTACK_TOKEN: $(browserstack_access_key)
|
||||
|
||||
- template: component-governance-component-detection-steps.yml
|
||||
parameters :
|
||||
|
|
|
|||
158
tools/python/upload_and_run_browserstack_tests.py
Normal file
158
tools/python/upload_and_run_browserstack_tests.py
Normal file
|
|
@ -0,0 +1,158 @@
|
|||
import argparse
|
||||
import os
|
||||
import sys
|
||||
import time
|
||||
from pathlib import Path
|
||||
|
||||
import requests
|
||||
|
||||
script_description = """
|
||||
After building ONNXRuntime for Android or iOS, use this script to upload the app and test files to BrowserStack then
|
||||
run the tests on the specified devices.
|
||||
|
||||
Find the Android test app in the repo here (as of rel-1.19.2):
|
||||
java/src/test/android
|
||||
"""
|
||||
|
||||
|
||||
def response_to_json(response):
|
||||
response.raise_for_status()
|
||||
response_json = response.json()
|
||||
print(response_json)
|
||||
print("-" * 30)
|
||||
return response_json
|
||||
|
||||
|
||||
def upload_apk_parse_json(post_url, apk_path, id, token):
|
||||
with open(apk_path, "rb") as apk_file:
|
||||
response = requests.post(post_url, files={"file": apk_file}, auth=(id, token), timeout=180)
|
||||
return response_to_json(response)
|
||||
|
||||
|
||||
def browserstack_build_request(devices, app_url, test_suite_url, test_platform, id, token):
|
||||
headers = {}
|
||||
|
||||
json_data = {
|
||||
"devices": devices,
|
||||
"app": app_url,
|
||||
"testSuite": test_suite_url,
|
||||
}
|
||||
|
||||
build_response = requests.post(
|
||||
f"https://api-cloud.browserstack.com/app-automate/{test_platform}/v2/build",
|
||||
headers=headers,
|
||||
json=json_data,
|
||||
auth=(id, token),
|
||||
timeout=180,
|
||||
)
|
||||
|
||||
return response_to_json(build_response)
|
||||
|
||||
|
||||
def build_query_loop(build_id, test_platform, id, token):
|
||||
"""Called after a build is initiated on Browserstack. Repeatedly queries the REST endpoint to check for the
|
||||
status of the tests in 30 second intervals.
|
||||
"""
|
||||
tests_status = "running"
|
||||
|
||||
while tests_status == "running":
|
||||
time.sleep(30)
|
||||
|
||||
test_response = requests.get(
|
||||
f"https://api-cloud.browserstack.com/app-automate/{test_platform}/v2/builds/{build_id}",
|
||||
auth=(id, token),
|
||||
timeout=30,
|
||||
)
|
||||
|
||||
test_response_json = response_to_json(test_response)
|
||||
tests_status = test_response_json["status"]
|
||||
|
||||
return tests_status
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
# handle cli args
|
||||
parser = argparse.ArgumentParser(script_description)
|
||||
|
||||
parser.add_argument(
|
||||
"--test_platform", type=str, help="Testing platform", choices=["espresso", "xcuitest"], required=True
|
||||
)
|
||||
parser.add_argument(
|
||||
"--app_apk_path",
|
||||
type=Path,
|
||||
help=(
|
||||
"Path to the app APK. "
|
||||
"Typically, the app APK is in "
|
||||
"{build_output_dir}/android_test/android/app/build/outputs/apk/debug/app-debug.apk"
|
||||
),
|
||||
required=True,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--test_apk_path",
|
||||
type=Path,
|
||||
help=(
|
||||
"Path to the test APK. "
|
||||
"Typically, the test APK is in "
|
||||
"{build_output_dir}/android_test/android/app/build/outputs/apk/androidTest/debug/app-debug-androidTest.apk"
|
||||
),
|
||||
required=True,
|
||||
)
|
||||
parser.add_argument(
|
||||
"--devices",
|
||||
type=str,
|
||||
nargs="+",
|
||||
help="List of devices to run the tests on. For more info, "
|
||||
"see https://www.browserstack.com/docs/app-automate/espresso/specify-devices",
|
||||
required=True,
|
||||
)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
try:
|
||||
browserstack_id = os.environ["BROWSERSTACK_ID"]
|
||||
browserstack_token = os.environ["BROWSERSTACK_TOKEN"]
|
||||
except KeyError:
|
||||
print("Please set the environment variables BROWSERSTACK_ID and BROWSERSTACK_TOKEN")
|
||||
print(
|
||||
"These values will be found at https://app-automate.browserstack.com/dashboard/v2 & clicking 'ACCESS KEY'"
|
||||
)
|
||||
sys.exit(1)
|
||||
|
||||
# Upload the app and test suites
|
||||
upload_app_json = upload_apk_parse_json(
|
||||
f"https://api-cloud.browserstack.com/app-automate/{args.test_platform}/v2/app",
|
||||
args.app_apk_path,
|
||||
browserstack_id,
|
||||
browserstack_token,
|
||||
)
|
||||
upload_test_json = upload_apk_parse_json(
|
||||
f"https://api-cloud.browserstack.com/app-automate/{args.test_platform}/v2/test-suite",
|
||||
args.test_apk_path,
|
||||
browserstack_id,
|
||||
browserstack_token,
|
||||
)
|
||||
|
||||
# Initiate build (send request to run the tests)
|
||||
build_response_json = browserstack_build_request(
|
||||
args.devices,
|
||||
upload_app_json["app_url"],
|
||||
upload_test_json["test_suite_url"],
|
||||
args.test_platform,
|
||||
browserstack_id,
|
||||
browserstack_token,
|
||||
)
|
||||
|
||||
# Get build status until the tests are no longer running
|
||||
tests_status = build_query_loop(
|
||||
build_response_json["build_id"], args.test_platform, browserstack_id, browserstack_token
|
||||
)
|
||||
|
||||
test_suite_details_url = (
|
||||
f"https://app-automate.browserstack.com/dashboard/v2/builds/{build_response_json['build_id']}"
|
||||
)
|
||||
|
||||
print("=" * 30)
|
||||
print("Test suite details: ", test_suite_details_url)
|
||||
print("=" * 30)
|
||||
if tests_status != "passed":
|
||||
raise Exception(f"Tests failed. Go to {test_suite_details_url} for more details.")
|
||||
Loading…
Reference in a new issue