From eedad80fe94a1920eeb65735de8f4c6bd8d42ea4 Mon Sep 17 00:00:00 2001 From: carzh Date: Wed, 15 Jan 2025 12:33:56 -0800 Subject: [PATCH] Working android MAUI app + project that runs android browserstack test --- .../.config/dotnet-tools.json | 13 +++ .../BrowserStackTest.cs | 74 ++++++++++++ ...rosoft.ML.OnnxRuntime.Tests.Android.csproj | 22 ++++ .../RunAllTest.cs | 107 ++++++++++++++++++ .../browserstack.yml | 13 +++ ...Microsoft.ML.OnnxRuntime.Tests.MAUI.csproj | 11 +- 6 files changed, 238 insertions(+), 2 deletions(-) create mode 100644 csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/.config/dotnet-tools.json create mode 100644 csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/BrowserStackTest.cs create mode 100644 csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/Microsoft.ML.OnnxRuntime.Tests.Android.csproj create mode 100644 csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/RunAllTest.cs create mode 100644 csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/browserstack.yml diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/.config/dotnet-tools.json b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/.config/dotnet-tools.json new file mode 100644 index 0000000000..67d39c423d --- /dev/null +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/.config/dotnet-tools.json @@ -0,0 +1,13 @@ +{ + "version": 1, + "isRoot": true, + "tools": { + "browserstack-sdk": { + "version": "1.16.13", + "commands": [ + "browserstack-sdk" + ], + "rollForward": false + } + } +} \ No newline at end of file diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/BrowserStackTest.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/BrowserStackTest.cs new file mode 100644 index 0000000000..594ba4f0e5 --- /dev/null +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/BrowserStackTest.cs @@ -0,0 +1,74 @@ +using Newtonsoft.Json; +using NUnit.Framework.Interfaces; +using NUnit.Framework; +using OpenQA.Selenium; +using OpenQA.Selenium.Appium; +using OpenQA.Selenium.Appium.Android; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Microsoft.ML.OnnxRuntime.Tests.Android +{ + public class BrowserStackTest + { + public AndroidDriver driver; + public BrowserStackTest() { } + + [SetUp] + public void Init() + { + var androidOptions = new AppiumOptions + { + AutomationName = "UIAutomator2", + PlatformName = "Android", + }; + + driver = new AndroidDriver(new Uri("http://127.0.0.1:4723/wd/hub"), androidOptions); + } + + /// + /// Sends a log to BrowserStack that is visible in the text logs and labelled as ANNOTATION. + /// + /// Log text to send. + /// Log level -- choose between info, debug, warning, and error + public void browserStackLog(String text, String logLevel = "info") + { + String jsonToSend = String.Format("browserstack_executor: {\"action\": \"annotate\", \"arguments\": {\"data\": {0}, \"level\": {1}}}", JsonConvert.ToString(text), JsonConvert.ToString(logLevel)); + + ((IJavaScriptExecutor)driver).ExecuteScript(jsonToSend); + } + + /// + /// Passes the correct test status to BrowserStack and ensures the driver quits. + /// + [TearDown] + public void Dispose() + { + try + { + // According to https://www.browserstack.com/docs/app-automate/appium/set-up-tests/mark-tests-as-pass-fail + // BrowserStack doesn't know whether test assertions have passed or failed. Below handles + // passing the test status to BrowserStack along with any relevant information. + if (TestContext.CurrentContext.Result.Outcome.Status == TestStatus.Failed) + { + String failureMessage = TestContext.CurrentContext.Result.Message; + String jsonToSendFailure = String.Format("browserstack_executor: {\"action\": \"setSessionStatus\", \"arguments\": {\"status\":\"failed\", \"reason\": {0}}}", JsonConvert.ToString(failureMessage)); + + ((IJavaScriptExecutor)driver).ExecuteScript(jsonToSendFailure); + } + else + { + ((IJavaScriptExecutor)driver).ExecuteScript("browserstack_executor: {\"action\": \"setSessionStatus\", \"arguments\": {\"status\":\"passed\", \"reason\": \"\"}}"); + } + } + finally + { + // will run even if exception is thrown by previous block + ((AndroidDriver)driver).Quit(); + } + } + } +} diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/Microsoft.ML.OnnxRuntime.Tests.Android.csproj b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/Microsoft.ML.OnnxRuntime.Tests.Android.csproj new file mode 100644 index 0000000000..9b9028d30c --- /dev/null +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/Microsoft.ML.OnnxRuntime.Tests.Android.csproj @@ -0,0 +1,22 @@ + + + + net8.0 + enable + enable + + false + true + + + + + + + + + + + + + diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/RunAllTest.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/RunAllTest.cs new file mode 100644 index 0000000000..72eb720a89 --- /dev/null +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/RunAllTest.cs @@ -0,0 +1,107 @@ +using OpenQA.Selenium.Appium; +using OpenQA.Selenium; +using NUnit.Framework; +using System; +using System.Collections.Generic; +using System.Linq; +using System.Text; +using System.Threading.Tasks; + +namespace Microsoft.ML.OnnxRuntime.Tests.Android +{ + [TestFixture] + public class RunAllTest : BrowserStackTest + { + public AppiumElement FindAppiumElement(String xpathQuery, String text) + { + IReadOnlyCollection appiumElements = driver.FindElements(By.XPath(xpathQuery)); + + foreach (var element in appiumElements) + { + if (element.Text.Contains(text)) + { + return element; + } + } + // was unable to find given element + throw new Exception(String.Format("Could not find {0}: {1} on the page.", xpathQuery, text)); + } + + public AppiumElement FindAppiumElementThenClick(String xpathQuery, String text) + { + AppiumElement appiumElement = FindAppiumElement(xpathQuery, text); + appiumElement.Click(); + return appiumElement; + } + + public (int, int) GetPassFailCount() + { + int numPassed = -1; + int numFailed = -1; + + IReadOnlyCollection labelElements = driver.FindElements(By.XPath("//android.widget.TextView")); + + for (int i = 0; i < labelElements.Count; i++) + { + AppiumElement element = labelElements.ElementAt(i); + + if (element.Text.Equals("✔")) + { + i++; + numPassed = int.Parse(labelElements.ElementAt(i).Text); + } + + if (element.Text.Equals("⛔")) + { + i++; + numFailed = int.Parse(labelElements.ElementAt(i).Text); + break; + } + } + + Assert.That(numPassed, Is.GreaterThanOrEqualTo(0), "Could not find number passed label."); + Assert.That(numFailed, Is.GreaterThanOrEqualTo(0), "Could not find number failed label."); + + return (numPassed, numFailed); + } + + [Test] + public async Task ClickRunAllTest() + { + AppiumElement runAllButton = FindAppiumElementThenClick("//android.widget.Button", "Run All"); + + while (!runAllButton.Enabled) + { + // waiting for unit tests to execute + await Task.Delay(500); + } + + var (numPassed, numFailed) = GetPassFailCount(); + + if (numFailed == 0) + { + Assert.Pass(); + return; + } + + // click into test results if tests have failed + FindAppiumElementThenClick("//android.widget.TextView", "⛔"); + await Task.Delay(500); + FindAppiumElementThenClick("//android.widget.EditText", "All"); + await Task.Delay(100); + FindAppiumElementThenClick("//android.widget.TextView", "Failed"); + await Task.Delay(500); + + StringBuilder sb = new StringBuilder(); + sb.AppendLine("PASSED TESTS: " + numPassed + " | FAILED TESTS: " + numFailed); + + IReadOnlyCollection textResults = driver.FindElements(By.XPath("//android.widget.TextView")); + foreach (var element in textResults) + { + sb.AppendLine(element.Text); + } + + Assert.That(numFailed, Is.EqualTo(0), sb.ToString()); + } + } +} diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/browserstack.yml b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/browserstack.yml new file mode 100644 index 0000000000..9efbc9fc6a --- /dev/null +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/browserstack.yml @@ -0,0 +1,13 @@ +app: ..\Microsoft.ML.OnnxRuntime.Tests.MAUI\bin\Release\net8.0-android\publish\ORT.CSharp.Tests.MAUI-Signed.apk +platforms: + - platformName: android + deviceName: Samsung Galaxy S22 Ultra + platformVersion: 12.0 +browserstackLocal: true +buildName: ORT android test +buildIdentifier: ${BUILD_NUMBER} +projectName: ORT-UITests +debug: true +networkLogs: false +testContextOptions: + skipSessionStatus: true \ No newline at end of file diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.MAUI/Microsoft.ML.OnnxRuntime.Tests.MAUI.csproj b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.MAUI/Microsoft.ML.OnnxRuntime.Tests.MAUI.csproj index e07448daee..c4a547748a 100644 --- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.MAUI/Microsoft.ML.OnnxRuntime.Tests.MAUI.csproj +++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.MAUI/Microsoft.ML.OnnxRuntime.Tests.MAUI.csproj @@ -7,7 +7,7 @@ - net8.0-android;net8.0-ios;net8.0-maccatalyst + net8.0;net8.0-android;net8.0-ios;net8.0-maccatalyst $(TargetFrameworks);net8.0-windows10.0.19041.0 - Exe + Exe Microsoft.ML.OnnxRuntime.Tests.MAUI true true @@ -101,6 +101,7 @@ + @@ -122,4 +123,10 @@ + + + false + en + +