diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/.config/dotnet-tools.json b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/.config/dotnet-tools.json
new file mode 100644
index 0000000000..67d39c423d
--- /dev/null
+++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/.config/dotnet-tools.json
@@ -0,0 +1,13 @@
+{
+ "version": 1,
+ "isRoot": true,
+ "tools": {
+ "browserstack-sdk": {
+ "version": "1.16.13",
+ "commands": [
+ "browserstack-sdk"
+ ],
+ "rollForward": false
+ }
+ }
+}
\ No newline at end of file
diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/BrowserStackTest.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/BrowserStackTest.cs
new file mode 100644
index 0000000000..594ba4f0e5
--- /dev/null
+++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/BrowserStackTest.cs
@@ -0,0 +1,74 @@
+using Newtonsoft.Json;
+using NUnit.Framework.Interfaces;
+using NUnit.Framework;
+using OpenQA.Selenium;
+using OpenQA.Selenium.Appium;
+using OpenQA.Selenium.Appium.Android;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Microsoft.ML.OnnxRuntime.Tests.Android
+{
+ public class BrowserStackTest
+ {
+ public AndroidDriver driver;
+ public BrowserStackTest() { }
+
+ [SetUp]
+ public void Init()
+ {
+ var androidOptions = new AppiumOptions
+ {
+ AutomationName = "UIAutomator2",
+ PlatformName = "Android",
+ };
+
+ driver = new AndroidDriver(new Uri("http://127.0.0.1:4723/wd/hub"), androidOptions);
+ }
+
+ ///
+ /// Sends a log to BrowserStack that is visible in the text logs and labelled as ANNOTATION.
+ ///
+ /// Log text to send.
+ /// Log level -- choose between info, debug, warning, and error
+ public void browserStackLog(String text, String logLevel = "info")
+ {
+ String jsonToSend = String.Format("browserstack_executor: {\"action\": \"annotate\", \"arguments\": {\"data\": {0}, \"level\": {1}}}", JsonConvert.ToString(text), JsonConvert.ToString(logLevel));
+
+ ((IJavaScriptExecutor)driver).ExecuteScript(jsonToSend);
+ }
+
+ ///
+ /// Passes the correct test status to BrowserStack and ensures the driver quits.
+ ///
+ [TearDown]
+ public void Dispose()
+ {
+ try
+ {
+ // According to https://www.browserstack.com/docs/app-automate/appium/set-up-tests/mark-tests-as-pass-fail
+ // BrowserStack doesn't know whether test assertions have passed or failed. Below handles
+ // passing the test status to BrowserStack along with any relevant information.
+ if (TestContext.CurrentContext.Result.Outcome.Status == TestStatus.Failed)
+ {
+ String failureMessage = TestContext.CurrentContext.Result.Message;
+ String jsonToSendFailure = String.Format("browserstack_executor: {\"action\": \"setSessionStatus\", \"arguments\": {\"status\":\"failed\", \"reason\": {0}}}", JsonConvert.ToString(failureMessage));
+
+ ((IJavaScriptExecutor)driver).ExecuteScript(jsonToSendFailure);
+ }
+ else
+ {
+ ((IJavaScriptExecutor)driver).ExecuteScript("browserstack_executor: {\"action\": \"setSessionStatus\", \"arguments\": {\"status\":\"passed\", \"reason\": \"\"}}");
+ }
+ }
+ finally
+ {
+ // will run even if exception is thrown by previous block
+ ((AndroidDriver)driver).Quit();
+ }
+ }
+ }
+}
diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/Microsoft.ML.OnnxRuntime.Tests.Android.csproj b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/Microsoft.ML.OnnxRuntime.Tests.Android.csproj
new file mode 100644
index 0000000000..9b9028d30c
--- /dev/null
+++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/Microsoft.ML.OnnxRuntime.Tests.Android.csproj
@@ -0,0 +1,22 @@
+
+
+
+ net8.0
+ enable
+ enable
+
+ false
+ true
+
+
+
+
+
+
+
+
+
+
+
+
+
diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/RunAllTest.cs b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/RunAllTest.cs
new file mode 100644
index 0000000000..72eb720a89
--- /dev/null
+++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/RunAllTest.cs
@@ -0,0 +1,107 @@
+using OpenQA.Selenium.Appium;
+using OpenQA.Selenium;
+using NUnit.Framework;
+using System;
+using System.Collections.Generic;
+using System.Linq;
+using System.Text;
+using System.Threading.Tasks;
+
+namespace Microsoft.ML.OnnxRuntime.Tests.Android
+{
+ [TestFixture]
+ public class RunAllTest : BrowserStackTest
+ {
+ public AppiumElement FindAppiumElement(String xpathQuery, String text)
+ {
+ IReadOnlyCollection appiumElements = driver.FindElements(By.XPath(xpathQuery));
+
+ foreach (var element in appiumElements)
+ {
+ if (element.Text.Contains(text))
+ {
+ return element;
+ }
+ }
+ // was unable to find given element
+ throw new Exception(String.Format("Could not find {0}: {1} on the page.", xpathQuery, text));
+ }
+
+ public AppiumElement FindAppiumElementThenClick(String xpathQuery, String text)
+ {
+ AppiumElement appiumElement = FindAppiumElement(xpathQuery, text);
+ appiumElement.Click();
+ return appiumElement;
+ }
+
+ public (int, int) GetPassFailCount()
+ {
+ int numPassed = -1;
+ int numFailed = -1;
+
+ IReadOnlyCollection labelElements = driver.FindElements(By.XPath("//android.widget.TextView"));
+
+ for (int i = 0; i < labelElements.Count; i++)
+ {
+ AppiumElement element = labelElements.ElementAt(i);
+
+ if (element.Text.Equals("✔"))
+ {
+ i++;
+ numPassed = int.Parse(labelElements.ElementAt(i).Text);
+ }
+
+ if (element.Text.Equals("⛔"))
+ {
+ i++;
+ numFailed = int.Parse(labelElements.ElementAt(i).Text);
+ break;
+ }
+ }
+
+ Assert.That(numPassed, Is.GreaterThanOrEqualTo(0), "Could not find number passed label.");
+ Assert.That(numFailed, Is.GreaterThanOrEqualTo(0), "Could not find number failed label.");
+
+ return (numPassed, numFailed);
+ }
+
+ [Test]
+ public async Task ClickRunAllTest()
+ {
+ AppiumElement runAllButton = FindAppiumElementThenClick("//android.widget.Button", "Run All");
+
+ while (!runAllButton.Enabled)
+ {
+ // waiting for unit tests to execute
+ await Task.Delay(500);
+ }
+
+ var (numPassed, numFailed) = GetPassFailCount();
+
+ if (numFailed == 0)
+ {
+ Assert.Pass();
+ return;
+ }
+
+ // click into test results if tests have failed
+ FindAppiumElementThenClick("//android.widget.TextView", "⛔");
+ await Task.Delay(500);
+ FindAppiumElementThenClick("//android.widget.EditText", "All");
+ await Task.Delay(100);
+ FindAppiumElementThenClick("//android.widget.TextView", "Failed");
+ await Task.Delay(500);
+
+ StringBuilder sb = new StringBuilder();
+ sb.AppendLine("PASSED TESTS: " + numPassed + " | FAILED TESTS: " + numFailed);
+
+ IReadOnlyCollection textResults = driver.FindElements(By.XPath("//android.widget.TextView"));
+ foreach (var element in textResults)
+ {
+ sb.AppendLine(element.Text);
+ }
+
+ Assert.That(numFailed, Is.EqualTo(0), sb.ToString());
+ }
+ }
+}
diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/browserstack.yml b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/browserstack.yml
new file mode 100644
index 0000000000..9efbc9fc6a
--- /dev/null
+++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.Android/browserstack.yml
@@ -0,0 +1,13 @@
+app: ..\Microsoft.ML.OnnxRuntime.Tests.MAUI\bin\Release\net8.0-android\publish\ORT.CSharp.Tests.MAUI-Signed.apk
+platforms:
+ - platformName: android
+ deviceName: Samsung Galaxy S22 Ultra
+ platformVersion: 12.0
+browserstackLocal: true
+buildName: ORT android test
+buildIdentifier: ${BUILD_NUMBER}
+projectName: ORT-UITests
+debug: true
+networkLogs: false
+testContextOptions:
+ skipSessionStatus: true
\ No newline at end of file
diff --git a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.MAUI/Microsoft.ML.OnnxRuntime.Tests.MAUI.csproj b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.MAUI/Microsoft.ML.OnnxRuntime.Tests.MAUI.csproj
index e07448daee..c4a547748a 100644
--- a/csharp/test/Microsoft.ML.OnnxRuntime.Tests.MAUI/Microsoft.ML.OnnxRuntime.Tests.MAUI.csproj
+++ b/csharp/test/Microsoft.ML.OnnxRuntime.Tests.MAUI/Microsoft.ML.OnnxRuntime.Tests.MAUI.csproj
@@ -7,7 +7,7 @@
- net8.0-android;net8.0-ios;net8.0-maccatalyst
+ net8.0;net8.0-android;net8.0-ios;net8.0-maccatalyst
$(TargetFrameworks);net8.0-windows10.0.19041.0
- Exe
+ Exe
Microsoft.ML.OnnxRuntime.Tests.MAUI
true
true
@@ -101,6 +101,7 @@
+
@@ -122,4 +123,10 @@
+
+
+ false
+ en
+
+