saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-08 17:17:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Move iOS e2e test to XCUITest (#9422)

Browse source 
* Move iOS test to user UITest

* minor update

* Update readme

* update test's ios deployment target

* address cr comments
This commit is contained in:
Guoyu Wang 2021-10-21 18:51:13 -07:00 committed by GitHub
parent 7f2f56633c
commit b64b2d48f3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 77 additions and 107 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
onnxruntime/test/platform/ios/ios_package_test/Podfile
View file

@ -4,7 +4,7 @@ target 'ios_package_test' do
# Comment the next line if you don't want to use dynamic frameworks
use_frameworks!
target 'ios_package_testTests' do
target 'ios_package_testUITests' do
inherit! :search_paths
pod 'onnxruntime-mobile-c', :podspec => './onnxruntime-mobile-c.podspec'
end

31
onnxruntime/test/platform/ios/ios_package_test/README.md
View file

@ -4,7 +4,7 @@ This End-to-End test app for iOS will test ORT Mobile C/C++ API framework using
## Requirements
- [Prerequisites for building ORT-Mobile for iOS](http://www.onnxruntime.ai/docs/how-to/build/android-ios.html#prerequisites-1)
- [Prerequisites for building ORT-Mobile for iOS](https://onnxruntime.ai/docs/build/android-ios.html#prerequisites-1)
- [CocoaPods](https://cocoapods.org/)
## iOS End-to-End Test App Overview
@ -14,34 +14,11 @@ The iOS End-to-End Test App will use CocoaPods to install the Onnx Runtime C/C++
### Model used
- [sigmoid ONNX model](https://github.com/onnx/onnx/blob/f9b0cc99344869c246b8f4011b8586a39841284c/onnx/backend/test/data/node/test_sigmoid/model.onnx) converted to ORT format
Here's the [document](http://www.onnxruntime.ai/docs/how-to/deploy-on-mobile.html#1-create-ort-format-model-and-configuration-file-with-required-operators) about how you can convert an ONNX model into ORT format.
Here's the [document](https://onnxruntime.ai/docs/tutorials/mobile/model-conversion.html) about how you can convert an ONNX model into ORT format.
### Tests
- [Tests for C API ](./ios_package_testTests/ios_package_test_c_api.m)
- [Tests for C++ API ](./ios_package_testTests/ios_package_test_cpp_api.mm)
- [Tests for C++ API ](./ios_package_testUITests/ios_package_uitest_cpp_api.mm)
## Build and Test iOS Framework using [build.py](../../../../../tools/ci_build/build.py)
Use the [build for iOS simulator](http://www.onnxruntime.ai/docs/how-to/build/android-ios.html#cross-build-for-ios-simulator) with `--build_apple_framework`
## Run the iOS End-to-End Test App standalone
### Requirements
- A pre-built ORT Mobile iOS framework, which can be built using the [instruction](#build-and-test-ios-framework-using-buildpy) above. The framework can be found as `<build_dir>/iOS/<build-config>/<build-config>-iphonesimulator/onnxruntime.framework`
### Steps
1. Go to this folder
2. Copy the [onnxruntime-mobile.podspec.template](./onnxruntime-mobile.podspec.template) to `onnxruntime-mobile.podspec`
3. Update the `onnxruntime-mobile.podspec`, replace `${ORT_BASE_FRAMEWORK_ARCHIVE}` with the path of a zip archive contains the pre-built ORT Mobile iOS framework
4. Run `pod install` to install the pre-built ORT Mobile iOS framework
5. Run the following command to perform the test
```
xcrun xcodebuild \
-workspace ./ios_package_test.xcworkspace \
-destination '<Your choice of test target device>' \
-scheme ios_package_test \
test
```
Use the [build for iOS simulator](https://onnxruntime.ai/docs/build/android-ios.html#cross-build-for-ios-simulator) with `--build_apple_framework`

104
onnxruntime/test/platform/ios/ios_package_test/ios_package_test.xcodeproj/project.pbxproj
View file

@ -11,13 +11,13 @@
229E592A265869BF006E41AE /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 229E5928265869BF006E41AE /* Main.storyboard */; };
229E592F265869C2006E41AE /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 229E592D265869C2006E41AE /* LaunchScreen.storyboard */; };
229E5932265869C2006E41AE /* main.m in Sources */ = {isa = PBXBuildFile; fileRef = 229E5931265869C2006E41AE /* main.m */; };
229E593C265869C2006E41AE /* ios_package_test_cpp_api.mm in Sources */ = {isa = PBXBuildFile; fileRef = 229E593B265869C2006E41AE /* ios_package_test_cpp_api.mm */; };
229E595926586B4A006E41AE /* sigmoid.ort in Resources */ = {isa = PBXBuildFile; fileRef = 229E595826586B4A006E41AE /* sigmoid.ort */; };
229E595A26586B4A006E41AE /* sigmoid.ort in Resources */ = {isa = PBXBuildFile; fileRef = 229E595826586B4A006E41AE /* sigmoid.ort */; };
22C1D8EA271A79FD002CEE67 /* ios_package_uitest_cpp_api.mm in Sources */ = {isa = PBXBuildFile; fileRef = 22C1D8E9271A79FD002CEE67 /* ios_package_uitest_cpp_api.mm */; };
22C1D8EB271A7A06002CEE67 /* sigmoid.ort in Resources */ = {isa = PBXBuildFile; fileRef = 229E595826586B4A006E41AE /* sigmoid.ort */; };
/* End PBXBuildFile section */
/* Begin PBXContainerItemProxy section */
229E5938265869C2006E41AE /* PBXContainerItemProxy */ = {
22C1D8E4271A79AF002CEE67 /* PBXContainerItemProxy */ = {
isa = PBXContainerItemProxy;
containerPortal = 229E5914265869BF006E41AE /* Project object */;
proxyType = 1;
@ -34,10 +34,9 @@
229E592E265869C2006E41AE /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = "<group>"; };
229E5930265869C2006E41AE /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
229E5931265869C2006E41AE /* main.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = main.m; sourceTree = "<group>"; };
229E5937265869C2006E41AE /* ios_package_testTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = ios_package_testTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; };
229E593B265869C2006E41AE /* ios_package_test_cpp_api.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = ios_package_test_cpp_api.mm; sourceTree = "<group>"; };
229E593D265869C2006E41AE /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = "<group>"; };
229E595826586B4A006E41AE /* sigmoid.ort */ = {isa = PBXFileReference; lastKnownFileType = file; path = sigmoid.ort; sourceTree = "<group>"; };
22C1D8DE271A79AF002CEE67 /* ios_package_testUITests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = ios_package_testUITests.xctest; sourceTree = BUILT_PRODUCTS_DIR; };
22C1D8E9271A79FD002CEE67 /* ios_package_uitest_cpp_api.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = ios_package_uitest_cpp_api.mm; sourceTree = "<group>"; };
/* End PBXFileReference section */
/* Begin PBXFrameworksBuildPhase section */
@ -48,7 +47,7 @@
);
runOnlyForDeploymentPostprocessing = 0;
};
229E5934265869C2006E41AE /* Frameworks */ = {
22C1D8DB271A79AF002CEE67 /* Frameworks */ = {
isa = PBXFrameworksBuildPhase;
buildActionMask = 2147483647;
files = (
@ -63,7 +62,7 @@
children = (
229E595426586A77006E41AE /* models */,
229E591E265869BF006E41AE /* ios_package_test */,
229E593A265869C2006E41AE /* ios_package_testTests */,
22C1D8DF271A79AF002CEE67 /* ios_package_testUITests */,
229E591D265869BF006E41AE /* Products */,
);
sourceTree = "<group>";
@ -72,7 +71,7 @@
isa = PBXGroup;
children = (
229E591C265869BF006E41AE /* ios_package_test.app */,
229E5937265869C2006E41AE /* ios_package_testTests.xctest */,
22C1D8DE271A79AF002CEE67 /* ios_package_testUITests.xctest */,
);
name = Products;
sourceTree = "<group>";
@ -90,15 +89,6 @@
path = ios_package_test;
sourceTree = "<group>";
};
229E593A265869C2006E41AE /* ios_package_testTests */ = {
isa = PBXGroup;
children = (
229E593B265869C2006E41AE /* ios_package_test_cpp_api.mm */,
229E593D265869C2006E41AE /* Info.plist */,
);
path = ios_package_testTests;
sourceTree = "<group>";
};
229E595426586A77006E41AE /* models */ = {
isa = PBXGroup;
children = (
@ -107,6 +97,14 @@
path = models;
sourceTree = "<group>";
};
22C1D8DF271A79AF002CEE67 /* ios_package_testUITests */ = {
isa = PBXGroup;
children = (
22C1D8E9271A79FD002CEE67 /* ios_package_uitest_cpp_api.mm */,
);
path = ios_package_testUITests;
sourceTree = "<group>";
};
/* End PBXGroup section */
/* Begin PBXNativeTarget section */
@ -127,23 +125,23 @@
productReference = 229E591C265869BF006E41AE /* ios_package_test.app */;
productType = "com.apple.product-type.application";
};
229E5936265869C2006E41AE /* ios_package_testTests */ = {
22C1D8DD271A79AF002CEE67 /* ios_package_testUITests */ = {
isa = PBXNativeTarget;
buildConfigurationList = 229E594E265869C2006E41AE /* Build configuration list for PBXNativeTarget "ios_package_testTests" */;
buildConfigurationList = 22C1D8E8271A79AF002CEE67 /* Build configuration list for PBXNativeTarget "ios_package_testUITests" */;
buildPhases = (
229E5933265869C2006E41AE /* Sources */,
229E5934265869C2006E41AE /* Frameworks */,
229E5935265869C2006E41AE /* Resources */,
22C1D8DA271A79AF002CEE67 /* Sources */,
22C1D8DB271A79AF002CEE67 /* Frameworks */,
22C1D8DC271A79AF002CEE67 /* Resources */,
);
buildRules = (
);
dependencies = (
229E5939265869C2006E41AE /* PBXTargetDependency */,
22C1D8E5271A79AF002CEE67 /* PBXTargetDependency */,
);
name = ios_package_testTests;
productName = ios_package_testTests;
productReference = 229E5937265869C2006E41AE /* ios_package_testTests.xctest */;
productType = "com.apple.product-type.bundle.unit-test";
name = ios_package_testUITests;
productName = ios_package_testUITests;
productReference = 22C1D8DE271A79AF002CEE67 /* ios_package_testUITests.xctest */;
productType = "com.apple.product-type.bundle.ui-testing";
};
/* End PBXNativeTarget section */
@ -156,8 +154,8 @@
229E591B265869BF006E41AE = {
CreatedOnToolsVersion = 12.5;
};
229E5936265869C2006E41AE = {
CreatedOnToolsVersion = 12.5;
22C1D8DD271A79AF002CEE67 = {
CreatedOnToolsVersion = 13.0;
TestTargetID = 229E591B265869BF006E41AE;
};
};
@ -176,7 +174,7 @@
projectRoot = "";
targets = (
229E591B265869BF006E41AE /* ios_package_test */,
229E5936265869C2006E41AE /* ios_package_testTests */,
22C1D8DD271A79AF002CEE67 /* ios_package_testUITests */,
);
};
/* End PBXProject section */
@ -192,11 +190,11 @@
);
runOnlyForDeploymentPostprocessing = 0;
};
229E5935265869C2006E41AE /* Resources */ = {
22C1D8DC271A79AF002CEE67 /* Resources */ = {
isa = PBXResourcesBuildPhase;
buildActionMask = 2147483647;
files = (
229E595A26586B4A006E41AE /* sigmoid.ort in Resources */,
22C1D8EB271A7A06002CEE67 /* sigmoid.ort in Resources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
@ -212,21 +210,21 @@
);
runOnlyForDeploymentPostprocessing = 0;
};
229E5933265869C2006E41AE /* Sources */ = {
22C1D8DA271A79AF002CEE67 /* Sources */ = {
isa = PBXSourcesBuildPhase;
buildActionMask = 2147483647;
files = (
229E593C265869C2006E41AE /* ios_package_test_cpp_api.mm in Sources */,
22C1D8EA271A79FD002CEE67 /* ios_package_uitest_cpp_api.mm in Sources */,
);
runOnlyForDeploymentPostprocessing = 0;
};
/* End PBXSourcesBuildPhase section */
/* Begin PBXTargetDependency section */
229E5939265869C2006E41AE /* PBXTargetDependency */ = {
22C1D8E5271A79AF002CEE67 /* PBXTargetDependency */ = {
isa = PBXTargetDependency;
target = 229E591B265869BF006E41AE /* ios_package_test */;
targetProxy = 229E5938265869C2006E41AE /* PBXContainerItemProxy */;
targetProxy = 22C1D8E4271A79AF002CEE67 /* PBXContainerItemProxy */;
};
/* End PBXTargetDependency section */
@ -396,41 +394,47 @@
};
name = Release;
};
229E594F265869C2006E41AE /* Debug */ = {
22C1D8E6271A79AF002CEE67 /* Debug */ = {
isa = XCBuildConfiguration;
buildSettings = {
BUNDLE_LOADER = "$(TEST_HOST)";
CLANG_CXX_LANGUAGE_STANDARD = "gnu++17";
CODE_SIGN_STYLE = Automatic;
INFOPLIST_FILE = ios_package_testTests/Info.plist;
CURRENT_PROJECT_VERSION = 1;
GENERATE_INFOPLIST_FILE = YES;
IPHONEOS_DEPLOYMENT_TARGET = 13.0;
LD_RUNPATH_SEARCH_PATHS = (
"$(inherited)",
"@executable_path/Frameworks",
"@loader_path/Frameworks",
);
PRODUCT_BUNDLE_IDENTIFIER = "ai.onnxruntime.tests.ios-package-testTests";
MARKETING_VERSION = 1.0;
PRODUCT_BUNDLE_IDENTIFIER = "ai.onnxruntime.tests.ios-package-testUITests";
PRODUCT_NAME = "$(TARGET_NAME)";
SWIFT_EMIT_LOC_STRINGS = NO;
TARGETED_DEVICE_FAMILY = "1,2";
TEST_HOST = "$(BUILT_PRODUCTS_DIR)/ios_package_test.app/ios_package_test";
TEST_TARGET_NAME = ios_package_test;
};
name = Debug;
};
229E5950265869C2006E41AE /* Release */ = {
22C1D8E7271A79AF002CEE67 /* Release */ = {
isa = XCBuildConfiguration;
buildSettings = {
BUNDLE_LOADER = "$(TEST_HOST)";
CLANG_CXX_LANGUAGE_STANDARD = "gnu++17";
CODE_SIGN_STYLE = Automatic;
INFOPLIST_FILE = ios_package_testTests/Info.plist;
CURRENT_PROJECT_VERSION = 1;
GENERATE_INFOPLIST_FILE = YES;
IPHONEOS_DEPLOYMENT_TARGET = 13.0;
LD_RUNPATH_SEARCH_PATHS = (
"$(inherited)",
"@executable_path/Frameworks",
"@loader_path/Frameworks",
);
PRODUCT_BUNDLE_IDENTIFIER = "ai.onnxruntime.tests.ios-package-testTests";
MARKETING_VERSION = 1.0;
PRODUCT_BUNDLE_IDENTIFIER = "ai.onnxruntime.tests.ios-package-testUITests";
PRODUCT_NAME = "$(TARGET_NAME)";
SWIFT_EMIT_LOC_STRINGS = NO;
TARGETED_DEVICE_FAMILY = "1,2";
TEST_HOST = "$(BUILT_PRODUCTS_DIR)/ios_package_test.app/ios_package_test";
TEST_TARGET_NAME = ios_package_test;
};
name = Release;
};
@ -455,11 +459,11 @@
defaultConfigurationIsVisible = 0;
defaultConfigurationName = Release;
};
229E594E265869C2006E41AE /* Build configuration list for PBXNativeTarget "ios_package_testTests" */ = {
22C1D8E8271A79AF002CEE67 /* Build configuration list for PBXNativeTarget "ios_package_testUITests" */ = {
isa = XCConfigurationList;
buildConfigurations = (
229E594F265869C2006E41AE /* Debug */,
229E5950265869C2006E41AE /* Release */,
22C1D8E6271A79AF002CEE67 /* Debug */,
22C1D8E7271A79AF002CEE67 /* Release */,
);
defaultConfigurationIsVisible = 0;
defaultConfigurationName = Release;

22
onnxruntime/test/platform/ios/ios_package_test/ios_package_testTests/Info.plist
View file

@ -1,22 +0,0 @@
<?xml version="1.0" encoding="UTF-8"?>
<!DOCTYPE plist PUBLIC "-//Apple//DTD PLIST 1.0//EN" "http://www.apple.com/DTDs/PropertyList-1.0.dtd">
<plist version="1.0">
<dict>
<key>CFBundleDevelopmentRegion</key>
<string>$(DEVELOPMENT_LANGUAGE)</string>
<key>CFBundleExecutable</key>
<string>$(EXECUTABLE_NAME)</string>
<key>CFBundleIdentifier</key>
<string>$(PRODUCT_BUNDLE_IDENTIFIER)</string>
<key>CFBundleInfoDictionaryVersion</key>
<string>6.0</string>
<key>CFBundleName</key>
<string>$(PRODUCT_NAME)</string>
<key>CFBundlePackageType</key>
<string>$(PRODUCT_BUNDLE_PACKAGE_TYPE)</string>
<key>CFBundleShortVersionString</key>
<string>1.0</string>
<key>CFBundleVersion</key>
<string>1</string>
</dict>
</plist>

25
onnxruntime/test/platform/ios/ios_package_test/ios_package_testTests/ios_package_test_cpp_api.mm → onnxruntime/test/platform/ios/ios_package_test/ios_package_testUITests/ios_package_uitest_cpp_api.mm
View file

@ -2,9 +2,9 @@
// Licensed under the MIT License.
//
// ios_package_test_cpp_api.mm
// ios_package_testTests
// ios_package_test_cpp_api
//
// This file hosts the tests of ORT C++ API, for tests of ORT C API, please see ios_package_test_c_api.mm
// This file hosts the tests of ORT C++ API
//
#import <XCTest/XCTest.h>
@ -21,7 +21,7 @@
#include <onnxruntime/coreml_provider_factory.h>
#endif
void testSigmoid(bool useCoreML) {
void testSigmoid(const char* modelPath, bool useCoreML) {
// This is an e2e test for ORT C++ API
Ort::Env env(ORT_LOGGING_LEVEL_WARNING, "testCppAPI");
@ -38,8 +38,7 @@ void testSigmoid(bool useCoreML) {
(void)useCoreML;
#endif
NSString* ns_model_path = [[NSBundle mainBundle] pathForResource:@"sigmoid" ofType:@"ort"];
Ort::Session session(env, ns_model_path.UTF8String, session_options);
Ort::Session session(env, modelPath, session_options);
size_t input_tensor_size = 3 * 4 * 5;
float input_tensor_values[input_tensor_size];
@ -78,19 +77,31 @@ void testSigmoid(bool useCoreML) {
- (void)setUp {
// Put setup code here. This method is called before the invocation of each test method in the class.
// In UI tests it is usually best to stop immediately when a failure occurs.
self.continueAfterFailure = YES;
// In UI tests it’s important to set the initial state - such as interface orientation - required for your tests before they run. The setUp method is a good place to do this.
}
- (void)tearDown {
// Put teardown code here. This method is called after the invocation of each test method in the class.
}
- (NSString*)getFilePath {
NSBundle* bundle = [NSBundle bundleForClass:[self class]];
NSString* ns_model_path = [bundle pathForResource:@"sigmoid" ofType:@"ort"];
XCTAssertNotNil(ns_model_path);
return ns_model_path;
}
- (void)testCppAPI_Basic {
testSigmoid(false /* useCoreML */);
testSigmoid([self getFilePath].UTF8String, false /* useCoreML */);
}
#if COREML_EP_AVAILABLE
- (void)testCppAPI_Basic_CoreML {
testSigmoid(true /* useCoreML */);
testSigmoid([self getFilePath].UTF8String, true /* useCoreML */);
}
#endif