From b64b2d48f302361cd490c0cb3a801a1e1ebdf66c Mon Sep 17 00:00:00 2001 From: Guoyu Wang <62914304+gwang-msft@users.noreply.github.com> Date: Thu, 21 Oct 2021 18:51:13 -0700 Subject: [PATCH] Move iOS e2e test to XCUITest (#9422) * Move iOS test to user UITest * minor update * Update readme * update test's ios deployment target * address cr comments --- .../platform/ios/ios_package_test/Podfile | 2 +- .../platform/ios/ios_package_test/README.md | 31 +----- .../project.pbxproj | 104 +++++++++--------- .../ios_package_testTests/Info.plist | 22 ---- .../ios_package_uitest_cpp_api.mm} | 25 +++-- 5 files changed, 77 insertions(+), 107 deletions(-) delete mode 100644 onnxruntime/test/platform/ios/ios_package_test/ios_package_testTests/Info.plist rename onnxruntime/test/platform/ios/ios_package_test/{ios_package_testTests/ios_package_test_cpp_api.mm => ios_package_testUITests/ios_package_uitest_cpp_api.mm} (75%) diff --git a/onnxruntime/test/platform/ios/ios_package_test/Podfile b/onnxruntime/test/platform/ios/ios_package_test/Podfile index eac73cbb84..80cae9562c 100644 --- a/onnxruntime/test/platform/ios/ios_package_test/Podfile +++ b/onnxruntime/test/platform/ios/ios_package_test/Podfile @@ -4,7 +4,7 @@ target 'ios_package_test' do # Comment the next line if you don't want to use dynamic frameworks use_frameworks! - target 'ios_package_testTests' do + target 'ios_package_testUITests' do inherit! :search_paths pod 'onnxruntime-mobile-c', :podspec => './onnxruntime-mobile-c.podspec' end diff --git a/onnxruntime/test/platform/ios/ios_package_test/README.md b/onnxruntime/test/platform/ios/ios_package_test/README.md index 980804b708..96c3e055ee 100644 --- a/onnxruntime/test/platform/ios/ios_package_test/README.md +++ b/onnxruntime/test/platform/ios/ios_package_test/README.md @@ -4,7 +4,7 @@ This End-to-End test app for iOS will test ORT Mobile C/C++ API framework using ## Requirements -- [Prerequisites for building ORT-Mobile for iOS](http://www.onnxruntime.ai/docs/how-to/build/android-ios.html#prerequisites-1) +- [Prerequisites for building ORT-Mobile for iOS](https://onnxruntime.ai/docs/build/android-ios.html#prerequisites-1) - [CocoaPods](https://cocoapods.org/) ## iOS End-to-End Test App Overview @@ -14,34 +14,11 @@ The iOS End-to-End Test App will use CocoaPods to install the Onnx Runtime C/C++ ### Model used - [sigmoid ONNX model](https://github.com/onnx/onnx/blob/f9b0cc99344869c246b8f4011b8586a39841284c/onnx/backend/test/data/node/test_sigmoid/model.onnx) converted to ORT format - Here's the [document](http://www.onnxruntime.ai/docs/how-to/deploy-on-mobile.html#1-create-ort-format-model-and-configuration-file-with-required-operators) about how you can convert an ONNX model into ORT format. + Here's the [document](https://onnxruntime.ai/docs/tutorials/mobile/model-conversion.html) about how you can convert an ONNX model into ORT format. ### Tests -- [Tests for C API ](./ios_package_testTests/ios_package_test_c_api.m) -- [Tests for C++ API ](./ios_package_testTests/ios_package_test_cpp_api.mm) +- [Tests for C++ API ](./ios_package_testUITests/ios_package_uitest_cpp_api.mm) ## Build and Test iOS Framework using [build.py](../../../../../tools/ci_build/build.py) -Use the [build for iOS simulator](http://www.onnxruntime.ai/docs/how-to/build/android-ios.html#cross-build-for-ios-simulator) with `--build_apple_framework` - -## Run the iOS End-to-End Test App standalone - -### Requirements - -- A pre-built ORT Mobile iOS framework, which can be built using the [instruction](#build-and-test-ios-framework-using-buildpy) above. The framework can be found as `/iOS//-iphonesimulator/onnxruntime.framework` - -### Steps - -1. Go to this folder -2. Copy the [onnxruntime-mobile.podspec.template](./onnxruntime-mobile.podspec.template) to `onnxruntime-mobile.podspec` -3. Update the `onnxruntime-mobile.podspec`, replace `${ORT_BASE_FRAMEWORK_ARCHIVE}` with the path of a zip archive contains the pre-built ORT Mobile iOS framework -4. Run `pod install` to install the pre-built ORT Mobile iOS framework -5. Run the following command to perform the test - -``` - xcrun xcodebuild \ - -workspace ./ios_package_test.xcworkspace \ - -destination '' \ - -scheme ios_package_test \ - test -``` \ No newline at end of file +Use the [build for iOS simulator](https://onnxruntime.ai/docs/build/android-ios.html#cross-build-for-ios-simulator) with `--build_apple_framework` diff --git a/onnxruntime/test/platform/ios/ios_package_test/ios_package_test.xcodeproj/project.pbxproj b/onnxruntime/test/platform/ios/ios_package_test/ios_package_test.xcodeproj/project.pbxproj index c0fd060a05..151db69323 100644 --- a/onnxruntime/test/platform/ios/ios_package_test/ios_package_test.xcodeproj/project.pbxproj +++ b/onnxruntime/test/platform/ios/ios_package_test/ios_package_test.xcodeproj/project.pbxproj @@ -11,13 +11,13 @@ 229E592A265869BF006E41AE /* Main.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 229E5928265869BF006E41AE /* Main.storyboard */; }; 229E592F265869C2006E41AE /* LaunchScreen.storyboard in Resources */ = {isa = PBXBuildFile; fileRef = 229E592D265869C2006E41AE /* LaunchScreen.storyboard */; }; 229E5932265869C2006E41AE /* main.m in Sources */ = {isa = PBXBuildFile; fileRef = 229E5931265869C2006E41AE /* main.m */; }; - 229E593C265869C2006E41AE /* ios_package_test_cpp_api.mm in Sources */ = {isa = PBXBuildFile; fileRef = 229E593B265869C2006E41AE /* ios_package_test_cpp_api.mm */; }; 229E595926586B4A006E41AE /* sigmoid.ort in Resources */ = {isa = PBXBuildFile; fileRef = 229E595826586B4A006E41AE /* sigmoid.ort */; }; - 229E595A26586B4A006E41AE /* sigmoid.ort in Resources */ = {isa = PBXBuildFile; fileRef = 229E595826586B4A006E41AE /* sigmoid.ort */; }; + 22C1D8EA271A79FD002CEE67 /* ios_package_uitest_cpp_api.mm in Sources */ = {isa = PBXBuildFile; fileRef = 22C1D8E9271A79FD002CEE67 /* ios_package_uitest_cpp_api.mm */; }; + 22C1D8EB271A7A06002CEE67 /* sigmoid.ort in Resources */ = {isa = PBXBuildFile; fileRef = 229E595826586B4A006E41AE /* sigmoid.ort */; }; /* End PBXBuildFile section */ /* Begin PBXContainerItemProxy section */ - 229E5938265869C2006E41AE /* PBXContainerItemProxy */ = { + 22C1D8E4271A79AF002CEE67 /* PBXContainerItemProxy */ = { isa = PBXContainerItemProxy; containerPortal = 229E5914265869BF006E41AE /* Project object */; proxyType = 1; @@ -34,10 +34,9 @@ 229E592E265869C2006E41AE /* Base */ = {isa = PBXFileReference; lastKnownFileType = file.storyboard; name = Base; path = Base.lproj/LaunchScreen.storyboard; sourceTree = ""; }; 229E5930265869C2006E41AE /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; 229E5931265869C2006E41AE /* main.m */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.c.objc; path = main.m; sourceTree = ""; }; - 229E5937265869C2006E41AE /* ios_package_testTests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = ios_package_testTests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; - 229E593B265869C2006E41AE /* ios_package_test_cpp_api.mm */ = {isa = PBXFileReference; lastKnownFileType = sourcecode.cpp.objcpp; path = ios_package_test_cpp_api.mm; sourceTree = ""; }; - 229E593D265869C2006E41AE /* Info.plist */ = {isa = PBXFileReference; lastKnownFileType = text.plist.xml; path = Info.plist; sourceTree = ""; }; 229E595826586B4A006E41AE /* sigmoid.ort */ = {isa = PBXFileReference; lastKnownFileType = file; path = sigmoid.ort; sourceTree = ""; }; + 22C1D8DE271A79AF002CEE67 /* ios_package_testUITests.xctest */ = {isa = PBXFileReference; explicitFileType = wrapper.cfbundle; includeInIndex = 0; path = ios_package_testUITests.xctest; sourceTree = BUILT_PRODUCTS_DIR; }; + 22C1D8E9271A79FD002CEE67 /* ios_package_uitest_cpp_api.mm */ = {isa = PBXFileReference; fileEncoding = 4; lastKnownFileType = sourcecode.cpp.objcpp; path = ios_package_uitest_cpp_api.mm; sourceTree = ""; }; /* End PBXFileReference section */ /* Begin PBXFrameworksBuildPhase section */ @@ -48,7 +47,7 @@ ); runOnlyForDeploymentPostprocessing = 0; }; - 229E5934265869C2006E41AE /* Frameworks */ = { + 22C1D8DB271A79AF002CEE67 /* Frameworks */ = { isa = PBXFrameworksBuildPhase; buildActionMask = 2147483647; files = ( @@ -63,7 +62,7 @@ children = ( 229E595426586A77006E41AE /* models */, 229E591E265869BF006E41AE /* ios_package_test */, - 229E593A265869C2006E41AE /* ios_package_testTests */, + 22C1D8DF271A79AF002CEE67 /* ios_package_testUITests */, 229E591D265869BF006E41AE /* Products */, ); sourceTree = ""; @@ -72,7 +71,7 @@ isa = PBXGroup; children = ( 229E591C265869BF006E41AE /* ios_package_test.app */, - 229E5937265869C2006E41AE /* ios_package_testTests.xctest */, + 22C1D8DE271A79AF002CEE67 /* ios_package_testUITests.xctest */, ); name = Products; sourceTree = ""; @@ -90,15 +89,6 @@ path = ios_package_test; sourceTree = ""; }; - 229E593A265869C2006E41AE /* ios_package_testTests */ = { - isa = PBXGroup; - children = ( - 229E593B265869C2006E41AE /* ios_package_test_cpp_api.mm */, - 229E593D265869C2006E41AE /* Info.plist */, - ); - path = ios_package_testTests; - sourceTree = ""; - }; 229E595426586A77006E41AE /* models */ = { isa = PBXGroup; children = ( @@ -107,6 +97,14 @@ path = models; sourceTree = ""; }; + 22C1D8DF271A79AF002CEE67 /* ios_package_testUITests */ = { + isa = PBXGroup; + children = ( + 22C1D8E9271A79FD002CEE67 /* ios_package_uitest_cpp_api.mm */, + ); + path = ios_package_testUITests; + sourceTree = ""; + }; /* End PBXGroup section */ /* Begin PBXNativeTarget section */ @@ -127,23 +125,23 @@ productReference = 229E591C265869BF006E41AE /* ios_package_test.app */; productType = "com.apple.product-type.application"; }; - 229E5936265869C2006E41AE /* ios_package_testTests */ = { + 22C1D8DD271A79AF002CEE67 /* ios_package_testUITests */ = { isa = PBXNativeTarget; - buildConfigurationList = 229E594E265869C2006E41AE /* Build configuration list for PBXNativeTarget "ios_package_testTests" */; + buildConfigurationList = 22C1D8E8271A79AF002CEE67 /* Build configuration list for PBXNativeTarget "ios_package_testUITests" */; buildPhases = ( - 229E5933265869C2006E41AE /* Sources */, - 229E5934265869C2006E41AE /* Frameworks */, - 229E5935265869C2006E41AE /* Resources */, + 22C1D8DA271A79AF002CEE67 /* Sources */, + 22C1D8DB271A79AF002CEE67 /* Frameworks */, + 22C1D8DC271A79AF002CEE67 /* Resources */, ); buildRules = ( ); dependencies = ( - 229E5939265869C2006E41AE /* PBXTargetDependency */, + 22C1D8E5271A79AF002CEE67 /* PBXTargetDependency */, ); - name = ios_package_testTests; - productName = ios_package_testTests; - productReference = 229E5937265869C2006E41AE /* ios_package_testTests.xctest */; - productType = "com.apple.product-type.bundle.unit-test"; + name = ios_package_testUITests; + productName = ios_package_testUITests; + productReference = 22C1D8DE271A79AF002CEE67 /* ios_package_testUITests.xctest */; + productType = "com.apple.product-type.bundle.ui-testing"; }; /* End PBXNativeTarget section */ @@ -156,8 +154,8 @@ 229E591B265869BF006E41AE = { CreatedOnToolsVersion = 12.5; }; - 229E5936265869C2006E41AE = { - CreatedOnToolsVersion = 12.5; + 22C1D8DD271A79AF002CEE67 = { + CreatedOnToolsVersion = 13.0; TestTargetID = 229E591B265869BF006E41AE; }; }; @@ -176,7 +174,7 @@ projectRoot = ""; targets = ( 229E591B265869BF006E41AE /* ios_package_test */, - 229E5936265869C2006E41AE /* ios_package_testTests */, + 22C1D8DD271A79AF002CEE67 /* ios_package_testUITests */, ); }; /* End PBXProject section */ @@ -192,11 +190,11 @@ ); runOnlyForDeploymentPostprocessing = 0; }; - 229E5935265869C2006E41AE /* Resources */ = { + 22C1D8DC271A79AF002CEE67 /* Resources */ = { isa = PBXResourcesBuildPhase; buildActionMask = 2147483647; files = ( - 229E595A26586B4A006E41AE /* sigmoid.ort in Resources */, + 22C1D8EB271A7A06002CEE67 /* sigmoid.ort in Resources */, ); runOnlyForDeploymentPostprocessing = 0; }; @@ -212,21 +210,21 @@ ); runOnlyForDeploymentPostprocessing = 0; }; - 229E5933265869C2006E41AE /* Sources */ = { + 22C1D8DA271A79AF002CEE67 /* Sources */ = { isa = PBXSourcesBuildPhase; buildActionMask = 2147483647; files = ( - 229E593C265869C2006E41AE /* ios_package_test_cpp_api.mm in Sources */, + 22C1D8EA271A79FD002CEE67 /* ios_package_uitest_cpp_api.mm in Sources */, ); runOnlyForDeploymentPostprocessing = 0; }; /* End PBXSourcesBuildPhase section */ /* Begin PBXTargetDependency section */ - 229E5939265869C2006E41AE /* PBXTargetDependency */ = { + 22C1D8E5271A79AF002CEE67 /* PBXTargetDependency */ = { isa = PBXTargetDependency; target = 229E591B265869BF006E41AE /* ios_package_test */; - targetProxy = 229E5938265869C2006E41AE /* PBXContainerItemProxy */; + targetProxy = 22C1D8E4271A79AF002CEE67 /* PBXContainerItemProxy */; }; /* End PBXTargetDependency section */ @@ -396,41 +394,47 @@ }; name = Release; }; - 229E594F265869C2006E41AE /* Debug */ = { + 22C1D8E6271A79AF002CEE67 /* Debug */ = { isa = XCBuildConfiguration; buildSettings = { - BUNDLE_LOADER = "$(TEST_HOST)"; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++17"; CODE_SIGN_STYLE = Automatic; - INFOPLIST_FILE = ios_package_testTests/Info.plist; + CURRENT_PROJECT_VERSION = 1; + GENERATE_INFOPLIST_FILE = YES; IPHONEOS_DEPLOYMENT_TARGET = 13.0; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/Frameworks", "@loader_path/Frameworks", ); - PRODUCT_BUNDLE_IDENTIFIER = "ai.onnxruntime.tests.ios-package-testTests"; + MARKETING_VERSION = 1.0; + PRODUCT_BUNDLE_IDENTIFIER = "ai.onnxruntime.tests.ios-package-testUITests"; PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_EMIT_LOC_STRINGS = NO; TARGETED_DEVICE_FAMILY = "1,2"; - TEST_HOST = "$(BUILT_PRODUCTS_DIR)/ios_package_test.app/ios_package_test"; + TEST_TARGET_NAME = ios_package_test; }; name = Debug; }; - 229E5950265869C2006E41AE /* Release */ = { + 22C1D8E7271A79AF002CEE67 /* Release */ = { isa = XCBuildConfiguration; buildSettings = { - BUNDLE_LOADER = "$(TEST_HOST)"; + CLANG_CXX_LANGUAGE_STANDARD = "gnu++17"; CODE_SIGN_STYLE = Automatic; - INFOPLIST_FILE = ios_package_testTests/Info.plist; + CURRENT_PROJECT_VERSION = 1; + GENERATE_INFOPLIST_FILE = YES; IPHONEOS_DEPLOYMENT_TARGET = 13.0; LD_RUNPATH_SEARCH_PATHS = ( "$(inherited)", "@executable_path/Frameworks", "@loader_path/Frameworks", ); - PRODUCT_BUNDLE_IDENTIFIER = "ai.onnxruntime.tests.ios-package-testTests"; + MARKETING_VERSION = 1.0; + PRODUCT_BUNDLE_IDENTIFIER = "ai.onnxruntime.tests.ios-package-testUITests"; PRODUCT_NAME = "$(TARGET_NAME)"; + SWIFT_EMIT_LOC_STRINGS = NO; TARGETED_DEVICE_FAMILY = "1,2"; - TEST_HOST = "$(BUILT_PRODUCTS_DIR)/ios_package_test.app/ios_package_test"; + TEST_TARGET_NAME = ios_package_test; }; name = Release; }; @@ -455,11 +459,11 @@ defaultConfigurationIsVisible = 0; defaultConfigurationName = Release; }; - 229E594E265869C2006E41AE /* Build configuration list for PBXNativeTarget "ios_package_testTests" */ = { + 22C1D8E8271A79AF002CEE67 /* Build configuration list for PBXNativeTarget "ios_package_testUITests" */ = { isa = XCConfigurationList; buildConfigurations = ( - 229E594F265869C2006E41AE /* Debug */, - 229E5950265869C2006E41AE /* Release */, + 22C1D8E6271A79AF002CEE67 /* Debug */, + 22C1D8E7271A79AF002CEE67 /* Release */, ); defaultConfigurationIsVisible = 0; defaultConfigurationName = Release; diff --git a/onnxruntime/test/platform/ios/ios_package_test/ios_package_testTests/Info.plist b/onnxruntime/test/platform/ios/ios_package_test/ios_package_testTests/Info.plist deleted file mode 100644 index 64d65ca495..0000000000 --- a/onnxruntime/test/platform/ios/ios_package_test/ios_package_testTests/Info.plist +++ /dev/null @@ -1,22 +0,0 @@ - - - - - CFBundleDevelopmentRegion - $(DEVELOPMENT_LANGUAGE) - CFBundleExecutable - $(EXECUTABLE_NAME) - CFBundleIdentifier - $(PRODUCT_BUNDLE_IDENTIFIER) - CFBundleInfoDictionaryVersion - 6.0 - CFBundleName - $(PRODUCT_NAME) - CFBundlePackageType - $(PRODUCT_BUNDLE_PACKAGE_TYPE) - CFBundleShortVersionString - 1.0 - CFBundleVersion - 1 - - diff --git a/onnxruntime/test/platform/ios/ios_package_test/ios_package_testTests/ios_package_test_cpp_api.mm b/onnxruntime/test/platform/ios/ios_package_test/ios_package_testUITests/ios_package_uitest_cpp_api.mm similarity index 75% rename from onnxruntime/test/platform/ios/ios_package_test/ios_package_testTests/ios_package_test_cpp_api.mm rename to onnxruntime/test/platform/ios/ios_package_test/ios_package_testUITests/ios_package_uitest_cpp_api.mm index f44e9f673b..d145a00b13 100644 --- a/onnxruntime/test/platform/ios/ios_package_test/ios_package_testTests/ios_package_test_cpp_api.mm +++ b/onnxruntime/test/platform/ios/ios_package_test/ios_package_testUITests/ios_package_uitest_cpp_api.mm @@ -2,9 +2,9 @@ // Licensed under the MIT License. // // ios_package_test_cpp_api.mm -// ios_package_testTests +// ios_package_test_cpp_api // -// This file hosts the tests of ORT C++ API, for tests of ORT C API, please see ios_package_test_c_api.mm +// This file hosts the tests of ORT C++ API // #import @@ -21,7 +21,7 @@ #include #endif -void testSigmoid(bool useCoreML) { +void testSigmoid(const char* modelPath, bool useCoreML) { // This is an e2e test for ORT C++ API Ort::Env env(ORT_LOGGING_LEVEL_WARNING, "testCppAPI"); @@ -38,8 +38,7 @@ void testSigmoid(bool useCoreML) { (void)useCoreML; #endif - NSString* ns_model_path = [[NSBundle mainBundle] pathForResource:@"sigmoid" ofType:@"ort"]; - Ort::Session session(env, ns_model_path.UTF8String, session_options); + Ort::Session session(env, modelPath, session_options); size_t input_tensor_size = 3 * 4 * 5; float input_tensor_values[input_tensor_size]; @@ -78,19 +77,31 @@ void testSigmoid(bool useCoreML) { - (void)setUp { // Put setup code here. This method is called before the invocation of each test method in the class. + + // In UI tests it is usually best to stop immediately when a failure occurs. + self.continueAfterFailure = YES; + + // In UI tests it’s important to set the initial state - such as interface orientation - required for your tests before they run. The setUp method is a good place to do this. } - (void)tearDown { // Put teardown code here. This method is called after the invocation of each test method in the class. } +- (NSString*)getFilePath { + NSBundle* bundle = [NSBundle bundleForClass:[self class]]; + NSString* ns_model_path = [bundle pathForResource:@"sigmoid" ofType:@"ort"]; + XCTAssertNotNil(ns_model_path); + return ns_model_path; +} + - (void)testCppAPI_Basic { - testSigmoid(false /* useCoreML */); + testSigmoid([self getFilePath].UTF8String, false /* useCoreML */); } #if COREML_EP_AVAILABLE - (void)testCppAPI_Basic_CoreML { - testSigmoid(true /* useCoreML */); + testSigmoid([self getFilePath].UTF8String, true /* useCoreML */); } #endif