Objective-C Add Support to Create and Query String ORTValues (#16764)

This pull request contains a few changes:

1. Adds support for string ort values.
2. Fixes the training minimal build (that was broken with #16601) by
putting custom op registration behind #ifdefs
3. Fixes the iOS pod package generation (that was again broken with
#16601) by explicitly providing paths to be copied during pod creation.

Package.swift

@@ -21,7 +21,7 @@ import class Foundation.ProcessInfo
 
 let package = Package(
     name: "onnxruntime",
-    platforms: [.iOS(.v11)],
+    platforms: [.iOS(.v12)],
     products: [
         .library(name: "onnxruntime",
                  type: .static,

objectivec/include/ort_enums.h

@@ -38,6 +38,7 @@ typedef NS_ENUM(int32_t, ORTTensorElementDataType) {
   ORTTensorElementDataTypeUInt32,
   ORTTensorElementDataTypeInt64,
   ORTTensorElementDataTypeUInt64,
+  ORTTensorElementDataTypeString,
 };
 
 /**

objectivec/include/ort_value.h

@@ -32,6 +32,21 @@ NS_ASSUME_NONNULL_BEGIN
                                       shape:(NSArray<NSNumber*>*)shape
                                       error:(NSError**)error;
 
+/**
+ * Creates a value that is a string tensor.
+ * The string data will be copied into a buffer owned by this ORTValue instance.
+ *
+ * Available since 1.16.
+ *
+ * @param tensorStringData The tensor string data.
+ * @param shape The tensor shape.
+ * @param error Optional error information set if an error occurs.
+ * @return The instance, or nil if an error occurs.
+ */
+- (nullable instancetype)initWithTensorStringData:(NSArray<NSString*>*)tensorStringData
+                                            shape:(NSArray<NSNumber*>*)shape
+                                            error:(NSError**)error;
+
 /**
  * Gets the type information.
  *
@@ -63,6 +78,19 @@ NS_ASSUME_NONNULL_BEGIN
  */
 - (nullable NSMutableData*)tensorDataWithError:(NSError**)error;
 
+/**
+ * Gets the tensor string data.
+ * This assumes that the value is a string tensor.
+ *
+ * This returns a copy of the value's underlying string data.
+ *
+ * Available since 1.16.
+ *
+ * @param error Optional error information set if an error occurs.
+ * @return The copy of the tensor string data, or nil if an error occurs.
+ */
+- (nullable NSArray<NSString*>*)tensorStringDataWithError:(NSError**)error;
+
 @end
 
 /**

objectivec/ort_enums.mm

@@ -4,6 +4,7 @@
 #import "ort_enums_internal.h"
 
 #include <algorithm>
+#include <optional>
 
 #import "cxx_api.h"
 
@@ -39,13 +40,13 @@ constexpr ValueTypeInfo kValueTypeInfos[]{
 struct TensorElementTypeInfo {
   ORTTensorElementDataType type;
   ONNXTensorElementDataType capi_type;
-  size_t element_size;
+  std::optional<size_t> element_size;
 };
 
 // supported ORT tensor element data types
 // define the mapping from ORTTensorElementDataType to C API ONNXTensorElementDataType here
 constexpr TensorElementTypeInfo kElementTypeInfos[]{
-    {ORTTensorElementDataTypeUndefined, ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED, 0},
+    {ORTTensorElementDataTypeUndefined, ONNX_TENSOR_ELEMENT_DATA_TYPE_UNDEFINED, std::nullopt},
     {ORTTensorElementDataTypeFloat, ONNX_TENSOR_ELEMENT_DATA_TYPE_FLOAT, sizeof(float)},
     {ORTTensorElementDataTypeInt8, ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8, sizeof(int8_t)},
     {ORTTensorElementDataTypeUInt8, ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT8, sizeof(uint8_t)},
@@ -53,6 +54,7 @@ constexpr TensorElementTypeInfo kElementTypeInfos[]{
     {ORTTensorElementDataTypeUInt32, ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT32, sizeof(uint32_t)},
     {ORTTensorElementDataTypeInt64, ONNX_TENSOR_ELEMENT_DATA_TYPE_INT64, sizeof(int64_t)},
     {ORTTensorElementDataTypeUInt64, ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT64, sizeof(uint64_t)},
+    {ORTTensorElementDataTypeString, ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING, std::nullopt},
 };
 
 struct GraphOptimizationLevelInfo {
@@ -119,9 +121,11 @@ ORTTensorElementDataType CAPIToPublicTensorElementType(ONNXTensorElementDataType
 size_t SizeOfCAPITensorElementType(ONNXTensorElementDataType capi_type) {
   return SelectAndTransform(
       kElementTypeInfos,
-      [capi_type](const auto& type_info) { return type_info.capi_type == capi_type; },
-      [](const auto& type_info) { return type_info.element_size; },
-      "unsupported tensor element type");
+      [capi_type](const auto& type_info) {
+        return type_info.element_size.has_value() && type_info.capi_type == capi_type;
+      },
+      [](const auto& type_info) { return *type_info.element_size; },
+      "unsupported tensor element type or tensor element type does not have a known size");
 }
 
 GraphOptimizationLevel PublicToCAPIGraphOptimizationLevel(ORTGraphOptimizationLevel opt_level) {

objectivec/ort_value.mm

@@ -71,6 +71,12 @@ bool SafeMultiply(size_t a, size_t b, size_t& out) {
                                       shape:(NSArray<NSNumber*>*)shape
                                       error:(NSError**)error {
   try {
+    if (elementType == ORTTensorElementDataTypeString) {
+      ORT_CXX_API_THROW(
+          "ORTTensorElementDataTypeString element type provided. "
+          "Please call initWithTensorStringData:shape:error: instead to create an ORTValue with string data.",
+          ORT_INVALID_ARGUMENT);
+    }
     const auto memoryInfo = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU);
     const auto ONNXElementType = PublicToCAPITensorElementType(elementType);
     const auto shapeVector = [shape]() {
@@ -92,6 +98,46 @@ bool SafeMultiply(size_t a, size_t b, size_t& out) {
   ORT_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error)
 }
 
+- (nullable instancetype)initWithTensorStringData:(NSArray<NSString*>*)tensorStringData
+                                            shape:(NSArray<NSNumber*>*)shape
+                                            error:(NSError**)error {
+  try {
+    Ort::AllocatorWithDefaultOptions allocator;
+    size_t tensorSize = 1U;
+    const auto shapeVector = [&tensorSize, shape]() {
+      std::vector<int64_t> result{};
+      result.reserve(shape.count);
+      for (NSNumber* dim in shape) {
+        const auto dimValue = dim.longLongValue;
+        if (dimValue < 0 || !SafeMultiply(static_cast<size_t>(dimValue), tensorSize, tensorSize)) {
+          ORT_CXX_API_THROW("Failed to compute the tensor size.", ORT_RUNTIME_EXCEPTION);
+        }
+        result.push_back(dimValue);
+      }
+      return result;
+    }();
+
+    if (tensorSize != [tensorStringData count]) {
+      ORT_CXX_API_THROW(
+          "Computed tensor size does not equal the length of the provided tensor string data.",
+          ORT_INVALID_ARGUMENT);
+    }
+
+    Ort::Value ortValue = Ort::Value::CreateTensor(
+        allocator, shapeVector.data(), shapeVector.size(), ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING);
+
+    size_t index = 0;
+    for (NSString* stringData in tensorStringData) {
+      ortValue.FillStringTensorElement([stringData UTF8String], index++);
+    }
+
+    return [self initWithCXXAPIOrtValue:std::move(ortValue)
+                     externalTensorData:nil
+                                  error:error];
+  }
+  ORT_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error)
+}
+
 - (nullable ORTValueTypeInfo*)typeInfoWithError:(NSError**)error {
   try {
     return CXXAPIToPublicValueTypeInfo(*_typeInfo);
@@ -110,6 +156,12 @@ bool SafeMultiply(size_t a, size_t b, size_t& out) {
 - (nullable NSMutableData*)tensorDataWithError:(NSError**)error {
   try {
     const auto tensorTypeAndShapeInfo = _typeInfo->GetTensorTypeAndShapeInfo();
+    if (tensorTypeAndShapeInfo.GetElementType() == ONNX_TENSOR_ELEMENT_DATA_TYPE_STRING) {
+      ORT_CXX_API_THROW(
+          "This ORTValue holds string data. Please call tensorStringDataWithError: "
+          "instead to retrieve the string data from this ORTValue.",
+          ORT_RUNTIME_EXCEPTION);
+    }
     const size_t elementCount = tensorTypeAndShapeInfo.GetElementCount();
     const size_t elementSize = SizeOfCAPITensorElementType(tensorTypeAndShapeInfo.GetElementType());
     size_t rawDataLength;
@@ -127,6 +179,29 @@ bool SafeMultiply(size_t a, size_t b, size_t& out) {
   ORT_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error)
 }
 
+- (nullable NSArray<NSString*>*)tensorStringDataWithError:(NSError**)error {
+  try {
+    const auto tensorTypeAndShapeInfo = _typeInfo->GetTensorTypeAndShapeInfo();
+    const size_t elementCount = tensorTypeAndShapeInfo.GetElementCount();
+    const size_t tensorStringDataLength = _value->GetStringTensorDataLength();
+    std::vector<char> tensorStringData(tensorStringDataLength, '\0');
+    std::vector<size_t> offsets(elementCount);
+    _value->GetStringTensorContent(tensorStringData.data(), tensorStringDataLength,
+                                   offsets.data(), offsets.size());
+
+    NSMutableArray<NSString*>* result = [NSMutableArray arrayWithCapacity:elementCount];
+    for (size_t idx = 0; idx < elementCount; ++idx) {
+      const size_t strLength = (idx == elementCount - 1) ? tensorStringDataLength - offsets[idx]
+                                                         : offsets[idx + 1] - offsets[idx];
+      [result addObject:[[NSString alloc] initWithBytes:tensorStringData.data() + offsets[idx]
+                                                 length:strLength
+                                               encoding:NSUTF8StringEncoding]];
+    }
+    return result;
+  }
+  ORT_OBJC_API_IMPL_CATCH_RETURNING_NULLABLE(error)
+}
+
 #pragma mark - Internal
 
 - (nullable instancetype)initWithCXXAPIOrtValue:(Ort::Value&&)existingCXXAPIOrtValue

objectivec/test/ort_session_test.mm

@@ -50,6 +50,13 @@ NS_ASSUME_NONNULL_BEGIN
   return path;
 }
 
++ (NSString*)getStringModelPath {
+  NSBundle* bundle = [NSBundle bundleForClass:[ORTSessionTest class]];
+  NSString* path = [bundle pathForResource:@"identity_string"
+                                    ofType:@"ort"];
+  return path;
+}
+
 + (NSMutableData*)dataWithScalarFloat:(float)value {
   NSMutableData* data = [[NSMutableData alloc] initWithBytes:&value length:sizeof(value)];
   return data;
@@ -259,6 +266,35 @@ static OrtStatus* _Nullable DummyRegisterCustomOpsFn(OrtSessionOptions* /*sessio
   XCTAssertEqual(gDummyRegisterCustomOpsFnCalled, true);
 }
 
+- (void)testStringInputs {
+  NSError* err = nil;
+  NSArray<NSString*>* stringData = @[ @"ONNX Runtime", @"is the", @"best", @"AI Framework" ];
+  ORTValue* stringValue = [[ORTValue alloc] initWithTensorStringData:stringData shape:@[ @2, @2 ] error:&err];
+  ORTAssertNullableResultSuccessful(stringValue, err);
+
+  ORTSession* session = [[ORTSession alloc] initWithEnv:self.ortEnv
+                                              modelPath:[ORTSessionTest getStringModelPath]
+                                         sessionOptions:[ORTSessionTest makeSessionOptions]
+                                                  error:&err];
+  ORTAssertNullableResultSuccessful(session, err);
+
+  NSDictionary<NSString*, ORTValue*>* outputs =
+      [session runWithInputs:@{@"input:0" : stringValue}
+                 outputNames:[NSSet setWithArray:@[ @"output:0" ]]
+                  runOptions:[ORTSessionTest makeRunOptions]
+                       error:&err];
+  ORTAssertNullableResultSuccessful(outputs, err);
+
+  ORTValue* outputStringValue = outputs[@"output:0"];
+  XCTAssertNotNil(outputStringValue);
+
+  NSArray<NSString*>* outputStringData = [outputStringValue tensorStringDataWithError:&err];
+  ORTAssertNullableResultSuccessful(outputStringData, err);
+
+  XCTAssertEqual([stringData count], [outputStringData count]);
+  XCTAssertTrue([stringData isEqualToArray:outputStringData]);
+}
+
 @end
 
 NS_ASSUME_NONNULL_END

objectivec/test/ort_value_test.mm

@@ -74,6 +74,19 @@ NS_ASSUME_NONNULL_BEGIN
   ORTAssertNullableResultUnsuccessful(ortValue, err);
 }
 
+- (void)testInitTensorWithStringDataSucceeds {
+  NSArray<NSString*>* stringData = @[ @"ONNX Runtime", @"is", @"the", @"best", @"AI", @"Framework" ];
+  NSError* err = nil;
+  ORTValue* stringValue = [[ORTValue alloc] initWithTensorStringData:stringData shape:@[ @3, @2 ] error:&err];
+  ORTAssertNullableResultSuccessful(stringValue, err);
+
+  NSArray<NSString*>* returnedStringData = [stringValue tensorStringDataWithError:&err];
+  ORTAssertNullableResultSuccessful(returnedStringData, err);
+
+  XCTAssertEqual([stringData count], [returnedStringData count]);
+  XCTAssertTrue([stringData isEqualToArray:returnedStringData]);
+}
+
 @end
 
 NS_ASSUME_NONNULL_END

orttraining/orttraining/training_api/module.cc

@@ -156,7 +156,7 @@ Module::Module(const std::string& train_model_path_or_bytes,
                const Environment& env,
                const std::vector<std::shared_ptr<IExecutionProvider>>& providers,
                const std::optional<std::string>& eval_model_path_or_bytes,
-               gsl::span<OrtCustomOpDomain* const> op_domains)
+               [[maybe_unused]] gsl::span<OrtCustomOpDomain* const> op_domains)
     : state_{state} {
   // Enforce weight prepacking is disabled
   // If the user explicitly enabled weight prepacking then return an error.
@@ -170,9 +170,11 @@ Module::Module(const std::string& train_model_path_or_bytes,
   }
 
   train_sess_ = std::make_unique<onnxruntime::InferenceSession>(session_options, env);
+#if !defined(ORT_MINIMAL_BUILD) || defined(ORT_MINIMAL_BUILD_CUSTOM_OPS)
   if (!op_domains.empty()) {
     ORT_THROW_IF_ERROR(train_sess_->AddCustomOpDomains(op_domains));
   }
+#endif
 
   ORT_THROW_IF_ERROR(train_sess_->Load(train_model_path_or_bytes));
   for (const auto& provider : providers) {
@@ -278,9 +280,11 @@ Module::Module(const std::string& train_model_path_or_bytes,
 
   if (eval_model_path_or_bytes.has_value()) {
     eval_sess_ = std::make_unique<onnxruntime::InferenceSession>(session_options, env);
+#if !defined(ORT_MINIMAL_BUILD) || defined(ORT_MINIMAL_BUILD_CUSTOM_OPS)
     if (!op_domains.empty()) {
       ORT_THROW_IF_ERROR(eval_sess_->AddCustomOpDomains(op_domains));
     }
+#endif
 
     ORT_THROW_IF_ERROR(eval_sess_->Load(eval_model_path_or_bytes.value()));
     for (const auto& provider : providers) {

orttraining/orttraining/training_api/optimizer.cc

@@ -225,10 +225,12 @@ Optimizer::Optimizer(const std::string& optim_path_or_bytes,
 
 void Optimizer::Initialize(const std::string& optim_path_or_bytes,
                            const std::vector<std::shared_ptr<IExecutionProvider>>& providers,
-                           gsl::span<OrtCustomOpDomain* const> op_domains) {
+                           [[maybe_unused]] gsl::span<OrtCustomOpDomain* const> op_domains) {
+#if !defined(ORT_MINIMAL_BUILD) || defined(ORT_MINIMAL_BUILD_CUSTOM_OPS)
   if (!op_domains.empty()) {
     ORT_THROW_IF_ERROR(optim_sess_->AddCustomOpDomains(op_domains));
   }
+#endif
 
   for (const auto& execution_provider : providers) {
     ORT_THROW_IF_ERROR(optim_sess_->RegisterExecutionProvider(execution_provider));

tools/ci_build/github/apple/objectivec/assemble_objc_pod_package.py

@@ -47,7 +47,10 @@ all_objc_files = {
     ],
     "test_resource_files": [
         "objectivec/test/testdata/*.ort",
-        "onnxruntime/test/testdata/training_api/*",
+        "onnxruntime/test/testdata/training_api/*.onnx",
+        "onnxruntime/test/testdata/training_api/*.ckpt",
+        "onnxruntime/test/testdata/training_api/*.in",
+        "onnxruntime/test/testdata/training_api/*.out",
     ],
 }
 
@@ -72,7 +75,10 @@ training_only_objc_files = {
         "objectivec/test/ort_training_utils_test.mm",
     ],
     "test_resource_files": [
-        "onnxruntime/test/testdata/training_api/*",
+        "onnxruntime/test/testdata/training_api/*.onnx",
+        "onnxruntime/test/testdata/training_api/*.ckpt",
+        "onnxruntime/test/testdata/training_api/*.in",
+        "onnxruntime/test/testdata/training_api/*.out",
     ],
 }