Reduce Heap contention in StringNormalizer (#20182)

### Description
<!-- Describe your changes. -->
Re-use pre-computed and pre-allocated buffers for UNICODE conversions.
Make sure we do not introduce unnecessary intermediate `std::string`
instances.
Create a Utf8Generic converter for use with non-Windows platforms.

### Motivation and Context
<!-- - Why is this change required? What problem does it solve?
- If it fixes an open issue, please link to the issue here. -->
This reduces heap contention in P1 customer.


![image](https://github.com/microsoft/onnxruntime/assets/11303988/fd39fb01-7361-47d2-8f83-69dbc3bbc65c)

onnxruntime/core/providers/cpu/text/string_normalizer.cc

@@ -4,27 +4,19 @@
 #include "string_normalizer.h"
 #include "core/common/common.h"
 #include "core/framework/tensor.h"
+// Used below HAS_DEPRECATED_DECLARATIONS
 #include "onnxruntime_config.h"
 
 #ifdef _MSC_VER
-#include <codecvt>
 #include <locale.h>
-#elif defined(__APPLE__) || defined(__ANDROID__)
-#include <codecvt>
-#else
-#include <limits>
-#include <iconv.h>
-
 #endif  // _MSC_VER
 
+#include <codecvt>
 #include <locale>
 #include <functional>
-#include <unordered_set>
 
 #if defined(__GNUC__)
-// Allow deprecated-declarations warning - std::wstring_convert is deprecated.
-// TODO find a suitable replacement
-// Note: GNU libiconv (e.g., on Apple platforms) is not suitable due to its LGPL license.
+// Allow deprecated-declarations warning - std::codecvt_utf8 is deprecatedd
 #if defined(HAS_DEPRECATED_DECLARATIONS)
 #pragma GCC diagnostic warning "-Wdeprecated-declarations"
 #endif  // defined(HAS_DEPRECATED_DECLARATIONS)
@@ -40,8 +32,196 @@ ONNX_CPU_OPERATOR_KERNEL(
     StringNormalizer);
 
 namespace string_normalizer {
-const std::string conv_error("Conversion Error");
-const std::wstring wconv_error(L"Conversion Error");
+
+// codecvt_utf8 is deprecated, we will want to replace it with our class
+class Utf8ConverterGeneric {
+ public:
+  size_t ComputeRequiredSizeToUtf8(const std::wstring& wstr) const {
+    if (wstr.empty()) {
+      return 0;
+    }
+
+    size_t result = 0;
+    std::mbstate_t state = std::mbstate_t();
+
+    const wchar_t* src = wstr.data();
+    const wchar_t* src_end = src + wstr.length();
+
+    char dummy_dest[128] = {0};
+
+    char* char_next = dummy_dest;
+    const wchar_t* wchar_next = src;
+
+    size_t converted = 0;
+
+    std::codecvt_base::result ret_code = std::codecvt_base::ok;
+
+    // Continue while we exhaust the sequence
+    while (converted < wstr.length()) {
+      ret_code = converter_.out(state,
+                                wchar_next,
+                                src_end,
+                                wchar_next,
+                                std::begin(dummy_dest),
+                                std::end(dummy_dest),
+                                char_next);
+      result += (char_next - dummy_dest);
+      converted = (wchar_next - src);
+
+      if (ret_code != std::codecvt_base::partial &&
+          ret_code != std::codecvt_base::ok) {
+        break;
+      }
+    }
+
+    ORT_ENFORCE(ret_code != std::codecvt_base::noconv, "Conversion is expected");
+
+    if (ret_code != std::codecvt_base::ok) {
+      ORT_THROW("Failed to compute size for UTF-8. Converted only first: ",
+                converted, " codepoints out of: ", wstr.length());
+    }
+
+    return result;
+  }
+
+  // We assume the caller pre-allocated the correct length
+  Status ConvertToUtf8(const std::wstring& wstr, std::string& str) const {
+    if (wstr.empty()) {
+      str.clear();
+      return Status::OK();
+    }
+
+    std::mbstate_t state = std::mbstate_t();
+
+    const wchar_t* src = wstr.data();
+    const wchar_t* src_end = src + wstr.length();
+
+    char* dest = str.data();
+    char* dest_end = dest + str.length();
+
+    char* char_next = dest;
+    const wchar_t* wchar_next = src;
+
+    std::codecvt_base::result ret_code = converter_.out(state,
+                                                        src,
+                                                        src_end,
+                                                        wchar_next,
+                                                        dest,
+                                                        dest_end,
+                                                        char_next);
+
+    if (ret_code != std::codecvt_base::ok) {
+      size_t converted = narrow<size_t>(wchar_next - wstr.data());
+      return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Failed to convert to UTF-8. Converted only first: ",
+                             converted, " codepoints out of: ", wstr.length());
+    }
+
+    str.resize(char_next - dest);
+
+    return Status::OK();
+  }
+
+  Status ComputeRequiredSizeToWideChar(const std::string& str, size_t& wchars) {
+    if (str.empty()) {
+      wchars = 0;
+      return Status::OK();
+    }
+
+    size_t result = 0;
+    std::mbstate_t state = std::mbstate_t();
+
+    const char* src = str.data();
+    const char* src_end = src + str.length();
+
+    wchar_t dummy_dest[128] = {0};
+    const char* char_next = src;
+    wchar_t* wchar_next = dummy_dest;
+
+    size_t converted = 0;
+
+    std::codecvt_base::result ret_code = std::codecvt_base::ok;
+    while (converted < str.length()) {
+      ret_code = converter_.in(state,
+                               char_next,
+                               src_end,
+                               char_next,
+                               std::begin(dummy_dest),
+                               std::end(dummy_dest),
+                               wchar_next);
+      result += (wchar_next - dummy_dest);
+      converted = (char_next - src);
+
+      if (ret_code != std::codecvt_base::partial &&
+          ret_code != std::codecvt_base::ok) {
+        break;
+      }
+    }
+
+    ORT_ENFORCE(ret_code != std::codecvt_base::noconv, "Conversion is expected");
+
+    if (ret_code != std::codecvt_base::ok) {
+      return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL,
+                             "Failed to compute buffer size for wchar_t. Converted only first: ",
+                             converted, " bytes out of: ", str.length(),
+                             " Source: ", src);
+    }
+
+    wchars = result;
+    return Status::OK();
+  }
+
+  // We assume the destination buffer is preallocated correctly
+  Status ConvertToWideChar(const std::string& str, std::wstring& wstr) {
+    if (str.empty()) {
+      // Preserve the buffer for re-use, just set size to 0
+      wstr.clear();
+      return Status::OK();
+    }
+
+    std::mbstate_t state = std::mbstate_t();
+    const char* src = str.data();
+    const char* src_end = src + str.length();
+
+    wchar_t* dest = wstr.data();
+    wchar_t* dest_end = dest + wstr.length();
+
+    const char* char_next = src;
+    wchar_t* wchar_next = dest;
+
+    std::codecvt_base::result ret_code = converter_.in(state,
+                                                       src,
+                                                       src_end,
+                                                       char_next,
+                                                       dest,
+                                                       dest_end,
+                                                       wchar_next);
+
+    if (ret_code != std::codecvt_base::ok) {
+      size_t converted = narrow<size_t>(char_next - str.data());
+      return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Failed to convert to wchar_t. Converted only first: ",
+                             converted, " bytes out of: ", str.length(),
+                             " Source: ", src);
+    }
+
+    wstr.resize(wchar_next - dest);
+
+    return Status::OK();
+  }
+
+  std::wstring from_bytes(const std::string& s) {
+    std::wstring result;
+
+    size_t wchars = 0;
+    ORT_THROW_IF_ERROR(ComputeRequiredSizeToWideChar(s, wchars));
+
+    result.resize(wchars);
+    ORT_THROW_IF_ERROR(ConvertToWideChar(s, result));
+    return result;
+  }
+
+ private:
+  std::codecvt_utf8<wchar_t> converter_;
+};
 
 // We need to specialize for MS as there is
 // a std::locale creation bug that affects different
@@ -83,10 +263,122 @@ class Locale {
   _locale_t loc_;
 };
 
-using Utf8Converter = std::wstring_convert<std::codecvt_utf8<wchar_t>>;
+class Utf8ConverterWindows {
+ public:
+  size_t ComputeRequiredSizeToUtf8(const std::wstring& wstr) const {
+    if (wstr.empty()) {
+      return 0;
+    }
+
+    int ret = WideCharToMultiByte(CP_UTF8,
+                                  0,
+                                  wstr.data(),
+                                  narrow<int>(wstr.length()),  // We specify the length so no trailing zero terminator
+                                  NULL,
+                                  0,      // indicates we need the buffer size.
+                                  NULL,   // Must be NULL for UTF-8
+                                  NULL);  // Must be NULL for UTF-8
+
+    // Failed. This is unlikely since the original UTF-8 to wchar_t succeeded.
+    // So we throw.
+    if (ret == 0) {
+      const auto error_code = GetLastError();
+      ORT_THROW("WideCharToMultiByte failed errcode = ",
+                error_code, " - ",
+                std::system_category().message(error_code));
+    }
+
+    return narrow<size_t>(ret);
+  }
+
+  Status ConvertToUtf8(const std::wstring& wstr, std::string& dest) const {
+    if (wstr.empty()) {
+      dest.clear();
+      return Status::OK();
+    }
+
+    const int ret = WideCharToMultiByte(CP_UTF8, 0,
+                                        wstr.data(),
+                                        narrow<int>(wstr.length()),
+                                        dest.data(),
+                                        narrow<int>(dest.length()),
+                                        nullptr,
+                                        nullptr);
+
+    if (ret == 0) {
+      const auto error_code = GetLastError();
+      return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "WideCharToMultiByte failed errcode = ",
+                             error_code, " - ",
+                             std::system_category().message(error_code));
+    }
+
+    dest.resize(narrow<size_t>(ret));
+    return Status::OK();
+  }
+
+  Status ComputeRequiredSizeToWideChar(const std::string& str, size_t& wchars) {
+    if (str.empty()) {
+      wchars = 0;
+      return Status::OK();
+    }
+
+    const int ret = MultiByteToWideChar(CP_UTF8,
+                                        MB_ERR_INVALID_CHARS,
+                                        str.data(),
+                                        narrow<int>(str.length()),
+                                        nullptr,
+                                        0);
+
+    if (ret == 0) {
+      const auto error_code = GetLastError();
+      return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "MultiByteToWideChar failed errcode = ",
+                             error_code, " - ",
+                             std::system_category().message(error_code));
+    }
+
+    wchars = narrow<size_t>(ret);
+    return Status::OK();
+  }
+
+  Status ConvertToWideChar(const std::string& str, std::wstring& wstr) {
+    if (str.empty()) {
+      // Preserve the buffer for re-use, just set size to 0
+      wstr.clear();
+      return Status::OK();
+    }
+
+    const int ret = MultiByteToWideChar(CP_UTF8, MB_ERR_INVALID_CHARS,
+                                        str.data(),
+                                        narrow<int>(str.length()),
+                                        wstr.data(),
+                                        narrow<int>(wstr.length()));
+
+    if (ret == 0) {
+      const auto error_code = GetLastError();
+      return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "MultiByteToWideChar failed errcode = ",
+                             error_code, " - ",
+                             std::system_category().message(error_code));
+    }
+
+    wstr.resize(narrow<size_t>(ret));
+    return Status::OK();
+  }
+
+  // Used only in the constructor to initialize stop_words
+  std::wstring from_bytes(const std::string& s) {
+    size_t size_required = 0;
+    ORT_THROW_IF_ERROR(ComputeRequiredSizeToWideChar(s, size_required));
+    std::wstring result;
+    result.resize(size_required);
+    ORT_THROW_IF_ERROR(ConvertToWideChar(s, result));
+    return result;
+  }
+};
 
 const std::string default_locale("en-US");
 
+using Utf8Converter = Utf8ConverterWindows;
+
 #else  // _MSC_VER
 
 class Locale {
@@ -123,81 +415,11 @@ class Locale {
 
 #if defined(__APPLE__) || defined(__ANDROID__)
 
-using Utf8Converter = std::wstring_convert<std::codecvt_utf8<wchar_t>>;
+using Utf8Converter = Utf8ConverterGeneric;
 
 #else
 
-// All others (not Windows, Apple, or Android)
-class Utf8Converter {
- public:
-  Utf8Converter(const std::string&, const std::wstring&) {}
-
-  std::wstring from_bytes(const std::string& s) const {
-    std::wstring result;
-    if (s.empty()) {
-      return result;
-    }
-    // Order of arguments is to, from
-    auto icvt = iconv_open("WCHAR_T", "UTF-8");
-    // CentOS is not happy with -1
-    if (std::numeric_limits<iconv_t>::max() == icvt) {
-      return wconv_error;
-    }
-
-    char* iconv_in = const_cast<char*>(s.c_str());
-    size_t iconv_in_bytes = s.length();
-    // Temporary buffer assumes 1 byte to 1 wchar_t
-    // to make sure it is enough.
-    const size_t buffer_len = iconv_in_bytes * sizeof(wchar_t);
-    auto buffer = std::make_unique<char[]>(buffer_len);
-    char* iconv_out = buffer.get();
-    size_t iconv_out_bytes = buffer_len;
-    auto ret = iconv(icvt, &iconv_in, &iconv_in_bytes, &iconv_out, &iconv_out_bytes);
-    if (static_cast<size_t>(-1) == ret) {
-      result = wconv_error;
-    } else {
-      size_t converted_bytes = buffer_len - iconv_out_bytes;
-      assert((converted_bytes % sizeof(wchar_t)) == 0);
-      result.assign(reinterpret_cast<const wchar_t*>(buffer.get()), converted_bytes / sizeof(wchar_t));
-    }
-    iconv_close(icvt);
-    return result;
-  }
-
-  std::string to_bytes(const std::wstring& wstr) const {
-    std::string result;
-    if (wstr.empty()) {
-      return result;
-    }
-    // Order of arguments is to, from
-    auto icvt = iconv_open("UTF-8", "WCHAR_T");
-    // CentOS is not happy with -1
-    if (std::numeric_limits<iconv_t>::max() == icvt) {
-      return conv_error;
-    }
-
-    // I hope this does not modify the incoming buffer
-    wchar_t* non_const_in = const_cast<wchar_t*>(wstr.c_str());
-    char* iconv_in = reinterpret_cast<char*>(non_const_in);
-    size_t iconv_in_bytes = wstr.length() * sizeof(wchar_t);
-    // Temp buffer, assume every code point converts into 3 bytes, this should be enough
-    // We do not convert terminating zeros
-    const size_t buffer_len = wstr.length() * 3;
-    auto buffer = std::make_unique<char[]>(buffer_len);
-
-    char* iconv_out = buffer.get();
-    size_t iconv_out_bytes = buffer_len;
-    auto ret = iconv(icvt, &iconv_in, &iconv_in_bytes, &iconv_out, &iconv_out_bytes);
-    if (static_cast<size_t>(-1) == ret) {
-      result = conv_error;
-    } else {
-      size_t converted_len = buffer_len - iconv_out_bytes;
-      result.assign(buffer.get(), converted_len);
-    }
-    iconv_close(icvt);
-    return result;
-  }
-};
+using Utf8Converter = Utf8ConverterGeneric;
 
 #endif
 
@@ -213,65 +435,11 @@ const std::string default_locale("en_US.UTF-8");  // All non-MS and not Apple
 #endif
 
 #endif  // _MSC_VER
-
-template <class ForwardIter>
-Status CopyCaseAction(ForwardIter first, ForwardIter end, OpKernelContext* ctx,
-                      const Locale& loc,
-                      Utf8Converter& converter,
-                      size_t N, size_t C,
-                      StringNormalizer::CaseAction caseaction) {
-  std::vector<int64_t> output_dims;
-  if (N == 1) {
-    output_dims.push_back(1);
-  }
-
-  // Empty output case
-  if (C == 0) {
-    output_dims.push_back(1);
-    TensorShape output_shape(output_dims);
-    // This will create one empty string
-    ctx->Output(0, output_shape);
-    return Status::OK();
-  }
-
-  output_dims.push_back(C);
-
-  TensorShape output_shape(output_dims);
-  auto output_tensor = ctx->Output(0, output_shape);
-  auto const output_data = output_tensor->MutableData<std::string>();
-
-  size_t output_idx = 0;
-  while (first != end) {
-    auto& s = *first;
-    if (caseaction == StringNormalizer::LOWER || caseaction == StringNormalizer::UPPER) {
-      std::wstring wstr = converter.from_bytes(s);
-      if (wstr == wconv_error) {
-        // Please do not include the input text in the error message as it could
-        // be deemed as a compliance violation by teams using this operator
-        return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT,
-                      "Input contains invalid utf8 chars");
-      }
-      // In place transform
-      loc.ChangeCase(caseaction, wstr);
-      *(output_data + output_idx) = converter.to_bytes(wstr);
-    } else {
-      assert(caseaction == StringNormalizer::NONE);
-      // Simple copy or move if the iterator points to a non-const string
-      *(output_data + output_idx) = std::move(s);
-    }
-    ++output_idx;
-    ++first;
-  }
-  return Status::OK();
-}
 }  // namespace string_normalizer
 
 using namespace string_normalizer;
 
-StringNormalizer::StringNormalizer(const OpKernelInfo& info) : OpKernel(info),
-                                                               is_case_sensitive_(true),
-                                                               case_change_action_(NONE),
-                                                               compare_caseaction_(NONE) {
+StringNormalizer::StringNormalizer(const OpKernelInfo& info) : OpKernel(info) {
   int64_t iscasesensitive = 0;
   Status status = info.GetAttr("is_case_sensitive", &iscasesensitive);
   ORT_ENFORCE(status.IsOK(), "attribute is_case_sensitive is not set");
@@ -290,27 +458,22 @@ StringNormalizer::StringNormalizer(const OpKernelInfo& info) : OpKernel(info),
     ORT_ENFORCE(false, "attribute case_change_action has invalid value");
   }
 
-  if (!is_case_sensitive_) {
-    // Convert stop words to a case which can help us preserve the case of filtered strings
-    compare_caseaction_ = (case_change_action_ == UPPER) ? UPPER : LOWER;
-  }
-
   locale_name_ = info.GetAttrOrDefault("locale", default_locale);
-  Locale locale(locale_name_);
-  Utf8Converter converter(conv_error, wconv_error);
 
-  std::vector<std::string> swords = info.GetAttrsOrDefault<std::string>("stopwords");
-  for (auto& sw : swords) {
-    ORT_ENFORCE(!sw.empty(), "Empty stopwords not allowed");
-    if (is_case_sensitive_) {
-      auto p = stopwords_.insert(std::move(sw));
-      ORT_ENFORCE(p.second, "Duplicate stopwords not allowed");
-    } else {
-      std::wstring wstr = converter.from_bytes(sw);
-      ORT_ENFORCE(wstr != wconv_error, "Stopword contains invalid utf8 chars");
+  std::vector<std::string> stop_words = info.GetAttrsOrDefault<std::string>("stopwords");
+  if (is_case_sensitive_) {
+    stopwords_.reserve(stop_words.size());
+    for (std::string& s : stop_words) {
+      stopwords_.insert(std::move(s));
+    }
+  } else {
+    Locale locale(locale_name_);
+    Utf8Converter converter;
+    wstopwords_.reserve(stop_words.size());
+    for (std::string& s : stop_words) {
+      std::wstring wstr = converter.from_bytes(s);
       locale.ChangeCase(compare_caseaction_, wstr);
-      auto p = wstopwords_.insert(std::move(wstr));
-      ORT_ENFORCE(p.second, "Duplicate stopwords not allowed");
+      wstopwords_.insert(std::move(wstr));
     }
   }
 }
@@ -319,95 +482,155 @@ Status StringNormalizer::Compute(OpKernelContext* ctx) const {
   using namespace string_normalizer;
 
   auto X = ctx->Input<Tensor>(0);
-  if (X == nullptr) return Status(common::ONNXRUNTIME, common::FAIL, "input count mismatch");
   auto input_dims = X->Shape().GetDims();
+  auto input_span = X->DataAsSpan<std::string>();
 
-  size_t N = 0;
-  size_t C = 0;
+  TensorShapeVector output_shape;
+  int64_t C = 0;
   if (input_dims.size() == 1) {
     if (input_dims[0] < 1) {
       return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT,
                     "Single dimension value must be greater than 0");
     }
-    C = onnxruntime::narrow<size_t>(input_dims[0]);
+    C = input_dims[0];
   } else if (input_dims.size() == 2) {
     if (input_dims[0] != 1 || input_dims[1] < 1) {
       return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT,
                     "Input dimensions are either[C > 0] or [1][C > 0] allowed");
     }
-    N = 1;
-    C = onnxruntime::narrow<size_t>(input_dims[1]);
+    output_shape.push_back(1);
+    C = input_dims[1];
   } else {
     return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT,
                   "Input dimensions are either[C > 0] or [1][C > 0] allowed");
   }
 
-  Status status;
+  // Special case, no filtering and no case change
+  if (case_change_action_ == NONE &&
+      ((is_case_sensitive_ && stopwords_.empty()) ||
+       (!is_case_sensitive_ && wstopwords_.empty()))) {
+    output_shape.push_back(C);
+    auto output_tensor = ctx->Output(0, output_shape);
+    auto const output_data = output_tensor->MutableData<std::string>();
+    std::copy(input_span.begin(), input_span.end(), output_data);
+    return Status::OK();
+  }
+
+  // We need to know the result dimension, and for that we need to filter
+  // the words first. If comparison mode is case sensitive, we just go ahead
+  // and compare with the original strings. Otherwise, we need to convert the string
+  // to widechar, lowercase it and then compare. Case-insensitive comparison is complicated
+  // for UTF-8 and requires additional dependency.
+
   Locale locale(locale_name_);
-  Utf8Converter converter(conv_error, wconv_error);
-  auto* const input_data = X->Data<std::string>();
-  using StrRef = std::reference_wrapper<const std::string>;
-  if (is_case_sensitive_) {
-    if (!stopwords_.empty()) {
-      InlinedVector<StrRef> filtered_strings;
-      filtered_strings.reserve(C);
-      auto first = input_data;
-      auto const last = input_data + C;
-      while (first != last) {
-        const std::string& s = *first;
-        if (0 == stopwords_.count(s)) {
-          filtered_strings.push_back(std::cref(s));
-        }
-        ++first;
+  Utf8Converter converter;
+
+  // Compute the largest widestring buffer needed.
+  size_t max_wide_buffer_len = 0;
+  for (const auto& s : input_span) {
+    size_t wchars = 0;
+    // Checks for invalid UTF-8 characters on Windows
+    ORT_RETURN_IF_ERROR(converter.ComputeRequiredSizeToWideChar(s, wchars));
+    max_wide_buffer_len = std::max(max_wide_buffer_len, wchars);
+  }
+
+  // Reuse reserved space
+  std::wstring wchar_buffer;
+  wchar_buffer.reserve(max_wide_buffer_len);
+
+  // Output everything and change case as required
+  auto output_no_filtering = [&](const TensorShape& output_shape) {
+    auto output_tensor = ctx->Output(0, output_shape);
+    auto const output_data = output_tensor->MutableData<std::string>();
+    for (size_t i = 0, lim = input_span.size(); i < lim; ++i) {
+      const std::string& s = input_span[i];
+      wchar_buffer.resize(max_wide_buffer_len);
+      ORT_RETURN_IF_ERROR(converter.ConvertToWideChar(s, wchar_buffer));
+      locale.ChangeCase(case_change_action_, wchar_buffer);
+
+      auto& dest = output_data[i];
+      size_t utf8_buffer_len = converter.ComputeRequiredSizeToUtf8(wchar_buffer);
+      dest.resize(utf8_buffer_len);
+      ORT_RETURN_IF_ERROR(converter.ConvertToUtf8(wchar_buffer, dest));
+    }
+    return Status::OK();
+  };
+
+  auto output_filtered = [&](const TensorShape& output_shape, gsl::span<const size_t> filtered_indices) {
+    auto output_tensor = ctx->Output(0, output_shape);
+    auto output_data = output_tensor->MutableData<std::string>();
+    for (size_t i : filtered_indices) {
+      const std::string& s = input_span[i];
+      if (case_change_action_ != NONE) {
+        wchar_buffer.resize(max_wide_buffer_len);
+        ORT_RETURN_IF_ERROR(converter.ConvertToWideChar(s, wchar_buffer));
+        locale.ChangeCase(case_change_action_, wchar_buffer);
+
+        auto& dest = *output_data++;
+        size_t utf8_buffer_len = converter.ComputeRequiredSizeToUtf8(wchar_buffer);
+        dest.resize(utf8_buffer_len);
+        ORT_RETURN_IF_ERROR(converter.ConvertToUtf8(wchar_buffer, dest));
+      } else {
+        *output_data++ = s;
       }
-      status = CopyCaseAction(filtered_strings.cbegin(), filtered_strings.cend(), ctx, locale, converter,
-                              N, filtered_strings.size(), case_change_action_);
+    }
+    return Status::OK();
+  };
+
+  Status status;
+
+  if (is_case_sensitive_) {
+    if (stopwords_.empty()) {
+      assert(case_change_action_ != NONE);
+      output_shape.push_back(C);
+      status = output_no_filtering(output_shape);
     } else {
-      // Nothing to filter. Copy input to output and change case if needed
-      status = CopyCaseAction(input_data, input_data + C, ctx, locale, converter, N, C, case_change_action_);
+      // we need to filter
+      InlinedVector<size_t> filtered_strings_indices;
+      filtered_strings_indices.reserve(input_span.size());
+
+      for (size_t i = 0, lim = input_span.size(); i < lim; ++i) {
+        const std::string& s = input_span[i];
+        if (stopwords_.count(s) == 0) {
+          filtered_strings_indices.push_back(i);
+        }
+      }
+
+      // According to the spec, if all strings are filtered out
+      // the output must have a shape of {1} with a single empty string.
+      const int64_t filtered_count = std::max<int64_t>(1, narrow<int64_t>(filtered_strings_indices.size()));
+      output_shape.push_back(filtered_count);
+      status = output_filtered(output_shape, filtered_strings_indices);
     }
   } else {
-    if (!wstopwords_.empty()) {
-      // Filter input. When no case action is required
-      // we simply store original string references.
-      // Otherwise, we store converted strings.
-      InlinedVector<StrRef> filtered_orignal_strings;
-      InlinedVector<std::string> filtered_cased_strings;
-      filtered_orignal_strings.reserve(C);
-      filtered_cased_strings.reserve(C);
-      auto first = input_data;
-      auto const last = input_data + C;
-      while (first != last) {
-        const std::string& s = *first;
-        std::wstring wstr = converter.from_bytes(s);
-        if (wstr == wconv_error) {
-          // Please do not include the input text in the error message as it could
-          // be deemed as a compliance violation by teams using this operator
-          return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT,
-                        "Input contains invalid utf8 chars");
-        }
-        locale.ChangeCase(compare_caseaction_, wstr);
-        if (0 == wstopwords_.count(wstr)) {
-          if (case_change_action_ == NONE) {
-            filtered_orignal_strings.push_back(std::cref(s));
-          } else {
-            filtered_cased_strings.push_back(converter.to_bytes(wstr));
-          }
-        }
-        ++first;
-      }
-      if (case_change_action_ == NONE) {
-        status = CopyCaseAction(filtered_orignal_strings.cbegin(), filtered_orignal_strings.cend(), ctx, locale, converter,
-                                N, filtered_orignal_strings.size(), NONE);
-      } else {
-        status = CopyCaseAction(filtered_cased_strings.begin(), filtered_cased_strings.end(), ctx, locale, converter,
-                                N, filtered_cased_strings.size(), NONE);
-      }
+    if (wstopwords_.empty()) {
+      assert(case_change_action_ != NONE);
+      output_shape.push_back(C);
+      status = output_no_filtering(output_shape);
     } else {
-      // Nothing to filter. Copy input to output and change case if needed
-      status = CopyCaseAction(input_data, input_data + C, ctx, locale, converter, N, C, case_change_action_);
+      // Case insensitive filtering is performed by converting the input strings
+      // to compare_caseaction_. For that we convert to wchar_t UNICODE.
+      // Otherwise, we need to pull ICU library on all platforms.
+      InlinedVector<size_t> filtered_strings_indices;
+      filtered_strings_indices.reserve(input_span.size());
+      for (size_t i = 0, lim = input_span.size(); i < lim; ++i) {
+        const std::string& s = input_span[i];
+        wchar_buffer.resize(max_wide_buffer_len);
+        ORT_RETURN_IF_ERROR(converter.ConvertToWideChar(s, wchar_buffer));
+        locale.ChangeCase(compare_caseaction_, wchar_buffer);
+        if (wstopwords_.count(wchar_buffer) == 0) {
+          filtered_strings_indices.push_back(i);
+        }
+      }
+
+      // According to the spec, if all strings are filtered out
+      // the output must have a shape of {1} with a single empty string.
+      const int64_t filtered_count = std::max<int64_t>(1, narrow<int64_t>(filtered_strings_indices.size()));
+      output_shape.push_back(filtered_count);
+      status = output_filtered(output_shape, filtered_strings_indices);
     }
   }
+
   return status;
 }
 }  // namespace onnxruntime

onnxruntime/core/providers/cpu/text/string_normalizer.h

@@ -25,9 +25,11 @@ class StringNormalizer : public OpKernel {
   Status Compute(OpKernelContext* ctx) const override;
 
  private:
-  bool is_case_sensitive_;
-  CaseAction case_change_action_;
-  CaseAction compare_caseaction_;  // used for case-insensitive compare
+  bool is_case_sensitive_{true};
+  CaseAction case_change_action_{NONE};
+  // Set this to lower because some characters do not have capital case.
+  // used for case-insensitive compare
+  CaseAction compare_caseaction_{LOWER};
   std::string locale_name_;
   // Either if these are populated but not both
   InlinedHashSet<std::string> stopwords_;

onnxruntime/test/providers/cpu/text/string_normalizer_test.cc

@@ -1,11 +1,16 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
-#if ((__cplusplus >= 201703L) || (defined(_MSVC_LANG) && (_MSVC_LANG >= 201703L)))
-// TODO: handle the u8string.
-#else
+
 #include "gtest/gtest.h"
 #include "test/providers/provider_test_utils.h"
 
+#ifdef __APPLE__
+#include <TargetConditionals.h>
+#if TARGET_OS_IOS
+#define ORT_IOS
+#endif
+#endif
+
 namespace onnxruntime {
 namespace test {
 
@@ -19,10 +24,10 @@ const std::string test_locale("en-US");
 const std::string test_locale("en_US.UTF-8");
 #endif
 
-void InitTestAttr(OpTester& test, const std::string& case_change_action,
-                  bool is_case_sensitive,
-                  const std::vector<std::string>& stopwords,
-                  const std::string& locale) {
+static void InitTestAttr(OpTester& test, const std::string& case_change_action,
+                         bool is_case_sensitive,
+                         const std::vector<std::string>& stopwords,
+                         const std::string& locale) {
   if (!case_change_action.empty()) {
     test.AddAttribute("case_change_action", case_change_action);
   }
@@ -38,170 +43,194 @@ void InitTestAttr(OpTester& test, const std::string& case_change_action,
 
 using namespace str_normalizer_test;
 
-TEST(ContribOpTest, StringNormalizerTest) {
+TEST(ContribOpTest, StringNormalizerSensitiveNoCase) {
   // - casesensitive approach
   // - no stopwords.
   // - No change case action, expecting default to take over
-  {
-    OpTester test("StringNormalizer", opset_ver, domain);
-    InitTestAttr(test, "", true, {}, test_locale);
-    std::vector<int64_t> dims{4};
-    std::vector<std::string> input = {std::string("monday"), std::string("tuesday"),
-                                      std::string("wednesday"), std::string("thursday")};
-    test.AddInput<std::string>("T", dims, input);
-    std::vector<std::string> output(input);  // do the same for now
-    test.AddOutput<std::string>("Y", dims, output);
-    test.Run(OpTester::ExpectResult::kExpectSuccess);
-  }
+  OpTester test("StringNormalizer", opset_ver, domain);
+  InitTestAttr(test, "", true, {}, test_locale);
+  std::vector<int64_t> dims{4};
+  std::vector<std::string> input = {std::string("monday"), std::string("tuesday"),
+                                    std::string("wednesday"), std::string("thursday")};
+  test.AddInput<std::string>("T", dims, input);
+  std::vector<std::string> output(input);  // do the same for now
+  test.AddOutput<std::string>("Y", dims, output);
+  test.Run(OpTester::ExpectResult::kExpectSuccess);
+}
+
+TEST(ContribOpTest, StringNormalizerSensitiveFilterOutNoCase) {
   // - casesensitive approach
   // - filter out monday
   // - No change case action
-  {
-    OpTester test("StringNormalizer", opset_ver, domain);
-    InitTestAttr(test, "NONE", true, {"monday"}, test_locale);
-    std::vector<int64_t> dims{4};
-    std::vector<std::string> input = {std::string("monday"), std::string("tuesday"),
-                                      std::string("wednesday"), std::string("thursday")};
-    test.AddInput<std::string>("T", dims, input);
 
-    std::vector<std::string> output = {std::string("tuesday"),
-                                       std::string("wednesday"), std::string("thursday")};
-    test.AddOutput<std::string>("Y", {3}, output);
-    test.Run(OpTester::ExpectResult::kExpectSuccess);
-  }
+  OpTester test("StringNormalizer", opset_ver, domain);
+  InitTestAttr(test, "NONE", true, {"monday"}, test_locale);
+  std::vector<int64_t> dims{4};
+  std::vector<std::string> input = {std::string("monday"), std::string("tuesday"),
+                                    std::string("wednesday"), std::string("thursday")};
+  test.AddInput<std::string>("T", dims, input);
+
+  std::vector<std::string> output = {std::string("tuesday"),
+                                     std::string("wednesday"), std::string("thursday")};
+  test.AddOutput<std::string>("Y", {3}, output);
+  test.Run(OpTester::ExpectResult::kExpectSuccess);
+}
+
+TEST(ContribOpTest, StringNormalizerSensitiveFilterOutLower) {
   // - casesensitive approach
   // - filter out monday
   // - LOWER should produce the same output as they are all lower.
-  {
-    OpTester test("StringNormalizer", opset_ver, domain);
-    InitTestAttr(test, "LOWER", true, {"monday"}, test_locale);
-    std::vector<int64_t> dims{4};
-    std::vector<std::string> input = {std::string("monday"), std::string("tuesday"),
-                                      std::string("wednesday"), std::string("thursday")};
-    test.AddInput<std::string>("T", dims, input);
+  OpTester test("StringNormalizer", opset_ver, domain);
+  InitTestAttr(test, "LOWER", true, {"monday"}, test_locale);
+  std::vector<int64_t> dims{4};
+  std::vector<std::string> input = {std::string("monday"), std::string("tuesday"),
+                                    std::string("wednesday"), std::string("thursday")};
+  test.AddInput<std::string>("T", dims, input);
 
-    std::vector<std::string> output = {std::string("tuesday"),
-                                       std::string("wednesday"), std::string("thursday")};
-    test.AddOutput<std::string>("Y", {3}, output);
-    test.Run(OpTester::ExpectResult::kExpectSuccess);
-  }
-  // - casesensitive approach
-  // - filter out monday
-  // - UPPER should produce the same output as they are all lower.
-  {
-    OpTester test("StringNormalizer", opset_ver, domain);
-    InitTestAttr(test, "UPPER", true, {"monday"}, test_locale);
-    std::vector<int64_t> dims{4};
-    std::vector<std::string> input = {std::string("monday"), std::string("tuesday"),
-                                      std::string("wednesday"), std::string("thursday")};
-    test.AddInput<std::string>("T", dims, input);
-
-    std::vector<std::string> output = {std::string("TUESDAY"),
-                                       std::string("WEDNESDAY"), std::string("THURSDAY")};
-    test.AddOutput<std::string>("Y", {3}, output);
-    test.Run(OpTester::ExpectResult::kExpectSuccess);
-  }
-  // - case-SENSETIVE approach en_US locale
-  // - we test the behavior of a mix of english, french, german, russian and chinese
-  //   with en_US locale
-  // - filter out monday
-  // - UPPER should produce the same output as they are all lower.
-  {
-    OpTester test("StringNormalizer", opset_ver, domain);
-    InitTestAttr(test, "UPPER", true, {u8"monday"}, test_locale);
-    std::vector<int64_t> dims{7};
-    std::vector<std::string> input = {std::string(u8"monday"),
-                                      std::string(u8"tuesday"),
-                                      std::string(u8"Besançon"),
-                                      std::string(u8"École élémentaire"),
-                                      std::string(u8"Понедельник"),
-                                      std::string(u8"mit freundlichen grüßen"),
-                                      std::string(u8"中文")};
-    test.AddInput<std::string>("T", dims, input);
-
-    // en_US results (default)
-    std::vector<std::string> output = {std::string(u8"TUESDAY"),
-                                       // It does upper case cecedille, accented E
-                                       // and german umlaut but fails
-                                       // with german eszett
-                                       std::string(u8"BESANÇON"),
-                                       std::string(u8"ÉCOLE ÉLÉMENTAIRE"),
-                                       // No issues with Cyrllic
-                                       std::string(u8"ПОНЕДЕЛЬНИК"),
-                                       std::string(u8"MIT FREUNDLICHEN GRÜßEN"),
-                                       // Chinese do not have cases
-                                       std::string(u8"中文")};
-    test.AddOutput<std::string>("Y", {6}, output);
-    test.Run(OpTester::ExpectResult::kExpectSuccess);
-  }
-  // - case-INSENSETIVE approach en_US locale
-  // - we test the behavior of a mix of english, french, german, russian and chinese
-  //   with en_US locale
-  // - filter out monday
-  // - UPPER should produce the same output as they are all lower.
-  {
-    OpTester test("StringNormalizer", opset_ver, domain);
-    InitTestAttr(test, "UPPER", false, {u8"monday"}, test_locale);
-    std::vector<int64_t> dims{7};
-    std::vector<std::string> input = {std::string(u8"monday"),
-                                      std::string(u8"tuesday"),
-                                      std::string(u8"Besançon"),
-                                      std::string(u8"École élémentaire"),
-                                      std::string(u8"Понедельник"),
-                                      std::string(u8"mit freundlichen grüßen"),
-                                      std::string(u8"中文")};
-    test.AddInput<std::string>("T", dims, input);
-
-    // en_US results (default)
-    std::vector<std::string> output = {std::string(u8"TUESDAY"),
-                                       // It does upper case cecedille, accented E
-                                       // and german umlaut but fails
-                                       // with german eszett
-                                       std::string(u8"BESANÇON"),
-                                       std::string(u8"ÉCOLE ÉLÉMENTAIRE"),
-                                       // No issues with Cyrllic
-                                       std::string(u8"ПОНЕДЕЛЬНИК"),
-                                       std::string(u8"MIT FREUNDLICHEN GRÜßEN"),
-                                       // Chinese do not have cases
-                                       std::string(u8"中文")};
-    test.AddOutput<std::string>("Y", {6}, output);
-    test.Run(OpTester::ExpectResult::kExpectSuccess);
-  }
-
-  // Empty output case
-  // - casesensitive approach
-  // - filter out monday
-  // - UPPER should produce the same output as they are all lower.
-  {
-    OpTester test("StringNormalizer", opset_ver, domain);
-    InitTestAttr(test, "UPPER", true, {"monday"}, test_locale);
-    std::vector<int64_t> dims{2};
-    std::vector<std::string> input = {std::string("monday"),
-                                      std::string("monday")};
-    test.AddInput<std::string>("T", dims, input);
-
-    std::vector<std::string> output{""};  // One empty string
-    test.AddOutput<std::string>("Y", {1}, output);
-    test.Run(OpTester::ExpectResult::kExpectSuccess);
-  }
-  // Empty output case
-  // - casesensitive approach
-  // - filter out monday
-  // - UPPER should produce the same output as they are all lower.
-  {
-    OpTester test("StringNormalizer", opset_ver, domain);
-    InitTestAttr(test, "UPPER", true, {"monday"}, "");
-    std::vector<int64_t> dims{1, 2};
-    std::vector<std::string> input = {std::string("monday"),
-                                      std::string("monday")};
-    test.AddInput<std::string>("T", dims, input);
-
-    std::vector<std::string> output{""};  // One empty string
-    test.AddOutput<std::string>("Y", {1, 1}, output);
-    test.Run(OpTester::ExpectResult::kExpectSuccess);
-  }
+  std::vector<std::string> output = {std::string("tuesday"),
+                                     std::string("wednesday"), std::string("thursday")};
+  test.AddOutput<std::string>("Y", {3}, output);
+  test.Run(OpTester::ExpectResult::kExpectSuccess);
 }
 
+TEST(ContribOpTest, StringNormalizerSensitiveFilterOutUpper) {
+  // - casesensitive approach
+  // - filter out monday
+  // - UPPER should produce the same output as they are all lower.
+
+  OpTester test("StringNormalizer", opset_ver, domain);
+  InitTestAttr(test, "UPPER", true, {"monday"}, test_locale);
+  std::vector<int64_t> dims{4};
+  std::vector<std::string> input = {std::string("monday"), std::string("tuesday"),
+                                    std::string("wednesday"), std::string("thursday")};
+  test.AddInput<std::string>("T", dims, input);
+
+  std::vector<std::string> output = {std::string("TUESDAY"),
+                                     std::string("WEDNESDAY"), std::string("THURSDAY")};
+  test.AddOutput<std::string>("Y", {3}, output);
+  test.Run(OpTester::ExpectResult::kExpectSuccess);
+}
+
+TEST(ContribOpTest, StringNormalizerSensitiveFilterOutUpperWithLocale) {
+  // - case-SENSITIVE approach en_US locale
+  // - we test the behavior of a mix of english, french, german, russian and chinese
+  //   with en_US locale
+  // - filter out monday
+  // - UPPER should produce the same output as they are all lower.
+
+  OpTester test("StringNormalizer", opset_ver, domain);
+  InitTestAttr(test, "UPPER", true, {"monday"}, test_locale);
+  std::vector<int64_t> dims{7};
+  std::vector<std::string> input = {"monday",
+                                    "tuesday",
+                                    "Besançon",
+                                    "École élémentaire",
+                                    "Понедельник",
+                                    "mit freundlichen grüßen",
+                                    "中文"};
+  test.AddInput<std::string>("T", dims, input);
+
+  // en_US results (default)
+  std::vector<std::string> output = {"TUESDAY",
+                                     // It does upper case cecedille, accented E
+                                     "BESANÇON",
+                                     "ÉCOLE ÉLÉMENTAIRE",
+                                     // No issues with Cyrllic
+                                     "ПОНЕДЕЛЬНИК",
+  // Works with german umlaut but fails
+  // with german Eszett. Reason being, capital case for Eszett
+  // was introduced only recently into encodings
+  // and some platforms produce it, but others do not
+#ifdef __wasm__
+                                     "MIT FREUNDLICHEN GRÜẞEN",
+#else
+                                     "MIT FREUNDLICHEN GRÜßEN",
+#endif
+                                     // Chinese do not have cases
+                                     "中文"};
+  test.AddOutput<std::string>("Y", {6}, output);
+  test.Run(OpTester::ExpectResult::kExpectSuccess);
+}
+
+TEST(ContribOpTest, StringNormalizerInsensitiveFilterOutUpperWithLocale) {
+  // - case-INSENSITIVE approach en_US locale
+  // - we test the behavior of a mix of english, french, german, russian and chinese
+  //   with en_US locale
+  // - filter out monday
+  // - UPPER should produce the same output as they are all lower.
+
+  OpTester test("StringNormalizer", opset_ver, domain);
+  InitTestAttr(test, "UPPER", false, {"monday"}, test_locale);
+  std::vector<int64_t> dims{7};
+  std::vector<std::string> input = {"monday",
+                                    "tuesday",
+                                    "Besançon",
+                                    "École élémentaire",
+                                    "Понедельник",
+                                    "mit freundlichen grüßen",
+                                    "中文"};
+  test.AddInput<std::string>("T", dims, input);
+
+  // en_US results (default)
+  std::vector<std::string> output = {"TUESDAY",
+                                     // It does upper case cecedille, and accented E
+                                     "BESANÇON",
+                                     "ÉCOLE ÉLÉMENTAIRE",
+                                     // No issues with Cyrllic
+                                     "ПОНЕДЕЛЬНИК",
+  // Works with german umlaut but fails
+  // with german Eszett (ß). Reason being, capital case for Eszett
+  // was introduced only recently into encodings
+  // and some platforms produce it, but others do not
+#ifdef __wasm__
+                                     "MIT FREUNDLICHEN GRÜẞEN",
+#else
+                                     "MIT FREUNDLICHEN GRÜßEN",
+#endif
+
+                                     // Chinese do not have cases
+                                     "中文"};
+  test.AddOutput<std::string>("Y", {6}, output);
+  test.Run(OpTester::ExpectResult::kExpectSuccess);
+}
+
+TEST(ContribOpTest, StringNormalizerSensitiveFilterOutUpperEmptyCase) {
+  // Empty output case
+  // - casesensitive approach
+  // - filter out monday
+  // - UPPER should produce the same output as they are all lower.
+
+  OpTester test("StringNormalizer", opset_ver, domain);
+  InitTestAttr(test, "UPPER", true, {"monday"}, test_locale);
+  std::vector<int64_t> dims{2};
+  std::vector<std::string> input = {"monday", "monday"};
+  test.AddInput<std::string>("T", dims, input);
+
+  std::vector<std::string> output{""};  // One empty string
+  test.AddOutput<std::string>("Y", {1}, output);
+  test.Run(OpTester::ExpectResult::kExpectSuccess);
+}
+
+// Fails on iOS because necessary locales are not installed
+// MacOS runs fine.
+#ifndef ORT_IOS
+TEST(ContribOpTest, StringNormalizerSensitiveFilterOutUpperSameOutput) {
+  // Empty output case
+  // - casesensitive approach
+  // - filter out monday
+  // - UPPER should produce the same output as they are all lower.
+  OpTester test("StringNormalizer", opset_ver, domain);
+  InitTestAttr(test, "UPPER", true, {"monday"}, "");
+  std::vector<int64_t> dims{1, 2};
+  std::vector<std::string> input = {std::string("monday"),
+                                    std::string("monday")};
+  test.AddInput<std::string>("T", dims, input);
+
+  std::vector<std::string> output{""};  // One empty string
+  test.AddOutput<std::string>("Y", {1, 1}, output);
+  test.Run(OpTester::ExpectResult::kExpectSuccess);
+}
+#endif
+
 }  // namespace test
 }  // namespace onnxruntime
-#endif