Add CounterVectorizer and TfIdfVectorizer. (#2981)

Add CounterVectorizer and TfIdfVectorizers.
  Since we do not support SparseTensors as of now, we convert
  the output to a dense Tensor.

onnxruntime/core/graph/featurizers_ops/featurizers_defs.cc

@@ -33,6 +33,7 @@ using ONNX_NAMESPACE::OPTIONAL;
 
 // Forward declarations
 static void RegisterCatImputerFeaturizerVer1();
+static void RegisterCountVectorizerFeaturizerVer1();
 static void RegisterDateTimeFeaturizerVer1();
 static void RegisterFromStringFeaturizerVer1();
 static void RegisterHashOneHotVectorizerFeaturizerVer1();
@@ -52,6 +53,7 @@ static void RegisterPCAFeaturizerVer1();
 static void RegisterRobustScalerFeaturizerVer1();
 static void RegisterStandardScaleWrapperFeaturizerVer1();
 static void RegisterStringFeaturizerVer1();
+static void RegisterTfidfVectorizerFeaturizerVer1();
 static void RegisterTimeSeriesImputerFeaturizerVer1();
 static void RegisterTruncatedSVDFeaturizerVer1();
 
@@ -60,6 +62,7 @@ static void RegisterTruncatedSVDFeaturizerVer1();
 // ----------------------------------------------------------------------
 void RegisterMSFeaturizersSchemas() {
   RegisterCatImputerFeaturizerVer1();
+  RegisterCountVectorizerFeaturizerVer1();
   RegisterDateTimeFeaturizerVer1();
   RegisterFromStringFeaturizerVer1();
   RegisterHashOneHotVectorizerFeaturizerVer1();
@@ -79,6 +82,7 @@ void RegisterMSFeaturizersSchemas() {
   RegisterNormalizeFeaturizerVer1();
   RegisterStandardScaleWrapperFeaturizerVer1();
   RegisterStringFeaturizerVer1();
+  RegisterTfidfVectorizerFeaturizerVer1();
   RegisterTimeSeriesImputerFeaturizerVer1();
   RegisterTruncatedSVDFeaturizerVer1();
 }
@@ -139,6 +143,70 @@ void RegisterCatImputerFeaturizerVer1() {
           });
 }
 
+void RegisterCountVectorizerFeaturizerVer1() {
+  static const char* doc = R"DOC(
+      Returns the count of the number of occurrances of each distinct item according to a
+      vocabulary established during training.
+
+      C++-style pseudo signature:
+        CountVector execute(std::string const &value);
+
+      Examples:
+        Assuming the training data is...
+        ["orange apple orange grape", "grape carrot carrot apple", "peach banana orange banana"]
+
+        The input data is...
+        "banana grape grape apple apple apple orange"
+
+        The result will be computed by...
+          categorize and compute each word's number of apperance in input data, we have "apple -> 3", "banana -> 1", "grape -> 2", "orange -> 1"
+          construct a dictionary and assign id for each unique word using training data, we have "apple -> 0", "banana -> 1", "grape -> 3", "orange -> 4"
+          generate TFStruct by combining <word's id, word's number of apperance>
+
+        The result is...
+        [3, 1, 0, 2, 1]
+  )DOC";
+
+  MS_FEATURIZERS_OPERATOR_SCHEMA(CountVectorizerTransformer)
+      .SinceVersion(1)
+      .SetDomain(kMSFeaturizersDomain)
+      .SetDoc(doc)
+      .Input(
+          0,
+          "State",
+          "State generated during training that is used for prediction",
+          "T0")
+      .Input(
+          1,
+          "Input",
+          "No information is available",
+          "T")
+      .Output(
+          0,
+          "Output",
+          "No information is available",
+          "T1")
+      .TypeConstraint(
+          "T0",
+          {"tensor(uint8)"},
+          "No information is available")
+      .TypeConstraint(
+          "T",
+          {"tensor(string)"},
+          "No information is available")
+      .TypeConstraint(
+          "T1",
+          {"tensor(uint32)"},
+          "No information is available")
+      .TypeAndShapeInferenceFunction(
+          [](ONNX_NAMESPACE::InferenceContext& ctx) {
+            propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT32, 0);
+            ONNX_NAMESPACE::TensorShapeProto shape_0;
+            shape_0.add_dim();  // unknown at this time
+            ONNX_NAMESPACE::updateOutputShape(ctx, 0, shape_0);
+          });
+}
+
 void RegisterDateTimeFeaturizerVer1() {
   //static const char* doc = R"DOC(
   //      Extracts various datetime-related values from a UTC time_point.
@@ -1408,6 +1476,78 @@ void RegisterStringFeaturizerVer1() {
           });
 }
 
+void RegisterTfidfVectorizerFeaturizerVer1() {
+  static const char* doc = R"DOC(
+      Convert a collection of raw documents to a matrix of TF-IDF features
+
+      C++-style pseudo signature:
+        TfidfVector execute(std::string const &value);
+
+      Examples:
+        Assuming the training data is...
+        ["this is the first document", "this document is the second document", "and this is the third one", "is this the first document"]
+
+        Assuming the input data is...
+        "this is the first document"
+        The default result will be...
+        [0. , 0.469791f, 0.580286f, 0.384085f, 0. , 0. , 0.384085f, 0. , 0.384085f]
+
+        Assuming the input data is...
+        "this document is the second document"
+        The default result will be...
+        [0. , 0.687624f, 0. , 0.281089f, 0. , 0.538648f, 0.281089f, 0. , 0.281089f]
+
+        Assuming the input data is...
+        "and this is the third one"
+        The default result will be...
+        [0.511849f, 0. , 0. , 0.267104f, 0.511849f, 0. , 0.267104f, 0.511849f, 0.267104f]
+
+        Assuming the input data is...
+        "is this the first document"
+        The default result will be...
+        [0. , 0.469791f, ,0.580286f, 0.384085f, 0. , 0. , 0.384085f, 0. , 0.384085f]
+  )DOC";
+
+  MS_FEATURIZERS_OPERATOR_SCHEMA(TfidfVectorizerTransformer)
+      .SinceVersion(1)
+      .SetDomain(kMSFeaturizersDomain)
+      .SetDoc(doc)
+      .Input(
+          0,
+          "State",
+          "State generated during training that is used for prediction",
+          "T0")
+      .Input(
+          1,
+          "Input",
+          "No information is available",
+          "T")
+      .Output(
+          0,
+          "Output",
+          "No information is available",
+          "T1")
+      .TypeConstraint(
+          "T0",
+          {"tensor(uint8)"},
+          "No information is available")
+      .TypeConstraint(
+          "T",
+          {"tensor(string)"},
+          "No information is available")
+      .TypeConstraint(
+          "T1",
+          {"tensor(float)"},
+          "No information is available")
+      .TypeAndShapeInferenceFunction(
+          [](ONNX_NAMESPACE::InferenceContext& ctx) {
+            propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_FLOAT, 0);
+            ONNX_NAMESPACE::TensorShapeProto shape_0;
+            shape_0.add_dim();  // unknown at this time
+            ONNX_NAMESPACE::updateOutputShape(ctx, 0, shape_0);
+          });
+}
+
 void RegisterTimeSeriesImputerFeaturizerVer1() {
   //static const char* doc = R"DOC(
   //  Imputes rows and column values such that the generated output does not contain any

onnxruntime/featurizers_ops/cpu/count_vectorizer_transfromer.cc

@@ -0,0 +1,66 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#include "core/common/common.h"
+#include "core/framework/op_kernel.h"
+
+#include "Featurizers/CountVectorizerFeaturizer.h"
+#include "Featurizers/../Archive.h"
+
+namespace NS = Microsoft::Featurizer;
+
+namespace onnxruntime {
+namespace featurizers {
+
+void CountVectorizerTransformerImpl(OpKernelContext* ctx) {
+  // Create the transformer
+  Microsoft::Featurizer::Featurizers::CountVectorizerTransformer transformer(
+      [ctx]() {
+        const auto* state_tensor(ctx->Input<Tensor>(0));
+        const uint8_t* const state_data(state_tensor->Data<uint8_t>());
+
+        Microsoft::Featurizer::Archive archive(state_data, state_tensor->Shape().GetDims()[0]);
+        return Microsoft::Featurizer::Featurizers::CountVectorizerTransformer(archive);
+      }());
+
+  // Get the input
+  const auto* input_tensor = ctx->Input<Tensor>(1);
+  const std::string* input_data = input_tensor->template Data<std::string>();
+  // Prepare the callback that would output directly to output memory
+  std::function<void(NS::Featurizers::SparseVectorEncoding<uint32_t>)> callback;
+  callback = [ctx](NS::Featurizers::SparseVectorEncoding<uint32_t> result) {
+    // Prepare output
+    ORT_ENFORCE(result.NumElements < static_cast<uint64_t>(std::numeric_limits<int64_t>::max()),
+        "NumElements in SparseVectorEncoding is GE than max(int64)");
+    auto* output_tensor = ctx->Output(0, TensorShape{static_cast<int64_t>(result.NumElements)});
+    uint32_t* output_data = output_tensor->template MutableData<uint32_t>();
+    std::fill(output_data, output_data + result.NumElements, 0);
+    for (const auto& el : result.Values) {
+      output_data[el.Index] = el.Value;
+    }
+  };
+  transformer.execute(*input_data, callback);
+};
+
+class CountVectorizerTransformer final : public OpKernel {
+ public:
+  explicit CountVectorizerTransformer(const OpKernelInfo& info) : OpKernel(info) {
+  }
+  Status Compute(OpKernelContext* ctx) const override {
+    CountVectorizerTransformerImpl(ctx);
+    return Status::OK();
+  }
+};
+
+ONNX_OPERATOR_KERNEL_EX(
+    CountVectorizerTransformer,
+    kMSFeaturizersDomain,
+    1,
+    kCpuExecutionProvider,
+    KernelDefBuilder()
+        .TypeConstraint("T0", DataTypeImpl::GetTensorType<uint8_t>())
+        .TypeConstraint("InputT", DataTypeImpl::GetTensorType<std::string>()),
+    CountVectorizerTransformer);
+
+}  // namespace featurizers
+}  // namespace onnxruntime

onnxruntime/featurizers_ops/cpu/tfidf_vectorizer_transformer.cc

@@ -0,0 +1,67 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#include "core/common/common.h"
+#include "core/framework/op_kernel.h"
+
+#include "Featurizers/TfidfVectorizerFeaturizer.h"
+#include "Featurizers/../Archive.h"
+
+namespace NS = Microsoft::Featurizer;
+
+namespace onnxruntime {
+namespace featurizers {
+
+void TfidfVectorizerTransformerImpl(OpKernelContext* ctx) {
+  // Create the transformer
+  Microsoft::Featurizer::Featurizers::TfidfVectorizerTransformer transformer(
+      [ctx]() {
+        const auto* state_tensor(ctx->Input<Tensor>(0));
+        const uint8_t* const state_data(state_tensor->Data<uint8_t>());
+
+        Microsoft::Featurizer::Archive archive(state_data, state_tensor->Shape().GetDims()[0]);
+        return Microsoft::Featurizer::Featurizers::TfidfVectorizerTransformer(archive);
+      }());
+
+  // Get the input
+  const auto* input_tensor = ctx->Input<Tensor>(1);
+  const std::string* input_data = input_tensor->template Data<std::string>();
+
+    // Prepare the callback that would output directly to output memory
+  std::function<void(NS::Featurizers::SparseVectorEncoding<float>)> callback;
+  callback = [ctx](NS::Featurizers::SparseVectorEncoding<float> result) {
+    // Prepare output
+    ORT_ENFORCE(result.NumElements < static_cast<uint64_t>(std::numeric_limits<int64_t>::max()),
+                "NumElements in SparseVectorEncoding is GE than max(int64)");
+    auto* output_tensor = ctx->Output(0, TensorShape{static_cast<int64_t>(result.NumElements)});
+    float* output_data = output_tensor->template MutableData<float>();
+    std::fill(output_data, output_data + result.NumElements, 0.f);
+    for (const auto& el : result.Values) {
+      output_data[el.Index] = el.Value;
+    }
+  };
+  transformer.execute(*input_data, callback);
+}
+
+class TfidfVectorizerTransformer final : public OpKernel {
+ public:
+  explicit TfidfVectorizerTransformer(const OpKernelInfo& info) : OpKernel(info) {
+  }
+  Status Compute(OpKernelContext* ctx) const override {
+    TfidfVectorizerTransformerImpl(ctx);
+    return Status::OK();
+  }
+};
+
+ONNX_OPERATOR_KERNEL_EX(
+    TfidfVectorizerTransformer,
+    kMSFeaturizersDomain,
+    1,
+    kCpuExecutionProvider,
+    KernelDefBuilder()
+        .TypeConstraint("T0", DataTypeImpl::GetTensorType<uint8_t>())
+        .TypeConstraint("InputT", DataTypeImpl::GetTensorType<std::string>()),
+    TfidfVectorizerTransformer);
+
+}  // namespace featurizers
+}  // namespace onnxruntime

onnxruntime/featurizers_ops/cpu_featurizers_kernels.cc

@@ -11,6 +11,7 @@ namespace featurizers {
 
 // Forward declarations
 class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, CatImputerTransformer);
+class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, CountVectorizerTransformer);
 class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, DateTimeTransformer);
 class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, FromStringTransformer);
 class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, HashOneHotVectorizerTransformer);
@@ -30,12 +31,14 @@ class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomai
 class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, NormalizeTransformer);
 class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, StandardScaleWrapperTransformer);
 class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, StringTransformer);
+class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, TfidfVectorizerTransformer);
 class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, TimeSeriesImputerTransformer);
 class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, TruncatedSVDTransformer);
 
 Status RegisterCpuMSFeaturizersKernels(KernelRegistry& kernel_registry) {
   static const BuildKernelCreateInfoFn function_table[] = {
       BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, CatImputerTransformer)>,
+      BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, CountVectorizerTransformer)>,
       BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, DateTimeTransformer)>,
       BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, FromStringTransformer)>,
       BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, HashOneHotVectorizerTransformer)>,
@@ -55,6 +58,7 @@ Status RegisterCpuMSFeaturizersKernels(KernelRegistry& kernel_registry) {
       BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, NormalizeTransformer)>,
       BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, StandardScaleWrapperTransformer)>,
       BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, StringTransformer)>,
+      BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, TfidfVectorizerTransformer)>,
       BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, TimeSeriesImputerTransformer)>,
       BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, TruncatedSVDTransformer)>
   };

onnxruntime/test/featurizers_ops/countvectorizer_transformer_test.cc

@@ -0,0 +1,72 @@
+#include "gtest/gtest.h"
+#include "test/providers/provider_test_utils.h"
+
+#include "Featurizers/CountVectorizerFeaturizer.h"
+#include "Featurizers/../Archive.h"
+#include "Featurizers/TestHelpers.h"
+
+namespace NS = Microsoft::Featurizer;
+
+using IndexMapType = std::unordered_map<std::string, std::uint32_t>;
+using AnalyzerMethod = NS::Featurizers::Components::AnalyzerMethod;
+
+namespace onnxruntime {
+namespace test {
+namespace {
+
+using InputType = std::string;
+using EstimatorT = NS::Featurizers::CountVectorizerEstimator<std::numeric_limits<size_t>::max()>;
+
+std::vector<uint8_t> GetStream(EstimatorT& estimator, const std::vector<std::vector<InputType>>& trainingBatches) {
+  NS::TestHelpers::Train<EstimatorT, InputType>(estimator, trainingBatches);
+  auto pTransformer = estimator.create_transformer();
+  NS::Archive ar;
+  pTransformer->save(ar);
+  return ar.commit();
+}
+}  // namespace
+
+// string - without binary with decorator, analyze word, maxdf = 1, mindf = 0, topk = null, empty vocabulary, ngram_min = 1, ngram_max = 1"
+TEST(FeaturizersTests, CountVectorizerTransformer_string_nobinary_with_decorator) {
+
+  EstimatorT estimator(NS::CreateTestAnnotationMapsPtr(1), 0, true, AnalyzerMethod::Word, "",
+                       1.0, 0, nonstd::optional<std::uint32_t>(), 1, 1, false);
+
+  auto trainingBatches = NS::TestHelpers::make_vector<std::vector<InputType>>(
+      NS::TestHelpers::make_vector<InputType>("oraNge apple oranGE grape"),
+      NS::TestHelpers::make_vector<InputType>("grApe caRrOt carrot apple"),
+      NS::TestHelpers::make_vector<InputType>("peach Banana orange banana"));
+
+  auto stream = GetStream(estimator, trainingBatches);
+  auto dim = static_cast<int64_t>(stream.size());
+  OpTester test("CountVectorizerTransformer", 1, onnxruntime::kMSFeaturizersDomain);
+  test.AddInput<uint8_t>("State", {dim}, stream);
+  test.AddInput<std::string>("Input", {1}, {"banana grape grape apple apple apple orange"});
+  test.AddOutput<uint32_t>("Output", {6}, {3, 1, 0, 2, 1, 0});
+
+  test.Run();
+}
+
+// string - with binary with decorator, analyze word, maxdf = 1, mindf = 0, topk = null, empty vocabulary, ngram_min = 1, ngram_max = 1
+TEST(FeaturizersTests, CountVectorizerTransformer_string_withbinary_withdecorator) {
+
+  EstimatorT estimator(NS::CreateTestAnnotationMapsPtr(1), 0, false, AnalyzerMethod::Word, "",
+                       1.0, 0, nonstd::optional<std::uint32_t>(), 1, 1, true);
+
+  auto trainingBatches = NS::TestHelpers::make_vector<std::vector<InputType>>(
+      NS::TestHelpers::make_vector<InputType>("orange apple orange grape"),
+      NS::TestHelpers::make_vector<InputType>("grape carrot carrot apple"),
+      NS::TestHelpers::make_vector<InputType>("peach banana orange banana"));
+
+  auto stream = GetStream(estimator, trainingBatches);
+  auto dim = static_cast<int64_t>(stream.size());
+  OpTester test("CountVectorizerTransformer", 1, onnxruntime::kMSFeaturizersDomain);
+  test.AddInput<uint8_t>("State", {dim}, stream);
+
+  test.AddInput<std::string>("Input", {1}, {"banana grape grape apple apple apple orange"});
+  test.AddOutput<uint32_t>("Output", {6}, {1, 1, 0, 1, 1, 0});
+  test.Run();
+}
+
+}  // namespace test
+}  // namespace onnxruntime

onnxruntime/test/featurizers_ops/tfidf_vectorizer_transformer_test.cc

@@ -0,0 +1,78 @@
+#include "gtest/gtest.h"
+#include "test/providers/provider_test_utils.h"
+
+#include "Featurizers/TfidfVectorizerFeaturizer.h"
+#include "Featurizers/../Archive.h"
+#include "Featurizers/TestHelpers.h"
+
+namespace NS = Microsoft::Featurizer;
+
+using IndexMapType = std::unordered_map<std::string, std::uint32_t>;
+using AnalyzerMethod = NS::Featurizers::Components::AnalyzerMethod;
+
+namespace onnxruntime {
+namespace test {
+namespace {
+
+using InputType = std::string;
+using TransformedType = NS::Featurizers::SparseVectorEncoding<std::float_t>;
+using EstimatorT = NS::Featurizers::TfidfVectorizerEstimator<>;
+
+std::vector<uint8_t> GetStream(EstimatorT& estimator, const std::vector<std::vector<InputType>>& trainingBatches) {
+  NS::TestHelpers::Train<EstimatorT, InputType>(estimator, trainingBatches);
+  auto pTransformer = estimator.create_transformer();
+  NS::Archive ar;
+  pTransformer->save(ar);
+  return ar.commit();
+}
+}  // namespace
+
+TEST(FeaturizersTests, TfidfVectorizerTransformer_string_standard_1_with_decorator) {
+  EstimatorT estimator(
+      NS::CreateTestAnnotationMapsPtr(1),
+      0,
+      true,
+      AnalyzerMethod::Word,
+      "");
+
+  auto trainingBatches = NS::TestHelpers::make_vector<std::vector<InputType>>(
+      NS::TestHelpers::make_vector<InputType>("this is THE first document"),
+      NS::TestHelpers::make_vector<InputType>("this DOCUMENT is the second document"),
+      NS::TestHelpers::make_vector<InputType>("and this is the THIRD one"),
+      NS::TestHelpers::make_vector<InputType>("IS this THE first document"));
+
+  auto stream = GetStream(estimator, trainingBatches);
+  auto dim = static_cast<int64_t>(stream.size());
+  OpTester test("TfidfVectorizerTransformer", 1, onnxruntime::kMSFeaturizersDomain);
+  test.AddInput<uint8_t>("State", {dim}, stream);
+  test.AddInput<std::string>("Input", {1}, {"THIS is the FIRST document"});
+  test.AddOutput<float>("Output", {9}, {0.f, 0.469791f, 0.580286f, 0.384085f, 0.f, 0.f, 0.384085f, 0.f, 0.384085f});
+  test.Run();
+}
+
+TEST(FeaturizersTests, TfidfVectorizerTransformer_string_standard_1_no_decorator) {
+
+   EstimatorT estimator(
+      NS::CreateTestAnnotationMapsPtr(1),
+      0,
+      false,
+      AnalyzerMethod::Word,
+      "");
+
+  auto trainingBatches = NS::TestHelpers::make_vector<std::vector<InputType>>(
+      NS::TestHelpers::make_vector<InputType>("this is the first document"),
+      NS::TestHelpers::make_vector<InputType>("this document is the second document"),
+      NS::TestHelpers::make_vector<InputType>("and this is the third one"),
+      NS::TestHelpers::make_vector<InputType>("is this the first document"));
+
+  auto stream = GetStream(estimator, trainingBatches);
+  auto dim = static_cast<int64_t>(stream.size());
+  OpTester test("TfidfVectorizerTransformer", 1, onnxruntime::kMSFeaturizersDomain);
+  test.AddInput<uint8_t>("State", {dim}, stream);
+
+  test.AddInput<std ::string>("Input", {1}, {"this is the first document"});
+  test.AddOutput<float>("Output", {9}, {0.f, 0.469791f, 0.580286f, 0.384085f, 0.f, 0.f, 0.384085f, 0.f, 0.384085f});
+  test.Run();
+}
+}  // namespace test
+}  // namespace onnxruntime