diff --git a/onnxruntime/core/graph/featurizers_ops/featurizers_defs.cc b/onnxruntime/core/graph/featurizers_ops/featurizers_defs.cc index 65e31ffba6..8318ccb1a4 100644 --- a/onnxruntime/core/graph/featurizers_ops/featurizers_defs.cc +++ b/onnxruntime/core/graph/featurizers_ops/featurizers_defs.cc @@ -33,6 +33,7 @@ using ONNX_NAMESPACE::OPTIONAL; // Forward declarations static void RegisterCatImputerFeaturizerVer1(); +static void RegisterCountVectorizerFeaturizerVer1(); static void RegisterDateTimeFeaturizerVer1(); static void RegisterFromStringFeaturizerVer1(); static void RegisterHashOneHotVectorizerFeaturizerVer1(); @@ -52,6 +53,7 @@ static void RegisterPCAFeaturizerVer1(); static void RegisterRobustScalerFeaturizerVer1(); static void RegisterStandardScaleWrapperFeaturizerVer1(); static void RegisterStringFeaturizerVer1(); +static void RegisterTfidfVectorizerFeaturizerVer1(); static void RegisterTimeSeriesImputerFeaturizerVer1(); static void RegisterTruncatedSVDFeaturizerVer1(); @@ -60,6 +62,7 @@ static void RegisterTruncatedSVDFeaturizerVer1(); // ---------------------------------------------------------------------- void RegisterMSFeaturizersSchemas() { RegisterCatImputerFeaturizerVer1(); + RegisterCountVectorizerFeaturizerVer1(); RegisterDateTimeFeaturizerVer1(); RegisterFromStringFeaturizerVer1(); RegisterHashOneHotVectorizerFeaturizerVer1(); @@ -79,6 +82,7 @@ void RegisterMSFeaturizersSchemas() { RegisterNormalizeFeaturizerVer1(); RegisterStandardScaleWrapperFeaturizerVer1(); RegisterStringFeaturizerVer1(); + RegisterTfidfVectorizerFeaturizerVer1(); RegisterTimeSeriesImputerFeaturizerVer1(); RegisterTruncatedSVDFeaturizerVer1(); } @@ -139,6 +143,70 @@ void RegisterCatImputerFeaturizerVer1() { }); } +void RegisterCountVectorizerFeaturizerVer1() { + static const char* doc = R"DOC( + Returns the count of the number of occurrances of each distinct item according to a + vocabulary established during training. + + C++-style pseudo signature: + CountVector execute(std::string const &value); + + Examples: + Assuming the training data is... + ["orange apple orange grape", "grape carrot carrot apple", "peach banana orange banana"] + + The input data is... + "banana grape grape apple apple apple orange" + + The result will be computed by... + categorize and compute each word's number of apperance in input data, we have "apple -> 3", "banana -> 1", "grape -> 2", "orange -> 1" + construct a dictionary and assign id for each unique word using training data, we have "apple -> 0", "banana -> 1", "grape -> 3", "orange -> 4" + generate TFStruct by combining + + The result is... + [3, 1, 0, 2, 1] + )DOC"; + + MS_FEATURIZERS_OPERATOR_SCHEMA(CountVectorizerTransformer) + .SinceVersion(1) + .SetDomain(kMSFeaturizersDomain) + .SetDoc(doc) + .Input( + 0, + "State", + "State generated during training that is used for prediction", + "T0") + .Input( + 1, + "Input", + "No information is available", + "T") + .Output( + 0, + "Output", + "No information is available", + "T1") + .TypeConstraint( + "T0", + {"tensor(uint8)"}, + "No information is available") + .TypeConstraint( + "T", + {"tensor(string)"}, + "No information is available") + .TypeConstraint( + "T1", + {"tensor(uint32)"}, + "No information is available") + .TypeAndShapeInferenceFunction( + [](ONNX_NAMESPACE::InferenceContext& ctx) { + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_UINT32, 0); + ONNX_NAMESPACE::TensorShapeProto shape_0; + shape_0.add_dim(); // unknown at this time + ONNX_NAMESPACE::updateOutputShape(ctx, 0, shape_0); + }); +} + void RegisterDateTimeFeaturizerVer1() { //static const char* doc = R"DOC( // Extracts various datetime-related values from a UTC time_point. @@ -1408,6 +1476,78 @@ void RegisterStringFeaturizerVer1() { }); } +void RegisterTfidfVectorizerFeaturizerVer1() { + static const char* doc = R"DOC( + Convert a collection of raw documents to a matrix of TF-IDF features + + C++-style pseudo signature: + TfidfVector execute(std::string const &value); + + Examples: + Assuming the training data is... + ["this is the first document", "this document is the second document", "and this is the third one", "is this the first document"] + + Assuming the input data is... + "this is the first document" + The default result will be... + [0. , 0.469791f, 0.580286f, 0.384085f, 0. , 0. , 0.384085f, 0. , 0.384085f] + + Assuming the input data is... + "this document is the second document" + The default result will be... + [0. , 0.687624f, 0. , 0.281089f, 0. , 0.538648f, 0.281089f, 0. , 0.281089f] + + Assuming the input data is... + "and this is the third one" + The default result will be... + [0.511849f, 0. , 0. , 0.267104f, 0.511849f, 0. , 0.267104f, 0.511849f, 0.267104f] + + Assuming the input data is... + "is this the first document" + The default result will be... + [0. , 0.469791f, ,0.580286f, 0.384085f, 0. , 0. , 0.384085f, 0. , 0.384085f] + )DOC"; + + MS_FEATURIZERS_OPERATOR_SCHEMA(TfidfVectorizerTransformer) + .SinceVersion(1) + .SetDomain(kMSFeaturizersDomain) + .SetDoc(doc) + .Input( + 0, + "State", + "State generated during training that is used for prediction", + "T0") + .Input( + 1, + "Input", + "No information is available", + "T") + .Output( + 0, + "Output", + "No information is available", + "T1") + .TypeConstraint( + "T0", + {"tensor(uint8)"}, + "No information is available") + .TypeConstraint( + "T", + {"tensor(string)"}, + "No information is available") + .TypeConstraint( + "T1", + {"tensor(float)"}, + "No information is available") + .TypeAndShapeInferenceFunction( + [](ONNX_NAMESPACE::InferenceContext& ctx) { + propagateElemTypeFromDtypeToOutput(ctx, ONNX_NAMESPACE::TensorProto_DataType_FLOAT, 0); + ONNX_NAMESPACE::TensorShapeProto shape_0; + shape_0.add_dim(); // unknown at this time + ONNX_NAMESPACE::updateOutputShape(ctx, 0, shape_0); + }); +} + void RegisterTimeSeriesImputerFeaturizerVer1() { //static const char* doc = R"DOC( // Imputes rows and column values such that the generated output does not contain any diff --git a/onnxruntime/featurizers_ops/cpu/count_vectorizer_transfromer.cc b/onnxruntime/featurizers_ops/cpu/count_vectorizer_transfromer.cc new file mode 100644 index 0000000000..28828572c8 --- /dev/null +++ b/onnxruntime/featurizers_ops/cpu/count_vectorizer_transfromer.cc @@ -0,0 +1,66 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" + +#include "Featurizers/CountVectorizerFeaturizer.h" +#include "Featurizers/../Archive.h" + +namespace NS = Microsoft::Featurizer; + +namespace onnxruntime { +namespace featurizers { + +void CountVectorizerTransformerImpl(OpKernelContext* ctx) { + // Create the transformer + Microsoft::Featurizer::Featurizers::CountVectorizerTransformer transformer( + [ctx]() { + const auto* state_tensor(ctx->Input(0)); + const uint8_t* const state_data(state_tensor->Data()); + + Microsoft::Featurizer::Archive archive(state_data, state_tensor->Shape().GetDims()[0]); + return Microsoft::Featurizer::Featurizers::CountVectorizerTransformer(archive); + }()); + + // Get the input + const auto* input_tensor = ctx->Input(1); + const std::string* input_data = input_tensor->template Data(); + // Prepare the callback that would output directly to output memory + std::function)> callback; + callback = [ctx](NS::Featurizers::SparseVectorEncoding result) { + // Prepare output + ORT_ENFORCE(result.NumElements < static_cast(std::numeric_limits::max()), + "NumElements in SparseVectorEncoding is GE than max(int64)"); + auto* output_tensor = ctx->Output(0, TensorShape{static_cast(result.NumElements)}); + uint32_t* output_data = output_tensor->template MutableData(); + std::fill(output_data, output_data + result.NumElements, 0); + for (const auto& el : result.Values) { + output_data[el.Index] = el.Value; + } + }; + transformer.execute(*input_data, callback); +}; + +class CountVectorizerTransformer final : public OpKernel { + public: + explicit CountVectorizerTransformer(const OpKernelInfo& info) : OpKernel(info) { + } + Status Compute(OpKernelContext* ctx) const override { + CountVectorizerTransformerImpl(ctx); + return Status::OK(); + } +}; + +ONNX_OPERATOR_KERNEL_EX( + CountVectorizerTransformer, + kMSFeaturizersDomain, + 1, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("T0", DataTypeImpl::GetTensorType()) + .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), + CountVectorizerTransformer); + +} // namespace featurizers +} // namespace onnxruntime diff --git a/onnxruntime/featurizers_ops/cpu/tfidf_vectorizer_transformer.cc b/onnxruntime/featurizers_ops/cpu/tfidf_vectorizer_transformer.cc new file mode 100644 index 0000000000..4b7b54c1bd --- /dev/null +++ b/onnxruntime/featurizers_ops/cpu/tfidf_vectorizer_transformer.cc @@ -0,0 +1,67 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" + +#include "Featurizers/TfidfVectorizerFeaturizer.h" +#include "Featurizers/../Archive.h" + +namespace NS = Microsoft::Featurizer; + +namespace onnxruntime { +namespace featurizers { + +void TfidfVectorizerTransformerImpl(OpKernelContext* ctx) { + // Create the transformer + Microsoft::Featurizer::Featurizers::TfidfVectorizerTransformer transformer( + [ctx]() { + const auto* state_tensor(ctx->Input(0)); + const uint8_t* const state_data(state_tensor->Data()); + + Microsoft::Featurizer::Archive archive(state_data, state_tensor->Shape().GetDims()[0]); + return Microsoft::Featurizer::Featurizers::TfidfVectorizerTransformer(archive); + }()); + + // Get the input + const auto* input_tensor = ctx->Input(1); + const std::string* input_data = input_tensor->template Data(); + + // Prepare the callback that would output directly to output memory + std::function)> callback; + callback = [ctx](NS::Featurizers::SparseVectorEncoding result) { + // Prepare output + ORT_ENFORCE(result.NumElements < static_cast(std::numeric_limits::max()), + "NumElements in SparseVectorEncoding is GE than max(int64)"); + auto* output_tensor = ctx->Output(0, TensorShape{static_cast(result.NumElements)}); + float* output_data = output_tensor->template MutableData(); + std::fill(output_data, output_data + result.NumElements, 0.f); + for (const auto& el : result.Values) { + output_data[el.Index] = el.Value; + } + }; + transformer.execute(*input_data, callback); +} + +class TfidfVectorizerTransformer final : public OpKernel { + public: + explicit TfidfVectorizerTransformer(const OpKernelInfo& info) : OpKernel(info) { + } + Status Compute(OpKernelContext* ctx) const override { + TfidfVectorizerTransformerImpl(ctx); + return Status::OK(); + } +}; + +ONNX_OPERATOR_KERNEL_EX( + TfidfVectorizerTransformer, + kMSFeaturizersDomain, + 1, + kCpuExecutionProvider, + KernelDefBuilder() + .TypeConstraint("T0", DataTypeImpl::GetTensorType()) + .TypeConstraint("InputT", DataTypeImpl::GetTensorType()), + TfidfVectorizerTransformer); + +} // namespace featurizers +} // namespace onnxruntime diff --git a/onnxruntime/featurizers_ops/cpu_featurizers_kernels.cc b/onnxruntime/featurizers_ops/cpu_featurizers_kernels.cc index 44cacbd526..2d74fe3f44 100644 --- a/onnxruntime/featurizers_ops/cpu_featurizers_kernels.cc +++ b/onnxruntime/featurizers_ops/cpu_featurizers_kernels.cc @@ -11,6 +11,7 @@ namespace featurizers { // Forward declarations class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, CatImputerTransformer); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, CountVectorizerTransformer); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, DateTimeTransformer); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, FromStringTransformer); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, HashOneHotVectorizerTransformer); @@ -30,12 +31,14 @@ class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomai class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, NormalizeTransformer); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, StandardScaleWrapperTransformer); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, StringTransformer); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, TfidfVectorizerTransformer); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, TimeSeriesImputerTransformer); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSFeaturizersDomain, 1, TruncatedSVDTransformer); Status RegisterCpuMSFeaturizersKernels(KernelRegistry& kernel_registry) { static const BuildKernelCreateInfoFn function_table[] = { BuildKernelCreateInfo, + BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, @@ -55,6 +58,7 @@ Status RegisterCpuMSFeaturizersKernels(KernelRegistry& kernel_registry) { BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, + BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo }; diff --git a/onnxruntime/test/featurizers_ops/countvectorizer_transformer_test.cc b/onnxruntime/test/featurizers_ops/countvectorizer_transformer_test.cc new file mode 100644 index 0000000000..417bd521f4 --- /dev/null +++ b/onnxruntime/test/featurizers_ops/countvectorizer_transformer_test.cc @@ -0,0 +1,72 @@ +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +#include "Featurizers/CountVectorizerFeaturizer.h" +#include "Featurizers/../Archive.h" +#include "Featurizers/TestHelpers.h" + +namespace NS = Microsoft::Featurizer; + +using IndexMapType = std::unordered_map; +using AnalyzerMethod = NS::Featurizers::Components::AnalyzerMethod; + +namespace onnxruntime { +namespace test { +namespace { + +using InputType = std::string; +using EstimatorT = NS::Featurizers::CountVectorizerEstimator::max()>; + +std::vector GetStream(EstimatorT& estimator, const std::vector>& trainingBatches) { + NS::TestHelpers::Train(estimator, trainingBatches); + auto pTransformer = estimator.create_transformer(); + NS::Archive ar; + pTransformer->save(ar); + return ar.commit(); +} +} // namespace + +// string - without binary with decorator, analyze word, maxdf = 1, mindf = 0, topk = null, empty vocabulary, ngram_min = 1, ngram_max = 1" +TEST(FeaturizersTests, CountVectorizerTransformer_string_nobinary_with_decorator) { + + EstimatorT estimator(NS::CreateTestAnnotationMapsPtr(1), 0, true, AnalyzerMethod::Word, "", + 1.0, 0, nonstd::optional(), 1, 1, false); + + auto trainingBatches = NS::TestHelpers::make_vector>( + NS::TestHelpers::make_vector("oraNge apple oranGE grape"), + NS::TestHelpers::make_vector("grApe caRrOt carrot apple"), + NS::TestHelpers::make_vector("peach Banana orange banana")); + + auto stream = GetStream(estimator, trainingBatches); + auto dim = static_cast(stream.size()); + OpTester test("CountVectorizerTransformer", 1, onnxruntime::kMSFeaturizersDomain); + test.AddInput("State", {dim}, stream); + test.AddInput("Input", {1}, {"banana grape grape apple apple apple orange"}); + test.AddOutput("Output", {6}, {3, 1, 0, 2, 1, 0}); + + test.Run(); +} + +// string - with binary with decorator, analyze word, maxdf = 1, mindf = 0, topk = null, empty vocabulary, ngram_min = 1, ngram_max = 1 +TEST(FeaturizersTests, CountVectorizerTransformer_string_withbinary_withdecorator) { + + EstimatorT estimator(NS::CreateTestAnnotationMapsPtr(1), 0, false, AnalyzerMethod::Word, "", + 1.0, 0, nonstd::optional(), 1, 1, true); + + auto trainingBatches = NS::TestHelpers::make_vector>( + NS::TestHelpers::make_vector("orange apple orange grape"), + NS::TestHelpers::make_vector("grape carrot carrot apple"), + NS::TestHelpers::make_vector("peach banana orange banana")); + + auto stream = GetStream(estimator, trainingBatches); + auto dim = static_cast(stream.size()); + OpTester test("CountVectorizerTransformer", 1, onnxruntime::kMSFeaturizersDomain); + test.AddInput("State", {dim}, stream); + + test.AddInput("Input", {1}, {"banana grape grape apple apple apple orange"}); + test.AddOutput("Output", {6}, {1, 1, 0, 1, 1, 0}); + test.Run(); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/featurizers_ops/tfidf_vectorizer_transformer_test.cc b/onnxruntime/test/featurizers_ops/tfidf_vectorizer_transformer_test.cc new file mode 100644 index 0000000000..b93cc7b1c6 --- /dev/null +++ b/onnxruntime/test/featurizers_ops/tfidf_vectorizer_transformer_test.cc @@ -0,0 +1,78 @@ +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" + +#include "Featurizers/TfidfVectorizerFeaturizer.h" +#include "Featurizers/../Archive.h" +#include "Featurizers/TestHelpers.h" + +namespace NS = Microsoft::Featurizer; + +using IndexMapType = std::unordered_map; +using AnalyzerMethod = NS::Featurizers::Components::AnalyzerMethod; + +namespace onnxruntime { +namespace test { +namespace { + +using InputType = std::string; +using TransformedType = NS::Featurizers::SparseVectorEncoding; +using EstimatorT = NS::Featurizers::TfidfVectorizerEstimator<>; + +std::vector GetStream(EstimatorT& estimator, const std::vector>& trainingBatches) { + NS::TestHelpers::Train(estimator, trainingBatches); + auto pTransformer = estimator.create_transformer(); + NS::Archive ar; + pTransformer->save(ar); + return ar.commit(); +} +} // namespace + +TEST(FeaturizersTests, TfidfVectorizerTransformer_string_standard_1_with_decorator) { + EstimatorT estimator( + NS::CreateTestAnnotationMapsPtr(1), + 0, + true, + AnalyzerMethod::Word, + ""); + + auto trainingBatches = NS::TestHelpers::make_vector>( + NS::TestHelpers::make_vector("this is THE first document"), + NS::TestHelpers::make_vector("this DOCUMENT is the second document"), + NS::TestHelpers::make_vector("and this is the THIRD one"), + NS::TestHelpers::make_vector("IS this THE first document")); + + auto stream = GetStream(estimator, trainingBatches); + auto dim = static_cast(stream.size()); + OpTester test("TfidfVectorizerTransformer", 1, onnxruntime::kMSFeaturizersDomain); + test.AddInput("State", {dim}, stream); + test.AddInput("Input", {1}, {"THIS is the FIRST document"}); + test.AddOutput("Output", {9}, {0.f, 0.469791f, 0.580286f, 0.384085f, 0.f, 0.f, 0.384085f, 0.f, 0.384085f}); + test.Run(); +} + +TEST(FeaturizersTests, TfidfVectorizerTransformer_string_standard_1_no_decorator) { + + EstimatorT estimator( + NS::CreateTestAnnotationMapsPtr(1), + 0, + false, + AnalyzerMethod::Word, + ""); + + auto trainingBatches = NS::TestHelpers::make_vector>( + NS::TestHelpers::make_vector("this is the first document"), + NS::TestHelpers::make_vector("this document is the second document"), + NS::TestHelpers::make_vector("and this is the third one"), + NS::TestHelpers::make_vector("is this the first document")); + + auto stream = GetStream(estimator, trainingBatches); + auto dim = static_cast(stream.size()); + OpTester test("TfidfVectorizerTransformer", 1, onnxruntime::kMSFeaturizersDomain); + test.AddInput("State", {dim}, stream); + + test.AddInput("Input", {1}, {"this is the first document"}); + test.AddOutput("Output", {9}, {0.f, 0.469791f, 0.580286f, 0.384085f, 0.f, 0.f, 0.384085f, 0.f, 0.384085f}); + test.Run(); +} +} // namespace test +} // namespace onnxruntime