saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-28 20:11:22 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fix batch mode output for TfIdfVectorizer (#3008)

Browse source 
Fix weights indexing in TfIdfVectorizer
  Add a test case to cover batching.
This commit is contained in:
Dmitri Smirnov 2020-02-12 10:46:20 -08:00 committed by GitHub
parent da653ccdac
commit c6b499bfc9
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 39 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
onnxruntime/core/providers/cpu/cpu_execution_provider.cc
View file

@ -262,9 +262,7 @@ class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, Aco
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, Atanh);
class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, 10, Scan);
class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, 10, Scatter);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, string, TfIdfVectorizer);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, int32_t, TfIdfVectorizer);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, int64_t, TfIdfVectorizer);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, TfIdfVectorizer);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, bool, NonZero);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, float, NonZero);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, int32_t, NonZero);
@ -881,12 +879,7 @@ Status RegisterOnnxOperatorKernels(KernelRegistry& kernel_registry) {
Scan)>,
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, 10,
Scatter)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, string,
TfIdfVectorizer)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, int32_t,
TfIdfVectorizer)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, int64_t,
TfIdfVectorizer)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, TfIdfVectorizer)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, bool,
NonZero)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 9, float,

44
onnxruntime/core/providers/cpu/nn/tfidfvectorizer.cc
View file

@ -13,30 +13,13 @@
namespace onnxruntime {
ONNX_CPU_OPERATOR_TYPED_KERNEL(
ONNX_CPU_OPERATOR_KERNEL(
TfIdfVectorizer,
9,
string,
KernelDefBuilder()
.TypeConstraint("T", DataTypeImpl::GetTensorType<std::string>())
.TypeConstraint("T1", DataTypeImpl::GetTensorType<float>()),
TfIdfVectorizer);
ONNX_CPU_OPERATOR_TYPED_KERNEL(
TfIdfVectorizer,
9,
int32_t,
KernelDefBuilder()
.TypeConstraint("T", DataTypeImpl::GetTensorType<int32_t>())
.TypeConstraint("T1", DataTypeImpl::GetTensorType<float>()),
TfIdfVectorizer);
ONNX_CPU_OPERATOR_TYPED_KERNEL(
TfIdfVectorizer,
9,
int64_t,
KernelDefBuilder()
.TypeConstraint("T", DataTypeImpl::GetTensorType<int64_t>())
.TypeConstraint("T", {DataTypeImpl::GetTensorType<std::string>(),
DataTypeImpl::GetTensorType<int32_t>(),
DataTypeImpl::GetTensorType<int64_t>()})
.TypeConstraint("T1", DataTypeImpl::GetTensorType<float>()),
TfIdfVectorizer);
@ -405,11 +388,14 @@ void TfIdfVectorizer::OutputResult(OpKernelContext* ctx, size_t B, const std::ve
std::vector<int64_t> output_dims;
if (B == 0) {
output_dims.push_back(impl.output_size_);
B = 1; // For use in the loops below
} else {
output_dims.push_back(B);
output_dims.push_back(impl.output_size_);
}
const auto row_size = impl.output_size_;
TensorShape output_shape(output_dims);
assert(frequences.size() == static_cast<size_t>(output_shape.Size()));
@ -424,9 +410,11 @@ void TfIdfVectorizer::OutputResult(OpKernelContext* ctx, size_t B, const std::ve
} break;
case kIDF: {
if (!w.empty()) {
assert(frequences.size() == w.size());
for (size_t i = 0; i < frequences.size(); ++i) {
*output_data++ = (frequences[i] > 0) ? w[i] : 0;
const auto* freqs = frequences.data();
for (size_t batch = 0; batch < B; ++batch) {
for (size_t i = 0; i < row_size; ++i) {
*output_data++ = (*freqs++ > 0) ? w[i] : 0;
}
}
} else {
for (auto f : frequences) {
@ -436,9 +424,11 @@ void TfIdfVectorizer::OutputResult(OpKernelContext* ctx, size_t B, const std::ve
} break;
case kTFIDF: {
if (!w.empty()) {
assert(frequences.size() == w.size());
for (size_t i = 0; i < frequences.size(); ++i) {
*output_data++ = frequences[i] * w[i];
const auto* freqs = frequences.data();
for (size_t batch = 0; batch < B; ++batch) {
for (size_t i = 0; i < row_size; ++i) {
*output_data++ = *freqs++ * w[i];
}
}
} else {
for (auto f : frequences) {

20
onnxruntime/test/providers/cpu/nn/tfidfvectorizer_test.cc
View file

@ -669,5 +669,25 @@ TEST(TfIdfVectorizerTest, String_TFIDFWeights_onlyBigrams_Skip5) {
test.Run(OpTester::ExpectResult::kExpectSuccess);
}
TEST(TfIdfVectorizerTest, String_TFIDFWeights_onlyBigrams_Skip5_2rows) {
OpTester test("TfIdfVectorizer", opset_ver);
// s=5, Min=Max=2, weights specified, string
InitTestAttr(test, "TFIDF", 2, 2, 5,
{0, 4},
{0, 1, 2, 3, 4, 5, 6}, //7 output indexes
{2.0f, 2.0f, 2.0f, 2.0f, 2.0f, 3.0f, 2.0f}, // weights
{},
{"two", "three", "five", "four", //1-grams
"five", "six", "seven", "eight", "six", "seven"}); //bi-grams
test.AddInput<std::string>("T", {2, 6}, {"one", "one", "three", "three", "three", "seven",
"eight", "six", "seven", "five", "six", "eight"});
test.AddOutput<float>("Y", {2, 7}, {0.f, 0.f, 0.f, 0.f, 0.f, 0.f, 0.f, // No bi-grams in the first row
0.f, 0.f, 0.f, 0.f, 2.f, 3.f, 2.f});
test.Run(OpTester::ExpectResult::kExpectSuccess);
}
} // namespace test
} // namespace onnxruntime