saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-28 20:11:22 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Support more types in MurmurhashOp (#3827)

Browse source 
* support more types for MurmurhashOp

* resolve comments

* fix comments
This commit is contained in:
Ye Wang 2020-05-07 20:43:11 -07:00 committed by GitHub
parent 19e3dc47d7
commit a9b43b05cb
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 46 additions and 7 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

15
onnxruntime/contrib_ops/cpu/murmur_hash3.cc
View file

@ -103,6 +103,10 @@ ONNX_OPERATOR_KERNEL_EX(
KernelDefBuilder()
.TypeConstraint("T1", std::vector<MLDataType>{DataTypeImpl::GetTensorType<int32_t>(),
DataTypeImpl::GetTensorType<uint32_t>(),
DataTypeImpl::GetTensorType<int64_t>(),
DataTypeImpl::GetTensorType<uint64_t>(),
DataTypeImpl::GetTensorType<float>(),
DataTypeImpl::GetTensorType<double>(),
DataTypeImpl::GetTensorType<std::string>()})
.TypeConstraint("T2", std::vector<MLDataType>{DataTypeImpl::GetTensorType<int32_t>(),
DataTypeImpl::GetTensorType<uint32_t>()}),
@ -193,14 +197,17 @@ Status MurmurHash3::Compute(OpKernelContext* ctx) const {
++output;
}
} else {
auto input = reinterpret_cast<const uint32_t*>(keys->DataRaw());
const auto input_end = input + input_count;
auto input = reinterpret_cast<const unsigned char*>(keys->DataRaw());
//input_element_bytes is 4, 8,.. less than 4 bytes is not allowed
int input_num_bytes = static_cast<int>(input_element_bytes);
ORT_ENFORCE(input_num_bytes % 4 == 0);
const auto input_end = input + input_count * input_num_bytes;
while (input != input_end) {
MurmurHash3_x86_32(input,
static_cast<int>(input_element_bytes),
input_num_bytes,
seed_,
output);
++input;
input += input_num_bytes;
++output;
}
}

2
onnxruntime/core/graph/contrib_ops/contrib_defs.cc
View file

@ -2084,7 +2084,7 @@ Output = Dequantize(Input) -> AveragePool on fp32 data -> Quantize(output)
.SetDoc(R"DOC(The underlying implementation is MurmurHash3_x86_32 generating low latency 32bits hash suitable for implementing lookup tables, Bloom filters, count min sketch or feature hashing.)DOC")
.Input(0, "X", "An input tensor to hash.", "T1")
.Output(0, "Y", "32-bit hash value.", "T2")
.TypeConstraint("T1", {"tensor(uint32)", "tensor(int32)", "tensor(string)"}, "Constrain input type to unsigned or signed 32-bit integer tensor, or string tensor. It should be utf-8 encoded if using unicode.")
.TypeConstraint("T1", {"tensor(uint32)", "tensor(int32)", "tensor(uint64)", "tensor(int64)", "tensor(float)", "tensor(double)", "tensor(string)"}, "Constrain input type to unsigned or signed 32-bit integer tensor, or string tensor. It should be utf-8 encoded if using unicode.")
.TypeConstraint("T2", {"tensor(uint32)", "tensor(int32)"}, "Constrain output type to unsigned and signed 32-bit integer tensor.")
.Attr(
"seed",

36
onnxruntime/test/contrib_ops/murmur_hash3_test.cc
View file

@ -9,7 +9,7 @@ namespace test {
TEST(MurmurHash3OpTest, UnsupportedInputType) {
OpTester test("MurmurHash3", 1, onnxruntime::kMSDomain);
test.AddInput<double>("X", {1}, {3.});
test.AddInput<int8_t>("X", {1}, {3});
test.AddAttribute<int64_t>("positive", 0);
test.AddOutput<int32_t>("Y", {1}, {847579505L});
// Unsupported input type
@ -49,7 +49,31 @@ TEST(MurmurHash3OpTest, ZeroSeedUIntResult2) {
test.Run();
}
TEST(MurmurHash3OpTest, MoreData) {
TEST(MurmurHash3OpTest, ZeroSeedUIntResult3) {
OpTester test("MurmurHash3", 1, onnxruntime::kMSDomain);
test.AddInput<int64_t>("X", {1}, {4LL});
test.AddAttribute<int64_t>("seed", 0LL);
test.AddOutput<uint32_t>("Y", {1}, {3491892518L});
test.Run();
}
TEST(MurmurHash3OpTest, ZeroSeedFloatResult) {
OpTester test("MurmurHash3", 1, onnxruntime::kMSDomain);
test.AddInput<float>("X", {1}, {3.});
test.AddAttribute<int64_t>("seed", 0LL);
test.AddOutput<uint32_t>("Y", {1}, {6814352L});
test.Run();
}
TEST(MurmurHash3OpTest, ZeroSeedDoubleResult) {
OpTester test("MurmurHash3", 1, onnxruntime::kMSDomain);
test.AddInput<double>("X", {1}, {3.});
test.AddAttribute<int64_t>("seed", 0LL);
test.AddOutput<uint32_t>("Y", {1}, {3554953595L});
test.Run();
}
TEST(MurmurHash3OpTest, MoreDataInt) {
OpTester test("MurmurHash3", 1, onnxruntime::kMSDomain);
test.AddInput<int32_t>("X", {2}, {3L, 4L});
test.AddAttribute<int64_t>("seed", 0LL);
@ -57,6 +81,14 @@ TEST(MurmurHash3OpTest, MoreData) {
test.Run();
}
TEST(MurmurHash3OpTest, MoreDataFloat) {
OpTester test("MurmurHash3", 1, onnxruntime::kMSDomain);
test.AddInput<float>("X", {2}, {3., 4.});
test.AddAttribute<int64_t>("seed", 0LL);
test.AddOutput<uint32_t>("Y", {2}, {6814352L, 313312394L});
test.Run();
}
TEST(MurmurHash3OpTest, NonZeroSeed) {
OpTester test("MurmurHash3", 1, onnxruntime::kMSDomain);
test.AddInput<int32_t>("X", {1}, {3L});