saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-27 20:02:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fix empty input handling in Tokenizer. (#826)

Browse source
This commit is contained in:
Dmitri Smirnov 2019-04-15 14:46:17 -07:00 committed by GitHub
parent 2c0b8e965e
commit 6194a92249
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 96 additions and 17 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

24
onnxruntime/contrib_ops/cpu/tokenizer.cc
View file

@ -613,21 +613,14 @@ Status Tokenizer::Compute(OpKernelContext* ctx) const {
"tensor(string) expected as input");
}
auto& input_dims = X->Shape().GetDims();
auto& input_shape = X->Shape();
auto& input_dims = input_shape.GetDims();
size_t N = 0;
size_t C = 0;
if (input_dims.size() == 1) {
N = 1;
if (input_dims[0] < 1) {
return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT,
"Invalid C dimension value");
}
C = input_dims[0];
} else if (input_dims.size() == 2) {
if (input_dims[0] < 1 || input_dims[1] < 1) {
return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT,
"Invalid N and/or C dimension values");
}
N = input_dims[0];
C = input_dims[1];
} else {
@ -635,7 +628,20 @@ Status Tokenizer::Compute(OpKernelContext* ctx) const {
"Input dimensions are either [C] or [N][C] allowed");
}
// Empty input
Status s;
if (input_shape.Size() == 0) {
std::vector<int64_t> output_dims;
if (input_dims.size() == 2) {
output_dims.push_back(input_dims[0]);
}
output_dims.push_back(0);
TensorShape output_shape(output_dims);
ctx->Output(0, output_shape);
return s;
}
if (char_tokenezation_) {
s = CharTokenize(ctx, N, C, input_dims);
} else {

39
onnxruntime/core/graph/contrib_ops/contrib_defs.cc
View file

@ -16,7 +16,7 @@
namespace ONNX_NAMESPACE {
void convPoolShapeInference(
ONNX_NAMESPACE::InferenceContext& ctx,
ONNX_NAMESPACE::InferenceContext& ctx,
bool use_dilation, bool require_kernel_shape,
int input1Idx,
int input2Idx);
@ -220,7 +220,8 @@ value at X[t][n] >= seqLengths[n].
ONNX_CONTRIB_OPERATOR_SCHEMA(GRUUnit)
.SinceVersion(1)
.SetDoc(GRUUnit_ver1_doc)
.Attr("drop_states", "Bool to determine if hidden state is zeroes or passed "
.Attr("drop_states",
"Bool to determine if hidden state is zeroes or passed "
"along for timesteps past the given sequence_length.",
AttributeProto::INT, OPTIONAL)
.Input(0, "hidden_prev", "The previous GRU hidden state.", "T")
@ -266,7 +267,7 @@ and op)DOC";
"tensor(float16)",
"tensor(float)",
"tensor(double)"},
"Constrain output types to bool, int32, int64, float16, float, double tensors.");
"Constrain output types to bool, int32, int64, float16, float, double tensors.");
ONNX_CONTRIB_OPERATOR_SCHEMA(GivenTensorFill)
.SinceVersion(10)
@ -367,7 +368,6 @@ and op)DOC";
"tensor(double)"},
"Constrain output types to bool, int32, int64, float16, float, double tensors.");
ONNX_OPERATOR_SCHEMA(MeanVarianceNormalization)
.SinceVersion(1)
.SetDoc(R"DOC(Perform mean variance normalization.)DOC")
@ -473,7 +473,7 @@ and op)DOC";
{"tensor(float16)", "tensor(float)", "tensor(double)"},
"Constrain input and output types to float tensors.")
.TypeAndShapeInferenceFunction(ONNX_NAMESPACE::propagateShapeAndTypeFromFirstInput);
// End of ONNX exp ops(Affine, Crop, ParametricSoftplus, ImageScaler, ThresholdedRelu, DynamicSlice, ScaledTanh, MVN) old version history maintenance
ONNX_CONTRIB_OPERATOR_SCHEMA(SampleOp)
@ -778,6 +778,13 @@ activation and leaky_relu_alpha.)DOC")
If input is ["Hello", "World"],
then the corresponding output would be [0x02, "Hello", "World", 0x03].
This implies that if mark is true, [C]/[N, C] - input's output shape becomes [C, D+2]/[N, C, D+2].
If tokenizer removes the entire content of [C]-input, it will produce [[]].
I.e. the output shape should be [C][0] or [N][C][0] if input shape was [N][C].
If the tokenizer receives empty input of [0] then the output is [0] if empty input
of [N, 0] then [N, 0].
)DOC";
ONNX_CONTRIB_OPERATOR_SCHEMA(Tokenizer)
@ -835,11 +842,27 @@ activation and leaky_relu_alpha.)DOC")
if (dims.size() < 1 || dims.size() > 2) {
fail_shape_inference("Input dimensions are either [C] or [N][C] allowed");
}
int64_t size = 1;
for (auto& dim : dims) {
*output_shape.add_dim() = dim;
if (dim.has_dim_value()) {
size *= dim.dim_value();
}
}
if (size > 0) {
for (auto& dim : dims) {
*output_shape.add_dim() = dim;
}
// Add the last unknown dimension
// only if the input is not empty
output_shape.add_dim();
} else if (size == 0) {
if (dims.size() == 2) {
*output_shape.add_dim() = dims[0];
}
output_shape.add_dim()->set_dim_value(0);
}
// Add the last unknown dimension
output_shape.add_dim();
updateOutputShape(ctx, 0, output_shape);
});

50
onnxruntime/test/contrib_ops/tokenizer_test.cc
View file

@ -830,5 +830,55 @@ TEST(ContribOpTest, TokenizerExpression_RegChar) {
test.Run(OpTester::ExpectResult::kExpectSuccess);
}
TEST(ContribOpTest, Tokenizer_EmptyInput) {
// Special case of empty input.
// For [C] empty input we should output [0]
{
OpTester test("Tokenizer", opset_ver, domain);
InitTestAttr(test, true, {""}, 1);
std::vector<int64_t> dims{0};
std::vector<std::string> input;
test.AddInput<std::string>("T", dims, input);
std::vector<int64_t> output_dims(dims);
std::vector<std::string> output;
test.AddOutput<std::string>("Y", output_dims, output);
test.Run(OpTester::ExpectResult::kExpectSuccess);
}
// For [N][C] empty input we output [N][0]
{
OpTester test("Tokenizer", opset_ver, domain);
InitTestAttr(test, true, {""}, 1);
std::vector<int64_t> dims{1, 0};
std::vector<std::string> input;
test.AddInput<std::string>("T", dims, input);
std::vector<int64_t> output_dims(dims);
std::vector<std::string> output;
test.AddOutput<std::string>("Y", output_dims, output);
test.Run(OpTester::ExpectResult::kExpectSuccess);
}
{
OpTester test("Tokenizer", opset_ver, domain);
InitTestAttr(test, true, {""}, 1);
std::vector<int64_t> dims{0, 1};
std::vector<std::string> input;
test.AddInput<std::string>("T", dims, input);
std::vector<int64_t> output_dims{0, 0};
std::vector<std::string> output;
test.AddOutput<std::string>("Y", output_dims, output);
test.Run(OpTester::ExpectResult::kExpectSuccess);
}
}
} // namespace test
} // namespace onnxruntime