diff --git a/onnxruntime/contrib_ops/cpu/tokenizer.cc b/onnxruntime/contrib_ops/cpu/tokenizer.cc index 404b395c96..c65e08344e 100644 --- a/onnxruntime/contrib_ops/cpu/tokenizer.cc +++ b/onnxruntime/contrib_ops/cpu/tokenizer.cc @@ -613,21 +613,14 @@ Status Tokenizer::Compute(OpKernelContext* ctx) const { "tensor(string) expected as input"); } - auto& input_dims = X->Shape().GetDims(); + auto& input_shape = X->Shape(); + auto& input_dims = input_shape.GetDims(); size_t N = 0; size_t C = 0; if (input_dims.size() == 1) { N = 1; - if (input_dims[0] < 1) { - return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, - "Invalid C dimension value"); - } C = input_dims[0]; } else if (input_dims.size() == 2) { - if (input_dims[0] < 1 || input_dims[1] < 1) { - return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, - "Invalid N and/or C dimension values"); - } N = input_dims[0]; C = input_dims[1]; } else { @@ -635,7 +628,20 @@ Status Tokenizer::Compute(OpKernelContext* ctx) const { "Input dimensions are either [C] or [N][C] allowed"); } + // Empty input Status s; + if (input_shape.Size() == 0) { + std::vector output_dims; + if (input_dims.size() == 2) { + output_dims.push_back(input_dims[0]); + } + output_dims.push_back(0); + + TensorShape output_shape(output_dims); + ctx->Output(0, output_shape); + return s; + } + if (char_tokenezation_) { s = CharTokenize(ctx, N, C, input_dims); } else { diff --git a/onnxruntime/core/graph/contrib_ops/contrib_defs.cc b/onnxruntime/core/graph/contrib_ops/contrib_defs.cc index 348b7714cf..9bc062223d 100644 --- a/onnxruntime/core/graph/contrib_ops/contrib_defs.cc +++ b/onnxruntime/core/graph/contrib_ops/contrib_defs.cc @@ -16,7 +16,7 @@ namespace ONNX_NAMESPACE { void convPoolShapeInference( - ONNX_NAMESPACE::InferenceContext& ctx, + ONNX_NAMESPACE::InferenceContext& ctx, bool use_dilation, bool require_kernel_shape, int input1Idx, int input2Idx); @@ -220,7 +220,8 @@ value at X[t][n] >= seqLengths[n]. ONNX_CONTRIB_OPERATOR_SCHEMA(GRUUnit) .SinceVersion(1) .SetDoc(GRUUnit_ver1_doc) - .Attr("drop_states", "Bool to determine if hidden state is zeroes or passed " + .Attr("drop_states", + "Bool to determine if hidden state is zeroes or passed " "along for timesteps past the given sequence_length.", AttributeProto::INT, OPTIONAL) .Input(0, "hidden_prev", "The previous GRU hidden state.", "T") @@ -266,7 +267,7 @@ and op)DOC"; "tensor(float16)", "tensor(float)", "tensor(double)"}, - "Constrain output types to bool, int32, int64, float16, float, double tensors."); + "Constrain output types to bool, int32, int64, float16, float, double tensors."); ONNX_CONTRIB_OPERATOR_SCHEMA(GivenTensorFill) .SinceVersion(10) @@ -367,7 +368,6 @@ and op)DOC"; "tensor(double)"}, "Constrain output types to bool, int32, int64, float16, float, double tensors."); - ONNX_OPERATOR_SCHEMA(MeanVarianceNormalization) .SinceVersion(1) .SetDoc(R"DOC(Perform mean variance normalization.)DOC") @@ -473,7 +473,7 @@ and op)DOC"; {"tensor(float16)", "tensor(float)", "tensor(double)"}, "Constrain input and output types to float tensors.") .TypeAndShapeInferenceFunction(ONNX_NAMESPACE::propagateShapeAndTypeFromFirstInput); - + // End of ONNX exp ops(Affine, Crop, ParametricSoftplus, ImageScaler, ThresholdedRelu, DynamicSlice, ScaledTanh, MVN) old version history maintenance ONNX_CONTRIB_OPERATOR_SCHEMA(SampleOp) @@ -778,6 +778,13 @@ activation and leaky_relu_alpha.)DOC") If input is ["Hello", "World"], then the corresponding output would be [0x02, "Hello", "World", 0x03]. This implies that if mark is true, [C]/[N, C] - input's output shape becomes [C, D+2]/[N, C, D+2]. + +If tokenizer removes the entire content of [C]-input, it will produce [[]]. +I.e. the output shape should be [C][0] or [N][C][0] if input shape was [N][C]. + +If the tokenizer receives empty input of [0] then the output is [0] if empty input +of [N, 0] then [N, 0]. + )DOC"; ONNX_CONTRIB_OPERATOR_SCHEMA(Tokenizer) @@ -835,11 +842,27 @@ activation and leaky_relu_alpha.)DOC") if (dims.size() < 1 || dims.size() > 2) { fail_shape_inference("Input dimensions are either [C] or [N][C] allowed"); } + + int64_t size = 1; for (auto& dim : dims) { - *output_shape.add_dim() = dim; + if (dim.has_dim_value()) { + size *= dim.dim_value(); + } + } + + if (size > 0) { + for (auto& dim : dims) { + *output_shape.add_dim() = dim; + } + // Add the last unknown dimension + // only if the input is not empty + output_shape.add_dim(); + } else if (size == 0) { + if (dims.size() == 2) { + *output_shape.add_dim() = dims[0]; + } + output_shape.add_dim()->set_dim_value(0); } - // Add the last unknown dimension - output_shape.add_dim(); updateOutputShape(ctx, 0, output_shape); }); diff --git a/onnxruntime/test/contrib_ops/tokenizer_test.cc b/onnxruntime/test/contrib_ops/tokenizer_test.cc index 0375b0f76e..a5741785e7 100644 --- a/onnxruntime/test/contrib_ops/tokenizer_test.cc +++ b/onnxruntime/test/contrib_ops/tokenizer_test.cc @@ -830,5 +830,55 @@ TEST(ContribOpTest, TokenizerExpression_RegChar) { test.Run(OpTester::ExpectResult::kExpectSuccess); } +TEST(ContribOpTest, Tokenizer_EmptyInput) { + // Special case of empty input. + // For [C] empty input we should output [0] + { + OpTester test("Tokenizer", opset_ver, domain); + InitTestAttr(test, true, {""}, 1); + + std::vector dims{0}; + std::vector input; + test.AddInput("T", dims, input); + + std::vector output_dims(dims); + std::vector output; + + test.AddOutput("Y", output_dims, output); + + test.Run(OpTester::ExpectResult::kExpectSuccess); + } + // For [N][C] empty input we output [N][0] + { + OpTester test("Tokenizer", opset_ver, domain); + InitTestAttr(test, true, {""}, 1); + + std::vector dims{1, 0}; + std::vector input; + test.AddInput("T", dims, input); + + std::vector output_dims(dims); + std::vector output; + + test.AddOutput("Y", output_dims, output); + + test.Run(OpTester::ExpectResult::kExpectSuccess); + } + { + OpTester test("Tokenizer", opset_ver, domain); + InitTestAttr(test, true, {""}, 1); + + std::vector dims{0, 1}; + std::vector input; + test.AddInput("T", dims, input); + + std::vector output_dims{0, 0}; + std::vector output; + + test.AddOutput("Y", output_dims, output); + + test.Run(OpTester::ExpectResult::kExpectSuccess); + } +} } // namespace test } // namespace onnxruntime