From 2490cf84c99407f1f41625cc82385f2181fd0dde Mon Sep 17 00:00:00 2001 From: JiCheng Date: Tue, 15 Nov 2022 12:02:17 +0800 Subject: [PATCH] [QLinearSoftmax]remove input_shape check in Ctor (#13489) ### Description In some case, we can't get node's shape to do pre-process. ### Motivation and Context --- .../cpu/quantization/qlinear_softmax.cc | 39 +-- .../cpu/quantization/qlinear_softmax.h | 2 +- .../contrib_ops/qlinear_lookup_table_test.cc | 224 ++++++++++-------- 3 files changed, 144 insertions(+), 121 deletions(-) diff --git a/onnxruntime/contrib_ops/cpu/quantization/qlinear_softmax.cc b/onnxruntime/contrib_ops/cpu/quantization/qlinear_softmax.cc index f20c04fa13..176ce21dfd 100644 --- a/onnxruntime/contrib_ops/cpu/quantization/qlinear_softmax.cc +++ b/onnxruntime/contrib_ops/cpu/quantization/qlinear_softmax.cc @@ -70,9 +70,6 @@ QLinearSoftmax::QLinearSoftmax(const OpKernelInfo& info) auto input_defs = node.InputDefs(); auto input_type = input_defs[0]->TypeAsProto()->tensor_type().elem_type(); is_signed_ = (input_type == ONNX_NAMESPACE::TensorProto_DataType_INT8); - const auto* x_shape = input_defs[0]->Shape(); - ORT_ENFORCE(x_shape != nullptr && x_shape->dim_size() > 0, "input_shape of QLinearSoftmax must be existed"); - int rank = x_shape->dim_size(); int64_t opset = -1; Status status = info.GetAttr("opset", &opset); @@ -89,12 +86,15 @@ QLinearSoftmax::QLinearSoftmax(const OpKernelInfo& info) axis_ = opset_ < OPSET13 ? 1 : -1; } - axis_ = static_cast(HandleNegativeAxis(axis_, int64_t(rank))); - auto input_shape = utils::GetTensorShapeFromTensorShapeProto(*x_shape); - int64_t reduce_size = opset_ < OPSET13 ? input_shape.SizeFromDimension(axis_) : input_shape[axis_]; - // reduce_size could be negative if input-shape has a dynamic axis - if (reduce_size > 0) { - BuildLookupTableIfFixed(info, fixed_lookup_table_, onnxruntime::narrow(reduce_size), is_signed_); + const auto* x_shape = input_defs[0]->Shape(); + if (x_shape != nullptr && x_shape->dim_size() > 0) { + axis_ = static_cast(HandleNegativeAxis(axis_, int64_t(x_shape->dim_size()))); + auto input_shape = utils::GetTensorShapeFromTensorShapeProto(*x_shape); + int64_t reduce_size = opset_ < OPSET13 ? input_shape.SizeFromDimension(axis_) : input_shape[axis_]; + // reduce_size could be negative if input-shape has a dynamic axis + if (reduce_size > 0) { + BuildLookupTableIfFixed(info, fixed_lookup_table_, reduce_size, is_signed_); + } } } @@ -102,21 +102,24 @@ QLinearSoftmax::QLinearSoftmax(const OpKernelInfo& info) Status QLinearSoftmax::Compute(OpKernelContext* ctx) const { const auto* X = ctx->Input(0); const auto& X_shape = X->Shape(); - auto* Y = ctx->Output(0, X_shape); - // edge case. one or more dims with value of 0. nothing to do if (X_shape.Size() == 0) { return Status::OK(); } + + auto axis = static_cast(HandleNegativeAxis(axis_, int64_t(X_shape.NumDimensions()))); + + auto* Y = ctx->Output(0, X_shape); + concurrency::ThreadPool* thread_pool = ctx->GetOperatorThreadPool(); - const size_t D = opset_ < OPSET13 ? onnxruntime::narrow(X_shape.SizeFromDimension(onnxruntime::narrow(axis_))) : onnxruntime::narrow(X_shape[onnxruntime::narrow(axis_)]); + const size_t D = opset_ < OPSET13 ? X_shape.SizeFromDimension(axis) : X_shape[axis]; EXP_OUT_DTYPE tmp_lookup_table[256]; gsl::span lookup_table = GetLookupTable(ctx, tmp_lookup_table, D); if (opset_ < OPSET13) { - return ComputeInternal(ctx, *X, *Y, lookup_table, axis_, thread_pool); + return ComputeInternal(ctx, *X, *Y, lookup_table, axis, thread_pool); } else { - return ComputeImplOpset13(ctx, *X, *Y, lookup_table, thread_pool); + return ComputeImplOpset13(ctx, *X, *Y, lookup_table, axis, thread_pool); } } @@ -291,12 +294,12 @@ Status QLinearSoftmax::ComputeInternal(OpKernelContext* context, const Tensor& i // opset-13 and above Status QLinearSoftmax::ComputeImplOpset13(OpKernelContext* context, const Tensor& input, Tensor& output, - gsl::span lookup_table, + gsl::span lookup_table, int axis, concurrency::ThreadPool* thread_pool) const { const auto& X_shape = input.Shape(); size_t rank = X_shape.NumDimensions(); - bool is_transpose_required = (size_t(axis_) != (rank - 1)); + bool is_transpose_required = (size_t(axis) != (rank - 1)); Tensor transposed_input; Tensor intermediate_output; // output that the softmax implementation will write into while using transposed input std::vector permutation(rank); @@ -307,8 +310,8 @@ Status QLinearSoftmax::ComputeImplOpset13(OpKernelContext* context, std::iota(std::begin(permutation), std::end(permutation), 0); // swap the innermost dim with the dim corresponding to axis - permutation[axis_] = rank - 1; - permutation[rank - 1] = axis_; + permutation[axis] = rank - 1; + permutation[rank - 1] = axis; std::vector transposed_input_dims(rank); std::transform(permutation.cbegin(), permutation.cend(), transposed_input_dims.begin(), [&X_shape](size_t e) { return X_shape[e]; }); diff --git a/onnxruntime/contrib_ops/cpu/quantization/qlinear_softmax.h b/onnxruntime/contrib_ops/cpu/quantization/qlinear_softmax.h index 08785f05a6..f47480300f 100644 --- a/onnxruntime/contrib_ops/cpu/quantization/qlinear_softmax.h +++ b/onnxruntime/contrib_ops/cpu/quantization/qlinear_softmax.h @@ -26,7 +26,7 @@ class QLinearSoftmax final : public OpKernel { int axis, concurrency::ThreadPool* thread_pool) const; Status ComputeImplOpset13(OpKernelContext* context, const Tensor& input, Tensor& output, - gsl::span lookup_table, + gsl::span lookup_table, int axis, concurrency::ThreadPool* thread_pool) const; private: diff --git a/onnxruntime/test/contrib_ops/qlinear_lookup_table_test.cc b/onnxruntime/test/contrib_ops/qlinear_lookup_table_test.cc index cbb0397af9..09ca669b4a 100644 --- a/onnxruntime/test/contrib_ops/qlinear_lookup_table_test.cc +++ b/onnxruntime/test/contrib_ops/qlinear_lookup_table_test.cc @@ -134,139 +134,159 @@ TEST(QLinearLookupTableBasedOperatorTests, QLinearSigmoid_UInt8_0_Y_ZP) { \see then followed by the [DOC](https://pytorch.org/docs/stable/quantization.html) */ TEST(QLinearLookupTableBasedOperatorTests, QLinearSoftmax_UInt8_v12) { - OpTester test("QLinearSoftmax", 1, onnxruntime::kMSDomain); - test.AddAttribute("axis", -2); - test.AddAttribute("opset", 12); - float X_scale = 0.166099221f; - // - uint8_t X_zero_point = 128; - float Y_scale = 1.0f / 256.0f; - uint8_t Y_zero_point = 0; - // + auto run_test = [](bool add_shape_to_input) { + OpTester test("QLinearSoftmax", 1, onnxruntime::kMSDomain); + test.AddAttribute("axis", -2); + test.AddAttribute("opset", 12); + float X_scale = 0.166099221f; + // + uint8_t X_zero_point = 128; + float Y_scale = 1.0f / 256.0f; + uint8_t Y_zero_point = 0; + // - std::vector dims = {2, 4, 5}; - auto x_in = std::vector{50, 67, 58, 68, 46, 69, 77, 91, 62, 74, 67, 72, 71, 70, 83, 88, 75, 54, 74, 88}; - auto y_out = std::vector { 0, 2, 0, 2, 0, 2, 8, 86, 1, 5, 2, 4, 3, 3, 23, 52, 6, 0, 5, 52 }; - for (int64_t i = 1; i < dims[0]; i++) { - for (int64_t j = 0; j < dims[1] * dims[2]; j++) { - x_in.push_back(x_in[j]); - y_out.push_back(y_out[j]); + std::vector dims = {2, 4, 5}; + auto x_in = std::vector{50, 67, 58, 68, 46, 69, 77, 91, 62, 74, 67, 72, 71, 70, 83, 88, 75, 54, 74, 88}; + auto y_out = std::vector{0, 2, 0, 2, 0, 2, 8, 86, 1, 5, 2, 4, 3, 3, 23, 52, 6, 0, 5, 52}; + for (int64_t i = 1; i < dims[0]; i++) { + for (int64_t j = 0; j < dims[1] * dims[2]; j++) { + x_in.push_back(x_in[j]); + y_out.push_back(y_out[j]); + } } - } - test.AddInput("X", dims, x_in); - test.AddInput("X_scale", {}, {X_scale}); - test.AddInput("X_zero_point", {}, {X_zero_point}); - test.AddInput("Y_scale", {}, {Y_scale}); - test.AddInput("Y_zero_point", {}, {Y_zero_point}); - test.AddOutput("Y", dims, y_out); - auto origin_round_mode = std::fegetround(); - std::fesetround(FE_TONEAREST); - test.Run(); - std::fesetround(origin_round_mode); + test.AddShapeToTensorData(add_shape_to_input); + test.AddInput("X", dims, x_in); + test.AddInput("X_scale", {}, {X_scale}); + test.AddInput("X_zero_point", {}, {X_zero_point}); + test.AddInput("Y_scale", {}, {Y_scale}); + test.AddInput("Y_zero_point", {}, {Y_zero_point}); + test.AddOutput("Y", dims, y_out); + auto origin_round_mode = std::fegetround(); + std::fesetround(FE_TONEAREST); + test.Run(); + std::fesetround(origin_round_mode); + }; + run_test(true); + run_test(false); } TEST(QLinearLookupTableBasedOperatorTests, QLinearSoftmax_UInt8_v13) { - OpTester test("QLinearSoftmax", 1, onnxruntime::kMSDomain); - test.AddAttribute("axis", -2); - test.AddAttribute("opset", 13); - float X_scale = 0.0304f; - // - uint8_t X_zero_point = 128; - float Y_scale = 0.0059f; - uint8_t Y_zero_point = 0; - // + auto run_test = [](bool add_shape_to_input) { + OpTester test("QLinearSoftmax", 1, onnxruntime::kMSDomain); + test.AddAttribute("axis", -2); + test.AddAttribute("opset", 13); + float X_scale = 0.0304f; + // + uint8_t X_zero_point = 128; + float Y_scale = 0.0059f; + uint8_t Y_zero_point = 0; + // - std::vector dims = {4, 4, 4}; - auto x_in = std::vector { - 62, 50, 71, 37, 68, 88, 64, 51, 59, 95, 41, 54, 55, 20, 77, 32, 92, + std::vector dims = {4, 4, 4}; + auto x_in = std::vector{ + 62, 50, 71, 37, 68, 88, 64, 51, 59, 95, 41, 54, 55, 20, 77, 32, 92, 63, 43, 13, 76, 82, 53, 43, 60, 18, 73, 74, 22, 89, 44, 106, 17, 95, 27, 35, 47, 57, 0, 78, 97, 66, 56, 28, 127, 33, 106, 71, 119, 64, 16, 0, 16, 79, 27, 89, 110, 126, 88, 90, 67, 11, 4, 90}; - auto y_out = std::vector { - 43, 20, 50, 33, 52, 63, 40, 51, 39, 78, + auto y_out = std::vector{ + 43, 20, 50, 33, 52, 63, 40, 51, 39, 78, 20, 56, 35, 8, 59, 29, 80, 32, 29, 6, 49, 57, 39, 16, 30, 8, 72, 40, 10, 71, 30, 107, 4, 90, 11, 20, 10, 28, 5, 74, 45, 37, 27, 16, 111, 14, 125, 59, 84, 18, 14, 4, 4, 28, 20, 54, 64, 119, 126, 56, 17, 4, 10, 56}; - test.AddInput("X", dims, x_in); - test.AddInput("X_scale", {}, {X_scale}); - test.AddInput("X_zero_point", {}, {X_zero_point}); - test.AddInput("Y_scale", {}, {Y_scale}); - test.AddInput("Y_zero_point", {}, {Y_zero_point}); - test.AddOutput("Y", dims, y_out); - auto origin_round_mode = std::fegetround(); - std::fesetround(FE_TONEAREST); - test.Run(); - std::fesetround(origin_round_mode); + test.AddShapeToTensorData(add_shape_to_input); + test.AddInput("X", dims, x_in); + test.AddInput("X_scale", {}, {X_scale}); + test.AddInput("X_zero_point", {}, {X_zero_point}); + test.AddInput("Y_scale", {}, {Y_scale}); + test.AddInput("Y_zero_point", {}, {Y_zero_point}); + test.AddOutput("Y", dims, y_out); + auto origin_round_mode = std::fegetround(); + std::fesetround(FE_TONEAREST); + test.Run(); + std::fesetround(origin_round_mode); + }; + run_test(true); + run_test(false); } TEST(QLinearLookupTableBasedOperatorTests, QLinearSoftmax_Int8_v13) { - OpTester test("QLinearSoftmax", 1, onnxruntime::kMSDomain); - test.AddAttribute("axis", -2); - test.AddAttribute("opset", 13); - float X_scale = 0.0304F; - // - int8_t X_zero_point = 0; - float Y_scale = 0.0059F; - int8_t Y_zero_point = -128; - // + auto run_test = [](bool add_shape_to_input) { + OpTester test("QLinearSoftmax", 1, onnxruntime::kMSDomain); + test.AddAttribute("axis", -2); + test.AddAttribute("opset", 13); + float X_scale = 0.0304F; + // + int8_t X_zero_point = 0; + float Y_scale = 0.0059F; + int8_t Y_zero_point = -128; + // - std::vector dims = {4, 4, 4}; - auto x_in = std::vector { - -4, -16, 5, -29, 2, 22, -2, -15, -7, 29, -25, -12, -11, -46, 11, -34, 26, + std::vector dims = {4, 4, 4}; + auto x_in = std::vector{ + -4, -16, 5, -29, 2, 22, -2, -15, -7, 29, -25, -12, -11, -46, 11, -34, 26, -3, -23, -53, 10, 16, -13, -23, -6, -48, 7, 8, -44, 23, -22, 40, -49, 29, -39, -31, -19, -9, -72, 12, 31, 0, -10, -38, 61, -33, 40, 5, 53, -2, -50, -66, -50, 13, -39, 23, 44, 60, 22, 24, 1, -55, -62, 24}; - auto y_out = std::vector { - -85, -108, -78, -95, -76, -65, -88, -77, -89, -50, -108, -72, -93, + auto y_out = std::vector{ + -85, -108, -78, -95, -76, -65, -88, -77, -89, -50, -108, -72, -93, -120, -69, -99, -48, -96, -99, -122, -79, -71, -89, -112, -98, -120, -56, -88, -118, -57, -98, -21, -124, -38, -117, -108, -118, -100, -124, -54, -83, -91, -100, -112, -17, -114, -2, -69, -44, -110, -114, -124, -124, -100, -108, -74, -64, -9, -2, -72, -111, -124, -118, -72}; - test.AddInput("X", dims, x_in); - test.AddInput("X_scale", {}, {X_scale}); - test.AddInput("X_zero_point", {}, {X_zero_point}); - test.AddInput("Y_scale", {}, {Y_scale}); - test.AddInput("Y_zero_point", {}, {Y_zero_point}); - test.AddOutput("Y", dims, y_out); - auto origin_round_mode = std::fegetround(); - std::fesetround(FE_TONEAREST); - test.Run(); - std::fesetround(origin_round_mode); + test.AddShapeToTensorData(add_shape_to_input); + test.AddInput("X", dims, x_in); + test.AddInput("X_scale", {}, {X_scale}); + test.AddInput("X_zero_point", {}, {X_zero_point}); + test.AddInput("Y_scale", {}, {Y_scale}); + test.AddInput("Y_zero_point", {}, {Y_zero_point}); + test.AddOutput("Y", dims, y_out); + auto origin_round_mode = std::fegetround(); + std::fesetround(FE_TONEAREST); + test.Run(); + std::fesetround(origin_round_mode); + }; + run_test(true); + run_test(false); } TEST(QLinearLookupTableBasedOperatorTests, QLinearSoftmax_Int8_v12) { - OpTester test("QLinearSoftmax", 1, onnxruntime::kMSDomain); - test.AddAttribute("axis", -2); - test.AddAttribute("opset", 12); - float X_scale = 0.166099221f; - // - int8_t X_zero_point = 0; - float Y_scale = 1.0f / 128.0f; - int8_t Y_zero_point = 0; - // + auto run_test = [](bool add_shape_to_input) { + OpTester test("QLinearSoftmax", 1, onnxruntime::kMSDomain); + test.AddAttribute("axis", -2); + test.AddAttribute("opset", 12); + float X_scale = 0.166099221f; + // + int8_t X_zero_point = 0; + float Y_scale = 1.0f / 128.0f; + int8_t Y_zero_point = 0; + // - std::vector dims = {2, 4, 5}; - auto x_in = std::vector{-28, -4, -4, -7, 3, -26, 4, -16, 23, 14, -7, 26, -8, 19, -16, -13, 7, 17, 27, 5}; - auto y_out = std::vector{0, 0, 0, 0, 1, 0, 1, 0, 22, 5, 0, 35, 0, 11, 0, 0, 2, 8, 42, 1}; - for (int64_t i = 1; i < dims[0]; i++) { - for (int64_t j = 0; j < dims[1] * dims[2]; j++) { - x_in.push_back(x_in[j]); - y_out.push_back(y_out[j]); + std::vector dims = {2, 4, 5}; + auto x_in = std::vector{-28, -4, -4, -7, 3, -26, 4, -16, 23, 14, -7, 26, -8, 19, -16, -13, 7, 17, 27, 5}; + auto y_out = std::vector{0, 0, 0, 0, 1, 0, 1, 0, 22, 5, 0, 35, 0, 11, 0, 0, 2, 8, 42, 1}; + for (int64_t i = 1; i < dims[0]; i++) { + for (int64_t j = 0; j < dims[1] * dims[2]; j++) { + x_in.push_back(x_in[j]); + y_out.push_back(y_out[j]); + } } - } - test.AddInput("X", dims, x_in); - test.AddInput("X_scale", {}, {X_scale}); - test.AddInput("X_zero_point", {}, {X_zero_point}); - test.AddInput("Y_scale", {}, {Y_scale}); - test.AddInput("Y_zero_point", {}, {Y_zero_point}); - test.AddOutput("Y", dims, y_out); - auto origin_round_mode = std::fegetround(); - std::fesetround(FE_TONEAREST); - test.Run(); - std::fesetround(origin_round_mode); + test.AddShapeToTensorData(add_shape_to_input); + test.AddInput("X", dims, x_in); + test.AddInput("X_scale", {}, {X_scale}); + test.AddInput("X_zero_point", {}, {X_zero_point}); + test.AddInput("Y_scale", {}, {Y_scale}); + test.AddInput("Y_zero_point", {}, {Y_zero_point}); + test.AddOutput("Y", dims, y_out); + auto origin_round_mode = std::fegetround(); + std::fesetround(FE_TONEAREST); + test.Run(); + std::fesetround(origin_round_mode); + }; + run_test(true); + run_test(false); } } // namespace test