From 5cfde7af29549b407daaaa1c228fb144a5375981 Mon Sep 17 00:00:00 2001 From: Rachel Guo <35738743+YUNQIUGUO@users.noreply.github.com> Date: Fri, 11 Feb 2022 10:42:08 -0800 Subject: [PATCH] [NNAPI QDQ] Add QDQTranspose op support (#10495) * Squashed commit of the following: commit 12380491a917493401a9bd65c6ed21122263a2df Author: Guoyu Wang Date: Mon Feb 7 12:59:04 2022 -0800 Add qdq mul support commit 9cadda7f2cad19ff9e67ee648c7f04d96dd98561 Merge: 7a32847761 0f5d0a091a Author: Guoyu Wang Date: Mon Feb 7 11:24:47 2022 -0800 Merge remote-tracking branch 'origin/master' into gwang-msft/qdq_mul commit 7a328477612cafe690a5479b1ce60ee9b87bd4c4 Author: Guoyu Wang Date: Mon Feb 7 00:41:30 2022 -0800 move test case to util commit c1a8f0d81e7e75e74dc7e9c45b27ccb77ca82282 Author: Guoyu Wang Date: Fri Feb 4 13:04:26 2022 -0800 update input/output check commit a6f0a0d50466f65d6b222ca7dfe31fe1a13c7113 Author: Guoyu Wang Date: Thu Feb 3 18:37:21 2022 -0800 update quantized io check functions commit 87f4d1dcfeafea1208b03e1ce06fb0b78a956964 Merge: 7849f07109 97b8f6f394 Author: Guoyu Wang Date: Wed Feb 2 17:22:58 2022 -0800 Merge remote-tracking branch 'origin/master' into gwang-msft/qdq_mul commit 7849f07109ddb5a5f82edf27431aded729dd3872 Author: Guoyu Wang Date: Wed Feb 2 17:22:55 2022 -0800 minor update commit 7196cdf4197d3275ba83d9a9bd6c6cbeb859b8a4 Author: Guoyu Wang Date: Wed Feb 2 10:50:10 2022 -0800 init change commit 84c00772a1eab9fb3a4002683e723152bb48c247 Merge: a8c7dce22f 7318361645 Author: Guoyu Wang Date: Tue Feb 1 18:21:17 2022 -0800 Merge remote-tracking branch 'origin/master' into gwang-msft/qdq_mul commit a8c7dce22f089def87be263d65357ca1d4293499 Merge: 55e536c182 ef7b4dc05c Author: Guoyu Wang Date: Tue Feb 1 13:51:04 2022 -0800 Merge remote-tracking branch 'origin/master' into gwang-msft/qdq_mul commit 55e536c182f151cbc360451164f78a548bd3ae71 Author: Guoyu Wang Date: Tue Feb 1 11:44:34 2022 -0800 address cr comments commit d460f5b776f53b2adc6c503fc7f0a120ea9c4775 Author: Guoyu Wang Date: Tue Feb 1 00:33:54 2022 -0800 fix android UT failure commit 52146cf06f5671299af27852fae6c31cf486f391 Author: Guoyu Wang Date: Mon Jan 31 16:01:13 2022 -0800 fix build break commit ec6d07df8bf59caf35d1b8e0d6b46d14b8676e49 Author: Guoyu Wang Date: Mon Jan 31 15:41:52 2022 -0800 minor update to UT commit 8ec8490b4fda73da91036935841792389b010ed8 Author: Guoyu Wang Date: Mon Jan 31 15:01:30 2022 -0800 Add NNAPI support of QDQ Resize * Update qdq add/mul test case, fix build break * Address CR comments * Add QLinearMul support * remove unused params * Address CR comments * wip * save * minor fix * fix * fix build * address pr comments * fix wrong ut tests * address comments * minor update * fix addinitializersskip Co-authored-by: Guoyu Wang Co-authored-by: rachguo --- .../nnapi/nnapi_builtin/builders/helper.cc | 3 ++- .../nnapi/nnapi_builtin/builders/helper.h | 1 + .../nnapi_builtin/builders/op_builder.cc | 26 +++++++++++++++++++ .../builders/op_support_checker.cc | 18 +++++++++++-- onnxruntime/test/optimizer/qdq_test_utils.h | 25 ++++++++++++++++++ .../test/optimizer/qdq_transformer_test.cc | 24 ++++------------- .../test/providers/nnapi/nnapi_basic_test.cc | 11 ++++++++ 7 files changed, 86 insertions(+), 22 deletions(-) diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc index f492a4cbbc..7ae031d45f 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc @@ -78,9 +78,10 @@ QuantizedOpType GetQuantizedOpType(const NodeUnit& node_unit) { return QuantizedOpType::QDQAdd; else if (op_type == "Mul") return QuantizedOpType::QDQMul; + else if (op_type == "Transpose") + return QuantizedOpType::QDQTranspose; } else { // throw? - // Do we want to throw here? seems got neglected last time } return QuantizedOpType::Unknown; diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h index 73ea329d0b..859a92b0bf 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h @@ -90,6 +90,7 @@ enum class QuantizedOpType : uint8_t { QDQAveragePool, QDQAdd, QDQMul, + QDQTranspose, // TODO, add other QDQ NodeUnit types }; diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc index 47e8fe2eef..cd467017c3 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc @@ -788,12 +788,29 @@ Status ReluOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N #pragma region op_transpose class TransposeOpBuilder : public BaseOpBuilder { + public: + void AddInitializersToSkip(ModelBuilder& model_builder, const NodeUnit& node_unit) const override; + private: Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override; + static bool IsQuantizedOp(const NodeUnit& node_unit) ORT_MUST_USE_RESULT; // TODO, see if we want to move this to BaseOpBuilder }; +void TransposeOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const NodeUnit& node_unit) const { + if (!IsQuantizedOp(node_unit)) + return; + + AddQuantizationScaleAndZeroPointToSkip(model_builder, *node_unit.Inputs()[0].quant_param); // x_scale, x_zp + AddQuantizationScaleAndZeroPointToSkip(model_builder, *node_unit.Outputs()[0].quant_param); // y_scale, y_zp +} + +/* static */ bool TransposeOpBuilder::IsQuantizedOp(const NodeUnit& node_unit) { + return GetQuantizedOpType(node_unit) == QuantizedOpType::QDQTranspose; +} + Status TransposeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const { auto& shaper(model_builder.GetShaper()); + const auto& initializers(model_builder.GetInitializerTensors()); const auto& input = node_unit.Inputs()[0].node_arg.Name(); const auto& output = node_unit.Outputs()[0].node_arg.Name(); @@ -816,6 +833,15 @@ Status TransposeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, co perm[i] = axis_nchw_to_nhwc[perm[i]]; } + // Check if the quantization scale and ZP are correct + if (IsQuantizedOp(node_unit)) { + float x_scale = 0.0f; + int32_t x_zero_point = 0; + ORT_RETURN_IF_ERROR(GetQuantizationScaleAndZeroPoint( + initializers, node_unit.Inputs()[0], node_unit.ModelPath(), x_scale, x_zero_point)); + ORT_RETURN_IF_ERROR(IsValidInputQuantizedType(model_builder, input, x_scale, x_zero_point)); + } + std::string perm_name = model_builder.GetUniqueName(node_unit.Name() + input + "perm"); // It is possible this onnx transpose operator can be nchw->nhwc, but so far I don't see diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_support_checker.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_support_checker.cc index 01b51fed39..5f0b4d840a 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_support_checker.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_support_checker.cc @@ -592,8 +592,14 @@ class TransposeOpSupportChecker : public BaseOpSupportChecker { bool HasSupportedInputOutputsImpl( const InitializedTensorSet& initializers, const NodeUnit& node_unit, const OpSupportCheckParams& params) const override; + bool IsNodeUnitTypeSupported(const NodeUnit& /* node_unit */) const override { return true; } + static bool IsQuantizedOp(const NodeUnit& node_unit) ORT_MUST_USE_RESULT; // TODO, see if we want to move this to BaseOpBuilder }; +/* static */ bool TransposeOpSupportChecker::IsQuantizedOp(const NodeUnit& node_unit) { + return GetQuantizedOpType(node_unit) == QuantizedOpType::QDQTranspose; +} + bool TransposeOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& /* initializers */, const NodeUnit& node_unit, const OpSupportCheckParams& /* params */) const { Shape input_shape; @@ -611,8 +617,8 @@ bool TransposeOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& /* } bool TransposeOpSupportChecker::HasSupportedInputOutputsImpl( - const InitializedTensorSet& /* initializers */, const NodeUnit& node_unit, - const OpSupportCheckParams& /* params */) const { + const InitializedTensorSet& initializers, const NodeUnit& node_unit, + const OpSupportCheckParams& params) const { int32_t input_type; if (!GetType(node_unit.Inputs()[0].node_arg, input_type)) return false; @@ -625,6 +631,14 @@ bool TransposeOpSupportChecker::HasSupportedInputOutputsImpl( return false; } + if (IsQuantizedOp(node_unit)) { + if (!IsQuantizedIOSupported(initializers, node_unit, {0}, params, IOKind::Input)) + return false; + + if (!IsQuantizedIOSupported(initializers, node_unit, {0}, params, IOKind::Output)) + return false; + } + return true; } diff --git a/onnxruntime/test/optimizer/qdq_test_utils.h b/onnxruntime/test/optimizer/qdq_test_utils.h index fbbb0b8af1..1a0c0f044c 100644 --- a/onnxruntime/test/optimizer/qdq_test_utils.h +++ b/onnxruntime/test/optimizer/qdq_test_utils.h @@ -187,5 +187,30 @@ GetQDQTestCaseFn BuildBinaryOpTestCase(const std::vector& input_shape, output_arg); }; } + +template +GetQDQTestCaseFn BuildQDQTransposeTestCase( + const std::vector& input_shape, + const std::vector& perms) { + return [input_shape, perms](ModelTestBuilder& builder) { + auto* input_arg = builder.MakeInput(input_shape, -128, 127); + auto* output_arg = builder.MakeOutput(); + + InputType dq_zp = std::numeric_limits::max() / 2; + OutputType q_zp = std::numeric_limits::max() / 2; + + // add DQ + auto* dq_output = builder.MakeIntermediate(); + builder.AddDequantizeLinearNode(input_arg, .003f, dq_zp, dq_output); + + // add Transpose + auto* transpose_output = builder.MakeIntermediate(); + Node& transpose_node = builder.AddNode("Transpose", {dq_output}, {transpose_output}); + transpose_node.AddAttribute("perm", perms); + + // add Q + builder.AddQuantizeLinearNode(transpose_output, .003f, q_zp, output_arg); + }; +} } // namespace test } // namespace onnxruntime \ No newline at end of file diff --git a/onnxruntime/test/optimizer/qdq_transformer_test.cc b/onnxruntime/test/optimizer/qdq_transformer_test.cc index 73b07e8adb..9eef77aa53 100644 --- a/onnxruntime/test/optimizer/qdq_transformer_test.cc +++ b/onnxruntime/test/optimizer/qdq_transformer_test.cc @@ -665,24 +665,7 @@ TEST(QDQTransformerTests, Gather) { } TEST(QDQTransformerTests, Transpose) { - auto test_case = [&](const std::vector& input1_shape, const std::vector& perms) { - auto build_test_case = [&](ModelTestBuilder& builder) { - auto* input1_arg = builder.MakeInput(input1_shape, -128, 127); - auto* output_arg = builder.MakeOutput(); - - // add DQ - auto* dq_output = builder.MakeIntermediate(); - builder.AddDequantizeLinearNode(input1_arg, .003f, 1, dq_output); - - // add Transpose - auto* transpose_output = builder.MakeIntermediate(); - Node& transpose_node = builder.AddNode("Transpose", {dq_output}, {transpose_output}); - transpose_node.AddAttribute("perm", perms); - - // add Q - builder.AddQuantizeLinearNode(transpose_output, .003f, 1, output_arg); - }; - + auto test_case = [&](const std::vector& input_shape, const std::vector& perms) { auto check_graph = [&](InferenceSessionWrapper& session) { auto op_to_count = CountOpsInGraph(session.GetGraph()); EXPECT_EQ(op_to_count["Transpose"], 1); @@ -690,7 +673,10 @@ TEST(QDQTransformerTests, Transpose) { EXPECT_EQ(op_to_count["DequantizeLinear"], 0); }; - TransformerTester(build_test_case, check_graph, TransformerLevel::Level1, TransformerLevel::Level2); + TransformerTester(BuildQDQTransposeTestCase(input_shape, perms), + check_graph, + TransformerLevel::Level1, + TransformerLevel::Level2); }; test_case({2, 13, 12, 37}, {0, 3, 1, 2}); diff --git a/onnxruntime/test/providers/nnapi/nnapi_basic_test.cc b/onnxruntime/test/providers/nnapi/nnapi_basic_test.cc index 834f0a7378..a136e0b22c 100644 --- a/onnxruntime/test/providers/nnapi/nnapi_basic_test.cc +++ b/onnxruntime/test/providers/nnapi/nnapi_basic_test.cc @@ -359,6 +359,17 @@ TEST(NnapiExecutionProviderTest, TestQDQMul) { }); } +TEST(NnapiExecutionProviderTest, TestQDQTranspose) { + RunQDQModelTest(BuildQDQTransposeTestCase( + {1, 3, 32, 32} /* input_shape */, + {0, 3, 1, 2} /* perms */), + "nnapi_qdq_test_graph_transpose", + { + true /* verify_entire_graph_use_ep */ + }); +} + #endif // !(ORT_MINIMAL_BUILD) TEST(NnapiExecutionProviderTest, NNAPIFlagsTest) {