From 8ec8490b4fda73da91036935841792389b010ed8 Mon Sep 17 00:00:00 2001 From: Guoyu Wang Date: Mon, 31 Jan 2022 15:01:30 -0800 Subject: [PATCH] Add NNAPI support of QDQ Resize --- .../nnapi/nnapi_builtin/builders/helper.cc | 2 + .../nnapi/nnapi_builtin/builders/helper.h | 1 + .../nnapi_builtin/builders/op_builder.cc | 21 ++++++++++ .../builders/op_support_checker.cc | 34 +++++++++++++++ onnxruntime/test/optimizer/qdq_test_utils.cc | 41 +++++++++++++++++++ onnxruntime/test/optimizer/qdq_test_utils.h | 10 +++-- .../test/optimizer/qdq_transformer_test.cc | 27 ++---------- .../test/providers/nnapi/nnapi_basic_test.cc | 25 ++++++++--- onnxruntime/test/util/test_utils.cc | 4 ++ 9 files changed, 132 insertions(+), 33 deletions(-) create mode 100644 onnxruntime/test/optimizer/qdq_test_utils.cc diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc index eecf188822..ee220b2a7e 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc @@ -68,6 +68,8 @@ QuantizedOpType GetQuantizedOpType(const NodeUnit& node_unit) { } else if (node_unit.UnitType() == NodeUnit::Type::QDQGroup) { if (op_type == "Conv") return QuantizedOpType::QDQConv; + else if (op_type == "Resize") + return QuantizedOpType::QDQResize; } else { // throw? } diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h index fe0b1ea51b..c5b3e1106d 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h @@ -86,6 +86,7 @@ enum class QuantizedOpType : uint8_t { // QLinearMul, // QLinearReduceMean, QDQConv, + QDQResize, // TODO, add other QDQ NodeUnit types }; diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc index 6272153cd2..65c593810c 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc @@ -2258,10 +2258,22 @@ class ResizeOpBuilder : public BaseOpBuilder { private: Status AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const override; + static bool IsQuantizedOp(const NodeUnit& node_unit) ORT_MUST_USE_RESULT; // TODO, see if we want to move this to BaseOpBuilder }; +/* static */ bool ResizeOpBuilder::IsQuantizedOp(const NodeUnit& node_unit) { + static bool is_quant_op_type = + GetQuantizedOpType(node_unit) == QuantizedOpType::QDQResize; + return is_quant_op_type; +} + void ResizeOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const NodeUnit& node_unit) const { const auto& inputs = node_unit.Inputs(); + if (IsQuantizedOp(node_unit)) { + AddQuantizationScaleAndZeroPointToSkip(model_builder, *inputs[0].quant_param); // x_scale, x_zp + AddQuantizationScaleAndZeroPointToSkip(model_builder, *node_unit.Outputs()[0].quant_param); // y_scale, y_zp + } + // We don't really use ROI here, so add them to skipped list model_builder.AddInitializerToSkip(inputs[1].node_arg.Name()); // ROI @@ -2296,6 +2308,15 @@ Status ResizeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const } } + // Check if the quantization scale and ZP is correct + if (IsQuantizedOp(node_unit)) { + float x_scale = 0.0f; + int32_t x_zero_point = 0; + ORT_RETURN_IF_ERROR(GetQuantizationScaleAndZeroPoint( + initializers, node_unit.Inputs()[0], node_unit.ModelPath(), x_scale, x_zero_point)); + ORT_RETURN_IF_ERROR(IsValidInputQuantizedType(model_builder, input, x_scale, x_zero_point)); + } + bool is_linear_resize = helper.Get("mode", "nearest") == "linear"; int32_t operationCode = is_linear_resize ? ANEURALNETWORKS_RESIZE_BILINEAR diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_support_checker.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_support_checker.cc index 3857de16f3..7c4329fc3c 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_support_checker.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_support_checker.cc @@ -1466,8 +1466,16 @@ class ResizeOpSupportChecker : public BaseOpSupportChecker { int GetMinSupportedOpSet(const NodeUnit& /* node_unit */) const override { return 11; } bool HasSupportedInputsImpl(const NodeUnit& node_unit) const override; + bool IsNodeUnitTypeSupported(const NodeUnit& /* node_unit */) const override { return true; } + static bool IsQuantizedOp(const NodeUnit& node_unit) ORT_MUST_USE_RESULT; // TODO, see if we want to move this to BaseOpBuilder }; +/* static */ bool ResizeOpSupportChecker::IsQuantizedOp(const NodeUnit& node_unit) { + static bool is_quant_op_type = + GetQuantizedOpType(node_unit) == QuantizedOpType::QDQResize; + return is_quant_op_type; +} + bool ResizeOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initializers, const NodeUnit& node_unit, const OpSupportCheckParams& params) const { Shape input_shape; @@ -1587,6 +1595,32 @@ bool ResizeOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initi } } } + + if (IsQuantizedOp(node_unit)) { + // For QDQResize, we only support uint8 output now + int32_t output_type; + if (!GetType(node_unit.Outputs()[0].node_arg, output_type)) + return false; + + if (output_type != ONNX_NAMESPACE::TensorProto_DataType_UINT8) { + LOGS_DEFAULT(VERBOSE) << "[Resize] output type: [" << output_type + << "] is not supported for now"; + return false; + } + + // Check input scales and ZPs + if (!HasValidQuantizationScales(initializers, node_unit, {0}, params, true /* is_input */)) + return false; + if (!HasValidQuantizationZeroPoints(initializers, node_unit, {0}, true /* is_input */)) + return false; + + // Check output scale and ZP + if (!HasValidQuantizationScales(initializers, node_unit, {0}, params, false /* is_input */)) + return false; + if (!HasValidQuantizationZeroPoints(initializers, node_unit, {0}, false /* is_input */)) + return false; + } + return true; } diff --git a/onnxruntime/test/optimizer/qdq_test_utils.cc b/onnxruntime/test/optimizer/qdq_test_utils.cc new file mode 100644 index 0000000000..6082a225d0 --- /dev/null +++ b/onnxruntime/test/optimizer/qdq_test_utils.cc @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "qdq_test_utils.h" + +namespace onnxruntime { +namespace test { + +GetQDQTestCaseFn BuildQDQResizeTestCase( + const std::vector& input_shape, + const std::vector& sizes_data) { + return [input_shape, sizes_data](ModelTestBuilder& builder) { + auto* input1_arg = builder.MakeInput(input_shape, + std::numeric_limits::min(), + std::numeric_limits::max()); + auto* roi = builder.MakeInitializer({0}, {}); + auto* scales = builder.MakeInitializer({0}, {}); + auto* sizes = builder.Make1DInitializer(sizes_data); + auto* output_arg = builder.MakeOutput(); + + // add DQ + auto* dq_output = builder.MakeIntermediate(); + builder.AddDequantizeLinearNode(input1_arg, .003f, 1, dq_output); + + // add Resize + auto* resize_output = builder.MakeIntermediate(); + Node& resize_node = builder.AddNode("Resize", {dq_output, roi, scales, sizes}, {resize_output}); + +// NNAPI EP does not support the default setting of Resize Op +// Use bi-linear and asymmetric for NNAPI EP only +#ifdef USE_NNAPI + resize_node.AddAttribute("mode", "linear"); + resize_node.AddAttribute("coordinate_transformation_mode", "asymmetric"); +#endif + // add Q + builder.AddQuantizeLinearNode(resize_output, .003f, 1, output_arg); + }; +} + +} // namespace test +} // namespace onnxruntime \ No newline at end of file diff --git a/onnxruntime/test/optimizer/qdq_test_utils.h b/onnxruntime/test/optimizer/qdq_test_utils.h index 2327ba7485..7ae23ba361 100644 --- a/onnxruntime/test/optimizer/qdq_test_utils.h +++ b/onnxruntime/test/optimizer/qdq_test_utils.h @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +#pragma once + #include "graph_transform_test_builder.h" #include "core/optimizer/qdq_transformer/selectors_actions/qdq_selector_action_transformer.h" @@ -12,7 +14,7 @@ namespace onnxruntime { namespace test { -using GetQDQConvTestCaseFn = std::function; +using GetQDQTestCaseFn = std::function; template typename std::enable_if::value, NodeArg*>::type @@ -24,10 +26,8 @@ AddQDQNodePair(ModelTestBuilder& builder, NodeArg* q_input, float scale, T zp = return dq_output; } -// TODO: for now it just builds a conv qdq graph. -// can be modified and made it shared among different qdq test graphs associated with other operators template -GetQDQConvTestCaseFn BuildQDQConvTestCase(const std::vector& input_shape, const std::vector& weights_shape) { +GetQDQTestCaseFn BuildQDQConvTestCase(const std::vector& input_shape, const std::vector& weights_shape) { return [input_shape, weights_shape](ModelTestBuilder& builder) { auto* input_arg = builder.MakeInput(input_shape, -1.f, 1.f); auto* output_arg = builder.MakeOutput(); @@ -78,5 +78,7 @@ GetQDQConvTestCaseFn BuildQDQConvTestCase(const std::vector& input_shap }; } +GetQDQTestCaseFn BuildQDQResizeTestCase(const std::vector& input_shape, const std::vector& sizes_data); + } // namespace test } // namespace onnxruntime \ No newline at end of file diff --git a/onnxruntime/test/optimizer/qdq_transformer_test.cc b/onnxruntime/test/optimizer/qdq_transformer_test.cc index 10d3bec7df..bca4eb737c 100644 --- a/onnxruntime/test/optimizer/qdq_transformer_test.cc +++ b/onnxruntime/test/optimizer/qdq_transformer_test.cc @@ -633,27 +633,6 @@ TEST(QDQTransformerTests, Transpose_No_Fusion) { TEST(QDQTransformerTests, Resize) { auto test_case = [&](const std::vector& input1_shape, const std::vector& sizes_shape) { - auto build_test_case = [&](ModelTestBuilder& builder) { - auto* input1_arg = builder.MakeInput(input1_shape, - std::numeric_limits::min(), - std::numeric_limits::max()); - auto* roi = builder.MakeInitializer({0}, {}); - auto* scales = builder.MakeInitializer({0}, {}); - auto* sizes = builder.MakeInitializer(sizes_shape, 1, 16); - auto* output_arg = builder.MakeOutput(); - - // add DQ - auto* dq_output = builder.MakeIntermediate(); - builder.AddDequantizeLinearNode(input1_arg, .003f, 1, dq_output); - - // add Resize - auto* resize_output = builder.MakeIntermediate(); - builder.AddNode("Resize", {dq_output, roi, scales, sizes}, {resize_output}); - - // add Q - builder.AddQuantizeLinearNode(resize_output, .003f, 1, output_arg); - }; - auto check_matmul_graph = [&](InferenceSessionWrapper& session) { auto op_to_count = CountOpsInGraph(session.GetGraph()); EXPECT_EQ(op_to_count["Resize"], 1); @@ -661,12 +640,14 @@ TEST(QDQTransformerTests, Resize) { EXPECT_EQ(op_to_count["DequantizeLinear"], 0); }; - TransformerTester(build_test_case, check_matmul_graph, + TransformerTester(BuildQDQResizeTestCase(input1_shape, sizes_shape), + check_matmul_graph, TransformerLevel::Level1, TransformerLevel::Level2); }; - test_case({2, 13, 12, 37}, {4}); + RandomValueGenerator rand_gen{optional{2345}}; + test_case({2, 13, 12, 37}, rand_gen.Uniform(std::vector{4}, 1, 16)); } TEST(QDQTransformerTests, Resize_No_Fusion) { diff --git a/onnxruntime/test/providers/nnapi/nnapi_basic_test.cc b/onnxruntime/test/providers/nnapi/nnapi_basic_test.cc index 4ac0c67e9e..71afbfe04b 100644 --- a/onnxruntime/test/providers/nnapi/nnapi_basic_test.cc +++ b/onnxruntime/test/providers/nnapi/nnapi_basic_test.cc @@ -240,13 +240,10 @@ TEST(NnapiExecutionProviderTest, TestNoShapeInputModel) { } #if defined(__ANDROID__) -TEST(NnapiExecutionProviderTest, TestQDQModel) { - onnxruntime::Model model("nnapi_qdq_test_graph", false, DefaultLoggingManager().DefaultLogger()); +static void RunQDQModelTest(const GetQDQTestCaseFn& build_test_case, const char* test_description) { + onnxruntime::Model model(test_description, false, DefaultLoggingManager().DefaultLogger()); Graph& graph = model.MainGraph(); ModelTestBuilder helper(graph); - - auto build_test_case = BuildQDQConvTestCase({1, 1, 5, 5} /*input_shape*/, - {1, 1, 3, 3} /*weights_shape*/); build_test_case(helper); helper.SetGraphOutputs(); ASSERT_STATUS_OK(model.MainGraph().Resolve()); @@ -259,7 +256,23 @@ TEST(NnapiExecutionProviderTest, TestQDQModel) { std::make_unique(0), helper.feeds_); - // TODO: can add test load only verfication here later + // TODO: can add test load only verification here later +} + +TEST(NnapiExecutionProviderTest, TestQDQConv) { + RunQDQModelTest(BuildQDQConvTestCase( + {1, 1, 5, 5} /*input_shape*/, + {1, 1, 3, 3} /*weights_shape*/), + "nnapi_qdq_test_graph_conv"); +} + +TEST(NnapiExecutionProviderTest, TestQDQResize) { + RunQDQModelTest(BuildQDQResizeTestCase({1, 3, 64, 64} /* input_shape */, + {1, 3, 32, 32} /* sizes_data */), + "nnapi_qdq_test_graph_conv"); } #endif // defined(__ANDROID__) diff --git a/onnxruntime/test/util/test_utils.cc b/onnxruntime/test/util/test_utils.cc index 476fed4282..220b1342f1 100644 --- a/onnxruntime/test/util/test_utils.cc +++ b/onnxruntime/test/util/test_utils.cc @@ -35,6 +35,10 @@ static void VerifyOutputs(const std::vector& output_names, EXPECT_THAT(ltensor.DataAsSpan(), ::testing::ContainerEq(rtensor.DataAsSpan())) << " mismatch for " << output_names[i]; break; + case ONNX_NAMESPACE::TensorProto_DataType_UINT8: + EXPECT_THAT(ltensor.DataAsSpan(), ::testing::ContainerEq(rtensor.DataAsSpan())) + << " mismatch for " << output_names[i]; + break; case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: { constexpr float abs_err = 1e-5f;