From 3cca32beec53d2b176a5b2846e36a74eb40a8bd9 Mon Sep 17 00:00:00 2001 From: Aung T Naing Date: Wed, 31 May 2023 10:12:35 -0700 Subject: [PATCH] [QNN EP] exapand convolution test coverage. (#15975) ### Description Convolution with Padding and Convolution with large inputs,outputs. ### Motivation and Context This is mainly to check the CPU vs QNN EP output mismatch for models. ./onnxruntime_test_all --gtest_filter=*.TestQDQConvU8U8S32* Failed tests with mismatch. [ FAILED ] 2 tests, listed below: [ FAILED ] QnnHTPBackendTests.TestQDQConvU8U8S32_large_input1_padding_bias_initializer [ FAILED ] QnnHTPBackendTests.TestQDQConvU8U8S32_large_input2_bias_initializer ./onnxruntime_test_all --gtest_filter=*.TestCPUConvf32_* [ FAILED ] QnnCPUBackendTests.TestCPUConvf32_large_input1_pad_bias_initializer --- onnxruntime/test/providers/qnn/conv_test.cc | 84 ++++++++++++++++++--- 1 file changed, 72 insertions(+), 12 deletions(-) diff --git a/onnxruntime/test/providers/qnn/conv_test.cc b/onnxruntime/test/providers/qnn/conv_test.cc index e3b41cbabf..a88b384bcf 100644 --- a/onnxruntime/test/providers/qnn/conv_test.cc +++ b/onnxruntime/test/providers/qnn/conv_test.cc @@ -16,8 +16,12 @@ namespace test { // Creates a graph with a single Conv operator. Used for testing CPU backend. static GetTestModelFn BuildConvTestCase(const std::vector& input_shape, const std::vector& weights_shape, - bool is_bias_initializer) { - return [input_shape, weights_shape, is_bias_initializer](ModelTestBuilder& builder) { + bool is_bias_initializer, + const std::vector& strides, + const std::vector& pads, + const std::vector& dilations, + const std::string& auto_pad = "NOTSET") { + return [input_shape, weights_shape, is_bias_initializer, strides, pads, dilations, auto_pad](ModelTestBuilder& builder) { auto* input = builder.MakeInput(input_shape, 0.0f, 10.0f); auto* output = builder.MakeOutput(); auto* weights = builder.MakeInitializer(weights_shape, 0.0f, 1.0f); @@ -30,7 +34,18 @@ static GetTestModelFn BuildConvTestCase(const std::vector& input_shape, bias = builder.MakeInput({weights_shape[0]}, -1.0f, 1.0f); } - builder.AddNode("Conv", {input, weights, bias}, {output}); + Node& convNode = builder.AddNode("Conv", {input, weights, bias}, {output}); + convNode.AddAttribute("auto_pad", auto_pad); + + if (!pads.empty() && auto_pad == "NOTSET") { + convNode.AddAttribute("pads", pads); + } + if (!strides.empty()) { + convNode.AddAttribute("strides", strides); + } + if (!dilations.empty()) { + convNode.AddAttribute("dilations", dilations); + } }; } @@ -39,6 +54,10 @@ static GetTestModelFn BuildConvTestCase(const std::vector& input_shape, static void RunCPUConvOpTest(const std::vector& input_shape, const std::vector& weights_shape, bool is_bias_initializer, + const std::vector& strides, + const std::vector& pads, + const std::vector& dilations, + const std::string& auto_pad, ExpectedEPNodeAssignment expected_ep_assignment, const char* test_description, int opset = 13) { ProviderOptions provider_options; @@ -50,7 +69,7 @@ static void RunCPUConvOpTest(const std::vector& input_shape, #endif constexpr int expected_nodes_in_partition = 1; - RunQnnModelTest(BuildConvTestCase(input_shape, weights_shape, is_bias_initializer), + RunQnnModelTest(BuildConvTestCase(input_shape, weights_shape, is_bias_initializer, strides, pads, dilations, auto_pad), provider_options, opset, expected_ep_assignment, @@ -62,8 +81,12 @@ static void RunCPUConvOpTest(const std::vector& input_shape, template GetTestModelFn BuildQDQConvTestCase(const std::vector& input_shape, const std::vector& weights_shape, - bool is_bias_initializer = true) { - return [input_shape, weights_shape, is_bias_initializer](ModelTestBuilder& builder) { + bool is_bias_initializer, + const std::vector& strides, + const std::vector& pads, + const std::vector& dilations, + const std::string& auto_pad = "NOTSET") { + return [input_shape, weights_shape, is_bias_initializer, strides, pads, dilations, auto_pad](ModelTestBuilder& builder) { auto* input_arg = builder.MakeInput(input_shape, -1.f, 1.f); auto* output_arg = builder.MakeOutput(); @@ -101,7 +124,7 @@ GetTestModelFn BuildQDQConvTestCase(const std::vector& input_shape, auto* conv_output = builder.MakeIntermediate(); auto* dq_output = AddQDQNodePair(builder, input_arg, .04f, (input_min_value + input_max_value) / 2 + 1); - builder.AddNode("Conv", {dq_output, dq_w_output, dq_bias_output}, {conv_output}); + Node& convNode = builder.AddNode("Conv", {dq_output, dq_w_output, dq_bias_output}, {conv_output}); auto* q_output = builder.MakeIntermediate(); builder.AddQuantizeLinearNode(conv_output, .039f, @@ -111,6 +134,17 @@ GetTestModelFn BuildQDQConvTestCase(const std::vector& input_shape, builder.AddDequantizeLinearNode(q_output, .039f, (OutputLimits::min() + OutputLimits::max()) / 2 + 1, output_arg); + convNode.AddAttribute("auto_pad", auto_pad); + + if (!pads.empty() && auto_pad == "NOTSET") { + convNode.AddAttribute("pads", pads); + } + if (!strides.empty()) { + convNode.AddAttribute("strides", strides); + } + if (!dilations.empty()) { + convNode.AddAttribute("dilations", dilations); + } }; } @@ -120,6 +154,10 @@ template & input_shape, const std::vector& weights_shape, bool is_bias_initializer, + const std::vector& strides, + const std::vector& pads, + const std::vector& dilations, + const std::string& auto_pad, ExpectedEPNodeAssignment expected_ep_assignment, const char* test_description, int opset = 13) { ProviderOptions provider_options; @@ -132,7 +170,8 @@ static void RunHTPConvOpTest(const std::vector& input_shape, constexpr int expected_nodes_in_partition = 1; RunQnnModelTest(BuildQDQConvTestCase(input_shape, weights_shape, - is_bias_initializer), + is_bias_initializer, + strides, pads, dilations, auto_pad), provider_options, opset, expected_ep_assignment, @@ -146,13 +185,23 @@ static void RunHTPConvOpTest(const std::vector& input_shape, // TODO: Enable this test when QNN CPU backend (QNN sdk 2.10.0) fixes bug that causes graph finalization to // throw a segfault when the bias is a non-static input. TEST_F(QnnCPUBackendTests, DISABLED_TestCPUConvf32_bias_input) { - RunCPUConvOpTest({1, 1, 3, 3}, {2, 1, 2, 2}, false, ExpectedEPNodeAssignment::All, "TestCPUConvf32_bias_input"); + RunCPUConvOpTest({1, 1, 3, 3}, {2, 1, 2, 2}, false, {1, 1}, {0, 0, 0, 0}, {1, 1}, "NOTSET", ExpectedEPNodeAssignment::All, "TestCPUConvf32_bias_input"); } // Check that QNN compiles DQ -> Conv -> Q as a single unit. // Tests bias as an initializer. TEST_F(QnnCPUBackendTests, TestCPUConvf32_bias_initializer) { - RunCPUConvOpTest({1, 1, 3, 3}, {2, 1, 2, 2}, true, ExpectedEPNodeAssignment::All, "TestCPUConvf32_bias_initializer"); + RunCPUConvOpTest({1, 1, 3, 3}, {2, 1, 2, 2}, true, {1, 1}, {0, 0, 0, 0}, {1, 1}, "NOTSET", ExpectedEPNodeAssignment::All, "TestCPUConvf32_bias_initializer"); +} + +// large input,output, pads +// TODO: re-enable tests once Padding issues are resolved +TEST_F(QnnCPUBackendTests, DISABLED_TestCPUConvf32_large_input1_pad_bias_initializer) { + RunCPUConvOpTest({1, 3, 60, 452}, {16, 3, 3, 3}, true, {1, 1}, {1, 1, 1, 1}, {1, 1}, "NOTSET", ExpectedEPNodeAssignment::All, "TestCPUConvf32_large_input1_pad_bias_initializer"); +} + +TEST_F(QnnCPUBackendTests, TestCPUConvf32_large_input2_nopad_bias_initializer) { + RunCPUConvOpTest({1, 32, 16, 113}, {16, 32, 1, 1}, true, {1, 1}, {0, 0, 0, 0}, {1, 1}, "NOTSET", ExpectedEPNodeAssignment::All, "TestCPUConvf32_large_input2_nopad_bias_initializer"); } #if defined(__aarch64__) || defined(_M_ARM64) || defined(__linux__) @@ -160,16 +209,27 @@ TEST_F(QnnCPUBackendTests, TestCPUConvf32_bias_initializer) { // Check that QNN compiles DQ -> Conv -> Q as a single unit. // Tests bias as an input. TEST_F(QnnHTPBackendTests, TestQDQConvU8U8S32_bias_input) { - RunHTPConvOpTest({1, 1, 5, 5}, {1, 1, 3, 3}, false, ExpectedEPNodeAssignment::All, + RunHTPConvOpTest({1, 1, 5, 5}, {1, 1, 3, 3}, false, {1, 1}, {0, 0, 0, 0}, {1, 1}, "NOTSET", ExpectedEPNodeAssignment::All, "TestQDQConvU8U8S32_bias_input"); } // Check that QNN compiles DQ -> Conv -> Q as a single unit. // Tests bias as an initializer. TEST_F(QnnHTPBackendTests, TestQDQConvU8U8S32_bias_initializer) { - RunHTPConvOpTest({1, 1, 5, 5}, {1, 1, 3, 3}, true, ExpectedEPNodeAssignment::All, + RunHTPConvOpTest({1, 1, 5, 5}, {1, 1, 3, 3}, true, {1, 1}, {0, 0, 0, 0}, {1, 1}, "NOTSET", ExpectedEPNodeAssignment::All, "TestQDQConvU8U8S32_bias_initializer"); } + +// TODO: re-enable tests once HTP issues are resolved +TEST_F(QnnHTPBackendTests, DISABLED_TestQDQConvU8U8S32_large_input1_padding_bias_initializer) { + RunHTPConvOpTest({1, 3, 60, 452}, {16, 3, 3, 3}, true, {1, 1}, {1, 1, 1, 1}, {1, 1}, "NOTSET", ExpectedEPNodeAssignment::All, + "TestQDQConvU8U8S32_large_input1_padding_bias_initializer"); +} + +TEST_F(QnnHTPBackendTests, DISABLED_TestQDQConvU8U8S32_large_input2_bias_initializer) { + RunHTPConvOpTest({1, 128, 8, 56}, {32, 128, 1, 1}, true, {1, 1}, {0, 0, 0, 0}, {1, 1}, "NOTSET", ExpectedEPNodeAssignment::All, + "TestQDQConvU8U8S32_large_input2_bias_initializer"); +} #endif // defined(__aarch64__) || defined(_M_ARM64) || defined(__linux__) } // namespace test