diff --git a/onnxruntime/test/optimizer/qdq_test_utils.cc b/onnxruntime/test/optimizer/qdq_test_utils.cc index 607049917f..9e3318dc0f 100644 --- a/onnxruntime/test/optimizer/qdq_test_utils.cc +++ b/onnxruntime/test/optimizer/qdq_test_utils.cc @@ -98,5 +98,36 @@ GetQDQTestCaseFn BuildQDQConcatTestCase(const std::vector>& }; } +GetQDQTestCaseFn BuildQDQConcatTestCaseUnsupportedInputScaleZp() { + return [](ModelTestBuilder& builder) { + const std::vector> input_shapes = { + {1, 6, 36}, + {1, 6, 8}, + {1, 6, 2}, + }; + int64_t axis = 2; + + std::vector input_args; + std::vector q_input_args; + + // set unmatched input scales/zp for test purpose + input_args.push_back(builder.MakeInput(input_shapes[0], -1.f, 1.f)); + q_input_args.push_back(AddQDQNodePair(builder, input_args.back(), 0.05f, 128)); + input_args.push_back(builder.MakeInput(input_shapes[1], -1.f, 1.f)); + q_input_args.push_back(AddQDQNodePair(builder, input_args.back(), 0.04f, 127)); + input_args.push_back(builder.MakeInput(input_shapes[2], -1.f, 1.f)); + q_input_args.push_back(AddQDQNodePair(builder, input_args.back(), 0.03f, 126)); + + auto* concat_output = builder.MakeIntermediate(); + Node& concat_node = builder.AddNode("Concat", q_input_args, {concat_output}); + concat_node.AddAttribute("axis", axis); + + auto* q_concat_output = builder.MakeIntermediate(); + builder.AddQuantizeLinearNode(concat_output, 0.05f, 128, q_concat_output); + auto* output_arg = builder.MakeOutput(); + builder.AddDequantizeLinearNode(q_concat_output, 0.05f, 128, output_arg); + }; +} + } // namespace test } // namespace onnxruntime \ No newline at end of file diff --git a/onnxruntime/test/optimizer/qdq_test_utils.h b/onnxruntime/test/optimizer/qdq_test_utils.h index affa5baf9d..2ee6abcb54 100644 --- a/onnxruntime/test/optimizer/qdq_test_utils.h +++ b/onnxruntime/test/optimizer/qdq_test_utils.h @@ -256,8 +256,9 @@ GetQDQTestCaseFn BuildQDQTransposeTestCase( } template -GetQDQTestCaseFn BuildQDQSoftMaxTestCase(const std::vector& input_shape, const int64_t& axis = -1) { - return [input_shape, axis](ModelTestBuilder& builder) { +GetQDQTestCaseFn BuildQDQSoftMaxTestCase(const std::vector& input_shape, const int64_t& axis, + float output_scales, OutputType output_zero_point) { + return [input_shape, axis, output_scales, output_zero_point](ModelTestBuilder& builder) { auto* input_arg = builder.MakeInput(input_shape, std::numeric_limits::min(), std::numeric_limits::max()); @@ -275,7 +276,7 @@ GetQDQTestCaseFn BuildQDQSoftMaxTestCase(const std::vector& input_shape softmax_node.AddAttribute("axis", axis); // add Q - builder.AddQuantizeLinearNode(softmax_output, 1.f / 256, 0, output_arg); + builder.AddQuantizeLinearNode(softmax_output, output_scales, output_zero_point, output_arg); }; } @@ -288,5 +289,7 @@ GetQDQTestCaseFn BuildQDQConcatTestCase(const std::vector>& bool has_input_int8 = false, bool has_output_int8 = false); +GetQDQTestCaseFn BuildQDQConcatTestCaseUnsupportedInputScaleZp(); + } // namespace test } // namespace onnxruntime diff --git a/onnxruntime/test/providers/nnapi/nnapi_basic_test.cc b/onnxruntime/test/providers/nnapi/nnapi_basic_test.cc index 5bdac54702..dd485636a0 100644 --- a/onnxruntime/test/providers/nnapi/nnapi_basic_test.cc +++ b/onnxruntime/test/providers/nnapi/nnapi_basic_test.cc @@ -4,7 +4,10 @@ #if !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD) #include "core/common/logging/logging.h" #include "core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.h" +#include "core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksTypes.h" +#include "core/providers/nnapi/nnapi_builtin/nnapi_lib/nnapi_implementation.h" #include "core/session/inference_session.h" +#include "core/framework/tensorprotoutils.h" #include "test/common/tensor_op_test_utils.h" #include "test/framework/test_utils.h" #include "test/util/include/asserts.h" @@ -271,9 +274,10 @@ TEST(NnapiExecutionProviderTest, TestNoShapeInputModel) { << "No node should be taken by the NNAPI EP"; } -static void RunQDQModelTest(const GetQDQTestCaseFn& build_test_case, - const char* test_description, - const EPVerificationParams& params = EPVerificationParams()) { +static void RunQDQModelTest( + const GetQDQTestCaseFn& build_test_case, + const char* test_description, + const EPVerificationParams& params = EPVerificationParams()) { onnxruntime::Model model(test_description, false, DefaultLoggingManager().DefaultLogger()); Graph& graph = model.MainGraph(); ModelTestBuilder helper(graph); @@ -290,15 +294,22 @@ static void RunQDQModelTest(const GetQDQTestCaseFn& build_test_case, std::make_unique(0), helper.feeds_, params); #else - ORT_UNUSED_PARAMETER(params); // test load only SessionOptions so; InferenceSessionWrapper session_object{so, GetEnvironment()}; ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(std::make_unique(0))); ASSERT_STATUS_OK(session_object.Load(model_data.data(), static_cast(model_data.size()))); ASSERT_STATUS_OK(session_object.Initialize()); - ASSERT_GT(CountAssignedNodes(session_object.GetGraph(), kNnapiExecutionProvider), 0) - << "Some nodes should have been taken by the NNAPI EP"; + if (params.ep_node_assignment == ExpectedEPNodeAssignment::None) { + ASSERT_EQ(CountAssignedNodes(session_object.GetGraph(), kNnapiExecutionProvider), 0) + << "No node should have been taken by the NNAPI EP"; + } else if (params.ep_node_assignment == ExpectedEPNodeAssignment::All) { + ASSERT_EQ(CountAssignedNodes(session_object.GetGraph(), kNnapiExecutionProvider), session_object.GetGraph().NumberOfNodes()) + << "All nodes should have been taken by the NNAPI EP"; + } else { + ASSERT_GT(CountAssignedNodes(session_object.GetGraph(), kNnapiExecutionProvider), 0) + << "Some nodes should have been taken by the NNAPI EP"; + } #endif } @@ -310,7 +321,7 @@ TEST(NnapiExecutionProviderTest, TestQDQConv) { {1, 1, 5, 5} /* input_shape */, {1, 1, 3, 3} /* weights_shape */), "nnapi_qdq_test_graph_conv", - {true /* verify_entire_graph_use_ep */}); + {ExpectedEPNodeAssignment::All}); } TEST(NnapiExecutionProviderTest, TestQDQResize) { @@ -326,7 +337,14 @@ TEST(NnapiExecutionProviderTest, TestQDQResize) { "linear" /* mode */, "asymmetric" /* coordinate_transformation_mode */), "nnapi_qdq_test_graph_resize", - {false /* verify_entire_graph_use_ep */}); + {ExpectedEPNodeAssignment::Some}); +} + +TEST(NnapiExecutionProviderTest, TestQDQResize_UnsupportedDefaultSetting) { + RunQDQModelTest(BuildQDQResizeTestCase({1, 3, 64, 64} /* input_shape */, + {1, 3, 32, 32} /* sizes_data */), + "nnapi_qdq_test_graph_resize_unsupported", + {ExpectedEPNodeAssignment::None}); } TEST(NnapiExecutionProviderTest, TestQDQAveragePool) { @@ -336,7 +354,7 @@ TEST(NnapiExecutionProviderTest, TestQDQAveragePool) { {1, 3, 32, 32} /* input_shape */), "nnapi_qdq_test_graph_averagepool", { - true /* verify_entire_graph_use_ep */, + ExpectedEPNodeAssignment::All, 1e-2f /* fp32_abs_err */, }); } @@ -348,7 +366,7 @@ TEST(NnapiExecutionProviderTest, TestQDQAdd) { {1, 23, 13, 13} /* input_shape */, "Add" /* op_type */), "nnapi_qdq_test_graph_add", - {true /* verify_entire_graph_use_ep */}); + {ExpectedEPNodeAssignment::All}); } TEST(NnapiExecutionProviderTest, TestQDQMul) { @@ -360,8 +378,8 @@ TEST(NnapiExecutionProviderTest, TestQDQMul) { "Mul" /* op_type */), "nnapi_qdq_test_graph_mul", { - true /* verify_entire_graph_use_ep */, - 1e-2f /* fp32_abs_err */, + ExpectedEPNodeAssignment::All, + 1e-2f /* fp32_abs_err */ }); } @@ -371,28 +389,36 @@ TEST(NnapiExecutionProviderTest, TestQDQTranspose) { {1, 3, 32, 32} /* input_shape */, {0, 3, 1, 2} /* perms */), "nnapi_qdq_test_graph_transpose", - { - true /* verify_entire_graph_use_ep */ - }); + {ExpectedEPNodeAssignment::All}); } TEST(NnapiExecutionProviderTest, TestQDQReshape) { RunQDQModelTest(BuildQDQReshapeTestCase({1, 3, 64, 64} /* input_shape */, {1, 64, 64, 3} /* reshape_shape */), "nnapi_qdq_test_graph_reshape", - { - true /* verify_entire_graph_use_ep */ - }); + {ExpectedEPNodeAssignment::All}); } TEST(NnapiExecutionProviderTest, TestQDQSoftMax) { RunQDQModelTest(BuildQDQSoftMaxTestCase( {1, 32} /* input_shape */, - static_cast(1) /* axis */), + static_cast(1) /* axis */, + 1.f / 256 /* output_scales */, + 0 /* output_zp */), "nnapi_qdq_test_graph_softmax", - { - true /* verify_entire_graph_use_ep */ - }); + {ExpectedEPNodeAssignment::All}); +} + +// This is to verify when Nnapi required scale and zero point are not satisfied +// the model can work as expected. (no nodes should be handled by Nnapi) +TEST(NnapiExecutionProviderTest, TestQDQSoftMax_UnsupportedOutputScaleAndZp) { + RunQDQModelTest(BuildQDQSoftMaxTestCase( + {1, 32} /* input_shape */, + static_cast(1) /* axis */, + 0.002f /* output_scales */, + 1 /* output_zp */), + "nnapi_qdq_test_graph_softmax_unsupported", + {ExpectedEPNodeAssignment::None}); } TEST(NnapiExecutionProviderTest, TestQDQConcat) { @@ -403,11 +429,26 @@ TEST(NnapiExecutionProviderTest, TestQDQConcat) { {1, 6, 2}, } /* input_shapes */, 2 /* axis */), - "nnapi_qdq_test_graph_concat", { - true /* verify_entire_graph_use_ep */ - }); + "nnapi_qdq_test_graph_concat", + {ExpectedEPNodeAssignment::All}); } +#if defined(__ANDROID__) +TEST(NnapiExecutionProviderTest, TestQDQConcat_UnsupportedInputScalesAndZp) { + // This is to verify all the inputs have the same scale and zp as input 0 for API 28- + // Currently, this test can only be run locally with a android emulator with API < 29 + // See https://developer.android.com/studio/run/emulator-commandline for some info on + // starting a testing android emulator in command line. (Run an android build with emulator started) + // TODO: consider to configure this and enable it to run in Android CI. + const auto* nnapi = NnApiImplementation(); + if (nnapi->nnapi_runtime_feature_level < ANEURALNETWORKS_FEATURE_LEVEL_3) { + RunQDQModelTest(BuildQDQConcatTestCaseUnsupportedInputScaleZp(), + "nnapi_qdq_test_graph_concat_unsupported", + {ExpectedEPNodeAssignment::None}); + } +} +#endif + #endif // !(ORT_MINIMAL_BUILD) TEST(NnapiExecutionProviderTest, NNAPIFlagsTest) { diff --git a/onnxruntime/test/util/include/test_utils.h b/onnxruntime/test/util/include/test_utils.h index 50859d826f..e1e2e8a542 100644 --- a/onnxruntime/test/util/include/test_utils.h +++ b/onnxruntime/test/util/include/test_utils.h @@ -15,11 +15,17 @@ class Graph; namespace test { +// If set to All: verify the entire graph is taken by ep +// If set to Some: verify that at least one node is assigned to ep +// If set to None: verify that no nodes is assigned to ep (typically for an expected failure path test case) +enum class ExpectedEPNodeAssignment { None, + Some, + All, }; + // struct to hold some verification params for RunAndVerifyOutputsWithEP struct EPVerificationParams { - // Verify the entire graph is taken by the EP - // if this is set to false, then will verify that at least one node is assigned to 'execution_provider' - bool verify_entire_graph_use_ep{false}; + + ExpectedEPNodeAssignment ep_node_assignment = ExpectedEPNodeAssignment::Some; // Some EP may use different rounding than ORT CPU EP, which may cause a bigger abs error than // the default of 1e-5f, especially for scenarios such as [Q -> Quantized op -> DQ] diff --git a/onnxruntime/test/util/test_utils.cc b/onnxruntime/test/util/test_utils.cc index b069b08810..ac79965812 100644 --- a/onnxruntime/test/util/test_utils.cc +++ b/onnxruntime/test/util/test_utils.cc @@ -123,9 +123,12 @@ void RunAndVerifyOutputsWithEP(const std::string& model_data, const char* log_id // make sure that some nodes are assigned to the EP, otherwise this test is pointless... const auto& graph2 = session_object2.GetGraph(); auto ep_nodes = CountAssignedNodes(graph2, provider_type); - if (params.verify_entire_graph_use_ep) { + if (params.ep_node_assignment == ExpectedEPNodeAssignment::All) { // Verify the entire graph is assigned to the EP ASSERT_EQ(ep_nodes, graph2.NumberOfNodes()) << "Not all nodes were assigned to " << provider_type; + } else if (params.ep_node_assignment == ExpectedEPNodeAssignment::None) { + // Check if expected failure path is correctly handled by ep. (only used in NNAPI EP QDQ model test case for now) + ASSERT_EQ(ep_nodes, 0) << "No nodes are supposed to be assigned to " << provider_type; } else { ASSERT_GT(ep_nodes, 0) << "No nodes were assigned to " << provider_type; }