saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-06-27 03:11:28 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add additional NNAPI QDQ test cases for expected failure path (#10769)

Browse source 
* add more qdq softmax test case

* add rest of additional nnapi qdq test case

* comment out

* update

* update concat test case

* extract zp/scale point retrieval out

* fix

* fix helper function for scale/zp and address some pr comments

* use enum class expectedepnodeassignments

* fix default value for ExpectedEPNodeAssignment

* update

* refine some comments

* refine some minor comments

* refine pr comments

* update

Co-authored-by: rachguo <rachguo@rachguos-Mini.attlocal.net>
This commit is contained in:
Rachel Guo 2022-03-14 19:22:22 -07:00 committed by GitHub
parent e53422c6d0
commit 7e9dfe627a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 116 additions and 32 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

31
onnxruntime/test/optimizer/qdq_test_utils.cc
View file

@ -98,5 +98,36 @@ GetQDQTestCaseFn BuildQDQConcatTestCase(const std::vector<std::vector<int64_t>>&
};
}
GetQDQTestCaseFn BuildQDQConcatTestCaseUnsupportedInputScaleZp() {
return [](ModelTestBuilder& builder) {
const std::vector<std::vector<int64_t>> input_shapes = {
{1, 6, 36},
{1, 6, 8},
{1, 6, 2},
};
int64_t axis = 2;
std::vector<NodeArg*> input_args;
std::vector<NodeArg*> q_input_args;
// set unmatched input scales/zp for test purpose
input_args.push_back(builder.MakeInput<float>(input_shapes[0], -1.f, 1.f));
q_input_args.push_back(AddQDQNodePair<uint8_t>(builder, input_args.back(), 0.05f, 128));
input_args.push_back(builder.MakeInput<float>(input_shapes[1], -1.f, 1.f));
q_input_args.push_back(AddQDQNodePair<uint8_t>(builder, input_args.back(), 0.04f, 127));
input_args.push_back(builder.MakeInput<float>(input_shapes[2], -1.f, 1.f));
q_input_args.push_back(AddQDQNodePair<uint8_t>(builder, input_args.back(), 0.03f, 126));
auto* concat_output = builder.MakeIntermediate();
Node& concat_node = builder.AddNode("Concat", q_input_args, {concat_output});
concat_node.AddAttribute("axis", axis);
auto* q_concat_output = builder.MakeIntermediate();
builder.AddQuantizeLinearNode<uint8_t>(concat_output, 0.05f, 128, q_concat_output);
auto* output_arg = builder.MakeOutput();
builder.AddDequantizeLinearNode<uint8_t>(q_concat_output, 0.05f, 128, output_arg);
};
}
} // namespace test
} // namespace onnxruntime

9
onnxruntime/test/optimizer/qdq_test_utils.h
View file

@ -256,8 +256,9 @@ GetQDQTestCaseFn BuildQDQTransposeTestCase(
}
template <typename InputType, typename OutputType>
GetQDQTestCaseFn BuildQDQSoftMaxTestCase(const std::vector<int64_t>& input_shape, const int64_t& axis = -1) {
return [input_shape, axis](ModelTestBuilder& builder) {
GetQDQTestCaseFn BuildQDQSoftMaxTestCase(const std::vector<int64_t>& input_shape, const int64_t& axis,
float output_scales, OutputType output_zero_point) {
return [input_shape, axis, output_scales, output_zero_point](ModelTestBuilder& builder) {
auto* input_arg = builder.MakeInput<InputType>(input_shape,
std::numeric_limits<InputType>::min(),
std::numeric_limits<InputType>::max());
@ -275,7 +276,7 @@ GetQDQTestCaseFn BuildQDQSoftMaxTestCase(const std::vector<int64_t>& input_shape
softmax_node.AddAttribute("axis", axis);
// add Q
builder.AddQuantizeLinearNode<OutputType>(softmax_output, 1.f / 256, 0, output_arg);
builder.AddQuantizeLinearNode<OutputType>(softmax_output, output_scales, output_zero_point, output_arg);
};
}
@ -288,5 +289,7 @@ GetQDQTestCaseFn BuildQDQConcatTestCase(const std::vector<std::vector<int64_t>>&
bool has_input_int8 = false,
bool has_output_int8 = false);
GetQDQTestCaseFn BuildQDQConcatTestCaseUnsupportedInputScaleZp();
} // namespace test
} // namespace onnxruntime

91
onnxruntime/test/providers/nnapi/nnapi_basic_test.cc
View file

@ -4,7 +4,10 @@
#if !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD)
#include "core/common/logging/logging.h"
#include "core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.h"
#include "core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksTypes.h"
#include "core/providers/nnapi/nnapi_builtin/nnapi_lib/nnapi_implementation.h"
#include "core/session/inference_session.h"
#include "core/framework/tensorprotoutils.h"
#include "test/common/tensor_op_test_utils.h"
#include "test/framework/test_utils.h"
#include "test/util/include/asserts.h"
@ -271,9 +274,10 @@ TEST(NnapiExecutionProviderTest, TestNoShapeInputModel) {
<< "No node should be taken by the NNAPI EP";
}
static void RunQDQModelTest(const GetQDQTestCaseFn& build_test_case,
const char* test_description,
const EPVerificationParams& params = EPVerificationParams()) {
static void RunQDQModelTest(
const GetQDQTestCaseFn& build_test_case,
const char* test_description,
const EPVerificationParams& params = EPVerificationParams()) {
onnxruntime::Model model(test_description, false, DefaultLoggingManager().DefaultLogger());
Graph& graph = model.MainGraph();
ModelTestBuilder helper(graph);
@ -290,15 +294,22 @@ static void RunQDQModelTest(const GetQDQTestCaseFn& build_test_case,
std::make_unique<NnapiExecutionProvider>(0),
helper.feeds_, params);
#else
ORT_UNUSED_PARAMETER(params);
// test load only
SessionOptions so;
InferenceSessionWrapper session_object{so, GetEnvironment()};
ASSERT_STATUS_OK(session_object.RegisterExecutionProvider(std::make_unique<NnapiExecutionProvider>(0)));
ASSERT_STATUS_OK(session_object.Load(model_data.data(), static_cast<int>(model_data.size())));
ASSERT_STATUS_OK(session_object.Initialize());
ASSERT_GT(CountAssignedNodes(session_object.GetGraph(), kNnapiExecutionProvider), 0)
<< "Some nodes should have been taken by the NNAPI EP";
if (params.ep_node_assignment == ExpectedEPNodeAssignment::None) {
ASSERT_EQ(CountAssignedNodes(session_object.GetGraph(), kNnapiExecutionProvider), 0)
<< "No node should have been taken by the NNAPI EP";
} else if (params.ep_node_assignment == ExpectedEPNodeAssignment::All) {
ASSERT_EQ(CountAssignedNodes(session_object.GetGraph(), kNnapiExecutionProvider), session_object.GetGraph().NumberOfNodes())
<< "All nodes should have been taken by the NNAPI EP";
} else {
ASSERT_GT(CountAssignedNodes(session_object.GetGraph(), kNnapiExecutionProvider), 0)
<< "Some nodes should have been taken by the NNAPI EP";
}
#endif
}
@ -310,7 +321,7 @@ TEST(NnapiExecutionProviderTest, TestQDQConv) {
{1, 1, 5, 5} /* input_shape */,
{1, 1, 3, 3} /* weights_shape */),
"nnapi_qdq_test_graph_conv",
{true /* verify_entire_graph_use_ep */});
{ExpectedEPNodeAssignment::All});
}
TEST(NnapiExecutionProviderTest, TestQDQResize) {
@ -326,7 +337,14 @@ TEST(NnapiExecutionProviderTest, TestQDQResize) {
"linear" /* mode */,
"asymmetric" /* coordinate_transformation_mode */),
"nnapi_qdq_test_graph_resize",
{false /* verify_entire_graph_use_ep */});
{ExpectedEPNodeAssignment::Some});
}
TEST(NnapiExecutionProviderTest, TestQDQResize_UnsupportedDefaultSetting) {
RunQDQModelTest(BuildQDQResizeTestCase({1, 3, 64, 64} /* input_shape */,
{1, 3, 32, 32} /* sizes_data */),
"nnapi_qdq_test_graph_resize_unsupported",
{ExpectedEPNodeAssignment::None});
}
TEST(NnapiExecutionProviderTest, TestQDQAveragePool) {
@ -336,7 +354,7 @@ TEST(NnapiExecutionProviderTest, TestQDQAveragePool) {
{1, 3, 32, 32} /* input_shape */),
"nnapi_qdq_test_graph_averagepool",
{
true /* verify_entire_graph_use_ep */,
ExpectedEPNodeAssignment::All,
1e-2f /* fp32_abs_err */,
});
}
@ -348,7 +366,7 @@ TEST(NnapiExecutionProviderTest, TestQDQAdd) {
{1, 23, 13, 13} /* input_shape */,
"Add" /* op_type */),
"nnapi_qdq_test_graph_add",
{true /* verify_entire_graph_use_ep */});
{ExpectedEPNodeAssignment::All});
}
TEST(NnapiExecutionProviderTest, TestQDQMul) {
@ -360,8 +378,8 @@ TEST(NnapiExecutionProviderTest, TestQDQMul) {
"Mul" /* op_type */),
"nnapi_qdq_test_graph_mul",
{
true /* verify_entire_graph_use_ep */,
1e-2f /* fp32_abs_err */,
ExpectedEPNodeAssignment::All,
1e-2f /* fp32_abs_err */
});
}
@ -371,28 +389,36 @@ TEST(NnapiExecutionProviderTest, TestQDQTranspose) {
{1, 3, 32, 32} /* input_shape */,
{0, 3, 1, 2} /* perms */),
"nnapi_qdq_test_graph_transpose",
{
true /* verify_entire_graph_use_ep */
});
{ExpectedEPNodeAssignment::All});
}
TEST(NnapiExecutionProviderTest, TestQDQReshape) {
RunQDQModelTest(BuildQDQReshapeTestCase({1, 3, 64, 64} /* input_shape */,
{1, 64, 64, 3} /* reshape_shape */),
"nnapi_qdq_test_graph_reshape",
{
true /* verify_entire_graph_use_ep */
});
{ExpectedEPNodeAssignment::All});
}
TEST(NnapiExecutionProviderTest, TestQDQSoftMax) {
RunQDQModelTest(BuildQDQSoftMaxTestCase<uint8_t, uint8_t>(
{1, 32} /* input_shape */,
static_cast<int64_t>(1) /* axis */),
static_cast<int64_t>(1) /* axis */,
1.f / 256 /* output_scales */,
0 /* output_zp */),
"nnapi_qdq_test_graph_softmax",
{
true /* verify_entire_graph_use_ep */
});
{ExpectedEPNodeAssignment::All});
}
// This is to verify when Nnapi required scale and zero point are not satisfied
// the model can work as expected. (no nodes should be handled by Nnapi)
TEST(NnapiExecutionProviderTest, TestQDQSoftMax_UnsupportedOutputScaleAndZp) {
RunQDQModelTest(BuildQDQSoftMaxTestCase<uint8_t, uint8_t>(
{1, 32} /* input_shape */,
static_cast<int64_t>(1) /* axis */,
0.002f /* output_scales */,
1 /* output_zp */),
"nnapi_qdq_test_graph_softmax_unsupported",
{ExpectedEPNodeAssignment::None});
}
TEST(NnapiExecutionProviderTest, TestQDQConcat) {
@ -403,11 +429,26 @@ TEST(NnapiExecutionProviderTest, TestQDQConcat) {
{1, 6, 2},
} /* input_shapes */,
2 /* axis */),
"nnapi_qdq_test_graph_concat", {
true /* verify_entire_graph_use_ep */
});
"nnapi_qdq_test_graph_concat",
{ExpectedEPNodeAssignment::All});
}
#if defined(__ANDROID__)
TEST(NnapiExecutionProviderTest, TestQDQConcat_UnsupportedInputScalesAndZp) {
// This is to verify all the inputs have the same scale and zp as input 0 for API 28-
// Currently, this test can only be run locally with a android emulator with API < 29
// See https://developer.android.com/studio/run/emulator-commandline for some info on
// starting a testing android emulator in command line. (Run an android build with emulator started)
// TODO: consider to configure this and enable it to run in Android CI.
const auto* nnapi = NnApiImplementation();
if (nnapi->nnapi_runtime_feature_level < ANEURALNETWORKS_FEATURE_LEVEL_3) {
RunQDQModelTest(BuildQDQConcatTestCaseUnsupportedInputScaleZp(),
"nnapi_qdq_test_graph_concat_unsupported",
{ExpectedEPNodeAssignment::None});
}
}
#endif
#endif // !(ORT_MINIMAL_BUILD)
TEST(NnapiExecutionProviderTest, NNAPIFlagsTest) {

12
onnxruntime/test/util/include/test_utils.h
View file

@ -15,11 +15,17 @@ class Graph;
namespace test {
// If set to All: verify the entire graph is taken by ep
// If set to Some: verify that at least one node is assigned to ep
// If set to None: verify that no nodes is assigned to ep (typically for an expected failure path test case)
enum class ExpectedEPNodeAssignment { None,
Some,
All, };
// struct to hold some verification params for RunAndVerifyOutputsWithEP
struct EPVerificationParams {
// Verify the entire graph is taken by the EP
// if this is set to false, then will verify that at least one node is assigned to 'execution_provider'
bool verify_entire_graph_use_ep{false};
ExpectedEPNodeAssignment ep_node_assignment = ExpectedEPNodeAssignment::Some;
// Some EP may use different rounding than ORT CPU EP, which may cause a bigger abs error than
// the default of 1e-5f, especially for scenarios such as [Q -> Quantized op -> DQ]

5
onnxruntime/test/util/test_utils.cc
View file

@ -123,9 +123,12 @@ void RunAndVerifyOutputsWithEP(const std::string& model_data, const char* log_id
// make sure that some nodes are assigned to the EP, otherwise this test is pointless...
const auto& graph2 = session_object2.GetGraph();
auto ep_nodes = CountAssignedNodes(graph2, provider_type);
if (params.verify_entire_graph_use_ep) {
if (params.ep_node_assignment == ExpectedEPNodeAssignment::All) {
// Verify the entire graph is assigned to the EP
ASSERT_EQ(ep_nodes, graph2.NumberOfNodes()) << "Not all nodes were assigned to " << provider_type;
} else if (params.ep_node_assignment == ExpectedEPNodeAssignment::None) {
// Check if expected failure path is correctly handled by ep. (only used in NNAPI EP QDQ model test case for now)
ASSERT_EQ(ep_nodes, 0) << "No nodes are supposed to be assigned to " << provider_type;
} else {
ASSERT_GT(ep_nodes, 0) << "No nodes were assigned to " << provider_type;
}