From 9b4c54bcef2a6f2d590f0db6ceacfbac7d524241 Mon Sep 17 00:00:00 2001 From: gwang-msft <62914304+gwang-msft@users.noreply.github.com> Date: Fri, 10 Jul 2020 13:34:44 -0700 Subject: [PATCH] Enable onnxruntime_test_all for NNAPI EP (#4476) --- .../nnapi_builtin/builders/op_builder.cc | 32 +++++++------ .../nnapi_builtin/nnapi_execution_provider.cc | 13 +++-- .../cpu/activation/activation_op_test.h | 47 +++++++++++-------- .../cpu/math/element_wise_ops_test.cc | 9 ++-- .../test/providers/cpu/math/softmax_test.cc | 17 ++++--- .../providers/cpu/tensor/concat_op_test.cc | 8 +++- .../test/providers/provider_test_utils.cc | 4 +- 7 files changed, 76 insertions(+), 54 deletions(-) diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc index 10864be333..695537efdd 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc @@ -1453,9 +1453,7 @@ void ConcatOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N inputs.reserve(node_input_size); if (all_input_have_same_layout) { // if all the inputs are of same layout, output will be the same layout - if (model_builder.IsOperandNHWC(input0)) { - output_is_nhwc = true; - } + output_is_nhwc = model_builder.IsOperandNHWC(input0); for (size_t i = 0; i < node_input_size; i++) { auto input = node.InputDefs()[i]->Name(); @@ -1546,24 +1544,30 @@ void SqueezeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeAttrHelper helper(node); vector axes = helper.Get("axes", vector()); - auto input_dims = shaper[input].size(); + const auto& input_shape(shaper[input]); + auto input_dims = input_shape.size(); for (auto& axis : axes) { if (axis < 0) axis += input_dims; } - std::vector input_indices; - input_indices.push_back(operand_indices.at(input)); // input - - if (!axes.empty()) { - const auto axes_name = model_builder.GetUniqueName(node.Name() + input + "_axes"); - Shape axes_dimen = {static_cast(axes.size())}; - shaper.AddShape(axes_name, axes_dimen); - const OperandType axes_operand_type(Type::TENSOR_INT32, axes_dimen); - model_builder.AddOperandFromPersistMemoryBuffer(axes_name, axes.data(), axes_operand_type); - input_indices.push_back(operand_indices.at(axes_name)); // axes + if (axes.empty()) { // Squeeze all + for (size_t i = 0; i < input_dims; i++) { + if (input_shape[i] == 1) + axes.push_back(i); + } } + const auto axes_name = model_builder.GetUniqueName(node.Name() + input + "_axes"); + Shape axes_dimen = {static_cast(axes.size())}; + shaper.AddShape(axes_name, axes_dimen); + const OperandType axes_operand_type(Type::TENSOR_INT32, axes_dimen); + model_builder.AddOperandFromPersistMemoryBuffer(axes_name, axes.data(), axes_operand_type); + + std::vector input_indices; + input_indices.push_back(operand_indices.at(input)); // input + input_indices.push_back(operand_indices.at(axes_name)); // axes + const auto& output = node.OutputDefs()[0]->Name(); shaper.Squeeze(input, axes, output); const OperandType output_operand_type(operand_types.at(input).type, shaper[output]); diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc index a2fe6b345e..6d72a5fec4 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc @@ -43,6 +43,11 @@ NnapiExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_view const std::vector& /*kernel_registries*/) const { std::vector> result; + // TODO: Task 812756: NNAPI EP, add support for subgraph (If and Loop operators) + if (graph_view.IsSubgraph()) { + return result; + } + std::unordered_set all_node_inputs; for (const auto& node : graph_view.Nodes()) { for (auto* input : node.InputDefs()) { @@ -240,10 +245,9 @@ common::Status NnapiExecutionProvider::Compile(const std::vectorGetMappedInputIdx(input_name); const OrtValue* input_tensor = ort.KernelContext_GetInput(context, input_idx); - const auto tensor_info = ort.GetTensorTypeAndShape(input_tensor); - const auto& tensor_shape = ort.GetTensorShape(tensor_info); + auto* tensor_info = ort.GetTensorTypeAndShape(input_tensor); std::vector dimensions; - for (const auto& dim : tensor_shape) + for (const auto& dim : ort.GetTensorShape(tensor_info)) dimensions.push_back(static_cast(dim)); // it is possible that the input has the detailed size while @@ -299,7 +303,8 @@ common::Status NnapiExecutionProvider::Compile(const std::vector>& input_vals_vec, @@ -38,26 +36,34 @@ inline void TestActivationOp(const char* szOp, const std::vector> input_values{{-1.0f, 0, 1.0f, // normal input values for activation - FLT_MIN, FLT_MIN / 10, -FLT_MIN / 10, // min, denorm, -denorm - FLT_MAX, -FLT_MAX}}; // max, -max, inf + std::vector> input_values{{-1.0f, 0, 1.0f, // normal input values for activation + FLT_MIN, FLT_MIN / 10, -FLT_MIN / 10, // min, denorm, -denorm + FLT_MAX, -FLT_MAX}}; // max, -max, inf void SetUp() override { float low = -1.0f, high = 1.0f; @@ -106,5 +112,6 @@ class ActivationOpNoInfTest : public ::testing::Test { } } }; -} -} \ No newline at end of file + +} // namespace test +} // namespace onnxruntime \ No newline at end of file diff --git a/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc b/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc index 9e3e7712a7..cf20e27fc8 100644 --- a/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc +++ b/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc @@ -13,8 +13,9 @@ namespace test { TEST(MathOpTest, DimWithZeroHandling) { auto run = [](OpTester& tester) { - // exclude NGraph and TensorRT as this isn't handled by those EPs - tester.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider, kNGraphExecutionProvider}); + // exclude NGraph, TensorRT and NNAPI as this isn't handled by those EPs + tester.Run(OpTester::ExpectResult::kExpectSuccess, "", + {kTensorrtExecutionProvider, kNGraphExecutionProvider, kNnapiExecutionProvider}); }; // test binary element-wise op broadcasting when there's a dim with value of zero @@ -183,7 +184,7 @@ TEST(MathOpTest, Add_Broadcast_0x0) { test.AddInput("A", {}, {10.0f}); test.AddInput("B", {}, {2.0f}); test.AddOutput("C", {}, {12.0f}); - test.Run(); + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kNnapiExecutionProvider}); // NNAPI: Add does not support scalar input } TEST(MathOpTest, Add_Broadcast_0x1) { @@ -814,7 +815,7 @@ TEST(MathOpTest, Sum_8_Test1) { //This test runs fine on CPU Plugin test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider, kOpenVINOExecutionProvider}); #else - test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: Expected output shape [{3,3,3}] did not match run output shape [{3,1,1}] for sum + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: Expected output shape [{3,3,3}] did not match run output shape [{3,1,1}] for sum #endif } diff --git a/onnxruntime/test/providers/cpu/math/softmax_test.cc b/onnxruntime/test/providers/cpu/math/softmax_test.cc index 34dd6091fc..a61c63d38f 100644 --- a/onnxruntime/test/providers/cpu/math/softmax_test.cc +++ b/onnxruntime/test/providers/cpu/math/softmax_test.cc @@ -12,7 +12,7 @@ static void RunTest(const std::vector& x_vals, const std::vector& expected_vals, const std::vector& dimensions, int64_t axis = 1, - bool is_tensorrt_supported = true, + const std::unordered_set& excluded_providers = {}, OpTester::ExpectResult expect_result = OpTester::ExpectResult::kExpectSuccess, const std::string& error_msg = "", int opset = 7) { @@ -24,10 +24,6 @@ static void RunTest(const std::vector& x_vals, test.AddInput("X", dimensions, x_vals); test.AddOutput("Y", dimensions, expected_vals); - std::unordered_set excluded_providers; - if (!is_tensorrt_supported) { - excluded_providers.insert(kTensorrtExecutionProvider); - } test.Run(expect_result, error_msg, excluded_providers); } @@ -97,7 +93,7 @@ TEST(SoftmaxOperator, ThreeDimsAxis0) { 0.017545262f, 0.0135920765f, 0.027506188f, 0.010684152f, 0.0049549243f, 0.01401341f, 0.011721271f, 0.027815264f, 0.021463264f, 0.014014485f}; - RunTest(x_vals_3dims, expected_vals, three_dimensions, /*axis*/ 0, false); // Axis=0 is not supported by TensorRT + RunTest(x_vals_3dims, expected_vals, three_dimensions, /*axis*/ 0, {kTensorrtExecutionProvider}); // Axis=0 is not supported by TensorRT } TEST(SoftmaxOperator, ThreeDimsAxis1) { @@ -123,7 +119,7 @@ TEST(SoftmaxOperator, ThreeDimsAxis1) { 0.050680935f, 0.03926183f, 0.079453886f, 0.030862054f, 0.014312706f, 0.040478885f, 0.033857856f, 0.080346674f, 0.06199841f, 0.040481992f}; - RunTest(x_vals_3dims, expected_vals, three_dimensions, /*axis*/ 1, false); + RunTest(x_vals_3dims, expected_vals, three_dimensions, /*axis*/ 1, {kTensorrtExecutionProvider}); } TEST(SoftmaxOperator, ThreeDimsAxis2) { @@ -187,7 +183,7 @@ TEST(SoftmaxOperator, InvalidAxis) { RunTest(x_vals, expected_vals, dimensions, - /* invalid axis */ -10, false, + /* invalid axis */ -10, {kTensorrtExecutionProvider}, OpTester::ExpectResult::kExpectFailure, // bug in ONNX error message currently. Message should be // "[ShapeInferenceError] 'axis' must be in [-2 , 1]. Its actual value is: -10" @@ -201,7 +197,10 @@ TEST(SoftmaxOperator, DimWithZero) { std::vector expected_vals = {}; std::vector dimensions = {1, 0}; // dim with value of 0 should be handled - RunTest(x_vals, expected_vals, dimensions, 0, false, OpTester::ExpectResult::kExpectSuccess, "", 10); + RunTest(x_vals, expected_vals, dimensions, 0, + {kTensorrtExecutionProvider, + kNnapiExecutionProvider}, // NNAPI softmax does not support empty input + OpTester::ExpectResult::kExpectSuccess, "", 10); } } // namespace test diff --git a/onnxruntime/test/providers/cpu/tensor/concat_op_test.cc b/onnxruntime/test/providers/cpu/tensor/concat_op_test.cc index 564f99d122..881c74e960 100644 --- a/onnxruntime/test/providers/cpu/tensor/concat_op_test.cc +++ b/onnxruntime/test/providers/cpu/tensor/concat_op_test.cc @@ -62,7 +62,9 @@ TEST(ConcatOpTest, Concat1D_2) { test.AddInput("input2", {2}, {2.0f, 3.0f}); test.AddInput("input3", {0}, {}); test.AddOutput("concat_result", {3}, {1.0f, 2.0f, 3.0f}); - test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: no support for dynamic shape tensor + test.Run(OpTester::ExpectResult::kExpectSuccess, "", + {kTensorrtExecutionProvider, //TensorRT: no support for dynamic shape tensor + kNnapiExecutionProvider}); // NNAPI: concat does not support 0 size input } TEST(ConcatOpTest, Concat2D_1) { @@ -104,7 +106,9 @@ TEST(ConcatOpTest, Concat2D_3) { test.AddInput("input2", {1, 0}, {}); test.AddInput("input3", {1, 0}, {}); test.AddOutput("concat_result", {1, 0}, {}); - test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: no support for dynamic shape tensor + test.Run(OpTester::ExpectResult::kExpectSuccess, "", + {kTensorrtExecutionProvider, //TensorRT: no support for dynamic shape tensor + kNnapiExecutionProvider}); // NNAPI: concat does not support 0 size input } TEST(ConcatOpTest, Concat3D_1) { diff --git a/onnxruntime/test/providers/provider_test_utils.cc b/onnxruntime/test/providers/provider_test_utils.cc index 889571ec27..4b9cfcfee1 100644 --- a/onnxruntime/test/providers/provider_test_utils.cc +++ b/onnxruntime/test/providers/provider_test_utils.cc @@ -723,6 +723,7 @@ void OpTester::Run( kDmlExecutionProvider, kAclExecutionProvider, kArmNNExecutionProvider, + kNnapiExecutionProvider, }; bool has_run = false; @@ -807,7 +808,8 @@ void OpTester::Run( if (provider_type == onnxruntime::kNGraphExecutionProvider || provider_type == onnxruntime::kOpenVINOExecutionProvider || provider_type == onnxruntime::kTensorrtExecutionProvider || - provider_type == onnxruntime::kNupharExecutionProvider) + provider_type == onnxruntime::kNupharExecutionProvider || + provider_type == onnxruntime::kNnapiExecutionProvider) continue; auto reg = execution_provider->GetKernelRegistry(); if (!KernelRegistry::HasImplementationOf(*reg, node, execution_provider->Type())) {