From d52b9aca680829715b69e2f8cd544a34d91bb526 Mon Sep 17 00:00:00 2001 From: Guoyu Wang <62914304+gwang-msft@users.noreply.github.com> Date: Wed, 25 Nov 2020 21:05:50 -0800 Subject: [PATCH] Enable scalar input/output for NNAPI EP (#5922) * Enable scalar input for NNAPI EP * Map scalar output ({1} tensor) of NNAPI back to scalar ({} tensor) if necessary * Enforce NNAPI scalar output has {1} shape * address CR comments * minor update --- .../nnapi_builtin/builders/model_builder.cc | 36 ++++++++++++++----- .../providers/nnapi/nnapi_builtin/model.cc | 8 +++++ .../providers/nnapi/nnapi_builtin/model.h | 10 ++++++ .../nnapi_builtin/nnapi_execution_provider.cc | 23 +++++++----- .../cpu/math/element_wise_ops_test.cc | 15 ++------ 5 files changed, 61 insertions(+), 31 deletions(-) diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc index 1aa179acd6..9bb7e16179 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc @@ -181,10 +181,12 @@ Status ModelBuilder::RegisterInitializers() { shape.push_back(SafeInt(dim)); } - // If we have an empty shape, this is a scalar initializer, since NNAPI does not allow empty shape, - // we will make the scalar initializer a {1} tensor - if (shape.empty()) + // If we have an empty shape, this is a scalar initializer, + // since NNAPI will treat empty shape input as dynamic ranking input, (onnx does not support dynamic ranking) + // we will make the scalar initializer as a {1} tensor + if (shape.empty()) { shape.push_back(1); + } Type type = Type::TENSOR_FLOAT32; switch (tensor.data_type()) { @@ -263,12 +265,12 @@ Status ModelBuilder::RegisterModelInputs() { shape.push_back(SafeInt(dim.dim_value())); } - // NNAPI has strict input type requirements which separates tensor inputs and scalar inputs - // For ONNX the we do not have clear line between scalar inputs and tensor inputs - // Also NNAPI treats a tensor input with empty shape as dynamic shape input - // Disable support of the scalar input (tensor input with an empty shape) for now - // TODO, add support for ONNX scalar input (tensor input with an empty shape) - ORT_RETURN_IF_NOT(!shape.empty(), "0-rank input is not currently supported, input name, ", input_name); + // If we have an empty shape, this is a scalar input, + // since NNAPI will treat empty shape input as dynamic ranking input, (onnx does not support dynamic ranking) + // we will make the scalar input as a {1} tensor + if (shape.empty()) { + shape.push_back(1); + } Type type = Type::TENSOR_FLOAT32; float scale = 0.0f; @@ -329,6 +331,22 @@ Status ModelBuilder::RegisterModelOutputs() { "The output of graph is not registered [", output_name, "]"); } + // Since for now all the shapes are deterministic for NNAPI, it's impossible we can have unknown output shape + const auto* shape_proto = node_arg->Shape(); + ORT_RETURN_IF(shape_proto == nullptr, "shape_proto cannot be null for output: ", output_name); + if (shape_proto->dim_size() == 0) { + // In NNAPI scalar output must have {1} shape + const auto& output_shape = shaper_[output_name]; + ORT_RETURN_IF_NOT(output_shape.size() == 1 && output_shape[0] == 1, + "scalar output [", output_name, "] must have {1} shape, ", + " actual shape, ", Shape2String(output_shape)); + + // Record the scalar output + // Since within NNAPI the scalar outputs will have {1} shapes, and for ORT scalar outputs will have {} shapes, + // we need to change the shapes of these scalar outputs back to {} when NNAPI EP returns these values to ORT + nnapi_model_->AddScalarOutput(output_name); + } + std::string nnapi_output_name = output_name; if (IsOperandNHWC(output_name)) { // We need to transpose the output still in nhwc back to nchw diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/model.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/model.cc index 1aff71f01a..440cccff2a 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/model.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/model.cc @@ -40,6 +40,14 @@ void Model::AddOutput(const std::string& onnx_output_name, operand_types_.emplace(nnapi_output_name, operand_type); } +bool Model::IsScalarOutput(const std::string& output_name) const { + return Contains(scalar_outputs_, output_name); +} + +void Model::AddScalarOutput(const std::string& output_name) { + scalar_outputs_.insert(output_name); +} + const std::vector& Model::GetInputs() const { return input_names_; } diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/model.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/model.h index ea32f567c1..fca0068939 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/model.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/model.h @@ -3,6 +3,8 @@ #pragma once +#include + #include "builders/shaper.h" #include "core/platform/ort_mutex.h" #include "nnapi_lib/NeuralNetworksWrapper.h" @@ -96,6 +98,11 @@ class Model { // Mutex for exclusive lock to this model object OrtMutex& GetMutex() { return mutex_; } + // If the given output is a scalar output + // Since NNAPI does not support tensor with empty shape (scalar), we use {1} tensor for scalar in NNAPI + // this output may need special handling + bool IsScalarOutput(const std::string& output_name) const; + Status PrepareForExecution(std::unique_ptr& execution) ORT_MUST_USE_RESULT; private: @@ -112,6 +119,7 @@ class Model { std::vector input_names_; std::vector output_names_; std::unordered_map operand_types_; + std::unordered_set scalar_outputs_; Shaper shaper_; @@ -133,6 +141,8 @@ class Model { const std::string& nnapi_output_name, const android::nn::wrapper::OperandType& operand_type); + void AddScalarOutput(const std::string& output_name); + void SetShaper(const Shaper shaper) { shaper_ = shaper; } int32_t GetAndroidSdkVer() const; diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc index bf27398a0b..e959a24c2b 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc @@ -318,15 +318,13 @@ common::Status NnapiExecutionProvider::Compile(const std::vector(dim)); - // NNAPI has strict input type requirements which separates tensor inputs and scalar inputs - // For ONNX the we do not have clear line between scalar inputs and tensor inputs - // Also NNAPI treats a tensor input with empty shape as dynamic shape input - // Disable support of the scalar input (tensor input with an empty shape) for now - // TODO, add support for ONNX scalar input (tensor input with an empty shape) + // If we have an empty shape, this is a scalar input, + // since NNAPI will treat empty shape input as dynamic ranking input, (onnx does not support dynamic ranking) + // we will make the scalar input as a {1} tensor if (dimensions.empty()) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, - "NNAPI does not support scalar input, input name, ", input_name); + dimensions.push_back(1); } + // it is possible that the input has the detailed size while // the model has an operand with unknown size, use the size // of the actual input @@ -367,9 +365,11 @@ common::Status NnapiExecutionProvider::Compile(const std::vector dynamic_shape_output_types; std::vector> dynamic_shape_output_buffers; for (size_t i = 0; i < num_outputs; i++) { - const auto output_name = model_outputs[i]; + const auto& output_name = model_outputs[i]; + + // Below 2 need to be copied since we will modify or take ownership const auto model_output_type = model->GetOutputType(output_name, *execution); - const auto output_shape = model_output_type.dimensions; + auto output_shape = model_output_type.dimensions; bool is_dynamic_shape_output = false; if (model_output_type.GetOperandBlobByteSize() == 0) { @@ -384,6 +384,11 @@ common::Status NnapiExecutionProvider::Compile(const std::vectorIsScalarOutput(output_name)) + output_shape.clear(); + ORT_RETURN_IF_ERROR(GetOutputBuffer(ort, context, *model, output_name, output_shape, model_output_type.type, diff --git a/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc b/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc index 4441bdf1e8..0d26ebfc0a 100644 --- a/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc +++ b/onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc @@ -192,7 +192,7 @@ TEST(MathOpTest, Add_Broadcast_0x0) { test.AddInput("A", {}, {10.0f}); test.AddInput("B", {}, {2.0f}); test.AddOutput("C", {}, {12.0f}); - test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kNnapiExecutionProvider}); // NNAPI: Add does not support scalar input + test.Run(OpTester::ExpectResult::kExpectSuccess, ""); } TEST(MathOpTest, Add_Broadcast_0x1) { @@ -205,11 +205,7 @@ TEST(MathOpTest, Add_Broadcast_0x1) { test.Run(OpTester::ExpectResult::kExpectSuccess, ""); }; - // NNAPI only supports scalar initializer, not scalar input -#ifndef USE_NNAPI run(false); -#endif - run(true); } @@ -222,11 +218,8 @@ TEST(MathOpTest, Add_Broadcast_1x0) { test.AddOutput("C", {1}, {12.0f}); test.Run(OpTester::ExpectResult::kExpectSuccess, ""); }; - // NNAPI only supports scalar initializer, not scalar input -#ifndef USE_NNAPI - run(false); -#endif + run(false); run(true); } @@ -398,11 +391,7 @@ TEST(MathOpTest, Sub_Broadcast_Scalar) { test.Run(OpTester::ExpectResult::kExpectSuccess, ""); }; - // NNAPI only supports scalar initializer, not scalar input -#ifndef USE_NNAPI run(false); -#endif - run(true); }