saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-27 20:02:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Enable scalar input/output for NNAPI EP (#5922)

Browse source 
* Enable scalar input for NNAPI EP

* Map scalar output ({1} tensor) of NNAPI back to scalar ({} tensor) if necessary

* Enforce NNAPI scalar output has {1} shape

* address CR comments

* minor update
This commit is contained in:
Guoyu Wang 2020-11-25 21:05:50 -08:00 committed by GitHub
parent 5fdd9f0fd2
commit d52b9aca68
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
5 changed files with 61 additions and 31 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

36
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc
View file

@ -181,10 +181,12 @@ Status ModelBuilder::RegisterInitializers() {
shape.push_back(SafeInt<uint32_t>(dim));
}
// If we have an empty shape, this is a scalar initializer, since NNAPI does not allow empty shape,
// we will make the scalar initializer a {1} tensor
if (shape.empty())
// If we have an empty shape, this is a scalar initializer,
// since NNAPI will treat empty shape input as dynamic ranking input, (onnx does not support dynamic ranking)
// we will make the scalar initializer as a {1} tensor
if (shape.empty()) {
shape.push_back(1);
}
Type type = Type::TENSOR_FLOAT32;
switch (tensor.data_type()) {
@ -263,12 +265,12 @@ Status ModelBuilder::RegisterModelInputs() {
shape.push_back(SafeInt<uint32_t>(dim.dim_value()));
}
// NNAPI has strict input type requirements which separates tensor inputs and scalar inputs
// For ONNX the we do not have clear line between scalar inputs and tensor inputs
// Also NNAPI treats a tensor input with empty shape as dynamic shape input
// Disable support of the scalar input (tensor input with an empty shape) for now
// TODO, add support for ONNX scalar input (tensor input with an empty shape)
ORT_RETURN_IF_NOT(!shape.empty(), "0-rank input is not currently supported, input name, ", input_name);
// If we have an empty shape, this is a scalar input,
// since NNAPI will treat empty shape input as dynamic ranking input, (onnx does not support dynamic ranking)
// we will make the scalar input as a {1} tensor
if (shape.empty()) {
shape.push_back(1);
}
Type type = Type::TENSOR_FLOAT32;
float scale = 0.0f;
@ -329,6 +331,22 @@ Status ModelBuilder::RegisterModelOutputs() {
"The output of graph is not registered [", output_name, "]");
}
// Since for now all the shapes are deterministic for NNAPI, it's impossible we can have unknown output shape
const auto* shape_proto = node_arg->Shape();
ORT_RETURN_IF(shape_proto == nullptr, "shape_proto cannot be null for output: ", output_name);
if (shape_proto->dim_size() == 0) {
// In NNAPI scalar output must have {1} shape
const auto& output_shape = shaper_[output_name];
ORT_RETURN_IF_NOT(output_shape.size() == 1 && output_shape[0] == 1,
"scalar output [", output_name, "] must have {1} shape, ",
" actual shape, ", Shape2String(output_shape));
// Record the scalar output
// Since within NNAPI the scalar outputs will have {1} shapes, and for ORT scalar outputs will have {} shapes,
// we need to change the shapes of these scalar outputs back to {} when NNAPI EP returns these values to ORT
nnapi_model_->AddScalarOutput(output_name);
}
std::string nnapi_output_name = output_name;
if (IsOperandNHWC(output_name)) {
// We need to transpose the output still in nhwc back to nchw

8
onnxruntime/core/providers/nnapi/nnapi_builtin/model.cc
View file

@ -40,6 +40,14 @@ void Model::AddOutput(const std::string& onnx_output_name,
operand_types_.emplace(nnapi_output_name, operand_type);
}
bool Model::IsScalarOutput(const std::string& output_name) const {
return Contains(scalar_outputs_, output_name);
}
void Model::AddScalarOutput(const std::string& output_name) {
scalar_outputs_.insert(output_name);
}
const std::vector<std::string>& Model::GetInputs() const {
return input_names_;
}

10
onnxruntime/core/providers/nnapi/nnapi_builtin/model.h
View file

@ -3,6 +3,8 @@
#pragma once
#include <unordered_set>
#include "builders/shaper.h"
#include "core/platform/ort_mutex.h"
#include "nnapi_lib/NeuralNetworksWrapper.h"
@ -96,6 +98,11 @@ class Model {
// Mutex for exclusive lock to this model object
OrtMutex& GetMutex() { return mutex_; }
// If the given output is a scalar output
// Since NNAPI does not support tensor with empty shape (scalar), we use {1} tensor for scalar in NNAPI
// this output may need special handling
bool IsScalarOutput(const std::string& output_name) const;
Status PrepareForExecution(std::unique_ptr<Execution>& execution) ORT_MUST_USE_RESULT;
private:
@ -112,6 +119,7 @@ class Model {
std::vector<std::string> input_names_;
std::vector<std::string> output_names_;
std::unordered_map<std::string, android::nn::wrapper::OperandType> operand_types_;
std::unordered_set<std::string> scalar_outputs_;
Shaper shaper_;
@ -133,6 +141,8 @@ class Model {
const std::string& nnapi_output_name,
const android::nn::wrapper::OperandType& operand_type);
void AddScalarOutput(const std::string& output_name);
void SetShaper(const Shaper shaper) { shaper_ = shaper; }
int32_t GetAndroidSdkVer() const;

23
onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc
View file

@ -318,15 +318,13 @@ common::Status NnapiExecutionProvider::Compile(const std::vector<FusedNodeAndGra
for (const auto& dim : ort.GetTensorShape(tensor_info))
dimensions.push_back(static_cast<uint32_t>(dim));
// NNAPI has strict input type requirements which separates tensor inputs and scalar inputs
// For ONNX the we do not have clear line between scalar inputs and tensor inputs
// Also NNAPI treats a tensor input with empty shape as dynamic shape input
// Disable support of the scalar input (tensor input with an empty shape) for now
// TODO, add support for ONNX scalar input (tensor input with an empty shape)
// If we have an empty shape, this is a scalar input,
// since NNAPI will treat empty shape input as dynamic ranking input, (onnx does not support dynamic ranking)
// we will make the scalar input as a {1} tensor
if (dimensions.empty()) {
return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
"NNAPI does not support scalar input, input name, ", input_name);
dimensions.push_back(1);
}
// it is possible that the input has the detailed size while
// the model has an operand with unknown size, use the size
// of the actual input
@ -367,9 +365,11 @@ common::Status NnapiExecutionProvider::Compile(const std::vector<FusedNodeAndGra
std::vector<OperandType> dynamic_shape_output_types;
std::vector<std::unique_ptr<uint8_t[]>> dynamic_shape_output_buffers;
for (size_t i = 0; i < num_outputs; i++) {
const auto output_name = model_outputs[i];
const auto& output_name = model_outputs[i];
// Below 2 need to be copied since we will modify or take ownership
const auto model_output_type = model->GetOutputType(output_name, *execution);
const auto output_shape = model_output_type.dimensions;
auto output_shape = model_output_type.dimensions;
bool is_dynamic_shape_output = false;
if (model_output_type.GetOperandBlobByteSize() == 0) {
@ -384,6 +384,11 @@ common::Status NnapiExecutionProvider::Compile(const std::vector<FusedNodeAndGra
void* output_buffer = nullptr;
size_t output_buffer_byte_size;
if (!is_dynamic_shape_output) {
// Since NNAPI use {1} tensor as scalar, if the model output should have empty shape
// We are going to replace the {1} shape of the output back to {}
if (model->IsScalarOutput(output_name))
output_shape.clear();
ORT_RETURN_IF_ERROR(GetOutputBuffer(ort, context,
*model,
output_name, output_shape, model_output_type.type,

15
onnxruntime/test/providers/cpu/math/element_wise_ops_test.cc
View file

@ -192,7 +192,7 @@ TEST(MathOpTest, Add_Broadcast_0x0) {
test.AddInput<float>("A", {}, {10.0f});
test.AddInput<float>("B", {}, {2.0f});
test.AddOutput<float>("C", {}, {12.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kNnapiExecutionProvider}); // NNAPI: Add does not support scalar input
test.Run(OpTester::ExpectResult::kExpectSuccess, "");
}
TEST(MathOpTest, Add_Broadcast_0x1) {
@ -205,11 +205,7 @@ TEST(MathOpTest, Add_Broadcast_0x1) {
test.Run(OpTester::ExpectResult::kExpectSuccess, "");
};
// NNAPI only supports scalar initializer, not scalar input
#ifndef USE_NNAPI
run(false);
#endif
run(true);
}
@ -222,11 +218,8 @@ TEST(MathOpTest, Add_Broadcast_1x0) {
test.AddOutput<float>("C", {1}, {12.0f});
test.Run(OpTester::ExpectResult::kExpectSuccess, "");
};
// NNAPI only supports scalar initializer, not scalar input
#ifndef USE_NNAPI
run(false);
#endif
run(false);
run(true);
}
@ -398,11 +391,7 @@ TEST(MathOpTest, Sub_Broadcast_Scalar) {
test.Run(OpTester::ExpectResult::kExpectSuccess, "");
};
// NNAPI only supports scalar initializer, not scalar input
#ifndef USE_NNAPI
run(false);
#endif
run(true);
}