saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-27 20:02:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

[NNAPI EP] Remove/Refactor shaper inference calculation code (#12618)

Browse source 
**Description**: Describe your changes.
As title.

The purpose of this pr is to eliminate as much of repetitive shape
inference code in nnapi ep shaper struct.

For ops (mainly require composed operations) :
-BatchNorm
-Reshape 
-Squeeze (in one case of gemm operator)
-BatchMatMul
still contains some shape calculation impl/logic.

Dynamic shape functions are not touched yet. 

**Motivation and Context**
- Why is this change required? What problem does it solve?
- If it fixes an open issue, please link to the issue here.

Clean up redundant code as cpu shape inference impl for NNAPI EP.
Get rid of the shape inference code in NNAPI EP by using the static
shape info in output NodeArg.
This commit is contained in:
Rachel Guo 2022-09-21 10:22:02 -07:00 committed by GitHub
parent 8356e3b9b0
commit 3810effe6e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
14 changed files with 359 additions and 890 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

22
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc
View file

@ -267,6 +267,26 @@ void GetFlattenOutputShape(const NodeUnit& node_unit, const Shape& input_shape,
dim_2 = std::accumulate(input_shape.cbegin() + axis, input_shape.cend(), 1, std::multiplies<int32_t>());
}
Shape GetShapeInfoFromNodeArg(const GraphViewer& graph_viewer, const std::string& name) {
// can be applied to both input and output
Shape shape;
const auto* node_arg = graph_viewer.GetNodeArg(name);
const auto* shape_proto = node_arg->Shape();
shape.reserve(shape_proto->dim_size());
for (const auto& shape_dim : shape_proto->dim()) {
// shape_dim here can possibly have dim_param, but as dynamic shape is not supported in NNAPI for now
// (checked already in BaseOpSupportChecker), call dim_value here only.
shape.push_back(SafeInt<uint32_t>(shape_dim.dim_value()));
}
// If we have an empty shape, (scalar input), we need to make it as {1} as
// nnapi will treat empty shape as dynamic ranking and onnx does not support that
if (shape_proto->dim_size() == 0) {
shape.push_back(1);
}
return shape;
}
bool IsValidSupportedNodeGroup(const std::vector<const Node*>& supported_node_partition) {
if (supported_node_partition.size() == 1) {
const auto* node = supported_node_partition[0];
@ -356,7 +376,7 @@ bool IsNodeSupportedInGroup(const NodeUnit& node_unit, const GraphViewer& graph_
return true;
}
std::string Shape2String(const std::vector<uint32_t>& shape) {
std::string Shape2String(const Shape& shape) {
std::ostringstream os;
os << "[ ";
for (const auto& dim : shape)

9
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h
View file

@ -5,6 +5,7 @@
#include <string>
#include <vector>
#include "core/common/inlined_containers.h"
#include "core/graph/basic_types.h"
#include "core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksTypes.h"
@ -24,7 +25,6 @@
namespace onnxruntime {
using Shape = std::vector<uint32_t>;
using InitializerMap = std::unordered_map<std::string, const ONNX_NAMESPACE::TensorProto&>;
class GraphViewer;
@ -37,6 +37,8 @@ struct NodeUnitIODef;
namespace nnapi {
using Shape = InlinedVector<uint32_t>;
class IOpSupportChecker;
struct OpSupportCheckParams;
@ -144,6 +146,9 @@ bool GetType(const NodeArg& node_arg, int32_t& type);
// Get the output shape of Flatten Op
void GetFlattenOutputShape(const NodeUnit& node_unit, const Shape& input_shape, int32_t& dim_1, int32_t& dim_2);
// Get the shape information from NodeArg
Shape GetShapeInfoFromNodeArg(const GraphViewer& graph_viewer, const std::string& name);
// If a node is supported by NNAPI
bool IsNodeSupported(const NodeUnit& node_unit, const GraphViewer& graph_viewer, const OpSupportCheckParams& params);
@ -157,7 +162,7 @@ bool IsNodeSupportedInGroup(const NodeUnit& node_unit, const GraphViewer& graph_
bool IsValidSupportedNodeGroup(const std::vector<const Node*>& supported_node_group);
// Get string representation of a Shape
std::string Shape2String(const std::vector<uint32_t>& shape);
std::string Shape2String(const Shape& shape);
uint32_t ShapeSize(const Shape& shape, size_t begin_idx, size_t end_idx);
inline uint32_t ShapeSize(const Shape& shape) {

14
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc
View file

@ -25,7 +25,7 @@ namespace onnxruntime {
namespace nnapi {
ModelBuilder::ModelBuilder(const GraphViewer& graph_viewer)
: nnapi_(NnApiImplementation()), graph_viewer_(graph_viewer) {}
: nnapi_(NnApiImplementation()), graph_viewer_(graph_viewer), shaper_{graph_viewer} {}
int32_t ModelBuilder::GetNNAPIFeatureLevel() const {
return nnapi_ ? static_cast<int32_t>(nnapi_->nnapi_runtime_feature_level) : 0;
@ -34,7 +34,7 @@ int32_t ModelBuilder::GetNNAPIFeatureLevel() const {
// Scalar operand is copied into the model, no need to persist
#define DEFINE_ADD_OPERAND_FROM_SCALAR(scalar_type, op_type) \
Status ModelBuilder::AddOperandFromScalar(scalar_type value, uint32_t& index) { \
OperandType operandType(Type::op_type, std::vector<uint32_t>{}); \
OperandType operandType(Type::op_type, InlinedVector<uint32_t>{}); \
ORT_RETURN_IF_ERROR(AddNewNNAPIOperand(operandType, index)); \
RETURN_STATUS_ON_ERROR_WITH_NOTE( \
nnapi_->ANeuralNetworksModel_setOperandValue( \
@ -54,7 +54,6 @@ void ModelBuilder::AddInitializerToSkip(const std::string& tensor_name) {
}
Status ModelBuilder::Prepare() {
nnapi_model_ = std::unique_ptr<Model>(new Model());
RETURN_STATUS_ON_ERROR(nnapi_->ANeuralNetworksModel_create(&nnapi_model_->model_));
ORT_RETURN_IF_ERROR(GetTargetDevices());
PreprocessNodeUnits();
@ -65,7 +64,6 @@ Status ModelBuilder::Prepare() {
ORT_RETURN_IF_ERROR(RegisterModelInputs());
ORT_RETURN_IF_ERROR(AddOperations());
ORT_RETURN_IF_ERROR(RegisterModelOutputs());
RegisterModelShaper();
return Status::OK();
}
@ -388,10 +386,6 @@ Status ModelBuilder::RegisterModelOutputs() {
return Status::OK();
}
void ModelBuilder::RegisterModelShaper() {
nnapi_model_->SetShaper(shaper_);
}
Status ModelBuilder::AddNewOperand(const std::string& name,
const OperandType& operand_type,
uint32_t& index) {
@ -512,10 +506,10 @@ Status ModelBuilder::AddOperations() {
return Status::OK();
}
Status ModelBuilder::AddOperation(int op, const std::vector<uint32_t>& input_indices,
Status ModelBuilder::AddOperation(int op, const InlinedVector<uint32_t>& input_indices,
const std::vector<std::string>& output_names,
const std::vector<OperandType>& output_types) {
std::vector<uint32_t> output_indices;
InlinedVector<uint32_t> output_indices;
for (size_t i = 0; i < output_types.size(); i++) {
uint32_t index = 0;
ORT_RETURN_IF_ERROR(AddNewOperand(output_names[i], output_types[i], index));

12
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h
View file

@ -5,6 +5,7 @@
#include <onnx/onnx_pb.h>
#include <unordered_set>
#include "core/common/inlined_containers.h"
#include "core/graph/basic_types.h"
#include "core/providers/nnapi/nnapi_builtin/model.h"
#include "core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.h"
@ -46,7 +47,7 @@ class ModelBuilder {
int32_t GetNNAPIFeatureLevel() const;
// Add an NNAPI operation (operator)
common::Status AddOperation(int op, const std::vector<uint32_t>& input_indices,
common::Status AddOperation(int op, const InlinedVector<uint32_t>& input_indices,
const std::vector<std::string>& output_names,
const std::vector<android::nn::wrapper::OperandType>& output_types);
@ -117,7 +118,7 @@ class ModelBuilder {
private:
const NnApi* nnapi_{nullptr};
const GraphViewer& graph_viewer_;
std::unique_ptr<Model> nnapi_model_;
std::unique_ptr<Model> nnapi_model_{std::make_unique<Model>()};
uint32_t name_token_{0};
@ -141,9 +142,8 @@ class ModelBuilder {
std::unordered_map<std::string, std::shared_ptr<IOpSupportChecker>> op_support_checkers_;
std::vector<uint32_t> input_index_vec_;
std::vector<uint32_t> output_index_vec_;
InlinedVector<uint32_t> input_index_vec_;
InlinedVector<uint32_t> output_index_vec_;
// Contains all quantized operators' input and the NodeUnit(s) using the input
// In the form of {input_name, [NodeUnit(s) using the input]}
@ -178,8 +178,6 @@ class ModelBuilder {
common::Status RegisterModelInputs();
common::Status AddOperations();
common::Status RegisterModelOutputs();
// After constructing the NNAPI model, will set the shape inferencing record to the Model
void RegisterModelShaper();
// Get all quantized inputs in the underlying graph_viewer
void GetAllQuantizedOpInputs();

262
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc
View file

@ -5,20 +5,21 @@
#include <onnx/onnx_pb.h>
#include <algorithm>
#include <unordered_set>
#include "core/common/logging/logging.h"
#include "core/common/safeint.h"
#include "core/framework/tensorprotoutils.h"
#include "core/graph/graph_viewer.h"
#include "core/providers/common.h"
#include "core/providers/shared/utils/utils.h"
#include "core/providers/shared/node_unit/node_unit.h"
#include "core/providers/cpu/tensor/slice_helper.h"
#include "core/providers/nnapi/nnapi_builtin/builders/op_builder_helpers.h"
#include "helper.h"
#include "model_builder.h"
#include "op_support_checker.h"
#include "core/optimizer/initializer.h"
#include "core/providers/common.h"
#include "core/providers/cpu/tensor/slice_helper.h"
#include "core/providers/nnapi/nnapi_builtin/builders/helper.h"
#include "core/providers/nnapi/nnapi_builtin/builders/model_builder.h"
#include "core/providers/nnapi/nnapi_builtin/builders/op_builder_helpers.h"
#include "core/providers/nnapi/nnapi_builtin/builders/op_support_checker.h"
#include "core/providers/shared/node_unit/node_unit.h"
#include "core/providers/shared/utils/utils.h"
using namespace android::nn::wrapper;
@ -38,8 +39,7 @@ struct OpBuilderRegistrations {
static Status AddBinaryOperator(int32_t op_type,
ModelBuilder& model_builder,
const std::string& input1,
const std::string& input2,
const std::string& input1, const std::string& input2,
bool add_activation,
int32_t fuse_code,
const std::string& output,
@ -49,7 +49,7 @@ static Status AddBinaryOperator(int32_t op_type,
const auto& operand_indices(model_builder.GetOperandIndices());
const auto& operand_types(model_builder.GetOperandTypes());
std::vector<uint32_t> input_indices;
InlinedVector<uint32_t> input_indices;
input_indices.push_back(operand_indices.at(input1)); // input 1
input_indices.push_back(operand_indices.at(input2)); // input 2
@ -57,7 +57,6 @@ static Status AddBinaryOperator(int32_t op_type,
ADD_SCALAR_OPERAND(model_builder, input_indices, fuse_code);
}
ORT_RETURN_IF_ERROR(shaper.Eltwise(input1, input2, output));
const OperandType output_operand_type(operand_types.at(input1).type, shaper[output],
output_scale, output_zero_point);
ORT_RETURN_IF_ERROR(model_builder.AddOperation(op_type, input_indices,
@ -65,10 +64,57 @@ static Status AddBinaryOperator(int32_t op_type,
return Status::OK();
}
static Status AddSqueezeOp(ModelBuilder& model_builder,
const std::string& node_name,
const std::string& input, const std::string& output,
std::vector<int32_t> axes) {
static Status AddNnapiBatchNormalization(ModelBuilder& model_builder,
const std::string& input1,
const std::string& input2,
const std::string& input3,
const std::string& output1,
const std::string& output2,
int32_t fuse_code,
float output_scale = 0.0f,
int32_t output_zero_point = 0) {
auto& shaper(model_builder.GetShaper());
const auto& operand_indices(model_builder.GetOperandIndices());
const auto& operand_types(model_builder.GetOperandTypes());
// Add Nnapi Mul
InlinedVector<uint32_t> input_indices;
input_indices.push_back(operand_indices.at(input1));
input_indices.push_back(operand_indices.at(input2));
ADD_SCALAR_OPERAND(model_builder, input_indices, ANEURALNETWORKS_FUSED_NONE);
const Shape& shape1 = shaper[input1];
const Shape& shape2 = shaper[input2];
Shape output1_shape;
// broadcasting support for eltwise shape operation
ORT_RETURN_IF_ERROR(op_builder_helpers::PerformBroadcasting(shape1, shape2, output1_shape));
const OperandType output_operand_type(operand_types.at(input1).type, output1_shape,
output_scale, output_zero_point);
ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_MUL, input_indices,
{output1}, {output_operand_type}));
// Add Nnapi Add
input_indices.clear();
input_indices.push_back(operand_indices.at(output1));
input_indices.push_back(operand_indices.at(input3));
ADD_SCALAR_OPERAND(model_builder, input_indices, fuse_code);
const Shape& shape3 = shaper[input3];
Shape output2_shape;
ORT_RETURN_IF_ERROR(op_builder_helpers::PerformBroadcasting(output1_shape, shape3, output2_shape));
const OperandType output_operand_type2(operand_types.at(input3).type, output2_shape,
output_scale, output_zero_point);
ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_ADD, input_indices,
{output2}, {output_operand_type2}));
return Status::OK();
}
static Status AddNnapiSqueeze(ModelBuilder& model_builder,
const std::string& node_name,
const std::string& input, const std::string& output,
std::vector<int32_t> axes) {
if (model_builder.GetNNAPIFeatureLevel() < ANEURALNETWORKS_FEATURE_LEVEL_2) {
return ORT_MAKE_STATUS(
ONNXRUNTIME, FAIL, "Squeeze is not supported on API level ", model_builder.GetNNAPIFeatureLevel());
@ -78,7 +124,7 @@ static Status AddSqueezeOp(ModelBuilder& model_builder,
const auto& operand_indices(model_builder.GetOperandIndices());
const auto& operand_types(model_builder.GetOperandTypes());
const auto& input_shape(shaper[input]);
const auto input_shape = shaper[input];
auto input_dims = input_shape.size();
for (auto& axis : axes) {
axis = static_cast<int32_t>(HandleNegativeAxis(axis, input_dims));
@ -102,12 +148,34 @@ static Status AddSqueezeOp(ModelBuilder& model_builder,
const OperandType axes_operand_type(Type::TENSOR_INT32, axes_dimen);
ORT_RETURN_IF_ERROR(model_builder.AddOperandFromPersistMemoryBuffer(axes_name, axes.data(), axes_operand_type));
std::vector<uint32_t> input_indices;
InlinedVector<uint32_t> input_indices;
input_indices.push_back(operand_indices.at(input)); // input
input_indices.push_back(operand_indices.at(axes_name)); // axes
ORT_RETURN_IF_ERROR(shaper.Squeeze(input, axes, output));
const OperandType output_operand_type(operand_types.at(input).type, shaper[output]);
// Shape inference calculation for squeeze
int32_t input_size = static_cast<int32_t>(input_shape.size());
std::unordered_set<int32_t> axes_to_be_squeezed;
// If the Op is squeezing all by not specifying axes, the axes is pre-populate
// with axes of all single dimensions by the caller
for (const auto& axis : axes)
axes_to_be_squeezed.insert(axis);
// Make output dimensions
InlinedVector<uint32_t> output_dimen;
output_dimen.reserve(input_size - axes_to_be_squeezed.size());
for (int32_t i = 0; i < input_size; i++) {
if (!Contains(axes_to_be_squeezed, i))
output_dimen.push_back(input_shape[i]);
}
// In case of a tensor has all 1's in dimension such as {1,1,1,1} and gets squeezed all
// the output shape will be {1}
if (output_dimen.empty())
output_dimen.push_back(1);
shaper.AddShape(output, output_dimen);
const OperandType output_operand_type(operand_types.at(input).type, output_dimen);
ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_SQUEEZE, input_indices,
{output}, {output_operand_type}));
return Status::OK();
@ -673,7 +741,6 @@ Status ReluOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N
const auto& input = node_unit.Inputs()[0].node_arg.Name();
const auto& output = node_unit.Outputs()[0].node_arg.Name();
ORT_RETURN_IF_ERROR(shaper.Identity(input, output));
const OperandType output_operand_type(operand_types.at(input).type, shaper[output]);
// skip this relu if it is some op's fuse output
@ -681,7 +748,7 @@ Status ReluOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N
LOGS_DEFAULT(VERBOSE) << "Relu Node [" << node_unit.Name() << "] fused";
model_builder.RegisterOperand(output, operand_indices.at(input), output_operand_type);
} else {
std::vector<uint32_t> input_indices;
InlinedVector<uint32_t> input_indices;
input_indices.push_back(operand_indices.at(input));
ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_RELU, input_indices,
{output}, {output_operand_type}));
@ -721,6 +788,7 @@ Status TransposeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, co
const auto& input = node_unit.Inputs()[0].node_arg.Name();
const auto& output = node_unit.Outputs()[0].node_arg.Name();
NodeAttrHelper helper(node_unit);
std::vector<int32_t> perm = helper.Get("perm", std::vector<int32_t>());
auto input_dims = static_cast<int32_t>(shaper[input].size());
@ -741,7 +809,6 @@ Status TransposeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, co
}
std::string perm_name = model_builder.GetUniqueName(node_unit.Name() + input + "perm");
ORT_RETURN_IF_ERROR(op_builder_helpers::AddNnapiTranspose(model_builder, input, perm_name, perm, output));
return Status::OK();
@ -854,13 +921,15 @@ bool ReshapeOpBuilder::IsQuantizedOp(const NodeUnit& node_unit) const {
const auto& operand_indices(model_builder.GetOperandIndices());
const auto& operand_types(model_builder.GetOperandTypes());
const auto& output = node_unit.Outputs()[0].node_arg.Name();
ORT_RETURN_IF_ERROR(shaper.Reshape(input, shape, output));
auto input_rank = shaper[input].size();
auto output_rank = shaper[output].size();
const auto input_shape = shaper[input];
const auto output_shape = shaper[output];
const auto input_rank = input_shape.size();
const auto output_rank = output_shape.size();
// For reshape, the output type should be the same as the input type except the shape is different
auto output_operand_type = operand_types.at(input);
output_operand_type.SetDimensions(shaper[output]);
output_operand_type.SetDimensions(output_shape);
// Since Reshape is not running using hardware in NNAPI for some CPU (e.g. Qualcomm SD for now)
// We will try to see if we the skip the Reshape to prevent context switching between
@ -871,8 +940,7 @@ bool ReshapeOpBuilder::IsQuantizedOp(const NodeUnit& node_unit) const {
} else {
// We still need to perform a reshape here
std::string shape_name = model_builder.GetUniqueName(node_unit.Name() + input + "newshape");
ORT_RETURN_IF_ERROR(op_builder_helpers::AddNnapiReshape(model_builder, input, shape_name, shape, output,
&shaper[output]));
ORT_RETURN_IF_ERROR(op_builder_helpers::AddNnapiReshape(model_builder, input, shape_name, shape, output));
}
return Status::OK();
@ -881,7 +949,6 @@ bool ReshapeOpBuilder::IsQuantizedOp(const NodeUnit& node_unit) const {
Status ReshapeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const {
auto& shaper(model_builder.GetShaper());
const auto& initializers(model_builder.GetInitializerTensors());
auto input = node_unit.Inputs()[0].node_arg.Name();
const auto& shape_tensor = *initializers.at(node_unit.Inputs()[1].node_arg.Name());
@ -889,7 +956,7 @@ Status ReshapeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, cons
auto raw_shape = unpacked_tensor.DataAsSpan<int64_t>();
const auto size = SafeInt<uint32_t>(shape_tensor.dims()[0]);
Shape input_shape = shaper[input];
const auto input_shape = shaper[input];
std::vector<int32_t> shape(size);
for (uint32_t i = 0; i < size; i++) {
int32_t dim = SafeInt<int32_t>(raw_shape[i]);
@ -929,7 +996,7 @@ void UnsqueezeOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, cons
}
Status UnsqueezeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const {
const auto& shaper(model_builder.GetShaper());
auto& shaper(model_builder.GetShaper());
const auto& input = node_unit.Inputs()[0].node_arg.Name();
// NNAPI does not support unsqueeze, here we utilize unsqueeze's axes input to compute output shape
@ -937,7 +1004,7 @@ Status UnsqueezeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, co
std::vector<int32_t> axes;
ORT_RETURN_IF_ERROR(GetAxesForSqueezeAndUnSqueeze(model_builder, node_unit, axes));
Shape input_shape = shaper[input];
const auto input_shape = shaper[input];
auto input_dims = input_shape.size();
std::vector<int32_t> shape;
const auto size = SafeInt<uint32_t>(input_dims + axes.size()); // "output rank"
@ -986,6 +1053,7 @@ Status BatchNormalizationOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_bu
// register a new operand with new shape
const auto& input = inputs[0].node_arg.Name();
const auto& output = node_unit.Outputs()[0].node_arg.Name();
const auto input_shape = shaper[input];
const auto& scale_tensor = *initializers.at(inputs[1].node_arg.Name());
const auto& bias_tensor = *initializers.at(inputs[2].node_arg.Name());
@ -1029,11 +1097,12 @@ Status BatchNormalizationOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_bu
// input shape {N, C} ==> tensor_a/b's shape {size}
// input shape {N, C, H} ==> tensor_a/b's shape {size, 1}
// input shape {N, C, H, W} ==> tensor_a/b's shape {size, 1, 1}
const auto input_rank = shaper[input].size();
const auto input_rank = input_shape.size();
for (size_t i = 2; i < input_rank; i++)
tensor_a_dimen.push_back(1);
}
shaper.AddShape(input, input_shape);
shaper.AddShape(tensor_a_name, tensor_a_dimen);
shaper.AddShape(tensor_b_name, tensor_a_dimen);
const OperandType a_operand_type(operand_types.at(input).type, tensor_a_dimen);
@ -1041,21 +1110,9 @@ Status BatchNormalizationOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_bu
const OperandType b_operand_type(operand_types.at(input).type, tensor_a_dimen);
ORT_RETURN_IF_ERROR(model_builder.AddOperandFromPersistMemoryBuffer(tensor_b_name, b.data(), b_operand_type));
// Mul
ORT_RETURN_IF_ERROR(AddBinaryOperator(ANEURALNETWORKS_MUL,
model_builder,
input, tensor_a_name,
true /* add_activation */, ANEURALNETWORKS_FUSED_NONE,
tensor_imm_product_name));
// Add
int32_t fuse_code = model_builder.FindActivation(node_unit);
ORT_RETURN_IF_ERROR(AddBinaryOperator(ANEURALNETWORKS_ADD,
model_builder,
tensor_imm_product_name, tensor_b_name,
true /* add_activation */, fuse_code,
output));
ORT_RETURN_IF_ERROR(AddNnapiBatchNormalization(model_builder, input, tensor_a_name, tensor_b_name,
tensor_imm_product_name, output, fuse_code));
return Status::OK();
}
@ -1124,7 +1181,7 @@ Status PoolOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N
std::vector<int32_t> onnx_pads, onnx_strides, kernel_shape;
bool use_auto_pad = false;
int32_t nnapi_padding_code = ANEURALNETWORKS_PADDING_VALID;
const auto& input_shape = shaper[input];
const auto input_shape = shaper[input];
if (is_average_pool || op_type == "MaxPool") {
const auto auto_pad_type = StringToAutoPadType(helper.Get("auto_pad", "NOTSET"));
kernel_shape = helper.Get("kernel_shape", std::vector<int32_t>{0, 0});
@ -1171,7 +1228,7 @@ Status PoolOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N
initializers, node_unit.Outputs()[0], node_unit.ModelPath(), y_scale, y_zero_point));
}
std::vector<uint32_t> input_indices;
InlinedVector<uint32_t> input_indices;
input_indices.push_back(operand_indices.at(input));
if (use_auto_pad) {
@ -1193,10 +1250,6 @@ Status PoolOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N
ADD_SCALAR_OPERAND(model_builder, input_indices, use_nchw);
}
ORT_RETURN_IF_ERROR(shaper.Pool(input,
onnx_pads, onnx_strides, kernel_shape,
use_nchw,
output));
const OperandType output_operand_type(operand_types.at(input).type, shaper[output], y_scale, y_zero_point);
ORT_RETURN_IF_ERROR(model_builder.AddOperation(op_code, input_indices,
{output}, {output_operand_type}));
@ -1385,7 +1438,7 @@ Status ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N
const auto auto_pad_type = StringToAutoPadType(helper.Get("auto_pad", "NOTSET"));
bool use_auto_pad = false;
int32_t nnapi_padding_code = ANEURALNETWORKS_PADDING_SAME;
const auto& input_shape = shaper[input];
const auto input_shape = shaper[input];
const auto& kernel_shape = shaper[weight];
const auto weight_size_y = kernel_shape[1];
const auto weight_size_x = kernel_shape[2];
@ -1395,7 +1448,7 @@ Status ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N
auto_pad_type, use_nchw,
onnx_pads, nnapi_padding_code, use_auto_pad));
std::vector<uint32_t> input_indices;
InlinedVector<uint32_t> input_indices;
input_indices.push_back(operand_indices.at(input));
input_indices.push_back(operand_indices.at(weight));
input_indices.push_back(operand_indices.at(bias));
@ -1443,16 +1496,8 @@ Status ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N
if (conv_2d || grouped_conv_2d) {
operationCode = conv_2d ? ANEURALNETWORKS_CONV_2D
: ANEURALNETWORKS_GROUPED_CONV_2D;
ORT_RETURN_IF_ERROR(shaper.Conv(input, weight,
onnx_pads, onnx_strides, onnx_dilations,
use_nchw,
output));
} else { // depthwise_conv_2d
operationCode = ANEURALNETWORKS_DEPTHWISE_CONV_2D;
ORT_RETURN_IF_ERROR(shaper.DepthwiseConv(input, weight,
onnx_pads, onnx_strides, onnx_dilations,
use_nchw,
output));
}
const OperandType output_operand_type(operand_types.at(input).type, shaper[output], y_scale, y_zero_point);
@ -1491,9 +1536,8 @@ Status CastOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N
return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Invalid cast to type: ", to);
}
std::vector<uint32_t> input_indices;
InlinedVector<uint32_t> input_indices;
input_indices.push_back(operand_indices.at(input));
ORT_RETURN_IF_ERROR(shaper.Identity(input, output));
const OperandType output_operand_type(type, shaper[output]);
ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_CAST, input_indices, {output},
{output_operand_type}));
@ -1523,7 +1567,7 @@ Status DepthToSpaceOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder,
int32_t blocksize = SafeInt<int32_t>(node_unit.GetNode().GetAttributes().at("blocksize").i());
std::vector<uint32_t> input_indices;
InlinedVector<uint32_t> input_indices;
input_indices.push_back(operand_indices.at(input));
ADD_SCALAR_OPERAND(model_builder, input_indices, blocksize);
@ -1531,8 +1575,6 @@ Status DepthToSpaceOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder,
// optional input to use nchw is available starting NNAPI feature level 3
ADD_SCALAR_OPERAND(model_builder, input_indices, use_nchw);
}
ORT_RETURN_IF_ERROR(shaper.DepthToSpace(input, blocksize, use_nchw, output));
const OperandType output_operand_type(operand_types.at(input).type, shaper[output]);
ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_DEPTH_TO_SPACE, input_indices, {output},
{output_operand_type}));
@ -1597,7 +1639,7 @@ Status SoftMaxOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, cons
const auto& output = node_unit.Outputs()[0].node_arg.Name();
float beta = 1.f;
std::vector<uint32_t> input_indices;
InlinedVector<uint32_t> input_indices;
input_indices.push_back(operand_indices.at(input));
ADD_SCALAR_OPERAND(model_builder, input_indices, beta);
@ -1606,7 +1648,6 @@ Status SoftMaxOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, cons
ADD_SCALAR_OPERAND(model_builder, input_indices, axis);
}
ORT_RETURN_IF_ERROR(shaper.Identity(input, output));
const OperandType output_operand_type(operand_types.at(input).type, shaper[output], y_scale, y_zero_point);
ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_SOFTMAX, input_indices,
{output}, {output_operand_type}));
@ -1632,10 +1673,9 @@ Status IdentityOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, con
const auto& input = node_unit.Inputs()[0].node_arg.Name();
const auto& output = node_unit.Outputs()[0].node_arg.Name();
std::vector<uint32_t> input_indices;
InlinedVector<uint32_t> input_indices;
input_indices.push_back(operand_indices.at(input)); // input
ORT_RETURN_IF_ERROR(shaper.Identity(input, output));
const OperandType output_operand_type(operand_types.at(input).type, shaper[output]);
model_builder.RegisterOperand(output, operand_indices.at(input), output_operand_type);
return Status::OK();
@ -1780,13 +1820,12 @@ Status GemmOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N
if (has_bias) {
const auto& bias = inputs[2].node_arg.Name();
if (!is_quant_gemm) {
// We need squeeze the input tensor to 1d if necessary
// We need to squeeze the input tensor to 1d if necessary
if (shaper[bias].size() > 1) {
std::string bias_squeezed = model_builder.GetUniqueName(node_unit.Name() + op + "_bias_squeezed");
// We will use squeeze all here
ORT_RETURN_IF_ERROR(AddSqueezeOp(model_builder, node_unit.Name(),
bias, bias_squeezed,
{} /* axes */));
ORT_RETURN_IF_ERROR(AddNnapiSqueeze(model_builder, node_unit.Name(),
bias, bias_squeezed, {} /* axes */));
bias_idx = operand_indices.at(bias_squeezed);
LOGS_DEFAULT(VERBOSE) << "GemmOpBuilder - Operand [" << bias << "] squeezed from "
<< Shape2String(shaper[bias])
@ -1833,14 +1872,13 @@ Status GemmOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N
bias_idx = operand_indices.at(bias);
}
std::vector<uint32_t> input_indices;
InlinedVector<uint32_t> input_indices;
input_indices.push_back(operand_indices.at(input1)); // A
input_indices.push_back(input_2_idx); // B
input_indices.push_back(bias_idx); // C
int32_t fuse_code = model_builder.FindActivation(node_unit);
ADD_SCALAR_OPERAND(model_builder, input_indices, fuse_code);
ORT_RETURN_IF_ERROR(shaper.FC(input1, input2, output));
const OperandType output_operand_type(operand_types.at(input1).type, shaper[output], y_scale, y_zero_point);
ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_FULLY_CONNECTED, input_indices,
{output}, {output_operand_type}));
@ -1900,7 +1938,7 @@ Status UnaryOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const
const auto& input = node_unit.Inputs()[0].node_arg.Name();
const auto& output = node_unit.Outputs()[0].node_arg.Name();
ORT_RETURN_IF_ERROR(shaper.Identity(input, output));
bool is_qlinear_sigmoid = op_type == "QLinearSigmoid";
int32_t op_code;
@ -1943,7 +1981,7 @@ Status UnaryOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const
y_zero_point = 0;
}
std::vector<uint32_t> input_indices;
InlinedVector<uint32_t> input_indices;
input_indices.push_back(operand_indices.at(input));
const OperandType output_operand_type(operand_types.at(input).type, shaper[output], y_scale, y_zero_point);
ORT_RETURN_IF_ERROR(model_builder.AddOperation(op_code, input_indices,
@ -1986,7 +2024,7 @@ Status ConcatOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const
NodeAttrHelper helper(node_unit);
const auto& inputs = node_unit.Inputs();
std::vector<uint32_t> input_indices;
InlinedVector<uint32_t> input_indices;
const auto& input0 = inputs[0].node_arg.Name();
const auto node_input_size = inputs.size();
@ -2062,7 +2100,6 @@ Status ConcatOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const
ADD_SCALAR_OPERAND(model_builder, input_indices, axis);
const auto& output = node_unit.Outputs()[0].node_arg.Name();
ORT_RETURN_IF_ERROR(shaper.Concat(input_names, axis, output));
OperandType output_operand_type(operand_types.at(input0).type, shaper[output], y_scale, y_zero_point);
ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_CONCATENATION, input_indices,
{output}, {output_operand_type}));
@ -2089,10 +2126,11 @@ void SqueezeOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const
Status SqueezeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const {
auto input = node_unit.Inputs()[0].node_arg.Name();
auto output = node_unit.Outputs()[0].node_arg.Name();
std::vector<int32_t> axes;
ORT_RETURN_IF_ERROR(GetAxesForSqueezeAndUnSqueeze(model_builder, node_unit, axes));
return AddSqueezeOp(model_builder, node_unit.Name(), input, node_unit.Outputs()[0].node_arg.Name(), axes);
return AddNnapiSqueeze(model_builder, node_unit.Name(), input, output, axes);
}
#pragma endregion
@ -2124,9 +2162,8 @@ Status QuantizeLinearOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builde
model_builder.GetInitializerTensors(), node_unit.Outputs()[0], node_unit.ModelPath(), scale, zero_point));
Type output_type = Type::TENSOR_QUANT8_ASYMM;
ORT_RETURN_IF_ERROR(shaper.Identity(input, output));
const OperandType output_operand_type(output_type, shaper[output], scale, zero_point);
std::vector<uint32_t> input_indices;
InlinedVector<uint32_t> input_indices;
input_indices.push_back(operand_indices.at(input));
ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_QUANTIZE, input_indices,
{output}, {output_operand_type}));
@ -2164,10 +2201,9 @@ Status DequantizeLinearOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_buil
ORT_RETURN_IF_ERROR(IsValidInputQuantizedType(model_builder, input, scale, zero_point));
ORT_RETURN_IF_ERROR(shaper.Identity(input, output));
const OperandType output_operand_type(Type::TENSOR_FLOAT32, shaper[output]);
std::vector<uint32_t> input_indices;
InlinedVector<uint32_t> input_indices;
input_indices.push_back(operand_indices.at(input));
ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_DEQUANTIZE, input_indices,
{output}, {output_operand_type}));
@ -2210,7 +2246,7 @@ Status LRNOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const No
const auto radius = (size - 1) / 2;
alpha /= size; // NNAPI's alpha is different than ONNX's alpha
std::vector<uint32_t> input_indices;
InlinedVector<uint32_t> input_indices;
input_indices.push_back(operand_indices.at(input));
ADD_SCALAR_OPERAND(model_builder, input_indices, radius);
ADD_SCALAR_OPERAND(model_builder, input_indices, bias);
@ -2226,7 +2262,6 @@ Status LRNOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const No
ADD_SCALAR_OPERAND(model_builder, input_indices, axis);
}
ORT_RETURN_IF_ERROR(shaper.Identity(input, output));
const OperandType output_operand_type(operand_types.at(input).type, shaper[output]);
ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_LOCAL_RESPONSE_NORMALIZATION, input_indices,
{output}, {output_operand_type}));
@ -2262,7 +2297,6 @@ Status ClipOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N
const auto& input = node_unit.Inputs()[0].node_arg.Name();
const auto& output = node_unit.Outputs()[0].node_arg.Name();
ORT_RETURN_IF_ERROR(shaper.Identity(input, output));
const OperandType output_operand_type(operand_types.at(input).type, shaper[output]);
if (Contains(model_builder.GetFusedActivations(), input)) {
@ -2284,7 +2318,7 @@ Status ClipOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N
return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "ClipOpBuilder, unsupported input [", min, ", ", max, "].",
"We should not reach here, ClipOpBuilder::IsOpSupportedImpl should have caught this.");
std::vector<uint32_t> input_indices;
InlinedVector<uint32_t> input_indices;
input_indices.push_back(operand_indices.at(input));
ORT_RETURN_IF_ERROR(model_builder.AddOperation(op_code, input_indices,
{output}, {output_operand_type}));
@ -2363,27 +2397,11 @@ Status ResizeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const
int h_idx = use_nchw ? 2 : 1;
int w_idx = use_nchw ? 3 : 2;
if (inputs.size() == 3) { // we are using scales
const auto& scales_name = inputs[2].node_arg.Name();
const auto& scales_tensor = *initializers.at(scales_name);
Initializer unpacked_tensor(scales_tensor);
auto scales_data = unpacked_tensor.DataAsSpan<float>();
ORT_RETURN_IF_ERROR(
shaper.ResizeUsingScales(input, scales_data[h_idx], scales_data[w_idx], use_nchw, output));
} else { // we are using sizes
const auto& sizes_name = inputs[3].node_arg.Name();
const auto& sizes_tensor = *initializers.at(sizes_name);
Initializer unpacked_tensor(sizes_tensor);
auto sizes_data = unpacked_tensor.DataAsSpan<int64_t>();
ORT_RETURN_IF_ERROR(
shaper.ResizeUsingOutputSizes(input, SafeInt<uint32_t>(sizes_data[h_idx]), SafeInt<uint32_t>(sizes_data[w_idx]), use_nchw, output));
}
const auto& output_shape = shaper[output];
int32_t output_h = output_shape[h_idx];
int32_t output_w = output_shape[w_idx];
std::vector<uint32_t> input_indices;
InlinedVector<uint32_t> input_indices;
input_indices.push_back(operand_indices.at(input));
ADD_SCALAR_OPERAND(model_builder, input_indices, output_w);
ADD_SCALAR_OPERAND(model_builder, input_indices, output_h);
@ -2475,7 +2493,7 @@ Status GatherOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const
int32_t rank = static_cast<int32_t>(shaper[input1].size());
int32_t axis = static_cast<int32_t>(HandleNegativeAxis(helper.Get("axis", 0), rank));
std::vector<uint32_t> input_indices;
InlinedVector<uint32_t> input_indices;
input_indices.push_back(operand_indices.at(input1));
ADD_SCALAR_OPERAND(model_builder, input_indices, axis);
@ -2510,9 +2528,8 @@ Status GatherOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const
ORT_RETURN_IF_ERROR(model_builder.AddOperandFromPersistMemoryBuffer(input2, indices.data(), indices_operand_type));
}
input_indices.push_back(operand_indices.at(input2));
ORT_RETURN_IF_ERROR(shaper.Gather(input1, input2, axis, output));
const OperandType output_operand_type(operand_types.at(input1).type, shaper[output]);
const OperandType output_operand_type(operand_types.at(input1).type, shaper[output]);
return model_builder.AddOperation(ANEURALNETWORKS_GATHER, input_indices,
{output}, {output_operand_type});
}
@ -2559,10 +2576,10 @@ class MinMaxOpBuilder : public BaseOpBuilder {
return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "MinMaxOpBuilder, unknown op: ", op_type);
}
std::vector<uint32_t> input_indices;
InlinedVector<uint32_t> input_indices;
input_indices.push_back(operand_indices.at(input1)); // input 1
input_indices.push_back(operand_indices.at(input2)); // input 2
ORT_RETURN_IF_ERROR(shaper.Eltwise(input1, input2, output));
const OperandType output_operand_type(operand_types.at(input1).type, shaper[output]);
ORT_RETURN_IF_ERROR(model_builder.AddOperation(op_code, input_indices,
{output}, {output_operand_type}));
@ -2593,11 +2610,11 @@ Status EluOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const No
const auto& operand_types(model_builder.GetOperandTypes());
const auto& input = node_unit.Inputs()[0].node_arg.Name();
const auto& output = node_unit.Outputs()[0].node_arg.Name();
ORT_RETURN_IF_ERROR(shaper.Identity(input, output));
const OperandType output_operand_type(operand_types.at(input).type, shaper[output]);
NodeAttrHelper helper(node_unit);
const auto alpha = helper.Get("alpha", 1.0f);
std::vector<uint32_t> input_indices;
InlinedVector<uint32_t> input_indices;
input_indices.push_back(operand_indices.at(input));
ADD_SCALAR_OPERAND(model_builder, input_indices, alpha);
return model_builder.AddOperation(ANEURALNETWORKS_ELU, input_indices,
@ -2634,7 +2651,7 @@ Status SliceOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const
const auto& operand_indices(model_builder.GetOperandIndices());
const auto& operand_types(model_builder.GetOperandTypes());
const auto& inputs = node_unit.Inputs();
const auto& input_shape = shaper[inputs[0].node_arg.Name()];
const auto input_shape = shaper[inputs[0].node_arg.Name()];
TensorShapeVector input_shape_64(input_shape.cbegin(), input_shape.cend());
SliceOp::PrepareForComputeMetadata compute_metadata(input_shape_64);
@ -2699,7 +2716,7 @@ Status SliceOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const
shaper.AddShape(output, nnapi_output_shape);
const OperandType output_operand_type(operand_types.at(input).type, shaper[output]);
std::vector<uint32_t> input_indices;
InlinedVector<uint32_t> input_indices;
input_indices.push_back(operand_indices.at(input));
// begin/end/strides of ANEURALNETWORKS_STRIDED_SLICE have the same shape
@ -2790,13 +2807,13 @@ void PadOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const Node
}
Status PadOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const {
auto& shaper = model_builder.GetShaper();
auto& shaper(model_builder.GetShaper());
const auto& operand_indices = model_builder.GetOperandIndices();
const auto& operand_types = model_builder.GetOperandTypes();
const auto& inputs = node_unit.Inputs();
const auto& outputs = node_unit.Outputs();
std::vector<uint32_t> input_indices{};
InlinedVector<uint32_t> input_indices{};
// `data` input
const auto& data = inputs[0].node_arg.Name();
@ -2805,7 +2822,7 @@ Status PadOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const No
// `pads` input
// convert from [begin_1, begin_2, ..., end_1, end_2, ...] to [begin_1, end_1, begin_2, end_2, ...]
// convert from int64_t to int32_t
const auto& data_shape = shaper[data];
const auto data_shape = shaper[data];
const uint32_t data_rank = SafeInt<uint32_t>(data_shape.size());
const auto& pads = inputs[1].node_arg.Name();
@ -2841,9 +2858,6 @@ Status PadOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const No
ADD_SCALAR_OPERAND(model_builder, input_indices, pad_value);
const auto& output = outputs[0].node_arg.Name();
ORT_RETURN_IF_ERROR(shaper.Pad(data, converted_pads_data, output));
const OperandType output_operand_type{operand_types.at(data).type, shaper[output]};
const auto op_code = ANEURALNETWORKS_PAD_V2;

149
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder_helpers.cc
View file

@ -4,10 +4,14 @@
#include "core/providers/nnapi/nnapi_builtin/builders/op_builder_helpers.h"
#include <algorithm>
#include <functional>
#include <utility>
#include "gsl/gsl"
#include "core/common/inlined_containers_fwd.h"
#include "core/common/safeint.h"
#include "core/common/span_utils.h"
#include "core/graph/node_arg.h"
#include "core/providers/common.h"
#include "core/providers/nnapi/nnapi_builtin/builders/helper.h"
@ -19,24 +23,36 @@ using android::nn::wrapper::OperandType, android::nn::wrapper::Type;
Status AddNnapiTranspose(ModelBuilder& model_builder,
const std::string& data_input,
const std::string& perm_input, const std::vector<int32_t>& perm,
const std::string& perm_input,
const gsl::span<const int32_t> perm,
const std::string& output) {
auto& shaper(model_builder.GetShaper());
// Calculate transpose output shape
const Shape input_dimen = shaper[data_input];
ORT_RETURN_IF_NOT(perm.size() == input_dimen.size(), "Invalid perm is given!");
size_t size = input_dimen.size();
Shape output_dimen(size);
for (size_t i = 0; i < size; i++)
output_dimen[i] = input_dimen[perm[i]];
shaper.AddShape(output, output_dimen);
const auto& operand_indices(model_builder.GetOperandIndices());
const auto& operand_types(model_builder.GetOperandTypes());
std::vector<uint32_t> input_indices;
InlinedVector<uint32_t> input_indices;
input_indices.push_back(operand_indices.at(data_input)); // input
Shape perm_dimen = {SafeInt<uint32_t>(perm.size())};
OperandType perm_operand_type(Type::TENSOR_INT32, perm_dimen);
ORT_RETURN_IF_ERROR(model_builder.AddOperandFromPersistMemoryBuffer(perm_input, perm.data(), perm_operand_type));
uint32_t perm_idx = operand_indices.at(perm_input);
input_indices.push_back(perm_idx); // permutation
ORT_RETURN_IF_ERROR(shaper.Transpose(data_input, perm, output));
OperandType output_operand_type = operand_types.at(data_input);
output_operand_type.SetDimensions(shaper[output]);
output_operand_type.SetDimensions(output_dimen);
return model_builder.AddOperation(ANEURALNETWORKS_TRANSPOSE, input_indices, {output},
{output_operand_type});
}
@ -44,18 +60,46 @@ Status AddNnapiTranspose(ModelBuilder& model_builder,
Status AddNnapiReshape(ModelBuilder& model_builder,
const std::string& data_input,
const std::string& shape_input, const std::vector<int32_t>& shape_value,
const std::string& output, const Shape* output_shape) {
if (output_shape == nullptr) {
auto& shaper = model_builder.GetShaper();
ORT_RETURN_IF_ERROR(shaper.Reshape(data_input, shape_value, output));
output_shape = &shaper[output];
const std::string& output) {
auto& shaper = model_builder.GetShaper();
// Calculate reshape output shape
const Shape input_dimen = shaper[data_input];
uint32_t input_size = ShapeSize(input_dimen);
Shape output_dimen(shape_value.size());
int64_t capacity = 1;
int unk_dim_idx = -1;
for (size_t i = 0; i < shape_value.size(); i++) {
int32_t dim_i = shape_value[i];
ORT_RETURN_IF_NOT(dim_i != 0, "NNAPI does not support 0 reshape dimension");
if (dim_i == -1) {
ORT_RETURN_IF_NOT(unk_dim_idx == -1, "Only one input dimension of Attr(shape) can be unknown!");
unk_dim_idx = static_cast<int>(i);
} else {
capacity *= dim_i;
output_dimen[i] = static_cast<uint32_t>(dim_i);
}
}
if (unk_dim_idx != -1) {
if (input_size == 0)
output_dimen[unk_dim_idx] = 0;
else
output_dimen[unk_dim_idx] = static_cast<uint32_t>(input_size / capacity);
capacity *= output_dimen[unk_dim_idx];
}
ORT_RETURN_IF_NOT(capacity == input_size, "Invalid shape is given!");
shaper.AddShape(output, output_dimen);
const auto& operand_indices = model_builder.GetOperandIndices();
const auto& operand_types = model_builder.GetOperandTypes();
// Add input
std::vector<uint32_t> input_indices;
InlinedVector<uint32_t> input_indices;
input_indices.push_back(operand_indices.at(data_input));
// Add new shape
@ -67,7 +111,7 @@ Status AddNnapiReshape(ModelBuilder& model_builder,
// For reshape, the output type should be the same as the input type except the shape is different
OperandType output_operand_type{operand_types.at(data_input)};
output_operand_type.SetDimensions(*output_shape);
output_operand_type.SetDimensions(output_dimen);
ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_RESHAPE,
input_indices, {output}, {output_operand_type}));
@ -85,12 +129,24 @@ Status AddNnapiSplit(ModelBuilder& model_builder,
const auto input_rank = shaper[input].size();
axis = static_cast<int32_t>(HandleNegativeAxis(axis, input_rank));
ORT_RETURN_IF_ERROR(shaper.Split(input, axis, outputs));
const auto count = gsl::narrow<int32_t>(outputs.size());
std::vector<uint32_t> input_indices;
// Calculate split output shape
{
const auto input_shape = shaper[input];
ORT_RETURN_IF_NOT(input_shape[axis] % count == 0,
"count [", count, "] does not evenly divide dimension ", axis, " [", input_shape[axis], "]");
Shape output_shape = input_shape;
output_shape[axis] = SafeInt<uint32_t>(input_shape[axis] / count);
for (const auto& output_name : outputs) {
shaper.AddShape(output_name, output_shape);
}
}
InlinedVector<uint32_t> input_indices;
input_indices.push_back(operand_indices.at(input));
ORT_RETURN_IF_ERROR(AddScalarOperand(model_builder, input_indices, axis));
const auto count = gsl::narrow<int32_t>(outputs.size());
ORT_RETURN_IF_ERROR(AddScalarOperand(model_builder, input_indices, count));
const OperandType& input_operand_type = operand_types.at(input);
@ -206,7 +262,7 @@ Status BuildBatchMatMul(ModelBuilder& model_builder, const NodeUnit& node_unit)
new_shape_i32.reserve(new_shape.size());
std::transform(new_shape.begin(), new_shape.end(), std::back_inserter(new_shape_i32),
[](uint32_t d) { return gsl::narrow<int32_t>(d); });
ORT_RETURN_IF_ERROR(AddNnapiReshape(model_builder, input, new_shape_name, new_shape_i32, output, nullptr));
ORT_RETURN_IF_ERROR(AddNnapiReshape(model_builder, input, new_shape_name, new_shape_i32, output));
return Status::OK();
};
@ -235,7 +291,8 @@ Status BuildBatchMatMul(ModelBuilder& model_builder, const NodeUnit& node_unit)
{
const std::string b_new_perm = model_builder.GetUniqueName(b + "/new_perm"),
b_transposed = model_builder.GetUniqueName(b + "/transposed");
ORT_RETURN_IF_ERROR(AddNnapiTranspose(model_builder, gemm_b_inputs.front(), b_new_perm, {0, 2, 1}, b_transposed));
ORT_RETURN_IF_ERROR(AddNnapiTranspose(model_builder, gemm_b_inputs.front(), b_new_perm,
AsSpan<int32_t>({0, 2, 1}), b_transposed));
gemm_b_inputs.front() = b_transposed;
}
@ -282,15 +339,18 @@ Status BuildBatchMatMul(ModelBuilder& model_builder, const NodeUnit& node_unit)
const auto& operand_indices = model_builder.GetOperandIndices();
const auto& operand_types = model_builder.GetOperandTypes();
auto& shaper = model_builder.GetShaper();
std::vector<uint32_t> input_indices;
InlinedVector<uint32_t> input_indices;
input_indices.push_back(operand_indices.at(a)); // A
input_indices.push_back(operand_indices.at(b_transposed)); // B'
input_indices.push_back(operand_indices.at(bias)); // C
int32_t fuse_code = ANEURALNETWORKS_FUSED_NONE;
ORT_RETURN_IF_ERROR(AddScalarOperand(model_builder, input_indices, fuse_code));
ORT_RETURN_IF_ERROR(shaper.FC(a, b_transposed, output));
const OperandType output_operand_type(operand_types.at(a).type, shaper[output]);
const auto a_dimen = shaper[a];
const auto b_transposed_dimen = shaper[b_transposed]; // num_units, input_size
Shape output_dimen{a_dimen[0], b_transposed_dimen[0]};
shaper.AddShape(output, output_dimen);
const OperandType output_operand_type(operand_types.at(a).type, output_dimen);
ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_FULLY_CONNECTED, input_indices,
{output}, {output_operand_type}));
return Status::OK();
@ -325,13 +385,35 @@ Status BuildBatchMatMul(ModelBuilder& model_builder, const NodeUnit& node_unit)
const auto& operand_indices = model_builder.GetOperandIndices();
const auto& operand_types = model_builder.GetOperandTypes();
auto& shaper = model_builder.GetShaper();
std::vector<uint32_t> input_indices;
InlinedVector<uint32_t> input_indices;
input_indices.reserve(inputs.size() + 1);
std::transform(inputs.begin(), inputs.end(), std::back_inserter(input_indices),
[&operand_indices](const std::string& input) { return operand_indices.at(input); });
const int32_t axis = 0;
ORT_RETURN_IF_ERROR(AddScalarOperand(model_builder, input_indices, axis));
ORT_RETURN_IF_ERROR(shaper.Concat(inputs, axis, output));
// Calculate_concat_output_shape
{
std::vector<Shape> dimens;
for (const auto& input_name : inputs) {
const Shape dimen = shaper[input_name];
dimens.push_back(dimen);
}
// If one of the inputs has dynamic shape (at axis), we will keep the dimen[axis] as 0 (dynamic)
auto output_dimen = dimens[0];
if (output_dimen[axis] != 0) {
for (size_t i = 1; i < dimens.size(); i++) {
if (dimens[i][axis] == 0) {
output_dimen[axis] = 0;
break;
}
output_dimen[axis] += dimens[i][axis];
}
}
shaper.AddShape(output, output_dimen);
}
OperandType output_operand_type = operand_types.at(inputs[0]);
output_operand_type.SetDimensions(shaper[output]);
ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_CONCATENATION,
@ -353,4 +435,27 @@ Status BuildBatchMatMul(ModelBuilder& model_builder, const NodeUnit& node_unit)
return Status::OK();
}
Status PerformBroadcasting(const Shape& shape1, const Shape& shape2, Shape& output_shape) {
bool shape1_is_bigger = shape1.size() >= shape2.size();
auto max_shape = shape1_is_bigger ? shape1 : shape2;
const auto& min_shape = shape1_is_bigger ? shape2 : shape1;
for (int i = gsl::narrow<int>(max_shape.size()) - 1,
j = gsl::narrow<int>(min_shape.size()) - 1;
i >= 0 && j >= 0;
i--, j--) {
int dim_max_shape = max_shape[i];
int dim_min_shape = min_shape[j];
if (dim_max_shape != dim_min_shape) {
ORT_RETURN_IF_NOT(dim_max_shape == 1 || dim_min_shape == 1,
"Dimensions are not compatible, dim1: ", dim_max_shape,
"dim2: ", dim_min_shape);
if (dim_max_shape == 1) {
max_shape[i] = dim_min_shape;
}
}
}
output_shape = std::move(max_shape);
return Status::OK();
}
} // namespace onnxruntime::nnapi::op_builder_helpers

10
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder_helpers.h
View file

@ -15,7 +15,7 @@ namespace onnxruntime::nnapi::op_builder_helpers {
// adds a scalar operand to the NNAPI model and appends its index to `input_indices`
template <typename T>
Status AddScalarOperand(ModelBuilder& model_builder, std::vector<uint32_t>& input_indices, T scalar_value) {
Status AddScalarOperand(ModelBuilder& model_builder, InlinedVector<uint32_t>& input_indices, T scalar_value) {
uint32_t index = 0;
ORT_RETURN_IF_ERROR(model_builder.AddOperandFromScalar(std::move(scalar_value), index));
input_indices.push_back(index);
@ -25,15 +25,14 @@ Status AddScalarOperand(ModelBuilder& model_builder, std::vector<uint32_t>& inpu
// adds ANEURALNETWORKS_TRANSPOSE operation
Status AddNnapiTranspose(ModelBuilder& model_builder,
const std::string& data_input,
const std::string& perm_input,
const std::vector<int32_t>& perm,
const std::string& perm_input, const gsl::span<const int32_t> perm,
const std::string& output);
// adds ANEURALNETWORKS_RESHAPE operation
Status AddNnapiReshape(ModelBuilder& model_builder,
const std::string& data_input,
const std::string& shape_input, const std::vector<int32_t>& shape_value,
const std::string& output, const Shape* output_shape);
const std::string& output);
// adds ANEURALNETWORKS_SPLIT operation
Status AddNnapiSplit(ModelBuilder& model_builder,
@ -48,4 +47,7 @@ bool IsSupportedBatchMatMul(const NodeUnit& node_unit, int32_t nnapi_feature_lev
// note: the pre-conditions of this function are checked in IsSupportedBatchMatMul()
Status BuildBatchMatMul(ModelBuilder& model_builder, const NodeUnit& node_unit);
// performs broadcasting operation on two shapes to make them compatible
Status PerformBroadcasting(const Shape& shape1, const Shape& shape2, Shape& output_shape);
} // namespace onnxruntime::nnapi::op_builder_helpers

558
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.cc
View file

@ -1,558 +0,0 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include "core/providers/common.h"
#include "shaper.h"
#include "helper.h"
namespace onnxruntime {
namespace nnapi {
std::pair<uint32_t, uint32_t> ComputeConvOutputShape(const uint32_t input_size_y, const uint32_t input_size_x,
const uint32_t weight_size_y, const uint32_t weight_size_x,
const std::vector<int32_t>& onnx_pads,
const std::vector<int32_t>& onnx_strides,
const std::vector<int32_t>& onnx_dilations) {
int32_t padding_top = onnx_pads[0];
int32_t padding_bottom = onnx_pads[2];
int32_t padding_left = onnx_pads[1];
int32_t padding_right = onnx_pads[3];
int32_t stride_y = onnx_strides[0];
int32_t stride_x = onnx_strides[1];
int32_t dilation_y = onnx_dilations[0];
int32_t dilation_x = onnx_dilations[1];
int64_t output_size_y =
0 == input_size_y
? 0
: onnxruntime::ComputeOutputShape(input_size_y,
stride_y, weight_size_y, dilation_y,
padding_top, padding_bottom);
int64_t output_size_x =
0 == input_size_x
? 0
: onnxruntime::ComputeOutputShape(input_size_x,
stride_x, weight_size_x, dilation_x,
padding_left, padding_right);
return std::make_pair(static_cast<uint32_t>(output_size_y), static_cast<uint32_t>(output_size_x));
}
#define SHAPER_FUNC(FUNC, ...) \
ORT_RETURN_IF_ERROR(FUNC##Impl(__VA_ARGS__)); \
shape_ops_.push_back( \
[__VA_ARGS__](Shaper& shaper) { \
return shaper.FUNC##Impl(__VA_ARGS__); \
}); \
return Status::OK();
Status Shaper::Conv(const std::string& input_name,
const std::string& weight_name,
const std::vector<int32_t>& onnx_pads,
const std::vector<int32_t>& onnx_strides,
const std::vector<int32_t>& onnx_dilations,
bool nchw,
const std::string& output_name) {
SHAPER_FUNC(Conv,
input_name, weight_name,
onnx_pads, onnx_strides, onnx_dilations,
nchw,
output_name);
}
Status Shaper::DepthwiseConv(const std::string& input_name,
const std::string& weight_name,
const std::vector<int32_t>& onnx_pads,
const std::vector<int32_t>& onnx_strides,
const std::vector<int32_t>& onnx_dilations,
bool nchw,
const std::string& output_name) {
SHAPER_FUNC(DepthwiseConv,
input_name, weight_name,
onnx_pads, onnx_strides, onnx_dilations,
nchw,
output_name);
}
Status Shaper::Pool(const std::string& input_name,
const std::vector<int32_t>& onnx_pads,
const std::vector<int32_t>& onnx_strides,
const std::vector<int32_t>& kernel_shape,
bool nchw,
const std::string& output_name) {
SHAPER_FUNC(Pool,
input_name,
onnx_pads, onnx_strides, kernel_shape,
nchw,
output_name);
}
Status Shaper::Reshape(const std::string& input_name,
const std::vector<int32_t>& shape,
const std::string& output_name) {
SHAPER_FUNC(Reshape, input_name, shape, output_name);
}
Status Shaper::Transpose(const std::string& input_name,
const std::vector<int32_t>& perm,
const std::string& output_name) {
SHAPER_FUNC(Transpose, input_name, perm, output_name);
}
Status Shaper::Eltwise(const std::string& input1_name,
const std::string& input2_name,
const std::string& output_name) {
SHAPER_FUNC(Eltwise, input1_name, input2_name, output_name);
}
Status Shaper::Identity(const std::string& input_name,
const std::string& output_name) {
SHAPER_FUNC(Identity, input_name, output_name);
}
Status Shaper::FC(const std::string& input1_name, const std::string& input2_name,
const std::string& output_name) {
SHAPER_FUNC(FC, input1_name, input2_name, output_name);
}
Status Shaper::Concat(const std::vector<std::string>& input_names,
const int32_t axis,
const std::string& output_name) {
SHAPER_FUNC(Concat, input_names, axis, output_name);
}
Status Shaper::Split(const std::string& input_name, int32_t axis,
const std::vector<std::string>& output_names) {
SHAPER_FUNC(Split, input_name, axis, output_names);
}
Status Shaper::Squeeze(const std::string& input_name,
const std::vector<int32_t>& axes,
const std::string& output_name) {
SHAPER_FUNC(Squeeze, input_name, axes, output_name);
}
Status Shaper::DepthToSpace(const std::string& input_name,
const int32_t blocksize,
bool nchw,
const std::string& output_name) {
SHAPER_FUNC(DepthToSpace, input_name, blocksize, nchw, output_name);
}
Status Shaper::Gather(const std::string& input_name1,
const std::string& input_name2,
const int32_t axis,
const std::string& output_name) {
SHAPER_FUNC(Gather, input_name1, input_name2, axis, output_name);
}
Status Shaper::ResizeUsingScales(const std::string& input_name,
const float scale_h, const float scale_w,
bool nchw,
const std::string& output_name) {
SHAPER_FUNC(ResizeUsingScales, input_name, scale_h, scale_w, nchw, output_name);
}
Status Shaper::ResizeUsingOutputSizes(const std::string& input_name,
const uint32_t output_h, const uint32_t output_w,
bool nchw,
const std::string& output_name) {
SHAPER_FUNC(ResizeUsingOutputSizes, input_name, output_h, output_w, nchw, output_name);
}
Status Shaper::Pad(const std::string& input_name,
const std::vector<int32_t>& pads,
const std::string& output_name) {
SHAPER_FUNC(Pad, input_name, pads, output_name);
}
#undef SHAPER_FUNC
Status Shaper::ConvImpl(const std::string& input_name,
const std::string& weight_name,
const std::vector<int32_t>& onnx_pads,
const std::vector<int32_t>& onnx_strides,
const std::vector<int32_t>& onnx_dilations,
bool nchw,
const std::string& output_name) {
const Shape& input_dimen = shape_map_.at(input_name);
const Shape& weight_dimen = shape_map_.at(weight_name); // num_output, height, width, num_input
const auto input_size_y = nchw ? input_dimen[2] : input_dimen[1];
const auto input_size_x = nchw ? input_dimen[3] : input_dimen[2];
const auto weight_size_y = weight_dimen[1];
const auto weight_size_x = weight_dimen[2];
uint32_t output_size_y, output_size_x;
std::tie(output_size_y, output_size_x) =
ComputeConvOutputShape(input_size_y, input_size_x,
weight_size_y, weight_size_x,
onnx_pads, onnx_strides, onnx_dilations);
Shape output_dimen;
if (nchw) {
output_dimen = {input_dimen[0], weight_dimen[0], output_size_y, output_size_x};
} else { // nhwc
output_dimen = {input_dimen[0], output_size_y, output_size_x, weight_dimen[0]};
}
shape_map_[output_name] = output_dimen;
return Status::OK();
}
Status Shaper::DepthwiseConvImpl(const std::string& input_name,
const std::string& weight_name,
const std::vector<int32_t>& onnx_pads,
const std::vector<int32_t>& onnx_strides,
const std::vector<int32_t>& onnx_dilations,
bool nchw,
const std::string& output_name) {
const Shape& input_dimen = shape_map_.at(input_name);
const Shape& weight_dimen = shape_map_.at(weight_name); // 1, height, width, num_output
const auto input_size_y = nchw ? input_dimen[2] : input_dimen[1];
const auto input_size_x = nchw ? input_dimen[3] : input_dimen[2];
const auto weight_size_y = weight_dimen[1];
const auto weight_size_x = weight_dimen[2];
uint32_t output_size_y, output_size_x;
std::tie(output_size_y, output_size_x) =
ComputeConvOutputShape(input_size_y, input_size_x,
weight_size_y, weight_size_x,
onnx_pads, onnx_strides, onnx_dilations);
Shape output_dimen;
if (nchw) {
output_dimen = {input_dimen[0], weight_dimen[3], output_size_y, output_size_x};
} else { // nhwc
output_dimen = {input_dimen[0], output_size_y, output_size_x, weight_dimen[3]};
}
shape_map_[output_name] = output_dimen;
return Status::OK();
}
Status Shaper::PoolImpl(const std::string& input_name,
const std::vector<int32_t>& onnx_pads,
const std::vector<int32_t>& onnx_strides,
const std::vector<int32_t>& kernel_shape,
bool nchw,
const std::string& output_name) {
const Shape& input_dimen = shape_map_.at(input_name);
const auto input_size_y = nchw ? input_dimen[2] : input_dimen[1];
const auto input_size_x = nchw ? input_dimen[3] : input_dimen[2];
const auto weight_size_y = kernel_shape[0];
const auto weight_size_x = kernel_shape[1];
uint32_t output_size_y, output_size_x;
std::tie(output_size_y, output_size_x) =
ComputeConvOutputShape(input_size_y, input_size_x,
weight_size_y, weight_size_x,
onnx_pads, onnx_strides, {1, 1} /* onnx_dilations */);
Shape output_dimen;
if (nchw) {
output_dimen = {input_dimen[0], input_dimen[1], output_size_y, output_size_x};
} else { // nhwc
output_dimen = {input_dimen[0], output_size_y, output_size_x, input_dimen[3]};
}
shape_map_[output_name] = output_dimen;
return Status::OK();
}
Status Shaper::ReshapeImpl(const std::string& input_name,
const std::vector<int32_t>& shape,
const std::string& output_name) {
const Shape& input_dimen = shape_map_.at(input_name);
uint32_t input_size = Product(input_dimen);
std::vector<uint32_t> output_dimen(shape.size());
int64_t capacity = 1;
int unk_dim_idx = -1;
for (size_t i = 0; i < shape.size(); i++) {
int32_t dim_i = shape[i];
ORT_RETURN_IF_NOT(dim_i != 0, "NNAPI does not support 0 reshape dimension");
if (dim_i == -1) {
ORT_RETURN_IF_NOT(unk_dim_idx == -1, "Only one input dimension of Attr(shape) can be unknown!");
unk_dim_idx = static_cast<int>(i);
} else {
capacity *= dim_i;
output_dimen[i] = static_cast<uint32_t>(dim_i);
}
}
if (unk_dim_idx != -1) {
if (input_size == 0)
output_dimen[unk_dim_idx] = 0;
else
output_dimen[unk_dim_idx] = static_cast<uint32_t>(input_size / capacity);
capacity *= output_dimen[unk_dim_idx];
}
ORT_RETURN_IF_NOT(capacity == input_size, "Invalid shape is given!");
shape_map_[output_name] = output_dimen;
return Status::OK();
}
Status Shaper::TransposeImpl(const std::string& input_name,
const std::vector<int32_t>& perm,
const std::string& output_name) {
const Shape& input_dimen = shape_map_.at(input_name);
ORT_RETURN_IF_NOT(perm.size() == input_dimen.size(), "Invalid perm is given!");
size_t size = input_dimen.size();
Shape output_dimen(size);
for (size_t i = 0; i < size; i++)
output_dimen[i] = input_dimen[perm[i]];
shape_map_[output_name] = output_dimen;
return Status::OK();
}
Status Shaper::EltwiseImpl(const std::string& input1_name,
const std::string& input2_name,
const std::string& output_name) {
const Shape& shape1 = shape_map_.at(input1_name);
const Shape& shape2 = shape_map_.at(input2_name);
// broadcasting support
bool shape1IsBigger = shape1.size() >= shape2.size();
auto max_shape = shape1IsBigger ? shape1 : shape2;
auto min_shape = shape1IsBigger ? shape2 : shape1;
for (int i = (int)max_shape.size() - 1,
j = (int)min_shape.size() - 1;
i >= 0 && j >= 0;
i--, j--) {
int dim_max_shape = max_shape[i];
int dim_min_shape = min_shape[j];
if (dim_max_shape != dim_min_shape) {
ORT_RETURN_IF_NOT(dim_max_shape == 1 || dim_min_shape == 1,
"Dimensions are not compatible, dim1: ", std::to_string(dim_max_shape),
"dim2: ", std::to_string(dim_min_shape));
}
if (dim_max_shape == 0 || dim_min_shape == 0) {
max_shape[i] = 0;
} else if (dim_max_shape < dim_min_shape) {
max_shape[i] = dim_min_shape;
}
}
shape_map_[output_name] = max_shape;
return Status::OK();
}
Status Shaper::IdentityImpl(const std::string& input_name,
const std::string& output_name) {
shape_map_[output_name] = shape_map_.at(input_name);
return Status::OK();
}
Status Shaper::FCImpl(const std::string& input1_name, const std::string& input2_name,
const std::string& output_name) {
// Currently we only support A*B'+C
const Shape& input1_dimen = shape_map_.at(input1_name);
const Shape& input2_dimen = shape_map_.at(input2_name); // num_units, input_size
Shape output_dimen{input1_dimen[0], input2_dimen[0]};
shape_map_[output_name] = output_dimen;
return Status::OK();
}
Status Shaper::ConcatImpl(const std::vector<std::string>& input_names,
const int32_t axis,
const std::string& output_name) {
std::vector<Shape> dimens;
for (const auto& input_name : input_names) {
const Shape& dimen = shape_map_.at(input_name);
dimens.push_back(dimen);
}
// If one of the inputs has dynamic shape (at axis), we will keep the dimen[axis] as 0 (dynamic)
auto output_dimen = dimens[0];
if (output_dimen[axis] != 0) {
for (size_t i = 1; i < dimens.size(); i++) {
if (dimens[i][axis] == 0) {
output_dimen[axis] = 0;
break;
}
output_dimen[axis] += dimens[i][axis];
}
}
shape_map_[output_name] = output_dimen;
return Status::OK();
}
Status Shaper::SplitImpl(const std::string& input_name, int32_t axis,
const std::vector<std::string>& output_names) {
const auto& input_shape = shape_map_.at(input_name);
const auto count = static_cast<uint32_t>(output_names.size());
ORT_RETURN_IF_NOT(input_shape[axis] % count == 0,
"count [", count, "] does not evenly divide dimension ", axis, " [", input_shape[axis], "]");
Shape output_shape = input_shape;
output_shape[axis] = input_shape[axis] / count;
for (const auto& output_name : output_names) {
shape_map_[output_name] = output_shape;
}
return Status::OK();
}
Status Shaper::SqueezeImpl(const std::string& input_name,
const std::vector<int32_t>& axes,
const std::string& output_name) {
const Shape& input_dimen = shape_map_.at(input_name);
int32_t input_size = static_cast<int32_t>(input_dimen.size());
std::unordered_set<int32_t> axes_to_be_squeezed;
// If the Op is squeezing all by not specifying axes, the axes is pre-populate
// with axes of all single dimensions by the caller
for (const auto& axis : axes)
axes_to_be_squeezed.insert(axis);
// Make output dimensions
std::vector<uint32_t> output_dimen;
output_dimen.reserve(input_size - axes_to_be_squeezed.size());
for (int32_t i = 0; i < input_size; i++) {
if (!Contains(axes_to_be_squeezed, i))
output_dimen.push_back(input_dimen[i]);
}
// In case of a tensor has all 1's in dimension such as {1,1,1,1} and gets squeezed all
// the output shape will be {1}
if (output_dimen.empty())
output_dimen.push_back(1);
shape_map_[output_name] = output_dimen;
return Status::OK();
}
Status Shaper::DepthToSpaceImpl(const std::string& input_name,
const int32_t blocksize,
bool nchw,
const std::string& output_name) {
const Shape& input_dimen = shape_map_.at(input_name);
// Make output dimensions
Shape output_dimen = shape_map_.at(input_name);
if (nchw) {
output_dimen[0] = input_dimen[0];
output_dimen[1] = input_dimen[1] / (blocksize * blocksize);
output_dimen[2] = input_dimen[2] * blocksize;
output_dimen[3] = input_dimen[3] * blocksize;
} else { // nhwc
output_dimen[0] = input_dimen[0];
output_dimen[1] = input_dimen[1] * blocksize;
output_dimen[2] = input_dimen[2] * blocksize;
output_dimen[3] = input_dimen[3] / (blocksize * blocksize);
}
shape_map_[output_name] = output_dimen;
return Status::OK();
}
Status Shaper::GatherImpl(const std::string& input_name1,
const std::string& input_name2,
const int32_t axis,
const std::string& output_name) {
const Shape& input_dimen = shape_map_.at(input_name1);
const Shape& indices_dimen = shape_map_.at(input_name2);
std::vector<uint32_t> output_dimen;
output_dimen.reserve(indices_dimen.size() + input_dimen.size() - 1);
// Calculate the output dim
for (int32_t i = 0; i < axis; ++i)
output_dimen.push_back(input_dimen[i]);
for (const auto dim : indices_dimen)
output_dimen.push_back(dim);
for (size_t i = axis + 1; i < input_dimen.size(); ++i)
output_dimen.push_back(input_dimen[i]);
shape_map_[output_name] = output_dimen;
return Status::OK();
}
Status Shaper::ResizeUsingScalesImpl(const std::string& input_name,
const float scale_h, const float scale_w,
bool nchw,
const std::string& output_name) {
Shape output_dimen = shape_map_.at(input_name);
if (nchw) {
output_dimen[2] = static_cast<uint32_t>(output_dimen[2] * scale_h);
output_dimen[3] = static_cast<uint32_t>(output_dimen[3] * scale_w);
} else { // nhwc
output_dimen[1] = static_cast<uint32_t>(output_dimen[1] * scale_h);
output_dimen[2] = static_cast<uint32_t>(output_dimen[2] * scale_w);
}
shape_map_[output_name] = output_dimen;
return Status::OK();
}
Status Shaper::ResizeUsingOutputSizesImpl(const std::string& input_name,
const uint32_t output_h, const uint32_t output_w,
bool nchw,
const std::string& output_name) {
Shape output_dimen = shape_map_.at(input_name);
if (nchw) {
output_dimen[2] = output_h;
output_dimen[3] = output_w;
} else { // nhwc
output_dimen[1] = output_h;
output_dimen[2] = output_w;
}
shape_map_[output_name] = output_dimen;
return Status::OK();
}
Status Shaper::PadImpl(const std::string& input_name,
const std::vector<int32_t>& pads,
const std::string& output_name) {
Shape padded_shape = shape_map_.at(input_name);
const size_t rank = padded_shape.size();
ORT_RETURN_IF_NOT(pads.size() == 2 * rank, "Expected 2*rank (", 2 * rank, ") pad values but got ", pads.size());
for (size_t i = 0; i < rank; ++i) {
padded_shape[i] += pads[2*i] + pads[2*i + 1];
}
shape_map_[output_name] = padded_shape;
return Status::OK();
}
void Shaper::AddShape(const std::string& name, const Shape& shape) {
shape_map_[name] = shape;
}
Status Shaper::UpdateShape(const std::string& name, const Shape& new_shape) {
const Shape& old_shape = shape_map_.at(name);
if (old_shape != new_shape) {
ORT_RETURN_IF_NOT(Product(old_shape) == 0 || !old_shape.empty(),
"The shape should be same size or old shape has size 0 (dynamic shape)");
shape_map_[name] = new_shape;
}
return Status::OK();
}
Status Shaper::UpdateDynamicDimensions() {
for (auto& shape_op : shape_ops_)
ORT_RETURN_IF_ERROR(shape_op(*this));
return Status::OK();
}
void Shaper::Clear() {
shape_map_.clear();
shape_ops_.clear();
}
} // namespace nnapi
} // namespace onnxruntime

147
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.h
View file

@ -6,149 +6,38 @@
#include <functional>
#include <string>
#include <unordered_map>
#include <vector>
#include "core/common/status.h"
#include "core/common/inlined_containers.h"
#include "core/providers/nnapi/nnapi_builtin/builders/helper.h"
namespace onnxruntime {
namespace nnapi {
class Shaper {
public:
using Shape = std::vector<uint32_t>;
using Shape = InlinedVector<uint32_t>;
void AddShape(const std::string& name, const Shape& shape);
inline const Shape& operator[](const std::string& key) const {
return shape_map_.at(key);
Shaper(const GraphViewer& graph_viewer) : graph_viewer_(&graph_viewer) {}
void AddShape(const std::string& name, const Shape& shape) {
shape_map_[name] = shape;
}
common::Status Conv(const std::string& input_name,
const std::string& weight_name,
const std::vector<int32_t>& onnx_pads,
const std::vector<int32_t>& onnx_strides,
const std::vector<int32_t>& onnx_dilations,
bool nchw,
const std::string& output_name);
inline Shape operator[](const std::string& key) const {
auto it = shape_map_.find(key);
if (it != shape_map_.end()) {
return it->second;
}
const auto shape = GetShapeInfoFromNodeArg(*graph_viewer_, key);
return shape;
}
common::Status DepthwiseConv(const std::string& input_name,
const std::string& weight_name,
const std::vector<int32_t>& onnx_pads,
const std::vector<int32_t>& onnx_strides,
const std::vector<int32_t>& onnx_dilations,
bool nchw,
const std::string& output_name);
common::Status Pool(const std::string& input_name,
const std::vector<int32_t>& onnx_pads,
const std::vector<int32_t>& onnx_strides,
const std::vector<int32_t>& kernel_shape,
bool nchw,
const std::string& output_name);
common::Status Reshape(const std::string& input_name, const std::vector<int32_t>& shape, const std::string& output_name);
common::Status Transpose(const std::string& input_name, const std::vector<int32_t>& perm, const std::string& output_name);
common::Status Eltwise(const std::string& input1_name, const std::string& input2_name, const std::string& output_name);
common::Status Identity(const std::string& input_name, const std::string& output_name);
common::Status FC(const std::string& input1_name, const std::string& input2_name, const std::string& output_name);
common::Status Concat(const std::vector<std::string>& input_names, const int32_t axis, const std::string& output_name);
common::Status Split(const std::string& input_name, int32_t axis, const std::vector<std::string>& output_names);
common::Status Squeeze(const std::string& input_name, const std::vector<int32_t>& axes, const std::string& output_name);
common::Status DepthToSpace(const std::string& input_name,
const int32_t blocksize,
bool nchw,
const std::string& output_name);
common::Status Gather(const std::string& input_name1,
const std::string& input_name2,
const int32_t axis,
const std::string& output_name);
common::Status ResizeUsingScales(const std::string& input_name,
const float scale_h, const float scale_w,
bool nchw,
const std::string& output_name);
common::Status ResizeUsingOutputSizes(const std::string& input_name,
const uint32_t output_h, const uint32_t output_w,
bool nchw,
const std::string& output_name);
// Note: `pads` should be in the layout expected by NNAPI, i.e., [begin_1, end_1, begin_2, end_2, ...]
common::Status Pad(const std::string& input_name,
const std::vector<int32_t>& pads,
const std::string& output_name);
// If the shape of certain input is dynamic
// Use the following 2 functions to update the particular shape
// and calculate the new output shape
// Only perform this when the NNAPI model is finalized!
common::Status UpdateShape(const std::string& name, const Shape& new_shape);
common::Status UpdateDynamicDimensions();
void Clear();
// Note: Original code to update shapes are removed for now due to lack of dynamic shape support in NNAPI EP.
// Can be added back and enhanced in the future if more support is available.
private:
common::Status ConvImpl(const std::string& input_name,
const std::string& weight_name,
const std::vector<int32_t>& onnx_pads,
const std::vector<int32_t>& onnx_strides,
const std::vector<int32_t>& onnx_dilations,
bool nchw,
const std::string& output_name);
common::Status DepthwiseConvImpl(const std::string& input_name,
const std::string& weight_name,
const std::vector<int32_t>& onnx_pads,
const std::vector<int32_t>& onnx_strides,
const std::vector<int32_t>& onnx_dilations,
bool nchw,
const std::string& output_name);
common::Status PoolImpl(const std::string& input_name,
const std::vector<int32_t>& onnx_pads,
const std::vector<int32_t>& onnx_strides,
const std::vector<int32_t>& kernel_shape,
bool nchw,
const std::string& output_name);
common::Status ReshapeImpl(const std::string& input_name, const std::vector<int32_t>& shape, const std::string& output_name);
common::Status TransposeImpl(const std::string& input_name, const std::vector<int32_t>& perm, const std::string& output_name);
common::Status EltwiseImpl(const std::string& input1_name, const std::string& input2_name, const std::string& output_name);
common::Status IdentityImpl(const std::string& input_name, const std::string& output_name);
common::Status FCImpl(const std::string& input1_name, const std::string& input2_name, const std::string& output_name);
common::Status ConcatImpl(const std::vector<std::string>& input_names, const int32_t axis, const std::string& output_name);
common::Status SplitImpl(const std::string& input_name, int32_t axis, const std::vector<std::string>& output_names);
common::Status SqueezeImpl(const std::string& input_names, const std::vector<int32_t>& axes, const std::string& output_name);
common::Status DepthToSpaceImpl(const std::string& input_names,
const int32_t blocksize,
bool nchw,
const std::string& output_name);
common::Status GatherImpl(const std::string& input_name1,
const std::string& input_name2,
const int32_t axis,
const std::string& output_name);
common::Status ResizeUsingScalesImpl(const std::string& input_name,
const float scale_h, const float scale_w,
bool nchw,
const std::string& output_name);
common::Status ResizeUsingOutputSizesImpl(const std::string& input_name,
const uint32_t output_h, const uint32_t output_w,
bool nchw,
const std::string& output_name);
common::Status PadImpl(const std::string& input_name,
const std::vector<int32_t>& pads,
const std::string& output_name);
std::unordered_map<std::string, Shape> shape_map_;
std::vector<std::function<common::Status(Shaper&)>> shape_ops_;
const GraphViewer* graph_viewer_;
};
} // namespace nnapi

26
onnxruntime/core/providers/nnapi/nnapi_builtin/model.cc
View file

@ -61,13 +61,14 @@ const android::nn::wrapper::OperandType& Model::GetInputType(const std::string&
return operand_types_.at(name);
}
android::nn::wrapper::OperandType Model::GetOutputType(const std::string& name, const Execution& execution) const {
const auto& nnapi_output_name = onnx_to_nnapi_output_map_.at(name);
const auto& output_type = operand_types_.at(nnapi_output_name);
android::nn::wrapper::OperandType type(
output_type.type, execution.GetShaper()[nnapi_output_name], output_type.operandType.scale, output_type.operandType.zeroPoint);
return type;
android::nn::wrapper::OperandType Model::GetOutputType(const std::string& name,
const Execution& /* execution */) const {
// Note: Before we validate if it's required to get Shaper from execution (if encounter dynamic shapes,
// shape can get updated during execution), we commented the usage here for now.
/* android::nn::wrapper::OperandType type(
output_type.type, execution.GetShaper()[nnapi_output_name], output_type.operandType.scale,
output_type.operandType.zeroPoint); */
return operand_types_.at(name);
}
void Model::SetInputMap(std::unordered_map<std::string, size_t>&& input_map) {
@ -99,7 +100,7 @@ Status Model::PrepareForExecution(std::unique_ptr<Execution>& execution) {
RETURN_STATUS_ON_ERROR(
nnapi_->ANeuralNetworksExecution_create(compilation_, &nnapi_execution));
execution.reset(new Execution(*nnapi_execution, shaper_));
execution.reset(new Execution(*nnapi_execution /*, shaper_*/));
return Status::OK();
}
@ -146,10 +147,9 @@ Model::NNMemory::NNMemory(const NnApi* /*nnapi*/, const char* name, size_t size)
#pragma region Execution
Execution::Execution(ANeuralNetworksExecution& execution, const Shaper& shaper)
Execution::Execution(ANeuralNetworksExecution& execution /*, const Shaper& shaper */)
: nnapi_(NnApiImplementation()),
execution_(&execution),
shaper_(shaper) {
execution_(&execution) {
}
Execution::~Execution() {
@ -160,10 +160,8 @@ Status Execution::SetInputBuffers(const std::vector<InputBuffer>& inputs) {
for (size_t i = 0; i < inputs.size(); i++) {
const auto& input(inputs[i]);
ORT_RETURN_IF_ERROR(SetInputBuffer(static_cast<int32_t>(i), input));
ORT_RETURN_IF_ERROR(shaper_.UpdateShape(input.name, input.type.dimensions));
}
ORT_RETURN_IF_ERROR(shaper_.UpdateDynamicDimensions());
return Status::OK();
}
@ -204,7 +202,7 @@ Status Execution::Predict(const std::vector<int32_t>& dynamic_outputs, std::vect
uint32_t output_rank = 0;
RETURN_STATUS_ON_ERROR(nnapi_->ANeuralNetworksExecution_getOutputOperandRank(execution_, i, &output_rank));
std::vector<uint32_t> output_shape(output_rank);
InlinedVector<uint32_t> output_shape(output_rank);
RETURN_STATUS_ON_ERROR(nnapi_->ANeuralNetworksExecution_getOutputOperandDimensions(execution_, i, output_shape.data()));
dynamic_output_shapes.push_back(output_shape);

16
onnxruntime/core/providers/nnapi/nnapi_builtin/model.h
View file

@ -57,6 +57,7 @@ class Model {
#endif
public:
Model();
~Model();
Model(const Model&) = delete;
Model& operator=(const Model&) = delete;
@ -70,7 +71,7 @@ class Model {
// Returns the data type and dimension of the given input/output
// Please note the output type will have updated dimensions
const android::nn::wrapper::OperandType& GetInputType(const std::string& name) const;
android::nn::wrapper::OperandType GetOutputType(const std::string& name, const Execution& execution) const;
android::nn::wrapper::OperandType GetOutputType(const std::string& name, const Execution& /* execution */) const;
// Set the mapping between input/output name and ORT kernel context
// input/output index, at execution time
@ -123,8 +124,6 @@ class Model {
std::unordered_map<std::string, android::nn::wrapper::OperandType> operand_types_;
std::unordered_set<std::string> scalar_outputs_;
Shaper shaper_;
std::unordered_map<std::string, size_t> input_map_;
std::unordered_map<std::string, size_t> output_map_;
@ -134,7 +133,6 @@ class Model {
OrtMutex mutex_;
Model();
void AddInput(const std::string& name, const android::nn::wrapper::OperandType& operand_type);
// It is possible that the actual output from NNAPI model is not the same as the name of
@ -145,8 +143,6 @@ class Model {
void AddScalarOutput(const std::string& output_name);
void SetShaper(const Shaper& shaper) { shaper_ = shaper; }
int32_t GetNNAPIFeatureLevel() const;
};
@ -165,12 +161,14 @@ class Execution {
};
public:
explicit Execution(ANeuralNetworksExecution& execution, const Shaper& shaper);
explicit Execution(ANeuralNetworksExecution& execution /* , const Shaper& shaper */);
~Execution();
Execution(const Execution&) = delete;
Execution& operator=(const Execution&) = delete;
const Shaper& GetShaper() const { return shaper_; }
// Before we validate if we actually need to keep a shaper instance for Execution (if we have dynamic shape
// outputs, shape can get updated during execution), we commented out Shaper here for now.
/* const Shaper& GetShaper() const { return shaper_; } */
// Set the input/output data buffers
// These need to be called before calling Predict()
@ -187,7 +185,7 @@ class Execution {
const NnApi* nnapi_{nullptr};
ANeuralNetworksExecution* execution_;
Shaper shaper_;
/* Shaper shaper_; */
};
} // namespace nnapi

6
onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc
View file

@ -208,7 +208,7 @@ static Status GetOutputBuffer(Ort::CustomOpApi& ort,
OrtKernelContext* context,
const nnapi::Model& model,
const std::string& output_name,
const std::vector<uint32_t>& output_shape,
const InlinedVector<uint32_t>& output_shape,
const android::nn::wrapper::Type output_type,
void** output_buffer) {
using namespace android::nn::wrapper;
@ -317,7 +317,7 @@ common::Status NnapiExecutionProvider::Compile(const std::vector<FusedNodeAndGra
auto input_idx = model->GetMappedInputIdx(input_name);
const OrtValue* input_tensor = ort.KernelContext_GetInput(context, input_idx);
auto* tensor_info = ort.GetTensorTypeAndShape(input_tensor);
std::vector<uint32_t> dimensions;
InlinedVector<uint32_t> dimensions;
for (const auto& dim : ort.GetTensorShape(tensor_info))
dimensions.push_back(static_cast<uint32_t>(dim));
@ -413,7 +413,7 @@ common::Status NnapiExecutionProvider::Compile(const std::vector<FusedNodeAndGra
}
ORT_RETURN_IF_ERROR(execution->SetOutputBuffers(outputs));
std::vector<std::vector<uint32_t>> dynamic_output_shapes;
std::vector<InlinedVector<uint32_t>> dynamic_output_shapes;
ORT_RETURN_IF_ERROR(
execution->Predict(dynamic_shape_output_indices, dynamic_output_shapes));

6
onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.cc
View file

@ -22,7 +22,7 @@ namespace android {
namespace nn {
namespace wrapper {
OperandType::OperandType(Type type, const std::vector<uint32_t>& d, float scale, int32_t zeroPoint)
OperandType::OperandType(Type type, const Shape& d, float scale, int32_t zeroPoint)
: type(type), dimensions(d) {
operandType = {
/*.type = */ static_cast<int32_t>(type),
@ -33,7 +33,7 @@ OperandType::OperandType(Type type, const std::vector<uint32_t>& d, float scale,
};
}
OperandType::OperandType(Type type, const std::vector<uint32_t>& d, SymmPerChannelQuantParams&& channelQuant)
OperandType::OperandType(Type type, const Shape& d, SymmPerChannelQuantParams&& channelQuant)
: type(type), dimensions(d), channelQuant(std::move(channelQuant)) {
operandType = {
/*.type = */ static_cast<int32_t>(type),
@ -101,7 +101,7 @@ size_t OperandType::GetOperandBlobByteSize() const {
return num_elements * GetElementByteSize();
}
void OperandType::SetDimensions(const std::vector<uint32_t>& d) {
void OperandType::SetDimensions(const Shape& d) {
dimensions = d;
operandType.dimensionCount = static_cast<uint32_t>(dimensions.size());
operandType.dimensions = dimensions.size() > 0 ? dimensions.data() : nullptr;

12
onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.h
View file

@ -22,6 +22,10 @@
#include "NeuralNetworksTypes.h"
#include "core/providers/nnapi/nnapi_builtin/builders/shaper.h"
using Shape = onnxruntime::nnapi::Shaper::Shape;
namespace android {
namespace nn {
namespace wrapper {
@ -122,11 +126,11 @@ struct SymmPerChannelQuantParams {
struct OperandType {
ANeuralNetworksOperandType operandType;
Type type;
std::vector<uint32_t> dimensions;
Shape dimensions;
std::optional<SymmPerChannelQuantParams> channelQuant;
explicit OperandType(Type type, const std::vector<uint32_t>& d, float scale = 0.0f, int32_t zeroPoint = 0);
explicit OperandType(Type type, const std::vector<uint32_t>& d, SymmPerChannelQuantParams&& channelQuant);
explicit OperandType(Type type, const Shape& d, float scale = 0.0f, int32_t zeroPoint = 0);
explicit OperandType(Type type, const Shape& d, SymmPerChannelQuantParams&& channelQuant);
OperandType(const OperandType& other);
OperandType& operator=(const OperandType& other);
@ -137,7 +141,7 @@ struct OperandType {
// Get the whole blob size in bytes
size_t GetOperandBlobByteSize() const;
void SetDimensions(const std::vector<uint32_t>& d);
void SetDimensions(const Shape& d);
operator ANeuralNetworksOperandType() const { return operandType; }
};