From 3810effe6e57544ca386046d7f56b7e8a281b590 Mon Sep 17 00:00:00 2001 From: Rachel Guo <35738743+YUNQIUGUO@users.noreply.github.com> Date: Wed, 21 Sep 2022 10:22:02 -0700 Subject: [PATCH] [NNAPI EP] Remove/Refactor shaper inference calculation code (#12618) **Description**: Describe your changes. As title. The purpose of this pr is to eliminate as much of repetitive shape inference code in nnapi ep shaper struct. For ops (mainly require composed operations) : -BatchNorm -Reshape -Squeeze (in one case of gemm operator) -BatchMatMul still contains some shape calculation impl/logic. Dynamic shape functions are not touched yet. **Motivation and Context** - Why is this change required? What problem does it solve? - If it fixes an open issue, please link to the issue here. Clean up redundant code as cpu shape inference impl for NNAPI EP. Get rid of the shape inference code in NNAPI EP by using the static shape info in output NodeArg. --- .../nnapi/nnapi_builtin/builders/helper.cc | 22 +- .../nnapi/nnapi_builtin/builders/helper.h | 9 +- .../nnapi_builtin/builders/model_builder.cc | 14 +- .../nnapi_builtin/builders/model_builder.h | 12 +- .../nnapi_builtin/builders/op_builder.cc | 262 ++++---- .../builders/op_builder_helpers.cc | 149 ++++- .../builders/op_builder_helpers.h | 10 +- .../nnapi/nnapi_builtin/builders/shaper.cc | 558 ------------------ .../nnapi/nnapi_builtin/builders/shaper.h | 147 +---- .../providers/nnapi/nnapi_builtin/model.cc | 26 +- .../providers/nnapi/nnapi_builtin/model.h | 16 +- .../nnapi_builtin/nnapi_execution_provider.cc | 6 +- .../nnapi_lib/NeuralNetworksWrapper.cc | 6 +- .../nnapi_lib/NeuralNetworksWrapper.h | 12 +- 14 files changed, 359 insertions(+), 890 deletions(-) delete mode 100644 onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.cc diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc index 09058eaad1..dbf38d557a 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc @@ -267,6 +267,26 @@ void GetFlattenOutputShape(const NodeUnit& node_unit, const Shape& input_shape, dim_2 = std::accumulate(input_shape.cbegin() + axis, input_shape.cend(), 1, std::multiplies()); } +Shape GetShapeInfoFromNodeArg(const GraphViewer& graph_viewer, const std::string& name) { + // can be applied to both input and output + Shape shape; + const auto* node_arg = graph_viewer.GetNodeArg(name); + const auto* shape_proto = node_arg->Shape(); + + shape.reserve(shape_proto->dim_size()); + for (const auto& shape_dim : shape_proto->dim()) { + // shape_dim here can possibly have dim_param, but as dynamic shape is not supported in NNAPI for now + // (checked already in BaseOpSupportChecker), call dim_value here only. + shape.push_back(SafeInt(shape_dim.dim_value())); + } + // If we have an empty shape, (scalar input), we need to make it as {1} as + // nnapi will treat empty shape as dynamic ranking and onnx does not support that + if (shape_proto->dim_size() == 0) { + shape.push_back(1); + } + return shape; +} + bool IsValidSupportedNodeGroup(const std::vector& supported_node_partition) { if (supported_node_partition.size() == 1) { const auto* node = supported_node_partition[0]; @@ -356,7 +376,7 @@ bool IsNodeSupportedInGroup(const NodeUnit& node_unit, const GraphViewer& graph_ return true; } -std::string Shape2String(const std::vector& shape) { +std::string Shape2String(const Shape& shape) { std::ostringstream os; os << "[ "; for (const auto& dim : shape) diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h index dce4c9088d..b1b294f4a0 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h @@ -5,6 +5,7 @@ #include #include +#include "core/common/inlined_containers.h" #include "core/graph/basic_types.h" #include "core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksTypes.h" @@ -24,7 +25,6 @@ namespace onnxruntime { -using Shape = std::vector; using InitializerMap = std::unordered_map; class GraphViewer; @@ -37,6 +37,8 @@ struct NodeUnitIODef; namespace nnapi { +using Shape = InlinedVector; + class IOpSupportChecker; struct OpSupportCheckParams; @@ -144,6 +146,9 @@ bool GetType(const NodeArg& node_arg, int32_t& type); // Get the output shape of Flatten Op void GetFlattenOutputShape(const NodeUnit& node_unit, const Shape& input_shape, int32_t& dim_1, int32_t& dim_2); +// Get the shape information from NodeArg +Shape GetShapeInfoFromNodeArg(const GraphViewer& graph_viewer, const std::string& name); + // If a node is supported by NNAPI bool IsNodeSupported(const NodeUnit& node_unit, const GraphViewer& graph_viewer, const OpSupportCheckParams& params); @@ -157,7 +162,7 @@ bool IsNodeSupportedInGroup(const NodeUnit& node_unit, const GraphViewer& graph_ bool IsValidSupportedNodeGroup(const std::vector& supported_node_group); // Get string representation of a Shape -std::string Shape2String(const std::vector& shape); +std::string Shape2String(const Shape& shape); uint32_t ShapeSize(const Shape& shape, size_t begin_idx, size_t end_idx); inline uint32_t ShapeSize(const Shape& shape) { diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc index 6164bf9743..d407751bca 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc @@ -25,7 +25,7 @@ namespace onnxruntime { namespace nnapi { ModelBuilder::ModelBuilder(const GraphViewer& graph_viewer) - : nnapi_(NnApiImplementation()), graph_viewer_(graph_viewer) {} + : nnapi_(NnApiImplementation()), graph_viewer_(graph_viewer), shaper_{graph_viewer} {} int32_t ModelBuilder::GetNNAPIFeatureLevel() const { return nnapi_ ? static_cast(nnapi_->nnapi_runtime_feature_level) : 0; @@ -34,7 +34,7 @@ int32_t ModelBuilder::GetNNAPIFeatureLevel() const { // Scalar operand is copied into the model, no need to persist #define DEFINE_ADD_OPERAND_FROM_SCALAR(scalar_type, op_type) \ Status ModelBuilder::AddOperandFromScalar(scalar_type value, uint32_t& index) { \ - OperandType operandType(Type::op_type, std::vector{}); \ + OperandType operandType(Type::op_type, InlinedVector{}); \ ORT_RETURN_IF_ERROR(AddNewNNAPIOperand(operandType, index)); \ RETURN_STATUS_ON_ERROR_WITH_NOTE( \ nnapi_->ANeuralNetworksModel_setOperandValue( \ @@ -54,7 +54,6 @@ void ModelBuilder::AddInitializerToSkip(const std::string& tensor_name) { } Status ModelBuilder::Prepare() { - nnapi_model_ = std::unique_ptr(new Model()); RETURN_STATUS_ON_ERROR(nnapi_->ANeuralNetworksModel_create(&nnapi_model_->model_)); ORT_RETURN_IF_ERROR(GetTargetDevices()); PreprocessNodeUnits(); @@ -65,7 +64,6 @@ Status ModelBuilder::Prepare() { ORT_RETURN_IF_ERROR(RegisterModelInputs()); ORT_RETURN_IF_ERROR(AddOperations()); ORT_RETURN_IF_ERROR(RegisterModelOutputs()); - RegisterModelShaper(); return Status::OK(); } @@ -388,10 +386,6 @@ Status ModelBuilder::RegisterModelOutputs() { return Status::OK(); } -void ModelBuilder::RegisterModelShaper() { - nnapi_model_->SetShaper(shaper_); -} - Status ModelBuilder::AddNewOperand(const std::string& name, const OperandType& operand_type, uint32_t& index) { @@ -512,10 +506,10 @@ Status ModelBuilder::AddOperations() { return Status::OK(); } -Status ModelBuilder::AddOperation(int op, const std::vector& input_indices, +Status ModelBuilder::AddOperation(int op, const InlinedVector& input_indices, const std::vector& output_names, const std::vector& output_types) { - std::vector output_indices; + InlinedVector output_indices; for (size_t i = 0; i < output_types.size(); i++) { uint32_t index = 0; ORT_RETURN_IF_ERROR(AddNewOperand(output_names[i], output_types[i], index)); diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h index 693583d307..5b4d1bd7e9 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h @@ -5,6 +5,7 @@ #include #include +#include "core/common/inlined_containers.h" #include "core/graph/basic_types.h" #include "core/providers/nnapi/nnapi_builtin/model.h" #include "core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.h" @@ -46,7 +47,7 @@ class ModelBuilder { int32_t GetNNAPIFeatureLevel() const; // Add an NNAPI operation (operator) - common::Status AddOperation(int op, const std::vector& input_indices, + common::Status AddOperation(int op, const InlinedVector& input_indices, const std::vector& output_names, const std::vector& output_types); @@ -117,7 +118,7 @@ class ModelBuilder { private: const NnApi* nnapi_{nullptr}; const GraphViewer& graph_viewer_; - std::unique_ptr nnapi_model_; + std::unique_ptr nnapi_model_{std::make_unique()}; uint32_t name_token_{0}; @@ -141,9 +142,8 @@ class ModelBuilder { std::unordered_map> op_support_checkers_; - - std::vector input_index_vec_; - std::vector output_index_vec_; + InlinedVector input_index_vec_; + InlinedVector output_index_vec_; // Contains all quantized operators' input and the NodeUnit(s) using the input // In the form of {input_name, [NodeUnit(s) using the input]} @@ -178,8 +178,6 @@ class ModelBuilder { common::Status RegisterModelInputs(); common::Status AddOperations(); common::Status RegisterModelOutputs(); - // After constructing the NNAPI model, will set the shape inferencing record to the Model - void RegisterModelShaper(); // Get all quantized inputs in the underlying graph_viewer void GetAllQuantizedOpInputs(); diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc index a3b0ea05c9..ffeeb14dce 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc @@ -5,20 +5,21 @@ #include #include +#include #include "core/common/logging/logging.h" #include "core/common/safeint.h" #include "core/framework/tensorprotoutils.h" #include "core/graph/graph_viewer.h" -#include "core/providers/common.h" -#include "core/providers/shared/utils/utils.h" -#include "core/providers/shared/node_unit/node_unit.h" -#include "core/providers/cpu/tensor/slice_helper.h" -#include "core/providers/nnapi/nnapi_builtin/builders/op_builder_helpers.h" -#include "helper.h" -#include "model_builder.h" -#include "op_support_checker.h" #include "core/optimizer/initializer.h" +#include "core/providers/common.h" +#include "core/providers/cpu/tensor/slice_helper.h" +#include "core/providers/nnapi/nnapi_builtin/builders/helper.h" +#include "core/providers/nnapi/nnapi_builtin/builders/model_builder.h" +#include "core/providers/nnapi/nnapi_builtin/builders/op_builder_helpers.h" +#include "core/providers/nnapi/nnapi_builtin/builders/op_support_checker.h" +#include "core/providers/shared/node_unit/node_unit.h" +#include "core/providers/shared/utils/utils.h" using namespace android::nn::wrapper; @@ -38,8 +39,7 @@ struct OpBuilderRegistrations { static Status AddBinaryOperator(int32_t op_type, ModelBuilder& model_builder, - const std::string& input1, - const std::string& input2, + const std::string& input1, const std::string& input2, bool add_activation, int32_t fuse_code, const std::string& output, @@ -49,7 +49,7 @@ static Status AddBinaryOperator(int32_t op_type, const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); - std::vector input_indices; + InlinedVector input_indices; input_indices.push_back(operand_indices.at(input1)); // input 1 input_indices.push_back(operand_indices.at(input2)); // input 2 @@ -57,7 +57,6 @@ static Status AddBinaryOperator(int32_t op_type, ADD_SCALAR_OPERAND(model_builder, input_indices, fuse_code); } - ORT_RETURN_IF_ERROR(shaper.Eltwise(input1, input2, output)); const OperandType output_operand_type(operand_types.at(input1).type, shaper[output], output_scale, output_zero_point); ORT_RETURN_IF_ERROR(model_builder.AddOperation(op_type, input_indices, @@ -65,10 +64,57 @@ static Status AddBinaryOperator(int32_t op_type, return Status::OK(); } -static Status AddSqueezeOp(ModelBuilder& model_builder, - const std::string& node_name, - const std::string& input, const std::string& output, - std::vector axes) { +static Status AddNnapiBatchNormalization(ModelBuilder& model_builder, + const std::string& input1, + const std::string& input2, + const std::string& input3, + const std::string& output1, + const std::string& output2, + int32_t fuse_code, + float output_scale = 0.0f, + int32_t output_zero_point = 0) { + auto& shaper(model_builder.GetShaper()); + const auto& operand_indices(model_builder.GetOperandIndices()); + const auto& operand_types(model_builder.GetOperandTypes()); + + // Add Nnapi Mul + InlinedVector input_indices; + input_indices.push_back(operand_indices.at(input1)); + input_indices.push_back(operand_indices.at(input2)); + + ADD_SCALAR_OPERAND(model_builder, input_indices, ANEURALNETWORKS_FUSED_NONE); + + const Shape& shape1 = shaper[input1]; + const Shape& shape2 = shaper[input2]; + Shape output1_shape; + // broadcasting support for eltwise shape operation + ORT_RETURN_IF_ERROR(op_builder_helpers::PerformBroadcasting(shape1, shape2, output1_shape)); + const OperandType output_operand_type(operand_types.at(input1).type, output1_shape, + output_scale, output_zero_point); + ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_MUL, input_indices, + {output1}, {output_operand_type})); + + // Add Nnapi Add + input_indices.clear(); + input_indices.push_back(operand_indices.at(output1)); + input_indices.push_back(operand_indices.at(input3)); + + ADD_SCALAR_OPERAND(model_builder, input_indices, fuse_code); + + const Shape& shape3 = shaper[input3]; + Shape output2_shape; + ORT_RETURN_IF_ERROR(op_builder_helpers::PerformBroadcasting(output1_shape, shape3, output2_shape)); + const OperandType output_operand_type2(operand_types.at(input3).type, output2_shape, + output_scale, output_zero_point); + ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_ADD, input_indices, + {output2}, {output_operand_type2})); + return Status::OK(); +} + +static Status AddNnapiSqueeze(ModelBuilder& model_builder, + const std::string& node_name, + const std::string& input, const std::string& output, + std::vector axes) { if (model_builder.GetNNAPIFeatureLevel() < ANEURALNETWORKS_FEATURE_LEVEL_2) { return ORT_MAKE_STATUS( ONNXRUNTIME, FAIL, "Squeeze is not supported on API level ", model_builder.GetNNAPIFeatureLevel()); @@ -78,7 +124,7 @@ static Status AddSqueezeOp(ModelBuilder& model_builder, const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); - const auto& input_shape(shaper[input]); + const auto input_shape = shaper[input]; auto input_dims = input_shape.size(); for (auto& axis : axes) { axis = static_cast(HandleNegativeAxis(axis, input_dims)); @@ -102,12 +148,34 @@ static Status AddSqueezeOp(ModelBuilder& model_builder, const OperandType axes_operand_type(Type::TENSOR_INT32, axes_dimen); ORT_RETURN_IF_ERROR(model_builder.AddOperandFromPersistMemoryBuffer(axes_name, axes.data(), axes_operand_type)); - std::vector input_indices; + InlinedVector input_indices; input_indices.push_back(operand_indices.at(input)); // input input_indices.push_back(operand_indices.at(axes_name)); // axes - ORT_RETURN_IF_ERROR(shaper.Squeeze(input, axes, output)); - const OperandType output_operand_type(operand_types.at(input).type, shaper[output]); + // Shape inference calculation for squeeze + int32_t input_size = static_cast(input_shape.size()); + std::unordered_set axes_to_be_squeezed; + + // If the Op is squeezing all by not specifying axes, the axes is pre-populate + // with axes of all single dimensions by the caller + for (const auto& axis : axes) + axes_to_be_squeezed.insert(axis); + + // Make output dimensions + InlinedVector output_dimen; + output_dimen.reserve(input_size - axes_to_be_squeezed.size()); + for (int32_t i = 0; i < input_size; i++) { + if (!Contains(axes_to_be_squeezed, i)) + output_dimen.push_back(input_shape[i]); + } + + // In case of a tensor has all 1's in dimension such as {1,1,1,1} and gets squeezed all + // the output shape will be {1} + if (output_dimen.empty()) + output_dimen.push_back(1); + + shaper.AddShape(output, output_dimen); + const OperandType output_operand_type(operand_types.at(input).type, output_dimen); ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_SQUEEZE, input_indices, {output}, {output_operand_type})); return Status::OK(); @@ -673,7 +741,6 @@ Status ReluOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N const auto& input = node_unit.Inputs()[0].node_arg.Name(); const auto& output = node_unit.Outputs()[0].node_arg.Name(); - ORT_RETURN_IF_ERROR(shaper.Identity(input, output)); const OperandType output_operand_type(operand_types.at(input).type, shaper[output]); // skip this relu if it is some op's fuse output @@ -681,7 +748,7 @@ Status ReluOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N LOGS_DEFAULT(VERBOSE) << "Relu Node [" << node_unit.Name() << "] fused"; model_builder.RegisterOperand(output, operand_indices.at(input), output_operand_type); } else { - std::vector input_indices; + InlinedVector input_indices; input_indices.push_back(operand_indices.at(input)); ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_RELU, input_indices, {output}, {output_operand_type})); @@ -721,6 +788,7 @@ Status TransposeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, co const auto& input = node_unit.Inputs()[0].node_arg.Name(); const auto& output = node_unit.Outputs()[0].node_arg.Name(); + NodeAttrHelper helper(node_unit); std::vector perm = helper.Get("perm", std::vector()); auto input_dims = static_cast(shaper[input].size()); @@ -741,7 +809,6 @@ Status TransposeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, co } std::string perm_name = model_builder.GetUniqueName(node_unit.Name() + input + "perm"); - ORT_RETURN_IF_ERROR(op_builder_helpers::AddNnapiTranspose(model_builder, input, perm_name, perm, output)); return Status::OK(); @@ -854,13 +921,15 @@ bool ReshapeOpBuilder::IsQuantizedOp(const NodeUnit& node_unit) const { const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); const auto& output = node_unit.Outputs()[0].node_arg.Name(); - ORT_RETURN_IF_ERROR(shaper.Reshape(input, shape, output)); - auto input_rank = shaper[input].size(); - auto output_rank = shaper[output].size(); + + const auto input_shape = shaper[input]; + const auto output_shape = shaper[output]; + const auto input_rank = input_shape.size(); + const auto output_rank = output_shape.size(); // For reshape, the output type should be the same as the input type except the shape is different auto output_operand_type = operand_types.at(input); - output_operand_type.SetDimensions(shaper[output]); + output_operand_type.SetDimensions(output_shape); // Since Reshape is not running using hardware in NNAPI for some CPU (e.g. Qualcomm SD for now) // We will try to see if we the skip the Reshape to prevent context switching between @@ -871,8 +940,7 @@ bool ReshapeOpBuilder::IsQuantizedOp(const NodeUnit& node_unit) const { } else { // We still need to perform a reshape here std::string shape_name = model_builder.GetUniqueName(node_unit.Name() + input + "newshape"); - ORT_RETURN_IF_ERROR(op_builder_helpers::AddNnapiReshape(model_builder, input, shape_name, shape, output, - &shaper[output])); + ORT_RETURN_IF_ERROR(op_builder_helpers::AddNnapiReshape(model_builder, input, shape_name, shape, output)); } return Status::OK(); @@ -881,7 +949,6 @@ bool ReshapeOpBuilder::IsQuantizedOp(const NodeUnit& node_unit) const { Status ReshapeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const { auto& shaper(model_builder.GetShaper()); const auto& initializers(model_builder.GetInitializerTensors()); - auto input = node_unit.Inputs()[0].node_arg.Name(); const auto& shape_tensor = *initializers.at(node_unit.Inputs()[1].node_arg.Name()); @@ -889,7 +956,7 @@ Status ReshapeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, cons auto raw_shape = unpacked_tensor.DataAsSpan(); const auto size = SafeInt(shape_tensor.dims()[0]); - Shape input_shape = shaper[input]; + const auto input_shape = shaper[input]; std::vector shape(size); for (uint32_t i = 0; i < size; i++) { int32_t dim = SafeInt(raw_shape[i]); @@ -929,7 +996,7 @@ void UnsqueezeOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, cons } Status UnsqueezeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const { - const auto& shaper(model_builder.GetShaper()); + auto& shaper(model_builder.GetShaper()); const auto& input = node_unit.Inputs()[0].node_arg.Name(); // NNAPI does not support unsqueeze, here we utilize unsqueeze's axes input to compute output shape @@ -937,7 +1004,7 @@ Status UnsqueezeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, co std::vector axes; ORT_RETURN_IF_ERROR(GetAxesForSqueezeAndUnSqueeze(model_builder, node_unit, axes)); - Shape input_shape = shaper[input]; + const auto input_shape = shaper[input]; auto input_dims = input_shape.size(); std::vector shape; const auto size = SafeInt(input_dims + axes.size()); // "output rank" @@ -986,6 +1053,7 @@ Status BatchNormalizationOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_bu // register a new operand with new shape const auto& input = inputs[0].node_arg.Name(); const auto& output = node_unit.Outputs()[0].node_arg.Name(); + const auto input_shape = shaper[input]; const auto& scale_tensor = *initializers.at(inputs[1].node_arg.Name()); const auto& bias_tensor = *initializers.at(inputs[2].node_arg.Name()); @@ -1029,11 +1097,12 @@ Status BatchNormalizationOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_bu // input shape {N, C} ==> tensor_a/b's shape {size} // input shape {N, C, H} ==> tensor_a/b's shape {size, 1} // input shape {N, C, H, W} ==> tensor_a/b's shape {size, 1, 1} - const auto input_rank = shaper[input].size(); + const auto input_rank = input_shape.size(); for (size_t i = 2; i < input_rank; i++) tensor_a_dimen.push_back(1); } + shaper.AddShape(input, input_shape); shaper.AddShape(tensor_a_name, tensor_a_dimen); shaper.AddShape(tensor_b_name, tensor_a_dimen); const OperandType a_operand_type(operand_types.at(input).type, tensor_a_dimen); @@ -1041,21 +1110,9 @@ Status BatchNormalizationOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_bu const OperandType b_operand_type(operand_types.at(input).type, tensor_a_dimen); ORT_RETURN_IF_ERROR(model_builder.AddOperandFromPersistMemoryBuffer(tensor_b_name, b.data(), b_operand_type)); - // Mul - ORT_RETURN_IF_ERROR(AddBinaryOperator(ANEURALNETWORKS_MUL, - model_builder, - input, tensor_a_name, - true /* add_activation */, ANEURALNETWORKS_FUSED_NONE, - tensor_imm_product_name)); - - // Add int32_t fuse_code = model_builder.FindActivation(node_unit); - ORT_RETURN_IF_ERROR(AddBinaryOperator(ANEURALNETWORKS_ADD, - model_builder, - tensor_imm_product_name, tensor_b_name, - true /* add_activation */, fuse_code, - output)); - + ORT_RETURN_IF_ERROR(AddNnapiBatchNormalization(model_builder, input, tensor_a_name, tensor_b_name, + tensor_imm_product_name, output, fuse_code)); return Status::OK(); } @@ -1124,7 +1181,7 @@ Status PoolOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N std::vector onnx_pads, onnx_strides, kernel_shape; bool use_auto_pad = false; int32_t nnapi_padding_code = ANEURALNETWORKS_PADDING_VALID; - const auto& input_shape = shaper[input]; + const auto input_shape = shaper[input]; if (is_average_pool || op_type == "MaxPool") { const auto auto_pad_type = StringToAutoPadType(helper.Get("auto_pad", "NOTSET")); kernel_shape = helper.Get("kernel_shape", std::vector{0, 0}); @@ -1171,7 +1228,7 @@ Status PoolOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N initializers, node_unit.Outputs()[0], node_unit.ModelPath(), y_scale, y_zero_point)); } - std::vector input_indices; + InlinedVector input_indices; input_indices.push_back(operand_indices.at(input)); if (use_auto_pad) { @@ -1193,10 +1250,6 @@ Status PoolOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N ADD_SCALAR_OPERAND(model_builder, input_indices, use_nchw); } - ORT_RETURN_IF_ERROR(shaper.Pool(input, - onnx_pads, onnx_strides, kernel_shape, - use_nchw, - output)); const OperandType output_operand_type(operand_types.at(input).type, shaper[output], y_scale, y_zero_point); ORT_RETURN_IF_ERROR(model_builder.AddOperation(op_code, input_indices, {output}, {output_operand_type})); @@ -1385,7 +1438,7 @@ Status ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N const auto auto_pad_type = StringToAutoPadType(helper.Get("auto_pad", "NOTSET")); bool use_auto_pad = false; int32_t nnapi_padding_code = ANEURALNETWORKS_PADDING_SAME; - const auto& input_shape = shaper[input]; + const auto input_shape = shaper[input]; const auto& kernel_shape = shaper[weight]; const auto weight_size_y = kernel_shape[1]; const auto weight_size_x = kernel_shape[2]; @@ -1395,7 +1448,7 @@ Status ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N auto_pad_type, use_nchw, onnx_pads, nnapi_padding_code, use_auto_pad)); - std::vector input_indices; + InlinedVector input_indices; input_indices.push_back(operand_indices.at(input)); input_indices.push_back(operand_indices.at(weight)); input_indices.push_back(operand_indices.at(bias)); @@ -1443,16 +1496,8 @@ Status ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N if (conv_2d || grouped_conv_2d) { operationCode = conv_2d ? ANEURALNETWORKS_CONV_2D : ANEURALNETWORKS_GROUPED_CONV_2D; - ORT_RETURN_IF_ERROR(shaper.Conv(input, weight, - onnx_pads, onnx_strides, onnx_dilations, - use_nchw, - output)); } else { // depthwise_conv_2d operationCode = ANEURALNETWORKS_DEPTHWISE_CONV_2D; - ORT_RETURN_IF_ERROR(shaper.DepthwiseConv(input, weight, - onnx_pads, onnx_strides, onnx_dilations, - use_nchw, - output)); } const OperandType output_operand_type(operand_types.at(input).type, shaper[output], y_scale, y_zero_point); @@ -1491,9 +1536,8 @@ Status CastOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Invalid cast to type: ", to); } - std::vector input_indices; + InlinedVector input_indices; input_indices.push_back(operand_indices.at(input)); - ORT_RETURN_IF_ERROR(shaper.Identity(input, output)); const OperandType output_operand_type(type, shaper[output]); ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_CAST, input_indices, {output}, {output_operand_type})); @@ -1523,7 +1567,7 @@ Status DepthToSpaceOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, int32_t blocksize = SafeInt(node_unit.GetNode().GetAttributes().at("blocksize").i()); - std::vector input_indices; + InlinedVector input_indices; input_indices.push_back(operand_indices.at(input)); ADD_SCALAR_OPERAND(model_builder, input_indices, blocksize); @@ -1531,8 +1575,6 @@ Status DepthToSpaceOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, // optional input to use nchw is available starting NNAPI feature level 3 ADD_SCALAR_OPERAND(model_builder, input_indices, use_nchw); } - - ORT_RETURN_IF_ERROR(shaper.DepthToSpace(input, blocksize, use_nchw, output)); const OperandType output_operand_type(operand_types.at(input).type, shaper[output]); ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_DEPTH_TO_SPACE, input_indices, {output}, {output_operand_type})); @@ -1597,7 +1639,7 @@ Status SoftMaxOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, cons const auto& output = node_unit.Outputs()[0].node_arg.Name(); float beta = 1.f; - std::vector input_indices; + InlinedVector input_indices; input_indices.push_back(operand_indices.at(input)); ADD_SCALAR_OPERAND(model_builder, input_indices, beta); @@ -1606,7 +1648,6 @@ Status SoftMaxOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, cons ADD_SCALAR_OPERAND(model_builder, input_indices, axis); } - ORT_RETURN_IF_ERROR(shaper.Identity(input, output)); const OperandType output_operand_type(operand_types.at(input).type, shaper[output], y_scale, y_zero_point); ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_SOFTMAX, input_indices, {output}, {output_operand_type})); @@ -1632,10 +1673,9 @@ Status IdentityOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, con const auto& input = node_unit.Inputs()[0].node_arg.Name(); const auto& output = node_unit.Outputs()[0].node_arg.Name(); - std::vector input_indices; + InlinedVector input_indices; input_indices.push_back(operand_indices.at(input)); // input - ORT_RETURN_IF_ERROR(shaper.Identity(input, output)); const OperandType output_operand_type(operand_types.at(input).type, shaper[output]); model_builder.RegisterOperand(output, operand_indices.at(input), output_operand_type); return Status::OK(); @@ -1780,13 +1820,12 @@ Status GemmOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N if (has_bias) { const auto& bias = inputs[2].node_arg.Name(); if (!is_quant_gemm) { - // We need squeeze the input tensor to 1d if necessary + // We need to squeeze the input tensor to 1d if necessary if (shaper[bias].size() > 1) { std::string bias_squeezed = model_builder.GetUniqueName(node_unit.Name() + op + "_bias_squeezed"); // We will use squeeze all here - ORT_RETURN_IF_ERROR(AddSqueezeOp(model_builder, node_unit.Name(), - bias, bias_squeezed, - {} /* axes */)); + ORT_RETURN_IF_ERROR(AddNnapiSqueeze(model_builder, node_unit.Name(), + bias, bias_squeezed, {} /* axes */)); bias_idx = operand_indices.at(bias_squeezed); LOGS_DEFAULT(VERBOSE) << "GemmOpBuilder - Operand [" << bias << "] squeezed from " << Shape2String(shaper[bias]) @@ -1833,14 +1872,13 @@ Status GemmOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N bias_idx = operand_indices.at(bias); } - std::vector input_indices; + InlinedVector input_indices; input_indices.push_back(operand_indices.at(input1)); // A input_indices.push_back(input_2_idx); // B input_indices.push_back(bias_idx); // C int32_t fuse_code = model_builder.FindActivation(node_unit); ADD_SCALAR_OPERAND(model_builder, input_indices, fuse_code); - ORT_RETURN_IF_ERROR(shaper.FC(input1, input2, output)); const OperandType output_operand_type(operand_types.at(input1).type, shaper[output], y_scale, y_zero_point); ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_FULLY_CONNECTED, input_indices, {output}, {output_operand_type})); @@ -1900,7 +1938,7 @@ Status UnaryOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const const auto& input = node_unit.Inputs()[0].node_arg.Name(); const auto& output = node_unit.Outputs()[0].node_arg.Name(); - ORT_RETURN_IF_ERROR(shaper.Identity(input, output)); + bool is_qlinear_sigmoid = op_type == "QLinearSigmoid"; int32_t op_code; @@ -1943,7 +1981,7 @@ Status UnaryOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const y_zero_point = 0; } - std::vector input_indices; + InlinedVector input_indices; input_indices.push_back(operand_indices.at(input)); const OperandType output_operand_type(operand_types.at(input).type, shaper[output], y_scale, y_zero_point); ORT_RETURN_IF_ERROR(model_builder.AddOperation(op_code, input_indices, @@ -1986,7 +2024,7 @@ Status ConcatOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeAttrHelper helper(node_unit); const auto& inputs = node_unit.Inputs(); - std::vector input_indices; + InlinedVector input_indices; const auto& input0 = inputs[0].node_arg.Name(); const auto node_input_size = inputs.size(); @@ -2062,7 +2100,6 @@ Status ConcatOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const ADD_SCALAR_OPERAND(model_builder, input_indices, axis); const auto& output = node_unit.Outputs()[0].node_arg.Name(); - ORT_RETURN_IF_ERROR(shaper.Concat(input_names, axis, output)); OperandType output_operand_type(operand_types.at(input0).type, shaper[output], y_scale, y_zero_point); ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_CONCATENATION, input_indices, {output}, {output_operand_type})); @@ -2089,10 +2126,11 @@ void SqueezeOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const Status SqueezeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const { auto input = node_unit.Inputs()[0].node_arg.Name(); + auto output = node_unit.Outputs()[0].node_arg.Name(); std::vector axes; ORT_RETURN_IF_ERROR(GetAxesForSqueezeAndUnSqueeze(model_builder, node_unit, axes)); - return AddSqueezeOp(model_builder, node_unit.Name(), input, node_unit.Outputs()[0].node_arg.Name(), axes); + return AddNnapiSqueeze(model_builder, node_unit.Name(), input, output, axes); } #pragma endregion @@ -2124,9 +2162,8 @@ Status QuantizeLinearOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builde model_builder.GetInitializerTensors(), node_unit.Outputs()[0], node_unit.ModelPath(), scale, zero_point)); Type output_type = Type::TENSOR_QUANT8_ASYMM; - ORT_RETURN_IF_ERROR(shaper.Identity(input, output)); const OperandType output_operand_type(output_type, shaper[output], scale, zero_point); - std::vector input_indices; + InlinedVector input_indices; input_indices.push_back(operand_indices.at(input)); ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_QUANTIZE, input_indices, {output}, {output_operand_type})); @@ -2164,10 +2201,9 @@ Status DequantizeLinearOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_buil ORT_RETURN_IF_ERROR(IsValidInputQuantizedType(model_builder, input, scale, zero_point)); - ORT_RETURN_IF_ERROR(shaper.Identity(input, output)); const OperandType output_operand_type(Type::TENSOR_FLOAT32, shaper[output]); - std::vector input_indices; + InlinedVector input_indices; input_indices.push_back(operand_indices.at(input)); ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_DEQUANTIZE, input_indices, {output}, {output_operand_type})); @@ -2210,7 +2246,7 @@ Status LRNOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const No const auto radius = (size - 1) / 2; alpha /= size; // NNAPI's alpha is different than ONNX's alpha - std::vector input_indices; + InlinedVector input_indices; input_indices.push_back(operand_indices.at(input)); ADD_SCALAR_OPERAND(model_builder, input_indices, radius); ADD_SCALAR_OPERAND(model_builder, input_indices, bias); @@ -2226,7 +2262,6 @@ Status LRNOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const No ADD_SCALAR_OPERAND(model_builder, input_indices, axis); } - ORT_RETURN_IF_ERROR(shaper.Identity(input, output)); const OperandType output_operand_type(operand_types.at(input).type, shaper[output]); ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_LOCAL_RESPONSE_NORMALIZATION, input_indices, {output}, {output_operand_type})); @@ -2262,7 +2297,6 @@ Status ClipOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N const auto& input = node_unit.Inputs()[0].node_arg.Name(); const auto& output = node_unit.Outputs()[0].node_arg.Name(); - ORT_RETURN_IF_ERROR(shaper.Identity(input, output)); const OperandType output_operand_type(operand_types.at(input).type, shaper[output]); if (Contains(model_builder.GetFusedActivations(), input)) { @@ -2284,7 +2318,7 @@ Status ClipOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "ClipOpBuilder, unsupported input [", min, ", ", max, "].", "We should not reach here, ClipOpBuilder::IsOpSupportedImpl should have caught this."); - std::vector input_indices; + InlinedVector input_indices; input_indices.push_back(operand_indices.at(input)); ORT_RETURN_IF_ERROR(model_builder.AddOperation(op_code, input_indices, {output}, {output_operand_type})); @@ -2363,27 +2397,11 @@ Status ResizeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const int h_idx = use_nchw ? 2 : 1; int w_idx = use_nchw ? 3 : 2; - if (inputs.size() == 3) { // we are using scales - const auto& scales_name = inputs[2].node_arg.Name(); - const auto& scales_tensor = *initializers.at(scales_name); - Initializer unpacked_tensor(scales_tensor); - auto scales_data = unpacked_tensor.DataAsSpan(); - ORT_RETURN_IF_ERROR( - shaper.ResizeUsingScales(input, scales_data[h_idx], scales_data[w_idx], use_nchw, output)); - } else { // we are using sizes - const auto& sizes_name = inputs[3].node_arg.Name(); - const auto& sizes_tensor = *initializers.at(sizes_name); - Initializer unpacked_tensor(sizes_tensor); - auto sizes_data = unpacked_tensor.DataAsSpan(); - ORT_RETURN_IF_ERROR( - shaper.ResizeUsingOutputSizes(input, SafeInt(sizes_data[h_idx]), SafeInt(sizes_data[w_idx]), use_nchw, output)); - } - const auto& output_shape = shaper[output]; int32_t output_h = output_shape[h_idx]; int32_t output_w = output_shape[w_idx]; - std::vector input_indices; + InlinedVector input_indices; input_indices.push_back(operand_indices.at(input)); ADD_SCALAR_OPERAND(model_builder, input_indices, output_w); ADD_SCALAR_OPERAND(model_builder, input_indices, output_h); @@ -2475,7 +2493,7 @@ Status GatherOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const int32_t rank = static_cast(shaper[input1].size()); int32_t axis = static_cast(HandleNegativeAxis(helper.Get("axis", 0), rank)); - std::vector input_indices; + InlinedVector input_indices; input_indices.push_back(operand_indices.at(input1)); ADD_SCALAR_OPERAND(model_builder, input_indices, axis); @@ -2510,9 +2528,8 @@ Status GatherOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const ORT_RETURN_IF_ERROR(model_builder.AddOperandFromPersistMemoryBuffer(input2, indices.data(), indices_operand_type)); } input_indices.push_back(operand_indices.at(input2)); - ORT_RETURN_IF_ERROR(shaper.Gather(input1, input2, axis, output)); - const OperandType output_operand_type(operand_types.at(input1).type, shaper[output]); + const OperandType output_operand_type(operand_types.at(input1).type, shaper[output]); return model_builder.AddOperation(ANEURALNETWORKS_GATHER, input_indices, {output}, {output_operand_type}); } @@ -2559,10 +2576,10 @@ class MinMaxOpBuilder : public BaseOpBuilder { return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "MinMaxOpBuilder, unknown op: ", op_type); } - std::vector input_indices; + InlinedVector input_indices; input_indices.push_back(operand_indices.at(input1)); // input 1 input_indices.push_back(operand_indices.at(input2)); // input 2 - ORT_RETURN_IF_ERROR(shaper.Eltwise(input1, input2, output)); + const OperandType output_operand_type(operand_types.at(input1).type, shaper[output]); ORT_RETURN_IF_ERROR(model_builder.AddOperation(op_code, input_indices, {output}, {output_operand_type})); @@ -2593,11 +2610,11 @@ Status EluOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const No const auto& operand_types(model_builder.GetOperandTypes()); const auto& input = node_unit.Inputs()[0].node_arg.Name(); const auto& output = node_unit.Outputs()[0].node_arg.Name(); - ORT_RETURN_IF_ERROR(shaper.Identity(input, output)); + const OperandType output_operand_type(operand_types.at(input).type, shaper[output]); NodeAttrHelper helper(node_unit); const auto alpha = helper.Get("alpha", 1.0f); - std::vector input_indices; + InlinedVector input_indices; input_indices.push_back(operand_indices.at(input)); ADD_SCALAR_OPERAND(model_builder, input_indices, alpha); return model_builder.AddOperation(ANEURALNETWORKS_ELU, input_indices, @@ -2634,7 +2651,7 @@ Status SliceOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); const auto& inputs = node_unit.Inputs(); - const auto& input_shape = shaper[inputs[0].node_arg.Name()]; + const auto input_shape = shaper[inputs[0].node_arg.Name()]; TensorShapeVector input_shape_64(input_shape.cbegin(), input_shape.cend()); SliceOp::PrepareForComputeMetadata compute_metadata(input_shape_64); @@ -2699,7 +2716,7 @@ Status SliceOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const shaper.AddShape(output, nnapi_output_shape); const OperandType output_operand_type(operand_types.at(input).type, shaper[output]); - std::vector input_indices; + InlinedVector input_indices; input_indices.push_back(operand_indices.at(input)); // begin/end/strides of ANEURALNETWORKS_STRIDED_SLICE have the same shape @@ -2790,13 +2807,13 @@ void PadOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const Node } Status PadOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const { - auto& shaper = model_builder.GetShaper(); + auto& shaper(model_builder.GetShaper()); const auto& operand_indices = model_builder.GetOperandIndices(); const auto& operand_types = model_builder.GetOperandTypes(); const auto& inputs = node_unit.Inputs(); const auto& outputs = node_unit.Outputs(); - std::vector input_indices{}; + InlinedVector input_indices{}; // `data` input const auto& data = inputs[0].node_arg.Name(); @@ -2805,7 +2822,7 @@ Status PadOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const No // `pads` input // convert from [begin_1, begin_2, ..., end_1, end_2, ...] to [begin_1, end_1, begin_2, end_2, ...] // convert from int64_t to int32_t - const auto& data_shape = shaper[data]; + const auto data_shape = shaper[data]; const uint32_t data_rank = SafeInt(data_shape.size()); const auto& pads = inputs[1].node_arg.Name(); @@ -2841,9 +2858,6 @@ Status PadOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const No ADD_SCALAR_OPERAND(model_builder, input_indices, pad_value); const auto& output = outputs[0].node_arg.Name(); - - ORT_RETURN_IF_ERROR(shaper.Pad(data, converted_pads_data, output)); - const OperandType output_operand_type{operand_types.at(data).type, shaper[output]}; const auto op_code = ANEURALNETWORKS_PAD_V2; diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder_helpers.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder_helpers.cc index 27e12ab649..898f0921cf 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder_helpers.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder_helpers.cc @@ -4,10 +4,14 @@ #include "core/providers/nnapi/nnapi_builtin/builders/op_builder_helpers.h" #include +#include +#include #include "gsl/gsl" +#include "core/common/inlined_containers_fwd.h" #include "core/common/safeint.h" +#include "core/common/span_utils.h" #include "core/graph/node_arg.h" #include "core/providers/common.h" #include "core/providers/nnapi/nnapi_builtin/builders/helper.h" @@ -19,24 +23,36 @@ using android::nn::wrapper::OperandType, android::nn::wrapper::Type; Status AddNnapiTranspose(ModelBuilder& model_builder, const std::string& data_input, - const std::string& perm_input, const std::vector& perm, + const std::string& perm_input, + const gsl::span perm, const std::string& output) { auto& shaper(model_builder.GetShaper()); + + // Calculate transpose output shape + + const Shape input_dimen = shaper[data_input]; + ORT_RETURN_IF_NOT(perm.size() == input_dimen.size(), "Invalid perm is given!"); + size_t size = input_dimen.size(); + Shape output_dimen(size); + for (size_t i = 0; i < size; i++) + output_dimen[i] = input_dimen[perm[i]]; + + shaper.AddShape(output, output_dimen); + const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); - std::vector input_indices; + InlinedVector input_indices; input_indices.push_back(operand_indices.at(data_input)); // input Shape perm_dimen = {SafeInt(perm.size())}; OperandType perm_operand_type(Type::TENSOR_INT32, perm_dimen); ORT_RETURN_IF_ERROR(model_builder.AddOperandFromPersistMemoryBuffer(perm_input, perm.data(), perm_operand_type)); uint32_t perm_idx = operand_indices.at(perm_input); - input_indices.push_back(perm_idx); // permutation - ORT_RETURN_IF_ERROR(shaper.Transpose(data_input, perm, output)); + OperandType output_operand_type = operand_types.at(data_input); - output_operand_type.SetDimensions(shaper[output]); + output_operand_type.SetDimensions(output_dimen); return model_builder.AddOperation(ANEURALNETWORKS_TRANSPOSE, input_indices, {output}, {output_operand_type}); } @@ -44,18 +60,46 @@ Status AddNnapiTranspose(ModelBuilder& model_builder, Status AddNnapiReshape(ModelBuilder& model_builder, const std::string& data_input, const std::string& shape_input, const std::vector& shape_value, - const std::string& output, const Shape* output_shape) { - if (output_shape == nullptr) { - auto& shaper = model_builder.GetShaper(); - ORT_RETURN_IF_ERROR(shaper.Reshape(data_input, shape_value, output)); - output_shape = &shaper[output]; + const std::string& output) { + auto& shaper = model_builder.GetShaper(); + // Calculate reshape output shape + + const Shape input_dimen = shaper[data_input]; + uint32_t input_size = ShapeSize(input_dimen); + Shape output_dimen(shape_value.size()); + + int64_t capacity = 1; + int unk_dim_idx = -1; + for (size_t i = 0; i < shape_value.size(); i++) { + int32_t dim_i = shape_value[i]; + ORT_RETURN_IF_NOT(dim_i != 0, "NNAPI does not support 0 reshape dimension"); + if (dim_i == -1) { + ORT_RETURN_IF_NOT(unk_dim_idx == -1, "Only one input dimension of Attr(shape) can be unknown!"); + unk_dim_idx = static_cast(i); + } else { + capacity *= dim_i; + output_dimen[i] = static_cast(dim_i); + } } + if (unk_dim_idx != -1) { + if (input_size == 0) + output_dimen[unk_dim_idx] = 0; + else + output_dimen[unk_dim_idx] = static_cast(input_size / capacity); + + capacity *= output_dimen[unk_dim_idx]; + } + + ORT_RETURN_IF_NOT(capacity == input_size, "Invalid shape is given!"); + + shaper.AddShape(output, output_dimen); + const auto& operand_indices = model_builder.GetOperandIndices(); const auto& operand_types = model_builder.GetOperandTypes(); // Add input - std::vector input_indices; + InlinedVector input_indices; input_indices.push_back(operand_indices.at(data_input)); // Add new shape @@ -67,7 +111,7 @@ Status AddNnapiReshape(ModelBuilder& model_builder, // For reshape, the output type should be the same as the input type except the shape is different OperandType output_operand_type{operand_types.at(data_input)}; - output_operand_type.SetDimensions(*output_shape); + output_operand_type.SetDimensions(output_dimen); ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_RESHAPE, input_indices, {output}, {output_operand_type})); @@ -85,12 +129,24 @@ Status AddNnapiSplit(ModelBuilder& model_builder, const auto input_rank = shaper[input].size(); axis = static_cast(HandleNegativeAxis(axis, input_rank)); - ORT_RETURN_IF_ERROR(shaper.Split(input, axis, outputs)); + const auto count = gsl::narrow(outputs.size()); - std::vector input_indices; + // Calculate split output shape + { + const auto input_shape = shaper[input]; + ORT_RETURN_IF_NOT(input_shape[axis] % count == 0, + "count [", count, "] does not evenly divide dimension ", axis, " [", input_shape[axis], "]"); + + Shape output_shape = input_shape; + output_shape[axis] = SafeInt(input_shape[axis] / count); + for (const auto& output_name : outputs) { + shaper.AddShape(output_name, output_shape); + } + } + + InlinedVector input_indices; input_indices.push_back(operand_indices.at(input)); ORT_RETURN_IF_ERROR(AddScalarOperand(model_builder, input_indices, axis)); - const auto count = gsl::narrow(outputs.size()); ORT_RETURN_IF_ERROR(AddScalarOperand(model_builder, input_indices, count)); const OperandType& input_operand_type = operand_types.at(input); @@ -206,7 +262,7 @@ Status BuildBatchMatMul(ModelBuilder& model_builder, const NodeUnit& node_unit) new_shape_i32.reserve(new_shape.size()); std::transform(new_shape.begin(), new_shape.end(), std::back_inserter(new_shape_i32), [](uint32_t d) { return gsl::narrow(d); }); - ORT_RETURN_IF_ERROR(AddNnapiReshape(model_builder, input, new_shape_name, new_shape_i32, output, nullptr)); + ORT_RETURN_IF_ERROR(AddNnapiReshape(model_builder, input, new_shape_name, new_shape_i32, output)); return Status::OK(); }; @@ -235,7 +291,8 @@ Status BuildBatchMatMul(ModelBuilder& model_builder, const NodeUnit& node_unit) { const std::string b_new_perm = model_builder.GetUniqueName(b + "/new_perm"), b_transposed = model_builder.GetUniqueName(b + "/transposed"); - ORT_RETURN_IF_ERROR(AddNnapiTranspose(model_builder, gemm_b_inputs.front(), b_new_perm, {0, 2, 1}, b_transposed)); + ORT_RETURN_IF_ERROR(AddNnapiTranspose(model_builder, gemm_b_inputs.front(), b_new_perm, + AsSpan({0, 2, 1}), b_transposed)); gemm_b_inputs.front() = b_transposed; } @@ -282,15 +339,18 @@ Status BuildBatchMatMul(ModelBuilder& model_builder, const NodeUnit& node_unit) const auto& operand_indices = model_builder.GetOperandIndices(); const auto& operand_types = model_builder.GetOperandTypes(); auto& shaper = model_builder.GetShaper(); - std::vector input_indices; + InlinedVector input_indices; input_indices.push_back(operand_indices.at(a)); // A input_indices.push_back(operand_indices.at(b_transposed)); // B' input_indices.push_back(operand_indices.at(bias)); // C int32_t fuse_code = ANEURALNETWORKS_FUSED_NONE; ORT_RETURN_IF_ERROR(AddScalarOperand(model_builder, input_indices, fuse_code)); - ORT_RETURN_IF_ERROR(shaper.FC(a, b_transposed, output)); - const OperandType output_operand_type(operand_types.at(a).type, shaper[output]); + const auto a_dimen = shaper[a]; + const auto b_transposed_dimen = shaper[b_transposed]; // num_units, input_size + Shape output_dimen{a_dimen[0], b_transposed_dimen[0]}; + shaper.AddShape(output, output_dimen); + const OperandType output_operand_type(operand_types.at(a).type, output_dimen); ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_FULLY_CONNECTED, input_indices, {output}, {output_operand_type})); return Status::OK(); @@ -325,13 +385,35 @@ Status BuildBatchMatMul(ModelBuilder& model_builder, const NodeUnit& node_unit) const auto& operand_indices = model_builder.GetOperandIndices(); const auto& operand_types = model_builder.GetOperandTypes(); auto& shaper = model_builder.GetShaper(); - std::vector input_indices; + InlinedVector input_indices; input_indices.reserve(inputs.size() + 1); std::transform(inputs.begin(), inputs.end(), std::back_inserter(input_indices), [&operand_indices](const std::string& input) { return operand_indices.at(input); }); const int32_t axis = 0; ORT_RETURN_IF_ERROR(AddScalarOperand(model_builder, input_indices, axis)); - ORT_RETURN_IF_ERROR(shaper.Concat(inputs, axis, output)); + + // Calculate_concat_output_shape + { + std::vector dimens; + for (const auto& input_name : inputs) { + const Shape dimen = shaper[input_name]; + dimens.push_back(dimen); + } + + // If one of the inputs has dynamic shape (at axis), we will keep the dimen[axis] as 0 (dynamic) + auto output_dimen = dimens[0]; + if (output_dimen[axis] != 0) { + for (size_t i = 1; i < dimens.size(); i++) { + if (dimens[i][axis] == 0) { + output_dimen[axis] = 0; + break; + } + output_dimen[axis] += dimens[i][axis]; + } + } + shaper.AddShape(output, output_dimen); + } + OperandType output_operand_type = operand_types.at(inputs[0]); output_operand_type.SetDimensions(shaper[output]); ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_CONCATENATION, @@ -353,4 +435,27 @@ Status BuildBatchMatMul(ModelBuilder& model_builder, const NodeUnit& node_unit) return Status::OK(); } +Status PerformBroadcasting(const Shape& shape1, const Shape& shape2, Shape& output_shape) { + bool shape1_is_bigger = shape1.size() >= shape2.size(); + auto max_shape = shape1_is_bigger ? shape1 : shape2; + const auto& min_shape = shape1_is_bigger ? shape2 : shape1; + for (int i = gsl::narrow(max_shape.size()) - 1, + j = gsl::narrow(min_shape.size()) - 1; + i >= 0 && j >= 0; + i--, j--) { + int dim_max_shape = max_shape[i]; + int dim_min_shape = min_shape[j]; + if (dim_max_shape != dim_min_shape) { + ORT_RETURN_IF_NOT(dim_max_shape == 1 || dim_min_shape == 1, + "Dimensions are not compatible, dim1: ", dim_max_shape, + "dim2: ", dim_min_shape); + if (dim_max_shape == 1) { + max_shape[i] = dim_min_shape; + } + } + } + output_shape = std::move(max_shape); + return Status::OK(); +} + } // namespace onnxruntime::nnapi::op_builder_helpers diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder_helpers.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder_helpers.h index 20ffaf230d..8500919a78 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder_helpers.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder_helpers.h @@ -15,7 +15,7 @@ namespace onnxruntime::nnapi::op_builder_helpers { // adds a scalar operand to the NNAPI model and appends its index to `input_indices` template -Status AddScalarOperand(ModelBuilder& model_builder, std::vector& input_indices, T scalar_value) { +Status AddScalarOperand(ModelBuilder& model_builder, InlinedVector& input_indices, T scalar_value) { uint32_t index = 0; ORT_RETURN_IF_ERROR(model_builder.AddOperandFromScalar(std::move(scalar_value), index)); input_indices.push_back(index); @@ -25,15 +25,14 @@ Status AddScalarOperand(ModelBuilder& model_builder, std::vector& inpu // adds ANEURALNETWORKS_TRANSPOSE operation Status AddNnapiTranspose(ModelBuilder& model_builder, const std::string& data_input, - const std::string& perm_input, - const std::vector& perm, + const std::string& perm_input, const gsl::span perm, const std::string& output); // adds ANEURALNETWORKS_RESHAPE operation Status AddNnapiReshape(ModelBuilder& model_builder, const std::string& data_input, const std::string& shape_input, const std::vector& shape_value, - const std::string& output, const Shape* output_shape); + const std::string& output); // adds ANEURALNETWORKS_SPLIT operation Status AddNnapiSplit(ModelBuilder& model_builder, @@ -48,4 +47,7 @@ bool IsSupportedBatchMatMul(const NodeUnit& node_unit, int32_t nnapi_feature_lev // note: the pre-conditions of this function are checked in IsSupportedBatchMatMul() Status BuildBatchMatMul(ModelBuilder& model_builder, const NodeUnit& node_unit); +// performs broadcasting operation on two shapes to make them compatible +Status PerformBroadcasting(const Shape& shape1, const Shape& shape2, Shape& output_shape); + } // namespace onnxruntime::nnapi::op_builder_helpers diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.cc deleted file mode 100644 index caa7835802..0000000000 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.cc +++ /dev/null @@ -1,558 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "core/providers/common.h" - -#include "shaper.h" -#include "helper.h" - -namespace onnxruntime { -namespace nnapi { - -std::pair ComputeConvOutputShape(const uint32_t input_size_y, const uint32_t input_size_x, - const uint32_t weight_size_y, const uint32_t weight_size_x, - const std::vector& onnx_pads, - const std::vector& onnx_strides, - const std::vector& onnx_dilations) { - int32_t padding_top = onnx_pads[0]; - int32_t padding_bottom = onnx_pads[2]; - int32_t padding_left = onnx_pads[1]; - int32_t padding_right = onnx_pads[3]; - int32_t stride_y = onnx_strides[0]; - int32_t stride_x = onnx_strides[1]; - int32_t dilation_y = onnx_dilations[0]; - int32_t dilation_x = onnx_dilations[1]; - - int64_t output_size_y = - 0 == input_size_y - ? 0 - : onnxruntime::ComputeOutputShape(input_size_y, - stride_y, weight_size_y, dilation_y, - padding_top, padding_bottom); - - int64_t output_size_x = - 0 == input_size_x - ? 0 - : onnxruntime::ComputeOutputShape(input_size_x, - stride_x, weight_size_x, dilation_x, - padding_left, padding_right); - - return std::make_pair(static_cast(output_size_y), static_cast(output_size_x)); -} - -#define SHAPER_FUNC(FUNC, ...) \ - ORT_RETURN_IF_ERROR(FUNC##Impl(__VA_ARGS__)); \ - shape_ops_.push_back( \ - [__VA_ARGS__](Shaper& shaper) { \ - return shaper.FUNC##Impl(__VA_ARGS__); \ - }); \ - return Status::OK(); - -Status Shaper::Conv(const std::string& input_name, - const std::string& weight_name, - const std::vector& onnx_pads, - const std::vector& onnx_strides, - const std::vector& onnx_dilations, - bool nchw, - const std::string& output_name) { - SHAPER_FUNC(Conv, - input_name, weight_name, - onnx_pads, onnx_strides, onnx_dilations, - nchw, - output_name); -} - -Status Shaper::DepthwiseConv(const std::string& input_name, - const std::string& weight_name, - const std::vector& onnx_pads, - const std::vector& onnx_strides, - const std::vector& onnx_dilations, - bool nchw, - const std::string& output_name) { - SHAPER_FUNC(DepthwiseConv, - input_name, weight_name, - onnx_pads, onnx_strides, onnx_dilations, - nchw, - output_name); -} - -Status Shaper::Pool(const std::string& input_name, - const std::vector& onnx_pads, - const std::vector& onnx_strides, - const std::vector& kernel_shape, - bool nchw, - const std::string& output_name) { - SHAPER_FUNC(Pool, - input_name, - onnx_pads, onnx_strides, kernel_shape, - nchw, - output_name); -} - -Status Shaper::Reshape(const std::string& input_name, - const std::vector& shape, - const std::string& output_name) { - SHAPER_FUNC(Reshape, input_name, shape, output_name); -} - -Status Shaper::Transpose(const std::string& input_name, - const std::vector& perm, - const std::string& output_name) { - SHAPER_FUNC(Transpose, input_name, perm, output_name); -} - -Status Shaper::Eltwise(const std::string& input1_name, - const std::string& input2_name, - const std::string& output_name) { - SHAPER_FUNC(Eltwise, input1_name, input2_name, output_name); -} - -Status Shaper::Identity(const std::string& input_name, - const std::string& output_name) { - SHAPER_FUNC(Identity, input_name, output_name); -} - -Status Shaper::FC(const std::string& input1_name, const std::string& input2_name, - const std::string& output_name) { - SHAPER_FUNC(FC, input1_name, input2_name, output_name); -} - -Status Shaper::Concat(const std::vector& input_names, - const int32_t axis, - const std::string& output_name) { - SHAPER_FUNC(Concat, input_names, axis, output_name); -} - -Status Shaper::Split(const std::string& input_name, int32_t axis, - const std::vector& output_names) { - SHAPER_FUNC(Split, input_name, axis, output_names); -} - -Status Shaper::Squeeze(const std::string& input_name, - const std::vector& axes, - const std::string& output_name) { - SHAPER_FUNC(Squeeze, input_name, axes, output_name); -} - -Status Shaper::DepthToSpace(const std::string& input_name, - const int32_t blocksize, - bool nchw, - const std::string& output_name) { - SHAPER_FUNC(DepthToSpace, input_name, blocksize, nchw, output_name); -} - -Status Shaper::Gather(const std::string& input_name1, - const std::string& input_name2, - const int32_t axis, - const std::string& output_name) { - SHAPER_FUNC(Gather, input_name1, input_name2, axis, output_name); -} - -Status Shaper::ResizeUsingScales(const std::string& input_name, - const float scale_h, const float scale_w, - bool nchw, - const std::string& output_name) { - SHAPER_FUNC(ResizeUsingScales, input_name, scale_h, scale_w, nchw, output_name); -} - -Status Shaper::ResizeUsingOutputSizes(const std::string& input_name, - const uint32_t output_h, const uint32_t output_w, - bool nchw, - const std::string& output_name) { - SHAPER_FUNC(ResizeUsingOutputSizes, input_name, output_h, output_w, nchw, output_name); -} - -Status Shaper::Pad(const std::string& input_name, - const std::vector& pads, - const std::string& output_name) { - SHAPER_FUNC(Pad, input_name, pads, output_name); -} - -#undef SHAPER_FUNC - -Status Shaper::ConvImpl(const std::string& input_name, - const std::string& weight_name, - const std::vector& onnx_pads, - const std::vector& onnx_strides, - const std::vector& onnx_dilations, - bool nchw, - const std::string& output_name) { - const Shape& input_dimen = shape_map_.at(input_name); - const Shape& weight_dimen = shape_map_.at(weight_name); // num_output, height, width, num_input - - const auto input_size_y = nchw ? input_dimen[2] : input_dimen[1]; - const auto input_size_x = nchw ? input_dimen[3] : input_dimen[2]; - const auto weight_size_y = weight_dimen[1]; - const auto weight_size_x = weight_dimen[2]; - - uint32_t output_size_y, output_size_x; - std::tie(output_size_y, output_size_x) = - ComputeConvOutputShape(input_size_y, input_size_x, - weight_size_y, weight_size_x, - onnx_pads, onnx_strides, onnx_dilations); - Shape output_dimen; - if (nchw) { - output_dimen = {input_dimen[0], weight_dimen[0], output_size_y, output_size_x}; - } else { // nhwc - output_dimen = {input_dimen[0], output_size_y, output_size_x, weight_dimen[0]}; - } - - shape_map_[output_name] = output_dimen; - return Status::OK(); -} - -Status Shaper::DepthwiseConvImpl(const std::string& input_name, - const std::string& weight_name, - const std::vector& onnx_pads, - const std::vector& onnx_strides, - const std::vector& onnx_dilations, - bool nchw, - const std::string& output_name) { - const Shape& input_dimen = shape_map_.at(input_name); - const Shape& weight_dimen = shape_map_.at(weight_name); // 1, height, width, num_output - - const auto input_size_y = nchw ? input_dimen[2] : input_dimen[1]; - const auto input_size_x = nchw ? input_dimen[3] : input_dimen[2]; - const auto weight_size_y = weight_dimen[1]; - const auto weight_size_x = weight_dimen[2]; - - uint32_t output_size_y, output_size_x; - std::tie(output_size_y, output_size_x) = - ComputeConvOutputShape(input_size_y, input_size_x, - weight_size_y, weight_size_x, - onnx_pads, onnx_strides, onnx_dilations); - - Shape output_dimen; - if (nchw) { - output_dimen = {input_dimen[0], weight_dimen[3], output_size_y, output_size_x}; - } else { // nhwc - output_dimen = {input_dimen[0], output_size_y, output_size_x, weight_dimen[3]}; - } - shape_map_[output_name] = output_dimen; - return Status::OK(); -} - -Status Shaper::PoolImpl(const std::string& input_name, - const std::vector& onnx_pads, - const std::vector& onnx_strides, - const std::vector& kernel_shape, - bool nchw, - const std::string& output_name) { - const Shape& input_dimen = shape_map_.at(input_name); - const auto input_size_y = nchw ? input_dimen[2] : input_dimen[1]; - const auto input_size_x = nchw ? input_dimen[3] : input_dimen[2]; - const auto weight_size_y = kernel_shape[0]; - const auto weight_size_x = kernel_shape[1]; - - uint32_t output_size_y, output_size_x; - std::tie(output_size_y, output_size_x) = - ComputeConvOutputShape(input_size_y, input_size_x, - weight_size_y, weight_size_x, - onnx_pads, onnx_strides, {1, 1} /* onnx_dilations */); - Shape output_dimen; - if (nchw) { - output_dimen = {input_dimen[0], input_dimen[1], output_size_y, output_size_x}; - } else { // nhwc - output_dimen = {input_dimen[0], output_size_y, output_size_x, input_dimen[3]}; - } - - shape_map_[output_name] = output_dimen; - return Status::OK(); -} - -Status Shaper::ReshapeImpl(const std::string& input_name, - const std::vector& shape, - const std::string& output_name) { - const Shape& input_dimen = shape_map_.at(input_name); - uint32_t input_size = Product(input_dimen); - std::vector output_dimen(shape.size()); - - int64_t capacity = 1; - int unk_dim_idx = -1; - for (size_t i = 0; i < shape.size(); i++) { - int32_t dim_i = shape[i]; - ORT_RETURN_IF_NOT(dim_i != 0, "NNAPI does not support 0 reshape dimension"); - if (dim_i == -1) { - ORT_RETURN_IF_NOT(unk_dim_idx == -1, "Only one input dimension of Attr(shape) can be unknown!"); - unk_dim_idx = static_cast(i); - } else { - capacity *= dim_i; - output_dimen[i] = static_cast(dim_i); - } - } - - if (unk_dim_idx != -1) { - if (input_size == 0) - output_dimen[unk_dim_idx] = 0; - else - output_dimen[unk_dim_idx] = static_cast(input_size / capacity); - - capacity *= output_dimen[unk_dim_idx]; - } - - ORT_RETURN_IF_NOT(capacity == input_size, "Invalid shape is given!"); - - shape_map_[output_name] = output_dimen; - return Status::OK(); -} - -Status Shaper::TransposeImpl(const std::string& input_name, - const std::vector& perm, - const std::string& output_name) { - const Shape& input_dimen = shape_map_.at(input_name); - - ORT_RETURN_IF_NOT(perm.size() == input_dimen.size(), "Invalid perm is given!"); - - size_t size = input_dimen.size(); - Shape output_dimen(size); - for (size_t i = 0; i < size; i++) - output_dimen[i] = input_dimen[perm[i]]; - - shape_map_[output_name] = output_dimen; - return Status::OK(); -} - -Status Shaper::EltwiseImpl(const std::string& input1_name, - const std::string& input2_name, - const std::string& output_name) { - const Shape& shape1 = shape_map_.at(input1_name); - const Shape& shape2 = shape_map_.at(input2_name); - - // broadcasting support - bool shape1IsBigger = shape1.size() >= shape2.size(); - auto max_shape = shape1IsBigger ? shape1 : shape2; - auto min_shape = shape1IsBigger ? shape2 : shape1; - for (int i = (int)max_shape.size() - 1, - j = (int)min_shape.size() - 1; - i >= 0 && j >= 0; - i--, j--) { - int dim_max_shape = max_shape[i]; - int dim_min_shape = min_shape[j]; - if (dim_max_shape != dim_min_shape) { - ORT_RETURN_IF_NOT(dim_max_shape == 1 || dim_min_shape == 1, - "Dimensions are not compatible, dim1: ", std::to_string(dim_max_shape), - "dim2: ", std::to_string(dim_min_shape)); - } - - if (dim_max_shape == 0 || dim_min_shape == 0) { - max_shape[i] = 0; - } else if (dim_max_shape < dim_min_shape) { - max_shape[i] = dim_min_shape; - } - } - - shape_map_[output_name] = max_shape; - return Status::OK(); -} - -Status Shaper::IdentityImpl(const std::string& input_name, - const std::string& output_name) { - shape_map_[output_name] = shape_map_.at(input_name); - return Status::OK(); -} - -Status Shaper::FCImpl(const std::string& input1_name, const std::string& input2_name, - const std::string& output_name) { - // Currently we only support A*B'+C - const Shape& input1_dimen = shape_map_.at(input1_name); - const Shape& input2_dimen = shape_map_.at(input2_name); // num_units, input_size - Shape output_dimen{input1_dimen[0], input2_dimen[0]}; - shape_map_[output_name] = output_dimen; - return Status::OK(); -} - -Status Shaper::ConcatImpl(const std::vector& input_names, - const int32_t axis, - const std::string& output_name) { - std::vector dimens; - for (const auto& input_name : input_names) { - const Shape& dimen = shape_map_.at(input_name); - dimens.push_back(dimen); - } - - // If one of the inputs has dynamic shape (at axis), we will keep the dimen[axis] as 0 (dynamic) - auto output_dimen = dimens[0]; - if (output_dimen[axis] != 0) { - for (size_t i = 1; i < dimens.size(); i++) { - if (dimens[i][axis] == 0) { - output_dimen[axis] = 0; - break; - } - output_dimen[axis] += dimens[i][axis]; - } - } - - shape_map_[output_name] = output_dimen; - return Status::OK(); -} - -Status Shaper::SplitImpl(const std::string& input_name, int32_t axis, - const std::vector& output_names) { - const auto& input_shape = shape_map_.at(input_name); - const auto count = static_cast(output_names.size()); - - ORT_RETURN_IF_NOT(input_shape[axis] % count == 0, - "count [", count, "] does not evenly divide dimension ", axis, " [", input_shape[axis], "]"); - - Shape output_shape = input_shape; - output_shape[axis] = input_shape[axis] / count; - - for (const auto& output_name : output_names) { - shape_map_[output_name] = output_shape; - } - - return Status::OK(); -} - -Status Shaper::SqueezeImpl(const std::string& input_name, - const std::vector& axes, - const std::string& output_name) { - const Shape& input_dimen = shape_map_.at(input_name); - int32_t input_size = static_cast(input_dimen.size()); - std::unordered_set axes_to_be_squeezed; - - // If the Op is squeezing all by not specifying axes, the axes is pre-populate - // with axes of all single dimensions by the caller - for (const auto& axis : axes) - axes_to_be_squeezed.insert(axis); - - // Make output dimensions - std::vector output_dimen; - output_dimen.reserve(input_size - axes_to_be_squeezed.size()); - for (int32_t i = 0; i < input_size; i++) { - if (!Contains(axes_to_be_squeezed, i)) - output_dimen.push_back(input_dimen[i]); - } - - // In case of a tensor has all 1's in dimension such as {1,1,1,1} and gets squeezed all - // the output shape will be {1} - if (output_dimen.empty()) - output_dimen.push_back(1); - - shape_map_[output_name] = output_dimen; - return Status::OK(); -} - -Status Shaper::DepthToSpaceImpl(const std::string& input_name, - const int32_t blocksize, - bool nchw, - const std::string& output_name) { - const Shape& input_dimen = shape_map_.at(input_name); - - // Make output dimensions - Shape output_dimen = shape_map_.at(input_name); - if (nchw) { - output_dimen[0] = input_dimen[0]; - output_dimen[1] = input_dimen[1] / (blocksize * blocksize); - output_dimen[2] = input_dimen[2] * blocksize; - output_dimen[3] = input_dimen[3] * blocksize; - } else { // nhwc - output_dimen[0] = input_dimen[0]; - output_dimen[1] = input_dimen[1] * blocksize; - output_dimen[2] = input_dimen[2] * blocksize; - output_dimen[3] = input_dimen[3] / (blocksize * blocksize); - } - - shape_map_[output_name] = output_dimen; - return Status::OK(); -} - -Status Shaper::GatherImpl(const std::string& input_name1, - const std::string& input_name2, - const int32_t axis, - const std::string& output_name) { - const Shape& input_dimen = shape_map_.at(input_name1); - const Shape& indices_dimen = shape_map_.at(input_name2); - - std::vector output_dimen; - output_dimen.reserve(indices_dimen.size() + input_dimen.size() - 1); - - // Calculate the output dim - for (int32_t i = 0; i < axis; ++i) - output_dimen.push_back(input_dimen[i]); - - for (const auto dim : indices_dimen) - output_dimen.push_back(dim); - - for (size_t i = axis + 1; i < input_dimen.size(); ++i) - output_dimen.push_back(input_dimen[i]); - - shape_map_[output_name] = output_dimen; - return Status::OK(); -} - -Status Shaper::ResizeUsingScalesImpl(const std::string& input_name, - const float scale_h, const float scale_w, - bool nchw, - const std::string& output_name) { - Shape output_dimen = shape_map_.at(input_name); - if (nchw) { - output_dimen[2] = static_cast(output_dimen[2] * scale_h); - output_dimen[3] = static_cast(output_dimen[3] * scale_w); - } else { // nhwc - output_dimen[1] = static_cast(output_dimen[1] * scale_h); - output_dimen[2] = static_cast(output_dimen[2] * scale_w); - } - shape_map_[output_name] = output_dimen; - return Status::OK(); -} - -Status Shaper::ResizeUsingOutputSizesImpl(const std::string& input_name, - const uint32_t output_h, const uint32_t output_w, - bool nchw, - const std::string& output_name) { - Shape output_dimen = shape_map_.at(input_name); - if (nchw) { - output_dimen[2] = output_h; - output_dimen[3] = output_w; - } else { // nhwc - output_dimen[1] = output_h; - output_dimen[2] = output_w; - } - shape_map_[output_name] = output_dimen; - return Status::OK(); -} - -Status Shaper::PadImpl(const std::string& input_name, - const std::vector& pads, - const std::string& output_name) { - Shape padded_shape = shape_map_.at(input_name); - const size_t rank = padded_shape.size(); - ORT_RETURN_IF_NOT(pads.size() == 2 * rank, "Expected 2*rank (", 2 * rank, ") pad values but got ", pads.size()); - for (size_t i = 0; i < rank; ++i) { - padded_shape[i] += pads[2*i] + pads[2*i + 1]; - } - shape_map_[output_name] = padded_shape; - return Status::OK(); -} - -void Shaper::AddShape(const std::string& name, const Shape& shape) { - shape_map_[name] = shape; -} - -Status Shaper::UpdateShape(const std::string& name, const Shape& new_shape) { - const Shape& old_shape = shape_map_.at(name); - if (old_shape != new_shape) { - ORT_RETURN_IF_NOT(Product(old_shape) == 0 || !old_shape.empty(), - "The shape should be same size or old shape has size 0 (dynamic shape)"); - - shape_map_[name] = new_shape; - } - - return Status::OK(); -} - -Status Shaper::UpdateDynamicDimensions() { - for (auto& shape_op : shape_ops_) - ORT_RETURN_IF_ERROR(shape_op(*this)); - - return Status::OK(); -} - -void Shaper::Clear() { - shape_map_.clear(); - shape_ops_.clear(); -} - -} // namespace nnapi -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.h index 31eb9aa9fa..2fbb2b221c 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.h @@ -6,149 +6,38 @@ #include #include #include -#include -#include "core/common/status.h" +#include "core/common/inlined_containers.h" +#include "core/providers/nnapi/nnapi_builtin/builders/helper.h" namespace onnxruntime { namespace nnapi { class Shaper { public: - using Shape = std::vector; + using Shape = InlinedVector; - void AddShape(const std::string& name, const Shape& shape); - inline const Shape& operator[](const std::string& key) const { - return shape_map_.at(key); + Shaper(const GraphViewer& graph_viewer) : graph_viewer_(&graph_viewer) {} + + void AddShape(const std::string& name, const Shape& shape) { + shape_map_[name] = shape; } - common::Status Conv(const std::string& input_name, - const std::string& weight_name, - const std::vector& onnx_pads, - const std::vector& onnx_strides, - const std::vector& onnx_dilations, - bool nchw, - const std::string& output_name); + inline Shape operator[](const std::string& key) const { + auto it = shape_map_.find(key); + if (it != shape_map_.end()) { + return it->second; + } + const auto shape = GetShapeInfoFromNodeArg(*graph_viewer_, key); + return shape; + } - common::Status DepthwiseConv(const std::string& input_name, - const std::string& weight_name, - const std::vector& onnx_pads, - const std::vector& onnx_strides, - const std::vector& onnx_dilations, - bool nchw, - const std::string& output_name); - - common::Status Pool(const std::string& input_name, - const std::vector& onnx_pads, - const std::vector& onnx_strides, - const std::vector& kernel_shape, - bool nchw, - const std::string& output_name); - - common::Status Reshape(const std::string& input_name, const std::vector& shape, const std::string& output_name); - - common::Status Transpose(const std::string& input_name, const std::vector& perm, const std::string& output_name); - - common::Status Eltwise(const std::string& input1_name, const std::string& input2_name, const std::string& output_name); - - common::Status Identity(const std::string& input_name, const std::string& output_name); - - common::Status FC(const std::string& input1_name, const std::string& input2_name, const std::string& output_name); - - common::Status Concat(const std::vector& input_names, const int32_t axis, const std::string& output_name); - - common::Status Split(const std::string& input_name, int32_t axis, const std::vector& output_names); - - common::Status Squeeze(const std::string& input_name, const std::vector& axes, const std::string& output_name); - - common::Status DepthToSpace(const std::string& input_name, - const int32_t blocksize, - bool nchw, - const std::string& output_name); - - common::Status Gather(const std::string& input_name1, - const std::string& input_name2, - const int32_t axis, - const std::string& output_name); - - common::Status ResizeUsingScales(const std::string& input_name, - const float scale_h, const float scale_w, - bool nchw, - const std::string& output_name); - common::Status ResizeUsingOutputSizes(const std::string& input_name, - const uint32_t output_h, const uint32_t output_w, - bool nchw, - const std::string& output_name); - - // Note: `pads` should be in the layout expected by NNAPI, i.e., [begin_1, end_1, begin_2, end_2, ...] - common::Status Pad(const std::string& input_name, - const std::vector& pads, - const std::string& output_name); - - // If the shape of certain input is dynamic - // Use the following 2 functions to update the particular shape - // and calculate the new output shape - // Only perform this when the NNAPI model is finalized! - common::Status UpdateShape(const std::string& name, const Shape& new_shape); - common::Status UpdateDynamicDimensions(); - - void Clear(); + // Note: Original code to update shapes are removed for now due to lack of dynamic shape support in NNAPI EP. + // Can be added back and enhanced in the future if more support is available. private: - common::Status ConvImpl(const std::string& input_name, - const std::string& weight_name, - const std::vector& onnx_pads, - const std::vector& onnx_strides, - const std::vector& onnx_dilations, - bool nchw, - const std::string& output_name); - - common::Status DepthwiseConvImpl(const std::string& input_name, - const std::string& weight_name, - const std::vector& onnx_pads, - const std::vector& onnx_strides, - const std::vector& onnx_dilations, - bool nchw, - const std::string& output_name); - - common::Status PoolImpl(const std::string& input_name, - const std::vector& onnx_pads, - const std::vector& onnx_strides, - const std::vector& kernel_shape, - bool nchw, - const std::string& output_name); - - common::Status ReshapeImpl(const std::string& input_name, const std::vector& shape, const std::string& output_name); - common::Status TransposeImpl(const std::string& input_name, const std::vector& perm, const std::string& output_name); - common::Status EltwiseImpl(const std::string& input1_name, const std::string& input2_name, const std::string& output_name); - common::Status IdentityImpl(const std::string& input_name, const std::string& output_name); - common::Status FCImpl(const std::string& input1_name, const std::string& input2_name, const std::string& output_name); - common::Status ConcatImpl(const std::vector& input_names, const int32_t axis, const std::string& output_name); - common::Status SplitImpl(const std::string& input_name, int32_t axis, const std::vector& output_names); - common::Status SqueezeImpl(const std::string& input_names, const std::vector& axes, const std::string& output_name); - common::Status DepthToSpaceImpl(const std::string& input_names, - const int32_t blocksize, - bool nchw, - const std::string& output_name); - common::Status GatherImpl(const std::string& input_name1, - const std::string& input_name2, - const int32_t axis, - const std::string& output_name); - common::Status ResizeUsingScalesImpl(const std::string& input_name, - const float scale_h, const float scale_w, - bool nchw, - const std::string& output_name); - common::Status ResizeUsingOutputSizesImpl(const std::string& input_name, - const uint32_t output_h, const uint32_t output_w, - bool nchw, - const std::string& output_name); - - common::Status PadImpl(const std::string& input_name, - const std::vector& pads, - const std::string& output_name); - std::unordered_map shape_map_; - std::vector> shape_ops_; + const GraphViewer* graph_viewer_; }; } // namespace nnapi diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/model.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/model.cc index 2201af8ce2..8d6b7ae544 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/model.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/model.cc @@ -61,13 +61,14 @@ const android::nn::wrapper::OperandType& Model::GetInputType(const std::string& return operand_types_.at(name); } -android::nn::wrapper::OperandType Model::GetOutputType(const std::string& name, const Execution& execution) const { - const auto& nnapi_output_name = onnx_to_nnapi_output_map_.at(name); - const auto& output_type = operand_types_.at(nnapi_output_name); - android::nn::wrapper::OperandType type( - output_type.type, execution.GetShaper()[nnapi_output_name], output_type.operandType.scale, output_type.operandType.zeroPoint); - - return type; +android::nn::wrapper::OperandType Model::GetOutputType(const std::string& name, + const Execution& /* execution */) const { + // Note: Before we validate if it's required to get Shaper from execution (if encounter dynamic shapes, + // shape can get updated during execution), we commented the usage here for now. + /* android::nn::wrapper::OperandType type( + output_type.type, execution.GetShaper()[nnapi_output_name], output_type.operandType.scale, + output_type.operandType.zeroPoint); */ + return operand_types_.at(name); } void Model::SetInputMap(std::unordered_map&& input_map) { @@ -99,7 +100,7 @@ Status Model::PrepareForExecution(std::unique_ptr& execution) { RETURN_STATUS_ON_ERROR( nnapi_->ANeuralNetworksExecution_create(compilation_, &nnapi_execution)); - execution.reset(new Execution(*nnapi_execution, shaper_)); + execution.reset(new Execution(*nnapi_execution /*, shaper_*/)); return Status::OK(); } @@ -146,10 +147,9 @@ Model::NNMemory::NNMemory(const NnApi* /*nnapi*/, const char* name, size_t size) #pragma region Execution -Execution::Execution(ANeuralNetworksExecution& execution, const Shaper& shaper) +Execution::Execution(ANeuralNetworksExecution& execution /*, const Shaper& shaper */) : nnapi_(NnApiImplementation()), - execution_(&execution), - shaper_(shaper) { + execution_(&execution) { } Execution::~Execution() { @@ -160,10 +160,8 @@ Status Execution::SetInputBuffers(const std::vector& inputs) { for (size_t i = 0; i < inputs.size(); i++) { const auto& input(inputs[i]); ORT_RETURN_IF_ERROR(SetInputBuffer(static_cast(i), input)); - ORT_RETURN_IF_ERROR(shaper_.UpdateShape(input.name, input.type.dimensions)); } - ORT_RETURN_IF_ERROR(shaper_.UpdateDynamicDimensions()); return Status::OK(); } @@ -204,7 +202,7 @@ Status Execution::Predict(const std::vector& dynamic_outputs, std::vect uint32_t output_rank = 0; RETURN_STATUS_ON_ERROR(nnapi_->ANeuralNetworksExecution_getOutputOperandRank(execution_, i, &output_rank)); - std::vector output_shape(output_rank); + InlinedVector output_shape(output_rank); RETURN_STATUS_ON_ERROR(nnapi_->ANeuralNetworksExecution_getOutputOperandDimensions(execution_, i, output_shape.data())); dynamic_output_shapes.push_back(output_shape); diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/model.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/model.h index 394c765d84..53884a3cb8 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/model.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/model.h @@ -57,6 +57,7 @@ class Model { #endif public: + Model(); ~Model(); Model(const Model&) = delete; Model& operator=(const Model&) = delete; @@ -70,7 +71,7 @@ class Model { // Returns the data type and dimension of the given input/output // Please note the output type will have updated dimensions const android::nn::wrapper::OperandType& GetInputType(const std::string& name) const; - android::nn::wrapper::OperandType GetOutputType(const std::string& name, const Execution& execution) const; + android::nn::wrapper::OperandType GetOutputType(const std::string& name, const Execution& /* execution */) const; // Set the mapping between input/output name and ORT kernel context // input/output index, at execution time @@ -123,8 +124,6 @@ class Model { std::unordered_map operand_types_; std::unordered_set scalar_outputs_; - Shaper shaper_; - std::unordered_map input_map_; std::unordered_map output_map_; @@ -134,7 +133,6 @@ class Model { OrtMutex mutex_; - Model(); void AddInput(const std::string& name, const android::nn::wrapper::OperandType& operand_type); // It is possible that the actual output from NNAPI model is not the same as the name of @@ -145,8 +143,6 @@ class Model { void AddScalarOutput(const std::string& output_name); - void SetShaper(const Shaper& shaper) { shaper_ = shaper; } - int32_t GetNNAPIFeatureLevel() const; }; @@ -165,12 +161,14 @@ class Execution { }; public: - explicit Execution(ANeuralNetworksExecution& execution, const Shaper& shaper); + explicit Execution(ANeuralNetworksExecution& execution /* , const Shaper& shaper */); ~Execution(); Execution(const Execution&) = delete; Execution& operator=(const Execution&) = delete; - const Shaper& GetShaper() const { return shaper_; } + // Before we validate if we actually need to keep a shaper instance for Execution (if we have dynamic shape + // outputs, shape can get updated during execution), we commented out Shaper here for now. + /* const Shaper& GetShaper() const { return shaper_; } */ // Set the input/output data buffers // These need to be called before calling Predict() @@ -187,7 +185,7 @@ class Execution { const NnApi* nnapi_{nullptr}; ANeuralNetworksExecution* execution_; - Shaper shaper_; + /* Shaper shaper_; */ }; } // namespace nnapi diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc index 782b384a63..0a80486db6 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc @@ -208,7 +208,7 @@ static Status GetOutputBuffer(Ort::CustomOpApi& ort, OrtKernelContext* context, const nnapi::Model& model, const std::string& output_name, - const std::vector& output_shape, + const InlinedVector& output_shape, const android::nn::wrapper::Type output_type, void** output_buffer) { using namespace android::nn::wrapper; @@ -317,7 +317,7 @@ common::Status NnapiExecutionProvider::Compile(const std::vectorGetMappedInputIdx(input_name); const OrtValue* input_tensor = ort.KernelContext_GetInput(context, input_idx); auto* tensor_info = ort.GetTensorTypeAndShape(input_tensor); - std::vector dimensions; + InlinedVector dimensions; for (const auto& dim : ort.GetTensorShape(tensor_info)) dimensions.push_back(static_cast(dim)); @@ -413,7 +413,7 @@ common::Status NnapiExecutionProvider::Compile(const std::vectorSetOutputBuffers(outputs)); - std::vector> dynamic_output_shapes; + std::vector> dynamic_output_shapes; ORT_RETURN_IF_ERROR( execution->Predict(dynamic_shape_output_indices, dynamic_output_shapes)); diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.cc index 9d18c8d228..1728640b9b 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.cc @@ -22,7 +22,7 @@ namespace android { namespace nn { namespace wrapper { -OperandType::OperandType(Type type, const std::vector& d, float scale, int32_t zeroPoint) +OperandType::OperandType(Type type, const Shape& d, float scale, int32_t zeroPoint) : type(type), dimensions(d) { operandType = { /*.type = */ static_cast(type), @@ -33,7 +33,7 @@ OperandType::OperandType(Type type, const std::vector& d, float scale, }; } -OperandType::OperandType(Type type, const std::vector& d, SymmPerChannelQuantParams&& channelQuant) +OperandType::OperandType(Type type, const Shape& d, SymmPerChannelQuantParams&& channelQuant) : type(type), dimensions(d), channelQuant(std::move(channelQuant)) { operandType = { /*.type = */ static_cast(type), @@ -101,7 +101,7 @@ size_t OperandType::GetOperandBlobByteSize() const { return num_elements * GetElementByteSize(); } -void OperandType::SetDimensions(const std::vector& d) { +void OperandType::SetDimensions(const Shape& d) { dimensions = d; operandType.dimensionCount = static_cast(dimensions.size()); operandType.dimensions = dimensions.size() > 0 ? dimensions.data() : nullptr; diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.h index a01f36d981..88eb94f831 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.h @@ -22,6 +22,10 @@ #include "NeuralNetworksTypes.h" +#include "core/providers/nnapi/nnapi_builtin/builders/shaper.h" + +using Shape = onnxruntime::nnapi::Shaper::Shape; + namespace android { namespace nn { namespace wrapper { @@ -122,11 +126,11 @@ struct SymmPerChannelQuantParams { struct OperandType { ANeuralNetworksOperandType operandType; Type type; - std::vector dimensions; + Shape dimensions; std::optional channelQuant; - explicit OperandType(Type type, const std::vector& d, float scale = 0.0f, int32_t zeroPoint = 0); - explicit OperandType(Type type, const std::vector& d, SymmPerChannelQuantParams&& channelQuant); + explicit OperandType(Type type, const Shape& d, float scale = 0.0f, int32_t zeroPoint = 0); + explicit OperandType(Type type, const Shape& d, SymmPerChannelQuantParams&& channelQuant); OperandType(const OperandType& other); OperandType& operator=(const OperandType& other); @@ -137,7 +141,7 @@ struct OperandType { // Get the whole blob size in bytes size_t GetOperandBlobByteSize() const; - void SetDimensions(const std::vector& d); + void SetDimensions(const Shape& d); operator ANeuralNetworksOperandType() const { return operandType; } };