diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc index 09058eaad1..dbf38d557a 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc @@ -267,6 +267,26 @@ void GetFlattenOutputShape(const NodeUnit& node_unit, const Shape& input_shape, dim_2 = std::accumulate(input_shape.cbegin() + axis, input_shape.cend(), 1, std::multiplies()); } +Shape GetShapeInfoFromNodeArg(const GraphViewer& graph_viewer, const std::string& name) { + // can be applied to both input and output + Shape shape; + const auto* node_arg = graph_viewer.GetNodeArg(name); + const auto* shape_proto = node_arg->Shape(); + + shape.reserve(shape_proto->dim_size()); + for (const auto& shape_dim : shape_proto->dim()) { + // shape_dim here can possibly have dim_param, but as dynamic shape is not supported in NNAPI for now + // (checked already in BaseOpSupportChecker), call dim_value here only. + shape.push_back(SafeInt(shape_dim.dim_value())); + } + // If we have an empty shape, (scalar input), we need to make it as {1} as + // nnapi will treat empty shape as dynamic ranking and onnx does not support that + if (shape_proto->dim_size() == 0) { + shape.push_back(1); + } + return shape; +} + bool IsValidSupportedNodeGroup(const std::vector& supported_node_partition) { if (supported_node_partition.size() == 1) { const auto* node = supported_node_partition[0]; @@ -356,7 +376,7 @@ bool IsNodeSupportedInGroup(const NodeUnit& node_unit, const GraphViewer& graph_ return true; } -std::string Shape2String(const std::vector& shape) { +std::string Shape2String(const Shape& shape) { std::ostringstream os; os << "[ "; for (const auto& dim : shape) diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h index dce4c9088d..b1b294f4a0 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h @@ -5,6 +5,7 @@ #include #include +#include "core/common/inlined_containers.h" #include "core/graph/basic_types.h" #include "core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksTypes.h" @@ -24,7 +25,6 @@ namespace onnxruntime { -using Shape = std::vector; using InitializerMap = std::unordered_map; class GraphViewer; @@ -37,6 +37,8 @@ struct NodeUnitIODef; namespace nnapi { +using Shape = InlinedVector; + class IOpSupportChecker; struct OpSupportCheckParams; @@ -144,6 +146,9 @@ bool GetType(const NodeArg& node_arg, int32_t& type); // Get the output shape of Flatten Op void GetFlattenOutputShape(const NodeUnit& node_unit, const Shape& input_shape, int32_t& dim_1, int32_t& dim_2); +// Get the shape information from NodeArg +Shape GetShapeInfoFromNodeArg(const GraphViewer& graph_viewer, const std::string& name); + // If a node is supported by NNAPI bool IsNodeSupported(const NodeUnit& node_unit, const GraphViewer& graph_viewer, const OpSupportCheckParams& params); @@ -157,7 +162,7 @@ bool IsNodeSupportedInGroup(const NodeUnit& node_unit, const GraphViewer& graph_ bool IsValidSupportedNodeGroup(const std::vector& supported_node_group); // Get string representation of a Shape -std::string Shape2String(const std::vector& shape); +std::string Shape2String(const Shape& shape); uint32_t ShapeSize(const Shape& shape, size_t begin_idx, size_t end_idx); inline uint32_t ShapeSize(const Shape& shape) { diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc index 6164bf9743..d407751bca 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc @@ -25,7 +25,7 @@ namespace onnxruntime { namespace nnapi { ModelBuilder::ModelBuilder(const GraphViewer& graph_viewer) - : nnapi_(NnApiImplementation()), graph_viewer_(graph_viewer) {} + : nnapi_(NnApiImplementation()), graph_viewer_(graph_viewer), shaper_{graph_viewer} {} int32_t ModelBuilder::GetNNAPIFeatureLevel() const { return nnapi_ ? static_cast(nnapi_->nnapi_runtime_feature_level) : 0; @@ -34,7 +34,7 @@ int32_t ModelBuilder::GetNNAPIFeatureLevel() const { // Scalar operand is copied into the model, no need to persist #define DEFINE_ADD_OPERAND_FROM_SCALAR(scalar_type, op_type) \ Status ModelBuilder::AddOperandFromScalar(scalar_type value, uint32_t& index) { \ - OperandType operandType(Type::op_type, std::vector{}); \ + OperandType operandType(Type::op_type, InlinedVector{}); \ ORT_RETURN_IF_ERROR(AddNewNNAPIOperand(operandType, index)); \ RETURN_STATUS_ON_ERROR_WITH_NOTE( \ nnapi_->ANeuralNetworksModel_setOperandValue( \ @@ -54,7 +54,6 @@ void ModelBuilder::AddInitializerToSkip(const std::string& tensor_name) { } Status ModelBuilder::Prepare() { - nnapi_model_ = std::unique_ptr(new Model()); RETURN_STATUS_ON_ERROR(nnapi_->ANeuralNetworksModel_create(&nnapi_model_->model_)); ORT_RETURN_IF_ERROR(GetTargetDevices()); PreprocessNodeUnits(); @@ -65,7 +64,6 @@ Status ModelBuilder::Prepare() { ORT_RETURN_IF_ERROR(RegisterModelInputs()); ORT_RETURN_IF_ERROR(AddOperations()); ORT_RETURN_IF_ERROR(RegisterModelOutputs()); - RegisterModelShaper(); return Status::OK(); } @@ -388,10 +386,6 @@ Status ModelBuilder::RegisterModelOutputs() { return Status::OK(); } -void ModelBuilder::RegisterModelShaper() { - nnapi_model_->SetShaper(shaper_); -} - Status ModelBuilder::AddNewOperand(const std::string& name, const OperandType& operand_type, uint32_t& index) { @@ -512,10 +506,10 @@ Status ModelBuilder::AddOperations() { return Status::OK(); } -Status ModelBuilder::AddOperation(int op, const std::vector& input_indices, +Status ModelBuilder::AddOperation(int op, const InlinedVector& input_indices, const std::vector& output_names, const std::vector& output_types) { - std::vector output_indices; + InlinedVector output_indices; for (size_t i = 0; i < output_types.size(); i++) { uint32_t index = 0; ORT_RETURN_IF_ERROR(AddNewOperand(output_names[i], output_types[i], index)); diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h index 693583d307..5b4d1bd7e9 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h @@ -5,6 +5,7 @@ #include #include +#include "core/common/inlined_containers.h" #include "core/graph/basic_types.h" #include "core/providers/nnapi/nnapi_builtin/model.h" #include "core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.h" @@ -46,7 +47,7 @@ class ModelBuilder { int32_t GetNNAPIFeatureLevel() const; // Add an NNAPI operation (operator) - common::Status AddOperation(int op, const std::vector& input_indices, + common::Status AddOperation(int op, const InlinedVector& input_indices, const std::vector& output_names, const std::vector& output_types); @@ -117,7 +118,7 @@ class ModelBuilder { private: const NnApi* nnapi_{nullptr}; const GraphViewer& graph_viewer_; - std::unique_ptr nnapi_model_; + std::unique_ptr nnapi_model_{std::make_unique()}; uint32_t name_token_{0}; @@ -141,9 +142,8 @@ class ModelBuilder { std::unordered_map> op_support_checkers_; - - std::vector input_index_vec_; - std::vector output_index_vec_; + InlinedVector input_index_vec_; + InlinedVector output_index_vec_; // Contains all quantized operators' input and the NodeUnit(s) using the input // In the form of {input_name, [NodeUnit(s) using the input]} @@ -178,8 +178,6 @@ class ModelBuilder { common::Status RegisterModelInputs(); common::Status AddOperations(); common::Status RegisterModelOutputs(); - // After constructing the NNAPI model, will set the shape inferencing record to the Model - void RegisterModelShaper(); // Get all quantized inputs in the underlying graph_viewer void GetAllQuantizedOpInputs(); diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc index a3b0ea05c9..ffeeb14dce 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc @@ -5,20 +5,21 @@ #include #include +#include #include "core/common/logging/logging.h" #include "core/common/safeint.h" #include "core/framework/tensorprotoutils.h" #include "core/graph/graph_viewer.h" -#include "core/providers/common.h" -#include "core/providers/shared/utils/utils.h" -#include "core/providers/shared/node_unit/node_unit.h" -#include "core/providers/cpu/tensor/slice_helper.h" -#include "core/providers/nnapi/nnapi_builtin/builders/op_builder_helpers.h" -#include "helper.h" -#include "model_builder.h" -#include "op_support_checker.h" #include "core/optimizer/initializer.h" +#include "core/providers/common.h" +#include "core/providers/cpu/tensor/slice_helper.h" +#include "core/providers/nnapi/nnapi_builtin/builders/helper.h" +#include "core/providers/nnapi/nnapi_builtin/builders/model_builder.h" +#include "core/providers/nnapi/nnapi_builtin/builders/op_builder_helpers.h" +#include "core/providers/nnapi/nnapi_builtin/builders/op_support_checker.h" +#include "core/providers/shared/node_unit/node_unit.h" +#include "core/providers/shared/utils/utils.h" using namespace android::nn::wrapper; @@ -38,8 +39,7 @@ struct OpBuilderRegistrations { static Status AddBinaryOperator(int32_t op_type, ModelBuilder& model_builder, - const std::string& input1, - const std::string& input2, + const std::string& input1, const std::string& input2, bool add_activation, int32_t fuse_code, const std::string& output, @@ -49,7 +49,7 @@ static Status AddBinaryOperator(int32_t op_type, const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); - std::vector input_indices; + InlinedVector input_indices; input_indices.push_back(operand_indices.at(input1)); // input 1 input_indices.push_back(operand_indices.at(input2)); // input 2 @@ -57,7 +57,6 @@ static Status AddBinaryOperator(int32_t op_type, ADD_SCALAR_OPERAND(model_builder, input_indices, fuse_code); } - ORT_RETURN_IF_ERROR(shaper.Eltwise(input1, input2, output)); const OperandType output_operand_type(operand_types.at(input1).type, shaper[output], output_scale, output_zero_point); ORT_RETURN_IF_ERROR(model_builder.AddOperation(op_type, input_indices, @@ -65,10 +64,57 @@ static Status AddBinaryOperator(int32_t op_type, return Status::OK(); } -static Status AddSqueezeOp(ModelBuilder& model_builder, - const std::string& node_name, - const std::string& input, const std::string& output, - std::vector axes) { +static Status AddNnapiBatchNormalization(ModelBuilder& model_builder, + const std::string& input1, + const std::string& input2, + const std::string& input3, + const std::string& output1, + const std::string& output2, + int32_t fuse_code, + float output_scale = 0.0f, + int32_t output_zero_point = 0) { + auto& shaper(model_builder.GetShaper()); + const auto& operand_indices(model_builder.GetOperandIndices()); + const auto& operand_types(model_builder.GetOperandTypes()); + + // Add Nnapi Mul + InlinedVector input_indices; + input_indices.push_back(operand_indices.at(input1)); + input_indices.push_back(operand_indices.at(input2)); + + ADD_SCALAR_OPERAND(model_builder, input_indices, ANEURALNETWORKS_FUSED_NONE); + + const Shape& shape1 = shaper[input1]; + const Shape& shape2 = shaper[input2]; + Shape output1_shape; + // broadcasting support for eltwise shape operation + ORT_RETURN_IF_ERROR(op_builder_helpers::PerformBroadcasting(shape1, shape2, output1_shape)); + const OperandType output_operand_type(operand_types.at(input1).type, output1_shape, + output_scale, output_zero_point); + ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_MUL, input_indices, + {output1}, {output_operand_type})); + + // Add Nnapi Add + input_indices.clear(); + input_indices.push_back(operand_indices.at(output1)); + input_indices.push_back(operand_indices.at(input3)); + + ADD_SCALAR_OPERAND(model_builder, input_indices, fuse_code); + + const Shape& shape3 = shaper[input3]; + Shape output2_shape; + ORT_RETURN_IF_ERROR(op_builder_helpers::PerformBroadcasting(output1_shape, shape3, output2_shape)); + const OperandType output_operand_type2(operand_types.at(input3).type, output2_shape, + output_scale, output_zero_point); + ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_ADD, input_indices, + {output2}, {output_operand_type2})); + return Status::OK(); +} + +static Status AddNnapiSqueeze(ModelBuilder& model_builder, + const std::string& node_name, + const std::string& input, const std::string& output, + std::vector axes) { if (model_builder.GetNNAPIFeatureLevel() < ANEURALNETWORKS_FEATURE_LEVEL_2) { return ORT_MAKE_STATUS( ONNXRUNTIME, FAIL, "Squeeze is not supported on API level ", model_builder.GetNNAPIFeatureLevel()); @@ -78,7 +124,7 @@ static Status AddSqueezeOp(ModelBuilder& model_builder, const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); - const auto& input_shape(shaper[input]); + const auto input_shape = shaper[input]; auto input_dims = input_shape.size(); for (auto& axis : axes) { axis = static_cast(HandleNegativeAxis(axis, input_dims)); @@ -102,12 +148,34 @@ static Status AddSqueezeOp(ModelBuilder& model_builder, const OperandType axes_operand_type(Type::TENSOR_INT32, axes_dimen); ORT_RETURN_IF_ERROR(model_builder.AddOperandFromPersistMemoryBuffer(axes_name, axes.data(), axes_operand_type)); - std::vector input_indices; + InlinedVector input_indices; input_indices.push_back(operand_indices.at(input)); // input input_indices.push_back(operand_indices.at(axes_name)); // axes - ORT_RETURN_IF_ERROR(shaper.Squeeze(input, axes, output)); - const OperandType output_operand_type(operand_types.at(input).type, shaper[output]); + // Shape inference calculation for squeeze + int32_t input_size = static_cast(input_shape.size()); + std::unordered_set axes_to_be_squeezed; + + // If the Op is squeezing all by not specifying axes, the axes is pre-populate + // with axes of all single dimensions by the caller + for (const auto& axis : axes) + axes_to_be_squeezed.insert(axis); + + // Make output dimensions + InlinedVector output_dimen; + output_dimen.reserve(input_size - axes_to_be_squeezed.size()); + for (int32_t i = 0; i < input_size; i++) { + if (!Contains(axes_to_be_squeezed, i)) + output_dimen.push_back(input_shape[i]); + } + + // In case of a tensor has all 1's in dimension such as {1,1,1,1} and gets squeezed all + // the output shape will be {1} + if (output_dimen.empty()) + output_dimen.push_back(1); + + shaper.AddShape(output, output_dimen); + const OperandType output_operand_type(operand_types.at(input).type, output_dimen); ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_SQUEEZE, input_indices, {output}, {output_operand_type})); return Status::OK(); @@ -673,7 +741,6 @@ Status ReluOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N const auto& input = node_unit.Inputs()[0].node_arg.Name(); const auto& output = node_unit.Outputs()[0].node_arg.Name(); - ORT_RETURN_IF_ERROR(shaper.Identity(input, output)); const OperandType output_operand_type(operand_types.at(input).type, shaper[output]); // skip this relu if it is some op's fuse output @@ -681,7 +748,7 @@ Status ReluOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N LOGS_DEFAULT(VERBOSE) << "Relu Node [" << node_unit.Name() << "] fused"; model_builder.RegisterOperand(output, operand_indices.at(input), output_operand_type); } else { - std::vector input_indices; + InlinedVector input_indices; input_indices.push_back(operand_indices.at(input)); ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_RELU, input_indices, {output}, {output_operand_type})); @@ -721,6 +788,7 @@ Status TransposeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, co const auto& input = node_unit.Inputs()[0].node_arg.Name(); const auto& output = node_unit.Outputs()[0].node_arg.Name(); + NodeAttrHelper helper(node_unit); std::vector perm = helper.Get("perm", std::vector()); auto input_dims = static_cast(shaper[input].size()); @@ -741,7 +809,6 @@ Status TransposeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, co } std::string perm_name = model_builder.GetUniqueName(node_unit.Name() + input + "perm"); - ORT_RETURN_IF_ERROR(op_builder_helpers::AddNnapiTranspose(model_builder, input, perm_name, perm, output)); return Status::OK(); @@ -854,13 +921,15 @@ bool ReshapeOpBuilder::IsQuantizedOp(const NodeUnit& node_unit) const { const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); const auto& output = node_unit.Outputs()[0].node_arg.Name(); - ORT_RETURN_IF_ERROR(shaper.Reshape(input, shape, output)); - auto input_rank = shaper[input].size(); - auto output_rank = shaper[output].size(); + + const auto input_shape = shaper[input]; + const auto output_shape = shaper[output]; + const auto input_rank = input_shape.size(); + const auto output_rank = output_shape.size(); // For reshape, the output type should be the same as the input type except the shape is different auto output_operand_type = operand_types.at(input); - output_operand_type.SetDimensions(shaper[output]); + output_operand_type.SetDimensions(output_shape); // Since Reshape is not running using hardware in NNAPI for some CPU (e.g. Qualcomm SD for now) // We will try to see if we the skip the Reshape to prevent context switching between @@ -871,8 +940,7 @@ bool ReshapeOpBuilder::IsQuantizedOp(const NodeUnit& node_unit) const { } else { // We still need to perform a reshape here std::string shape_name = model_builder.GetUniqueName(node_unit.Name() + input + "newshape"); - ORT_RETURN_IF_ERROR(op_builder_helpers::AddNnapiReshape(model_builder, input, shape_name, shape, output, - &shaper[output])); + ORT_RETURN_IF_ERROR(op_builder_helpers::AddNnapiReshape(model_builder, input, shape_name, shape, output)); } return Status::OK(); @@ -881,7 +949,6 @@ bool ReshapeOpBuilder::IsQuantizedOp(const NodeUnit& node_unit) const { Status ReshapeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const { auto& shaper(model_builder.GetShaper()); const auto& initializers(model_builder.GetInitializerTensors()); - auto input = node_unit.Inputs()[0].node_arg.Name(); const auto& shape_tensor = *initializers.at(node_unit.Inputs()[1].node_arg.Name()); @@ -889,7 +956,7 @@ Status ReshapeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, cons auto raw_shape = unpacked_tensor.DataAsSpan(); const auto size = SafeInt(shape_tensor.dims()[0]); - Shape input_shape = shaper[input]; + const auto input_shape = shaper[input]; std::vector shape(size); for (uint32_t i = 0; i < size; i++) { int32_t dim = SafeInt(raw_shape[i]); @@ -929,7 +996,7 @@ void UnsqueezeOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, cons } Status UnsqueezeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const { - const auto& shaper(model_builder.GetShaper()); + auto& shaper(model_builder.GetShaper()); const auto& input = node_unit.Inputs()[0].node_arg.Name(); // NNAPI does not support unsqueeze, here we utilize unsqueeze's axes input to compute output shape @@ -937,7 +1004,7 @@ Status UnsqueezeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, co std::vector axes; ORT_RETURN_IF_ERROR(GetAxesForSqueezeAndUnSqueeze(model_builder, node_unit, axes)); - Shape input_shape = shaper[input]; + const auto input_shape = shaper[input]; auto input_dims = input_shape.size(); std::vector shape; const auto size = SafeInt(input_dims + axes.size()); // "output rank" @@ -986,6 +1053,7 @@ Status BatchNormalizationOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_bu // register a new operand with new shape const auto& input = inputs[0].node_arg.Name(); const auto& output = node_unit.Outputs()[0].node_arg.Name(); + const auto input_shape = shaper[input]; const auto& scale_tensor = *initializers.at(inputs[1].node_arg.Name()); const auto& bias_tensor = *initializers.at(inputs[2].node_arg.Name()); @@ -1029,11 +1097,12 @@ Status BatchNormalizationOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_bu // input shape {N, C} ==> tensor_a/b's shape {size} // input shape {N, C, H} ==> tensor_a/b's shape {size, 1} // input shape {N, C, H, W} ==> tensor_a/b's shape {size, 1, 1} - const auto input_rank = shaper[input].size(); + const auto input_rank = input_shape.size(); for (size_t i = 2; i < input_rank; i++) tensor_a_dimen.push_back(1); } + shaper.AddShape(input, input_shape); shaper.AddShape(tensor_a_name, tensor_a_dimen); shaper.AddShape(tensor_b_name, tensor_a_dimen); const OperandType a_operand_type(operand_types.at(input).type, tensor_a_dimen); @@ -1041,21 +1110,9 @@ Status BatchNormalizationOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_bu const OperandType b_operand_type(operand_types.at(input).type, tensor_a_dimen); ORT_RETURN_IF_ERROR(model_builder.AddOperandFromPersistMemoryBuffer(tensor_b_name, b.data(), b_operand_type)); - // Mul - ORT_RETURN_IF_ERROR(AddBinaryOperator(ANEURALNETWORKS_MUL, - model_builder, - input, tensor_a_name, - true /* add_activation */, ANEURALNETWORKS_FUSED_NONE, - tensor_imm_product_name)); - - // Add int32_t fuse_code = model_builder.FindActivation(node_unit); - ORT_RETURN_IF_ERROR(AddBinaryOperator(ANEURALNETWORKS_ADD, - model_builder, - tensor_imm_product_name, tensor_b_name, - true /* add_activation */, fuse_code, - output)); - + ORT_RETURN_IF_ERROR(AddNnapiBatchNormalization(model_builder, input, tensor_a_name, tensor_b_name, + tensor_imm_product_name, output, fuse_code)); return Status::OK(); } @@ -1124,7 +1181,7 @@ Status PoolOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N std::vector onnx_pads, onnx_strides, kernel_shape; bool use_auto_pad = false; int32_t nnapi_padding_code = ANEURALNETWORKS_PADDING_VALID; - const auto& input_shape = shaper[input]; + const auto input_shape = shaper[input]; if (is_average_pool || op_type == "MaxPool") { const auto auto_pad_type = StringToAutoPadType(helper.Get("auto_pad", "NOTSET")); kernel_shape = helper.Get("kernel_shape", std::vector{0, 0}); @@ -1171,7 +1228,7 @@ Status PoolOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N initializers, node_unit.Outputs()[0], node_unit.ModelPath(), y_scale, y_zero_point)); } - std::vector input_indices; + InlinedVector input_indices; input_indices.push_back(operand_indices.at(input)); if (use_auto_pad) { @@ -1193,10 +1250,6 @@ Status PoolOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N ADD_SCALAR_OPERAND(model_builder, input_indices, use_nchw); } - ORT_RETURN_IF_ERROR(shaper.Pool(input, - onnx_pads, onnx_strides, kernel_shape, - use_nchw, - output)); const OperandType output_operand_type(operand_types.at(input).type, shaper[output], y_scale, y_zero_point); ORT_RETURN_IF_ERROR(model_builder.AddOperation(op_code, input_indices, {output}, {output_operand_type})); @@ -1385,7 +1438,7 @@ Status ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N const auto auto_pad_type = StringToAutoPadType(helper.Get("auto_pad", "NOTSET")); bool use_auto_pad = false; int32_t nnapi_padding_code = ANEURALNETWORKS_PADDING_SAME; - const auto& input_shape = shaper[input]; + const auto input_shape = shaper[input]; const auto& kernel_shape = shaper[weight]; const auto weight_size_y = kernel_shape[1]; const auto weight_size_x = kernel_shape[2]; @@ -1395,7 +1448,7 @@ Status ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N auto_pad_type, use_nchw, onnx_pads, nnapi_padding_code, use_auto_pad)); - std::vector input_indices; + InlinedVector input_indices; input_indices.push_back(operand_indices.at(input)); input_indices.push_back(operand_indices.at(weight)); input_indices.push_back(operand_indices.at(bias)); @@ -1443,16 +1496,8 @@ Status ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N if (conv_2d || grouped_conv_2d) { operationCode = conv_2d ? ANEURALNETWORKS_CONV_2D : ANEURALNETWORKS_GROUPED_CONV_2D; - ORT_RETURN_IF_ERROR(shaper.Conv(input, weight, - onnx_pads, onnx_strides, onnx_dilations, - use_nchw, - output)); } else { // depthwise_conv_2d operationCode = ANEURALNETWORKS_DEPTHWISE_CONV_2D; - ORT_RETURN_IF_ERROR(shaper.DepthwiseConv(input, weight, - onnx_pads, onnx_strides, onnx_dilations, - use_nchw, - output)); } const OperandType output_operand_type(operand_types.at(input).type, shaper[output], y_scale, y_zero_point); @@ -1491,9 +1536,8 @@ Status CastOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Invalid cast to type: ", to); } - std::vector input_indices; + InlinedVector input_indices; input_indices.push_back(operand_indices.at(input)); - ORT_RETURN_IF_ERROR(shaper.Identity(input, output)); const OperandType output_operand_type(type, shaper[output]); ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_CAST, input_indices, {output}, {output_operand_type})); @@ -1523,7 +1567,7 @@ Status DepthToSpaceOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, int32_t blocksize = SafeInt(node_unit.GetNode().GetAttributes().at("blocksize").i()); - std::vector input_indices; + InlinedVector input_indices; input_indices.push_back(operand_indices.at(input)); ADD_SCALAR_OPERAND(model_builder, input_indices, blocksize); @@ -1531,8 +1575,6 @@ Status DepthToSpaceOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, // optional input to use nchw is available starting NNAPI feature level 3 ADD_SCALAR_OPERAND(model_builder, input_indices, use_nchw); } - - ORT_RETURN_IF_ERROR(shaper.DepthToSpace(input, blocksize, use_nchw, output)); const OperandType output_operand_type(operand_types.at(input).type, shaper[output]); ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_DEPTH_TO_SPACE, input_indices, {output}, {output_operand_type})); @@ -1597,7 +1639,7 @@ Status SoftMaxOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, cons const auto& output = node_unit.Outputs()[0].node_arg.Name(); float beta = 1.f; - std::vector input_indices; + InlinedVector input_indices; input_indices.push_back(operand_indices.at(input)); ADD_SCALAR_OPERAND(model_builder, input_indices, beta); @@ -1606,7 +1648,6 @@ Status SoftMaxOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, cons ADD_SCALAR_OPERAND(model_builder, input_indices, axis); } - ORT_RETURN_IF_ERROR(shaper.Identity(input, output)); const OperandType output_operand_type(operand_types.at(input).type, shaper[output], y_scale, y_zero_point); ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_SOFTMAX, input_indices, {output}, {output_operand_type})); @@ -1632,10 +1673,9 @@ Status IdentityOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, con const auto& input = node_unit.Inputs()[0].node_arg.Name(); const auto& output = node_unit.Outputs()[0].node_arg.Name(); - std::vector input_indices; + InlinedVector input_indices; input_indices.push_back(operand_indices.at(input)); // input - ORT_RETURN_IF_ERROR(shaper.Identity(input, output)); const OperandType output_operand_type(operand_types.at(input).type, shaper[output]); model_builder.RegisterOperand(output, operand_indices.at(input), output_operand_type); return Status::OK(); @@ -1780,13 +1820,12 @@ Status GemmOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N if (has_bias) { const auto& bias = inputs[2].node_arg.Name(); if (!is_quant_gemm) { - // We need squeeze the input tensor to 1d if necessary + // We need to squeeze the input tensor to 1d if necessary if (shaper[bias].size() > 1) { std::string bias_squeezed = model_builder.GetUniqueName(node_unit.Name() + op + "_bias_squeezed"); // We will use squeeze all here - ORT_RETURN_IF_ERROR(AddSqueezeOp(model_builder, node_unit.Name(), - bias, bias_squeezed, - {} /* axes */)); + ORT_RETURN_IF_ERROR(AddNnapiSqueeze(model_builder, node_unit.Name(), + bias, bias_squeezed, {} /* axes */)); bias_idx = operand_indices.at(bias_squeezed); LOGS_DEFAULT(VERBOSE) << "GemmOpBuilder - Operand [" << bias << "] squeezed from " << Shape2String(shaper[bias]) @@ -1833,14 +1872,13 @@ Status GemmOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N bias_idx = operand_indices.at(bias); } - std::vector input_indices; + InlinedVector input_indices; input_indices.push_back(operand_indices.at(input1)); // A input_indices.push_back(input_2_idx); // B input_indices.push_back(bias_idx); // C int32_t fuse_code = model_builder.FindActivation(node_unit); ADD_SCALAR_OPERAND(model_builder, input_indices, fuse_code); - ORT_RETURN_IF_ERROR(shaper.FC(input1, input2, output)); const OperandType output_operand_type(operand_types.at(input1).type, shaper[output], y_scale, y_zero_point); ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_FULLY_CONNECTED, input_indices, {output}, {output_operand_type})); @@ -1900,7 +1938,7 @@ Status UnaryOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const const auto& input = node_unit.Inputs()[0].node_arg.Name(); const auto& output = node_unit.Outputs()[0].node_arg.Name(); - ORT_RETURN_IF_ERROR(shaper.Identity(input, output)); + bool is_qlinear_sigmoid = op_type == "QLinearSigmoid"; int32_t op_code; @@ -1943,7 +1981,7 @@ Status UnaryOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const y_zero_point = 0; } - std::vector input_indices; + InlinedVector input_indices; input_indices.push_back(operand_indices.at(input)); const OperandType output_operand_type(operand_types.at(input).type, shaper[output], y_scale, y_zero_point); ORT_RETURN_IF_ERROR(model_builder.AddOperation(op_code, input_indices, @@ -1986,7 +2024,7 @@ Status ConcatOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeAttrHelper helper(node_unit); const auto& inputs = node_unit.Inputs(); - std::vector input_indices; + InlinedVector input_indices; const auto& input0 = inputs[0].node_arg.Name(); const auto node_input_size = inputs.size(); @@ -2062,7 +2100,6 @@ Status ConcatOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const ADD_SCALAR_OPERAND(model_builder, input_indices, axis); const auto& output = node_unit.Outputs()[0].node_arg.Name(); - ORT_RETURN_IF_ERROR(shaper.Concat(input_names, axis, output)); OperandType output_operand_type(operand_types.at(input0).type, shaper[output], y_scale, y_zero_point); ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_CONCATENATION, input_indices, {output}, {output_operand_type})); @@ -2089,10 +2126,11 @@ void SqueezeOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const Status SqueezeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const { auto input = node_unit.Inputs()[0].node_arg.Name(); + auto output = node_unit.Outputs()[0].node_arg.Name(); std::vector axes; ORT_RETURN_IF_ERROR(GetAxesForSqueezeAndUnSqueeze(model_builder, node_unit, axes)); - return AddSqueezeOp(model_builder, node_unit.Name(), input, node_unit.Outputs()[0].node_arg.Name(), axes); + return AddNnapiSqueeze(model_builder, node_unit.Name(), input, output, axes); } #pragma endregion @@ -2124,9 +2162,8 @@ Status QuantizeLinearOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builde model_builder.GetInitializerTensors(), node_unit.Outputs()[0], node_unit.ModelPath(), scale, zero_point)); Type output_type = Type::TENSOR_QUANT8_ASYMM; - ORT_RETURN_IF_ERROR(shaper.Identity(input, output)); const OperandType output_operand_type(output_type, shaper[output], scale, zero_point); - std::vector input_indices; + InlinedVector input_indices; input_indices.push_back(operand_indices.at(input)); ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_QUANTIZE, input_indices, {output}, {output_operand_type})); @@ -2164,10 +2201,9 @@ Status DequantizeLinearOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_buil ORT_RETURN_IF_ERROR(IsValidInputQuantizedType(model_builder, input, scale, zero_point)); - ORT_RETURN_IF_ERROR(shaper.Identity(input, output)); const OperandType output_operand_type(Type::TENSOR_FLOAT32, shaper[output]); - std::vector input_indices; + InlinedVector input_indices; input_indices.push_back(operand_indices.at(input)); ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_DEQUANTIZE, input_indices, {output}, {output_operand_type})); @@ -2210,7 +2246,7 @@ Status LRNOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const No const auto radius = (size - 1) / 2; alpha /= size; // NNAPI's alpha is different than ONNX's alpha - std::vector input_indices; + InlinedVector input_indices; input_indices.push_back(operand_indices.at(input)); ADD_SCALAR_OPERAND(model_builder, input_indices, radius); ADD_SCALAR_OPERAND(model_builder, input_indices, bias); @@ -2226,7 +2262,6 @@ Status LRNOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const No ADD_SCALAR_OPERAND(model_builder, input_indices, axis); } - ORT_RETURN_IF_ERROR(shaper.Identity(input, output)); const OperandType output_operand_type(operand_types.at(input).type, shaper[output]); ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_LOCAL_RESPONSE_NORMALIZATION, input_indices, {output}, {output_operand_type})); @@ -2262,7 +2297,6 @@ Status ClipOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N const auto& input = node_unit.Inputs()[0].node_arg.Name(); const auto& output = node_unit.Outputs()[0].node_arg.Name(); - ORT_RETURN_IF_ERROR(shaper.Identity(input, output)); const OperandType output_operand_type(operand_types.at(input).type, shaper[output]); if (Contains(model_builder.GetFusedActivations(), input)) { @@ -2284,7 +2318,7 @@ Status ClipOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const N return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "ClipOpBuilder, unsupported input [", min, ", ", max, "].", "We should not reach here, ClipOpBuilder::IsOpSupportedImpl should have caught this."); - std::vector input_indices; + InlinedVector input_indices; input_indices.push_back(operand_indices.at(input)); ORT_RETURN_IF_ERROR(model_builder.AddOperation(op_code, input_indices, {output}, {output_operand_type})); @@ -2363,27 +2397,11 @@ Status ResizeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const int h_idx = use_nchw ? 2 : 1; int w_idx = use_nchw ? 3 : 2; - if (inputs.size() == 3) { // we are using scales - const auto& scales_name = inputs[2].node_arg.Name(); - const auto& scales_tensor = *initializers.at(scales_name); - Initializer unpacked_tensor(scales_tensor); - auto scales_data = unpacked_tensor.DataAsSpan(); - ORT_RETURN_IF_ERROR( - shaper.ResizeUsingScales(input, scales_data[h_idx], scales_data[w_idx], use_nchw, output)); - } else { // we are using sizes - const auto& sizes_name = inputs[3].node_arg.Name(); - const auto& sizes_tensor = *initializers.at(sizes_name); - Initializer unpacked_tensor(sizes_tensor); - auto sizes_data = unpacked_tensor.DataAsSpan(); - ORT_RETURN_IF_ERROR( - shaper.ResizeUsingOutputSizes(input, SafeInt(sizes_data[h_idx]), SafeInt(sizes_data[w_idx]), use_nchw, output)); - } - const auto& output_shape = shaper[output]; int32_t output_h = output_shape[h_idx]; int32_t output_w = output_shape[w_idx]; - std::vector input_indices; + InlinedVector input_indices; input_indices.push_back(operand_indices.at(input)); ADD_SCALAR_OPERAND(model_builder, input_indices, output_w); ADD_SCALAR_OPERAND(model_builder, input_indices, output_h); @@ -2475,7 +2493,7 @@ Status GatherOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const int32_t rank = static_cast(shaper[input1].size()); int32_t axis = static_cast(HandleNegativeAxis(helper.Get("axis", 0), rank)); - std::vector input_indices; + InlinedVector input_indices; input_indices.push_back(operand_indices.at(input1)); ADD_SCALAR_OPERAND(model_builder, input_indices, axis); @@ -2510,9 +2528,8 @@ Status GatherOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const ORT_RETURN_IF_ERROR(model_builder.AddOperandFromPersistMemoryBuffer(input2, indices.data(), indices_operand_type)); } input_indices.push_back(operand_indices.at(input2)); - ORT_RETURN_IF_ERROR(shaper.Gather(input1, input2, axis, output)); - const OperandType output_operand_type(operand_types.at(input1).type, shaper[output]); + const OperandType output_operand_type(operand_types.at(input1).type, shaper[output]); return model_builder.AddOperation(ANEURALNETWORKS_GATHER, input_indices, {output}, {output_operand_type}); } @@ -2559,10 +2576,10 @@ class MinMaxOpBuilder : public BaseOpBuilder { return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "MinMaxOpBuilder, unknown op: ", op_type); } - std::vector input_indices; + InlinedVector input_indices; input_indices.push_back(operand_indices.at(input1)); // input 1 input_indices.push_back(operand_indices.at(input2)); // input 2 - ORT_RETURN_IF_ERROR(shaper.Eltwise(input1, input2, output)); + const OperandType output_operand_type(operand_types.at(input1).type, shaper[output]); ORT_RETURN_IF_ERROR(model_builder.AddOperation(op_code, input_indices, {output}, {output_operand_type})); @@ -2593,11 +2610,11 @@ Status EluOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const No const auto& operand_types(model_builder.GetOperandTypes()); const auto& input = node_unit.Inputs()[0].node_arg.Name(); const auto& output = node_unit.Outputs()[0].node_arg.Name(); - ORT_RETURN_IF_ERROR(shaper.Identity(input, output)); + const OperandType output_operand_type(operand_types.at(input).type, shaper[output]); NodeAttrHelper helper(node_unit); const auto alpha = helper.Get("alpha", 1.0f); - std::vector input_indices; + InlinedVector input_indices; input_indices.push_back(operand_indices.at(input)); ADD_SCALAR_OPERAND(model_builder, input_indices, alpha); return model_builder.AddOperation(ANEURALNETWORKS_ELU, input_indices, @@ -2634,7 +2651,7 @@ Status SliceOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); const auto& inputs = node_unit.Inputs(); - const auto& input_shape = shaper[inputs[0].node_arg.Name()]; + const auto input_shape = shaper[inputs[0].node_arg.Name()]; TensorShapeVector input_shape_64(input_shape.cbegin(), input_shape.cend()); SliceOp::PrepareForComputeMetadata compute_metadata(input_shape_64); @@ -2699,7 +2716,7 @@ Status SliceOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const shaper.AddShape(output, nnapi_output_shape); const OperandType output_operand_type(operand_types.at(input).type, shaper[output]); - std::vector input_indices; + InlinedVector input_indices; input_indices.push_back(operand_indices.at(input)); // begin/end/strides of ANEURALNETWORKS_STRIDED_SLICE have the same shape @@ -2790,13 +2807,13 @@ void PadOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const Node } Status PadOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const NodeUnit& node_unit) const { - auto& shaper = model_builder.GetShaper(); + auto& shaper(model_builder.GetShaper()); const auto& operand_indices = model_builder.GetOperandIndices(); const auto& operand_types = model_builder.GetOperandTypes(); const auto& inputs = node_unit.Inputs(); const auto& outputs = node_unit.Outputs(); - std::vector input_indices{}; + InlinedVector input_indices{}; // `data` input const auto& data = inputs[0].node_arg.Name(); @@ -2805,7 +2822,7 @@ Status PadOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const No // `pads` input // convert from [begin_1, begin_2, ..., end_1, end_2, ...] to [begin_1, end_1, begin_2, end_2, ...] // convert from int64_t to int32_t - const auto& data_shape = shaper[data]; + const auto data_shape = shaper[data]; const uint32_t data_rank = SafeInt(data_shape.size()); const auto& pads = inputs[1].node_arg.Name(); @@ -2841,9 +2858,6 @@ Status PadOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const No ADD_SCALAR_OPERAND(model_builder, input_indices, pad_value); const auto& output = outputs[0].node_arg.Name(); - - ORT_RETURN_IF_ERROR(shaper.Pad(data, converted_pads_data, output)); - const OperandType output_operand_type{operand_types.at(data).type, shaper[output]}; const auto op_code = ANEURALNETWORKS_PAD_V2; diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder_helpers.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder_helpers.cc index 27e12ab649..898f0921cf 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder_helpers.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder_helpers.cc @@ -4,10 +4,14 @@ #include "core/providers/nnapi/nnapi_builtin/builders/op_builder_helpers.h" #include +#include +#include #include "gsl/gsl" +#include "core/common/inlined_containers_fwd.h" #include "core/common/safeint.h" +#include "core/common/span_utils.h" #include "core/graph/node_arg.h" #include "core/providers/common.h" #include "core/providers/nnapi/nnapi_builtin/builders/helper.h" @@ -19,24 +23,36 @@ using android::nn::wrapper::OperandType, android::nn::wrapper::Type; Status AddNnapiTranspose(ModelBuilder& model_builder, const std::string& data_input, - const std::string& perm_input, const std::vector& perm, + const std::string& perm_input, + const gsl::span perm, const std::string& output) { auto& shaper(model_builder.GetShaper()); + + // Calculate transpose output shape + + const Shape input_dimen = shaper[data_input]; + ORT_RETURN_IF_NOT(perm.size() == input_dimen.size(), "Invalid perm is given!"); + size_t size = input_dimen.size(); + Shape output_dimen(size); + for (size_t i = 0; i < size; i++) + output_dimen[i] = input_dimen[perm[i]]; + + shaper.AddShape(output, output_dimen); + const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); - std::vector input_indices; + InlinedVector input_indices; input_indices.push_back(operand_indices.at(data_input)); // input Shape perm_dimen = {SafeInt(perm.size())}; OperandType perm_operand_type(Type::TENSOR_INT32, perm_dimen); ORT_RETURN_IF_ERROR(model_builder.AddOperandFromPersistMemoryBuffer(perm_input, perm.data(), perm_operand_type)); uint32_t perm_idx = operand_indices.at(perm_input); - input_indices.push_back(perm_idx); // permutation - ORT_RETURN_IF_ERROR(shaper.Transpose(data_input, perm, output)); + OperandType output_operand_type = operand_types.at(data_input); - output_operand_type.SetDimensions(shaper[output]); + output_operand_type.SetDimensions(output_dimen); return model_builder.AddOperation(ANEURALNETWORKS_TRANSPOSE, input_indices, {output}, {output_operand_type}); } @@ -44,18 +60,46 @@ Status AddNnapiTranspose(ModelBuilder& model_builder, Status AddNnapiReshape(ModelBuilder& model_builder, const std::string& data_input, const std::string& shape_input, const std::vector& shape_value, - const std::string& output, const Shape* output_shape) { - if (output_shape == nullptr) { - auto& shaper = model_builder.GetShaper(); - ORT_RETURN_IF_ERROR(shaper.Reshape(data_input, shape_value, output)); - output_shape = &shaper[output]; + const std::string& output) { + auto& shaper = model_builder.GetShaper(); + // Calculate reshape output shape + + const Shape input_dimen = shaper[data_input]; + uint32_t input_size = ShapeSize(input_dimen); + Shape output_dimen(shape_value.size()); + + int64_t capacity = 1; + int unk_dim_idx = -1; + for (size_t i = 0; i < shape_value.size(); i++) { + int32_t dim_i = shape_value[i]; + ORT_RETURN_IF_NOT(dim_i != 0, "NNAPI does not support 0 reshape dimension"); + if (dim_i == -1) { + ORT_RETURN_IF_NOT(unk_dim_idx == -1, "Only one input dimension of Attr(shape) can be unknown!"); + unk_dim_idx = static_cast(i); + } else { + capacity *= dim_i; + output_dimen[i] = static_cast(dim_i); + } } + if (unk_dim_idx != -1) { + if (input_size == 0) + output_dimen[unk_dim_idx] = 0; + else + output_dimen[unk_dim_idx] = static_cast(input_size / capacity); + + capacity *= output_dimen[unk_dim_idx]; + } + + ORT_RETURN_IF_NOT(capacity == input_size, "Invalid shape is given!"); + + shaper.AddShape(output, output_dimen); + const auto& operand_indices = model_builder.GetOperandIndices(); const auto& operand_types = model_builder.GetOperandTypes(); // Add input - std::vector input_indices; + InlinedVector input_indices; input_indices.push_back(operand_indices.at(data_input)); // Add new shape @@ -67,7 +111,7 @@ Status AddNnapiReshape(ModelBuilder& model_builder, // For reshape, the output type should be the same as the input type except the shape is different OperandType output_operand_type{operand_types.at(data_input)}; - output_operand_type.SetDimensions(*output_shape); + output_operand_type.SetDimensions(output_dimen); ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_RESHAPE, input_indices, {output}, {output_operand_type})); @@ -85,12 +129,24 @@ Status AddNnapiSplit(ModelBuilder& model_builder, const auto input_rank = shaper[input].size(); axis = static_cast(HandleNegativeAxis(axis, input_rank)); - ORT_RETURN_IF_ERROR(shaper.Split(input, axis, outputs)); + const auto count = gsl::narrow(outputs.size()); - std::vector input_indices; + // Calculate split output shape + { + const auto input_shape = shaper[input]; + ORT_RETURN_IF_NOT(input_shape[axis] % count == 0, + "count [", count, "] does not evenly divide dimension ", axis, " [", input_shape[axis], "]"); + + Shape output_shape = input_shape; + output_shape[axis] = SafeInt(input_shape[axis] / count); + for (const auto& output_name : outputs) { + shaper.AddShape(output_name, output_shape); + } + } + + InlinedVector input_indices; input_indices.push_back(operand_indices.at(input)); ORT_RETURN_IF_ERROR(AddScalarOperand(model_builder, input_indices, axis)); - const auto count = gsl::narrow(outputs.size()); ORT_RETURN_IF_ERROR(AddScalarOperand(model_builder, input_indices, count)); const OperandType& input_operand_type = operand_types.at(input); @@ -206,7 +262,7 @@ Status BuildBatchMatMul(ModelBuilder& model_builder, const NodeUnit& node_unit) new_shape_i32.reserve(new_shape.size()); std::transform(new_shape.begin(), new_shape.end(), std::back_inserter(new_shape_i32), [](uint32_t d) { return gsl::narrow(d); }); - ORT_RETURN_IF_ERROR(AddNnapiReshape(model_builder, input, new_shape_name, new_shape_i32, output, nullptr)); + ORT_RETURN_IF_ERROR(AddNnapiReshape(model_builder, input, new_shape_name, new_shape_i32, output)); return Status::OK(); }; @@ -235,7 +291,8 @@ Status BuildBatchMatMul(ModelBuilder& model_builder, const NodeUnit& node_unit) { const std::string b_new_perm = model_builder.GetUniqueName(b + "/new_perm"), b_transposed = model_builder.GetUniqueName(b + "/transposed"); - ORT_RETURN_IF_ERROR(AddNnapiTranspose(model_builder, gemm_b_inputs.front(), b_new_perm, {0, 2, 1}, b_transposed)); + ORT_RETURN_IF_ERROR(AddNnapiTranspose(model_builder, gemm_b_inputs.front(), b_new_perm, + AsSpan({0, 2, 1}), b_transposed)); gemm_b_inputs.front() = b_transposed; } @@ -282,15 +339,18 @@ Status BuildBatchMatMul(ModelBuilder& model_builder, const NodeUnit& node_unit) const auto& operand_indices = model_builder.GetOperandIndices(); const auto& operand_types = model_builder.GetOperandTypes(); auto& shaper = model_builder.GetShaper(); - std::vector input_indices; + InlinedVector input_indices; input_indices.push_back(operand_indices.at(a)); // A input_indices.push_back(operand_indices.at(b_transposed)); // B' input_indices.push_back(operand_indices.at(bias)); // C int32_t fuse_code = ANEURALNETWORKS_FUSED_NONE; ORT_RETURN_IF_ERROR(AddScalarOperand(model_builder, input_indices, fuse_code)); - ORT_RETURN_IF_ERROR(shaper.FC(a, b_transposed, output)); - const OperandType output_operand_type(operand_types.at(a).type, shaper[output]); + const auto a_dimen = shaper[a]; + const auto b_transposed_dimen = shaper[b_transposed]; // num_units, input_size + Shape output_dimen{a_dimen[0], b_transposed_dimen[0]}; + shaper.AddShape(output, output_dimen); + const OperandType output_operand_type(operand_types.at(a).type, output_dimen); ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_FULLY_CONNECTED, input_indices, {output}, {output_operand_type})); return Status::OK(); @@ -325,13 +385,35 @@ Status BuildBatchMatMul(ModelBuilder& model_builder, const NodeUnit& node_unit) const auto& operand_indices = model_builder.GetOperandIndices(); const auto& operand_types = model_builder.GetOperandTypes(); auto& shaper = model_builder.GetShaper(); - std::vector input_indices; + InlinedVector input_indices; input_indices.reserve(inputs.size() + 1); std::transform(inputs.begin(), inputs.end(), std::back_inserter(input_indices), [&operand_indices](const std::string& input) { return operand_indices.at(input); }); const int32_t axis = 0; ORT_RETURN_IF_ERROR(AddScalarOperand(model_builder, input_indices, axis)); - ORT_RETURN_IF_ERROR(shaper.Concat(inputs, axis, output)); + + // Calculate_concat_output_shape + { + std::vector dimens; + for (const auto& input_name : inputs) { + const Shape dimen = shaper[input_name]; + dimens.push_back(dimen); + } + + // If one of the inputs has dynamic shape (at axis), we will keep the dimen[axis] as 0 (dynamic) + auto output_dimen = dimens[0]; + if (output_dimen[axis] != 0) { + for (size_t i = 1; i < dimens.size(); i++) { + if (dimens[i][axis] == 0) { + output_dimen[axis] = 0; + break; + } + output_dimen[axis] += dimens[i][axis]; + } + } + shaper.AddShape(output, output_dimen); + } + OperandType output_operand_type = operand_types.at(inputs[0]); output_operand_type.SetDimensions(shaper[output]); ORT_RETURN_IF_ERROR(model_builder.AddOperation(ANEURALNETWORKS_CONCATENATION, @@ -353,4 +435,27 @@ Status BuildBatchMatMul(ModelBuilder& model_builder, const NodeUnit& node_unit) return Status::OK(); } +Status PerformBroadcasting(const Shape& shape1, const Shape& shape2, Shape& output_shape) { + bool shape1_is_bigger = shape1.size() >= shape2.size(); + auto max_shape = shape1_is_bigger ? shape1 : shape2; + const auto& min_shape = shape1_is_bigger ? shape2 : shape1; + for (int i = gsl::narrow(max_shape.size()) - 1, + j = gsl::narrow(min_shape.size()) - 1; + i >= 0 && j >= 0; + i--, j--) { + int dim_max_shape = max_shape[i]; + int dim_min_shape = min_shape[j]; + if (dim_max_shape != dim_min_shape) { + ORT_RETURN_IF_NOT(dim_max_shape == 1 || dim_min_shape == 1, + "Dimensions are not compatible, dim1: ", dim_max_shape, + "dim2: ", dim_min_shape); + if (dim_max_shape == 1) { + max_shape[i] = dim_min_shape; + } + } + } + output_shape = std::move(max_shape); + return Status::OK(); +} + } // namespace onnxruntime::nnapi::op_builder_helpers diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder_helpers.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder_helpers.h index 20ffaf230d..8500919a78 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder_helpers.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder_helpers.h @@ -15,7 +15,7 @@ namespace onnxruntime::nnapi::op_builder_helpers { // adds a scalar operand to the NNAPI model and appends its index to `input_indices` template -Status AddScalarOperand(ModelBuilder& model_builder, std::vector& input_indices, T scalar_value) { +Status AddScalarOperand(ModelBuilder& model_builder, InlinedVector& input_indices, T scalar_value) { uint32_t index = 0; ORT_RETURN_IF_ERROR(model_builder.AddOperandFromScalar(std::move(scalar_value), index)); input_indices.push_back(index); @@ -25,15 +25,14 @@ Status AddScalarOperand(ModelBuilder& model_builder, std::vector& inpu // adds ANEURALNETWORKS_TRANSPOSE operation Status AddNnapiTranspose(ModelBuilder& model_builder, const std::string& data_input, - const std::string& perm_input, - const std::vector& perm, + const std::string& perm_input, const gsl::span perm, const std::string& output); // adds ANEURALNETWORKS_RESHAPE operation Status AddNnapiReshape(ModelBuilder& model_builder, const std::string& data_input, const std::string& shape_input, const std::vector& shape_value, - const std::string& output, const Shape* output_shape); + const std::string& output); // adds ANEURALNETWORKS_SPLIT operation Status AddNnapiSplit(ModelBuilder& model_builder, @@ -48,4 +47,7 @@ bool IsSupportedBatchMatMul(const NodeUnit& node_unit, int32_t nnapi_feature_lev // note: the pre-conditions of this function are checked in IsSupportedBatchMatMul() Status BuildBatchMatMul(ModelBuilder& model_builder, const NodeUnit& node_unit); +// performs broadcasting operation on two shapes to make them compatible +Status PerformBroadcasting(const Shape& shape1, const Shape& shape2, Shape& output_shape); + } // namespace onnxruntime::nnapi::op_builder_helpers diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.cc deleted file mode 100644 index caa7835802..0000000000 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.cc +++ /dev/null @@ -1,558 +0,0 @@ -// Copyright (c) Microsoft Corporation. All rights reserved. -// Licensed under the MIT License. - -#include "core/providers/common.h" - -#include "shaper.h" -#include "helper.h" - -namespace onnxruntime { -namespace nnapi { - -std::pair ComputeConvOutputShape(const uint32_t input_size_y, const uint32_t input_size_x, - const uint32_t weight_size_y, const uint32_t weight_size_x, - const std::vector& onnx_pads, - const std::vector& onnx_strides, - const std::vector& onnx_dilations) { - int32_t padding_top = onnx_pads[0]; - int32_t padding_bottom = onnx_pads[2]; - int32_t padding_left = onnx_pads[1]; - int32_t padding_right = onnx_pads[3]; - int32_t stride_y = onnx_strides[0]; - int32_t stride_x = onnx_strides[1]; - int32_t dilation_y = onnx_dilations[0]; - int32_t dilation_x = onnx_dilations[1]; - - int64_t output_size_y = - 0 == input_size_y - ? 0 - : onnxruntime::ComputeOutputShape(input_size_y, - stride_y, weight_size_y, dilation_y, - padding_top, padding_bottom); - - int64_t output_size_x = - 0 == input_size_x - ? 0 - : onnxruntime::ComputeOutputShape(input_size_x, - stride_x, weight_size_x, dilation_x, - padding_left, padding_right); - - return std::make_pair(static_cast(output_size_y), static_cast(output_size_x)); -} - -#define SHAPER_FUNC(FUNC, ...) \ - ORT_RETURN_IF_ERROR(FUNC##Impl(__VA_ARGS__)); \ - shape_ops_.push_back( \ - [__VA_ARGS__](Shaper& shaper) { \ - return shaper.FUNC##Impl(__VA_ARGS__); \ - }); \ - return Status::OK(); - -Status Shaper::Conv(const std::string& input_name, - const std::string& weight_name, - const std::vector& onnx_pads, - const std::vector& onnx_strides, - const std::vector& onnx_dilations, - bool nchw, - const std::string& output_name) { - SHAPER_FUNC(Conv, - input_name, weight_name, - onnx_pads, onnx_strides, onnx_dilations, - nchw, - output_name); -} - -Status Shaper::DepthwiseConv(const std::string& input_name, - const std::string& weight_name, - const std::vector& onnx_pads, - const std::vector& onnx_strides, - const std::vector& onnx_dilations, - bool nchw, - const std::string& output_name) { - SHAPER_FUNC(DepthwiseConv, - input_name, weight_name, - onnx_pads, onnx_strides, onnx_dilations, - nchw, - output_name); -} - -Status Shaper::Pool(const std::string& input_name, - const std::vector& onnx_pads, - const std::vector& onnx_strides, - const std::vector& kernel_shape, - bool nchw, - const std::string& output_name) { - SHAPER_FUNC(Pool, - input_name, - onnx_pads, onnx_strides, kernel_shape, - nchw, - output_name); -} - -Status Shaper::Reshape(const std::string& input_name, - const std::vector& shape, - const std::string& output_name) { - SHAPER_FUNC(Reshape, input_name, shape, output_name); -} - -Status Shaper::Transpose(const std::string& input_name, - const std::vector& perm, - const std::string& output_name) { - SHAPER_FUNC(Transpose, input_name, perm, output_name); -} - -Status Shaper::Eltwise(const std::string& input1_name, - const std::string& input2_name, - const std::string& output_name) { - SHAPER_FUNC(Eltwise, input1_name, input2_name, output_name); -} - -Status Shaper::Identity(const std::string& input_name, - const std::string& output_name) { - SHAPER_FUNC(Identity, input_name, output_name); -} - -Status Shaper::FC(const std::string& input1_name, const std::string& input2_name, - const std::string& output_name) { - SHAPER_FUNC(FC, input1_name, input2_name, output_name); -} - -Status Shaper::Concat(const std::vector& input_names, - const int32_t axis, - const std::string& output_name) { - SHAPER_FUNC(Concat, input_names, axis, output_name); -} - -Status Shaper::Split(const std::string& input_name, int32_t axis, - const std::vector& output_names) { - SHAPER_FUNC(Split, input_name, axis, output_names); -} - -Status Shaper::Squeeze(const std::string& input_name, - const std::vector& axes, - const std::string& output_name) { - SHAPER_FUNC(Squeeze, input_name, axes, output_name); -} - -Status Shaper::DepthToSpace(const std::string& input_name, - const int32_t blocksize, - bool nchw, - const std::string& output_name) { - SHAPER_FUNC(DepthToSpace, input_name, blocksize, nchw, output_name); -} - -Status Shaper::Gather(const std::string& input_name1, - const std::string& input_name2, - const int32_t axis, - const std::string& output_name) { - SHAPER_FUNC(Gather, input_name1, input_name2, axis, output_name); -} - -Status Shaper::ResizeUsingScales(const std::string& input_name, - const float scale_h, const float scale_w, - bool nchw, - const std::string& output_name) { - SHAPER_FUNC(ResizeUsingScales, input_name, scale_h, scale_w, nchw, output_name); -} - -Status Shaper::ResizeUsingOutputSizes(const std::string& input_name, - const uint32_t output_h, const uint32_t output_w, - bool nchw, - const std::string& output_name) { - SHAPER_FUNC(ResizeUsingOutputSizes, input_name, output_h, output_w, nchw, output_name); -} - -Status Shaper::Pad(const std::string& input_name, - const std::vector& pads, - const std::string& output_name) { - SHAPER_FUNC(Pad, input_name, pads, output_name); -} - -#undef SHAPER_FUNC - -Status Shaper::ConvImpl(const std::string& input_name, - const std::string& weight_name, - const std::vector& onnx_pads, - const std::vector& onnx_strides, - const std::vector& onnx_dilations, - bool nchw, - const std::string& output_name) { - const Shape& input_dimen = shape_map_.at(input_name); - const Shape& weight_dimen = shape_map_.at(weight_name); // num_output, height, width, num_input - - const auto input_size_y = nchw ? input_dimen[2] : input_dimen[1]; - const auto input_size_x = nchw ? input_dimen[3] : input_dimen[2]; - const auto weight_size_y = weight_dimen[1]; - const auto weight_size_x = weight_dimen[2]; - - uint32_t output_size_y, output_size_x; - std::tie(output_size_y, output_size_x) = - ComputeConvOutputShape(input_size_y, input_size_x, - weight_size_y, weight_size_x, - onnx_pads, onnx_strides, onnx_dilations); - Shape output_dimen; - if (nchw) { - output_dimen = {input_dimen[0], weight_dimen[0], output_size_y, output_size_x}; - } else { // nhwc - output_dimen = {input_dimen[0], output_size_y, output_size_x, weight_dimen[0]}; - } - - shape_map_[output_name] = output_dimen; - return Status::OK(); -} - -Status Shaper::DepthwiseConvImpl(const std::string& input_name, - const std::string& weight_name, - const std::vector& onnx_pads, - const std::vector& onnx_strides, - const std::vector& onnx_dilations, - bool nchw, - const std::string& output_name) { - const Shape& input_dimen = shape_map_.at(input_name); - const Shape& weight_dimen = shape_map_.at(weight_name); // 1, height, width, num_output - - const auto input_size_y = nchw ? input_dimen[2] : input_dimen[1]; - const auto input_size_x = nchw ? input_dimen[3] : input_dimen[2]; - const auto weight_size_y = weight_dimen[1]; - const auto weight_size_x = weight_dimen[2]; - - uint32_t output_size_y, output_size_x; - std::tie(output_size_y, output_size_x) = - ComputeConvOutputShape(input_size_y, input_size_x, - weight_size_y, weight_size_x, - onnx_pads, onnx_strides, onnx_dilations); - - Shape output_dimen; - if (nchw) { - output_dimen = {input_dimen[0], weight_dimen[3], output_size_y, output_size_x}; - } else { // nhwc - output_dimen = {input_dimen[0], output_size_y, output_size_x, weight_dimen[3]}; - } - shape_map_[output_name] = output_dimen; - return Status::OK(); -} - -Status Shaper::PoolImpl(const std::string& input_name, - const std::vector& onnx_pads, - const std::vector& onnx_strides, - const std::vector& kernel_shape, - bool nchw, - const std::string& output_name) { - const Shape& input_dimen = shape_map_.at(input_name); - const auto input_size_y = nchw ? input_dimen[2] : input_dimen[1]; - const auto input_size_x = nchw ? input_dimen[3] : input_dimen[2]; - const auto weight_size_y = kernel_shape[0]; - const auto weight_size_x = kernel_shape[1]; - - uint32_t output_size_y, output_size_x; - std::tie(output_size_y, output_size_x) = - ComputeConvOutputShape(input_size_y, input_size_x, - weight_size_y, weight_size_x, - onnx_pads, onnx_strides, {1, 1} /* onnx_dilations */); - Shape output_dimen; - if (nchw) { - output_dimen = {input_dimen[0], input_dimen[1], output_size_y, output_size_x}; - } else { // nhwc - output_dimen = {input_dimen[0], output_size_y, output_size_x, input_dimen[3]}; - } - - shape_map_[output_name] = output_dimen; - return Status::OK(); -} - -Status Shaper::ReshapeImpl(const std::string& input_name, - const std::vector& shape, - const std::string& output_name) { - const Shape& input_dimen = shape_map_.at(input_name); - uint32_t input_size = Product(input_dimen); - std::vector output_dimen(shape.size()); - - int64_t capacity = 1; - int unk_dim_idx = -1; - for (size_t i = 0; i < shape.size(); i++) { - int32_t dim_i = shape[i]; - ORT_RETURN_IF_NOT(dim_i != 0, "NNAPI does not support 0 reshape dimension"); - if (dim_i == -1) { - ORT_RETURN_IF_NOT(unk_dim_idx == -1, "Only one input dimension of Attr(shape) can be unknown!"); - unk_dim_idx = static_cast(i); - } else { - capacity *= dim_i; - output_dimen[i] = static_cast(dim_i); - } - } - - if (unk_dim_idx != -1) { - if (input_size == 0) - output_dimen[unk_dim_idx] = 0; - else - output_dimen[unk_dim_idx] = static_cast(input_size / capacity); - - capacity *= output_dimen[unk_dim_idx]; - } - - ORT_RETURN_IF_NOT(capacity == input_size, "Invalid shape is given!"); - - shape_map_[output_name] = output_dimen; - return Status::OK(); -} - -Status Shaper::TransposeImpl(const std::string& input_name, - const std::vector& perm, - const std::string& output_name) { - const Shape& input_dimen = shape_map_.at(input_name); - - ORT_RETURN_IF_NOT(perm.size() == input_dimen.size(), "Invalid perm is given!"); - - size_t size = input_dimen.size(); - Shape output_dimen(size); - for (size_t i = 0; i < size; i++) - output_dimen[i] = input_dimen[perm[i]]; - - shape_map_[output_name] = output_dimen; - return Status::OK(); -} - -Status Shaper::EltwiseImpl(const std::string& input1_name, - const std::string& input2_name, - const std::string& output_name) { - const Shape& shape1 = shape_map_.at(input1_name); - const Shape& shape2 = shape_map_.at(input2_name); - - // broadcasting support - bool shape1IsBigger = shape1.size() >= shape2.size(); - auto max_shape = shape1IsBigger ? shape1 : shape2; - auto min_shape = shape1IsBigger ? shape2 : shape1; - for (int i = (int)max_shape.size() - 1, - j = (int)min_shape.size() - 1; - i >= 0 && j >= 0; - i--, j--) { - int dim_max_shape = max_shape[i]; - int dim_min_shape = min_shape[j]; - if (dim_max_shape != dim_min_shape) { - ORT_RETURN_IF_NOT(dim_max_shape == 1 || dim_min_shape == 1, - "Dimensions are not compatible, dim1: ", std::to_string(dim_max_shape), - "dim2: ", std::to_string(dim_min_shape)); - } - - if (dim_max_shape == 0 || dim_min_shape == 0) { - max_shape[i] = 0; - } else if (dim_max_shape < dim_min_shape) { - max_shape[i] = dim_min_shape; - } - } - - shape_map_[output_name] = max_shape; - return Status::OK(); -} - -Status Shaper::IdentityImpl(const std::string& input_name, - const std::string& output_name) { - shape_map_[output_name] = shape_map_.at(input_name); - return Status::OK(); -} - -Status Shaper::FCImpl(const std::string& input1_name, const std::string& input2_name, - const std::string& output_name) { - // Currently we only support A*B'+C - const Shape& input1_dimen = shape_map_.at(input1_name); - const Shape& input2_dimen = shape_map_.at(input2_name); // num_units, input_size - Shape output_dimen{input1_dimen[0], input2_dimen[0]}; - shape_map_[output_name] = output_dimen; - return Status::OK(); -} - -Status Shaper::ConcatImpl(const std::vector& input_names, - const int32_t axis, - const std::string& output_name) { - std::vector dimens; - for (const auto& input_name : input_names) { - const Shape& dimen = shape_map_.at(input_name); - dimens.push_back(dimen); - } - - // If one of the inputs has dynamic shape (at axis), we will keep the dimen[axis] as 0 (dynamic) - auto output_dimen = dimens[0]; - if (output_dimen[axis] != 0) { - for (size_t i = 1; i < dimens.size(); i++) { - if (dimens[i][axis] == 0) { - output_dimen[axis] = 0; - break; - } - output_dimen[axis] += dimens[i][axis]; - } - } - - shape_map_[output_name] = output_dimen; - return Status::OK(); -} - -Status Shaper::SplitImpl(const std::string& input_name, int32_t axis, - const std::vector& output_names) { - const auto& input_shape = shape_map_.at(input_name); - const auto count = static_cast(output_names.size()); - - ORT_RETURN_IF_NOT(input_shape[axis] % count == 0, - "count [", count, "] does not evenly divide dimension ", axis, " [", input_shape[axis], "]"); - - Shape output_shape = input_shape; - output_shape[axis] = input_shape[axis] / count; - - for (const auto& output_name : output_names) { - shape_map_[output_name] = output_shape; - } - - return Status::OK(); -} - -Status Shaper::SqueezeImpl(const std::string& input_name, - const std::vector& axes, - const std::string& output_name) { - const Shape& input_dimen = shape_map_.at(input_name); - int32_t input_size = static_cast(input_dimen.size()); - std::unordered_set axes_to_be_squeezed; - - // If the Op is squeezing all by not specifying axes, the axes is pre-populate - // with axes of all single dimensions by the caller - for (const auto& axis : axes) - axes_to_be_squeezed.insert(axis); - - // Make output dimensions - std::vector output_dimen; - output_dimen.reserve(input_size - axes_to_be_squeezed.size()); - for (int32_t i = 0; i < input_size; i++) { - if (!Contains(axes_to_be_squeezed, i)) - output_dimen.push_back(input_dimen[i]); - } - - // In case of a tensor has all 1's in dimension such as {1,1,1,1} and gets squeezed all - // the output shape will be {1} - if (output_dimen.empty()) - output_dimen.push_back(1); - - shape_map_[output_name] = output_dimen; - return Status::OK(); -} - -Status Shaper::DepthToSpaceImpl(const std::string& input_name, - const int32_t blocksize, - bool nchw, - const std::string& output_name) { - const Shape& input_dimen = shape_map_.at(input_name); - - // Make output dimensions - Shape output_dimen = shape_map_.at(input_name); - if (nchw) { - output_dimen[0] = input_dimen[0]; - output_dimen[1] = input_dimen[1] / (blocksize * blocksize); - output_dimen[2] = input_dimen[2] * blocksize; - output_dimen[3] = input_dimen[3] * blocksize; - } else { // nhwc - output_dimen[0] = input_dimen[0]; - output_dimen[1] = input_dimen[1] * blocksize; - output_dimen[2] = input_dimen[2] * blocksize; - output_dimen[3] = input_dimen[3] / (blocksize * blocksize); - } - - shape_map_[output_name] = output_dimen; - return Status::OK(); -} - -Status Shaper::GatherImpl(const std::string& input_name1, - const std::string& input_name2, - const int32_t axis, - const std::string& output_name) { - const Shape& input_dimen = shape_map_.at(input_name1); - const Shape& indices_dimen = shape_map_.at(input_name2); - - std::vector output_dimen; - output_dimen.reserve(indices_dimen.size() + input_dimen.size() - 1); - - // Calculate the output dim - for (int32_t i = 0; i < axis; ++i) - output_dimen.push_back(input_dimen[i]); - - for (const auto dim : indices_dimen) - output_dimen.push_back(dim); - - for (size_t i = axis + 1; i < input_dimen.size(); ++i) - output_dimen.push_back(input_dimen[i]); - - shape_map_[output_name] = output_dimen; - return Status::OK(); -} - -Status Shaper::ResizeUsingScalesImpl(const std::string& input_name, - const float scale_h, const float scale_w, - bool nchw, - const std::string& output_name) { - Shape output_dimen = shape_map_.at(input_name); - if (nchw) { - output_dimen[2] = static_cast(output_dimen[2] * scale_h); - output_dimen[3] = static_cast(output_dimen[3] * scale_w); - } else { // nhwc - output_dimen[1] = static_cast(output_dimen[1] * scale_h); - output_dimen[2] = static_cast(output_dimen[2] * scale_w); - } - shape_map_[output_name] = output_dimen; - return Status::OK(); -} - -Status Shaper::ResizeUsingOutputSizesImpl(const std::string& input_name, - const uint32_t output_h, const uint32_t output_w, - bool nchw, - const std::string& output_name) { - Shape output_dimen = shape_map_.at(input_name); - if (nchw) { - output_dimen[2] = output_h; - output_dimen[3] = output_w; - } else { // nhwc - output_dimen[1] = output_h; - output_dimen[2] = output_w; - } - shape_map_[output_name] = output_dimen; - return Status::OK(); -} - -Status Shaper::PadImpl(const std::string& input_name, - const std::vector& pads, - const std::string& output_name) { - Shape padded_shape = shape_map_.at(input_name); - const size_t rank = padded_shape.size(); - ORT_RETURN_IF_NOT(pads.size() == 2 * rank, "Expected 2*rank (", 2 * rank, ") pad values but got ", pads.size()); - for (size_t i = 0; i < rank; ++i) { - padded_shape[i] += pads[2*i] + pads[2*i + 1]; - } - shape_map_[output_name] = padded_shape; - return Status::OK(); -} - -void Shaper::AddShape(const std::string& name, const Shape& shape) { - shape_map_[name] = shape; -} - -Status Shaper::UpdateShape(const std::string& name, const Shape& new_shape) { - const Shape& old_shape = shape_map_.at(name); - if (old_shape != new_shape) { - ORT_RETURN_IF_NOT(Product(old_shape) == 0 || !old_shape.empty(), - "The shape should be same size or old shape has size 0 (dynamic shape)"); - - shape_map_[name] = new_shape; - } - - return Status::OK(); -} - -Status Shaper::UpdateDynamicDimensions() { - for (auto& shape_op : shape_ops_) - ORT_RETURN_IF_ERROR(shape_op(*this)); - - return Status::OK(); -} - -void Shaper::Clear() { - shape_map_.clear(); - shape_ops_.clear(); -} - -} // namespace nnapi -} // namespace onnxruntime diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.h index 31eb9aa9fa..2fbb2b221c 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.h @@ -6,149 +6,38 @@ #include #include #include -#include -#include "core/common/status.h" +#include "core/common/inlined_containers.h" +#include "core/providers/nnapi/nnapi_builtin/builders/helper.h" namespace onnxruntime { namespace nnapi { class Shaper { public: - using Shape = std::vector; + using Shape = InlinedVector; - void AddShape(const std::string& name, const Shape& shape); - inline const Shape& operator[](const std::string& key) const { - return shape_map_.at(key); + Shaper(const GraphViewer& graph_viewer) : graph_viewer_(&graph_viewer) {} + + void AddShape(const std::string& name, const Shape& shape) { + shape_map_[name] = shape; } - common::Status Conv(const std::string& input_name, - const std::string& weight_name, - const std::vector& onnx_pads, - const std::vector& onnx_strides, - const std::vector& onnx_dilations, - bool nchw, - const std::string& output_name); + inline Shape operator[](const std::string& key) const { + auto it = shape_map_.find(key); + if (it != shape_map_.end()) { + return it->second; + } + const auto shape = GetShapeInfoFromNodeArg(*graph_viewer_, key); + return shape; + } - common::Status DepthwiseConv(const std::string& input_name, - const std::string& weight_name, - const std::vector& onnx_pads, - const std::vector& onnx_strides, - const std::vector& onnx_dilations, - bool nchw, - const std::string& output_name); - - common::Status Pool(const std::string& input_name, - const std::vector& onnx_pads, - const std::vector& onnx_strides, - const std::vector& kernel_shape, - bool nchw, - const std::string& output_name); - - common::Status Reshape(const std::string& input_name, const std::vector& shape, const std::string& output_name); - - common::Status Transpose(const std::string& input_name, const std::vector& perm, const std::string& output_name); - - common::Status Eltwise(const std::string& input1_name, const std::string& input2_name, const std::string& output_name); - - common::Status Identity(const std::string& input_name, const std::string& output_name); - - common::Status FC(const std::string& input1_name, const std::string& input2_name, const std::string& output_name); - - common::Status Concat(const std::vector& input_names, const int32_t axis, const std::string& output_name); - - common::Status Split(const std::string& input_name, int32_t axis, const std::vector& output_names); - - common::Status Squeeze(const std::string& input_name, const std::vector& axes, const std::string& output_name); - - common::Status DepthToSpace(const std::string& input_name, - const int32_t blocksize, - bool nchw, - const std::string& output_name); - - common::Status Gather(const std::string& input_name1, - const std::string& input_name2, - const int32_t axis, - const std::string& output_name); - - common::Status ResizeUsingScales(const std::string& input_name, - const float scale_h, const float scale_w, - bool nchw, - const std::string& output_name); - common::Status ResizeUsingOutputSizes(const std::string& input_name, - const uint32_t output_h, const uint32_t output_w, - bool nchw, - const std::string& output_name); - - // Note: `pads` should be in the layout expected by NNAPI, i.e., [begin_1, end_1, begin_2, end_2, ...] - common::Status Pad(const std::string& input_name, - const std::vector& pads, - const std::string& output_name); - - // If the shape of certain input is dynamic - // Use the following 2 functions to update the particular shape - // and calculate the new output shape - // Only perform this when the NNAPI model is finalized! - common::Status UpdateShape(const std::string& name, const Shape& new_shape); - common::Status UpdateDynamicDimensions(); - - void Clear(); + // Note: Original code to update shapes are removed for now due to lack of dynamic shape support in NNAPI EP. + // Can be added back and enhanced in the future if more support is available. private: - common::Status ConvImpl(const std::string& input_name, - const std::string& weight_name, - const std::vector& onnx_pads, - const std::vector& onnx_strides, - const std::vector& onnx_dilations, - bool nchw, - const std::string& output_name); - - common::Status DepthwiseConvImpl(const std::string& input_name, - const std::string& weight_name, - const std::vector& onnx_pads, - const std::vector& onnx_strides, - const std::vector& onnx_dilations, - bool nchw, - const std::string& output_name); - - common::Status PoolImpl(const std::string& input_name, - const std::vector& onnx_pads, - const std::vector& onnx_strides, - const std::vector& kernel_shape, - bool nchw, - const std::string& output_name); - - common::Status ReshapeImpl(const std::string& input_name, const std::vector& shape, const std::string& output_name); - common::Status TransposeImpl(const std::string& input_name, const std::vector& perm, const std::string& output_name); - common::Status EltwiseImpl(const std::string& input1_name, const std::string& input2_name, const std::string& output_name); - common::Status IdentityImpl(const std::string& input_name, const std::string& output_name); - common::Status FCImpl(const std::string& input1_name, const std::string& input2_name, const std::string& output_name); - common::Status ConcatImpl(const std::vector& input_names, const int32_t axis, const std::string& output_name); - common::Status SplitImpl(const std::string& input_name, int32_t axis, const std::vector& output_names); - common::Status SqueezeImpl(const std::string& input_names, const std::vector& axes, const std::string& output_name); - common::Status DepthToSpaceImpl(const std::string& input_names, - const int32_t blocksize, - bool nchw, - const std::string& output_name); - common::Status GatherImpl(const std::string& input_name1, - const std::string& input_name2, - const int32_t axis, - const std::string& output_name); - common::Status ResizeUsingScalesImpl(const std::string& input_name, - const float scale_h, const float scale_w, - bool nchw, - const std::string& output_name); - common::Status ResizeUsingOutputSizesImpl(const std::string& input_name, - const uint32_t output_h, const uint32_t output_w, - bool nchw, - const std::string& output_name); - - common::Status PadImpl(const std::string& input_name, - const std::vector& pads, - const std::string& output_name); - std::unordered_map shape_map_; - std::vector> shape_ops_; + const GraphViewer* graph_viewer_; }; } // namespace nnapi diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/model.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/model.cc index 2201af8ce2..8d6b7ae544 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/model.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/model.cc @@ -61,13 +61,14 @@ const android::nn::wrapper::OperandType& Model::GetInputType(const std::string& return operand_types_.at(name); } -android::nn::wrapper::OperandType Model::GetOutputType(const std::string& name, const Execution& execution) const { - const auto& nnapi_output_name = onnx_to_nnapi_output_map_.at(name); - const auto& output_type = operand_types_.at(nnapi_output_name); - android::nn::wrapper::OperandType type( - output_type.type, execution.GetShaper()[nnapi_output_name], output_type.operandType.scale, output_type.operandType.zeroPoint); - - return type; +android::nn::wrapper::OperandType Model::GetOutputType(const std::string& name, + const Execution& /* execution */) const { + // Note: Before we validate if it's required to get Shaper from execution (if encounter dynamic shapes, + // shape can get updated during execution), we commented the usage here for now. + /* android::nn::wrapper::OperandType type( + output_type.type, execution.GetShaper()[nnapi_output_name], output_type.operandType.scale, + output_type.operandType.zeroPoint); */ + return operand_types_.at(name); } void Model::SetInputMap(std::unordered_map&& input_map) { @@ -99,7 +100,7 @@ Status Model::PrepareForExecution(std::unique_ptr& execution) { RETURN_STATUS_ON_ERROR( nnapi_->ANeuralNetworksExecution_create(compilation_, &nnapi_execution)); - execution.reset(new Execution(*nnapi_execution, shaper_)); + execution.reset(new Execution(*nnapi_execution /*, shaper_*/)); return Status::OK(); } @@ -146,10 +147,9 @@ Model::NNMemory::NNMemory(const NnApi* /*nnapi*/, const char* name, size_t size) #pragma region Execution -Execution::Execution(ANeuralNetworksExecution& execution, const Shaper& shaper) +Execution::Execution(ANeuralNetworksExecution& execution /*, const Shaper& shaper */) : nnapi_(NnApiImplementation()), - execution_(&execution), - shaper_(shaper) { + execution_(&execution) { } Execution::~Execution() { @@ -160,10 +160,8 @@ Status Execution::SetInputBuffers(const std::vector& inputs) { for (size_t i = 0; i < inputs.size(); i++) { const auto& input(inputs[i]); ORT_RETURN_IF_ERROR(SetInputBuffer(static_cast(i), input)); - ORT_RETURN_IF_ERROR(shaper_.UpdateShape(input.name, input.type.dimensions)); } - ORT_RETURN_IF_ERROR(shaper_.UpdateDynamicDimensions()); return Status::OK(); } @@ -204,7 +202,7 @@ Status Execution::Predict(const std::vector& dynamic_outputs, std::vect uint32_t output_rank = 0; RETURN_STATUS_ON_ERROR(nnapi_->ANeuralNetworksExecution_getOutputOperandRank(execution_, i, &output_rank)); - std::vector output_shape(output_rank); + InlinedVector output_shape(output_rank); RETURN_STATUS_ON_ERROR(nnapi_->ANeuralNetworksExecution_getOutputOperandDimensions(execution_, i, output_shape.data())); dynamic_output_shapes.push_back(output_shape); diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/model.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/model.h index 394c765d84..53884a3cb8 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/model.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/model.h @@ -57,6 +57,7 @@ class Model { #endif public: + Model(); ~Model(); Model(const Model&) = delete; Model& operator=(const Model&) = delete; @@ -70,7 +71,7 @@ class Model { // Returns the data type and dimension of the given input/output // Please note the output type will have updated dimensions const android::nn::wrapper::OperandType& GetInputType(const std::string& name) const; - android::nn::wrapper::OperandType GetOutputType(const std::string& name, const Execution& execution) const; + android::nn::wrapper::OperandType GetOutputType(const std::string& name, const Execution& /* execution */) const; // Set the mapping between input/output name and ORT kernel context // input/output index, at execution time @@ -123,8 +124,6 @@ class Model { std::unordered_map operand_types_; std::unordered_set scalar_outputs_; - Shaper shaper_; - std::unordered_map input_map_; std::unordered_map output_map_; @@ -134,7 +133,6 @@ class Model { OrtMutex mutex_; - Model(); void AddInput(const std::string& name, const android::nn::wrapper::OperandType& operand_type); // It is possible that the actual output from NNAPI model is not the same as the name of @@ -145,8 +143,6 @@ class Model { void AddScalarOutput(const std::string& output_name); - void SetShaper(const Shaper& shaper) { shaper_ = shaper; } - int32_t GetNNAPIFeatureLevel() const; }; @@ -165,12 +161,14 @@ class Execution { }; public: - explicit Execution(ANeuralNetworksExecution& execution, const Shaper& shaper); + explicit Execution(ANeuralNetworksExecution& execution /* , const Shaper& shaper */); ~Execution(); Execution(const Execution&) = delete; Execution& operator=(const Execution&) = delete; - const Shaper& GetShaper() const { return shaper_; } + // Before we validate if we actually need to keep a shaper instance for Execution (if we have dynamic shape + // outputs, shape can get updated during execution), we commented out Shaper here for now. + /* const Shaper& GetShaper() const { return shaper_; } */ // Set the input/output data buffers // These need to be called before calling Predict() @@ -187,7 +185,7 @@ class Execution { const NnApi* nnapi_{nullptr}; ANeuralNetworksExecution* execution_; - Shaper shaper_; + /* Shaper shaper_; */ }; } // namespace nnapi diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc index 782b384a63..0a80486db6 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc @@ -208,7 +208,7 @@ static Status GetOutputBuffer(Ort::CustomOpApi& ort, OrtKernelContext* context, const nnapi::Model& model, const std::string& output_name, - const std::vector& output_shape, + const InlinedVector& output_shape, const android::nn::wrapper::Type output_type, void** output_buffer) { using namespace android::nn::wrapper; @@ -317,7 +317,7 @@ common::Status NnapiExecutionProvider::Compile(const std::vectorGetMappedInputIdx(input_name); const OrtValue* input_tensor = ort.KernelContext_GetInput(context, input_idx); auto* tensor_info = ort.GetTensorTypeAndShape(input_tensor); - std::vector dimensions; + InlinedVector dimensions; for (const auto& dim : ort.GetTensorShape(tensor_info)) dimensions.push_back(static_cast(dim)); @@ -413,7 +413,7 @@ common::Status NnapiExecutionProvider::Compile(const std::vectorSetOutputBuffers(outputs)); - std::vector> dynamic_output_shapes; + std::vector> dynamic_output_shapes; ORT_RETURN_IF_ERROR( execution->Predict(dynamic_shape_output_indices, dynamic_output_shapes)); diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.cc index 9d18c8d228..1728640b9b 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.cc @@ -22,7 +22,7 @@ namespace android { namespace nn { namespace wrapper { -OperandType::OperandType(Type type, const std::vector& d, float scale, int32_t zeroPoint) +OperandType::OperandType(Type type, const Shape& d, float scale, int32_t zeroPoint) : type(type), dimensions(d) { operandType = { /*.type = */ static_cast(type), @@ -33,7 +33,7 @@ OperandType::OperandType(Type type, const std::vector& d, float scale, }; } -OperandType::OperandType(Type type, const std::vector& d, SymmPerChannelQuantParams&& channelQuant) +OperandType::OperandType(Type type, const Shape& d, SymmPerChannelQuantParams&& channelQuant) : type(type), dimensions(d), channelQuant(std::move(channelQuant)) { operandType = { /*.type = */ static_cast(type), @@ -101,7 +101,7 @@ size_t OperandType::GetOperandBlobByteSize() const { return num_elements * GetElementByteSize(); } -void OperandType::SetDimensions(const std::vector& d) { +void OperandType::SetDimensions(const Shape& d) { dimensions = d; operandType.dimensionCount = static_cast(dimensions.size()); operandType.dimensions = dimensions.size() > 0 ? dimensions.data() : nullptr; diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.h index a01f36d981..88eb94f831 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.h @@ -22,6 +22,10 @@ #include "NeuralNetworksTypes.h" +#include "core/providers/nnapi/nnapi_builtin/builders/shaper.h" + +using Shape = onnxruntime::nnapi::Shaper::Shape; + namespace android { namespace nn { namespace wrapper { @@ -122,11 +126,11 @@ struct SymmPerChannelQuantParams { struct OperandType { ANeuralNetworksOperandType operandType; Type type; - std::vector dimensions; + Shape dimensions; std::optional channelQuant; - explicit OperandType(Type type, const std::vector& d, float scale = 0.0f, int32_t zeroPoint = 0); - explicit OperandType(Type type, const std::vector& d, SymmPerChannelQuantParams&& channelQuant); + explicit OperandType(Type type, const Shape& d, float scale = 0.0f, int32_t zeroPoint = 0); + explicit OperandType(Type type, const Shape& d, SymmPerChannelQuantParams&& channelQuant); OperandType(const OperandType& other); OperandType& operator=(const OperandType& other); @@ -137,7 +141,7 @@ struct OperandType { // Get the whole blob size in bytes size_t GetOperandBlobByteSize() const; - void SetDimensions(const std::vector& d); + void SetDimensions(const Shape& d); operator ANeuralNetworksOperandType() const { return operandType; } };