saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-30 20:18:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

[QDQ] Move NNAPI EP to use NodeUnitIODef for non-QDQ ops (#10237)

Browse source
This commit is contained in:
Guoyu Wang 2022-01-18 12:54:58 -08:00 committed by GitHub
parent 33dd2f8f5e
commit 6ae22d562b
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 1202 additions and 1187 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

207
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc
View file

@ -5,22 +5,18 @@
#include <string>
#include <vector>
#include <core/common/safeint.h>
#include <core/common/logging/logging.h>
#include <core/framework/tensorprotoutils.h>
#include <core/graph/graph.h>
#include <core/graph/graph_viewer.h>
#include <core/providers/common.h>
#include "helper.h"
#include "core/common/safeint.h"
#include "core/common/logging/logging.h"
#include "core/framework/tensorprotoutils.h"
#include "core/graph/graph.h"
#include "core/graph/graph_viewer.h"
#include "core/providers/common.h"
#include "core/providers/shared/node_unit/node_unit.h"
#include "core/providers/shared/utils/utils.h"
#include "helper.h"
#include "op_support_checker.h"
using onnxruntime::NodeUnit;
using std::string;
using std::vector;
namespace onnxruntime {
namespace nnapi {
@ -72,16 +68,11 @@ QLinearOpType GetQLinearOpType(const onnxruntime::Node& node) {
return QLinearOpType::Unknown;
}
ConvType GetConvType(const onnxruntime::Node& node, const InitializedTensorSet& initializers) {
const auto& op_type = node.OpType();
bool is_qlinear_conv = (op_type == "QLinearConv");
ORT_ENFORCE(op_type == "Conv" || is_qlinear_conv);
NodeAttrHelper helper(node);
ConvType GetConvType(const NodeUnit& node_unit, const InitializedTensorSet& initializers) {
NodeAttrHelper helper(node_unit);
const auto group = helper.Get("group", 1);
size_t w_idx = is_qlinear_conv ? 3 : 1;
const auto& weight = node.InputDefs()[w_idx]->Name();
const auto& weight = node_unit.Inputs()[1].node_arg.Name();
const auto& weight_tensor = *initializers.at(weight);
// For ONNX we only have 1 conv ops
@ -104,13 +95,13 @@ bool IsQLinearBinaryOp(QLinearOpType qlinear_op_type) {
qlinear_op_type == QLinearOpType::QLinearAdd;
}
bool HasValidUnaryOpQuantizedInputs(const Node& node) {
bool HasValidUnaryOpQuantizedInputs(const NodeUnit& node_unit) {
int32_t input_type;
if (!GetType(*node.InputDefs()[0], input_type))
if (!GetType(node_unit.Inputs()[0].node_arg, input_type))
return false;
if (input_type != ONNX_NAMESPACE::TensorProto_DataType_UINT8) {
LOGS_DEFAULT(VERBOSE) << "[" << node.OpType()
LOGS_DEFAULT(VERBOSE) << "[" << node_unit.OpType()
<< "] Input type: [" << input_type
<< "] is not supported for now";
return false;
@ -119,18 +110,18 @@ bool HasValidUnaryOpQuantizedInputs(const Node& node) {
return true;
}
bool HasValidBinaryOpQuantizedInputs(const Node& node) {
auto op_type = GetQLinearOpType(node);
bool HasValidBinaryOpQuantizedInputs(const NodeUnit& node_unit) {
auto op_type = GetQLinearOpType(node_unit.GetNode());
int32_t a_input_type, b_input_type;
if (!IsQLinearBinaryOp(op_type)) {
LOGS_DEFAULT(VERBOSE) << "[" << node.OpType() << "] is not a binary qlinear op";
LOGS_DEFAULT(VERBOSE) << "[" << node_unit.OpType() << "] is not a binary qlinear op";
return false;
}
const auto input_defs(node.InputDefs());
if (!GetType(*input_defs[0], a_input_type))
const auto& inputs = node_unit.Inputs();
if (!GetType(inputs[0].node_arg, a_input_type))
return false;
if (!GetType(*input_defs[3], b_input_type))
if (!GetType(inputs[1].node_arg, b_input_type))
return false;
// QlinearConv supports u8u8 or u8s8
@ -143,7 +134,7 @@ bool HasValidBinaryOpQuantizedInputs(const Node& node) {
if (a_input_type != ONNX_NAMESPACE::TensorProto_DataType_UINT8 ||
(!is_qlinear_conv && a_input_type != b_input_type) ||
(is_qlinear_conv && !has_valid_qlinear_conv_weight)) {
LOGS_DEFAULT(VERBOSE) << "[" << node.OpType()
LOGS_DEFAULT(VERBOSE) << "[" << node_unit.OpType()
<< "] A Input type: [" << a_input_type
<< "] B Input type: [" << b_input_type
<< "] is not supported for now";
@ -153,32 +144,41 @@ bool HasValidBinaryOpQuantizedInputs(const Node& node) {
return true;
}
bool HasValidQuantizationScales(const InitializedTensorSet& initializers, const Node& node,
const std::vector<size_t>& indices, const OpSupportCheckParams& params) {
const auto& op_type = node.OpType();
auto qlinear_op_type = GetQLinearOpType(node);
bool HasValidQuantizationScales(const InitializedTensorSet& initializers, const NodeUnit& node_unit,
const std::vector<size_t>& indices, const OpSupportCheckParams& params, bool is_input) {
const auto& op_type = node_unit.OpType();
auto qlinear_op_type = GetQLinearOpType(node_unit.GetNode());
bool is_qlinear_conv = (qlinear_op_type == QLinearOpType::QLinearConv);
bool is_qlinear_matmul = (qlinear_op_type == QLinearOpType::QLinearMatMul);
const auto input_defs(node.InputDefs());
const auto& io_defs = is_input ? node_unit.Inputs() : node_unit.Outputs();
for (const auto idx : indices) {
if (idx >= input_defs.size()) {
LOGS_DEFAULT(VERBOSE) << "HasValidQuantizationScales, Input index, " << idx
<< " >= input number, " << input_defs.size();
if (idx >= io_defs.size()) {
LOGS_DEFAULT(VERBOSE) << (is_input ? "Input" : "Output") << " index, " << idx
<< " >= size, " << io_defs.size()
<< " of NodeUnit: " << node_unit.Name();
return false;
}
const auto scale_name = input_defs[idx]->Name();
const auto& io_def = io_defs[idx];
if (!io_def.quant_param.has_value()) {
LOGS_DEFAULT(VERBOSE) << "HasValidQuantizationZeroPoints, Input index, " << idx
<< " has no quant_param";
return false;
}
const auto scale_name = io_def.quant_param->scale.Name();
if (!Contains(initializers, scale_name)) {
LOGS_DEFAULT(VERBOSE) << "The scale of " << op_type << " must be an initializer tensor";
return false;
}
// If this op is Qlinear[Conv/MatMul], we want to check u8s8 support for weight tensor (or B tensor for QlinearMatMul)
bool is_conv_matmul_weight = (is_qlinear_conv || is_qlinear_matmul) && idx == 4;
bool is_conv_matmul_weight = is_input && (is_qlinear_conv || is_qlinear_matmul) && idx == 1;
bool is_conv_matmul_u8s8_weight = false;
if (is_conv_matmul_weight) {
const auto& weight_tensor = *initializers.at(node.InputDefs()[3]->Name());
const auto& weight_tensor = *initializers.at(io_def.node_arg.Name());
is_conv_matmul_u8s8_weight = weight_tensor.data_type() == ONNX_NAMESPACE::TensorProto_DataType_INT8;
}
@ -205,7 +205,7 @@ bool HasValidQuantizationScales(const InitializedTensorSet& initializers, const
return false;
}
const auto& weight_tensor = *initializers.at(node.InputDefs()[3]->Name());
const auto& weight_tensor = *initializers.at(io_def.node_arg.Name());
if (weight_tensor.dims()[0] != scales_dim) {
LOGS_DEFAULT(VERBOSE) << op_type << " mismatch int8 per-channel quantization weight,"
<< " weight dimension[0] " << weight_tensor.dims()[0]
@ -218,30 +218,44 @@ bool HasValidQuantizationScales(const InitializedTensorSet& initializers, const
return true;
}
bool HasValidQuantizationZeroPoints(const InitializedTensorSet& initializers, const Node& node,
const std::vector<size_t>& indices) {
const auto& op_type = node.OpType();
auto qlinear_op_type = GetQLinearOpType(node);
bool HasValidQuantizationZeroPoints(const InitializedTensorSet& initializers, const NodeUnit& node_unit,
const std::vector<size_t>& indices, bool is_input) {
const auto& op_type = node_unit.OpType();
auto qlinear_op_type = GetQLinearOpType(node_unit.GetNode());
bool is_qlinear_conv = (qlinear_op_type == QLinearOpType::QLinearConv);
bool is_qlinear_matmul = (qlinear_op_type == QLinearOpType::QLinearMatMul);
const auto input_defs(node.InputDefs());
const auto& io_defs = is_input ? node_unit.Inputs() : node_unit.Outputs();
for (const auto idx : indices) {
if (idx >= input_defs.size()) {
LOGS_DEFAULT(VERBOSE) << "HasValidQuantizationZeroPoints, Input index, " << idx
<< " >= input number, " << input_defs.size();
if (idx >= io_defs.size()) {
LOGS_DEFAULT(VERBOSE) << "HasValidQuantizationZeroPoints, "
<< (is_input ? "Input" : "Output") << " index, " << idx
<< " >= size, " << io_defs.size();
return false;
}
const auto zero_point_name = input_defs[idx]->Name();
const auto& io_def = io_defs[idx];
if (!io_def.quant_param.has_value()) {
LOGS_DEFAULT(VERBOSE) << "HasValidQuantizationZeroPoints, Input index, " << idx
<< " has no quant_param";
return false;
}
// zero point is optional here
if (!io_def.quant_param->zero_point)
return true;
const auto& zero_point_name = io_def.quant_param->zero_point->Name();
if (!Contains(initializers, zero_point_name)) {
LOGS_DEFAULT(VERBOSE) << "The zero point of " << op_type << " must be an initializer tensor";
return false;
}
bool is_conv_matmul_weight = is_qlinear_conv && idx == 5;
bool is_conv_matmul_weight = is_input && (is_qlinear_conv || is_qlinear_matmul) && idx == 1;
bool is_conv_matmul_u8s8_weight = false;
if (is_conv_matmul_weight) {
const auto& weight_tensor = *initializers.at(node.InputDefs()[3]->Name());
const auto& weight_tensor = *initializers.at(io_def.node_arg.Name());
is_conv_matmul_u8s8_weight = weight_tensor.data_type() == ONNX_NAMESPACE::TensorProto_DataType_INT8;
}
@ -275,7 +289,7 @@ bool HasValidQuantizationZeroPoints(const InitializedTensorSet& initializers, co
// or a tensor with same channel as weight, for NNAPI we only support it be
// 0 (scalar) or all 0 (tensor), NNAPI will assume the zero point for per-channel
// quantization is 0 there is no input for it
const auto& weight_tensor = *initializers.at(node.InputDefs()[3]->Name());
const auto& weight_tensor = *initializers.at(io_def.node_arg.Name());
if (weight_tensor.dims()[0] != zero_dim && zero_dim != 1) {
LOGS_DEFAULT(VERBOSE) << op_type << " mismatch int8 per-channel quantization weight,"
<< " weight dimension[0] " << weight_tensor.dims()[0]
@ -284,7 +298,7 @@ bool HasValidQuantizationZeroPoints(const InitializedTensorSet& initializers, co
}
std::vector<uint8_t> unpacked_tensor;
auto status = onnxruntime::utils::UnpackInitializerData(zero_tensor, node.ModelPath(), unpacked_tensor);
auto status = onnxruntime::utils::UnpackInitializerData(zero_tensor, node_unit.ModelPath(), unpacked_tensor);
if (!status.IsOK()) {
LOGS_DEFAULT(ERROR) << "Qlinear[Conv/MatMul] error when unpack zero tensor: " << zero_point_name
<< ", error msg: " << status.ErrorMessage();
@ -306,33 +320,61 @@ bool HasValidQuantizationZeroPoints(const InitializedTensorSet& initializers, co
return true;
}
common::Status GetQuantizationScale(const InitializedTensorSet& initializers, const Node& node,
size_t idx, float& scale) {
std::vector<uint8_t> unpacked_tensor;
const auto& name = node.InputDefs()[idx]->Name();
const auto& scale_tensor = *initializers.at(name);
ORT_RETURN_IF_ERROR(
onnxruntime::utils::UnpackInitializerData(scale_tensor, node.ModelPath(), unpacked_tensor));
common::Status GetQuantizationScaleAndZeroPoint(
const InitializedTensorSet& initializers, const NodeUnitIODef& io_def, const Path& model_path,
float& scale, int32_t& zero_point) {
scale = 0.0f;
zero_point = 0;
if (!io_def.quant_param) { // Not a quantized IO
return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
"NodeArg: ", io_def.node_arg.Name(), " is not quantized");
}
const auto unpack_tensor = [&model_path](const InitializedTensorSet& initializers,
const std::string& name, std::vector<uint8_t>& unpacked_tensor) {
const auto& tensor = *initializers.at(name);
ORT_RETURN_IF_ERROR(
onnxruntime::utils::UnpackInitializerData(tensor, model_path, unpacked_tensor));
return Status::OK();
};
const auto& quant_param = *io_def.quant_param;
{ // get the scale
std::vector<uint8_t> unpacked_tensor;
const auto& name = quant_param.scale.Name();
ORT_RETURN_IF_ERROR(unpack_tensor(initializers, name, unpacked_tensor));
// The scale should be one or more floats
ORT_RETURN_IF(unpacked_tensor.size() < 4,
"The initializer [", name, "] should have one or more floats ",
"with size no less than 4, actual size: ", unpacked_tensor.size());
scale = reinterpret_cast<const float*>(unpacked_tensor.data())[0];
}
if (quant_param.zero_point) { // get the zero point if it's there
std::vector<uint8_t> unpacked_tensor;
const auto& name = quant_param.zero_point->Name();
ORT_RETURN_IF_ERROR(unpack_tensor(initializers, name, unpacked_tensor));
ORT_RETURN_IF(unpacked_tensor.empty(), "The initializer [", name, "] is empty");
// Onnx quantization uses uint8 [int8 not yet supported], need to cast to int32_t used by NNAPI
zero_point = static_cast<int32_t>(unpacked_tensor[0]);
}
// The scale should be one or more floats
ORT_RETURN_IF(unpacked_tensor.size() < 4, "The initializer [", name, "] should have one or more floats ",
"with size no less than 4, actual size: ", unpacked_tensor.size());
scale = reinterpret_cast<const float*>(unpacked_tensor.data())[0];
return Status::OK();
}
common::Status GetQuantizationZeroPoint(const InitializedTensorSet& initializers,
const Node& node, size_t idx, int32_t& zero_point) {
std::vector<uint8_t> unpacked_tensor;
const auto& name = node.InputDefs()[idx]->Name();
const auto& zero_point_tensor = *initializers.at(name);
ORT_RETURN_IF_ERROR(
onnxruntime::utils::UnpackInitializerData(zero_point_tensor, node.ModelPath(), unpacked_tensor));
common::Status GetQuantizationScaleAndZeroPoint(
const InitializedTensorSet& initializers, const NodeUnit& node_unit, const std::string& name,
float& scale, int32_t& zero_point, bool is_input) {
const auto& io_defs = is_input ? node_unit.Inputs() : node_unit.Outputs();
for (const auto& io_def : io_defs) {
if (io_def.node_arg.Name() == name)
return GetQuantizationScaleAndZeroPoint(initializers, io_def, node_unit.ModelPath(),
scale, zero_point);
}
ORT_RETURN_IF(unpacked_tensor.empty(), "The initializer [", name, "] is empty");
// Onnx quantization uses uint8 [int8 not yet supported], need to cast to int32_t used by NNAPI
zero_point = static_cast<int32_t>(unpacked_tensor[0]);
return Status::OK();
return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
"Unknown input: ", name, ", for NodeUnit with node index: ", node_unit.Index());
}
bool GetShape(const NodeArg& node_arg, Shape& shape) {
@ -363,9 +405,9 @@ bool GetType(const NodeArg& node_arg, int32_t& type) {
return true;
}
void GetFlattenOutputShape(const Node& node, const Shape& input_shape, int32_t& dim_1, int32_t& dim_2) {
void GetFlattenOutputShape(const NodeUnit& node_unit, const Shape& input_shape, int32_t& dim_1, int32_t& dim_2) {
int32_t rank = static_cast<int>(input_shape.size());
NodeAttrHelper helper(node);
NodeAttrHelper helper(node_unit);
int32_t axis = helper.Get("axis", 1);
// axis == rank is a valid input, but invalid for HandleNegativeAxis
// Skip non-negative axis here
@ -491,10 +533,11 @@ std::string Shape2String(const std::vector<uint32_t>& shape) {
return os.str();
}
bool CheckIsInitializer(const InitializedTensorSet& initializers, const Node& node,
size_t input_idx, const char* input_name) {
if (!Contains(initializers, node.InputDefs()[input_idx]->Name())) {
LOGS_DEFAULT(VERBOSE) << input_name << " of " << node.OpType() << " must be an initializer tensor";
bool CheckIsInitializer(const InitializedTensorSet& initializers, const NodeUnit& node_unit,
const std::string& input_name, const char* input_description) {
if (!Contains(initializers, input_name)) {
LOGS_DEFAULT(VERBOSE) << input_description << " of " << node_unit.Name() << "of type ["
<< node_unit.OpType() << "] must be an initializer tensor";
return false;
}

39
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h
View file

@ -26,10 +26,13 @@ namespace onnxruntime {
using Shape = std::vector<uint32_t>;
using InitializerMap = std::unordered_map<std::string, const ONNX_NAMESPACE::TensorProto&>;
class GraphViewer;
class Node;
class NodeArg;
class NodeUnit;
class GraphViewer;
class Path;
struct NodeUnitIODef;
namespace nnapi {
@ -94,28 +97,32 @@ QLinearOpType GetQLinearOpType(const onnxruntime::Node& node);
// Return the type of the conv ops,
// This function assumes the input is a 2d conv node
ConvType GetConvType(const onnxruntime::Node& node, const InitializedTensorSet& initializers);
ConvType GetConvType(const NodeUnit& node_unit, const InitializedTensorSet& initializers);
// This qlinear op is an operator takes 2 inputs and produces 1 output
// Such as QLinearConv, QLinearMatMul, QLinearAdd, ...
bool IsQLinearBinaryOp(QLinearOpType qlinear_op_type);
// Check if a qlinear unary op has valid inputs, Qlinear[Sigmoid/AveragePool]
bool HasValidUnaryOpQuantizedInputs(const Node& node);
bool HasValidUnaryOpQuantizedInputs(const NodeUnit& node_unit);
// Check if a qlinear binary op has valid inputs, Qlinear[Conv/MatMul/Add]
bool HasValidBinaryOpQuantizedInputs(const Node& node);
bool HasValidBinaryOpQuantizedInputs(const NodeUnit& node_unit);
// Check if a qlinear op has valid scales for given indices
bool HasValidQuantizationScales(const InitializedTensorSet& initializers, const Node& node,
const std::vector<size_t>& indices, const OpSupportCheckParams& params);
bool HasValidQuantizationScales(const InitializedTensorSet& initializers, const NodeUnit& node_unit,
const std::vector<size_t>& indices, const OpSupportCheckParams& params, bool is_input);
// Check if a qlinear op has valid zero points for given indices
bool HasValidQuantizationZeroPoints(const InitializedTensorSet& initializers, const Node& node,
const std::vector<size_t>& indices);
bool HasValidQuantizationZeroPoints(const InitializedTensorSet& initializers, const NodeUnit& node_unit,
const std::vector<size_t>& indices, bool is_input);
common::Status GetQuantizationScale(const InitializedTensorSet& initializers, const Node& node,
size_t idx, float& scale);
common::Status GetQuantizationScaleAndZeroPoint(
const InitializedTensorSet& initializers, const NodeUnitIODef& io_def, const Path& model_path,
float& scale, int32_t& zero_point);
common::Status GetQuantizationZeroPoint(const InitializedTensorSet& initializers,
const Node& node, size_t idx, int32_t& zero_point) ORT_MUST_USE_RESULT;
common::Status GetQuantizationScaleAndZeroPoint(
const InitializedTensorSet& initializers, const NodeUnit& node_unit, const std::string& name,
float& scale, int32_t& zero_point, bool is_input = true);
// Get Shape/Type of a NodeArg
// TODO, move to shared_utils
@ -123,7 +130,7 @@ bool GetShape(const NodeArg& node_arg, Shape& shape);
bool GetType(const NodeArg& node_arg, int32_t& type);
// Get the output shape of Flatten Op
void GetFlattenOutputShape(const Node& node, const Shape& input_shape, int32_t& dim_1, int32_t& dim_2);
void GetFlattenOutputShape(const NodeUnit& node_unit, const Shape& input_shape, int32_t& dim_1, int32_t& dim_2);
// If a node is supported by NNAPI
bool IsNodeSupported(const NodeUnit& node_unit, const GraphViewer& graph_viewer, const OpSupportCheckParams& params);
@ -144,8 +151,10 @@ bool IsValidSupportedNodeGroup(const std::vector<const Node*>& supported_node_gr
std::string Shape2String(const std::vector<uint32_t>& shape);
// Check the given input is an initializer tensor
bool CheckIsInitializer(const InitializedTensorSet& initializers, const Node& node,
size_t index, const char* input_name) ORT_MUST_USE_RESULT;
// input_name is the name of the initializer
// input_description is the string describing the input in the output message (if any)
bool CheckIsInitializer(const InitializedTensorSet& initializers, const NodeUnit& node_unit,
const std::string& input_name, const char* input_description);
} // namespace nnapi
} // namespace onnxruntime

175
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc
View file

@ -1,22 +1,23 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include <core/common/logging/logging.h>
#include <core/common/safeint.h>
#include <core/framework/tensorprotoutils.h>
#include "model_builder.h"
#include "core/common/logging/logging.h"
#include "core/common/safeint.h"
#include "core/common/status.h"
#include "core/framework/tensorprotoutils.h"
#include "core/graph/graph_viewer.h"
#include "core/providers/common.h"
#include "core/providers/shared/node_unit/node_unit.h"
#include "core/providers/shared/utils/utils.h"
#include "core/providers/nnapi/nnapi_builtin/nnapi_lib/nnapi_implementation.h"
#include "helper.h"
#include "model_builder.h"
#include "op_builder.h"
#include "op_support_checker.h"
using onnxruntime::NodeUnit;
using namespace android::nn::wrapper;
using std::vector;
namespace onnxruntime {
namespace nnapi {
@ -31,7 +32,7 @@ int32_t ModelBuilder::GetNNAPIFeatureLevel() const {
// Scalar operand is copied into the model, no need to persist
#define DEFINE_ADD_OPERAND_FROM_SCALAR(scalar_type, op_type) \
Status ModelBuilder::AddOperandFromScalar(scalar_type value, uint32_t& index) { \
OperandType operandType(Type::op_type, vector<uint32_t>{}); \
OperandType operandType(Type::op_type, std::vector<uint32_t>{}); \
ORT_RETURN_IF_ERROR(AddNewNNAPIOperand(operandType, index)); \
RETURN_STATUS_ON_ERROR_WITH_NOTE( \
nnapi_->ANeuralNetworksModel_setOperandValue( \
@ -50,13 +51,12 @@ void ModelBuilder::AddInitializerToSkip(const std::string& tensor_name) {
skipped_initializers_.insert(tensor_name);
}
static std::unordered_map<std::string, vector<const Node*>> GetAllQuantizedOpInputs(const GraphViewer& graph_viewer);
Status ModelBuilder::Prepare() {
nnapi_model_ = std::unique_ptr<Model>(new Model());
RETURN_STATUS_ON_ERROR(nnapi_->ANeuralNetworksModel_create(&nnapi_model_->model_));
ORT_RETURN_IF_ERROR(GetTargetDevices());
all_quantized_op_inputs_ = GetAllQuantizedOpInputs(graph_viewer_);
PreprocessNodeUnits();
GetAllQuantizedOpInputs();
PreprocessInitializers();
PreprocessActivations();
ORT_RETURN_IF_ERROR(RegisterInitializers());
@ -118,74 +118,87 @@ Status ModelBuilder::GetTargetDevices() {
}
void ModelBuilder::PreprocessInitializers() {
const auto& node_indices = graph_viewer_.GetNodesInTopologicalOrder();
for (size_t i = 0; i < node_indices.size(); i++) {
const auto* node(graph_viewer_.GetNode(node_indices[i]));
if (const auto* op_builder = GetOpBuilder(*node)) {
const NodeUnit node_unit(*node);
op_builder->AddInitializersToSkip(*this, node_unit);
for (const auto& node_unit : node_unit_holder_) {
if (const auto* op_builder = GetOpBuilder(*node_unit)) {
op_builder->AddInitializersToSkip(*this, *node_unit);
}
}
}
void ModelBuilder::PreprocessActivations() {
const auto& node_indices = graph_viewer_.GetNodesInTopologicalOrder();
for (size_t i = 0; i < node_indices.size(); i++) {
const auto* node(graph_viewer_.GetNode(node_indices[i]));
const auto& op_type(node->OpType());
for (const auto& node_unit : node_unit_holder_) {
const auto& node = node_unit->GetNode();
const auto& op_type(node.OpType());
if (op_type == "Relu") {
activation_nodes_.emplace(node->Index(), ANEURALNETWORKS_FUSED_RELU);
activation_node_units_.emplace(node_unit.get(), ANEURALNETWORKS_FUSED_RELU);
} else if (op_type == "Clip") { // Relu1 or Relu6
float min, max;
if (!GetClipMinMax(GetInitializerTensors(), *node, min, max, logging::LoggingManager::DefaultLogger()))
if (!GetClipMinMax(GetInitializerTensors(), node, min, max, logging::LoggingManager::DefaultLogger()))
continue;
if (min == -1.0f && max == 1.0f) {
activation_nodes_.emplace(node->Index(), ANEURALNETWORKS_FUSED_RELU1);
activation_node_units_.emplace(node_unit.get(), ANEURALNETWORKS_FUSED_RELU1);
} else if (min == 0.0f && max == 6.0f) {
activation_nodes_.emplace(node->Index(), ANEURALNETWORKS_FUSED_RELU6);
activation_node_units_.emplace(node_unit.get(), ANEURALNETWORKS_FUSED_RELU6);
}
}
}
}
// Help to get all quantized operators' input and the node(s) using the input
static std::unordered_map<std::string, vector<const Node*>> GetAllQuantizedOpInputs(const GraphViewer& graph_viewer) {
std::unordered_map<std::string, vector<const Node*>> all_quantized_op_inputs;
const auto& node_indices = graph_viewer.GetNodesInTopologicalOrder();
for (const auto& node_idx : node_indices) {
const auto* node(graph_viewer.GetNode(node_idx));
auto qlinear_op_type = GetQLinearOpType(*node);
const NodeUnit& ModelBuilder::GetNodeUnit(const Node* node) const {
// In theory, if node_unit_map_ is generated correctly, see PreprocessNodeUnits(), a NodeUnit can be
// found for any single node in the graph_viewer_, unless the given node is not from graph_viewer_
return *node_unit_map_.at(node);
}
void ModelBuilder::PreprocessNodeUnits() {
// TODO, hookup shared QDQ selectors here to identify all the qdq NodeUnit in the graph
const auto& node_indices = graph_viewer_.GetNodesInTopologicalOrder();
for (size_t i = 0; i < node_indices.size(); i++) {
const auto node_idx = node_indices[i];
// TODO, check if the node is already part of a qdq group
const auto* node(graph_viewer_.GetNode(node_idx));
auto node_unit = std::make_unique<NodeUnit>(*node);
node_unit_map_.insert({node, node_unit.get()});
node_unit_holder_.push_back(std::move(node_unit));
}
}
// Help to get all quantized operators' input and the NodeUnit(s) using the input
void ModelBuilder::GetAllQuantizedOpInputs() {
for (const auto& node_unit : node_unit_holder_) {
// TODO, hookup getting quantized inputs with QDQ NodeUnits and remove the ORT_ENFORCE
ORT_ENFORCE(node_unit->UnitType() == NodeUnit::Type::SingleNode, "QDQ NodeUnit is not yet implemented");
auto qlinear_op_type = GetQLinearOpType(node_unit->GetNode());
// Not a qlinear op
// TODO, add handling for QDQ NodeUnit
if (qlinear_op_type == QLinearOpType::Unknown)
continue;
const auto add_quantized_input =
[&all_quantized_op_inputs = all_quantized_op_inputs_](const NodeUnit& node_unit, size_t input_idx) {
const auto& input_name = node_unit.Inputs()[input_idx].node_arg.Name();
all_quantized_op_inputs[input_name].push_back(&node_unit);
};
// All qlinear ops EXCEPT QuantizeLinear has quantized input
if (qlinear_op_type != QLinearOpType::QuantizeLinear) {
const auto& input_name = node->InputDefs()[0]->Name();
if (Contains(all_quantized_op_inputs, input_name))
all_quantized_op_inputs.at(input_name).push_back(node);
else
all_quantized_op_inputs.emplace(input_name, vector<const Node*>{node});
add_quantized_input(*node_unit, 0);
}
if (IsQLinearBinaryOp(qlinear_op_type)) {
const auto& input_name = node->InputDefs()[3]->Name();
if (Contains(all_quantized_op_inputs, input_name))
all_quantized_op_inputs.at(input_name).push_back(node);
else
all_quantized_op_inputs.emplace(input_name, vector<const Node*>{node});
add_quantized_input(*node_unit, 1);
}
}
return all_quantized_op_inputs;
// TODO, add handling for varidiac nodes such as QLinearConcat
}
}
static Status GetInputDataType(
const InitializedTensorSet& initializers,
const std::unordered_map<std::string, std::vector<const Node*>>& all_quantized_op_inputs,
const std::unordered_map<std::string, std::vector<const NodeUnit*>>& all_quantized_op_inputs,
const std::string& name, int32_t data_type, const Shape& shape,
OperandType& operand_type) {
Type type = Type::TENSOR_FLOAT32;
@ -208,10 +221,9 @@ static Status GetInputDataType(
}
// TODO, verify the scale and zero point match if there are multiple op using same input
const auto* node = all_quantized_op_inputs.at(name)[0];
const NodeUnit node_unit(*node);
ORT_RETURN_IF_ERROR(GetQuantizedInputScaleAndZeroPoint(
initializers, node_unit, name, scale, zero_point));
const auto* node_unit = all_quantized_op_inputs.at(name)[0];
ORT_RETURN_IF_ERROR(GetQuantizationScaleAndZeroPoint(
initializers, *node_unit, name, scale, zero_point, true /* is_input */));
break;
}
// case ONNX_NAMESPACE::TensorProto_DataType_INT8:
@ -491,15 +503,23 @@ Status ModelBuilder::AddOperandFromPersistMemoryBuffer(
Status ModelBuilder::AddOperations() {
const auto& node_indices = graph_viewer_.GetNodesInTopologicalOrder();
std::unordered_set<const NodeUnit*> processed_node_units;
for (size_t i = 0; i < node_indices.size(); i++) {
const auto* node(graph_viewer_.GetNode(node_indices[i]));
if (const auto* op_builder = GetOpBuilder(*node)) {
const NodeUnit node_unit(*node);
const NodeUnit& node_unit = GetNodeUnit(node);
// Since a NodeUnit may contain multiple nodes, avoid processing the same NodeUnit multiple times
if (Contains(processed_node_units, &node_unit))
continue;
if (const auto* op_builder = GetOpBuilder(node_unit)) {
ORT_RETURN_IF_ERROR(op_builder->AddToModelBuilder(*this, node_unit));
} else {
return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT,
"Node [", node->Name(), "], type [", node->OpType(), "] is not supported");
"Node [", node_unit.Name(), "], type [", node_unit.OpType(), "] is not supported");
}
processed_node_units.insert(&node_unit);
}
return Status::OK();
@ -605,20 +625,40 @@ Status ModelBuilder::Compile(std::unique_ptr<Model>& model) {
return Status::OK();
}
int32_t ModelBuilder::FindActivation(const Node& node, const NodeArg& output) {
int32_t ModelBuilder::FindActivation(const NodeUnit& node_unit) {
int32_t fuse_code = ANEURALNETWORKS_FUSED_NONE;
const auto& output_nodes = node_unit.GetOutputNodes();
if (node_unit.GetOutputNodes().size() != 1) {
LOGS_DEFAULT(VERBOSE) << "FindActivation does not support, NodeUnit [" << node_unit.Name()
<< "] type [" << node_unit.OpType()
<< "], with " << output_nodes.size() << " output nodes";
return fuse_code;
}
const auto& outputs = node_unit.Outputs();
if (outputs.size() != 1) {
LOGS_DEFAULT(VERBOSE) << "FindActivation does not support, NodeUnit [" << node_unit.Name()
<< "] type [" << node_unit.OpType()
<< "], with " << outputs.size() << " outputs";
return fuse_code;
}
const NodeArg& output = outputs[0].node_arg;
const auto& output_node = *output_nodes[0];
// TODO, add support of activation fusion for quantized node group (qdq or qlinear)
// We do not support activation fusion for quantized operators for now
auto qlinear_op_type = GetQLinearOpType(node);
auto qlinear_op_type = GetQLinearOpType(node_unit.GetNode());
if (qlinear_op_type != QLinearOpType::Unknown)
return fuse_code;
for (auto it = node.OutputEdgesBegin(), end = node.OutputEdgesEnd(); it != end; ++it) {
for (auto it = output_node.OutputEdgesBegin(), end = output_node.OutputEdgesEnd(); it != end; ++it) {
const auto& dst_node = it->GetNode();
const auto* dst_input = dst_node.InputDefs()[it->GetDstArgIndex()];
if (Contains(activation_nodes_, dst_node.Index())) {
const auto& dst_node_unit = GetNodeUnit(&dst_node);
if (Contains(activation_node_units_, &dst_node_unit)) {
if (&output == dst_input) {
fuse_code = activation_nodes_.at(dst_node.Index());
fuse_code = activation_node_units_.at(&dst_node_unit);
}
} else {
// if there is any other non-relu node using the output
@ -628,14 +668,14 @@ int32_t ModelBuilder::FindActivation(const Node& node, const NodeArg& output) {
}
}
// if output is a graph output, will add relu separately
// if output is a graph output, will add activation separately
if (fuse_code != ANEURALNETWORKS_FUSED_NONE) {
for (const auto* graph_output : graph_viewer_.GetOutputs()) {
if (&output == graph_output)
return ANEURALNETWORKS_FUSED_NONE;
const auto& graph_outputs = graph_viewer_.GetOutputs();
if (std::find(graph_outputs.cbegin(), graph_outputs.cend(), &output) != graph_outputs.cend()) {
return ANEURALNETWORKS_FUSED_NONE;
}
LOGS_DEFAULT(VERBOSE) << "Node [" << node.Name() << "] type [" << node.OpType()
LOGS_DEFAULT(VERBOSE) << "Node [" << node_unit.Name() << "] type [" << node_unit.OpType()
<< "], fused the output [" << output.Name() << "]";
fused_activations_.insert(output.Name());
@ -644,12 +684,13 @@ int32_t ModelBuilder::FindActivation(const Node& node, const NodeArg& output) {
return fuse_code;
}
/* static */ const IOpBuilder* ModelBuilder::GetOpBuilder(const Node& node) {
/* static */ const IOpBuilder* ModelBuilder::GetOpBuilder(const NodeUnit& node_unit) {
const auto& op_builders = GetOpBuilders();
if (!Contains(op_builders, node.OpType()))
const auto& op_type = node_unit.GetNode().OpType();
if (!Contains(op_builders, op_type))
return nullptr;
return op_builders.at(node.OpType());
return op_builders.at(op_type);
}
std::string ModelBuilder::GetUniqueName(const std::string& base_name) {
@ -663,6 +704,10 @@ std::string ModelBuilder::GetUniqueName(const std::string& base_name) {
return unique_name;
}
const InitializedTensorSet& ModelBuilder::GetInitializerTensors() const {
return graph_viewer_.GetAllInitializedTensors();
}
void ModelBuilder::RegisterNHWCOperand(const std::string& name) {
nhwc_operands_.insert(name);
}

98
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.h
View file

@ -5,16 +5,22 @@
#include <onnx/onnx_pb.h>
#include <unordered_set>
#include <core/graph/graph_viewer.h>
#include "core/graph/basic_types.h"
#include "core/providers/nnapi/nnapi_builtin/model.h"
#include "core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.h"
#include "op_support_checker.h"
#include "shaper.h"
namespace onnxruntime {
class GraphViewer;
class NodeUnit;
class Node;
class NodeArg;
namespace nnapi {
class IOpBuilder;
class IOpSupportChecker;
class ModelBuilder {
public:
@ -33,30 +39,30 @@ class ModelBuilder {
};
ModelBuilder(const GraphViewer& graph_viewer);
~ModelBuilder() = default;
Status Compile(std::unique_ptr<Model>& model) ORT_MUST_USE_RESULT;
common::Status Compile(std::unique_ptr<Model>& model);
int32_t GetNNAPIFeatureLevel() const;
// Add an NNAPI operation (operator)
Status AddOperation(int op, const std::vector<uint32_t>& input_indices,
const std::vector<std::string>& output_names,
const std::vector<android::nn::wrapper::OperandType>& types,
const std::vector<bool>& is_nhwc_vec) ORT_MUST_USE_RESULT;
common::Status AddOperation(int op, const std::vector<uint32_t>& input_indices,
const std::vector<std::string>& output_names,
const std::vector<android::nn::wrapper::OperandType>& types,
const std::vector<bool>& is_nhwc_vec);
// Find if an output has a fuseable activation (Relu)
int32_t FindActivation(const Node& node, const NodeArg& output);
// Find if the given node_unit has a fuseable activation (Relu/Relu1/Relu6)
// For now we only support node_unit with a single output
int32_t FindActivation(const NodeUnit& node_unit);
// Add an NNAPI scalar operand
Status AddOperandFromScalar(bool value, uint32_t& index) ORT_MUST_USE_RESULT;
Status AddOperandFromScalar(float value, uint32_t& index) ORT_MUST_USE_RESULT;
Status AddOperandFromScalar(int32_t value, uint32_t& index) ORT_MUST_USE_RESULT;
common::Status AddOperandFromScalar(bool value, uint32_t& index);
common::Status AddOperandFromScalar(float value, uint32_t& index);
common::Status AddOperandFromScalar(int32_t value, uint32_t& index);
// Add an NNAPI tensor operand (and allocate persist buffer)
Status AddOperandFromPersistMemoryBuffer(
common::Status AddOperandFromPersistMemoryBuffer(
const std::string& name, const void* buffer,
const android::nn::wrapper::OperandType& operand_type) ORT_MUST_USE_RESULT;
const android::nn::wrapper::OperandType& operand_type);
// The initializer will be processed separately, skip it as an initializer
void AddInitializerToSkip(const std::string& tensor_name);
@ -96,7 +102,7 @@ class ModelBuilder {
const std::unordered_set<std::string>&
GetFusedActivations() const { return fused_activations_; }
const InitializedTensorSet& GetInitializerTensors() const { return graph_viewer_.GetAllInitializedTensors(); }
const InitializedTensorSet& GetInitializerTensors() const;
const GraphViewer& GetGraphViewer() const { return graph_viewer_; }
@ -107,10 +113,13 @@ class ModelBuilder {
bool GetNCHWOperand(const std::string& nhwc_name, std::string& nchw_name);
bool GetNHWCOperand(const std::string& nchw_name, std::string& nhwc_name);
Status SetNHWCToNCHWOperandMap(const std::string& nhwc_name,
const std::string& nchw_name) ORT_MUST_USE_RESULT;
Status SetNCHWToNHWCOperandMap(const std::string& nchw_name,
const std::string& nhwc_name) ORT_MUST_USE_RESULT;
// Get the NodeUnit which contains the given node
const NodeUnit& GetNodeUnit(const Node* node) const;
common::Status SetNHWCToNCHWOperandMap(const std::string& nhwc_name,
const std::string& nchw_name);
common::Status SetNCHWToNHWCOperandMap(const std::string& nchw_name,
const std::string& nhwc_name);
private:
const NnApi* nnapi_{nullptr};
@ -134,8 +143,8 @@ class ModelBuilder {
std::unordered_set<std::string> skipped_initializers_;
// All activation nodes (Relu, Relu1, Relu6) as a map <NodeIndex, activation_code>
std::unordered_map<NodeIndex, int32_t> activation_nodes_;
// All activation nodes (Relu, Relu1, Relu6) as a map <const NodeUnit*, activation_code>
std::unordered_map<const NodeUnit*, int32_t> activation_node_units_;
std::unordered_map<std::string, std::shared_ptr<IOpSupportChecker>> op_support_checkers_;
@ -149,9 +158,14 @@ class ModelBuilder {
std::vector<uint32_t> input_index_vec_;
std::vector<uint32_t> output_index_vec_;
// Contains all quantized operators' input and the node(s) using the input
// In the form of {input_name, [node(s) using the input]}
std::unordered_map<std::string, std::vector<const Node*>> all_quantized_op_inputs_;
// Contains all quantized operators' input and the NodeUnit(s) using the input
// In the form of {input_name, [NodeUnit(s) using the input]}
std::unordered_map<std::string, std::vector<const NodeUnit*>> all_quantized_op_inputs_;
// Holder for the NodeUnits in the graph, this will guarantee the NodeUnits is
// valid throughout the lifetime of the ModelBuilder
std::vector<std::unique_ptr<NodeUnit>> node_unit_holder_;
std::unordered_map<const Node*, const NodeUnit*> node_unit_map_;
std::unordered_set<std::string> unique_names_;
@ -164,32 +178,38 @@ class ModelBuilder {
uint32_t next_index_ = 0;
// Convert the onnx model to ANeuralNetworksModel
Status Prepare() ORT_MUST_USE_RESULT;
common::Status Prepare();
Status GetTargetDevices() ORT_MUST_USE_RESULT;
common::Status GetTargetDevices();
// If a NNAPI operation will use initializers directly, we will add the initializers to the skip list
void PreprocessInitializers();
// Preprocess all the activation nodes (Relu/Relu1/Relu6) for easy query later
void PreprocessActivations();
// Copy and process all the initializers to NNAPI model
Status RegisterInitializers() ORT_MUST_USE_RESULT;
Status RegisterModelInputs() ORT_MUST_USE_RESULT;
Status AddOperations() ORT_MUST_USE_RESULT;
Status RegisterModelOutputs() ORT_MUST_USE_RESULT;
common::Status RegisterInitializers();
common::Status RegisterModelInputs();
common::Status AddOperations();
common::Status RegisterModelOutputs();
// After constructing the NNAPI model, will set the shape inferencing record to the Model
void RegisterModelShaper();
Status SetOperandValue(uint32_t index, Model::NNMemory* memory,
size_t size, size_t offset) ORT_MUST_USE_RESULT;
// Get all quantized inputs in the underlying graph_viewer
void GetAllQuantizedOpInputs();
Status AddNewNNAPIOperand(const android::nn::wrapper::OperandType& type, uint32_t& index) ORT_MUST_USE_RESULT;
Status AddNewOperand(const std::string& name,
const android::nn::wrapper::OperandType& operand_type,
bool is_nhwc,
uint32_t& index) ORT_MUST_USE_RESULT;
// Go through the underlying graph_viewer, and generate NodeUnits, Many initializing functions are
// using the result of PreprocessNodeUnits, this need to run early in the Prepare()
void PreprocessNodeUnits();
static const IOpBuilder* GetOpBuilder(const Node& node);
common::Status SetOperandValue(uint32_t index, Model::NNMemory* memory, size_t size, size_t offset);
common::Status AddNewNNAPIOperand(const android::nn::wrapper::OperandType& type, uint32_t& index);
common::Status AddNewOperand(const std::string& name,
const android::nn::wrapper::OperandType& operand_type,
bool is_nhwc,
uint32_t& index);
static const IOpBuilder* GetOpBuilder(const NodeUnit& node_unit);
};
} // namespace nnapi

948
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc
View file

File diff suppressed because it is too large Load diff

11
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.h
View file

@ -7,7 +7,6 @@
#include <unordered_map>
#include "core/graph/basic_types.h"
#include "core/session/onnxruntime_c_api.h"
namespace onnxruntime {
@ -31,7 +30,7 @@ class IOpBuilder {
virtual void AddInitializersToSkip(ModelBuilder& model_builder, const NodeUnit& node_unit) const = 0;
// Add the operator to NNAPI model
virtual common::Status AddToModelBuilder(ModelBuilder& model_builder, const NodeUnit& node_unit) const ORT_MUST_USE_RESULT = 0;
virtual common::Status AddToModelBuilder(ModelBuilder& model_builder, const NodeUnit& node_unit) const = 0;
};
// Get the lookup table with IOpBuilder delegates for different onnx operators
@ -40,13 +39,7 @@ class IOpBuilder {
const std::unordered_map<std::string, const IOpBuilder*>& GetOpBuilders();
// Transpose the NHWC input to NCHW output
common::Status TransposeNHWCToNCHW(ModelBuilder& model_builder, const std::string& input, const std::string& output)
ORT_MUST_USE_RESULT;
// Get the quantized input's scale and zero point for the given input
common::Status GetQuantizedInputScaleAndZeroPoint(const InitializedTensorSet& initializers,
const NodeUnit& node_unit, const std::string& input_name,
float& scale, int32_t& zero_point) ORT_MUST_USE_RESULT;
common::Status TransposeNHWCToNCHW(ModelBuilder& model_builder, const std::string& input, const std::string& output);
} // namespace nnapi
} // namespace onnxruntime

461
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_support_checker.cc
View file

@ -1,19 +1,16 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include <core/common/logging/logging.h>
#include <core/common/safeint.h>
#include <core/framework/tensorprotoutils.h>
#include <core/graph/graph.h>
#include "op_support_checker.h"
#include "core/common/logging/logging.h"
#include "core/common/safeint.h"
#include "core/framework/tensorprotoutils.h"
#include "core/graph/graph.h"
#include "core/providers/common.h"
#include "core/providers/shared/node_unit/node_unit.h"
#include "core/providers/shared/utils/utils.h"
#include "helper.h"
#include "op_support_checker.h"
using onnxruntime::NodeUnit;
using std::vector;
namespace onnxruntime {
namespace nnapi {
@ -25,19 +22,37 @@ struct OpSupportCheckerRegistrations {
std::unordered_map<std::string, const IOpSupportChecker*> op_support_checker_map;
};
bool HasExternalInitializer(const InitializedTensorSet& initializers, const Node& node) {
for (const auto* node_arg : node.InputDefs()) {
const auto& input_name(node_arg->Name());
if (!Contains(initializers, input_name))
bool HasExternalInitializer(const InitializedTensorSet& initializers, const NodeUnit& node_unit) {
const auto is_ext_initializer =
[&](const NodeArg& node_arg) {
const auto& input_name(node_arg.Name());
if (!Contains(initializers, input_name))
return false;
const auto& tensor = *initializers.at(input_name);
if (tensor.has_data_location() &&
tensor.data_location() == ONNX_NAMESPACE::TensorProto_DataLocation_EXTERNAL) {
LOGS_DEFAULT(VERBOSE) << "Initializer [" << input_name
<< "] with external data location are not currently supported";
return true;
}
return false;
};
const auto& inputs = node_unit.Inputs();
for (const auto& input : inputs) {
if (is_ext_initializer(input.node_arg))
return true;
if (!input.quant_param)
continue;
const auto& tensor = *initializers.at(input_name);
if (tensor.has_data_location() &&
tensor.data_location() == ONNX_NAMESPACE::TensorProto_DataLocation_EXTERNAL) {
LOGS_DEFAULT(VERBOSE) << "Initializer [" << input_name
<< "] with external data location are not currently supported";
if (is_ext_initializer(input.quant_param->scale))
return true;
if (input.quant_param->zero_point && is_ext_initializer(*input.quant_param->zero_point))
return true;
}
}
return false;
@ -118,10 +133,8 @@ bool BaseOpSupportChecker::IsOpSupported(const InitializedTensorSet& initializer
if (!HasSupportedInputs(node_unit))
return false;
const auto& node = node_unit.GetNode();
// We do not support external initializers for now
if (HasExternalInitializer(initializers, node))
if (HasExternalInitializer(initializers, node_unit))
return false;
if (!HasSupportedOpSet(node_unit))
@ -247,30 +260,25 @@ int BinaryOpSupportChecker::GetMinSupportedOpSet(const NodeUnit& node_unit) cons
}
bool BinaryOpSupportChecker::HasSupportedInputsImpl(const NodeUnit& node_unit) const {
// TODO, change to use node unit and quant_param of IODef
const auto& node = node_unit.GetNode();
bool is_qlinear_add = node.OpType() == "QLinearAdd";
bool is_pow = node.OpType() == "Pow";
bool is_qlinear_add = node_unit.OpType() == "QLinearAdd";
bool is_pow = node_unit.OpType() == "Pow";
if (!is_qlinear_add && !is_pow)
return BaseOpSupportChecker::HasSupportedInputsImpl(node_unit);
if (is_qlinear_add) {
// QLinearAdd
if (!HasValidBinaryOpQuantizedInputs(node))
if (!HasValidBinaryOpQuantizedInputs(node_unit))
return false;
}
// Pow we only support both input as fp32 now
if (is_pow) {
const auto& input1 = *node.InputDefs()[0];
const auto& input2 = *node.InputDefs()[1];
int32_t input_type_1;
if (!GetType(input1, input_type_1))
if (!GetType(node_unit.Inputs()[0].node_arg, input_type_1))
return false;
int32_t input_type_2;
if (!GetType(input2, input_type_2))
if (!GetType(node_unit.Inputs()[1].node_arg, input_type_2))
return false;
if (input_type_1 != ONNX_NAMESPACE::TensorProto_DataType_FLOAT || input_type_1 != input_type_2) {
@ -286,24 +294,18 @@ bool BinaryOpSupportChecker::HasSupportedInputsImpl(const NodeUnit& node_unit) c
bool BinaryOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initializers, const NodeUnit& node_unit,
const OpSupportCheckParams& params) const {
const auto& node = node_unit.GetNode();
const auto& op_type(node.OpType());
const auto input_defs(node.InputDefs());
const auto& op_type(node_unit.OpType());
const auto& inputs = node_unit.Inputs();
bool op_is_qlinear = op_type == "QLinearAdd";
size_t a_idx = 0, b_idx = 1;
if (op_is_qlinear) {
b_idx = 3;
}
Shape input1_shape, input2_shape;
if (!GetShape(*input_defs[a_idx], input1_shape) ||
!GetShape(*input_defs[b_idx], input2_shape))
if (!GetShape(inputs[0].node_arg, input1_shape) ||
!GetShape(inputs[1].node_arg, input2_shape))
return false;
const auto input1_size = input1_shape.size();
const auto input2_size = input2_shape.size();
if (input1_size > 4 || input2_size > 4) {
LOGS_DEFAULT(VERBOSE) << node.OpType() << " only support up to 4d shape, input1 is "
LOGS_DEFAULT(VERBOSE) << op_type << " only support up to 4d shape, input1 is "
<< input1_size << "d shape, input 2 is "
<< input2_size << "d shape";
return false;
@ -312,7 +314,7 @@ bool BinaryOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initi
if (op_is_qlinear) {
// For QLinearAdd, we only support uint8 output now
int32_t output_type;
if (!GetType(*node.OutputDefs()[0], output_type))
if (!GetType(node_unit.Outputs()[0].node_arg, output_type))
return false;
if (output_type != ONNX_NAMESPACE::TensorProto_DataType_UINT8) {
@ -322,13 +324,16 @@ bool BinaryOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initi
return false;
}
// All scale/zero points are initializer scalars
// a/b/y_scale
if (!HasValidQuantizationScales(initializers, node, {1, 4, 6}, params))
// Check input scales and ZPs
if (!HasValidQuantizationScales(initializers, node_unit, {0, 1}, params, true /* is_input */))
return false;
if (!HasValidQuantizationZeroPoints(initializers, node_unit, {0, 1}, true /* is_input */))
return false;
// a/b/y_zero_point
if (!HasValidQuantizationZeroPoints(initializers, node, {2, 5, 7}))
// Check output scale and ZP
if (!HasValidQuantizationScales(initializers, node_unit, {0}, params, false /* is_input */))
return false;
if (!HasValidQuantizationZeroPoints(initializers, node_unit, {0}, false /* is_input */))
return false;
}
@ -354,9 +359,8 @@ class TransposeOpSupportChecker : public BaseOpSupportChecker {
bool TransposeOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& /* initializers */, const NodeUnit& node_unit,
const OpSupportCheckParams& /* params */) const {
const auto& node = node_unit.GetNode();
Shape input_shape;
if (!GetShape(*node.InputDefs()[0], input_shape))
if (!GetShape(node_unit.Inputs()[0].node_arg, input_shape))
return false;
const auto input_size = input_shape.size();
@ -400,15 +404,15 @@ class ReshapeOpSupportChecker : public BaseOpSupportChecker {
bool ReshapeOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initializers, const NodeUnit& node_unit,
const OpSupportCheckParams& /* params */) const {
const auto& node = node_unit.GetNode();
const auto& perm_name = node.InputDefs()[1]->Name();
const auto& inputs = node_unit.Inputs();
const auto& perm_name = inputs[1].node_arg.Name();
if (!Contains(initializers, perm_name)) {
LOGS_DEFAULT(VERBOSE) << "New shape of reshape must be known";
return false;
}
Shape input_shape;
if (!GetShape(*node.InputDefs()[0], input_shape))
if (!GetShape(inputs[0].node_arg, input_shape))
return false;
if (input_shape.size() > 4 || input_shape.empty()) {
@ -427,7 +431,7 @@ bool ReshapeOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& init
const int64_t* raw_perm = reinterpret_cast<const int64_t*>(unpacked_tensor.data());
const auto perm_size = SafeInt<uint32_t>(perm_tensor.dims()[0]);
NodeAttrHelper helper(node);
NodeAttrHelper helper(node_unit);
const bool allow_zero = helper.Get("allowzero ", 0) == 1;
for (uint32_t i = 0; i < perm_size; i++) {
// NNAPI reshape does not support 0 as dimension
@ -462,16 +466,15 @@ class BatchNormalizationOpSupportChecker : public BaseOpSupportChecker {
bool BatchNormalizationOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initializers, const NodeUnit& node_unit,
const OpSupportCheckParams& /* params */) const {
const auto& node = node_unit.GetNode();
if (node.OutputDefs().size() != 1) {
if (node_unit.Outputs().size() != 1) {
LOGS_DEFAULT(VERBOSE) << "Your onnx model may be in training mode, please export "
"it in test mode.";
return false;
}
const auto& input_defs = node.InputDefs();
const auto& inputs = node_unit.Inputs();
Shape input_shape;
if (!GetShape(*input_defs[0], input_shape))
if (!GetShape(inputs[0].node_arg, input_shape))
return false;
const auto input_size = input_shape.size();
@ -481,17 +484,17 @@ bool BatchNormalizationOpSupportChecker::IsOpSupportedImpl(const InitializedTens
return false;
}
NodeAttrHelper helper(node);
NodeAttrHelper helper(node_unit);
const auto spatial = helper.Get("spatial", 1);
if (spatial != 1) {
LOGS_DEFAULT(VERBOSE) << "Non-spatial BN is not supported";
return false;
}
const auto& scale_name = input_defs[1]->Name();
const auto& b_name = input_defs[2]->Name();
const auto& mean_name = input_defs[3]->Name();
const auto& var_name = input_defs[4]->Name();
const auto& scale_name = inputs[1].node_arg.Name();
const auto& b_name = inputs[2].node_arg.Name();
const auto& mean_name = inputs[3].node_arg.Name();
const auto& var_name = inputs[4].node_arg.Name();
if (!Contains(initializers, scale_name)) {
LOGS_DEFAULT(VERBOSE) << "Scale of BN must be known";
return false;
@ -548,25 +551,24 @@ class PoolOpSupportChecker : public BaseOpSupportChecker {
bool PoolOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initializers, const NodeUnit& node_unit,
const OpSupportCheckParams& params) const {
const auto& node = node_unit.GetNode();
const auto& op_name = node.Name();
const auto& op_type = node.OpType();
const auto& input_defs = node.InputDefs();
const auto& op_name = node_unit.Name();
const auto& op_type = node_unit.OpType();
const auto& inputs = node_unit.Inputs();
Shape input_shape;
if (!GetShape(*input_defs[0], input_shape))
if (!GetShape(inputs[0].node_arg, input_shape))
return false;
const auto input_size = input_shape.size();
if (input_size != 4) {
LOGS_DEFAULT(VERBOSE)
<< op_type << " only supports rank-4 tensor, input ["
<< input_defs[0]->Name() << "] has actual dim count " << input_size;
<< inputs[0].node_arg.Name() << "] has actual dim count " << input_size;
return false;
}
bool is_qlinear_average_pool = op_type == "QLinearAveragePool";
if (op_type == "AveragePool" || op_type == "MaxPool" || is_qlinear_average_pool) {
NodeAttrHelper helper(node);
NodeAttrHelper helper(node_unit);
const auto count_include_pad = helper.Get("count_include_pad", 0);
if (count_include_pad == 1) {
@ -596,7 +598,7 @@ bool PoolOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initial
return false;
}
if (node.OutputDefs().size() != 1) {
if (node_unit.Outputs().size() != 1) {
LOGS_DEFAULT(VERBOSE) << "Argmax in maxpooling is not supported";
return false;
}
@ -607,37 +609,38 @@ bool PoolOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initial
// We need to check if we have valid scales and zero points for QLinearAveragePool
if (is_qlinear_average_pool) {
if (input_defs.size() < 4)
// Check input scales and ZPs
if (!HasValidQuantizationScales(initializers, node_unit, {0}, params, true /* is_input */))
return false;
if (!HasValidQuantizationZeroPoints(initializers, node_unit, {0}, true /* is_input */))
return false;
// the output zero point can be optional
bool has_output_zp = input_defs.size() == 5;
// Check output scale and ZP
if (!HasValidQuantizationScales(initializers, node, {1, 3}, params))
if (!HasValidQuantizationScales(initializers, node_unit, {0}, params, false /* is_input */))
return false;
if (!HasValidQuantizationZeroPoints(initializers, node,
has_output_zp
? std::vector<size_t>{2}
: std::vector<size_t>{2, 4})) {
if (!HasValidQuantizationZeroPoints(initializers, node_unit, {0}, false /* is_input */))
return false;
}
// NNAPI requires Quantized Average Pool has same scale and zero point for both input and output
float input_scale = 0.0f;
auto status = GetQuantizationScale(initializers, node, 1, input_scale);
int32_t input_zp = 0;
auto status = GetQuantizationScaleAndZeroPoint(
initializers, node_unit.Inputs()[0], node_unit.ModelPath(), input_scale, input_zp);
if (!status.IsOK()) {
LOGS_DEFAULT(ERROR) << "Op [" << op_type << "] name [" << op_name
<< "] GetQuantizationScale for input_scale failed, message: "
<< "] GetQuantizationScaleAndZeroPoint for input_scale/zp failed, message: "
<< status.ErrorMessage();
return false;
}
float output_scale = 0.0f;
status = GetQuantizationScale(initializers, node, 3, output_scale);
int32_t output_zp = 0;
status = GetQuantizationScaleAndZeroPoint(
initializers, node_unit.Outputs()[0], node_unit.ModelPath(), output_scale, output_zp);
if (!status.IsOK()) {
LOGS_DEFAULT(ERROR) << "Op [" << op_type << "] name [" << op_name
<< "] GetQuantizationScale for output_scale failed, message: "
<< "] GetQuantizationScaleAndZeroPoint for output_scale/zp failed, message: "
<< status.ErrorMessage();
return false;
}
@ -649,26 +652,6 @@ bool PoolOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initial
return false;
}
int32_t input_zp = 0;
int32_t output_zp = 0;
status = GetQuantizationZeroPoint(initializers, node, 2, input_zp);
if (!status.IsOK()) {
LOGS_DEFAULT(ERROR) << "Op [" << op_type << "] name [" << op_name
<< "] GetQuantizationZeroPoint for input_zp failed, message: "
<< status.ErrorMessage();
return false;
}
if (has_output_zp) {
status = GetQuantizationZeroPoint(initializers, node, 4, output_zp);
if (!status.IsOK()) {
LOGS_DEFAULT(ERROR) << "Op [" << op_type << "] name [" << op_name
<< "] GetQuantizationZeroPoint for output_zp failed, message: "
<< status.ErrorMessage();
return false;
}
}
if (input_zp != output_zp) {
LOGS_DEFAULT(VERBOSE) << "Op [" << op_type << "] name [" << op_name
<< "] has different input_zp: " << input_zp
@ -681,26 +664,24 @@ bool PoolOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initial
}
bool PoolOpSupportChecker::HasSupportedInputsImpl(const NodeUnit& node_unit) const {
// TODO, change to use node unit and quant_param of IODef
const auto& node = node_unit.GetNode();
bool is_max_pool = node.OpType() == "MaxPool";
bool is_qlinear_average_pool = node.OpType() == "QLinearAveragePool";
bool is_max_pool = node_unit.OpType() == "MaxPool";
bool is_qlinear_average_pool = node_unit.OpType() == "QLinearAveragePool";
if (!is_max_pool && !is_qlinear_average_pool)
return BaseOpSupportChecker::HasSupportedInputsImpl(node_unit);
if (is_qlinear_average_pool) {
return HasValidUnaryOpQuantizedInputs(node);
return HasValidUnaryOpQuantizedInputs(node_unit);
}
// is_max_pool
// For max pool, we can support both float and uint8 input
int32_t input_type;
if (!GetType(*node.InputDefs()[0], input_type))
if (!GetType(node_unit.Inputs()[0].node_arg, input_type))
return false;
if (input_type != ONNX_NAMESPACE::TensorProto_DataType_FLOAT &&
input_type != ONNX_NAMESPACE::TensorProto_DataType_UINT8) {
LOGS_DEFAULT(VERBOSE) << "[" << node.OpType()
LOGS_DEFAULT(VERBOSE) << "[" << node_unit.OpType()
<< "] Input type: [" << input_type
<< "] is not supported for now";
return false;
@ -741,13 +722,11 @@ class ConvOpSupportChecker : public BaseOpSupportChecker {
}
bool ConvOpSupportChecker::HasSupportedInputsImpl(const NodeUnit& node_unit) const {
// TODO, change to use node unit and quant_param of IODef
const auto& node = node_unit.GetNode();
if (node.OpType() != "QLinearConv")
if (node_unit.OpType() != "QLinearConv")
return BaseOpSupportChecker::HasSupportedInputsImpl(node_unit);
// QLinearConv only supports input of uint8 for now
if (!HasValidBinaryOpQuantizedInputs(node))
if (!HasValidBinaryOpQuantizedInputs(node_unit))
return false;
return true;
@ -755,21 +734,19 @@ bool ConvOpSupportChecker::HasSupportedInputsImpl(const NodeUnit& node_unit) con
bool ConvOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initializers, const NodeUnit& node_unit,
const OpSupportCheckParams& params) const {
const auto& node = node_unit.GetNode();
const auto& op_type = node.OpType();
const auto& op_type = node_unit.OpType();
const bool is_qlinear_conv = (op_type == "QLinearConv");
// We don't support nhwc com.microsoft.QLinearConv for now
if (is_qlinear_conv && node.Domain() == kMSDomain) {
if (is_qlinear_conv && node_unit.Domain() == kMSDomain) {
LOGS_DEFAULT(VERBOSE) << "com.microsoft.QLinearConv is not supported";
return false;
}
const auto input_defs = node.InputDefs();
NodeAttrHelper helper(node);
size_t w_idx = is_qlinear_conv ? 3 : 1;
const auto& inputs = node_unit.Inputs();
NodeAttrHelper helper(node_unit);
const auto group = helper.Get("group", 1);
const auto weight_name = input_defs[w_idx]->Name();
const auto weight_name = inputs[1].node_arg.Name();
if (Contains(initializers, weight_name)) {
const auto& tensor = *initializers.at(weight_name);
if (tensor.dims().size() != 4) {
@ -777,8 +754,8 @@ bool ConvOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initial
return false;
}
const auto onnx_dilations = helper.Get("dilations", vector<int>{1, 1});
if (onnx_dilations != vector<int>{1, 1}) {
const auto onnx_dilations = helper.Get("dilations", std::vector<int>{1, 1});
if (onnx_dilations != std::vector<int>{1, 1}) {
if (group != 1 && tensor.dims()[1] != 1) {
LOGS_DEFAULT(VERBOSE) << "dilation is not supported on grouped conv";
return false;
@ -798,7 +775,7 @@ bool ConvOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initial
if (is_qlinear_conv) {
// For QLinearConv, we only support uint8 output now
int32_t output_type;
if (!GetType(*node.OutputDefs()[0], output_type))
if (!GetType(node_unit.Outputs()[0].node_arg, output_type))
return false;
if (output_type != ONNX_NAMESPACE::TensorProto_DataType_UINT8) {
@ -808,17 +785,21 @@ bool ConvOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initial
return false;
}
if (input_defs.size() > 8 && !Contains(initializers, input_defs[8]->Name())) {
if (inputs.size() > 2 && !Contains(initializers, inputs[2].node_arg.Name())) {
LOGS_DEFAULT(VERBOSE) << "Bias of QLinearConv must be known";
return false;
}
// a/b/y_scale
if (!HasValidQuantizationScales(initializers, node, {1, 4, 6}, params))
// Check input scales and ZPs
if (!HasValidQuantizationScales(initializers, node_unit, {0, 1}, params, true /* is_input */))
return false;
if (!HasValidQuantizationZeroPoints(initializers, node_unit, {0, 1}, true /* is_input */))
return false;
// a/b/y_zero_point
if (!HasValidQuantizationZeroPoints(initializers, node, {2, 5, 7}))
// Check output scale and ZP
if (!HasValidQuantizationScales(initializers, node_unit, {0}, params, false /* is_input */))
return false;
if (!HasValidQuantizationZeroPoints(initializers, node_unit, {0}, false /* is_input */))
return false;
}
@ -845,8 +826,7 @@ class CastOpSupportChecker : public BaseOpSupportChecker {
bool CastOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& /* initializers */, const NodeUnit& node_unit,
const OpSupportCheckParams& /* params */) const {
const auto& node = node_unit.GetNode();
NodeAttrHelper helper(node);
NodeAttrHelper helper(node_unit);
const auto to = helper.Get("to", 0);
if (to != ONNX_NAMESPACE::TensorProto::FLOAT &&
to != ONNX_NAMESPACE::TensorProto::INT32) {
@ -874,9 +854,8 @@ class SoftMaxOpSupportChecker : public BaseOpSupportChecker {
bool SoftMaxOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& /* initializers */, const NodeUnit& node_unit,
const OpSupportCheckParams& params) const {
const auto& node = node_unit.GetNode();
Shape input_shape;
if (!GetShape(*node.InputDefs()[0], input_shape))
if (!GetShape(node_unit.Inputs()[0].node_arg, input_shape))
return false;
const auto input_size = input_shape.size();
@ -887,7 +866,7 @@ bool SoftMaxOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& /* i
}
if (params.android_feature_level < ANEURALNETWORKS_FEATURE_LEVEL_3) {
NodeAttrHelper helper(node);
NodeAttrHelper helper(node_unit);
int32_t axis = helper.Get("axis", 1);
if (axis != 1) {
LOGS_DEFAULT(VERBOSE)
@ -917,13 +896,11 @@ class GemmOpSupportChecker : public BaseOpSupportChecker {
};
bool GemmOpSupportChecker::HasSupportedInputsImpl(const NodeUnit& node_unit) const {
// TODO, change to use node unit and quant_param of IODef
const auto& node = node_unit.GetNode();
if (node.OpType() != "QLinearMatMul")
if (node_unit.OpType() != "QLinearMatMul")
return BaseOpSupportChecker::HasSupportedInputsImpl(node_unit);
// QLinearMatMul
if (!HasValidBinaryOpQuantizedInputs(node))
if (!HasValidBinaryOpQuantizedInputs(node_unit))
return false;
return true;
@ -985,19 +962,13 @@ int GemmOpSupportChecker::GetMinSupportedOpSet(const NodeUnit& node_unit) const
bool GemmOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initializers, const NodeUnit& node_unit,
const OpSupportCheckParams& params) const {
const auto& node = node_unit.GetNode();
const auto& op_type = node.OpType();
const auto input_defs(node.InputDefs());
size_t a_idx = 0, b_idx = 1, c_idx = 2; // A*B+C
const auto& op_type = node_unit.OpType();
const auto& inputs = node_unit.Inputs();
bool is_qlinear_matmul = op_type == "QLinearMatMul";
if (is_qlinear_matmul) {
a_idx = 0;
b_idx = 3;
}
Shape a_shape;
{
if (!GetShape(*input_defs[a_idx], a_shape))
if (!GetShape(inputs[0].node_arg, a_shape))
return false;
if (a_shape.size() != 2) {
@ -1008,7 +979,7 @@ bool GemmOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initial
Shape b_shape;
{
if (!GetShape(*input_defs[b_idx], b_shape))
if (!GetShape(inputs[1].node_arg, b_shape))
return false;
if (b_shape.size() != 2) {
@ -1021,7 +992,7 @@ bool GemmOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initial
// Only support
// 1. A*B'+C
// 2. A*B+C and B is an initializer
NodeAttrHelper helper(node);
NodeAttrHelper helper(node_unit);
const auto transA = helper.Get("transA", 0);
const auto transB = helper.Get("transB", 0);
const auto alpha = helper.Get("alpha", 1.0f);
@ -1037,14 +1008,14 @@ bool GemmOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initial
return false;
}
if (transB == 0 && !Contains(initializers, input_defs[b_idx]->Name())) {
if (transB == 0 && !Contains(initializers, inputs[1].node_arg.Name())) {
LOGS_DEFAULT(VERBOSE) << "B of Gemm must be known if transB != 1";
return false;
}
if (input_defs.size() == 3) {
if (inputs.size() == 3) {
Shape c_shape;
if (!GetShape(*input_defs[c_idx], c_shape))
if (!GetShape(inputs[2].node_arg, c_shape))
return false;
uint32_t c_size;
@ -1062,7 +1033,7 @@ bool GemmOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initial
}
} else if (op_type == "MatMul" || is_qlinear_matmul) {
// Only support A*B B is an initializer
if (!Contains(initializers, input_defs[b_idx]->Name())) {
if (!Contains(initializers, inputs[1].node_arg.Name())) {
LOGS_DEFAULT(VERBOSE) << "B of MatMul must be known";
return false;
}
@ -1070,7 +1041,7 @@ bool GemmOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initial
if (is_qlinear_matmul) {
// For QLinearMatMul, we only support uint8 output now
int32_t output_type;
if (!GetType(*node.OutputDefs()[0], output_type))
if (!GetType(node_unit.Outputs()[0].node_arg, output_type))
return false;
if (output_type != ONNX_NAMESPACE::TensorProto_DataType_UINT8) {
@ -1081,12 +1052,16 @@ bool GemmOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initial
}
// All scale/zero points are initializer scalars
// a/b/y_scale
if (!HasValidQuantizationScales(initializers, node, {1, 4, 6}, params))
// Check input scales and ZPs
if (!HasValidQuantizationScales(initializers, node_unit, {0, 1}, params, true /* is_input */))
return false;
if (!HasValidQuantizationZeroPoints(initializers, node_unit, {0, 1}, true /* is_input */))
return false;
// a/b/y_zero_point
if (!HasValidQuantizationZeroPoints(initializers, node, {2, 5, 7}))
// Check output scale and ZP
if (!HasValidQuantizationScales(initializers, node_unit, {0}, params, false /* is_input */))
return false;
if (!HasValidQuantizationZeroPoints(initializers, node_unit, {0}, false /* is_input */))
return false;
}
} else {
@ -1116,7 +1091,7 @@ class UnaryOpSupportChecker : public BaseOpSupportChecker {
int GetMinSupportedOpSet(const NodeUnit& node_unit) const override;
static bool IsQuantizedOpSupported(const InitializedTensorSet& initializers, const Node& node,
static bool IsQuantizedOpSupported(const InitializedTensorSet& initializers, const NodeUnit& node_unit,
const OpSupportCheckParams& params);
};
@ -1140,9 +1115,8 @@ class UnaryOpSupportChecker : public BaseOpSupportChecker {
bool UnaryOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initializers, const NodeUnit& node_unit,
const OpSupportCheckParams& params) const {
const auto& node = node_unit.GetNode();
if (node.OpType() == "QLinearSigmoid")
return IsQuantizedOpSupported(initializers, node, params);
if (node_unit.OpType() == "QLinearSigmoid")
return IsQuantizedOpSupported(initializers, node_unit, params);
else // Everything except "QLinearSigmoid" are by default supported
return true;
}
@ -1163,13 +1137,11 @@ int32_t UnaryOpSupportChecker::GetMinSupportedNNAPIFeatureLevel(const NodeUnit&
}
bool UnaryOpSupportChecker::HasSupportedInputsImpl(const NodeUnit& node_unit) const {
// TODO, change to use node unit and quant_param of IODef
const auto& node = node_unit.GetNode();
// We only need to override input check for QLinearSigmoid
if (node.OpType() != "QLinearSigmoid")
if (node_unit.OpType() != "QLinearSigmoid")
return BaseOpSupportChecker::HasSupportedInputsImpl(node_unit);
return HasValidUnaryOpQuantizedInputs(node);
return HasValidUnaryOpQuantizedInputs(node_unit);
}
// All ops except "Sin" opset 5- uses consumed_inputs attribute which is not supported for now
@ -1183,35 +1155,35 @@ int UnaryOpSupportChecker::GetMinSupportedOpSet(const NodeUnit& node_unit) const
}
/* static */ bool UnaryOpSupportChecker::IsQuantizedOpSupported(
const InitializedTensorSet& initializers, const Node& node, const OpSupportCheckParams& params) {
const auto& op_type = node.OpType();
const InitializedTensorSet& initializers, const NodeUnit& node_unit, const OpSupportCheckParams& params) {
const auto& op_type = node_unit.OpType();
ORT_ENFORCE(op_type == "QLinearSigmoid");
const auto& op_name = node.Name();
const auto input_defs(node.InputDefs());
// const auto output_defs(node.OutputDefs());
const auto& op_name = node_unit.Name();
if (input_defs.size() < 4)
// Check input scales and ZPs
if (!HasValidQuantizationScales(initializers, node_unit, {0}, params, true /* is_input */))
return false;
if (!HasValidQuantizationZeroPoints(initializers, node_unit, {0}, true /* is_input */))
return false;
bool has_output_zp = input_defs.size() == 5;
if (!HasValidQuantizationScales(initializers, node, {1, 3}, params))
// Check output scale and ZP
if (!HasValidQuantizationScales(initializers, node_unit, {0}, params, false /* is_input */))
return false;
if (!HasValidQuantizationZeroPoints(initializers, node_unit, {0}, false /* is_input */))
return false;
if (!HasValidQuantizationZeroPoints(initializers, node,
has_output_zp
? std::vector<size_t>{2}
: std::vector<size_t>{2, 4}))
return false;
return false;
// NNAPI requires the scale be 1.f/256 and zero point to be 0
// See https://android.googlesource.com/platform/frameworks/ml/+/refs/heads/android10-c2f2-release/nn/common/operations/Activation.cpp#180
float output_scale = 0.0f;
auto status = GetQuantizationScale(initializers, node, 3, output_scale);
int32_t output_zp = 0;
auto status = GetQuantizationScaleAndZeroPoint(initializers, node_unit.Outputs()[0], node_unit.ModelPath(),
output_scale, output_zp);
if (!status.IsOK()) {
LOGS_DEFAULT(ERROR) << "Op [" << op_type << "] name [" << op_name
<< "] GetQuantizationScale failed, message: " << status.ErrorMessage();
<< "] GetQuantizationScaleAndZeroPoint failed, message: " << status.ErrorMessage();
return false;
}
@ -1221,20 +1193,10 @@ int UnaryOpSupportChecker::GetMinSupportedOpSet(const NodeUnit& node_unit) const
return false;
}
int32_t output_zp;
if (has_output_zp) {
status = GetQuantizationZeroPoint(initializers, node, 4, output_zp);
if (!status.IsOK()) {
LOGS_DEFAULT(ERROR) << "Op [" << op_type << "] name [" << op_name
<< "] GetQuantizationZeroPoint failed, message: " << status.ErrorMessage();
return false;
}
if (output_zp != 0) {
LOGS_DEFAULT(VERBOSE) << "Op [" << op_type << "] name [" << op_name
<< "] output zero point can only be 0, actual zero point: " << output_scale;
return false;
}
if (output_zp != 0) {
LOGS_DEFAULT(VERBOSE) << "Op [" << op_type << "] name [" << op_name
<< "] output zero point can only be 0, actual zero point: " << output_scale;
return false;
}
return true;
@ -1254,9 +1216,8 @@ class ConcatOpSupportChecker : public BaseOpSupportChecker {
bool ConcatOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& /* initializers */, const NodeUnit& node_unit,
const OpSupportCheckParams& /* params */) const {
const auto& node = node_unit.GetNode();
Shape input_shape;
if (!GetShape(*node.InputDefs()[0], input_shape))
if (GetShape(node_unit.Inputs()[0].node_arg, input_shape))
return false;
const auto input_size = input_shape.size();
@ -1302,21 +1263,21 @@ class SqueezeOpSupportChecker : public BaseOpSupportChecker {
bool SqueezeOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initializers, const NodeUnit& node_unit,
const OpSupportCheckParams& /* params */) const {
const auto& node = node_unit.GetNode();
const auto& inputs = node_unit.Inputs();
Shape input_shape;
if (!GetShape(*node.InputDefs()[0], input_shape))
if (!GetShape(inputs[0].node_arg, input_shape))
return false;
const auto input_size = input_shape.size();
if (input_size > 4 || input_size == 0) {
const auto input_rank = input_shape.size();
if (input_rank > 4 || input_rank == 0) {
LOGS_DEFAULT(VERBOSE) << "Squeeze only supports 1-4d shape, input is "
<< input_size << "d shape";
<< input_rank << "d shape";
return false;
}
// Squeeze opset 13 use input 1 as axes, if we have input 1 then it need to be an initializer
if (node.SinceVersion() > 12 && node.InputDefs().size() > 1) {
const auto& axes_name = node.InputDefs()[1]->Name();
if (node_unit.SinceVersion() > 12 && inputs.size() > 1) {
const auto& axes_name = inputs[1].node_arg.Name();
if (!Contains(initializers, axes_name)) {
LOGS_DEFAULT(VERBOSE) << "Input axes of Squeeze must be known";
return false;
@ -1343,28 +1304,23 @@ class QuantizeLinearOpSupportChecker : public BaseOpSupportChecker {
bool QuantizeLinearOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initializers, const NodeUnit& node_unit,
const OpSupportCheckParams& params) const {
const auto& node = node_unit.GetNode();
const auto input_defs(node.InputDefs());
const auto output_defs(node.OutputDefs());
int32_t output_type;
if (!GetType(*output_defs[0], output_type))
if (!GetType(node_unit.Outputs()[0].node_arg, output_type))
return false;
if (output_type != ONNX_NAMESPACE::TensorProto_DataType_UINT8) {
LOGS_DEFAULT(VERBOSE) << "[" << node.OpType()
LOGS_DEFAULT(VERBOSE) << "[" << node_unit.OpType()
<< "] output type: [" << output_type
<< "] is not supported for now";
return false;
}
if (!HasValidQuantizationScales(initializers, node, {1}, params))
// For QuantizeLinear only output is quantized
// Check output scale and ZP
if (!HasValidQuantizationScales(initializers, node_unit, {0}, params, false /* is_input */))
return false;
if (!HasValidQuantizationZeroPoints(initializers, node_unit, {0}, false /* is_input */))
return false;
if (input_defs.size() == 3) { // has zero_point input
if (!HasValidQuantizationZeroPoints(initializers, node, {2}))
return false;
}
return true;
}
@ -1387,15 +1343,12 @@ class DequantizeLinearOpSupportChecker : public BaseOpSupportChecker {
bool DequantizeLinearOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initializers, const NodeUnit& node_unit,
const OpSupportCheckParams& params) const {
const auto& node = node_unit.GetNode();
const auto input_defs(node.InputDefs());
if (!HasValidQuantizationScales(initializers, node, {1}, params))
// For DequantizeLinear only input is quantized
// Check input scale and ZP
if (!HasValidQuantizationScales(initializers, node_unit, {0}, params, true /* is_input */))
return false;
if (!HasValidQuantizationZeroPoints(initializers, node_unit, {0}, true /* is_input */))
return false;
if (input_defs.size() == 3) { // has zero_point input
if (!HasValidQuantizationZeroPoints(initializers, node, {2}))
return false;
}
return true;
}
@ -1432,9 +1385,8 @@ class LRNOpSupportChecker : public BaseOpSupportChecker {
bool LRNOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& /* initializers */, const NodeUnit& node_unit,
const OpSupportCheckParams& /* params */) const {
const auto& node = node_unit.GetNode();
Shape input_shape;
if (!GetShape(*node.InputDefs()[0], input_shape))
if (!GetShape(node_unit.Inputs()[0].node_arg, input_shape))
return false;
const auto input_size = input_shape.size();
@ -1459,9 +1411,8 @@ class ClipOpSupportChecker : public BaseOpSupportChecker {
bool ClipOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initializers, const NodeUnit& node_unit,
const OpSupportCheckParams& /* params */) const {
const auto& node = node_unit.GetNode();
float min, max;
if (!GetClipMinMax(initializers, node, min, max, logging::LoggingManager::DefaultLogger()))
if (!GetClipMinMax(initializers, node_unit.GetNode(), min, max, logging::LoggingManager::DefaultLogger()))
return false;
// We only supoort relu6 or relu1
@ -1496,9 +1447,8 @@ class ResizeOpSupportChecker : public BaseOpSupportChecker {
bool ResizeOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initializers, const NodeUnit& node_unit,
const OpSupportCheckParams& params) const {
const auto& node = node_unit.GetNode();
Shape input_shape;
if (!GetShape(*node.InputDefs()[0], input_shape))
if (!GetShape(node_unit.Inputs()[0].node_arg, input_shape))
return false;
const auto input_size = input_shape.size();
@ -1509,7 +1459,7 @@ bool ResizeOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initi
}
{ // check attributes
NodeAttrHelper helper(node);
NodeAttrHelper helper(node_unit);
const auto mode = helper.Get("mode", "nearest");
bool is_linear_resize = mode == "linear";
bool is_nearest_resize = mode == "nearest";
@ -1556,27 +1506,27 @@ bool ResizeOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initi
}
{ // scales and sizes (if present) must be initializers
const auto input_defs = node.InputDefs();
if (input_defs.size() < 3) {
const auto inputs = node_unit.Inputs();
if (inputs.size() < 3) {
LOGS_DEFAULT(VERBOSE) << "Input scales or sizes of Resize must be known";
return false;
}
// scales
if (input_defs.size() == 3 && !Contains(initializers, input_defs[2]->Name())) {
if (inputs.size() == 3 && !Contains(initializers, inputs[2].node_arg.Name())) {
LOGS_DEFAULT(VERBOSE) << "Input scales of Resize must be known";
return false;
}
// sizes
if (input_defs.size() > 3 && !Contains(initializers, input_defs[3]->Name())) {
if (inputs.size() > 3 && !Contains(initializers, inputs[3].node_arg.Name())) {
LOGS_DEFAULT(VERBOSE) << "Input sizes of Resize must be known";
return false;
}
// We want to check if the scales or sizes are not trying to resize on N/C channels here
if (input_defs.size() == 3) { // we are using scales
const auto& scales_tensor = *initializers.at(input_defs[2]->Name());
if (inputs.size() == 3) { // we are using scales
const auto& scales_tensor = *initializers.at(inputs[2].node_arg.Name());
std::vector<uint8_t> unpacked_tensor;
auto status = onnxruntime::utils::UnpackInitializerData(scales_tensor, unpacked_tensor);
if (!status.IsOK()) {
@ -1594,7 +1544,7 @@ bool ResizeOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initi
}
} else {
// we are using sizes
const auto& sizes_name = input_defs[3]->Name();
const auto& sizes_name = inputs[3].node_arg.Name();
const auto& sizes_tensor = *initializers.at(sizes_name);
std::vector<uint8_t> unpacked_tensor;
auto status = onnxruntime::utils::UnpackInitializerData(sizes_tensor, unpacked_tensor);
@ -1659,9 +1609,8 @@ class FlattenOpSupportChecker : public BaseOpSupportChecker {
bool FlattenOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& /* initializers */, const NodeUnit& node_unit,
const OpSupportCheckParams& /* params */) const {
const auto& node = node_unit.GetNode();
Shape input_shape;
if (!GetShape(*node.InputDefs()[0], input_shape))
if (!GetShape(node_unit.Inputs()[0].node_arg, input_shape))
return false;
if (input_shape.size() > 4 || input_shape.empty()) {
@ -1672,7 +1621,7 @@ bool FlattenOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& /* i
int32_t dim_1 = 1;
int32_t dim_2 = 1;
GetFlattenOutputShape(node, input_shape, dim_1, dim_2);
GetFlattenOutputShape(node_unit, input_shape, dim_1, dim_2);
if (dim_1 == 0 && dim_2 == 0) {
LOGS_DEFAULT(VERBOSE) << "The dynamical input shape " << Shape2String(input_shape)
@ -1717,11 +1666,10 @@ class MinMaxOpSupportChecker : public BaseOpSupportChecker {
bool MinMaxOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& /* initializers */, const NodeUnit& node_unit,
const OpSupportCheckParams& /* params */) const {
const auto& node = node_unit.GetNode();
// TODO support 2+ inputs for Min/Max op
if (node.InputDefs().size() != 2) {
LOGS_DEFAULT(VERBOSE) << "[" << node.OpType() << "] only supports 2 inputs, "
<< "actual input number, " << node.InputDefs().size();
if (node_unit.Inputs().size() != 2) {
LOGS_DEFAULT(VERBOSE) << "[" << node_unit.OpType() << "] only supports 2 inputs, "
<< "actual input number, " << node_unit.Inputs().size();
return false;
}
@ -1763,9 +1711,8 @@ class SliceOpSupportChecker : public BaseOpSupportChecker {
bool SliceOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initializers, const NodeUnit& node_unit,
const OpSupportCheckParams& /* params */) const {
const auto& node = node_unit.GetNode();
Shape input_shape;
if (!GetShape(*node.InputDefs()[0], input_shape))
if (!GetShape(node_unit.Inputs()[0].node_arg, input_shape))
return false;
if (input_shape.size() > 4) {
@ -1780,19 +1727,19 @@ bool SliceOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initia
return false;
}
if (!CheckIsInitializer(initializers, node, 1, "starts")) {
if (!CheckIsInitializer(initializers, node_unit, node_unit.Inputs()[1].node_arg.Name(), "starts")) {
return false;
}
if (!CheckIsInitializer(initializers, node, 2, "ends")) {
if (!CheckIsInitializer(initializers, node_unit, node_unit.Inputs()[2].node_arg.Name(), "ends")) {
return false;
}
const auto& input_defs = node.InputDefs();
if (input_defs.size() > 3) {
if (!CheckIsInitializer(initializers, node, 3, "axes")) {
const auto& inputs = node_unit.Inputs();
if (inputs.size() > 3) {
if (!CheckIsInitializer(initializers, node_unit, node_unit.Inputs()[3].node_arg.Name(), "axes")) {
return false;
}
if (input_defs.size() > 4) {
if (!CheckIsInitializer(initializers, node, 4, "steps")) {
if (inputs.size() > 4) {
if (!CheckIsInitializer(initializers, node_unit, node_unit.Inputs()[4].node_arg.Name(), "steps")) {
return false;
}
}

23
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.cc
View file

@ -3,20 +3,17 @@
#include "core/providers/common.h"
#include "helper.h"
#include "shaper.h"
#include "helper.h"
namespace onnxruntime {
namespace nnapi {
using std::string;
using std::vector;
std::pair<uint32_t, uint32_t> ComputeConvOutputShape(const uint32_t input_size_y, const uint32_t input_size_x,
const uint32_t weight_size_y, const uint32_t weight_size_x,
const vector<int32_t>& onnx_pads,
const vector<int32_t>& onnx_strides,
const vector<int32_t>& onnx_dilations) {
const std::vector<int32_t>& onnx_pads,
const std::vector<int32_t>& onnx_strides,
const std::vector<int32_t>& onnx_dilations) {
int32_t padding_top = onnx_pads[0];
int32_t padding_bottom = onnx_pads[2];
int32_t padding_left = onnx_pads[1];
@ -53,9 +50,9 @@ std::pair<uint32_t, uint32_t> ComputeConvOutputShape(const uint32_t input_size_y
Status Shaper::Conv(const std::string& input_name,
const std::string& weight_name,
const vector<int32_t>& onnx_pads,
const vector<int32_t>& onnx_strides,
const vector<int32_t>& onnx_dilations,
const std::vector<int32_t>& onnx_pads,
const std::vector<int32_t>& onnx_strides,
const std::vector<int32_t>& onnx_dilations,
bool nchw,
const std::string& output_name) {
SHAPER_FUNC(Conv,
@ -150,9 +147,9 @@ Status Shaper::ResizeUsingOutputSizes(const std::string& input_name,
Status Shaper::ConvImpl(const std::string& input_name,
const std::string& weight_name,
const vector<int32_t>& onnx_pads,
const vector<int32_t>& onnx_strides,
const vector<int32_t>& onnx_dilations,
const std::vector<int32_t>& onnx_pads,
const std::vector<int32_t>& onnx_strides,
const std::vector<int32_t>& onnx_dilations,
bool nchw,
const std::string& output_name) {
const Shape& input_dimen = shape_map_.at(input_name);

161
onnxruntime/core/providers/nnapi/nnapi_builtin/builders/shaper.h
View file

@ -6,7 +6,8 @@
#include <string>
#include <unordered_map>
#include <vector>
#include <core/session/onnxruntime_c_api.h>
#include "core/common/status.h"
namespace onnxruntime {
namespace nnapi {
@ -20,115 +21,103 @@ class Shaper {
return shape_map_.at(key);
}
Status Conv(const std::string& input_name,
const std::string& weight_name,
const std::vector<int32_t>& onnx_pads,
const std::vector<int32_t>& onnx_strides,
const std::vector<int32_t>& onnx_dilations,
bool nchw,
const std::string& output_name) ORT_MUST_USE_RESULT;
common::Status Conv(const std::string& input_name,
const std::string& weight_name,
const std::vector<int32_t>& onnx_pads,
const std::vector<int32_t>& onnx_strides,
const std::vector<int32_t>& onnx_dilations,
bool nchw,
const std::string& output_name);
Status DepthwiseConv(const std::string& input_name,
const std::string& weight_name,
const std::vector<int32_t>& onnx_pads,
const std::vector<int32_t>& onnx_strides,
const std::vector<int32_t>& onnx_dilations,
bool nchw,
const std::string& output_name) ORT_MUST_USE_RESULT;
common::Status DepthwiseConv(const std::string& input_name,
const std::string& weight_name,
const std::vector<int32_t>& onnx_pads,
const std::vector<int32_t>& onnx_strides,
const std::vector<int32_t>& onnx_dilations,
bool nchw,
const std::string& output_name);
Status Pool(const std::string& input_name,
const std::vector<int32_t>& onnx_pads,
const std::vector<int32_t>& onnx_strides,
const std::vector<int32_t>& kernel_shape,
bool nchw,
const std::string& output_name) ORT_MUST_USE_RESULT;
common::Status Pool(const std::string& input_name,
const std::vector<int32_t>& onnx_pads,
const std::vector<int32_t>& onnx_strides,
const std::vector<int32_t>& kernel_shape,
bool nchw,
const std::string& output_name);
Status Reshape(const std::string& input_name, const std::vector<int32_t>& shape, const std::string& output_name)
ORT_MUST_USE_RESULT;
common::Status Reshape(const std::string& input_name, const std::vector<int32_t>& shape, const std::string& output_name);
Status Transpose(const std::string& input_name, const std::vector<int32_t>& perm, const std::string& output_name)
ORT_MUST_USE_RESULT;
common::Status Transpose(const std::string& input_name, const std::vector<int32_t>& perm, const std::string& output_name);
Status Eltwise(const std::string& input1_name, const std::string& input2_name, const std::string& output_name)
ORT_MUST_USE_RESULT;
common::Status Eltwise(const std::string& input1_name, const std::string& input2_name, const std::string& output_name);
Status Identity(const std::string& input_name, const std::string& output_name) ORT_MUST_USE_RESULT;
common::Status Identity(const std::string& input_name, const std::string& output_name);
Status FC(const std::string& input1_name, const std::string& input2_name, const std::string& output_name)
ORT_MUST_USE_RESULT;
common::Status FC(const std::string& input1_name, const std::string& input2_name, const std::string& output_name);
Status Concat(const std::vector<std::string>& input_names, const int32_t axis, const std::string& output_name)
ORT_MUST_USE_RESULT;
common::Status Concat(const std::vector<std::string>& input_names, const int32_t axis, const std::string& output_name);
Status Squeeze(const std::string& input_name, const std::vector<int32_t>& axes, const std::string& output_name)
ORT_MUST_USE_RESULT;
common::Status Squeeze(const std::string& input_name, const std::vector<int32_t>& axes, const std::string& output_name);
Status ResizeUsingScales(const std::string& input_name,
const float scale_h, const float scale_w,
bool nchw,
const std::string& output_name) ORT_MUST_USE_RESULT;
Status ResizeUsingOutputSizes(const std::string& input_name,
const uint32_t output_h, const uint32_t output_w,
bool nchw,
const std::string& output_name) ORT_MUST_USE_RESULT;
common::Status ResizeUsingScales(const std::string& input_name,
const float scale_h, const float scale_w,
bool nchw,
const std::string& output_name);
common::Status ResizeUsingOutputSizes(const std::string& input_name,
const uint32_t output_h, const uint32_t output_w,
bool nchw,
const std::string& output_name);
// If the shape of certain input is dynamic
// Use the following 2 functions to update the particular shape
// and calculate the new output shape
// Only perform this when the NNAPI model is finalized!
Status UpdateShape(const std::string& name, const Shape& new_shape) ORT_MUST_USE_RESULT;
Status UpdateDynamicDimensions() ORT_MUST_USE_RESULT;
common::Status UpdateShape(const std::string& name, const Shape& new_shape);
common::Status UpdateDynamicDimensions();
void Clear();
private:
Status ConvImpl(const std::string& input_name,
const std::string& weight_name,
const std::vector<int32_t>& onnx_pads,
const std::vector<int32_t>& onnx_strides,
const std::vector<int32_t>& onnx_dilations,
bool nchw,
const std::string& output_name) ORT_MUST_USE_RESULT;
common::Status ConvImpl(const std::string& input_name,
const std::string& weight_name,
const std::vector<int32_t>& onnx_pads,
const std::vector<int32_t>& onnx_strides,
const std::vector<int32_t>& onnx_dilations,
bool nchw,
const std::string& output_name);
Status DepthwiseConvImpl(const std::string& input_name,
const std::string& weight_name,
const std::vector<int32_t>& onnx_pads,
const std::vector<int32_t>& onnx_strides,
const std::vector<int32_t>& onnx_dilations,
bool nchw,
const std::string& output_name) ORT_MUST_USE_RESULT;
common::Status DepthwiseConvImpl(const std::string& input_name,
const std::string& weight_name,
const std::vector<int32_t>& onnx_pads,
const std::vector<int32_t>& onnx_strides,
const std::vector<int32_t>& onnx_dilations,
bool nchw,
const std::string& output_name);
Status PoolImpl(const std::string& input_name,
const std::vector<int32_t>& onnx_pads,
const std::vector<int32_t>& onnx_strides,
const std::vector<int32_t>& kernel_shape,
bool nchw,
const std::string& output_name) ORT_MUST_USE_RESULT;
common::Status PoolImpl(const std::string& input_name,
const std::vector<int32_t>& onnx_pads,
const std::vector<int32_t>& onnx_strides,
const std::vector<int32_t>& kernel_shape,
bool nchw,
const std::string& output_name);
Status ReshapeImpl(const std::string& input_name, const std::vector<int32_t>& shape, const std::string& output_name)
ORT_MUST_USE_RESULT;
Status TransposeImpl(const std::string& input_name, const std::vector<int32_t>& perm, const std::string& output_name)
ORT_MUST_USE_RESULT;
Status EltwiseImpl(const std::string& input1_name, const std::string& input2_name, const std::string& output_name)
ORT_MUST_USE_RESULT;
Status IdentityImpl(const std::string& input_name, const std::string& output_name) ORT_MUST_USE_RESULT;
Status FCImpl(const std::string& input1_name, const std::string& input2_name, const std::string& output_name)
ORT_MUST_USE_RESULT;
Status ConcatImpl(const std::vector<std::string>& input_names, const int32_t axis, const std::string& output_name)
ORT_MUST_USE_RESULT;
Status SqueezeImpl(const std::string& input_names, const std::vector<int32_t>& axes, const std::string& output_name)
ORT_MUST_USE_RESULT;
Status ResizeUsingScalesImpl(const std::string& input_name,
const float scale_h, const float scale_w,
bool nchw,
const std::string& output_name) ORT_MUST_USE_RESULT;
Status ResizeUsingOutputSizesImpl(const std::string& input_name,
const uint32_t output_h, const uint32_t output_w,
bool nchw,
const std::string& output_name) ORT_MUST_USE_RESULT;
common::Status ReshapeImpl(const std::string& input_name, const std::vector<int32_t>& shape, const std::string& output_name);
common::Status TransposeImpl(const std::string& input_name, const std::vector<int32_t>& perm, const std::string& output_name);
common::Status EltwiseImpl(const std::string& input1_name, const std::string& input2_name, const std::string& output_name);
common::Status IdentityImpl(const std::string& input_name, const std::string& output_name);
common::Status FCImpl(const std::string& input1_name, const std::string& input2_name, const std::string& output_name);
common::Status ConcatImpl(const std::vector<std::string>& input_names, const int32_t axis, const std::string& output_name);
common::Status SqueezeImpl(const std::string& input_names, const std::vector<int32_t>& axes, const std::string& output_name);
common::Status ResizeUsingScalesImpl(const std::string& input_name,
const float scale_h, const float scale_w,
bool nchw,
const std::string& output_name);
common::Status ResizeUsingOutputSizesImpl(const std::string& input_name,
const uint32_t output_h, const uint32_t output_w,
bool nchw,
const std::string& output_name);
std::unordered_map<std::string, Shape> shape_map_;
std::vector<std::function<Status(Shaper&)>> shape_ops_;
std::vector<std::function<common::Status(Shaper&)>> shape_ops_;
};
} // namespace nnapi

4
onnxruntime/core/providers/nnapi/nnapi_builtin/model.cc
View file

@ -1,9 +1,9 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include <core/common/logging/logging.h>
#include "model.h"
#include "core/common/logging/logging.h"
#include "core/providers/common.h"
#include "core/providers/nnapi/nnapi_builtin/builders/helper.h"
#include "core/providers/nnapi/nnapi_builtin/nnapi_lib/nnapi_implementation.h"

15
onnxruntime/core/providers/nnapi/nnapi_builtin/model.h
View file

@ -103,7 +103,7 @@ class Model {
// this output may need special handling
bool IsScalarOutput(const std::string& output_name) const;
Status PrepareForExecution(std::unique_ptr<Execution>& execution) ORT_MUST_USE_RESULT;
common::Status PrepareForExecution(std::unique_ptr<Execution>& execution);
private:
const NnApi* nnapi_{nullptr};
@ -143,7 +143,7 @@ class Model {
void AddScalarOutput(const std::string& output_name);
void SetShaper(const Shaper shaper) { shaper_ = shaper; }
void SetShaper(const Shaper& shaper) { shaper_ = shaper; }
int32_t GetNNAPIFeatureLevel() const;
};
@ -172,17 +172,16 @@ class Execution {
// Set the input/output data buffers
// These need to be called before calling Predict()
Status SetInputBuffers(const std::vector<InputBuffer>& inputs) ORT_MUST_USE_RESULT;
Status SetOutputBuffers(const std::vector<OutputBuffer>& outputs) ORT_MUST_USE_RESULT;
common::Status SetInputBuffers(const std::vector<InputBuffer>& inputs);
common::Status SetOutputBuffers(const std::vector<OutputBuffer>& outputs);
// Execute the NNAPI model
// if there is dynamic output shape, will output the actual output shapes
Status Predict(const std::vector<int32_t>& dynamic_outputs, std::vector<Shaper::Shape>& dynamic_output_shapes)
ORT_MUST_USE_RESULT;
common::Status Predict(const std::vector<int32_t>& dynamic_outputs, std::vector<Shaper::Shape>& dynamic_output_shapes);
private:
Status SetInputBuffer(const int32_t index, const InputBuffer& input) ORT_MUST_USE_RESULT;
Status SetOutputBuffer(const int32_t index, const OutputBuffer& output) ORT_MUST_USE_RESULT;
common::Status SetInputBuffer(const int32_t index, const InputBuffer& input);
common::Status SetOutputBuffer(const int32_t index, const OutputBuffer& output);
const NnApi* nnapi_{nullptr};
ANeuralNetworksExecution* execution_;

10
onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc
View file

@ -21,8 +21,6 @@
#include "core/providers/nnapi/nnapi_builtin/model.h"
#endif
using onnxruntime::NodeUnit;
namespace onnxruntime {
namespace {
@ -189,14 +187,6 @@ NnapiExecutionProvider::GetCapability(const onnxruntime::GraphViewer& graph_view
}
#ifdef __ANDROID__
static Status GetOutputBuffer(Ort::CustomOpApi& ort,
OrtKernelContext* context,
const nnapi::Model& model,
const std::string& output_name,
const std::vector<uint32_t>& output_shape,
const android::nn::wrapper::Type output_type,
void** output_buffer) ORT_MUST_USE_RESULT;
static Status GetOutputBuffer(Ort::CustomOpApi& ort,
OrtKernelContext* context,
const nnapi::Model& model,

180
onnxruntime/core/providers/shared/node_unit/node_unit.cc
View file

@ -6,40 +6,170 @@
namespace onnxruntime {
namespace {
// The QLinearOpType GetQLinearOpType, is very similar to the one in NNAPI
// However, the NNAPI ones are only the subset of the ones here,
// TODO, make these shared
enum class QLinearOpType : uint8_t {
Unknown, // Unknown or not a linear quantized op
DequantizeLinear,
QuantizeLinear,
QLinearConv,
QLinearMatMul,
QLinearAdd,
QLinearSigmoid,
QLinearAveragePool,
QLinearMul,
QLinearReduceMean,
QLinearConcat,
QLinearGlobalAveragePool,
QLinearLeakyRelu,
};
QLinearOpType GetQLinearOpType(const onnxruntime::Node& node) {
const auto& op_type = node.OpType();
if (op_type == "DequantizeLinear")
return QLinearOpType::DequantizeLinear;
else if (op_type == "QuantizeLinear")
return QLinearOpType::QuantizeLinear;
else if (op_type == "QLinearConv")
return QLinearOpType::QLinearConv;
else if (op_type == "QLinearMatMul")
return QLinearOpType::QLinearMatMul;
else if (op_type == "QLinearAdd")
return QLinearOpType::QLinearAdd;
else if (op_type == "QLinearSigmoid")
return QLinearOpType::QLinearSigmoid;
else if (op_type == "QLinearAveragePool")
return QLinearOpType::QLinearAveragePool;
else if (op_type == "QLinearMul")
return QLinearOpType::QLinearMul;
else if (op_type == "QLinearReduceMean")
return QLinearOpType::QLinearReduceMean;
else if (op_type == "QLinearConcat")
return QLinearOpType::QLinearConcat;
else if (op_type == "QLinearGlobalAveragePool")
return QLinearOpType::QLinearGlobalAveragePool;
else if (op_type == "QLinearLeakyRelu")
return QLinearOpType::QLinearLeakyRelu;
return QLinearOpType::Unknown;
}
// Ops have 1 input
bool IsUnaryQLinearOp(QLinearOpType type) {
return type == QLinearOpType::QLinearSigmoid ||
type == QLinearOpType::QLinearAveragePool ||
type == QLinearOpType::QLinearGlobalAveragePool ||
type == QLinearOpType::QLinearLeakyRelu ||
type == QLinearOpType::QLinearReduceMean;
}
// Ops have 2 inputs
bool IsBinaryQLinearOp(QLinearOpType type) {
return type == QLinearOpType::QLinearConv ||
type == QLinearOpType::QLinearMatMul ||
type == QLinearOpType::QLinearAdd ||
type == QLinearOpType::QLinearMul;
}
// Ops have 1 or more inputs
bool IsVariadicQLinearOp(QLinearOpType type) {
return type == QLinearOpType::QLinearConcat;
}
} // namespace
NodeUnit::NodeUnit(const Node& node)
: nodes_{&node},
node_(node),
: output_nodes_{&node},
target_node_(node),
type_(Type::SingleNode) {
InitForNode();
}
const std::string& NodeUnit::Domain() const noexcept { return node_.Domain(); }
const std::string& NodeUnit::OpType() const noexcept { return node_.OpType(); }
const std::string& NodeUnit::Name() const noexcept { return node_.Name(); }
int NodeUnit::SinceVersion() const noexcept { return node_.SinceVersion(); }
NodeIndex NodeUnit::Index() const noexcept { return node_.Index(); }
const Path& NodeUnit::ModelPath() const noexcept { return node_.ModelPath(); }
ProviderType NodeUnit::GetExecutionProviderType() const noexcept { return node_.GetExecutionProviderType(); }
const std::string& NodeUnit::Domain() const noexcept { return target_node_.Domain(); }
const std::string& NodeUnit::OpType() const noexcept { return target_node_.OpType(); }
const std::string& NodeUnit::Name() const noexcept { return target_node_.Name(); }
int NodeUnit::SinceVersion() const noexcept { return target_node_.SinceVersion(); }
NodeIndex NodeUnit::Index() const noexcept { return target_node_.Index(); }
const Path& NodeUnit::ModelPath() const noexcept { return target_node_.ModelPath(); }
ProviderType NodeUnit::GetExecutionProviderType() const noexcept { return target_node_.GetExecutionProviderType(); }
void NodeUnit::InitForNode() {
const auto& input_defs = node_.InputDefs();
const auto& output_defs = node_.OutputDefs();
// The 1st step is to hookup the NodeUnit with the NNAPI builder interface
// So we are not handling quantization here now
// TODO, enable quantization
// auto qlinear_type = GetQLinearOpType(node_);
// if (qlinear_type == QLinearOpType::Unknown) {
// Not a Qlinear op, add all inputs/outputs
auto add_all_io = [](std::vector<IODef>& defs,
const ConstPointerContainer<std::vector<NodeArg*>>& node_defs) {
defs.reserve(node_defs.size());
const auto& input_defs = target_node_.InputDefs();
const auto& output_defs = target_node_.OutputDefs();
auto qlinear_type = GetQLinearOpType(target_node_);
if (qlinear_type == QLinearOpType::Unknown ||
IsVariadicQLinearOp(qlinear_type)) { // TODO, add variadic support
// Not a Qlinear op, add all inputs / outputs
auto add_all_io = [](std::vector<NodeUnitIODef>& defs,
const ConstPointerContainer<std::vector<NodeArg*>>& node_defs) {
defs.reserve(node_defs.size());
for (const auto def : node_defs) {
defs.push_back(NodeUnit::IODef{*def, std::nullopt});
for (const auto def : node_defs) {
defs.push_back(NodeUnitIODef{*def, std::nullopt});
}
};
add_all_io(inputs_, input_defs);
add_all_io(outputs_, output_defs);
} else if (IsUnaryQLinearOp(qlinear_type)) {
// Unary QLinear Op has 5 inputs
// x, x_scale, x_zp, y_scale, y_zp (optional)
inputs_.push_back(NodeUnitIODef{
*input_defs[0],
NodeUnitIODef::QuantParam{*input_defs[1], input_defs[2]}});
outputs_.push_back(NodeUnitIODef{
*output_defs[0],
NodeUnitIODef::QuantParam{*input_defs[3],
input_defs.size() > 4
? input_defs[4]
: nullptr}});
} else if (IsBinaryQLinearOp(qlinear_type)) {
// Binary QLinear Op has 9 inputs
// x1, x1_scale, x1_zp, x2/w, x2_scale, x2_zp, y_scale , y_zp, B
inputs_.push_back(NodeUnitIODef{
*input_defs[0],
NodeUnitIODef::QuantParam{*input_defs[1], input_defs[2]}});
inputs_.push_back(NodeUnitIODef{
*input_defs[3],
NodeUnitIODef::QuantParam{*input_defs[4], input_defs[5]}});
if (input_defs.size() == 9) { // has Bias
inputs_.push_back(NodeUnitIODef{
*input_defs[8],
std::nullopt}); // for Bias the scale and zp are optional
}
};
add_all_io(input_defs_, input_defs);
add_all_io(output_defs_, output_defs);
outputs_.push_back(NodeUnitIODef{
*output_defs[0],
NodeUnitIODef::QuantParam{*input_defs[6], input_defs[7]}});
} else if (qlinear_type == QLinearOpType::DequantizeLinear) {
// DequantizeLinear has 3 inputs
// x, x_scale, x_zp
// output is not quantized
inputs_.push_back(NodeUnitIODef{
*input_defs[0],
NodeUnitIODef::QuantParam{*input_defs[1],
input_defs.size() == 3
? input_defs[2]
: nullptr}});
outputs_.push_back(NodeUnitIODef{*output_defs[0], std::nullopt});
} else if (qlinear_type == QLinearOpType::QuantizeLinear) {
// QuantizeLinear the input is not quantized and has 3 inputs
// x, y_scale, y_zp (optional)
// The output is quantized
inputs_.push_back(NodeUnitIODef{*input_defs[0], std::nullopt});
outputs_.push_back(NodeUnitIODef{
*output_defs[0],
NodeUnitIODef::QuantParam{*input_defs[1],
input_defs.size() == 3
? input_defs[2]
: nullptr}});
} else {
ORT_THROW("The QLinear op [", static_cast<uint8_t>(qlinear_type), "] is not supported");
}
}
} // namespace onnxruntime

45
onnxruntime/core/providers/shared/node_unit/node_unit.h
View file

@ -21,6 +21,20 @@ namespace QDQ {
struct NodeGroup;
}
// Definition of one input or output
// If the optional quant_param is present, then this is a quantized input,
// otherwise this is a regular input
struct NodeUnitIODef {
// The quantization parameter, scale is manadatory, and zero_point is optional
struct QuantParam {
const NodeArg& scale;
const NodeArg* zero_point{nullptr};
};
const NodeArg& node_arg;
const std::optional<QuantParam> quant_param;
};
/**
@class NodeUnit
Class to represent a single node or a QDQ group of nodes, which will be used as a single unit.
@ -33,27 +47,13 @@ class NodeUnit {
QDQGroup, // The NodeUnit contain a QDQ group of nodes, such as "DQ->Sigmoid->Q"
};
// Definition of one input or output
// If the optional quant_param is present, then this is a quantized input,
// otherwise this is a regular input
struct IODef {
// The quantization parmeter, scale is manadatory, and zero_point is optional
struct QuantParam {
const NodeArg& scale;
const NodeArg* zero_point{nullptr};
};
const NodeArg& node_arg;
const std::optional<QuantParam> quant_param;
};
public:
explicit NodeUnit(const Node& node);
Type UnitType() const noexcept { return type_; }
const std::vector<IODef>& Inputs() const noexcept { return input_defs_; }
const std::vector<IODef>& Outputs() const noexcept { return output_defs_; }
const std::vector<NodeUnitIODef>& Inputs() const noexcept { return inputs_; }
const std::vector<NodeUnitIODef>& Outputs() const noexcept { return outputs_; }
const std::string& Domain() const noexcept;
const std::string& OpType() const noexcept;
@ -63,16 +63,15 @@ class NodeUnit {
const Path& ModelPath() const noexcept;
ProviderType GetExecutionProviderType() const noexcept;
const Node& GetNode() const noexcept { return node_; }
const std::vector<const Node*> GetAllNodes() const noexcept { return nodes_; }
const Node& GetNode() const noexcept { return target_node_; }
const std::vector<const Node*> GetOutputNodes() const noexcept { return output_nodes_; }
private:
std::vector<IODef> input_defs_;
std::vector<IODef> output_defs_;
std::vector<NodeUnitIODef> inputs_;
std::vector<NodeUnitIODef> outputs_;
const std::vector<const Node*> nodes_; // all nodes in this NodeUnit
const Node& node_; // target Node
const std::vector<const Node*> output_nodes_; // all the nodes producing outputs for this NodeUnit
const Node& target_node_;
Type type_;
void InitForNode(); // Initializing for single Node

4
onnxruntime/core/providers/shared/utils/utils.cc
View file

@ -8,6 +8,7 @@
#include <core/framework/tensorprotoutils.h>
#include <core/graph/graph.h>
#include <core/providers/common.h>
#include "core/providers/shared/node_unit/node_unit.h"
namespace onnxruntime {
@ -81,6 +82,9 @@ bool GetClipMinMax(const InitializedTensorSet& initializers, const Node& node,
NodeAttrHelper::NodeAttrHelper(const onnxruntime::Node& node)
: node_attributes_(node.GetAttributes()) {}
NodeAttrHelper::NodeAttrHelper(const NodeUnit& node_unit)
: node_attributes_(node_unit.GetNode().GetAttributes()) {}
float NodeAttrHelper::Get(const std::string& key, float def_val) const {
if (!HasAttr(key))
return def_val;

8
onnxruntime/core/providers/shared/utils/utils.h
View file

@ -17,6 +17,7 @@ class Logger;
class Node;
class NodeArg;
class NodeUnit;
// Get the min/max of a Clip operator.
// If min/max are not known initializer tensors, will return false
@ -34,7 +35,10 @@ bool GetType(const NodeArg& node_arg, int32_t& type, const logging::Logger& logg
*/
class NodeAttrHelper {
public:
NodeAttrHelper(const onnxruntime::Node& node);
explicit NodeAttrHelper(const Node& node);
// Get the attributes from the target node of the node_unit
explicit NodeAttrHelper(const NodeUnit& node_unit);
float Get(const std::string& key, float def_val) const;
@ -52,7 +56,7 @@ class NodeAttrHelper {
bool HasAttr(const std::string& key) const;
private:
const onnxruntime::NodeAttributes& node_attributes_;
const NodeAttributes& node_attributes_;
};
} // namespace onnxruntime