diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc index 324f2a8bee..ede0a3086d 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.cc @@ -109,21 +109,6 @@ bool IsQuantizedBinaryOp(QuantizedOpType quant_op_type) { IsQuantizedConv(quant_op_type); } -bool HasValidUnaryOpQuantizedInputs(const NodeUnit& node_unit) { - int32_t input_type; - if (!GetType(node_unit.Inputs()[0].node_arg, input_type)) - return false; - - if (input_type != ONNX_NAMESPACE::TensorProto_DataType_UINT8) { - LOGS_DEFAULT(VERBOSE) << "[" << node_unit.OpType() - << "] Input type: [" << input_type - << "] is not supported for now"; - return false; - } - - return true; -} - bool HasValidBinaryOpQuantizedInputTypes(const NodeUnit& node_unit) { auto quant_op_type = GetQuantizedOpType(node_unit); int32_t a_input_type, b_input_type; @@ -159,387 +144,6 @@ bool HasValidBinaryOpQuantizedInputTypes(const NodeUnit& node_unit) { return true; } -bool IsQuantizationScaleSupported(const InitializedTensorSet& initializers, - const NodeUnitIODef& io_def, - const OpSupportCheckParams& params, - const std::string& op_type, - bool is_quant_matmul, - bool is_conv_matmul_weight, - bool is_conv_matmul_u8s8_weight) { - const auto scale_name = io_def.quant_param->scale.Name(); - if (!Contains(initializers, scale_name)) { - LOGS_DEFAULT(VERBOSE) << "The scale of " << op_type << " must be an initializer tensor"; - return false; - } - - const auto& scale_tensor = *initializers.at(scale_name); - int64_t scales_dim = scale_tensor.dims().empty() ? 1 : scale_tensor.dims()[0]; - if (!is_conv_matmul_u8s8_weight) { - if (scales_dim != 1) { - LOGS_DEFAULT(VERBOSE) << op_type << " does not support per-channel quantization, " - << " for now, only u8s8 QlinearConv supports per-channel quantization on API 29+"; - return false; - } - } else if (scales_dim != 1) { - // For u8s8 Qlinear[Conv/MatMul], we support - // 1. Per-tensor, the weight will be transformed to uint8 later - // 2. Per-channel, only from Android API level 29 - if (is_quant_matmul) { - LOGS_DEFAULT(VERBOSE) << "QLinearMatMul does not support per-channel quantization"; - return false; - } - - if (params.android_feature_level < ANEURALNETWORKS_FEATURE_LEVEL_3) { - LOGS_DEFAULT(VERBOSE) << op_type << " only supports per-channel quantization on Android API 29+, " - << "system NNAPI feature level: " << params.android_feature_level; - return false; - } - - Shape weight_shape; - if (!GetShape(io_def.node_arg, weight_shape)) - return false; - - if (weight_shape[0] != scales_dim) { - LOGS_DEFAULT(VERBOSE) << op_type << " mismatch int8 per-channel quantization weight," - << " weight dimension[0] " << weight_shape[0] - << " scale dimension " << scales_dim; - return false; - } - } - - return true; -} - -bool IsQuantizationZeroPointSupported(const InitializedTensorSet& initializers, - const NodeUnitIODef& io_def, - const OpSupportCheckParams& params, - const std::string& op_type, - const Path& model_path, - bool is_quant_matmul, - bool is_conv_matmul_weight, - bool is_conv_matmul_u8s8_weight) { - // zero point is optional here - if (!io_def.quant_param->zero_point) - return true; - - const auto& zero_point_name = io_def.quant_param->zero_point->Name(); - if (!Contains(initializers, zero_point_name)) { - LOGS_DEFAULT(VERBOSE) << "The zero point of " << op_type << " must be an initializer tensor"; - return false; - } - - const auto& zero_tensor = *initializers.at(zero_point_name); - int64_t zero_dim = zero_tensor.dims().empty() ? 1 : zero_tensor.dims()[0]; - - if (!is_conv_matmul_u8s8_weight) { - if (zero_dim != 1) { - LOGS_DEFAULT(VERBOSE) << op_type << " does not support per-channel quantization, " - << " for now, only u8s8 QlinearConv supports per-channel quantization on API 29+"; - return false; - } - } else { - // For u8s8 Qlinear[Conv/MatMul], we support - // 1. Per-tensor, the weight will be transformed to uint8 later - // 2. Per-channel, only from Android API level 29 - if (zero_tensor.data_type() != ONNX_NAMESPACE::TensorProto_DataType_INT8) { - LOGS_DEFAULT(VERBOSE) << "u8s8 Qlinear[Conv/MatMul] only supports int8 zero point for weight, " - << "actual zero point type: [" << zero_tensor.data_type() << "]"; - return false; - } - - if (zero_dim != 1) { - if (is_quant_matmul) { - LOGS_DEFAULT(VERBOSE) << "QLinearMatMul does not support per-channel quantization"; - return false; - } - } - - // For onnx, u8s8 QlinearConv, the weight zero point can be a scalar, - // or a tensor with same channel as weight, for NNAPI we only support it be - // 0 (scalar) or all 0 (tensor), NNAPI will assume the zero point for per-channel - // quantization is 0 there is no input for it - Shape weight_shape; - if (!GetShape(io_def.node_arg, weight_shape)) - return false; - - if (weight_shape[0] != zero_dim && zero_dim != 1) { - LOGS_DEFAULT(VERBOSE) << op_type << " mismatch int8 per-channel quantization weight," - << " weight dimension[0] " << weight_shape[0] - << " zero point dimension " << zero_dim; - return false; - } - - std::vector unpacked_tensor; - auto status = onnxruntime::utils::UnpackInitializerData(zero_tensor, model_path, unpacked_tensor); - if (!status.IsOK()) { - LOGS_DEFAULT(ERROR) << "Qlinear[Conv/MatMul] error when unpack zero tensor: " << zero_point_name - << ", error msg: " << status.ErrorMessage(); - return false; - } - - // Verify all onnx weight zero point(s) are 0(s) - const int8_t* zero_points = reinterpret_cast(unpacked_tensor.data()); - for (size_t i = 0; i < unpacked_tensor.size(); i++) { - if (zero_points[i] != 0) { - LOGS_DEFAULT(VERBOSE) << "u8s8 Qlinear[Conv/MatMul] only support 0 as zero point, " - << "zero_points[" << i << "] has value: " << zero_points[i]; - return false; - } - } - } - - return true; -} - -bool IsQuantizedIOSupported(const InitializedTensorSet& initializers, const NodeUnit& node_unit, - const std::vector& indices, const OpSupportCheckParams& params, bool is_input) { - const auto& op_type = node_unit.OpType(); - auto quant_op_type = GetQuantizedOpType(node_unit); - - ORT_ENFORCE(quant_op_type != QuantizedOpType::QLinearMatMul, "[", op_type, "] is not a quantized op"); - - bool is_quant_conv = IsQuantizedConv(quant_op_type); - bool is_quant_matmul = (quant_op_type == QuantizedOpType::QLinearMatMul); - const auto& io_defs = is_input ? node_unit.Inputs() : node_unit.Outputs(); - - for (const auto idx : indices) { - if (idx >= io_defs.size()) { - LOGS_DEFAULT(VERBOSE) << (is_input ? "Input" : "Output") << " index, " << idx - << " >= size, " << io_defs.size() - << " of NodeUnit: " << node_unit.Name(); - return false; - } - - const auto& io_def = io_defs[idx]; - ORT_ENFORCE(io_def.quant_param.has_value(), "Input index, ", idx, " has no quant_param"); - - // If this op is Qlinear[Conv/MatMul], we want to check u8s8 support for weight tensor (or B tensor for QlinearMatMul) - bool is_conv_matmul_weight = is_input && (is_quant_conv || is_quant_matmul) && idx == 1; - bool is_conv_matmul_u8s8_weight = false; - - if (is_conv_matmul_weight) { - int32_t weight_type; - if (!GetType(io_def.node_arg, weight_type)) - return false; - is_conv_matmul_u8s8_weight = weight_type == ONNX_NAMESPACE::TensorProto_DataType_INT8; - } - - int32_t input_type; - if (!GetType(io_def.node_arg, input_type)) - return false; - - // We only support s8 for most of the inputs and all outputs, with the exception for Quantized MatMul and Conv, - // which allows s8 weight (u8s8) - // TODO, add support of s8s8 - if (input_type != ONNX_NAMESPACE::TensorProto_DataType_INT8 && !is_conv_matmul_u8s8_weight) { - LOGS_DEFAULT(VERBOSE) << op_type << "NodeUnit [" << node_unit.Name() - << "], type [" << op_type << "]'s " - << (is_input ? "Input" : "Output") << " index [" << idx - << "] has unspoorted type [" << input_type << "]"; - return false; - } - - // Check scale and zero point - if (!IsQuantizationScaleSupported(initializers, io_def, params, op_type, - is_quant_matmul, is_conv_matmul_weight, is_conv_matmul_u8s8_weight)) { - return false; - } - - if (!IsQuantizationZeroPointSupported(initializers, io_def, params, op_type, node_unit.ModelPath(), - is_quant_matmul, is_conv_matmul_weight, is_conv_matmul_u8s8_weight)) { - return false; - } - } - - return true; -} - -bool HasValidQuantizationScales(const InitializedTensorSet& initializers, const NodeUnit& node_unit, - const std::vector& indices, const OpSupportCheckParams& params, bool is_input) { - const auto& op_type = node_unit.OpType(); - auto quant_op_type = GetQuantizedOpType(node_unit); - bool is_quant_conv = IsQuantizedConv(quant_op_type); - bool is_quant_matmul = (quant_op_type == QuantizedOpType::QLinearMatMul); - const auto& io_defs = is_input ? node_unit.Inputs() : node_unit.Outputs(); - for (const auto idx : indices) { - if (idx >= io_defs.size()) { - LOGS_DEFAULT(VERBOSE) << (is_input ? "Input" : "Output") << " index, " << idx - << " >= size, " << io_defs.size() - << " of NodeUnit: " << node_unit.Name(); - return false; - } - - const auto& io_def = io_defs[idx]; - if (!io_def.quant_param.has_value()) { - LOGS_DEFAULT(VERBOSE) << "HasValidQuantizationZeroPoints, Input index, " << idx - << " has no quant_param"; - return false; - } - - const auto scale_name = io_def.quant_param->scale.Name(); - - if (!Contains(initializers, scale_name)) { - LOGS_DEFAULT(VERBOSE) << "The scale of " << op_type << " must be an initializer tensor"; - return false; - } - - // If this op is Qlinear[Conv/MatMul], we want to check u8s8 support for weight tensor (or B tensor for QlinearMatMul) - bool is_conv_matmul_weight = is_input && (is_quant_conv || is_quant_matmul) && idx == 1; - bool is_conv_matmul_u8s8_weight = false; - - if (is_conv_matmul_weight) { - int32_t weight_type; - if (!GetType(io_def.node_arg, weight_type)) - return false; - is_conv_matmul_u8s8_weight = weight_type == ONNX_NAMESPACE::TensorProto_DataType_INT8; - } - - const auto& scale_tensor = *initializers.at(scale_name); - int64_t scales_dim = scale_tensor.dims().empty() ? 1 : scale_tensor.dims()[0]; - if (!is_conv_matmul_u8s8_weight) { - if (scales_dim != 1) { - LOGS_DEFAULT(VERBOSE) << op_type << " does not support per-channel quantization, " - << " for now, only u8s8 QlinearConv supports per-channel quantization on API 29+"; - return false; - } - } else if (scales_dim != 1) { - // For u8s8 Qlinear[Conv/MatMul], we support - // 1. Per-tensor, the weight will be transformed to uint8 later - // 2. Per-channel, only from Android API level 29 - if (is_quant_matmul) { - LOGS_DEFAULT(VERBOSE) << "QLinearMatMul does not support per-channel quantization"; - return false; - } - - if (params.android_feature_level < ANEURALNETWORKS_FEATURE_LEVEL_3) { - LOGS_DEFAULT(VERBOSE) << op_type << " only supports per-channel quantization on Android API 29+, " - << "system NNAPI feature level: " << params.android_feature_level; - return false; - } - - Shape weight_shape; - if (!GetShape(io_def.node_arg, weight_shape)) - return false; - - if (weight_shape[0] != scales_dim) { - LOGS_DEFAULT(VERBOSE) << op_type << " mismatch int8 per-channel quantization weight," - << " weight dimension[0] " << weight_shape[0] - << " scale dimension " << scales_dim; - return false; - } - } - } - - return true; -} - -bool HasValidQuantizationZeroPoints(const InitializedTensorSet& initializers, const NodeUnit& node_unit, - const std::vector& indices, bool is_input) { - const auto& op_type = node_unit.OpType(); - auto quant_op_type = GetQuantizedOpType(node_unit); - bool is_quant_conv = IsQuantizedConv(quant_op_type); - bool is_quant_matmul = (quant_op_type == QuantizedOpType::QLinearMatMul); - - const auto& io_defs = is_input ? node_unit.Inputs() : node_unit.Outputs(); - for (const auto idx : indices) { - if (idx >= io_defs.size()) { - LOGS_DEFAULT(VERBOSE) << "HasValidQuantizationZeroPoints, " - << (is_input ? "Input" : "Output") << " index, " << idx - << " >= size, " << io_defs.size(); - return false; - } - - const auto& io_def = io_defs[idx]; - if (!io_def.quant_param.has_value()) { - LOGS_DEFAULT(VERBOSE) << "HasValidQuantizationZeroPoints, Input index, " << idx - << " has no quant_param"; - return false; - } - - // zero point is optional here - if (!io_def.quant_param->zero_point) - return true; - - const auto& zero_point_name = io_def.quant_param->zero_point->Name(); - if (!Contains(initializers, zero_point_name)) { - LOGS_DEFAULT(VERBOSE) << "The zero point of " << op_type << " must be an initializer tensor"; - return false; - } - - bool is_conv_matmul_weight = is_input && (is_quant_conv || is_quant_matmul) && idx == 1; - bool is_conv_matmul_u8s8_weight = false; - - if (is_conv_matmul_weight) { - int32_t weight_type; - if (!GetType(io_def.node_arg, weight_type)) - return false; - is_conv_matmul_u8s8_weight = weight_type == ONNX_NAMESPACE::TensorProto_DataType_INT8; - } - - const auto& zero_tensor = *initializers.at(zero_point_name); - int64_t zero_dim = zero_tensor.dims().empty() ? 1 : zero_tensor.dims()[0]; - - if (!is_conv_matmul_u8s8_weight) { - if (zero_dim != 1) { - LOGS_DEFAULT(VERBOSE) << op_type << " does not support per-channel quantization, " - << " for now, only u8s8 QlinearConv supports per-channel quantization on API 29+"; - return false; - } - } else { - // For u8s8 Qlinear[Conv/MatMul], we support - // 1. Per-tensor, the weight will be transformed to uint8 later - // 2. Per-channel, only from Android API level 29 - if (zero_tensor.data_type() != ONNX_NAMESPACE::TensorProto_DataType_INT8) { - LOGS_DEFAULT(VERBOSE) << "u8s8 Qlinear[Conv/MatMul] only supports int8 zero point for weight, " - << "actual zero point type: [" << zero_tensor.data_type() << "]"; - return false; - } - - if (zero_dim != 1) { - if (is_quant_matmul) { - LOGS_DEFAULT(VERBOSE) << "QLinearMatMul does not support per-channel quantization"; - return false; - } - } - - // For onnx, u8s8 QlinearConv, the weight zero point can be a scalar, - // or a tensor with same channel as weight, for NNAPI we only support it be - // 0 (scalar) or all 0 (tensor), NNAPI will assume the zero point for per-channel - // quantization is 0 there is no input for it - Shape weight_shape; - if (!GetShape(io_def.node_arg, weight_shape)) - return false; - - if (weight_shape[0] != zero_dim && zero_dim != 1) { - LOGS_DEFAULT(VERBOSE) << op_type << " mismatch int8 per-channel quantization weight," - << " weight dimension[0] " << weight_shape[0] - << " zero point dimension " << zero_dim; - return false; - } - - std::vector unpacked_tensor; - auto status = onnxruntime::utils::UnpackInitializerData(zero_tensor, node_unit.ModelPath(), unpacked_tensor); - if (!status.IsOK()) { - LOGS_DEFAULT(ERROR) << "Qlinear[Conv/MatMul] error when unpack zero tensor: " << zero_point_name - << ", error msg: " << status.ErrorMessage(); - return false; - } - - // Verify all onnx weight zero point(s) are 0(s) - const int8_t* zero_points = reinterpret_cast(unpacked_tensor.data()); - for (size_t i = 0; i < unpacked_tensor.size(); i++) { - if (zero_points[i] != 0) { - LOGS_DEFAULT(VERBOSE) << "u8s8 Qlinear[Conv/MatMul] only support 0 as zero point, " - << "zero_points[" << i << "] has value: " << zero_points[i]; - return false; - } - } - } - } - - return true; -} - common::Status GetQuantizationScaleAndZeroPoint( const InitializedTensorSet& initializers, const NodeUnitIODef& io_def, const Path& model_path, float& scale, int32_t& zero_point) { diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h index cb9d96aa32..ad8e76fe07 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/helper.h @@ -109,23 +109,9 @@ bool IsQuantizedConv(QuantizedOpType quant_op_type); // Such as QLinearConv, QLinearMatMul, QLinearAdd, QDQConv,... bool IsQuantizedBinaryOp(QuantizedOpType quant_op_type); -// Check if a qlinear unary op has valid inputs, Qlinear[Sigmoid/AveragePool] -bool HasValidUnaryOpQuantizedInputs(const NodeUnit& node_unit); // Check if a qlinear binary op has valid inputs, Qlinear[Conv/MatMul/Add] bool HasValidBinaryOpQuantizedInputTypes(const NodeUnit& node_unit); -// Check if the given quantized input(s) or output(s) is supported -bool IsQuantizedIOSupported(const InitializedTensorSet& initializers, const NodeUnit& node_unit, - const std::vector& indices, const OpSupportCheckParams& params, bool is_input); - -// Check if a qlinear op has valid scales for given indices -bool HasValidQuantizationScales(const InitializedTensorSet& initializers, const NodeUnit& node_unit, - const std::vector& indices, const OpSupportCheckParams& params, bool is_input); - -// Check if a qlinear op has valid zero points for given indices -bool HasValidQuantizationZeroPoints(const InitializedTensorSet& initializers, const NodeUnit& node_unit, - const std::vector& indices, bool is_input); - common::Status GetQuantizationScaleAndZeroPoint( const InitializedTensorSet& initializers, const NodeUnitIODef& io_def, const Path& model_path, float& scale, int32_t& zero_point); diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_support_checker.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_support_checker.cc index 783642b7f9..37ea601ba9 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_support_checker.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_support_checker.cc @@ -22,7 +22,21 @@ struct OpSupportCheckerRegistrations { std::unordered_map op_support_checker_map; }; -bool HasExternalInitializer(const InitializedTensorSet& initializers, const NodeUnit& node_unit) { +template +void CreateSharedOpSupportCheckerImpl(const std::string& op_type, + OpSupportCheckerRegistrations& op_registrations, + const std::vector& op_types) { + // The shared OpSupportChecker is already in the OpSupportCheckerRegistrations + if (op_registrations.op_support_checker_map.find(op_type) != op_registrations.op_support_checker_map.cend()) + return; + + op_registrations.support_checkers.push_back(std::make_unique()); + for (const auto& op : op_types) { + op_registrations.op_support_checker_map.emplace(op, op_registrations.support_checkers.back().get()); + } +} + +static bool HasExternalInitializer(const InitializedTensorSet& initializers, const NodeUnit& node_unit) { const auto is_ext_initializer = [&](const NodeArg& node_arg) { const auto& input_name(node_arg.Name()); @@ -58,18 +72,201 @@ bool HasExternalInitializer(const InitializedTensorSet& initializers, const Node return false; } -template -void CreateSharedOpSupportCheckerImpl(const std::string& op_type, - OpSupportCheckerRegistrations& op_registrations, - const std::vector& op_types) { - // The shared OpSupportChecker is already in the OpSupportCheckerRegistrations - if (op_registrations.op_support_checker_map.find(op_type) != op_registrations.op_support_checker_map.cend()) - return; - - op_registrations.support_checkers.push_back(std::make_unique()); - for (const auto& op : op_types) { - op_registrations.op_support_checker_map.emplace(op, op_registrations.support_checkers.back().get()); +static bool IsQuantizationScaleSupported(const InitializedTensorSet& initializers, + const NodeUnitIODef& io_def, + const OpSupportCheckParams& params, + const std::string& op_type, + bool is_quant_matmul, + bool is_conv_matmul_weight, + bool is_conv_matmul_u8s8_weight) { + const auto scale_name = io_def.quant_param->scale.Name(); + if (!Contains(initializers, scale_name)) { + LOGS_DEFAULT(VERBOSE) << "The scale of " << op_type << " must be an initializer tensor"; + return false; } + + const auto& scale_tensor = *initializers.at(scale_name); + int64_t scales_dim = scale_tensor.dims().empty() ? 1 : scale_tensor.dims()[0]; + if (!is_conv_matmul_u8s8_weight) { + if (scales_dim != 1) { + LOGS_DEFAULT(VERBOSE) << op_type << " does not support per-channel quantization, " + << " for now, only u8s8 QlinearConv supports per-channel quantization on API 29+"; + return false; + } + } else if (scales_dim != 1) { + // For u8s8 Qlinear[Conv/MatMul], we support + // 1. Per-tensor, the weight will be transformed to uint8 later + // 2. Per-channel, only from Android API level 29 + if (is_quant_matmul) { + LOGS_DEFAULT(VERBOSE) << "QLinearMatMul does not support per-channel quantization"; + return false; + } + + if (params.android_feature_level < ANEURALNETWORKS_FEATURE_LEVEL_3) { + LOGS_DEFAULT(VERBOSE) << op_type << " only supports per-channel quantization on Android API 29+, " + << "system NNAPI feature level: " << params.android_feature_level; + return false; + } + + Shape weight_shape; + if (!GetShape(io_def.node_arg, weight_shape)) + return false; + + if (weight_shape[0] != scales_dim) { + LOGS_DEFAULT(VERBOSE) << op_type << " mismatch int8 per-channel quantization weight," + << " weight dimension[0] " << weight_shape[0] + << " scale dimension " << scales_dim; + return false; + } + } + + return true; +} + +static bool IsQuantizationZeroPointSupported(const InitializedTensorSet& initializers, + const NodeUnitIODef& io_def, + const OpSupportCheckParams& params, + const std::string& op_type, + const Path& model_path, + bool is_quant_matmul, + bool is_conv_matmul_weight, + bool is_conv_matmul_u8s8_weight) { + // zero point is optional here + if (!io_def.quant_param->zero_point) + return true; + + const auto& zero_point_name = io_def.quant_param->zero_point->Name(); + if (!Contains(initializers, zero_point_name)) { + LOGS_DEFAULT(VERBOSE) << "The zero point of " << op_type << " must be an initializer tensor"; + return false; + } + + const auto& zero_tensor = *initializers.at(zero_point_name); + int64_t zero_dim = zero_tensor.dims().empty() ? 1 : zero_tensor.dims()[0]; + + if (!is_conv_matmul_u8s8_weight) { + if (zero_dim != 1) { + LOGS_DEFAULT(VERBOSE) << op_type << " does not support per-channel quantization, " + << " for now, only u8s8 QlinearConv supports per-channel quantization on API 29+"; + return false; + } + } else { + // For u8s8 Qlinear[Conv/MatMul], we support + // 1. Per-tensor, the weight will be transformed to uint8 later + // 2. Per-channel, only from Android API level 29 + if (zero_tensor.data_type() != ONNX_NAMESPACE::TensorProto_DataType_INT8) { + LOGS_DEFAULT(VERBOSE) << "u8s8 Qlinear[Conv/MatMul] only supports int8 zero point for weight, " + << "actual zero point type: [" << zero_tensor.data_type() << "]"; + return false; + } + + if (zero_dim != 1) { + if (is_quant_matmul) { + LOGS_DEFAULT(VERBOSE) << "QLinearMatMul does not support per-channel quantization"; + return false; + } + } + + // For onnx, u8s8 QlinearConv, the weight zero point can be a scalar, + // or a tensor with same channel as weight, for NNAPI we only support it be + // 0 (scalar) or all 0 (tensor), NNAPI will assume the zero point for per-channel + // quantization is 0 there is no input for it + Shape weight_shape; + if (!GetShape(io_def.node_arg, weight_shape)) + return false; + + if (weight_shape[0] != zero_dim && zero_dim != 1) { + LOGS_DEFAULT(VERBOSE) << op_type << " mismatch int8 per-channel quantization weight," + << " weight dimension[0] " << weight_shape[0] + << " zero point dimension " << zero_dim; + return false; + } + + std::vector unpacked_tensor; + auto status = onnxruntime::utils::UnpackInitializerData(zero_tensor, model_path, unpacked_tensor); + if (!status.IsOK()) { + LOGS_DEFAULT(ERROR) << "Qlinear[Conv/MatMul] error when unpack zero tensor: " << zero_point_name + << ", error msg: " << status.ErrorMessage(); + return false; + } + + // Verify all onnx weight zero point(s) are 0(s) + const int8_t* zero_points = reinterpret_cast(unpacked_tensor.data()); + for (size_t i = 0; i < unpacked_tensor.size(); i++) { + if (zero_points[i] != 0) { + LOGS_DEFAULT(VERBOSE) << "u8s8 Qlinear[Conv/MatMul] only support 0 as zero point, " + << "zero_points[" << i << "] has value: " << zero_points[i]; + return false; + } + } + } + + return true; +} + +// Check if the given quantized input(s) or output(s) is supported +static bool IsQuantizedIOSupported(const InitializedTensorSet& initializers, const NodeUnit& node_unit, + const std::vector& indices, const OpSupportCheckParams& params, bool is_input) { + const auto& op_type = node_unit.OpType(); + auto quant_op_type = GetQuantizedOpType(node_unit); + + ORT_ENFORCE(quant_op_type != QuantizedOpType::QLinearMatMul, "[", op_type, "] is not a quantized op"); + + bool is_quant_conv = IsQuantizedConv(quant_op_type); + bool is_quant_matmul = (quant_op_type == QuantizedOpType::QLinearMatMul); + const auto& io_defs = is_input ? node_unit.Inputs() : node_unit.Outputs(); + + for (const auto idx : indices) { + if (idx >= io_defs.size()) { + LOGS_DEFAULT(VERBOSE) << (is_input ? "Input" : "Output") << " index, " << idx + << " >= size, " << io_defs.size() + << " of NodeUnit: " << node_unit.Name(); + return false; + } + + const auto& io_def = io_defs[idx]; + ORT_ENFORCE(io_def.quant_param.has_value(), "Input index, ", idx, " has no quant_param"); + + // If this op is Qlinear[Conv/MatMul], we want to check u8s8 support for weight tensor (or B tensor for QlinearMatMul) + bool is_conv_matmul_weight = is_input && (is_quant_conv || is_quant_matmul) && idx == 1; + bool is_conv_matmul_u8s8_weight = false; + + if (is_conv_matmul_weight) { + int32_t weight_type; + if (!GetType(io_def.node_arg, weight_type)) + return false; + is_conv_matmul_u8s8_weight = weight_type == ONNX_NAMESPACE::TensorProto_DataType_INT8; + } + + int32_t input_type; + if (!GetType(io_def.node_arg, input_type)) + return false; + + // We only support u8 for most of the inputs and all outputs, with the exception for Quantized MatMul and Conv, + // which allows s8 weight (u8s8) + // TODO, add support of s8s8 + if (input_type != ONNX_NAMESPACE::TensorProto_DataType_UINT8 && + !(input_type == ONNX_NAMESPACE::TensorProto_DataType_INT8 && is_conv_matmul_u8s8_weight)) { + LOGS_DEFAULT(VERBOSE) << op_type << "NodeUnit [" << node_unit.Name() + << "], type [" << op_type << "]'s " + << (is_input ? "Input" : "Output") << " index [" << idx + << "] has unspoorted type [" << input_type << "]"; + return false; + } + + // Check scale and zero point + if (!IsQuantizationScaleSupported(initializers, io_def, params, op_type, + is_quant_matmul, is_conv_matmul_weight, is_conv_matmul_u8s8_weight)) { + return false; + } + + if (!IsQuantizationZeroPointSupported(initializers, io_def, params, op_type, node_unit.ModelPath(), + is_quant_matmul, is_conv_matmul_weight, is_conv_matmul_u8s8_weight)) { + return false; + } + } + + return true; } #pragma endregion helpers @@ -561,8 +758,8 @@ class PoolOpSupportChecker : public BaseOpSupportChecker { } bool HasSupportedInputOutputsImpl( - const InitializedTensorSet& /* initializers */, const NodeUnit& node_unit, - const OpSupportCheckParams& /* params */) const override; + const InitializedTensorSet& initializers, const NodeUnit& node_unit, + const OpSupportCheckParams& params) const override; }; /* static */ void PoolOpSupportChecker::CreateSharedOpSupportChecker( @@ -638,19 +835,6 @@ bool PoolOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initial // We need to check if we have valid scales and zero points for QLinearAveragePool if (is_qlinear_average_pool) { - // Check input scales and ZPs - if (!HasValidQuantizationScales(initializers, node_unit, {0}, params, true /* is_input */)) - return false; - if (!HasValidQuantizationZeroPoints(initializers, node_unit, {0}, true /* is_input */)) - return false; - - // Check output scale and ZP - - if (!HasValidQuantizationScales(initializers, node_unit, {0}, params, false /* is_input */)) - return false; - if (!HasValidQuantizationZeroPoints(initializers, node_unit, {0}, false /* is_input */)) - return false; - // NNAPI requires Quantized Average Pool has same scale and zero point for both input and output float input_scale = 0.0f; int32_t input_zp = 0; @@ -701,7 +885,11 @@ bool PoolOpSupportChecker::HasSupportedInputOutputsImpl( return BaseOpSupportChecker::HasSupportedInputOutputsImpl(initializers, node_unit, params); if (is_qlinear_average_pool) { - return HasValidUnaryOpQuantizedInputs(node_unit); + if (!IsQuantizedIOSupported(initializers, node_unit, {0}, params, true /* is_input */)) + return false; + + if (!IsQuantizedIOSupported(initializers, node_unit, {0}, params, false /* is_input */)) + return false; } // is_max_pool @@ -770,6 +958,12 @@ bool ConvOpSupportChecker::HasSupportedInputOutputsImpl( if (!HasValidBinaryOpQuantizedInputTypes(node_unit)) return false; + if (!IsQuantizedIOSupported(initializers, node_unit, {0, 1}, params, true /* is_input */)) + return false; + + if (!IsQuantizedIOSupported(initializers, node_unit, {0}, params, false /* is_input */)) + return false; + return true; } @@ -814,34 +1008,10 @@ bool ConvOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initial } if (is_quant_conv) { - // For QLinearConv, we only support uint8 output now - int32_t output_type; - if (!GetType(node_unit.Outputs()[0].node_arg, output_type)) - return false; - - if (output_type != ONNX_NAMESPACE::TensorProto_DataType_UINT8) { - LOGS_DEFAULT(VERBOSE) << "[" << op_type - << "] output type: [" << output_type - << "] is not supported for now"; - return false; - } - if (inputs.size() > 2 && !Contains(initializers, inputs[2].node_arg.Name())) { LOGS_DEFAULT(VERBOSE) << "Bias of QLinearConv must be known"; return false; } - - // Check input scales and ZPs - if (!HasValidQuantizationScales(initializers, node_unit, {0, 1}, params, true /* is_input */)) - return false; - if (!HasValidQuantizationZeroPoints(initializers, node_unit, {0, 1}, true /* is_input */)) - return false; - - // Check output scale and ZP - if (!HasValidQuantizationScales(initializers, node_unit, {0}, params, false /* is_input */)) - return false; - if (!HasValidQuantizationZeroPoints(initializers, node_unit, {0}, false /* is_input */)) - return false; } return true; @@ -948,6 +1118,12 @@ bool GemmOpSupportChecker::HasSupportedInputOutputsImpl( if (!HasValidBinaryOpQuantizedInputTypes(node_unit)) return false; + if (!IsQuantizedIOSupported(initializers, node_unit, {0, 1}, params, true /* is_input */)) + return false; + + if (!IsQuantizedIOSupported(initializers, node_unit, {0}, params, false /* is_input */)) + return false; + return true; } @@ -1082,33 +1258,6 @@ bool GemmOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initial LOGS_DEFAULT(VERBOSE) << "B of MatMul must be known"; return false; } - - if (is_qlinear_matmul) { - // For QLinearMatMul, we only support uint8 output now - int32_t output_type; - if (!GetType(node_unit.Outputs()[0].node_arg, output_type)) - return false; - - if (output_type != ONNX_NAMESPACE::TensorProto_DataType_UINT8) { - LOGS_DEFAULT(VERBOSE) << "[" << op_type - << "] output type: [" << output_type - << "] is not supported for now"; - return false; - } - - // All scale/zero points are initializer scalars - // Check input scales and ZPs - if (!HasValidQuantizationScales(initializers, node_unit, {0, 1}, params, true /* is_input */)) - return false; - if (!HasValidQuantizationZeroPoints(initializers, node_unit, {0, 1}, true /* is_input */)) - return false; - - // Check output scale and ZP - if (!HasValidQuantizationScales(initializers, node_unit, {0}, params, false /* is_input */)) - return false; - if (!HasValidQuantizationZeroPoints(initializers, node_unit, {0}, false /* is_input */)) - return false; - } } else { LOGS_DEFAULT(VERBOSE) << "GemmOpSupportChecker, unknown op: " << op_type; } @@ -1190,7 +1339,13 @@ bool UnaryOpSupportChecker::HasSupportedInputOutputsImpl( if (node_unit.OpType() != "QLinearSigmoid") return BaseOpSupportChecker::HasSupportedInputOutputsImpl(initializers, node_unit, params); - return HasValidUnaryOpQuantizedInputs(node_unit); + if (!IsQuantizedIOSupported(initializers, node_unit, {0}, params, true /* is_input */)) + return false; + + if (!IsQuantizedIOSupported(initializers, node_unit, {0}, params, false /* is_input */)) + return false; + + return true; } // All ops except "Sin" opset 5- uses consumed_inputs attribute which is not supported for now @@ -1207,21 +1362,8 @@ int UnaryOpSupportChecker::GetMinSupportedOpSet(const NodeUnit& node_unit) const const InitializedTensorSet& initializers, const NodeUnit& node_unit, const OpSupportCheckParams& params) { const auto& op_type = node_unit.OpType(); ORT_ENFORCE(op_type == "QLinearSigmoid"); - const auto& op_name = node_unit.Name(); - // Check input scales and ZPs - if (!HasValidQuantizationScales(initializers, node_unit, {0}, params, true /* is_input */)) - return false; - if (!HasValidQuantizationZeroPoints(initializers, node_unit, {0}, true /* is_input */)) - return false; - - // Check output scale and ZP - if (!HasValidQuantizationScales(initializers, node_unit, {0}, params, false /* is_input */)) - return false; - if (!HasValidQuantizationZeroPoints(initializers, node_unit, {0}, false /* is_input */)) - return false; - // NNAPI requires the scale be 1.f/256 and zero point to be 0 // See https://android.googlesource.com/platform/frameworks/ml/+/refs/heads/android10-c2f2-release/nn/common/operations/Activation.cpp#180 float output_scale = 0.0f; @@ -1344,37 +1486,17 @@ bool SqueezeOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& init class QuantizeLinearOpSupportChecker : public BaseOpSupportChecker { private: - bool IsOpSupportedImpl(const InitializedTensorSet& initializers, const NodeUnit& node_unit, - const OpSupportCheckParams& params) const override; - int32_t GetMinSupportedNNAPIFeatureLevel(const NodeUnit& /* node_unit */, const OpSupportCheckParams& /* params */) const override { return ANEURALNETWORKS_FEATURE_LEVEL_3; } -}; -bool QuantizeLinearOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initializers, const NodeUnit& node_unit, - const OpSupportCheckParams& params) const { - int32_t output_type; - if (!GetType(node_unit.Outputs()[0].node_arg, output_type)) - return false; - - if (output_type != ONNX_NAMESPACE::TensorProto_DataType_UINT8) { - LOGS_DEFAULT(VERBOSE) << "[" << node_unit.OpType() - << "] output type: [" << output_type - << "] is not supported for now"; - return false; + bool HasSupportedInputOutputsImpl( + const InitializedTensorSet& initializers, const NodeUnit& node_unit, + const OpSupportCheckParams& params) const override { + return IsQuantizedIOSupported(initializers, node_unit, {0}, params, false /* is_input */); } - - // For QuantizeLinear only output is quantized - // Check output scale and ZP - if (!HasValidQuantizationScales(initializers, node_unit, {0}, params, false /* is_input */)) - return false; - if (!HasValidQuantizationZeroPoints(initializers, node_unit, {0}, false /* is_input */)) - return false; - - return true; -} +}; #pragma endregion @@ -1382,46 +1504,17 @@ bool QuantizeLinearOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSe class DequantizeLinearOpSupportChecker : public BaseOpSupportChecker { private: - bool IsOpSupportedImpl(const InitializedTensorSet& initializers, const NodeUnit& node_unit, - const OpSupportCheckParams& params) const override; - int32_t GetMinSupportedNNAPIFeatureLevel(const NodeUnit& /* node_unit */, const OpSupportCheckParams& /* params */) const override { return ANEURALNETWORKS_FEATURE_LEVEL_1; } + bool HasSupportedInputOutputsImpl( - const InitializedTensorSet& /* initializers */, const NodeUnit& node_unit, - const OpSupportCheckParams& /* params */) const override; -}; - -bool DequantizeLinearOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initializers, const NodeUnit& node_unit, - const OpSupportCheckParams& params) const { - // For DequantizeLinear only input is quantized - // Check input scale and ZP - if (!HasValidQuantizationScales(initializers, node_unit, {0}, params, true /* is_input */)) - return false; - if (!HasValidQuantizationZeroPoints(initializers, node_unit, {0}, true /* is_input */)) - return false; - - return true; -} - -bool DequantizeLinearOpSupportChecker::HasSupportedInputOutputsImpl( - const InitializedTensorSet& /* initializers */, const NodeUnit& node_unit, - const OpSupportCheckParams& /* params */) const { - int32_t input_type; - if (!GetType(node_unit.Inputs()[0].node_arg, input_type)) - return false; - - if (input_type != ONNX_NAMESPACE::TensorProto_DataType_UINT8) { - LOGS_DEFAULT(VERBOSE) << "[" << node_unit.OpType() - << "] Input type: [" << input_type - << "] is not supported for now"; - return false; + const InitializedTensorSet& initializers, const NodeUnit& node_unit, + const OpSupportCheckParams& params) const override { + return IsQuantizedIOSupported(initializers, node_unit, {0}, params, true /* is_input */); } - - return true; -} +}; #pragma endregion @@ -1438,8 +1531,8 @@ class LRNOpSupportChecker : public BaseOpSupportChecker { } }; -bool LRNOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& /* initializers */, const NodeUnit& node_unit, - const OpSupportCheckParams& /* params */) const { +bool LRNOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initializers, const NodeUnit& node_unit, + const OpSupportCheckParams& params) const { Shape input_shape; if (!GetShape(node_unit.Inputs()[0].node_arg, input_shape)) return false; @@ -1628,33 +1721,6 @@ bool ResizeOpSupportChecker::IsOpSupportedImpl(const InitializedTensorSet& initi } } - if (IsQuantizedOp(node_unit)) { - // For QDQResize, we only support uint8 output now - // TODO, add int8 support to NNAPI, and maybe move all the output type check into a virtual function - // similar to HasSupportedInputsImpl - int32_t output_type; - if (!GetType(node_unit.Outputs()[0].node_arg, output_type)) - return false; - - if (output_type != ONNX_NAMESPACE::TensorProto_DataType_UINT8) { - LOGS_DEFAULT(VERBOSE) << "[Resize] output type: [" << output_type - << "] is not supported for now"; - return false; - } - - // Check input scales and ZPs - if (!HasValidQuantizationScales(initializers, node_unit, {0}, params, true /* is_input */)) - return false; - if (!HasValidQuantizationZeroPoints(initializers, node_unit, {0}, true /* is_input */)) - return false; - - // Check output scale and ZP - if (!HasValidQuantizationScales(initializers, node_unit, {0}, params, false /* is_input */)) - return false; - if (!HasValidQuantizationZeroPoints(initializers, node_unit, {0}, false /* is_input */)) - return false; - } - return true; } @@ -1673,8 +1739,8 @@ int32_t ResizeOpSupportChecker::GetMinSupportedNNAPIFeatureLevel(const NodeUnit& } bool ResizeOpSupportChecker::HasSupportedInputOutputsImpl( - const InitializedTensorSet& /* initializers */, const NodeUnit& node_unit, - const OpSupportCheckParams& /* params */) const { + const InitializedTensorSet& initializers, const NodeUnit& node_unit, + const OpSupportCheckParams& params) const { int32_t input_type; if (!GetType(node_unit.Inputs()[0].node_arg, input_type)) return false; @@ -1687,6 +1753,14 @@ bool ResizeOpSupportChecker::HasSupportedInputOutputsImpl( return false; } + if (IsQuantizedOp(node_unit)) { + if (!IsQuantizedIOSupported(initializers, node_unit, {0}, params, true /* is_input */)) + return false; + + if (!IsQuantizedIOSupported(initializers, node_unit, {0}, params, false /* is_input */)) + return false; + } + return true; }