From ca0dd8246c132e315c0cbee8f0c428bc12329c8c Mon Sep 17 00:00:00 2001 From: gwang-msft <62914304+gwang-msft@users.noreply.github.com> Date: Mon, 20 Jul 2020 16:43:31 -0700 Subject: [PATCH] NNAPI EP, add quantization support (#4530) * nnapi quantization work, 1. add SymmPerChannelQuantParams to operand types * add ways for operand_type to change dimension * remove per-channel quantization * Squashed commit of the following: commit 4857c3a732298c2f96efb61800b7621251d61c1b Author: gwang Date: Tue Jul 14 23:02:28 2020 -0700 remove per-channel quantization commit 775e4b2960f531496b8d11eef27d64e5b85c3c66 Author: gwang Date: Mon Jul 13 11:54:02 2020 -0700 add ways for operand_type to change dimension commit e56a494de67c66f8122d908270fbc2bb17e38423 Author: gwang Date: Wed Jul 8 15:18:55 2020 -0700 nnapi quantization work, 1. add SymmPerChannelQuantParams to operand types * add support for QuantizeLinear * add dequantizelinear support * minor style update * minor bug fix * add quantization support for qlinearmatmul, minor issue fix * add quantized input support, minor bug fix * fix issues in the qlinearmatmul * add verify scale and zeropoint for qlinearmatmul * add test for [de]qunatizelinear ops * add qlinearconv support * fixed small issue causing test failure * fix test exception * fix for centos test failure * fix centos test failure * fix issue causing win-tensorRT ci failure * addressed comments --- .../nnapi_builtin/builders/model_builder.cc | 49 +- .../nnapi_builtin/builders/op_builder.cc | 863 +++++++++++++++--- .../nnapi/nnapi_builtin/builders/op_builder.h | 6 +- .../nnapi_builtin/nnapi_execution_provider.cc | 3 + .../nnapi_lib/NeuralNetworksWrapper.cc | 43 +- .../nnapi_lib/NeuralNetworksWrapper.h | 2 + .../test/contrib_ops/quantize_ops_test.cc | 108 ++- .../test/providers/cpu/math/gemm_test.cc | 75 +- .../cpu/math/quantize_linear_matmul_test.cc | 43 +- .../test/providers/cpu/nn/conv_op_test.cc | 32 +- .../providers/cpu/nn/qlinearconv_op_test.cc | 43 +- 11 files changed, 1034 insertions(+), 233 deletions(-) diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc index c92aa2a11d..d64261a2d4 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/model_builder.cc @@ -186,6 +186,33 @@ void ModelBuilder::PreprocessInitializers() { } } +// Help to get all quantized operators' input and the node(s) using the input +std::unordered_map> GetAllQuantizedOpInputs(const GraphViewer& graph_view) { + std::unordered_map> all_quantized_op_inputs; + const auto& node_indices = graph_view.GetNodesInTopologicalOrder(); + for (const auto& node_idx : node_indices) { + const auto* node(graph_view.GetNode(node_idx)); + const auto& op_type = node->OpType(); + if (op_type == "DequantizeLinear" || op_type == "QLinearMatMul" || op_type == "QLinearConv") { + const auto& input_name = node->InputDefs()[0]->Name(); + if (Contains(all_quantized_op_inputs, input_name)) + all_quantized_op_inputs.at(input_name).push_back(node); + else + all_quantized_op_inputs.emplace(input_name, vector{node}); + } + + if (op_type == "QLinearMatMul" || op_type == "QLinearConv") { + const auto& input_name = node->InputDefs()[3]->Name(); + if (Contains(all_quantized_op_inputs, input_name)) + all_quantized_op_inputs.at(input_name).push_back(node); + else + all_quantized_op_inputs.emplace(input_name, vector{node}); + } + } + + return all_quantized_op_inputs; +} + void ModelBuilder::RegisterInitializers() { // First pass to get all the stats of the initializers auto initializer_size = initializers_.size(); @@ -256,6 +283,7 @@ void ModelBuilder::RegisterInitializers() { } void ModelBuilder::RegisterModelInputs() { + const auto all_quantized_op_inputs = GetAllQuantizedOpInputs(graph_view_); for (const auto* node_arg : graph_view_.GetInputs()) { const auto& input_name = node_arg->Name(); @@ -277,6 +305,8 @@ void ModelBuilder::RegisterModelInputs() { } Type type = Type::TENSOR_FLOAT32; + float scale = 0.0f; + int32_t zero_point = 0; const auto* type_proto = node_arg->TypeAsProto(); if (!type_proto || !type_proto->tensor_type().has_elem_type()) { ORT_THROW("The input of graph doesn't have elem_type: " + input_name); @@ -285,6 +315,23 @@ void ModelBuilder::RegisterModelInputs() { case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: type = Type::TENSOR_FLOAT32; break; + case ONNX_NAMESPACE::TensorProto_DataType_UINT8: { + // For ONNX the quantized input does not carry scale and zero point info + // So we will need to search the operator using this input + // And dig out the scale and zero point as the input initializers to the operator + type = Type::TENSOR_QUANT8_ASYMM; + if (!Contains(all_quantized_op_inputs, input_name)) { + // We current do not support uint8 input if it is not a quantized input + ORT_THROW("The input of graph doesn't have valid type, name: " + + input_name + " type: " + + std::to_string(type_proto->tensor_type().elem_type())); + } + + // TODO, verify the scale and zero point match if there are multiple op using same input + std::tie(scale, zero_point) = + GetQuantizedInputScaleAndZeroPoint(*this, *all_quantized_op_inputs.at(input_name)[0], input_name); + break; + } default: // TODO: support other type ORT_THROW("The input of graph doesn't have valid type, name: " + @@ -293,7 +340,7 @@ void ModelBuilder::RegisterModelInputs() { } } - OperandType operand_type(type, shape); + OperandType operand_type(type, shape, scale, zero_point); shaper_.AddShape(input_name, operand_type.dimensions); auto index = AddNewOperand(input_name, operand_type, false /* is_nhwc */); diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc index dcb87c1f75..4575f5c226 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.cc @@ -3,6 +3,7 @@ #include #include +#include #include #include @@ -19,17 +20,59 @@ using Shape = Shaper::Shape; #pragma region helpers -const int64_t* GetTensorInt64Data(const ONNX_NAMESPACE::TensorProto& tensor) { - return tensor.int64_data().empty() - ? reinterpret_cast(tensor.raw_data().data()) - : tensor.int64_data().data(); -} +#define GET_TENSOR_DATA(FUNC_NAME, ELEMENT_TYPE, DATA) \ + static const ELEMENT_TYPE* GetTensor##FUNC_NAME(const ONNX_NAMESPACE::TensorProto& tensor) { \ + return tensor.DATA().empty() \ + ? reinterpret_cast(tensor.raw_data().data()) \ + : tensor.DATA().data(); \ + } -const float* GetTensorFloatData(const ONNX_NAMESPACE::TensorProto& tensor) { - return tensor.float_data().empty() - ? reinterpret_cast(tensor.raw_data().data()) - : tensor.float_data().data(); +GET_TENSOR_DATA(FloatData, float, float_data) +GET_TENSOR_DATA(Int32Data, int32_t, int32_data) +GET_TENSOR_DATA(Int64Data, int64_t, int64_data) + +#undef GET_TENSOR_DATA + +// TODO, move this to a shared location +#define CASE_UNPACK(TYPE, ELEMENT_TYPE, DATA_SIZE) \ + case ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_##TYPE: { \ + size_t element_count = initializer.has_raw_data() \ + ? initializer.raw_data().size() \ + : initializer.DATA_SIZE(); \ + tensor_byte_size = element_count * sizeof(ELEMENT_TYPE); \ + unpacked_tensor.reset(new uint8_t[tensor_byte_size]); \ + return onnxruntime::utils::UnpackTensor( \ + initializer, \ + initializer.has_raw_data() ? initializer.raw_data().data() : nullptr, \ + initializer.has_raw_data() ? initializer.raw_data().size() : 0, \ + reinterpret_cast(unpacked_tensor.get()), element_count); \ + break; \ + } + +static Status UnpackInitializerTensor(const onnx::TensorProto& initializer, + std::unique_ptr& unpacked_tensor, + size_t& tensor_byte_size) { + switch (initializer.data_type()) { + CASE_UNPACK(FLOAT, float, float_data_size); + CASE_UNPACK(DOUBLE, double, double_data_size); + CASE_UNPACK(BOOL, bool, int32_data_size); + CASE_UNPACK(INT8, int8_t, int32_data_size); + CASE_UNPACK(INT16, int16_t, int32_data_size); + CASE_UNPACK(INT32, int32_t, int32_data_size); + CASE_UNPACK(INT64, int64_t, int64_data_size); + CASE_UNPACK(UINT8, uint8_t, int32_data_size); + CASE_UNPACK(UINT16, uint16_t, int32_data_size); + CASE_UNPACK(UINT32, uint32_t, uint64_data_size); + CASE_UNPACK(UINT64, uint64_t, int64_data_size); + CASE_UNPACK(FLOAT16, onnxruntime::MLFloat16, int32_data_size); + CASE_UNPACK(BFLOAT16, onnxruntime::BFloat16, int32_data_size); + default: + break; + } + return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, + "Unsupported type: " + std::to_string(initializer.data_type())); } +#undef CASE_UNPACK void AddTransposeOperator(ModelBuilder& model_builder, const std::string& input, @@ -51,7 +94,8 @@ void AddTransposeOperator(ModelBuilder& model_builder, input_indices.push_back(perm_idx); // permutation shaper.Transpose(input, perm, output); - const OperandType output_operand_type(operand_types.at(input).type, shaper[output]); + OperandType output_operand_type = operand_types.at(input); + output_operand_type.SetDimensions(shaper[output]); model_builder.AddOperation(ANEURALNETWORKS_TRANSPOSE, input_indices, {output}, {output_operand_type}, {output_is_nhwc}); } @@ -105,13 +149,13 @@ void TransposeNCHWToNHWC(ModelBuilder& model_builder, TransposeBetweenNCHWAndNHWC(model_builder, input, output, true /* nchw_to_nhwc */); } -void AddBinaryOperator(int32_t op_type, - ModelBuilder& model_builder, - const std::string& input1, - const std::string& input2, - int32_t fuse_code, - const std::string& output, - bool output_is_nhwc) { +static void AddBinaryOperator(int32_t op_type, + ModelBuilder& model_builder, + const std::string& input1, + const std::string& input2, + int32_t fuse_code, + const std::string& output, + bool output_is_nhwc) { auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); @@ -125,7 +169,7 @@ void AddBinaryOperator(int32_t op_type, model_builder.AddOperation(op_type, input_indices, {output}, {output_operand_type}, {output_is_nhwc}); } -bool GetType(const NodeArg& node_arg, int32_t& type) { +static bool GetType(const NodeArg& node_arg, int32_t& type) { type = ONNX_NAMESPACE::TensorProto_DataType_UNDEFINED; const auto* type_proto = node_arg.TypeAsProto(); if (!type_proto || !type_proto->has_tensor_type() || !type_proto->tensor_type().has_elem_type()) { @@ -137,7 +181,7 @@ bool GetType(const NodeArg& node_arg, int32_t& type) { return true; } -bool GetShape(const NodeArg& node_arg, Shape& shape) { +static bool GetShape(const NodeArg& node_arg, Shape& shape) { shape.clear(); const auto* shape_proto = node_arg.Shape(); @@ -159,37 +203,50 @@ enum DataLayout { }; // TODO, replace this with more efficient code in optimizers -uint32_t AddInitializerInNewLayout(ModelBuilder& model_builder, - const std::string& name, - DataLayout new_layout) { +static uint32_t AddInitializerInNewLayout(ModelBuilder& model_builder, + const std::string& name, + const OperandType& source_operand_type, + DataLayout new_layout) { const auto& tensor = model_builder.GetInitializerTensors().at(name); - Shape shape; - for (auto dim : tensor.dims()) - shape.push_back(SafeInt(dim)); - + const Shape& shape = source_operand_type.dimensions; ORT_ENFORCE(shape.size() == 4, "The initializer is not 4D: " + name + " actual dim " + std::to_string(shape.size())); // TODO support other data types - Type type; - if (tensor.data_type() == ONNX_NAMESPACE::TensorProto_DataType_FLOAT) { - type = Type::TENSOR_FLOAT32; - } else { - ORT_THROW("The initializer of graph doesn't have valid type: " + name); + const uint8_t* src = nullptr; + std::unique_ptr unpacked_tensor; + size_t tensor_byte_size; + + switch (tensor.data_type()) { + case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: + src = reinterpret_cast(GetTensorFloatData(tensor)); + break; + case ONNX_NAMESPACE::TensorProto_DataType_UINT8: + case ONNX_NAMESPACE::TensorProto_DataType_INT8: { + ORT_THROW_IF_ERROR( + UnpackInitializerTensor(tensor, unpacked_tensor, tensor_byte_size)); + src = unpacked_tensor.get(); + break; + } + default: + ORT_THROW("The initializer of graph " + name + + " doesn't have valid type: " + std::to_string(tensor.data_type())); } - auto out_t = shape[0], in_t = shape[1], - h_t = shape[2], w_t = shape[3]; + const auto out_t = shape[0], in_t = shape[1], + h_t = shape[2], w_t = shape[3]; Shape dest_shape; if (new_layout == L_0231) dest_shape = {out_t, h_t, w_t, in_t}; // L_0231 else dest_shape = {in_t, h_t, w_t, out_t}; // L_1230 for depthwise conv weight - const float* src = GetTensorFloatData(tensor); - float* buffer = new float[Product(shape)]; - const OperandType operand_type(type, dest_shape); + OperandType operand_type = source_operand_type; + operand_type.SetDimensions(dest_shape); + std::unique_ptr buffer_holder(new uint8_t[operand_type.GetOperandBlobByteSize()]); + uint8_t* buffer = buffer_holder.get(); + size_t element_size = operand_type.GetElementByteSize(); for (uint32_t out = 0; out < out_t; out++) { for (uint32_t in = 0; in < in_t; in++) { for (uint32_t h = 0; h < h_t; h++) { @@ -212,51 +269,64 @@ uint32_t AddInitializerInNewLayout(ModelBuilder& model_builder, out; } - buffer[nnapi_idx] = src[onnx_idx]; + for (size_t i = 0; i < element_size; i++) { + buffer[element_size * nnapi_idx + i] = src[element_size * onnx_idx + i]; + } } } } } - auto operand_idx = model_builder.AddOperandFromPersistMemoryBuffer(name, &buffer[0], operand_type); - delete[] buffer; - return operand_idx; + return model_builder.AddOperandFromPersistMemoryBuffer(name, &buffer[0], operand_type); } // TODO, replace this with more efficient code in optimizers -uint32_t AddInitializerTransposed(ModelBuilder& model_builder, - const std::string& name) { +static uint32_t AddInitializerTransposed(ModelBuilder& model_builder, + const OperandType& source_operand_type, + const std::string& name) { const auto& tensor = model_builder.GetInitializerTensors().at(name); - Shape shape; - for (auto dim : tensor.dims()) - shape.push_back(SafeInt(dim)); + const Shape& shape = source_operand_type.dimensions; ORT_ENFORCE(shape.size() == 2, "The initializer is not 2D: " + name + " actual dim " + std::to_string(shape.size())); // TODO support other data types - Type type; - if (tensor.data_type() == ONNX_NAMESPACE::TensorProto_DataType_FLOAT) { - type = Type::TENSOR_FLOAT32; - } else { - ORT_THROW("The initializer of graph doesn't have valid type: " + name); + const uint8_t* src = nullptr; + std::unique_ptr unpacked_tensor; + size_t tensor_byte_size; + switch (tensor.data_type()) { + case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: + src = reinterpret_cast(GetTensorFloatData(tensor)); + break; + case ONNX_NAMESPACE::TensorProto_DataType_UINT8: + case ONNX_NAMESPACE::TensorProto_DataType_INT8: { + ORT_THROW_IF_ERROR( + UnpackInitializerTensor(tensor, unpacked_tensor, tensor_byte_size)); + src = unpacked_tensor.get(); + break; + } + default: + ORT_THROW("The initializer of graph " + name + + " doesn't have valid type: " + std::to_string(tensor.data_type())); } - auto x_t = shape[0], y_t = shape[1]; + const auto x_t = shape[0], y_t = shape[1]; Shape dest_shape = {y_t, x_t}; - const OperandType operand_type(type, dest_shape); - const float* src = GetTensorFloatData(tensor); - float* buffer = new float[Product(shape)]; + OperandType operand_type = source_operand_type; + operand_type.SetDimensions(dest_shape); + std::unique_ptr buffer_holder(new uint8_t[operand_type.GetOperandBlobByteSize()]); + uint8_t* buffer = buffer_holder.get(); + size_t element_size = operand_type.GetElementByteSize(); for (uint32_t x = 0; x < x_t; x++) { for (uint32_t y = 0; y < y_t; y++) { - buffer[y * x_t + x] = src[x * y_t + y]; + for (size_t i = 0; i < element_size; i++) { + buffer[element_size * (y * x_t + x) + i] = src[element_size * (x * y_t + y) + i]; + } } } - auto operand_idx = model_builder.AddOperandFromPersistMemoryBuffer(name, &buffer[0], operand_type); - delete[] buffer; - return operand_idx; + return model_builder.AddOperandFromPersistMemoryBuffer(name, &buffer[0], operand_type); } static vector ComputeConvPads( @@ -309,7 +379,8 @@ static void HandleAutoPad(const Shape& input_shape, nnapi_padding_code = (AutoPadType::VALID == auto_pad_type) ? ANEURALNETWORKS_PADDING_VALID : ANEURALNETWORKS_PADDING_SAME; } - } else { + } else if (onnx_dilations == std::vector{1, 1}) { + // Since NNAPI runs more efficiently using auto_pad, we try to map the NOTSET padding to auto_pad const auto same_upper_pads = ComputeConvPads(input_shape, weight_size_y, weight_size_x, onnx_pads, onnx_strides, onnx_dilations, AutoPadType::SAME_UPPER, use_nchw); @@ -320,6 +391,112 @@ static void HandleAutoPad(const Shape& input_shape, } } +static bool IsQuantizationScaleSupported( + const ModelBuilder& model_builder, const Node& node, const std::vector& idx_vec) { + const auto& op = node.OpType(); + for (const auto idx : idx_vec) { + const auto scale_name = node.InputDefs()[idx]->Name(); + if (Contains(model_builder.GetInitializerTensors(), scale_name)) { + const auto& tensor = model_builder.GetInitializerTensors().at(scale_name); + if (!tensor.dims().empty() && tensor.dims()[0] != 1) { + LOGS_DEFAULT(VERBOSE) << op << " does not support per-channel quantization"; + return false; + } + } else { + LOGS_DEFAULT(VERBOSE) << "The scale of " << op << " must be known"; + return false; + } + } + + return true; +} + +static bool IsQuantizationZeroPointSupported( + const ModelBuilder& model_builder, const Node& node, const std::vector& idx_vec) { + const auto& op = node.OpType(); + for (const auto idx : idx_vec) { + const auto zero_point_name = node.InputDefs()[idx]->Name(); + if (Contains(model_builder.GetInitializerTensors(), zero_point_name)) { + const auto& tensor = model_builder.GetInitializerTensors().at(zero_point_name); + if (!tensor.dims().empty() && tensor.dims()[0] != 1) { + LOGS_DEFAULT(VERBOSE) << op << " does not support per-channel quantization"; + return false; + } + if (tensor.data_type() != ONNX_NAMESPACE::TensorProto_DataType_UINT8) { + LOGS_DEFAULT(VERBOSE) << op << " does not support zero point data type " + << std::to_string(tensor.data_type()); + return false; + } + } else { + LOGS_DEFAULT(VERBOSE) << "The zero point of " << op << " must be known"; + return false; + } + } + + return true; +} + +static float GetQuantizationScale(const ModelBuilder& model_builder, const Node& node, size_t idx) { + const auto& scale_tensor = model_builder.GetInitializerTensors().at(node.InputDefs()[idx]->Name()); + return GetTensorFloatData(scale_tensor)[0]; +} + +static int32_t GetQuantizationZeroPoint(const ModelBuilder& model_builder, const Node& node, size_t idx) { + std::unique_ptr unpacked_tensor; + size_t tensor_byte_size; + const auto& zero_point_tensor = model_builder.GetInitializerTensors().at(node.InputDefs()[idx]->Name()); + ORT_THROW_IF_ERROR( + UnpackInitializerTensor(zero_point_tensor, unpacked_tensor, tensor_byte_size)); + return static_cast(unpacked_tensor.get()[0]); +} + +static void VerifyValidInputQuantizedType(const std::string& input_name, + const OperandType& input_operand_type, + float scale, int32_t zero_point) { + ORT_ENFORCE(input_operand_type.operandType.scale == scale, + "Input [" + input_name + "] NNAPI input: " + " scale: " + + std::to_string(input_operand_type.operandType.scale) + + ", ONNX input scale: " + std::to_string(scale)); + + ORT_ENFORCE(input_operand_type.operandType.zeroPoint == zero_point, + "Input [" + input_name + "] NNNAPI input zero point: " + + std::to_string(input_operand_type.operandType.zeroPoint) + + ", ONNX input zero point: " + std::to_string(zero_point)); +} + +std::pair GetQuantizedInputScaleAndZeroPoint(const ModelBuilder& model_builder, + const Node& node, + const std::string& input_name) { + const auto& op_type = node.OpType(); + assert(op_type == "QLinearMatMul" || op_type == "QLinearConv" || op_type == "DequantizeLinear"); + size_t scale_idx, zero_point_idx; + if (op_type == "DequantizeLinear") { + scale_idx = 1; + zero_point_idx = 2; + } else if (op_type == "QLinearMatMul" || op_type == "QLinearConv") { + const auto input_defs(node.InputDefs()); + if (input_name == input_defs[0]->Name()) { + scale_idx = 1; + zero_point_idx = 2; + } else if (input_name == input_defs[3]->Name()) { + scale_idx = 4; + zero_point_idx = 5; + } else { + ORT_THROW("Unknown input: " + input_name + ", for op: " + op_type); + } + } else { + ORT_THROW("Unsupported op: " + op_type); + } + + float scale = GetQuantizationScale(model_builder, node, scale_idx); + int32_t zero_point = 0; + if (node.InputDefs().size() > 2) { + zero_point = GetQuantizationZeroPoint(model_builder, node, zero_point_idx); + } + + return std::make_pair(scale, zero_point); +} + #pragma endregion helpers #pragma region op_base @@ -426,6 +603,7 @@ class BinaryOpBuilder : public BaseOpBuilder { int32_t GetMinSupportedSdkVer(ModelBuilder& model_builder, const Node& node) const override; private: + bool IsOpSupportedImpl(ModelBuilder& /* model_builder */, const Node& node) override; void AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) override; }; @@ -438,6 +616,24 @@ int32_t BinaryOpBuilder::GetMinSupportedSdkVer(ModelBuilder& /* model_builder */ return 27; } +bool BinaryOpBuilder::IsOpSupportedImpl(ModelBuilder& /* model_builder */, const Node& node) { + Shape input1_shape, input2_shape; + if (!GetShape(*node.InputDefs()[0], input1_shape) || + !GetShape(*node.InputDefs()[1], input2_shape)) + return false; + + const auto input1_size = input1_shape.size(); + const auto input2_size = input2_shape.size(); + if (input1_size > 4 || input2_size > 4) { + LOGS_DEFAULT(VERBOSE) << node.OpType() << " only support up to 4d shape, input1 is " + << input1_size << "d shape, input 2 is " + << input2_size << "d shape"; + return false; + } + + return true; +} + void BinaryOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) { const auto& op(node.OpType()); int32_t op_code; @@ -967,25 +1163,84 @@ class ConvOpBuilder : public BaseOpBuilder { private: bool IsOpSupportedImpl(ModelBuilder& model_builder, const Node& node) override; + int32_t GetMinSupportedSdkVer(ModelBuilder& model_builder, const Node& /* node */) const override { + return model_builder.UseNCHW() ? 29 : 28; + } + + bool HasSupportedInputs(const Node& node) override; void AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) override; }; +bool ConvOpBuilder::HasSupportedInputs(const Node& node) { + if (node.OpType() != "QLinearConv") + return BaseOpBuilder::HasSupportedInputs(node); + + // QLinearConv only supports input of uint8 for now + int32_t x_input_type, w_input_type; + if (!GetType(*node.InputDefs()[0], x_input_type)) + return false; + + if (!GetType(*node.InputDefs()[3], w_input_type)) + return false; + + if (x_input_type != ONNX_NAMESPACE::TensorProto_DataType_UINT8 || x_input_type != w_input_type) { + LOGS_DEFAULT(VERBOSE) << "[" << node.OpType() + << "] x Input type: [" << x_input_type + << "] w Input type: [" << w_input_type + << "] is not supported for now"; + return false; + } + + return true; +} + void ConvOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const Node& node) { + const auto& op = node.OpType(); + const auto input_defs = node.InputDefs(); + // skip the weight for conv as we need to transpose - model_builder.AddInitializerToSkip(node.InputDefs()[1]->Name()); + if (op == "QLinearConv") { + model_builder.AddInitializerToSkip(input_defs[1]->Name()); // a_scale + model_builder.AddInitializerToSkip(input_defs[2]->Name()); // x_zero_point + model_builder.AddInitializerToSkip(input_defs[3]->Name()); // w + model_builder.AddInitializerToSkip(input_defs[4]->Name()); // w_scale + model_builder.AddInitializerToSkip(input_defs[5]->Name()); // w_zero_point + model_builder.AddInitializerToSkip(input_defs[6]->Name()); // y_scale + model_builder.AddInitializerToSkip(input_defs[7]->Name()); // y_zero_point + if (input_defs.size() > 8) + model_builder.AddInitializerToSkip(input_defs[8]->Name()); // B + } else { + model_builder.AddInitializerToSkip(input_defs[1]->Name()); // w + } } bool ConvOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, const Node& node) { + const auto& op_type = node.OpType(); + const auto input_defs = node.InputDefs(); + const auto& initializers(model_builder.GetInitializerTensors()); NodeAttrHelper helper(node); + bool is_qlinear_conv = (op_type == "QLinearConv"); + size_t w_idx = is_qlinear_conv ? 3 : 1; const auto group = helper.Get("group", 1); - const auto weight_name = node.InputDefs()[1]->Name(); - if (Contains(model_builder.GetInitializerTensors(), weight_name)) { - const auto& tensor = model_builder.GetInitializerTensors().at(weight_name); + const auto weight_name = input_defs[w_idx]->Name(); + if (Contains(initializers, weight_name)) { + const auto& tensor = initializers.at(weight_name); if (tensor.dims().size() != 4) { LOGS_DEFAULT(VERBOSE) << "Only conv 2d is supported."; return false; } + + const auto onnx_dilations = helper.Get("dilations", vector{1, 1}); + if (onnx_dilations != vector{1, 1}) { + const auto android_sdk_ver = model_builder.GetAndroidSdkVer(); + if (android_sdk_ver < 29) { + LOGS_DEFAULT(VERBOSE) << op_type << " dilations is only supported on Android API levle 29+, " + << "actual API level: " << android_sdk_ver; + return false; + } + } + if (group != 1 && tensor.dims()[1] != 1) { LOGS_DEFAULT(VERBOSE) << "group != 1 is not supported"; return false; @@ -995,6 +1250,33 @@ bool ConvOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, const Node& n return false; } + if (is_qlinear_conv) { + // For QLinearConv, we only support uint8 output now + int32_t output_type; + if (!GetType(*node.OutputDefs()[0], output_type)) + return false; + + if (output_type != ONNX_NAMESPACE::TensorProto_DataType_UINT8) { + LOGS_DEFAULT(VERBOSE) << "[" << op_type + << "] output type: [" << output_type + << "] is not supported for now"; + return false; + } + + if (input_defs.size() > 8 && !Contains(initializers, input_defs[8]->Name())) { + LOGS_DEFAULT(VERBOSE) << "Bias of QLinearConv must be known"; + return false; + } + + // a/b/y_scale + if (!IsQuantizationScaleSupported(model_builder, node, {1, 4, 6})) + return false; + + // a/b/y_zero_point + if (!IsQuantizationZeroPointSupported(model_builder, node, {2, 5, 7})) + return false; + } + return true; } @@ -1004,6 +1286,9 @@ void ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Nod const auto& operand_types(model_builder.GetOperandTypes()); const auto& initializers(model_builder.GetInitializerTensors()); NodeAttrHelper helper(node); + const auto input_defs = node.InputDefs(); + const auto& op_type = node.OpType(); + bool is_qlinear_conv = (op_type == "QLinearConv"); // onnx strides are in the order height, width // while nnapi strides are in the order width, height @@ -1018,7 +1303,11 @@ void ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Nod const auto onnx_dilations = helper.Get("dilations", vector{1, 1}); const auto group = helper.Get("group", 1); - auto input = node.InputDefs()[0]->Name(); + size_t x_idx = 0, + w_idx = is_qlinear_conv ? 3 : 1, + b_idx = is_qlinear_conv ? 8 : 2; + + auto input = input_defs[x_idx]->Name(); bool use_nchw = model_builder.UseNCHW(); bool input_is_nhwc = model_builder.IsOperandNHWC(input); bool output_is_nhwc = false; @@ -1027,7 +1316,7 @@ void ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Nod } else { output_is_nhwc = true; if (!input_is_nhwc) { - const auto& nchw_input = node.InputDefs()[0]->Name(); + const auto& nchw_input = input_defs[x_idx]->Name(); if (!model_builder.GetNHWCOperand(nchw_input, input)) { input = model_builder.GetUniqueName(nchw_input + "_nchw_to_nhwc"); TransposeNCHWToNHWC(model_builder, nchw_input, input); @@ -1035,27 +1324,71 @@ void ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Nod } } - const auto& weight = node.InputDefs()[1]->Name(); - const auto& output = node.OutputDefs()[0]->Name(); + float x_scale = 0.0f, + w_scale = 0.0f, + y_scale = 0.0f; + int32_t x_zero_point = 0, + w_zero_point = 0, + y_zero_point = 0; + + if (is_qlinear_conv) { + x_scale = GetQuantizationScale(model_builder, node, 1); + w_scale = GetQuantizationScale(model_builder, node, 4); + y_scale = GetQuantizationScale(model_builder, node, 6); + + x_zero_point = GetQuantizationZeroPoint(model_builder, node, 2); + w_zero_point = GetQuantizationZeroPoint(model_builder, node, 5); + y_zero_point = GetQuantizationZeroPoint(model_builder, node, 7); + } + + const auto& weight = input_defs[w_idx]->Name(); - bool conv2d = (group == 1); const auto& weight_tensor = initializers.at(weight); + bool conv2d = (group == 1); bool depthwise_conv2d = (weight_tensor.dims()[1] == 1); + Shape onnx_weight_shape; + for (auto dim : weight_tensor.dims()) + onnx_weight_shape.push_back(SafeInt(dim)); + + Type onnx_weight_type; + switch (weight_tensor.data_type()) { + case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: + onnx_weight_type = Type::TENSOR_FLOAT32; + break; + case ONNX_NAMESPACE::TensorProto_DataType_UINT8: + onnx_weight_type = Type::TENSOR_QUANT8_ASYMM; + break; + default: + ORT_THROW("The initializer of graph " + weight + " doesn't have valid type: " + + std::to_string(weight_tensor.data_type())); + } + + OperandType onnx_weight_operand_type(onnx_weight_type, onnx_weight_shape, w_scale, w_zero_point); + // Pre-process weights if (conv2d) { - AddInitializerInNewLayout(model_builder, weight, L_0231); + AddInitializerInNewLayout(model_builder, weight, onnx_weight_operand_type, L_0231); } else { // depthwise_conv2d - AddInitializerInNewLayout(model_builder, weight, L_1230); + AddInitializerInNewLayout(model_builder, weight, onnx_weight_operand_type, L_1230); } - bool hasBias = (node.InputDefs().size() >= 3); - std::string bias = hasBias ? node.InputDefs()[2]->Name() : weight + "_bias"; + if (is_qlinear_conv) { + // Verify if the scale and zero point matchs from onnx input/weight and nnapi input/weight + const OperandType& x_operand_type = operand_types.at(input); + ORT_ENFORCE(x_operand_type.type == Type::TENSOR_QUANT8_ASYMM, + "input type is " + TypeToStr(x_operand_type.type)); + VerifyValidInputQuantizedType(input, x_operand_type, x_scale, x_zero_point); - uint32_t bias_idx_val; - if (hasBias) { - bias_idx_val = operand_indices.at(bias); - } else { + const OperandType& w_operand_type = operand_types.at(weight); + ORT_ENFORCE(w_operand_type.type == Type::TENSOR_QUANT8_ASYMM, + "input type is " + TypeToStr(w_operand_type.type)); + VerifyValidInputQuantizedType(weight, w_operand_type, w_scale, w_zero_point); + } + + bool hasBias = (input_defs.size() > b_idx); + std::string bias = hasBias ? input_defs[b_idx]->Name() : weight + "_bias"; + if (!hasBias) { const auto weight_dimen = shaper[weight]; Shape bias_dimen; if (conv2d) @@ -1065,16 +1398,27 @@ void ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Nod const auto& weight_type = operand_types.at(weight).type; if (weight_type == Type::TENSOR_FLOAT32) { - vector buffer(bias_dimen[0]); - for (uint32_t i = 0; i < buffer.size(); i++) { - buffer[i] = 0.f; - } - OperandType bias_operand_type(Type::TENSOR_FLOAT32, bias_dimen); - bias_idx_val = model_builder.AddOperandFromPersistMemoryBuffer( - bias, buffer.data(), bias_operand_type); + vector buffer(bias_dimen[0], 0.0f); + OperandType bias_operand_type(Type::TENSOR_FLOAT32, bias_dimen, x_scale * w_scale); + model_builder.AddOperandFromPersistMemoryBuffer(bias, buffer.data(), bias_operand_type); + } else if (weight_type == Type::TENSOR_QUANT8_ASYMM) { + vector buffer(bias_dimen[0], 0); + OperandType bias_operand_type(Type::TENSOR_INT32, bias_dimen, x_scale * w_scale); + model_builder.AddOperandFromPersistMemoryBuffer(bias, buffer.data(), bias_operand_type); } else { ORT_THROW("Unknown weight type " + TypeToStr(weight_type)); } + } else if (is_qlinear_conv) { // QLinearConv's bias type need special handling + const auto& bias_tensor = model_builder.GetInitializerTensors().at(bias); + ORT_ENFORCE(bias_tensor.data_type() == ONNX_NAMESPACE::TensorProto_DataType_INT32, + "bias of QLinearConv should be int32, actual type: " + std::to_string(bias_tensor.data_type())); + Shape bias_dimen; + for (auto dim : bias_tensor.dims()) + bias_dimen.push_back(SafeInt(dim)); + + const void* buffer = GetTensorInt32Data(bias_tensor); + OperandType bias_operand_type(Type::TENSOR_INT32, bias_dimen, x_scale * w_scale); + model_builder.AddOperandFromPersistMemoryBuffer(bias, buffer, bias_operand_type); } const auto auto_pad_type = StringToAutoPadType(helper.Get("auto_pad", "NOTSET")); @@ -1092,7 +1436,7 @@ void ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Nod std::vector input_indices; input_indices.push_back(operand_indices.at(input)); input_indices.push_back(operand_indices.at(weight)); - input_indices.push_back(bias_idx_val); + input_indices.push_back(operand_indices.at(bias)); if (use_auto_pad) { input_indices.push_back(model_builder.AddOperandFromScalar(nnapi_padding_code)); @@ -1123,6 +1467,7 @@ void ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Nod } int32_t operationCode; + const auto& output = node.OutputDefs()[0]->Name(); if (conv2d) { operationCode = ANEURALNETWORKS_CONV_2D; shaper.Conv(input, weight, @@ -1137,7 +1482,7 @@ void ConvOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Nod output); } - const OperandType output_operand_type(operand_types.at(input).type, shaper[output]); + const OperandType output_operand_type(operand_types.at(input).type, shaper[output], y_scale, y_zero_point); model_builder.AddOperation(operationCode, input_indices, {output}, {output_operand_type}, {output_is_nhwc}); } @@ -1296,17 +1641,45 @@ class GemmOpBuilder : public BaseOpBuilder { private: bool IsOpSupportedImpl(ModelBuilder& model_builder, const Node& node) override; - + bool HasSupportedInputs(const Node& node) override; void AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) override; }; +bool GemmOpBuilder::HasSupportedInputs(const Node& node) { + if (node.OpType() != "QLinearMatMul") + return BaseOpBuilder::HasSupportedInputs(node); + + // QLinearMatMul + int32_t a_input_type, b_input_type; + if (!GetType(*node.InputDefs()[0], a_input_type)) + return false; + if (!GetType(*node.InputDefs()[3], b_input_type)) + return false; + + if (a_input_type != ONNX_NAMESPACE::TensorProto_DataType_UINT8 || a_input_type != b_input_type) { + LOGS_DEFAULT(VERBOSE) << "[" << node.OpType() + << "] A Input type: [" << a_input_type + << "] B Input type: [" << b_input_type + << "] is not supported for now"; + return false; + } + + return true; +} + bool GemmOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, const Node& node) { const auto& op = node.OpType(); + const auto input_defs(node.InputDefs()); const auto& initializers(model_builder.GetInitializerTensors()); + size_t a_idx = 0, b_idx = 1, c_idx = 2; // A*B+C + if (op == "QLinearMatMul") { + a_idx = 0; + b_idx = 3; + } Shape a_shape; { - if (!GetShape(*node.InputDefs()[0], a_shape)) + if (!GetShape(*input_defs[a_idx], a_shape)) return false; if (a_shape.size() != 2) { @@ -1317,7 +1690,7 @@ bool GemmOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, const Node& n Shape b_shape; { - if (!GetShape(*node.InputDefs()[1], b_shape)) + if (!GetShape(*input_defs[b_idx], b_shape)) return false; if (b_shape.size() != 2) { @@ -1326,11 +1699,39 @@ bool GemmOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, const Node& n } } - if (op == "MatMul") { // Only support A*B B is an initializer - if (!Contains(initializers, node.InputDefs()[1]->Name())) { + if (op == "MatMul") { + // Only support A*B B is an initializer + if (!Contains(initializers, input_defs[b_idx]->Name())) { LOGS_DEFAULT(VERBOSE) << "B of MatMul must be known"; return false; } + } else if (op == "QLinearMatMul") { + // For QLinearMatMul, we only support uint8 output now + int32_t output_type; + if (!GetType(*node.OutputDefs()[0], output_type)) + return false; + + if (output_type != ONNX_NAMESPACE::TensorProto_DataType_UINT8) { + LOGS_DEFAULT(VERBOSE) << "[" << op + << "] output type: [" << output_type + << "] is not supported for now"; + return false; + } + + // Only support A*B B is an initializer + // And all scale/zero points are initializer scalars + if (!Contains(initializers, input_defs[b_idx]->Name())) { + LOGS_DEFAULT(VERBOSE) << "B of MatMul must be known"; + return false; + } + + // a/b/y_scale + if (!IsQuantizationScaleSupported(model_builder, node, {1, 4, 6})) + return false; + + // a/b/y_zero_point + if (!IsQuantizationZeroPointSupported(model_builder, node, {2, 5, 7})) + return false; } else if (op == "Gemm") { // Only support // 1. A*B'+C @@ -1347,14 +1748,14 @@ bool GemmOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, const Node& n return false; } - if (transB == 0 && !Contains(initializers, node.InputDefs()[1]->Name())) { + if (transB == 0 && !Contains(initializers, input_defs[b_idx]->Name())) { LOGS_DEFAULT(VERBOSE) << "B of Gemm must be known if transB != 1"; return false; } - if (node.InputDefs().size() == 3) { + if (input_defs.size() == 3) { Shape c_shape; - if (!GetShape(*node.InputDefs()[2], c_shape)) + if (!GetShape(*input_defs[c_idx], c_shape)) return false; if (c_shape.size() != 1 || @@ -1373,53 +1774,116 @@ bool GemmOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, const Node& n void GemmOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const Node& node) { const auto& op = node.OpType(); + const auto input_defs(node.InputDefs()); if (op == "MatMul") { - model_builder.AddInitializerToSkip(node.InputDefs()[1]->Name()); + model_builder.AddInitializerToSkip(input_defs[1]->Name()); } else if (op == "Gemm") { NodeAttrHelper helper(node); const auto transB = helper.Get("transB", 0); if (transB == 0) - model_builder.AddInitializerToSkip(node.InputDefs()[1]->Name()); + model_builder.AddInitializerToSkip(input_defs[1]->Name()); + } else if (op == "QLinearMatMul") { + model_builder.AddInitializerToSkip(input_defs[1]->Name()); // a_scale + model_builder.AddInitializerToSkip(input_defs[2]->Name()); // a_zero_point + model_builder.AddInitializerToSkip(input_defs[3]->Name()); // b + model_builder.AddInitializerToSkip(input_defs[4]->Name()); // b_scale + model_builder.AddInitializerToSkip(input_defs[5]->Name()); // b_zero_point + model_builder.AddInitializerToSkip(input_defs[6]->Name()); // y_scale + model_builder.AddInitializerToSkip(input_defs[7]->Name()); // y_zero_point } } void GemmOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) { - const auto& op = node.OpType(); auto& shaper(model_builder.GetShaper()); const auto& operand_indices(model_builder.GetOperandIndices()); const auto& operand_types(model_builder.GetOperandTypes()); - NodeAttrHelper helper(node); + const auto& initializers(model_builder.GetInitializerTensors()); - const auto& input1 = node.InputDefs()[0]->Name(); - const auto& input2 = node.InputDefs()[1]->Name(); + const auto& op = node.OpType(); + const auto input_defs(node.InputDefs()); + NodeAttrHelper helper(node); + bool is_qlinear_matmul = op == "QLinearMatMul"; + + size_t a_idx = 0, + b_idx = is_qlinear_matmul ? 3 : 1, + c_idx = 2; // QLinearMatMul has no bias + + const auto& input1 = input_defs[a_idx]->Name(); + const auto& input2 = input_defs[b_idx]->Name(); const auto& output = node.OutputDefs()[0]->Name(); const auto transB = helper.Get("transB", 0); + float a_scale = 0.0f, + b_scale = 0.0f, + y_scale = 0.0f; + int32_t a_zero_point = 0, + b_zero_point = 0, + y_zero_point = 0; + + if (is_qlinear_matmul) { + a_scale = GetQuantizationScale(model_builder, node, 1); + b_scale = GetQuantizationScale(model_builder, node, 4); + y_scale = GetQuantizationScale(model_builder, node, 6); + + a_zero_point = GetQuantizationZeroPoint(model_builder, node, 2); + b_zero_point = GetQuantizationZeroPoint(model_builder, node, 5); + y_zero_point = GetQuantizationZeroPoint(model_builder, node, 7); + } + uint32_t input_2_idx; if (transB == 0) { - input_2_idx = AddInitializerTransposed(model_builder, input2); + Type onnx_mat_b_type; + if (!is_qlinear_matmul) + onnx_mat_b_type = Type::TENSOR_FLOAT32; + else + onnx_mat_b_type = Type::TENSOR_QUANT8_ASYMM; + + const auto& mat_b_tensor = initializers.at(input2); + Shape onnx_mat_b_shape; + for (auto dim : mat_b_tensor.dims()) + onnx_mat_b_shape.push_back(SafeInt(dim)); + + const OperandType onnx_mat_b_operand_type(onnx_mat_b_type, onnx_mat_b_shape, b_scale, b_zero_point); + input_2_idx = AddInitializerTransposed(model_builder, onnx_mat_b_operand_type, input2); } else { input_2_idx = operand_indices.at(input2); } + // Verify if the scale and zero point matchs from onnx input and nnapi input + if (is_qlinear_matmul) { + const OperandType& a_operand_type = operand_types.at(input1); + ORT_ENFORCE(a_operand_type.type == Type::TENSOR_QUANT8_ASYMM, + "input type is " + TypeToStr(a_operand_type.type)); + VerifyValidInputQuantizedType(input1, a_operand_type, a_scale, a_zero_point); + + const OperandType& b_operand_type = operand_types.at(input2); + ORT_ENFORCE(b_operand_type.type == Type::TENSOR_QUANT8_ASYMM, + "input type is " + TypeToStr(b_operand_type.type)); + VerifyValidInputQuantizedType(input2, b_operand_type, b_scale, b_zero_point); + } + uint32_t bias_idx; - if (node.InputDefs().size() == 2) { + bool has_bias = (op == "Gemm") && (input_defs.size() > 2); + if (has_bias) { + bias_idx = operand_indices.at(input_defs[c_idx]->Name()); + } else { + // No C supplied, we need a vector of 0 std::string bias = node.Name() + op + "_bias"; - const auto& B_type = operand_types.at(input2).type; + const auto& bias_type = operand_types.at(input2).type; Shape bias_dimen = {shaper[input2][0]}; - if (B_type == Type::TENSOR_FLOAT32) { - float buffer[bias_dimen[0]]; - for (uint32_t i = 0; i < bias_dimen[0]; i++) { - buffer[i] = 0.f; - } + if (bias_type == Type::TENSOR_FLOAT32) { + std::vector buffer(bias_dimen[0], 0.f); OperandType bias_operand_type(Type::TENSOR_FLOAT32, bias_dimen); bias_idx = model_builder.AddOperandFromPersistMemoryBuffer( - bias, &buffer[0], bias_operand_type); + bias, buffer.data(), bias_operand_type); + } else if (bias_type == Type::TENSOR_QUANT8_ASYMM) { + std::vector buffer(bias_dimen[0], 0); + OperandType bias_operand_type(Type::TENSOR_INT32, bias_dimen, a_scale * b_scale, 0); + bias_idx = model_builder.AddOperandFromPersistMemoryBuffer( + bias, buffer.data(), bias_operand_type); } else { - ORT_THROW("Unknown weight type " + TypeToStr(B_type)); + ORT_THROW("Unknown weight type " + TypeToStr(bias_type)); } - } else { - bias_idx = operand_indices.at(node.InputDefs()[2]->Name()); } std::vector input_indices; @@ -1430,7 +1894,7 @@ void GemmOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Nod input_indices.push_back(model_builder.AddOperandFromScalar(fuse_code)); shaper.FC(input1, input2, output); - const OperandType output_operand_type(operand_types.at(input1).type, shaper[output]); + const OperandType output_operand_type(operand_types.at(input1).type, shaper[output], y_scale, y_zero_point); model_builder.AddOperation(ANEURALNETWORKS_FULLY_CONNECTED, input_indices, {output}, {output_operand_type}, {false}); } @@ -1674,6 +2138,172 @@ void SqueezeOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const #pragma endregion +#pragma region op_quantizelinear + +class QuantizeLinearOpBuilder : public BaseOpBuilder { + public: + void AddInitializersToSkip(ModelBuilder& model_builder, const Node& node) override; + + private: + bool IsOpSupportedImpl(ModelBuilder& model_builder, const Node& node) override; + + int32_t GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, const Node& /* node */) const override { + return 27; + } + + void AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) override; +}; + +void QuantizeLinearOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const Node& node) { + const auto input_defs(node.InputDefs()); + + model_builder.AddInitializerToSkip(input_defs[1]->Name()); + + if (input_defs.size() == 3) // has zero_point input + model_builder.AddInitializerToSkip(input_defs[2]->Name()); +} + +bool QuantizeLinearOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, const Node& node) { + const auto input_defs(node.InputDefs()); + const auto output_defs(node.OutputDefs()); + + int32_t output_type; + if (!GetType(*output_defs[0], output_type)) + return false; + + if (output_type != ONNX_NAMESPACE::TensorProto_DataType_UINT8) { + LOGS_DEFAULT(VERBOSE) << "[" << node.OpType() + << "] output type: [" << output_type + << "] is not supported for now"; + return false; + } + + if (!IsQuantizationScaleSupported(model_builder, node, {1})) + return false; + + if (input_defs.size() == 3) { // has zero_point input + if (!IsQuantizationZeroPointSupported(model_builder, node, {2})) + return false; + } + + return true; +} + +void QuantizeLinearOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) { + auto& shaper(model_builder.GetShaper()); + const auto& operand_indices(model_builder.GetOperandIndices()); + const auto input_defs(node.InputDefs()); + + const auto& input = input_defs[0]->Name(); + const auto& output = node.OutputDefs()[0]->Name(); + bool output_is_nhwc = model_builder.IsOperandNHWC(input); + + float scale = GetQuantizationScale(model_builder, node, 1); + int32_t zero_point = 0; + Type output_type = Type::TENSOR_QUANT8_ASYMM; + + if (input_defs.size() == 3) { // Get zero point + zero_point = GetQuantizationZeroPoint(model_builder, node, 2); + } + + LOGS_DEFAULT(VERBOSE) << "scale: " << scale << " zp: " << zero_point; + + shaper.Identity(input, output); + const OperandType output_operand_type(output_type, shaper[output], scale, zero_point); + std::vector input_indices; + input_indices.push_back(operand_indices.at(input)); + model_builder.AddOperation(ANEURALNETWORKS_QUANTIZE, input_indices, {output}, {output_operand_type}, {output_is_nhwc}); +} + +#pragma endregion + +#pragma region op_dequantizelinear + +class DequantizeLinearOpBuilder : public BaseOpBuilder { + public: + void AddInitializersToSkip(ModelBuilder& model_builder, const Node& node) override; + + private: + bool IsOpSupportedImpl(ModelBuilder& model_builder, const Node& node) override; + + int32_t GetMinSupportedSdkVer(ModelBuilder& /* model_builder */, const Node& /* node */) const override { + return 29; + } + + bool HasSupportedInputs(const Node& node) override; + void AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) override; +}; + +bool DequantizeLinearOpBuilder::HasSupportedInputs(const Node& node) { + int32_t input_type; + if (!GetType(*node.InputDefs()[0], input_type)) + return false; + + if (input_type != ONNX_NAMESPACE::TensorProto_DataType_UINT8) { + LOGS_DEFAULT(VERBOSE) << "[" << node.OpType() + << "] Input type: [" << input_type + << "] is not supported for now"; + return false; + } + + return true; +} + +void DequantizeLinearOpBuilder::AddInitializersToSkip(ModelBuilder& model_builder, const Node& node) { + const auto input_defs(node.InputDefs()); + + model_builder.AddInitializerToSkip(input_defs[1]->Name()); + + if (input_defs.size() == 3) // has zero_point input + model_builder.AddInitializerToSkip(input_defs[2]->Name()); +} + +bool DequantizeLinearOpBuilder::IsOpSupportedImpl(ModelBuilder& model_builder, const Node& node) { + const auto input_defs(node.InputDefs()); + + if (!IsQuantizationScaleSupported(model_builder, node, {1})) + return false; + + if (input_defs.size() == 3) { // has zero_point input + if (!IsQuantizationZeroPointSupported(model_builder, node, {2})) + return false; + } + + return true; +} + +void DequantizeLinearOpBuilder::AddToModelBuilderImpl(ModelBuilder& model_builder, const Node& node) { + auto& shaper(model_builder.GetShaper()); + const auto& operand_indices(model_builder.GetOperandIndices()); + const auto& operand_types(model_builder.GetOperandTypes()); + const auto input_defs(node.InputDefs()); + + const auto& input = input_defs[0]->Name(); + const auto& output = node.OutputDefs()[0]->Name(); + bool output_is_nhwc = model_builder.IsOperandNHWC(input); + + float scale = GetQuantizationScale(model_builder, node, 1); + int32_t zero_point = 0; + if (input_defs.size() == 3) { // Get zero point + zero_point = GetQuantizationZeroPoint(model_builder, node, 2); + } + + const OperandType& input_operand_type = operand_types.at(input); + ORT_ENFORCE(input_operand_type.type == Type::TENSOR_QUANT8_ASYMM, + "input type is " + TypeToStr(input_operand_type.type)); + + VerifyValidInputQuantizedType(input, input_operand_type, scale, zero_point); + + shaper.Identity(input, output); + const OperandType output_operand_type(Type::TENSOR_FLOAT32, shaper[output]); + + std::vector input_indices; + input_indices.push_back(operand_indices.at(input)); + model_builder.AddOperation(ANEURALNETWORKS_DEQUANTIZE, input_indices, {output}, {output_operand_type}, {output_is_nhwc}); +} + +#pragma endregion + #pragma region CreateOpBuilders std::unordered_map> @@ -1701,7 +2331,11 @@ CreateOpBuilders() { op_map.emplace("MaxPool", pool_op_builder); } - op_map.emplace("Conv", std::make_shared()); + { + op_map.emplace("Conv", std::make_shared()); + op_map.emplace("QLinearConv", std::make_shared()); + } + op_map.emplace("Cast", std::make_shared()); op_map.emplace("Softmax", std::make_shared()); op_map.emplace("Identity", std::make_shared()); @@ -1710,6 +2344,7 @@ CreateOpBuilders() { auto gemm_op_builder = std::make_shared(); op_map.emplace("Gemm", gemm_op_builder); op_map.emplace("MatMul", gemm_op_builder); + op_map.emplace("QLinearMatMul", gemm_op_builder); } { @@ -1727,6 +2362,8 @@ CreateOpBuilders() { op_map.emplace("Concat", std::make_shared()); op_map.emplace("Squeeze", std::make_shared()); + op_map.emplace("QuantizeLinear", std::make_shared()); + op_map.emplace("DequantizeLinear", std::make_shared()); return op_map; } diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.h index d800a59b12..d2e30ac75f 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/builders/op_builder.h @@ -27,8 +27,12 @@ class IOpBuilder { // for different onnx operators std::unordered_map> CreateOpBuilders(); -// Transpose the NHWCinput to NCHW output +// Transpose the NHWC input to NCHW output void TransposeNHWCToNCHW(ModelBuilder& model_builder, const std::string& input, const std::string& output); +// Get the quantized input's scale and zero point for the given input +std::pair GetQuantizedInputScaleAndZeroPoint(const ModelBuilder& model_builder, + const Node& node, const std::string& input_name); + } // namespace nnapi } // namespace onnxruntime diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc index 6d72a5fec4..3699d73652 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_execution_provider.cc @@ -296,6 +296,9 @@ common::Status NnapiExecutionProvider::Compile(const std::vector(output_tensor); break; + case Type::TENSOR_QUANT8_ASYMM: + output_buffer = ort.GetTensorMutableData(output_tensor); + break; default: return Status(common::ONNXRUNTIME, common::FAIL, "Unsupported output type: " + TypeToStr(model_output_type.type)); diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.cc b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.cc index 1fd31d352c..e16e460d71 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.cc +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.cc @@ -21,14 +21,14 @@ namespace android { namespace nn { namespace wrapper { -bool isScalarType(const Type& type) { +bool IsScalarType(const Type& type) { return type == Type::FLOAT16 || type == Type::FLOAT32 || type == Type::INT32 || type == Type::BOOL || type == Type::UINT32; } OperandType::OperandType(Type type, const std::vector& d, float scale, int32_t zeroPoint) - : type(type), dimensions(std::move(d)) { + : type(type), dimensions(d) { if (dimensions.empty()) { - if (!isScalarType(type)) { + if (!IsScalarType(type)) { dimensions = {1}; } } @@ -45,38 +45,30 @@ OperandType::OperandType(Type type, const std::vector& d, float scale, OperandType::OperandType(const OperandType& other) { type = other.type; dimensions = other.dimensions; + if (dimensions.empty()) { - if (!isScalarType(type)) { + if (!IsScalarType(type)) { dimensions = {1}; } } - operandType = { - .type = static_cast(type), - .dimensionCount = static_cast(dimensions.size()), - .dimensions = dimensions.size() > 0 ? dimensions.data() : nullptr, - .scale = other.operandType.scale, - .zeroPoint = other.operandType.zeroPoint, - }; -} // namespace wrapper + operandType = other.operandType; + operandType.dimensions = dimensions.size() > 0 ? dimensions.data() : nullptr; +} OperandType& OperandType::operator=(const OperandType& other) { if (this != &other) { type = other.type; dimensions = other.dimensions; + if (dimensions.empty()) { - if (!isScalarType(type)) { + if (!IsScalarType(type)) { dimensions = {1}; } } - operandType = { - .type = static_cast(type), - .dimensionCount = static_cast(dimensions.size()), - .dimensions = dimensions.size() > 0 ? dimensions.data() : nullptr, - .scale = other.operandType.scale, - .zeroPoint = other.operandType.zeroPoint, - }; + operandType = other.operandType; + operandType.dimensions = dimensions.size() > 0 ? dimensions.data() : nullptr; } return *this; @@ -121,6 +113,17 @@ size_t OperandType::GetOperandBlobByteSize() const { return Product(dimensions) * GetElementByteSize(); } +void OperandType::SetDimensions(const std::vector& d) { + dimensions = d; + if (dimensions.empty()) { + if (!IsScalarType(type)) { + dimensions = {1}; + } + } + operandType.dimensionCount = dimensions.size(); + operandType.dimensions = dimensions.size() > 0 ? dimensions.data() : nullptr; +} + } // namespace wrapper } // namespace nn } // namespace android \ No newline at end of file diff --git a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.h b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.h index 426a10ea7f..c61fc458e9 100644 --- a/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.h +++ b/onnxruntime/core/providers/nnapi/nnapi_builtin/nnapi_lib/NeuralNetworksWrapper.h @@ -113,6 +113,8 @@ struct OperandType { // Get the whole blob size in bytes size_t GetOperandBlobByteSize() const; + void SetDimensions(const std::vector& d); + operator ANeuralNetworksOperandType() const { return operandType; } }; } // namespace wrapper diff --git a/onnxruntime/test/contrib_ops/quantize_ops_test.cc b/onnxruntime/test/contrib_ops/quantize_ops_test.cc index 9baae4b1f4..c38322e29f 100644 --- a/onnxruntime/test/contrib_ops/quantize_ops_test.cc +++ b/onnxruntime/test/contrib_ops/quantize_ops_test.cc @@ -9,16 +9,25 @@ namespace onnxruntime { namespace test { // scalar zero & scale with uint8 -TEST(DequantizeLinearOpTest, DequantizeLinear_per_tensor_float_uint8) { +void TestDequantizeLinearPerTensorFloatUint8(bool use_initializer_except_x) { OpTester test("DequantizeLinear", 1, onnxruntime::kMSDomain); std::vector dims{4}; test.AddInput("x", dims, {0, 3, 128, 255}); - test.AddInput("x_scale", {}, {2.0f}); - test.AddInput("x_zero_point", {}, {128}); + test.AddInput("x_scale", {}, {2.0f}, use_initializer_except_x); + test.AddInput("x_zero_point", {}, {128}, use_initializer_except_x); test.AddOutput("y", dims, {-256.0f, -250.0f, 0.0f, 254.0f}); test.Run(); } +TEST(DequantizeLinearOpTest, DequantizeLinear_per_tensor_float_uint8) { + TestDequantizeLinearPerTensorFloatUint8(false); +} + +// NNAPI EP requires weight to be an initializer +TEST(DequantizeLinearOpTest, DequantizeLinear_per_tensor_float_uint8_use_initializer_except_x) { + TestDequantizeLinearPerTensorFloatUint8(true); +} + // scalar zero & scale with int8 TEST(DequantizeLinearOpTest, DequantizeLinear_per_tensor_float_int8) { OpTester test("DequantizeLinear", 1, onnxruntime::kMSDomain); @@ -166,11 +175,11 @@ TEST(DequantizeLinearContribOpTest, DequantizeLinear_3) { } // quantize with scalar zero point and scale -TEST(QuantizeLinearContribOpTest, QuantizeLinear_per_tensor_float_uint8) { +void TestQuantizeLinearPerTensorFloatUint8(bool use_initializer_except_x) { OpTester test("QuantizeLinear", 1, onnxruntime::kMSDomain); std::vector dims{16}; test.AddInput("x", dims, { - 0.f, 2.f, + 0.f, 2.f, // 3.f, -3.f, // rounding half to even 2.9f, -2.9f, // low case 3.1f, -3.1f, // up case @@ -179,24 +188,36 @@ TEST(QuantizeLinearContribOpTest, QuantizeLinear_per_tensor_float_uint8) { 256.f, -258.f, // critical point 1000.f, -1000.f // saturate case }); - test.AddInput("y_scale", {}, {2.0f}); - test.AddInput("y_zero_point", {}, {128}); - test.AddOutput("y", dims, {128, 129, - 130, 126, - 129, 127, - 130, 126, - 255, 0, - 255, 0, - 255, 0, - 255, 0}); + test.AddInput("y_scale", {}, {2.0f}, use_initializer_except_x); + test.AddInput("y_zero_point", {}, {128}, use_initializer_except_x); + test.AddOutput("y", dims, + {128, 129, + 130, 126, + 129, 127, + 130, 126, + 255, 0, + 255, 0, + 255, 0, + 255, 0}); test.Run(); } +TEST(QuantizeLinearContribOpTest, QuantizeLinear_per_tensor_float_uint8) { + TestQuantizeLinearPerTensorFloatUint8(false); +} + +// Only NNAPI EP requires weight to be an initializer +#ifdef USE_NNAPI +TEST(QuantizeLinearContribOpTest, QuantizeLinear_per_tensor_float_uint8_use_initializer_except_x) { + TestQuantizeLinearPerTensorFloatUint8(true); +} +#endif + TEST(QuantizeLinearContribOpTest, QuantizeLinear_per_tensor_float_int8) { OpTester test("QuantizeLinear", 1, onnxruntime::kMSDomain); std::vector dims{16}; test.AddInput("x", dims, { - 0.f, 2.f, + 0.f, 2.f, // 3.f, -3.f, // rounding half to even 2.9f, -2.9f, // low case 3.1f, -3.1f, // up case @@ -207,14 +228,15 @@ TEST(QuantizeLinearContribOpTest, QuantizeLinear_per_tensor_float_int8) { }); test.AddInput("y_scale", {}, {2.0f}); test.AddInput("y_zero_point", {}, {1}); - test.AddOutput("y", dims, {1, 2, - 3, -1, - 2, 0, - 3, -1, - 127, -127, - 127, -127, - 127, -128, - 127, -128}); + test.AddOutput("y", dims, + {1, 2, + 3, -1, + 2, 0, + 3, -1, + 127, -127, + 127, -127, + 127, -128, + 127, -128}); test.Run(); } @@ -223,7 +245,7 @@ TEST(QuantizeLinearContribOpTest, QuantizeLinear_per_tensor_half_uint8) { OpTester test("QuantizeLinear", 1, onnxruntime::kMSDomain); std::vector dims{16}; test.AddInput("x", dims, ToFloat16({ - 0.f, 2.f, + 0.f, 2.f, // 3.f, -3.f, // rounding half to even 2.9f, -2.9f, // low case 3.1f, -3.1f, // up case @@ -234,14 +256,15 @@ TEST(QuantizeLinearContribOpTest, QuantizeLinear_per_tensor_half_uint8) { })); test.AddInput("y_scale", {}, ToFloat16({2.0f})); test.AddInput("y_zero_point", {}, {128}); - test.AddOutput("y", dims, {128, 129, - 130, 126, - 129, 127, - 130, 126, - 255, 0, - 255, 0, - 255, 0, - 255, 0}); + test.AddOutput("y", dims, + {128, 129, + 130, 126, + 129, 127, + 130, 126, + 255, 0, + 255, 0, + 255, 0, + 255, 0}); test.Run(); } @@ -249,7 +272,7 @@ TEST(QuantizeLinearContribOpTest, QuantizeLinear_per_tensor_half_int8) { OpTester test("QuantizeLinear", 1, onnxruntime::kMSDomain); std::vector dims{16}; test.AddInput("x", dims, ToFloat16({ - 0.f, 2.f, + 0.f, 2.f, // 3.f, -3.f, // rounding half to even 2.9f, -2.9f, // low case 3.1f, -3.1f, // up case @@ -260,14 +283,15 @@ TEST(QuantizeLinearContribOpTest, QuantizeLinear_per_tensor_half_int8) { })); test.AddInput("y_scale", {}, ToFloat16({2.0f})); test.AddInput("y_zero_point", {}, {1}); - test.AddOutput("y", dims, {1, 2, - 3, -1, - 2, 0, - 3, -1, - 127, -127, - 127, -127, - 127, -128, - 127, -128}); + test.AddOutput("y", dims, + {1, 2, + 3, -1, + 2, 0, + 3, -1, + 127, -127, + 127, -127, + 127, -128, + 127, -128}); test.Run(); } #endif diff --git a/onnxruntime/test/providers/cpu/math/gemm_test.cc b/onnxruntime/test/providers/cpu/math/gemm_test.cc index 2d9f6aa8cd..b53e477c49 100644 --- a/onnxruntime/test/providers/cpu/math/gemm_test.cc +++ b/onnxruntime/test/providers/cpu/math/gemm_test.cc @@ -8,7 +8,7 @@ namespace onnxruntime { namespace test { -TEST(GemmOpTest, GemmNoTrans) { +void TestGemmNoTrans(bool b_is_initializer) { OpTester test("Gemm"); test.AddAttribute("transA", (int64_t)0); @@ -19,7 +19,7 @@ TEST(GemmOpTest, GemmNoTrans) { test.AddInput("A", {2, 4}, {1.0f, 2.0f, 3.0f, 4.0f, -1.0f, -2.0f, -3.0f, -4.0f}); - test.AddInput("B", {4, 3}, std::vector(12, 1.0f)); + test.AddInput("B", {4, 3}, std::vector(12, 1.0f), b_is_initializer); test.AddInput("C", {2, 3}, std::vector(6, 1.0f)); test.AddOutput("Y", {2, 3}, {11.0f, 11.0f, 11.0f, @@ -27,6 +27,15 @@ TEST(GemmOpTest, GemmNoTrans) { test.Run(); } +TEST(GemmOpTest, GemmNoTrans) { + TestGemmNoTrans(false); +} + +// NNAPI EP requires weight to be an initializer +TEST(GemmOpTest, GemmNoTransBIsInitializer) { + TestGemmNoTrans(true); +} + // Only CUDA kernel has float 16 support #ifdef USE_CUDA TEST(GemmOpTest, GemmNoTrans_f16) { @@ -43,7 +52,7 @@ TEST(GemmOpTest, GemmNoTrans_f16) { test.AddAttribute("beta", 1.0f); std::vector A{1.0f, 2.0f, 3.0f, 4.0f, - -1.0f, -2.0f, -3.0f, -4.0f}; + -1.0f, -2.0f, -3.0f, -4.0f}; std::vector B(12, 1.0f); std::vector C(6, 1.0f); std::vector Y{11.0f, 11.0f, 11.0f, @@ -62,7 +71,7 @@ TEST(GemmOpTest, GemmNoTrans_f16) { test.AddInput("B", {4, 3}, f_B); test.AddInput("C", {2, 3}, f_C); test.AddOutput("Y", {2, 3}, f_Y); - test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: fp16 is not supported + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: fp16 is not supported } #endif @@ -82,11 +91,11 @@ TEST(GemmOpTest, GemmBroadcast) { test.AddOutput("Y", {2, 3}, {11.0f, 12.0f, 13.0f, -9.0f, -8.0f, -7.0f}); - #if defined(OPENVINO_CONFIG_GPU_FP16) || defined(OPENVINO_CONFIG_GPU_FP32) - test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kOpenVINOExecutionProvider}); // OpenVINO : Temporarily disabled due to accuracy issues - #else - test.Run(); - #endif +#if defined(OPENVINO_CONFIG_GPU_FP16) || defined(OPENVINO_CONFIG_GPU_FP32) + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kOpenVINOExecutionProvider}); // OpenVINO : Temporarily disabled due to accuracy issues +#else + test.Run(); +#endif } TEST(GemmOpTest, GemmTrans) { @@ -107,11 +116,35 @@ TEST(GemmOpTest, GemmTrans) { test.AddOutput("Y", {2, 3}, {11.0f, 11.0f, 11.0f, -9.0f, -9.0f, -9.0f}); - #if defined(OPENVINO_CONFIG_GPU_FP16) || defined(OPENVINO_CONFIG_GPU_FP32) - test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kOpenVINOExecutionProvider}); // OpenVINO: Temporarily disabled due to accuracy issues - #else - test.Run(); - #endif +#if defined(OPENVINO_CONFIG_GPU_FP16) || defined(OPENVINO_CONFIG_GPU_FP32) + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kOpenVINOExecutionProvider}); // OpenVINO: Temporarily disabled due to accuracy issues +#else + test.Run(); +#endif +} + +// NNAPI EP's GEMM only works as A*B', add case only B is transposed +TEST(GemmOpTest, GemmTransB) { + OpTester test("Gemm"); + + test.AddAttribute("transA", (int64_t)0); + test.AddAttribute("transB", (int64_t)1); + test.AddAttribute("alpha", 1.0f); + test.AddAttribute("beta", 1.0f); + + test.AddInput("A", {2, 4}, + {1.0f, 2.0f, 3.0f, 4.0f, + -1.0f, -2.0f, -3.0f, -4.0f}); + test.AddInput("B", {3, 4}, std::vector(12, 1.0f)); + test.AddInput("C", {3}, std::vector(3, 1.0f)); + test.AddOutput("Y", {2, 3}, + {11.0f, 11.0f, 11.0f, + -9.0f, -9.0f, -9.0f}); +#if defined(OPENVINO_CONFIG_GPU_FP16) || defined(OPENVINO_CONFIG_GPU_FP32) + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kOpenVINOExecutionProvider}); // OpenVINO: Temporarily disabled due to accuracy issues +#else + test.Run(); +#endif } TEST(GemmOpTest, GemmAlphaBeta) { @@ -130,11 +163,11 @@ TEST(GemmOpTest, GemmAlphaBeta) { test.AddOutput("Y", {2, 3}, {7.0f, 7.0f, 7.0f, -3.0f, -3.0f, -3.0f}); - #if defined(OPENVINO_CONFIG_GPU_FP16) || defined(OPENVINO_CONFIG_GPU_FP32) - test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider,kOpenVINOExecutionProvider}); // OpenVINO: Temporarily disabled due to accuracy issues - #else - test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: Seg fault in parser - #endif +#if defined(OPENVINO_CONFIG_GPU_FP16) || defined(OPENVINO_CONFIG_GPU_FP32) + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider, kOpenVINOExecutionProvider}); // OpenVINO: Temporarily disabled due to accuracy issues +#else + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: Seg fault in parser +#endif } TEST(GemmOpTest, GemmNaN) { @@ -153,7 +186,7 @@ TEST(GemmOpTest, GemmNaN) { test.AddOutput("Y", {2, 3}, {10.0f, 10.0f, 10.0f, -10.0f, -10.0f, -10.0f}); - test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: Seg fault in parser + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: Seg fault in parser } TEST(GemmOpTest, GemmScalarBroadcast) { @@ -247,7 +280,7 @@ TEST(GemmOpTest, GemmEmptyTensor) { test.AddInput("C", {3}, std::vector(3, 1.0f)); test.AddOutput("Y", {0, 3}, {}); - test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider, kDnnlExecutionProvider}); //TensorRT: doesn't support dynamic shape yet + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider, kDnnlExecutionProvider}); //TensorRT: doesn't support dynamic shape yet } TEST(GemmOpTest, GemmNoBiasOpset11) { diff --git a/onnxruntime/test/providers/cpu/math/quantize_linear_matmul_test.cc b/onnxruntime/test/providers/cpu/math/quantize_linear_matmul_test.cc index 83160f97de..32095a8a15 100644 --- a/onnxruntime/test/providers/cpu/math/quantize_linear_matmul_test.cc +++ b/onnxruntime/test/providers/cpu/math/quantize_linear_matmul_test.cc @@ -12,7 +12,7 @@ TEST(QuantizeLinearMatmulOpTest, QLinearMatMul3D) { test.AddInput("T1", {2, 2, 4}, {208, 236, 0, 238, 3, 214, 255, 29, - + 208, 236, 0, 238, 3, 214, 255, 29}); @@ -21,42 +21,51 @@ TEST(QuantizeLinearMatmulOpTest, QLinearMatMul3D) { test.AddInput("T2", {2, 4, 3}, {152, 51, 244, - 60, 26, 255, - 0, 127, 246, + 60, 26, 255, + 0, 127, 246, 127, 254, 247, 152, 51, 244, 60, 26, 255, 0, 127, 246, 127, 254, 247}); - + test.AddInput("b_scale", {}, {0.00705f}); test.AddInput("b_zero_point", {}, {114}); test.AddInput("y_scale", {}, {0.0107f}); test.AddInput("y_zero_point", {}, {118}); - test.AddOutput("T3", {2, 2, 3}, + test.AddOutput("T3", {2, 2, 3}, {168, 115, 255, 1, 66, 151, 168, 115, 255, 1, 66, 151}); - + + test.Run(); +} + +static void QLinearMatMul2DTest(bool only_t1_not_initializer) { + OpTester test("QLinearMatMul", 10); + test.AddInput("T1", {2, 4}, {208, 236, 0, 238, 3, 214, 255, 29}); + test.AddInput("a_scale", {1}, {0.0066f}, only_t1_not_initializer); + test.AddInput("a_zero_point", {1}, {113}, only_t1_not_initializer); + test.AddInput("T2", {4, 3}, {152, 51, 244, 60, 26, 255, 0, 127, 246, 127, 254, 247}, only_t1_not_initializer); + test.AddInput("b_scale", {1}, {0.00705f}, only_t1_not_initializer); + test.AddInput("b_zero_point", {1}, {114}, only_t1_not_initializer); + test.AddInput("y_scale", {1}, {0.0107f}, only_t1_not_initializer); + test.AddInput("y_zero_point", {1}, {118}, only_t1_not_initializer); + test.AddOutput("T3", {2, 3}, {168, 115, 255, 1, 66, 151}); test.Run(); } TEST(QuantizeLinearMatmulOpTest, QLinearMatMul) { - OpTester test("QLinearMatMul", 10); - test.AddInput("T1", {2, 4}, {208, 236, 0, 238, 3, 214, 255, 29}); - test.AddInput("a_scale", {}, {0.0066f}); - test.AddInput("a_zero_point", {}, {113}); - test.AddInput("T2", {4, 3}, {152, 51, 244, 60, 26, 255, 0, 127, 246, 127, 254, 247}); - test.AddInput("b_scale", {}, {0.00705f}); - test.AddInput("b_zero_point", {}, {114}); - test.AddInput("y_scale", {}, {0.0107f}); - test.AddInput("y_zero_point", {}, {118}); - test.AddOutput("T3", {2, 3}, {168, 115, 255, 1, 66, 151}); - test.Run(); + QLinearMatMul2DTest(false); +} + +// NNAPI EP requires weight to be an initializer +TEST(QuantizeLinearMatmulOpTest, QLinearMatMulAllInputExceptT1AreInitializers) { + QLinearMatMul2DTest(true); } } // namespace test } // namespace onnxruntime diff --git a/onnxruntime/test/providers/cpu/nn/conv_op_test.cc b/onnxruntime/test/providers/cpu/nn/conv_op_test.cc index 30707e8388..9d9fab549f 100644 --- a/onnxruntime/test/providers/cpu/nn/conv_op_test.cc +++ b/onnxruntime/test/providers/cpu/nn/conv_op_test.cc @@ -24,6 +24,7 @@ void TestConvOp(const ConvOpAndTestAttributes& attributes, const vector>& input_shapes, const std::initializer_list& expected_output, const vector& expected_output_shape, + bool weight_is_initializer = false, OpTester::ExpectResult expect_result = OpTester::ExpectResult::kExpectSuccess, const std::string& err_str = "", int opset = 7) { @@ -46,9 +47,11 @@ void TestConvOp(const ConvOpAndTestAttributes& attributes, ORT_ENFORCE(inputs.size() <= 3, "Our name array is only setup to handle 3 inputs"); const char* szNames[] = {"X", "W", "B"}; - for (size_t i = 0; i < inputs.size(); i++) { - test.AddInput(szNames[i], input_shapes[i], inputs[i]); - } + test.AddInput(szNames[0], input_shapes[0], inputs[0]); + test.AddInput(szNames[1], input_shapes[1], inputs[1], weight_is_initializer); + if (inputs.size() == 3) + test.AddInput(szNames[2], input_shapes[2], inputs[2]); + test.AddOutput("Y", expected_output_shape, expected_output); std::unordered_set excluded_providers(attributes.excluded_providers); @@ -199,6 +202,9 @@ TEST(ConvTest, Conv2D_1) { attrs.excluded_providers.insert(kCudaExecutionProvider); // asymmetric padding is not supported by cudnn TestConvOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape); + + // NNAPI EP requires weight to be an initializer + TestConvOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape, true); } TEST(ConvTest, Conv1D_Invalid_Input_Shape) { @@ -216,7 +222,7 @@ TEST(ConvTest, Conv1D_Invalid_Input_Shape) { vector X_shape = {1, 1, 1}; vector dummy_shape = {1, 1, 2}; auto dummy_vals = {0.0f, 0.0f}; - TestConvOp(attrs, {X, dummy_vals}, {X_shape, dummy_shape}, dummy_vals, dummy_shape, + TestConvOp(attrs, {X, dummy_vals}, {X_shape, dummy_shape}, dummy_vals, dummy_shape, false, OpTester::ExpectResult::kExpectFailure, "Node:node1 Output:Y [ShapeInferenceError] Can't merge shape info. " "Both source and target dimension have values but they differ. Source=0 Target=2 Dimension=2", @@ -239,7 +245,7 @@ TEST(ConvTest, Conv2D_Invalid_Input_Shape) { vector dummy_shape = {2, 2, 1, 2}; auto dummy_vals = {-0.0f, 0.0f, -0.0f, -0.0f, -0.0f, 0.0f, -0.0f, -0.0f}; - TestConvOp(attrs, {X, dummy_vals}, {X_shape, dummy_shape}, dummy_vals, dummy_shape, + TestConvOp(attrs, {X, dummy_vals}, {X_shape, dummy_shape}, dummy_vals, dummy_shape, false, OpTester::ExpectResult::kExpectFailure, "Node:node1 Output:Y [ShapeInferenceError] Can't merge shape info. " "Both source and target dimension have values but they differ. Source=1 Target=2 Dimension=0", @@ -289,6 +295,9 @@ TEST(ConvTest, Conv2D_2) { 0.06516310572624207f, -0.015176207758486271f, 0.14682966470718384f, -0.02665453404188156f, -0.18779225647449493f}; TestConvOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape); + + // NNAPI EP requires weight to be an initializer + TestConvOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape, true); } TEST(ConvTest, Conv2D_Bias_1) { @@ -312,6 +321,9 @@ TEST(ConvTest, Conv2D_Bias_1) { auto expected_vals = {13.0f, 17.0f, 25.0f, 29.0f, 11.0f, 15.0f, 23.0f, 27.0f}; TestConvOp(attrs, {X, W, B}, {X_shape, W_shape, B_shape}, expected_vals, Y_shape); + + // NNAPI EP requires weight to be an initializer + TestConvOp(attrs, {X, W, B}, {X_shape, W_shape, B_shape}, expected_vals, Y_shape, true); } // Conv48 @@ -363,6 +375,8 @@ TEST(ConvTest, Conv2D_Bias_2) { attrs.excluded_providers.insert(kCudaExecutionProvider); // asymmetric padding is not supported by cudnn TestConvOp(attrs, {X, W, B}, {X_shape, W_shape, B_shape}, expected_vals, Y_shape); + + TestConvOp(attrs, {X, W, B}, {X_shape, W_shape, B_shape}, expected_vals, Y_shape, true); } TEST(ConvTest, Conv2D_AutoPad1) { @@ -390,6 +404,9 @@ TEST(ConvTest, Conv2D_AutoPad1) { 27.0f, 36.0f, 36.0f, 36.0f, 21.0f, 12.0f, 15.0f, 15.0f, 15.0f, 8.0f}; TestConvOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape); + + // NNAPI EP requires weight to be an initializer + TestConvOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape, true); } TEST(ConvTest, Conv2D_AutoPad2) { @@ -421,6 +438,9 @@ TEST(ConvTest, Conv2D_AutoPad2) { 12.0f, 24.0f, 12.0f, 24.0f, 12.0f, 5.0f, 10.0f, 5.0f, 10.0f, 5.0f}; TestConvOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape); + + // NNAPI EP requires weight to be an initializer + TestConvOp(attrs, {X, W}, {X_shape, W_shape}, expected_vals, Y_shape, true); } // Conv10 @@ -629,7 +649,7 @@ TEST(ConvTest, ConvDimWithZero) { attrs.excluded_providers.insert(kNGraphExecutionProvider); attrs.excluded_providers.insert(kAclExecutionProvider); - TestConvOp(attrs, {X, W}, {X_shape, W_shape}, {}, out_shape, OpTester::ExpectResult::kExpectSuccess, "", 10); + TestConvOp(attrs, {X, W}, {X_shape, W_shape}, {}, out_shape, false, OpTester::ExpectResult::kExpectSuccess, "", 10); } } // namespace test diff --git a/onnxruntime/test/providers/cpu/nn/qlinearconv_op_test.cc b/onnxruntime/test/providers/cpu/nn/qlinearconv_op_test.cc index 04a3994903..3f6678f2f4 100644 --- a/onnxruntime/test/providers/cpu/nn/qlinearconv_op_test.cc +++ b/onnxruntime/test/providers/cpu/nn/qlinearconv_op_test.cc @@ -73,22 +73,22 @@ void TestQLinearConvOp(OpTester& test, const QuantizedBiasTensor* B, const QuantizedTensor& Y, const std::vector& Y_shape, + bool all_input_initializer_except_x = false, const std::unordered_set& excluded_provider_types = {}) { - test.AddInput("x", X_shape, X.quantized_); - test.AddInput("x_scale", {}, {X.scale_}); - test.AddInput("x_zero_point", {}, {X.zero_point_}); + test.AddInput("x_scale", {}, {X.scale_}, all_input_initializer_except_x); + test.AddInput("x_zero_point", {}, {X.zero_point_}, all_input_initializer_except_x); - test.AddInput("w", W_shape, W.quantized_); - test.AddInput("w_scale", {}, {W.scale_}); - test.AddInput("w_zero_point", {}, {W.zero_point_}); + test.AddInput("w", W_shape, W.quantized_, all_input_initializer_except_x); + test.AddInput("w_scale", {}, {W.scale_}, all_input_initializer_except_x); + test.AddInput("w_zero_point", {}, {W.zero_point_}, all_input_initializer_except_x); - test.AddInput("y_scale", {}, {Y.scale_}); - test.AddInput("y_zero_point", {}, {Y.zero_point_}); + test.AddInput("y_scale", {}, {Y.scale_}, all_input_initializer_except_x); + test.AddInput("y_zero_point", {}, {Y.zero_point_}, all_input_initializer_except_x); if (B != nullptr) { const std::vector B_shape{static_cast(B->quantized_.size())}; - test.AddInput("b", B_shape, B->quantized_); + test.AddInput("b", B_shape, B->quantized_, all_input_initializer_except_x); } test.AddOutput("y", Y_shape, Y.quantized_); @@ -96,7 +96,7 @@ void TestQLinearConvOp(OpTester& test, test.Run(OpTester::ExpectResult::kExpectSuccess, "", excluded_provider_types); } -TEST(QLinearConvTest, Conv2DTest) { +void RunConv2DTest(bool all_input_initializer_except_x) { QuantizedTensor X({0.45246148109436035f, 0.15498268604278564f, 0.11199361085891724f, -0.39421093463897705f, 0.2626858949661255f, 0.13414543867111206f, -0.27184486389160156f, -0.43028733134269714f, -0.26825493574142456f, 0.3893144130706787f, -0.13631996512413025f, -0.009590476751327515f, @@ -131,7 +131,16 @@ TEST(QLinearConvTest, Conv2DTest) { X, {1, 1, 7, 7}, W, {1, 1, 1, 1}, nullptr, - Y, {1, 1, 7, 7}); + Y, {1, 1, 7, 7}, + all_input_initializer_except_x); +} + +TEST(QLinearConvTest, Conv2DTest) { + RunConv2DTest(false); +} + +TEST(QLinearConvTest, Conv2DTestAllInputInitializerExceptX) { + RunConv2DTest(true); } TEST(QLinearConvTest, Conv3DTest) { @@ -172,7 +181,7 @@ TEST(QLinearConvTest, Conv3DTest) { Y, {1, 1, 4, 4, 4}); } -TEST(QLinearConvTest, WithBias_2D) { +void RunConv2DWithBiasTest(bool all_input_initializer_except_x) { QuantizedTensor X({6, 81, 214, 151, 234, 42, 50, 89, 30, 91, 125, 141, 52, 31, 58, 224, 84, 251, 67, 137, 223, 119, 79, 220, 249, 75, 131, 246, 113, 56, 54, 197, 110, 142, 126, 171, 53, 228, 240, 83, 229, 218, 185, 9, 80, 116, 176, 193, 175, 253}, @@ -203,9 +212,18 @@ TEST(QLinearConvTest, WithBias_2D) { W, {4, 2, 3, 3}, &B, Y, {1, 4, 5, 5}, + all_input_initializer_except_x, {kNGraphExecutionProvider}); } +TEST(QLinearConvTest, WithBias_2D) { + RunConv2DWithBiasTest(false); +} + +TEST(QLinearConvTest, WithBias_2D_AllInputInitializerExceptX) { + RunConv2DWithBiasTest(true); +} + TEST(QLinearConvTest, WithGroup_2D) { QuantizedTensor X({98, 166, 219, 195, 46, 97, 27, 211, 239, 1, 28, 208, 143, 144, 215, 252, 79, 5, 154, 56, 122, 191, 94, 25, 221, 48, 37, 182, 68, 245, 210, 206, 183, 22, 163, 104, 242, @@ -236,6 +254,7 @@ TEST(QLinearConvTest, WithGroup_2D) { W, {6, 2, 2, 2}, &B, Y, {1, 6, 2, 3}, + false, {kNGraphExecutionProvider}); }