mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-30 20:18:08 +00:00
add data type check before quantizing (#11840)
This commit is contained in:
Yufeng Li
GitHub
parent
607afbe1c0
commit
80d8c4c7ff
9 changed files with 92 additions and 45 deletions
|
|
@ -329,28 +329,6 @@ class ONNXQuantizer:
|
|||
return self.parent.find_initializer_in_path(initializer_name)
|
||||
return False
|
||||
|
||||
def should_quantize(self, node):
|
||||
if (
|
||||
self.nodes_to_quantize is not None
|
||||
and len(self.nodes_to_quantize) != 0
|
||||
and node.name not in self.nodes_to_quantize
|
||||
):
|
||||
return False
|
||||
|
||||
if node.op_type not in self.op_types_to_quantize:
|
||||
return False
|
||||
|
||||
if self.nodes_to_exclude is not None and node.name in self.nodes_to_exclude:
|
||||
return False
|
||||
|
||||
# do not quantize non-constant B matrices for matmul
|
||||
if self.q_matmul_const_b_only:
|
||||
if node.op_type == "MatMul" and (not self.find_initializer_in_path(node.input[1])):
|
||||
print("Ignore MatMul due to non constant B: {}[{}]".format(self.graph_scope, node.name))
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def add_new_nodes(self, nodes):
|
||||
self.new_nodes.extend(nodes)
|
||||
for node in nodes:
|
||||
|
|
@ -370,11 +348,7 @@ class ONNXQuantizer:
|
|||
node = self.quantize_node_with_sub_graph(node)
|
||||
|
||||
number_of_existing_new_nodes = len(self.new_nodes)
|
||||
if self.should_quantize(node):
|
||||
op_quantizer = CreateOpQuantizer(self, node)
|
||||
else:
|
||||
op_quantizer = CreateDefaultOpQuantizer(self, node)
|
||||
|
||||
op_quantizer = CreateOpQuantizer(self, node)
|
||||
op_quantizer.quantize()
|
||||
for i in range(number_of_existing_new_nodes, len(self.new_nodes)):
|
||||
for output_name in self.new_nodes[i].output:
|
||||
|
|
@ -425,6 +399,40 @@ class ONNXQuantizer:
|
|||
return False
|
||||
return self.parent.is_valid_quantize_weight(weight_name)
|
||||
|
||||
def is_float_tensor(self, tensor_name):
|
||||
if self.is_input_a_weight(tensor_name):
|
||||
return self.is_valid_quantize_weight(tensor_name)
|
||||
|
||||
if tensor_name in self.value_infos.keys():
|
||||
vi = self.value_infos[tensor_name]
|
||||
if vi.type.HasField("tensor_type") and vi.type.tensor_type.elem_type == onnx_proto.TensorProto.FLOAT:
|
||||
return True
|
||||
elif self.enable_subgraph_quantization and self.parent:
|
||||
return self.parent.is_float_tensor(tensor_name)
|
||||
else:
|
||||
logging.warning(
|
||||
"Failed to infer data type of tensor: {}. Please add data type info for this tensor "
|
||||
"if your model has customized operators.".format(tensor_name)
|
||||
)
|
||||
|
||||
return False
|
||||
|
||||
def should_quantize_node(self, node):
|
||||
if (
|
||||
self.nodes_to_quantize is not None
|
||||
and len(self.nodes_to_quantize) != 0
|
||||
and node.name not in self.nodes_to_quantize
|
||||
):
|
||||
return False
|
||||
|
||||
if node.op_type not in self.op_types_to_quantize:
|
||||
return False
|
||||
|
||||
if self.nodes_to_exclude is not None and node.name in self.nodes_to_exclude:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def _get_dynamic_input_quantization_params(self, input_name, nodes_list, qType):
|
||||
"""
|
||||
Create nodes for dynamic quantization of input and add them to nodes_list.
|
||||
|
|
@ -1091,7 +1099,9 @@ class ONNXQuantizer:
|
|||
for node in self.model.nodes():
|
||||
if node.op_type not in ["Clip", "Relu"]:
|
||||
continue
|
||||
if not self.should_quantize(node):
|
||||
if self.is_activation_symmetric:
|
||||
continue
|
||||
if not self.should_quantize_node(node):
|
||||
continue
|
||||
if len(self.model.input_name_to_nodes()[node.input[0]]) != 1:
|
||||
continue
|
||||
|
|
|
|||
|
|
@ -13,6 +13,9 @@ class AttentionQuant(QuantOperatorBase):
|
|||
def __init__(self, onnx_quantizer, onnx_node):
|
||||
super().__init__(onnx_quantizer, onnx_node)
|
||||
|
||||
def should_quantize(self):
|
||||
return self.quantizer.should_quantize_node(self.node)
|
||||
|
||||
def quantize(self):
|
||||
"""
|
||||
parameter node: Attention node.
|
||||
|
|
|
|||
|
|
@ -3,6 +3,12 @@ class QuantOperatorBase:
|
|||
self.quantizer = onnx_quantizer
|
||||
self.node = onnx_node
|
||||
|
||||
def should_quantize(self):
|
||||
if not self.quantizer.should_quantize_node(self.node):
|
||||
return False
|
||||
|
||||
return self.quantizer.is_float_tensor(self.node.input[0])
|
||||
|
||||
def quantize(self):
|
||||
"""
|
||||
Given a node which does not support quantization, this method checks whether the input to
|
||||
|
|
@ -11,8 +17,7 @@ class QuantOperatorBase:
|
|||
parameter new_nodes_list: List of new nodes created before processing current node
|
||||
return: List of new nodes created
|
||||
"""
|
||||
nodes = []
|
||||
for index, node_input in enumerate(self.node.input):
|
||||
for _, node_input in enumerate(self.node.input):
|
||||
dequantize_node = self.quantizer._dequantize_value(node_input)
|
||||
if dequantize_node is not None:
|
||||
self.quantizer.new_nodes.append(dequantize_node)
|
||||
|
|
|
|||
|
|
@ -18,6 +18,9 @@ class EmbedLayerNormalizationQuant(QuantOperatorBase):
|
|||
def __init__(self, onnx_quantizer, onnx_node):
|
||||
super().__init__(onnx_quantizer, onnx_node)
|
||||
|
||||
def should_quantize(self):
|
||||
return self.quantizer.should_quantize_node(self.node)
|
||||
|
||||
def quantize(self):
|
||||
node = self.node
|
||||
assert node.op_type == "EmbedLayerNormalization"
|
||||
|
|
|
|||
|
|
@ -13,12 +13,15 @@ class GatherQuant(QuantOperatorBase):
|
|||
def __init__(self, onnx_quantizer, onnx_node):
|
||||
super().__init__(onnx_quantizer, onnx_node)
|
||||
|
||||
def should_quantize(self):
|
||||
if not self.quantizer.should_quantize_node(self.node):
|
||||
return False
|
||||
|
||||
return self.quantizer.is_valid_quantize_weight(self.node.input[0])
|
||||
|
||||
def quantize(self):
|
||||
node = self.node
|
||||
assert node.op_type == "Gather"
|
||||
if not self.quantizer.is_valid_quantize_weight(node.input[0]):
|
||||
super().quantize()
|
||||
return
|
||||
|
||||
(
|
||||
quantized_input_names,
|
||||
|
|
|
|||
|
|
@ -6,6 +6,7 @@ from onnx import onnx_pb as onnx_proto
|
|||
|
||||
from ..quant_utils import QuantizedValue, QuantizedValueType, attribute_to_kwarg, find_by_name, get_mul_node, ms_domain
|
||||
from .base_operator import QuantOperatorBase
|
||||
from .matmul import QOpMatMul
|
||||
from .qdq_base_operator import QDQOperatorBase
|
||||
|
||||
|
||||
|
|
@ -33,7 +34,7 @@ def set_default_beta(gemm_node):
|
|||
return 1.0
|
||||
|
||||
|
||||
class QLinearGemm(QuantOperatorBase):
|
||||
class QLinearGemm(QOpMatMul):
|
||||
def __init__(self, onnx_quantizer, onnx_node):
|
||||
super().__init__(onnx_quantizer, onnx_node)
|
||||
|
||||
|
|
|
|||
|
|
@ -7,12 +7,34 @@ from ..quant_utils import QuantizedValue, QuantizedValueType, find_by_name, get_
|
|||
from .base_operator import QuantOperatorBase
|
||||
from .qdq_base_operator import QDQOperatorBase
|
||||
|
||||
|
||||
class QOpMatMul(QuantOperatorBase):
|
||||
def __init__(self, onnx_quantizer, onnx_node):
|
||||
super().__init__(onnx_quantizer, onnx_node)
|
||||
|
||||
def should_quantize(self):
|
||||
if not self.quantizer.should_quantize_node(self.node):
|
||||
return False
|
||||
|
||||
if (not self.quantizer.is_float_tensor(self.node.input[1])) and (
|
||||
not self.quantizer.is_float_tensor(self.node.input[0])
|
||||
):
|
||||
return False
|
||||
|
||||
# do not quantize non-constant B matrices for matmul
|
||||
if self.quantizer.q_matmul_const_b_only:
|
||||
if not self.quantizer.find_initializer_in_path(self.node.input[1]):
|
||||
print("Ignore MatMul due to non constant B: {}[{}]".format(self.quantizer.graph_scope, self.node.name))
|
||||
return False
|
||||
return True
|
||||
|
||||
|
||||
"""
|
||||
Used when quantize mode is QuantizationMode.IntegerOps.
|
||||
"""
|
||||
|
||||
|
||||
class MatMulInteger(QuantOperatorBase):
|
||||
class MatMulInteger(QOpMatMul):
|
||||
def __init__(self, onnx_quantizer, onnx_node):
|
||||
super().__init__(onnx_quantizer, onnx_node)
|
||||
|
||||
|
|
@ -83,7 +105,7 @@ class MatMulInteger(QuantOperatorBase):
|
|||
"""
|
||||
|
||||
|
||||
class QLinearMatMul(QuantOperatorBase):
|
||||
class QLinearMatMul(QOpMatMul):
|
||||
def __init__(self, onnx_quantizer, onnx_node):
|
||||
super().__init__(onnx_quantizer, onnx_node)
|
||||
|
||||
|
|
|
|||
|
|
@ -151,19 +151,17 @@ class QDQQuantizer(ONNXQuantizer):
|
|||
self.model.remove_nodes(self.nodes_to_remove)
|
||||
|
||||
def quantize_model(self):
|
||||
if self.dedicated_qdq_pair:
|
||||
for node in self.model.nodes():
|
||||
if self.should_quantize(node):
|
||||
for node in self.model.nodes():
|
||||
if self.should_quantize_node(node):
|
||||
op_quantizer = CreateQDQQuantizer(self, node)
|
||||
op_quantizer.quantize()
|
||||
|
||||
if self.dedicated_qdq_pair:
|
||||
for tensor_name in node.input:
|
||||
if tensor_name not in self.tensor_to_its_receiving_nodes:
|
||||
self.tensor_to_its_receiving_nodes[tensor_name] = []
|
||||
self.tensor_to_its_receiving_nodes[tensor_name].append(node)
|
||||
|
||||
for node in self.model.nodes():
|
||||
if self.should_quantize(node):
|
||||
op_quantizer = CreateQDQQuantizer(self, node)
|
||||
op_quantizer.quantize()
|
||||
|
||||
self.quantize_tensors()
|
||||
self.quantize_weights_per_channel()
|
||||
self.quantize_bias_tensors()
|
||||
|
|
|
|||
|
|
@ -82,7 +82,9 @@ def CreateDefaultOpQuantizer(onnx_quantizer, node):
|
|||
def CreateOpQuantizer(onnx_quantizer, node):
|
||||
registry = IntegerOpsRegistry if onnx_quantizer.mode == QuantizationMode.IntegerOps else QLinearOpsRegistry
|
||||
if node.op_type in registry.keys():
|
||||
return registry[node.op_type](onnx_quantizer, node)
|
||||
op_quantizer = registry[node.op_type](onnx_quantizer, node)
|
||||
if op_quantizer.should_quantize():
|
||||
return op_quantizer
|
||||
return QuantOperatorBase(onnx_quantizer, node)
|
||||
|
||||
|
||||
|
|
|
|||
Loading…
Reference in a new issue