saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-29 20:14:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add QDQ for output of node (#9134)

Browse source 
* Add QDQ for output of node

* keep output of removable activation
This commit is contained in:
Yufeng Li 2021-09-27 16:55:23 -07:00 committed by GitHub
parent f16bb37fae
commit 1a71687102
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
13 changed files with 51 additions and 28 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

5
onnxruntime/python/tools/quantization/onnx_quantizer.py
View file

@ -43,7 +43,8 @@ class ONNXQuantizer:
self.fuse_dynamic_quant = False
self.enable_subgraph_quantization = 'EnableSubgraph' in self.extra_options and self.extra_options['EnableSubgraph']
self.q_matmul_const_b_only = 'MatMulConstBOnly' in self.extra_options and self.extra_options['MatMulConstBOnly']
self.is_weight_symmetric = True if 'WeightSymmetric' not in self.extra_options else self.extra_options['WeightSymmetric']
is_weight_int8 = weight_qType == QuantType.QInt8
self.is_weight_symmetric = is_weight_int8 if 'WeightSymmetric' not in self.extra_options else self.extra_options['WeightSymmetric']
self.is_activation_symmetric = False if 'ActivationSymmetric' not in self.extra_options else self.extra_options['ActivationSymmetric']
self.input_qType = onnx_proto.TensorProto.INT8 if input_qType == QuantType.QInt8 else onnx_proto.TensorProto.UINT8
@ -742,7 +743,7 @@ class ONNXQuantizer:
per_channel_data = weights.take(i, channel_axis)
rmin, rmax, zero_point, scale, quantized_per_channel_data = quantize_data(
per_channel_data.flatten().tolist(), weight_qType,
self.is_weight_symmetric, self.reduce_range and reduce_range)
self.is_weight_symmetric or weight_qType == onnx_proto.TensorProto.INT8, self.reduce_range and reduce_range)
rmin_list.append(rmin)
rmax_list.append(rmax)
zero_point_list.append(zero_point)

2
onnxruntime/python/tools/quantization/operators/conv.py
View file

@ -174,6 +174,8 @@ class QDQConv(QDQOperatorBase):
assert (node.op_type == "Conv")
self.quantizer.quantize_tensor(node.input[0])
self.quantizer.quantize_tensor(node.output[0])
if self.quantizer.is_per_channel():
self.quantizer.quantize_tensor_per_channel(node.input[1], 0)
else:

1
onnxruntime/python/tools/quantization/operators/direct_q8.py
View file

@ -35,3 +35,4 @@ class QDQDirect8BitOp(QDQOperatorBase):
def quantize(self):
self.quantizer.quantize_tensor(self.node.input[0])
self.quantizer.quantize_tensor(self.node.output[0])

7
onnxruntime/python/tools/quantization/operators/qdq_base_operator.py
View file

@ -1,5 +1,4 @@
import onnx
import numpy as np
import itertools
from .base_operator import QuantOperatorBase
from ..quant_utils import QuantizedValue, QuantizedValueType, attribute_to_kwarg, quantize_nparray
@ -12,5 +11,5 @@ class QDQOperatorBase:
def quantize(self):
node = self.node
for input_name in node.input:
self.quantizer.quantize_tensor(input_name)
for tensor_name in itertools.chain(node.input, node.output):
self.quantizer.quantize_tensor(tensor_name)

28
onnxruntime/python/tools/quantization/qdq_quantizer.py
View file

@ -96,6 +96,7 @@ class QDQQuantizer(ONNXQuantizer):
len(self.model.input_name_to_nodes()[upstream_output_name]) == 1 and \
not self.model.is_graph_output(output_name):
self.model.replace_output_of_all_nodes(upstream_output_name, output_name)
self.tensors_to_quantize.remove(upstream_output_name)
return True
return False
@ -123,17 +124,28 @@ class QDQQuantizer(ONNXQuantizer):
"In static mode quantization params for inputs and outputs of nodes to be quantized are required."
.format(tensor_name))
qlinear_node = onnx.helper.make_node("QuantizeLinear", [tensor_name, scale_name, zp_name],
[tensor_name + "_QuantizeLinear"], tensor_name + "_QuantizeLinear")
dequant_node = onnx.helper.make_node("DequantizeLinear",
[tensor_name + "_QuantizeLinear", scale_name, zp_name],
[tensor_name + "_DequantizeLinear"],
tensor_name + "_DequantizeLinear")
self.model.replace_input_of_all_nodes(tensor_name, tensor_name + "_DequantizeLinear")
q_input = tensor_name
q_output = tensor_name + "_QuantizeLinear"
dq_input = q_output
dq_output = tensor_name + "_DequantizeLinear"
if self.model.is_graph_output(tensor_name):
q_input = tensor_name + "_QuantizeLinearInput"
dq_output = tensor_name
self.model.replace_output_of_all_nodes(tensor_name, q_input)
else:
self.model.replace_input_of_all_nodes(tensor_name, dq_output)
quant_node_name = tensor_name + "_QuantizeLinear"
dequant_node_name = tensor_name + "_DequantizeLinear"
qlinear_node = onnx.helper.make_node("QuantizeLinear", [q_input, scale_name, zp_name],
[q_output], quant_node_name)
dequant_node = onnx.helper.make_node("DequantizeLinear",
[dq_input, scale_name, zp_name],
[dq_output],
dequant_node_name)
self.model.add_nodes([qlinear_node, dequant_node])
quantized_value = QuantizedValue(tensor_name, tensor_name + "_QuantizeLinear", scale_name, zp_name,
quantized_value = QuantizedValue(tensor_name, dq_output, scale_name, zp_name,
QuantizedValueType.Input)
self.quantized_value_map[tensor_name] = quantized_value

4
onnxruntime/test/python/quantization/test_op_gemm.py
View file

@ -143,8 +143,8 @@ class TestOpGEMM(unittest.TestCase):
data_reader.rewind()
quant_nodes = {'MatMul' : 2,
'Add' : 2,
'QuantizeLinear' : 4,
'DequantizeLinear' : 8}
'QuantizeLinear' : 5,
'DequantizeLinear' : 9}
check_op_type_count(self, model_int8_path, **quant_nodes)
check_model_correctness(self, model_fp32_path, model_int8_path, data_reader.get_next())

2
onnxruntime/test/python/quantization/test_op_maxpool.py
View file

@ -83,7 +83,7 @@ class TestOpMaxPool(unittest.TestCase):
# Verify QDQ mode
data_reader.rewind()
quantize_static(model_fp32_path, model_uint8_qdq_path, data_reader, quant_format=QuantFormat.QDQ)
qdqnode_counts = {'Conv': 1, 'QuantizeLinear': 2, 'DequantizeLinear': 3, 'MaxPool': 1}
qdqnode_counts = {'Conv': 1, 'QuantizeLinear': 3, 'DequantizeLinear': 4, 'MaxPool': 1}
check_op_type_count(self, model_uint8_qdq_path, **qdqnode_counts)
data_reader.rewind()
check_model_correctness(self, model_fp32_path, model_uint8_qdq_path, data_reader.get_next())

2
onnxruntime/test/python/quantization/test_op_pooling.py
View file

@ -80,7 +80,7 @@ class TestOpAveragePool(unittest.TestCase):
# Verify QDQ mode
data_reader.rewind()
quantize_static(model_fp32_path, model_uint8_qdq_path, data_reader, quant_format=QuantFormat.QDQ)
qdqnode_counts = {'Conv': 1, 'QuantizeLinear': 2, 'DequantizeLinear': 3, 'AveragePool': 1}
qdqnode_counts = {'Conv': 1, 'QuantizeLinear': 3, 'DequantizeLinear': 4, 'AveragePool': 1}
check_op_type_count(self, model_uint8_qdq_path, **qdqnode_counts)
data_reader.rewind()
check_model_correctness(self, model_fp32_path, model_uint8_qdq_path, data_reader.get_next())

2
onnxruntime/test/python/quantization/test_op_reshape.py
View file

@ -91,7 +91,7 @@ class TestOpReshape(unittest.TestCase):
# Verify QDQ mode
data_reader.rewind()
quantize_static(model_fp32_path, model_uint8_qdq_path, data_reader, quant_format=QuantFormat.QDQ)
qdqnode_counts = {'MatMul': 1, 'QuantizeLinear': 2, 'DequantizeLinear': 3, 'Reshape': 1}
qdqnode_counts = {'MatMul': 1, 'QuantizeLinear': 3, 'DequantizeLinear': 4, 'Reshape': 1}
check_op_type_count(self, model_uint8_qdq_path, **qdqnode_counts)
data_reader.rewind()
check_model_correctness(self, model_fp32_path, model_uint8_qdq_path, data_reader.get_next())

2
onnxruntime/test/python/quantization/test_op_resize.py
View file

@ -109,7 +109,7 @@ class TestOpResize(unittest.TestCase):
# Verify QDQ mode
data_reader.rewind()
quantize_static(model_fp32_path, model_uint8_qdq_path, data_reader, quant_format=QuantFormat.QDQ)
qdqnode_counts = {'Conv': 1, 'QuantizeLinear': 2, 'DequantizeLinear': 3, 'Resize': 1}
qdqnode_counts = {'Conv': 1, 'QuantizeLinear': 3, 'DequantizeLinear': 4, 'Resize': 1}
check_op_type_count(self, model_uint8_qdq_path, **qdqnode_counts)
data_reader.rewind()
check_model_correctness(self, model_fp32_path, model_uint8_qdq_path, data_reader.get_next())

2
onnxruntime/test/python/quantization/test_op_squeeze_unsqueeze.py
View file

@ -104,7 +104,7 @@ class TestOpSqueezeUnsqueeze(unittest.TestCase):
# Verify QDQ mode
data_reader.rewind()
quantize_static(model_fp32_path, model_uint8_qdq_path, data_reader, quant_format=QuantFormat.QDQ)
qdqnode_counts = {'Conv': 3, 'QuantizeLinear': 8, 'DequantizeLinear': 11}
qdqnode_counts = {'Conv': 3, 'QuantizeLinear': 9, 'DequantizeLinear': 12}
check_op_type_count(self, model_uint8_qdq_path, **qdqnode_counts)
data_reader.rewind()
check_model_correctness(self, model_fp32_path, model_uint8_qdq_path, data_reader.get_next(), rtol=0.01, atol=0.5)

2
onnxruntime/test/python/quantization/test_op_transpose.py
View file

@ -83,7 +83,7 @@ class TestOpTranspose(unittest.TestCase):
# Verify QDQ model
data_reader.rewind()
quantize_static(model_fp32_path, model_uint8_qdq_path, data_reader, quant_format=QuantFormat.QDQ)
qdqnode_counts = {'MatMul': 1, 'QuantizeLinear': 2, 'DequantizeLinear': 3, 'Transpose': 1}
qdqnode_counts = {'MatMul': 1, 'QuantizeLinear': 3, 'DequantizeLinear': 4, 'Transpose': 1}
check_op_type_count(self, model_uint8_qdq_path, **qdqnode_counts)
data_reader.rewind()
check_model_correctness(self, model_fp32_path, model_uint8_qdq_path, data_reader.get_next())

20
onnxruntime/test/python/quantization/test_qdq.py
View file

@ -79,7 +79,7 @@ class TestQDQFormatConv(TestQDQFormat):
reduce_range = per_channel
)
data_reader.rewind()
qdq_nodes = {'Conv': 1, 'QuantizeLinear': 1, 'DequantizeLinear': 3 if has_bias else 2}
qdq_nodes = {'Conv': 1, 'QuantizeLinear': 2, 'DequantizeLinear': 4 if has_bias else 3}
check_op_type_count(self, model_int8_qdq_path, **qdq_nodes)
check_model_correctness(self, model_fp32_path, model_int8_qdq_path, data_reader.get_next())
@ -110,6 +110,8 @@ class TestQDQFormatConvClip(TestQDQFormat):
# |
# Clip
# |
# Reshape
# |
# (output)
input_name = 'input'
output_name = 'output'
@ -128,17 +130,23 @@ class TestQDQFormatConvClip(TestQDQFormat):
clip_min_name = 'clip_min'
clip_max_name = 'clip_max'
clip_inputs = [conv_outputs[0], clip_min_name, clip_max_name]
clip_outputs = [output_name]
clip_outputs = ['clip_output']
clip_name = 'clip_node'
initializers.append(onnx.numpy_helper.from_array(np.array(-1.0, dtype=np.float32), name=clip_min_name))
initializers.append(onnx.numpy_helper.from_array(np.array(1.0, dtype=np.float32), name=clip_max_name))
clip_node = onnx.helper.make_node('Clip', clip_inputs, clip_outputs, name=clip_name)
# make Identity node
reshape_name = 'reshape_node'
reshape_shape = 'reshape_shape'
initializers.append(onnx.numpy_helper.from_array(np.array([-1], dtype=np.int64), name=reshape_shape))
reshape_node = onnx.helper.make_node('Reshape', ['clip_output', reshape_shape], [output_name], name=reshape_name)
# make graph
input_tensor = helper.make_tensor_value_info(input_name, TensorProto.FLOAT, input_shape)
output_tensor = helper.make_tensor_value_info(output_name, TensorProto.FLOAT, output_shape)
graph_name = 'QDQ_Test_Conv_clip'
graph = helper.make_graph([conv_node, clip_node], graph_name,
graph = helper.make_graph([conv_node, clip_node, reshape_node], graph_name,
[input_tensor], [output_tensor], initializer=initializers)
model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 13)])
model.ir_version = 7 # use stable onnx ir version
@ -154,7 +162,7 @@ class TestQDQFormatConvClip(TestQDQFormat):
self.construct_model_conv_clip(model_fp32_path,
[1, 8, 33, 33],
[16, 8, 3, 3],
[1, 16, 31, 31])
[15376])
quantize_static(model_fp32_path,
model_int8_qdq_path,
data_reader,
@ -164,7 +172,7 @@ class TestQDQFormatConvClip(TestQDQFormat):
)
data_reader.rewind()
#topo sort check
check_op_type_order(self, model_int8_qdq_path, ['DequantizeLinear', 'QuantizeLinear', 'DequantizeLinear', 'Conv', 'Clip'])
check_op_type_order(self, model_int8_qdq_path, ['DequantizeLinear', 'QuantizeLinear', 'DequantizeLinear', 'Conv', 'QuantizeLinear', 'DequantizeLinear', 'Reshape', 'QuantizeLinear', 'DequantizeLinear'])
check_model_correctness(self, model_fp32_path, model_int8_qdq_path, data_reader.get_next())
data_reader.rewind()
@ -182,7 +190,7 @@ class TestQDQFormatConvClip(TestQDQFormat):
def test_quantize_conv_without_bias(self):
self.verify(False) # per_channel:False
self.verify(True) # per_channel:True
#self.verify(True) # per_channel:True
if __name__ == '__main__':
unittest.main()