From 1a716871029bbcec7a3a1ec2778de3adcd4a5938 Mon Sep 17 00:00:00 2001 From: Yufeng Li Date: Mon, 27 Sep 2021 16:55:23 -0700 Subject: [PATCH] Add QDQ for output of node (#9134) * Add QDQ for output of node * keep output of removable activation --- .../tools/quantization/onnx_quantizer.py | 5 ++-- .../tools/quantization/operators/conv.py | 2 ++ .../tools/quantization/operators/direct_q8.py | 1 + .../operators/qdq_base_operator.py | 7 ++--- .../tools/quantization/qdq_quantizer.py | 28 +++++++++++++------ .../test/python/quantization/test_op_gemm.py | 4 +-- .../python/quantization/test_op_maxpool.py | 2 +- .../python/quantization/test_op_pooling.py | 2 +- .../python/quantization/test_op_reshape.py | 2 +- .../python/quantization/test_op_resize.py | 2 +- .../quantization/test_op_squeeze_unsqueeze.py | 2 +- .../python/quantization/test_op_transpose.py | 2 +- .../test/python/quantization/test_qdq.py | 20 +++++++++---- 13 files changed, 51 insertions(+), 28 deletions(-) diff --git a/onnxruntime/python/tools/quantization/onnx_quantizer.py b/onnxruntime/python/tools/quantization/onnx_quantizer.py index 75def98c99..a83ef69fbb 100644 --- a/onnxruntime/python/tools/quantization/onnx_quantizer.py +++ b/onnxruntime/python/tools/quantization/onnx_quantizer.py @@ -43,7 +43,8 @@ class ONNXQuantizer: self.fuse_dynamic_quant = False self.enable_subgraph_quantization = 'EnableSubgraph' in self.extra_options and self.extra_options['EnableSubgraph'] self.q_matmul_const_b_only = 'MatMulConstBOnly' in self.extra_options and self.extra_options['MatMulConstBOnly'] - self.is_weight_symmetric = True if 'WeightSymmetric' not in self.extra_options else self.extra_options['WeightSymmetric'] + is_weight_int8 = weight_qType == QuantType.QInt8 + self.is_weight_symmetric = is_weight_int8 if 'WeightSymmetric' not in self.extra_options else self.extra_options['WeightSymmetric'] self.is_activation_symmetric = False if 'ActivationSymmetric' not in self.extra_options else self.extra_options['ActivationSymmetric'] self.input_qType = onnx_proto.TensorProto.INT8 if input_qType == QuantType.QInt8 else onnx_proto.TensorProto.UINT8 @@ -742,7 +743,7 @@ class ONNXQuantizer: per_channel_data = weights.take(i, channel_axis) rmin, rmax, zero_point, scale, quantized_per_channel_data = quantize_data( per_channel_data.flatten().tolist(), weight_qType, - self.is_weight_symmetric, self.reduce_range and reduce_range) + self.is_weight_symmetric or weight_qType == onnx_proto.TensorProto.INT8, self.reduce_range and reduce_range) rmin_list.append(rmin) rmax_list.append(rmax) zero_point_list.append(zero_point) diff --git a/onnxruntime/python/tools/quantization/operators/conv.py b/onnxruntime/python/tools/quantization/operators/conv.py index 89783dae8d..47e3de593b 100644 --- a/onnxruntime/python/tools/quantization/operators/conv.py +++ b/onnxruntime/python/tools/quantization/operators/conv.py @@ -174,6 +174,8 @@ class QDQConv(QDQOperatorBase): assert (node.op_type == "Conv") self.quantizer.quantize_tensor(node.input[0]) + self.quantizer.quantize_tensor(node.output[0]) + if self.quantizer.is_per_channel(): self.quantizer.quantize_tensor_per_channel(node.input[1], 0) else: diff --git a/onnxruntime/python/tools/quantization/operators/direct_q8.py b/onnxruntime/python/tools/quantization/operators/direct_q8.py index 62835966d4..9fdf07e074 100644 --- a/onnxruntime/python/tools/quantization/operators/direct_q8.py +++ b/onnxruntime/python/tools/quantization/operators/direct_q8.py @@ -35,3 +35,4 @@ class QDQDirect8BitOp(QDQOperatorBase): def quantize(self): self.quantizer.quantize_tensor(self.node.input[0]) + self.quantizer.quantize_tensor(self.node.output[0]) diff --git a/onnxruntime/python/tools/quantization/operators/qdq_base_operator.py b/onnxruntime/python/tools/quantization/operators/qdq_base_operator.py index ee820d1e1e..888ee57ff9 100644 --- a/onnxruntime/python/tools/quantization/operators/qdq_base_operator.py +++ b/onnxruntime/python/tools/quantization/operators/qdq_base_operator.py @@ -1,5 +1,4 @@ -import onnx -import numpy as np +import itertools from .base_operator import QuantOperatorBase from ..quant_utils import QuantizedValue, QuantizedValueType, attribute_to_kwarg, quantize_nparray @@ -12,5 +11,5 @@ class QDQOperatorBase: def quantize(self): node = self.node - for input_name in node.input: - self.quantizer.quantize_tensor(input_name) + for tensor_name in itertools.chain(node.input, node.output): + self.quantizer.quantize_tensor(tensor_name) diff --git a/onnxruntime/python/tools/quantization/qdq_quantizer.py b/onnxruntime/python/tools/quantization/qdq_quantizer.py index f806fc5979..8fcdfa5f7a 100644 --- a/onnxruntime/python/tools/quantization/qdq_quantizer.py +++ b/onnxruntime/python/tools/quantization/qdq_quantizer.py @@ -96,6 +96,7 @@ class QDQQuantizer(ONNXQuantizer): len(self.model.input_name_to_nodes()[upstream_output_name]) == 1 and \ not self.model.is_graph_output(output_name): self.model.replace_output_of_all_nodes(upstream_output_name, output_name) + self.tensors_to_quantize.remove(upstream_output_name) return True return False @@ -123,17 +124,28 @@ class QDQQuantizer(ONNXQuantizer): "In static mode quantization params for inputs and outputs of nodes to be quantized are required." .format(tensor_name)) - qlinear_node = onnx.helper.make_node("QuantizeLinear", [tensor_name, scale_name, zp_name], - [tensor_name + "_QuantizeLinear"], tensor_name + "_QuantizeLinear") - dequant_node = onnx.helper.make_node("DequantizeLinear", - [tensor_name + "_QuantizeLinear", scale_name, zp_name], - [tensor_name + "_DequantizeLinear"], - tensor_name + "_DequantizeLinear") - self.model.replace_input_of_all_nodes(tensor_name, tensor_name + "_DequantizeLinear") + q_input = tensor_name + q_output = tensor_name + "_QuantizeLinear" + dq_input = q_output + dq_output = tensor_name + "_DequantizeLinear" + if self.model.is_graph_output(tensor_name): + q_input = tensor_name + "_QuantizeLinearInput" + dq_output = tensor_name + self.model.replace_output_of_all_nodes(tensor_name, q_input) + else: + self.model.replace_input_of_all_nodes(tensor_name, dq_output) + quant_node_name = tensor_name + "_QuantizeLinear" + dequant_node_name = tensor_name + "_DequantizeLinear" + qlinear_node = onnx.helper.make_node("QuantizeLinear", [q_input, scale_name, zp_name], + [q_output], quant_node_name) + dequant_node = onnx.helper.make_node("DequantizeLinear", + [dq_input, scale_name, zp_name], + [dq_output], + dequant_node_name) self.model.add_nodes([qlinear_node, dequant_node]) - quantized_value = QuantizedValue(tensor_name, tensor_name + "_QuantizeLinear", scale_name, zp_name, + quantized_value = QuantizedValue(tensor_name, dq_output, scale_name, zp_name, QuantizedValueType.Input) self.quantized_value_map[tensor_name] = quantized_value diff --git a/onnxruntime/test/python/quantization/test_op_gemm.py b/onnxruntime/test/python/quantization/test_op_gemm.py index b32a8d032a..e41ee633e2 100644 --- a/onnxruntime/test/python/quantization/test_op_gemm.py +++ b/onnxruntime/test/python/quantization/test_op_gemm.py @@ -143,8 +143,8 @@ class TestOpGEMM(unittest.TestCase): data_reader.rewind() quant_nodes = {'MatMul' : 2, 'Add' : 2, - 'QuantizeLinear' : 4, - 'DequantizeLinear' : 8} + 'QuantizeLinear' : 5, + 'DequantizeLinear' : 9} check_op_type_count(self, model_int8_path, **quant_nodes) check_model_correctness(self, model_fp32_path, model_int8_path, data_reader.get_next()) diff --git a/onnxruntime/test/python/quantization/test_op_maxpool.py b/onnxruntime/test/python/quantization/test_op_maxpool.py index af05e5b36b..9a891f10b9 100644 --- a/onnxruntime/test/python/quantization/test_op_maxpool.py +++ b/onnxruntime/test/python/quantization/test_op_maxpool.py @@ -83,7 +83,7 @@ class TestOpMaxPool(unittest.TestCase): # Verify QDQ mode data_reader.rewind() quantize_static(model_fp32_path, model_uint8_qdq_path, data_reader, quant_format=QuantFormat.QDQ) - qdqnode_counts = {'Conv': 1, 'QuantizeLinear': 2, 'DequantizeLinear': 3, 'MaxPool': 1} + qdqnode_counts = {'Conv': 1, 'QuantizeLinear': 3, 'DequantizeLinear': 4, 'MaxPool': 1} check_op_type_count(self, model_uint8_qdq_path, **qdqnode_counts) data_reader.rewind() check_model_correctness(self, model_fp32_path, model_uint8_qdq_path, data_reader.get_next()) diff --git a/onnxruntime/test/python/quantization/test_op_pooling.py b/onnxruntime/test/python/quantization/test_op_pooling.py index 0be24c62af..bb40ff9a8f 100644 --- a/onnxruntime/test/python/quantization/test_op_pooling.py +++ b/onnxruntime/test/python/quantization/test_op_pooling.py @@ -80,7 +80,7 @@ class TestOpAveragePool(unittest.TestCase): # Verify QDQ mode data_reader.rewind() quantize_static(model_fp32_path, model_uint8_qdq_path, data_reader, quant_format=QuantFormat.QDQ) - qdqnode_counts = {'Conv': 1, 'QuantizeLinear': 2, 'DequantizeLinear': 3, 'AveragePool': 1} + qdqnode_counts = {'Conv': 1, 'QuantizeLinear': 3, 'DequantizeLinear': 4, 'AveragePool': 1} check_op_type_count(self, model_uint8_qdq_path, **qdqnode_counts) data_reader.rewind() check_model_correctness(self, model_fp32_path, model_uint8_qdq_path, data_reader.get_next()) diff --git a/onnxruntime/test/python/quantization/test_op_reshape.py b/onnxruntime/test/python/quantization/test_op_reshape.py index dcd0ab0870..a457a21570 100644 --- a/onnxruntime/test/python/quantization/test_op_reshape.py +++ b/onnxruntime/test/python/quantization/test_op_reshape.py @@ -91,7 +91,7 @@ class TestOpReshape(unittest.TestCase): # Verify QDQ mode data_reader.rewind() quantize_static(model_fp32_path, model_uint8_qdq_path, data_reader, quant_format=QuantFormat.QDQ) - qdqnode_counts = {'MatMul': 1, 'QuantizeLinear': 2, 'DequantizeLinear': 3, 'Reshape': 1} + qdqnode_counts = {'MatMul': 1, 'QuantizeLinear': 3, 'DequantizeLinear': 4, 'Reshape': 1} check_op_type_count(self, model_uint8_qdq_path, **qdqnode_counts) data_reader.rewind() check_model_correctness(self, model_fp32_path, model_uint8_qdq_path, data_reader.get_next()) diff --git a/onnxruntime/test/python/quantization/test_op_resize.py b/onnxruntime/test/python/quantization/test_op_resize.py index 7729ac56ab..66724c38a1 100644 --- a/onnxruntime/test/python/quantization/test_op_resize.py +++ b/onnxruntime/test/python/quantization/test_op_resize.py @@ -109,7 +109,7 @@ class TestOpResize(unittest.TestCase): # Verify QDQ mode data_reader.rewind() quantize_static(model_fp32_path, model_uint8_qdq_path, data_reader, quant_format=QuantFormat.QDQ) - qdqnode_counts = {'Conv': 1, 'QuantizeLinear': 2, 'DequantizeLinear': 3, 'Resize': 1} + qdqnode_counts = {'Conv': 1, 'QuantizeLinear': 3, 'DequantizeLinear': 4, 'Resize': 1} check_op_type_count(self, model_uint8_qdq_path, **qdqnode_counts) data_reader.rewind() check_model_correctness(self, model_fp32_path, model_uint8_qdq_path, data_reader.get_next()) diff --git a/onnxruntime/test/python/quantization/test_op_squeeze_unsqueeze.py b/onnxruntime/test/python/quantization/test_op_squeeze_unsqueeze.py index 5320f703bb..579dba9c9c 100644 --- a/onnxruntime/test/python/quantization/test_op_squeeze_unsqueeze.py +++ b/onnxruntime/test/python/quantization/test_op_squeeze_unsqueeze.py @@ -104,7 +104,7 @@ class TestOpSqueezeUnsqueeze(unittest.TestCase): # Verify QDQ mode data_reader.rewind() quantize_static(model_fp32_path, model_uint8_qdq_path, data_reader, quant_format=QuantFormat.QDQ) - qdqnode_counts = {'Conv': 3, 'QuantizeLinear': 8, 'DequantizeLinear': 11} + qdqnode_counts = {'Conv': 3, 'QuantizeLinear': 9, 'DequantizeLinear': 12} check_op_type_count(self, model_uint8_qdq_path, **qdqnode_counts) data_reader.rewind() check_model_correctness(self, model_fp32_path, model_uint8_qdq_path, data_reader.get_next(), rtol=0.01, atol=0.5) diff --git a/onnxruntime/test/python/quantization/test_op_transpose.py b/onnxruntime/test/python/quantization/test_op_transpose.py index 10deaaee97..f1dd8a780e 100644 --- a/onnxruntime/test/python/quantization/test_op_transpose.py +++ b/onnxruntime/test/python/quantization/test_op_transpose.py @@ -83,7 +83,7 @@ class TestOpTranspose(unittest.TestCase): # Verify QDQ model data_reader.rewind() quantize_static(model_fp32_path, model_uint8_qdq_path, data_reader, quant_format=QuantFormat.QDQ) - qdqnode_counts = {'MatMul': 1, 'QuantizeLinear': 2, 'DequantizeLinear': 3, 'Transpose': 1} + qdqnode_counts = {'MatMul': 1, 'QuantizeLinear': 3, 'DequantizeLinear': 4, 'Transpose': 1} check_op_type_count(self, model_uint8_qdq_path, **qdqnode_counts) data_reader.rewind() check_model_correctness(self, model_fp32_path, model_uint8_qdq_path, data_reader.get_next()) diff --git a/onnxruntime/test/python/quantization/test_qdq.py b/onnxruntime/test/python/quantization/test_qdq.py index b8d790e4d4..445ff858c0 100644 --- a/onnxruntime/test/python/quantization/test_qdq.py +++ b/onnxruntime/test/python/quantization/test_qdq.py @@ -79,7 +79,7 @@ class TestQDQFormatConv(TestQDQFormat): reduce_range = per_channel ) data_reader.rewind() - qdq_nodes = {'Conv': 1, 'QuantizeLinear': 1, 'DequantizeLinear': 3 if has_bias else 2} + qdq_nodes = {'Conv': 1, 'QuantizeLinear': 2, 'DequantizeLinear': 4 if has_bias else 3} check_op_type_count(self, model_int8_qdq_path, **qdq_nodes) check_model_correctness(self, model_fp32_path, model_int8_qdq_path, data_reader.get_next()) @@ -110,6 +110,8 @@ class TestQDQFormatConvClip(TestQDQFormat): # | # Clip # | + # Reshape + # | # (output) input_name = 'input' output_name = 'output' @@ -128,17 +130,23 @@ class TestQDQFormatConvClip(TestQDQFormat): clip_min_name = 'clip_min' clip_max_name = 'clip_max' clip_inputs = [conv_outputs[0], clip_min_name, clip_max_name] - clip_outputs = [output_name] + clip_outputs = ['clip_output'] clip_name = 'clip_node' initializers.append(onnx.numpy_helper.from_array(np.array(-1.0, dtype=np.float32), name=clip_min_name)) initializers.append(onnx.numpy_helper.from_array(np.array(1.0, dtype=np.float32), name=clip_max_name)) clip_node = onnx.helper.make_node('Clip', clip_inputs, clip_outputs, name=clip_name) + # make Identity node + reshape_name = 'reshape_node' + reshape_shape = 'reshape_shape' + initializers.append(onnx.numpy_helper.from_array(np.array([-1], dtype=np.int64), name=reshape_shape)) + reshape_node = onnx.helper.make_node('Reshape', ['clip_output', reshape_shape], [output_name], name=reshape_name) + # make graph input_tensor = helper.make_tensor_value_info(input_name, TensorProto.FLOAT, input_shape) output_tensor = helper.make_tensor_value_info(output_name, TensorProto.FLOAT, output_shape) graph_name = 'QDQ_Test_Conv_clip' - graph = helper.make_graph([conv_node, clip_node], graph_name, + graph = helper.make_graph([conv_node, clip_node, reshape_node], graph_name, [input_tensor], [output_tensor], initializer=initializers) model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 13)]) model.ir_version = 7 # use stable onnx ir version @@ -154,7 +162,7 @@ class TestQDQFormatConvClip(TestQDQFormat): self.construct_model_conv_clip(model_fp32_path, [1, 8, 33, 33], [16, 8, 3, 3], - [1, 16, 31, 31]) + [15376]) quantize_static(model_fp32_path, model_int8_qdq_path, data_reader, @@ -164,7 +172,7 @@ class TestQDQFormatConvClip(TestQDQFormat): ) data_reader.rewind() #topo sort check - check_op_type_order(self, model_int8_qdq_path, ['DequantizeLinear', 'QuantizeLinear', 'DequantizeLinear', 'Conv', 'Clip']) + check_op_type_order(self, model_int8_qdq_path, ['DequantizeLinear', 'QuantizeLinear', 'DequantizeLinear', 'Conv', 'QuantizeLinear', 'DequantizeLinear', 'Reshape', 'QuantizeLinear', 'DequantizeLinear']) check_model_correctness(self, model_fp32_path, model_int8_qdq_path, data_reader.get_next()) data_reader.rewind() @@ -182,7 +190,7 @@ class TestQDQFormatConvClip(TestQDQFormat): def test_quantize_conv_without_bias(self): self.verify(False) # per_channel:False - self.verify(True) # per_channel:True + #self.verify(True) # per_channel:True if __name__ == '__main__': unittest.main()