From f6f457aa5734e095726bdec1e46eb6f427c5fb1d Mon Sep 17 00:00:00 2001 From: Yufeng Li Date: Tue, 7 Jun 2022 18:02:31 -0700 Subject: [PATCH] not remove relu/clip for symmetric activation (#11696) * not remove relu/clip for symmetric activation --- .../quantization/operators/activation.py | 10 +- .../python/quantization/test_op_attention.py | 113 +++++++++ .../test/python/quantization/test_op_gemm.py | 98 +------- .../test/python/quantization/test_op_relu.py | 221 ++++++++++++++++++ 4 files changed, 352 insertions(+), 90 deletions(-) create mode 100644 onnxruntime/test/python/quantization/test_op_attention.py create mode 100644 onnxruntime/test/python/quantization/test_op_relu.py diff --git a/onnxruntime/python/tools/quantization/operators/activation.py b/onnxruntime/python/tools/quantization/operators/activation.py index ec2c2298e7..f224ef8dae 100644 --- a/onnxruntime/python/tools/quantization/operators/activation.py +++ b/onnxruntime/python/tools/quantization/operators/activation.py @@ -17,9 +17,9 @@ class QLinearActivation(QuantOperatorBase): # When mode is QLinearOps, the output quantization params are calculated based on outputs from # activation nodes, therefore these nodes can be removed from the graph if they follow a quantized op. # If input to this node is not quantized then keep this node - if node.input[0] not in self.quantizer.quantized_value_map: - self.quantizer.new_nodes += [node] - return + # If activation is symmetric, not quantize the op and simply return + if node.input[0] not in self.quantizer.quantized_value_map or self.quantizer.is_activation_symmetric: + return super().quantize() quantized_value = self.quantizer.quantized_value_map[node.input[0]] self.quantizer.quantized_value_map[node.output[0]] = quantized_value @@ -103,7 +103,9 @@ class QDQRemovableActivation(QDQOperatorBase): def quantize(self): node = self.node - if self.quantizer.try_replacing_upstream_output(node.input[0], node.output[0]): + if not self.quantizer.is_activation_symmetric and self.quantizer.try_replacing_upstream_output( + node.input[0], node.output[0] + ): self.quantizer.remove_node(self.node) else: self.quantizer.quantize_tensor(node.input[0]) diff --git a/onnxruntime/test/python/quantization/test_op_attention.py b/onnxruntime/test/python/quantization/test_op_attention.py new file mode 100644 index 0000000000..47bc342aad --- /dev/null +++ b/onnxruntime/test/python/quantization/test_op_attention.py @@ -0,0 +1,113 @@ +#!/usr/bin/env python +# coding: utf-8 +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +import unittest + +import numpy as np +import onnx +from onnx import TensorProto, helper +from op_test_utils import TestDataFeeds, check_model_correctness, check_op_type_count + +from onnxruntime.quantization import quantize_dynamic + + +class TestOpAttention(unittest.TestCase): + def input_feeds(self, n, name2shape): + input_data_list = [] + for i in range(n): + inputs = {} + for name, shape in name2shape.items(): + inputs.update({name: np.random.randint(-1, 2, shape).astype(np.float32)}) + input_data_list.extend([inputs]) + dr = TestDataFeeds(input_data_list) + return dr + + def construct_model_attention_and_matmul(self, output_model_path): + # (input) + # | + # Attention + # | + # MatMul + # | + # (output) + input_name = "input" + output_name = "output" + initializers = [] + + def make_attention_node(input_name, weight_shape, weight_name, bias_shape, bias_name, output_name): + weight_data = np.random.normal(0, 0.1, weight_shape).astype(np.float32) + initializers.append(onnx.numpy_helper.from_array(weight_data, name=weight_name)) + + bias_data = np.random.normal(0, 0.1, bias_shape).astype(np.float32) + initializers.append(onnx.numpy_helper.from_array(bias_data, name=bias_name)) + + return onnx.helper.make_node("Attention", [input_name, weight_name, bias_name], [output_name]) + + def make_matmul_node(input_name, weight_shape, weight_name, output_name): + weight_data = np.random.normal(0, 0.1, weight_shape).astype(np.float32) + initializers.append(onnx.numpy_helper.from_array(weight_data, name=weight_name)) + + return onnx.helper.make_node("MatMul", [input_name, weight_name], [output_name]) + + # make attention node + attention_output_name = "attention_output" + attention_node = make_attention_node( + input_name, [10, 30], "qkv.weight", [30], "qkv.bias", attention_output_name + ) + attention_node.domain = "com.microsoft" + attention_node.attribute.extend([helper.make_attribute("num_heads", 5)]) + + # make matmul node + matmul_node = make_matmul_node(attention_output_name, [10, 10], "matmul.weight", output_name) + + # make graph + input_tensor = helper.make_tensor_value_info(input_name, TensorProto.FLOAT, [1, -1, 10]) + output_tensor = helper.make_tensor_value_info(output_name, TensorProto.FLOAT, [1, -1, 10]) + graph_name = "attention_test" + graph = helper.make_graph( + [attention_node, matmul_node], + graph_name, + [input_tensor], + [output_tensor], + initializer=initializers, + ) + model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 13)]) + model.ir_version = onnx.IR_VERSION + + onnx.save(model, output_model_path) + + def dynamic_attention_quant_test(self, model_fp32_path, model_int8_path, per_channel, reduce_range): + quantize_dynamic( + model_fp32_path, + model_int8_path, + per_channel=per_channel, + reduce_range=reduce_range, + ) + quant_nodes = {"QAttention": 1, "MatMulInteger": 1} + check_op_type_count(self, model_int8_path, **quant_nodes) + check_model_correctness( + self, + model_fp32_path, + model_int8_path, + {"input": np.random.rand(1, 5, 10).astype(np.float32)}, + ) + + def test_quantize_attention(self): + np.random.seed(1) + model_fp32_path = "attention_fp32.onnx" + model_int8_path = "attention_fp32.quant.onnx" + self.construct_model_attention_and_matmul(model_fp32_path) + + self.dynamic_attention_quant_test(model_fp32_path, model_int8_path, True, True) + self.dynamic_attention_quant_test(model_fp32_path, model_int8_path, True, False) + self.dynamic_attention_quant_test(model_fp32_path, model_int8_path, False, True) + self.dynamic_attention_quant_test(model_fp32_path, model_int8_path, False, False) + + +if __name__ == "__main__": + unittest.main() diff --git a/onnxruntime/test/python/quantization/test_op_gemm.py b/onnxruntime/test/python/quantization/test_op_gemm.py index 692e4ebe9f..7475ffb470 100644 --- a/onnxruntime/test/python/quantization/test_op_gemm.py +++ b/onnxruntime/test/python/quantization/test_op_gemm.py @@ -16,7 +16,7 @@ from op_test_utils import TestDataFeeds, check_model_correctness, check_op_type_ from onnxruntime.quantization import QuantFormat, QuantType, quantize_dynamic, quantize_static -class TestOpGEMM(unittest.TestCase): +class TestOpGemm(unittest.TestCase): def input_feeds(self, n, name2shape): input_data_list = [] for i in range(n): @@ -30,11 +30,11 @@ class TestOpGEMM(unittest.TestCase): def construct_model_gemm(self, output_model_path): # (input) # | - # GEMM + # Gemm # | # Clip # | - # GEMM + # Gemm # | # (output) input_name = "input" @@ -104,60 +104,6 @@ class TestOpGEMM(unittest.TestCase): onnx.save(model, output_model_path) - def construct_model_attention_and_matmul(self, output_model_path): - # (input) - # | - # Attention - # | - # MatMul - # | - # (output) - input_name = "input" - output_name = "output" - initializers = [] - - def make_attention_node(input_name, weight_shape, weight_name, bias_shape, bias_name, output_name): - weight_data = np.random.normal(0, 0.1, weight_shape).astype(np.float32) - initializers.append(onnx.numpy_helper.from_array(weight_data, name=weight_name)) - - bias_data = np.random.normal(0, 0.1, bias_shape).astype(np.float32) - initializers.append(onnx.numpy_helper.from_array(bias_data, name=bias_name)) - - return onnx.helper.make_node("Attention", [input_name, weight_name, bias_name], [output_name]) - - def make_matmul_node(input_name, weight_shape, weight_name, output_name): - weight_data = np.random.normal(0, 0.1, weight_shape).astype(np.float32) - initializers.append(onnx.numpy_helper.from_array(weight_data, name=weight_name)) - - return onnx.helper.make_node("MatMul", [input_name, weight_name], [output_name]) - - # make attention node - attention_output_name = "attention_output" - attention_node = make_attention_node( - input_name, [10, 30], "qkv.weight", [30], "qkv.bias", attention_output_name - ) - attention_node.domain = "com.microsoft" - attention_node.attribute.extend([helper.make_attribute("num_heads", 5)]) - - # make matmul node - matmul_node = make_matmul_node(attention_output_name, [10, 10], "matmul.weight", output_name) - - # make graph - input_tensor = helper.make_tensor_value_info(input_name, TensorProto.FLOAT, [1, -1, 10]) - output_tensor = helper.make_tensor_value_info(output_name, TensorProto.FLOAT, [1, -1, 10]) - graph_name = "attention_test" - graph = helper.make_graph( - [attention_node, matmul_node], - graph_name, - [input_tensor], - [output_tensor], - initializer=initializers, - ) - model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 13)]) - model.ir_version = onnx.IR_VERSION - - onnx.save(model, output_model_path) - def static_quant_test( self, model_fp32_path, @@ -181,7 +127,10 @@ class TestOpGEMM(unittest.TestCase): weight_type=weight_type, extra_options=extra_options, ) - quant_nodes = {"QGemm": 2, "QuantizeLinear": 1, "DequantizeLinear": 1} + + qdq_count = 1 if activation_type == QuantType.QUInt8 else 2 + clip_count = 0 if activation_type == QuantType.QUInt8 else 1 + quant_nodes = {"QGemm": 2, "QuantizeLinear": qdq_count, "DequantizeLinear": qdq_count, "Clip": clip_count} check_op_type_count(self, model_int8_path, **quant_nodes) qnode_io_qtypes = { "QuantizeLinear": [ @@ -217,7 +166,11 @@ class TestOpGEMM(unittest.TestCase): weight_type=weight_type, extra_options=extra_options, ) - quant_nodes = {"Gemm": 2, "QuantizeLinear": 3, "DequantizeLinear": 7} + + clip_count = 0 if activation_type == QuantType.QUInt8 else 1 + q_count = 3 if activation_type == QuantType.QUInt8 else 4 + dq_count = 7 if activation_type == QuantType.QUInt8 else 8 + quant_nodes = {"Gemm": 2, "QuantizeLinear": q_count, "DequantizeLinear": dq_count, "Clip": clip_count} check_op_type_count(self, model_int8_path, **quant_nodes) qnode_io_qtypes = { "QuantizeLinear": [ @@ -260,22 +213,6 @@ class TestOpGEMM(unittest.TestCase): {"input": np.random.rand(5, 10).astype(np.float32)}, ) - def dynamic_attention_quant_test(self, model_fp32_path, model_int8_path, per_channel, reduce_range): - quantize_dynamic( - model_fp32_path, - model_int8_path, - per_channel=per_channel, - reduce_range=reduce_range, - ) - quant_nodes = {"QAttention": 1, "MatMulInteger": 1} - check_op_type_count(self, model_int8_path, **quant_nodes) - check_model_correctness( - self, - model_fp32_path, - model_int8_path, - {"input": np.random.rand(1, 5, 10).astype(np.float32)}, - ) - def test_quantize_gemm(self): np.random.seed(1) model_fp32_path = "gemm_fp32.onnx" @@ -326,17 +263,6 @@ class TestOpGEMM(unittest.TestCase): # self.dynamic_quant_test(model_fp32_path, data_reader, activation_type=QuantType.QInt8, weight_type=QuantType.QInt8, # extra_options={'ActivationSymmetric': True}) - def test_quantize_attention(self): - np.random.seed(1) - model_fp32_path = "attention_fp32.onnx" - model_int8_path = "attention_fp32.quant.onnx" - self.construct_model_attention_and_matmul(model_fp32_path) - - self.dynamic_attention_quant_test(model_fp32_path, model_int8_path, True, True) - self.dynamic_attention_quant_test(model_fp32_path, model_int8_path, True, False) - self.dynamic_attention_quant_test(model_fp32_path, model_int8_path, False, True) - self.dynamic_attention_quant_test(model_fp32_path, model_int8_path, False, False) - if __name__ == "__main__": unittest.main() diff --git a/onnxruntime/test/python/quantization/test_op_relu.py b/onnxruntime/test/python/quantization/test_op_relu.py new file mode 100644 index 0000000000..bf72b96508 --- /dev/null +++ b/onnxruntime/test/python/quantization/test_op_relu.py @@ -0,0 +1,221 @@ +#!/usr/bin/env python +# coding: utf-8 +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +import unittest + +import numpy as np +import onnx +from onnx import TensorProto, helper +from op_test_utils import TestDataFeeds, check_model_correctness, check_op_type_count, check_qtype_by_node_type + +from onnxruntime.quantization import QuantFormat, QuantType, quantize_static + + +class TestOpRelu(unittest.TestCase): + def input_feeds(self, n, name2shape): + input_data_list = [] + for i in range(n): + inputs = {} + for name, shape in name2shape.items(): + inputs.update({name: np.random.randint(-1, 2, shape).astype(np.float32)}) + input_data_list.extend([inputs]) + dr = TestDataFeeds(input_data_list) + return dr + + def construct_model_gemm(self, output_model_path): + # (input) + # | + # Gemm + # | + # Relu + # | + # Gemm + # | + # (output) + input_name = "input" + output_name = "output" + initializers = [] + + def make_gemm(input_name, weight_shape, weight_name, bias_shape, bias_name, output_name): + weight_data = np.random.normal(0, 0.1, weight_shape).astype(np.float32) + initializers.append(onnx.numpy_helper.from_array(weight_data, name=weight_name)) + + bias_data = np.random.normal(0, 0.1, bias_shape).astype(np.float32) + initializers.append(onnx.numpy_helper.from_array(bias_data, name=bias_name)) + + return onnx.helper.make_node( + "Gemm", + [input_name, weight_name, bias_name], + [output_name], + alpha=1.0, + beta=1.0, + transB=1, + ) + + # make gemm1 node + gemm1_output_name = "gemm1_output" + gemm1_node = make_gemm( + input_name, + [100, 10], + "linear1.weight", + [100], + "linear1.bias", + gemm1_output_name, + ) + + # make Relu + relu_output = "relu_output" + relu_node = onnx.helper.make_node("Relu", [gemm1_output_name], [relu_output]) + + # make gemm2 node + gemm2_node = make_gemm( + relu_output, + [10, 100], + "linear2.weight", + [10], + "linear2.bias", + output_name, + ) + + # make graph + input_tensor = helper.make_tensor_value_info(input_name, TensorProto.FLOAT, [-1, 10]) + output_tensor = helper.make_tensor_value_info(output_name, TensorProto.FLOAT, [-1, 10]) + graph_name = "relu_test" + graph = helper.make_graph( + [gemm1_node, relu_node, gemm2_node], + graph_name, + [input_tensor], + [output_tensor], + initializer=initializers, + ) + model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 13)]) + model.ir_version = onnx.IR_VERSION + + onnx.save(model, output_model_path) + + def static_quant_test( + self, + model_fp32_path, + data_reader, + activation_type, + weight_type, + extra_options={}, + ): + activation_proto_qtype = TensorProto.UINT8 if activation_type == QuantType.QUInt8 else TensorProto.INT8 + activation_type_str = "u8" if (activation_type == QuantType.QUInt8) else "s8" + weight_type_str = "u8" if (weight_type == QuantType.QUInt8) else "s8" + model_int8_path = "relu_fp32.quant_{}{}.onnx".format(activation_type_str, weight_type_str) + + data_reader.rewind() + quantize_static( + model_fp32_path, + model_int8_path, + data_reader, + quant_format=QuantFormat.QOperator, + activation_type=activation_type, + weight_type=weight_type, + extra_options=extra_options, + ) + + qdq_count = 1 if activation_type == QuantType.QUInt8 else 2 + relu_count = 0 if activation_type == QuantType.QUInt8 else 1 + quant_nodes = {"QGemm": 2, "QuantizeLinear": qdq_count, "DequantizeLinear": qdq_count, "Relu": relu_count} + check_op_type_count(self, model_int8_path, **quant_nodes) + qnode_io_qtypes = { + "QuantizeLinear": [ + ["i", 2, activation_proto_qtype], + ["o", 0, activation_proto_qtype], + ] + } + qnode_io_qtypes.update({"DequantizeLinear": [["i", 2, activation_proto_qtype]]}) + check_qtype_by_node_type(self, model_int8_path, qnode_io_qtypes) + data_reader.rewind() + check_model_correctness(self, model_fp32_path, model_int8_path, data_reader.get_next()) + + def static_quant_test_qdq( + self, + model_fp32_path, + data_reader, + activation_type, + weight_type, + extra_options={}, + ): + activation_proto_qtype = TensorProto.UINT8 if activation_type == QuantType.QUInt8 else TensorProto.INT8 + activation_type_str = "u8" if (activation_type == QuantType.QUInt8) else "s8" + weight_type_str = "u8" if (weight_type == QuantType.QUInt8) else "s8" + model_int8_path = "relu_fp32.quant_dqd_{}{}.onnx".format(activation_type_str, weight_type_str) + + data_reader.rewind() + quantize_static( + model_fp32_path, + model_int8_path, + data_reader, + quant_format=QuantFormat.QDQ, + activation_type=activation_type, + weight_type=weight_type, + extra_options=extra_options, + ) + + relu_count = 0 if activation_type == QuantType.QUInt8 else 1 + q_count = 3 if activation_type == QuantType.QUInt8 else 4 + dq_count = 7 if activation_type == QuantType.QUInt8 else 8 + quant_nodes = {"Gemm": 2, "QuantizeLinear": q_count, "DequantizeLinear": dq_count, "Relu": relu_count} + check_op_type_count(self, model_int8_path, **quant_nodes) + qnode_io_qtypes = { + "QuantizeLinear": [ + ["i", 2, activation_proto_qtype], + ["o", 0, activation_proto_qtype], + ] + } + check_qtype_by_node_type(self, model_int8_path, qnode_io_qtypes) + data_reader.rewind() + check_model_correctness(self, model_fp32_path, model_int8_path, data_reader.get_next()) + + def test_quantize_gemm(self): + np.random.seed(1) + model_fp32_path = "relu_fp32.onnx" + self.construct_model_gemm(model_fp32_path) + data_reader = self.input_feeds(1, {"input": [5, 10]}) + + self.static_quant_test( + model_fp32_path, + data_reader, + activation_type=QuantType.QUInt8, + weight_type=QuantType.QUInt8, + ) + self.static_quant_test_qdq( + model_fp32_path, + data_reader, + activation_type=QuantType.QUInt8, + weight_type=QuantType.QUInt8, + ) + + def test_quantize_relu_s8s8(self): + np.random.seed(1) + model_fp32_path = "relu_fp32.onnx" + self.construct_model_gemm(model_fp32_path) + data_reader = self.input_feeds(1, {"input": [5, 10]}) + + self.static_quant_test( + model_fp32_path, + data_reader, + activation_type=QuantType.QInt8, + weight_type=QuantType.QInt8, + extra_options={"ActivationSymmetric": True}, + ) + self.static_quant_test_qdq( + model_fp32_path, + data_reader, + activation_type=QuantType.QInt8, + weight_type=QuantType.QInt8, + extra_options={"ActivationSymmetric": True}, + ) + + +if __name__ == "__main__": + unittest.main()