From eb36258df4764a73d6af5b82c05f24124775cd99 Mon Sep 17 00:00:00 2001 From: SilvanK4t1qbit <48225758+SilvanK4t1qbit@users.noreply.github.com> Date: Thu, 24 Jun 2021 14:42:22 -0700 Subject: [PATCH] Enable signed int8 data type for activations in static quantization (#7029) * Add support for signed int8 static activation quantization. Make symmetrization in quantization switcheable --- .../tools/quantization/onnx_quantizer.py | 27 ++-- .../python/tools/quantization/quant_utils.py | 80 +++++++---- .../python/tools/quantization/quantize.py | 9 +- .../quantization/test_symmetric_flag.py | 132 ++++++++++++++++++ 4 files changed, 200 insertions(+), 48 deletions(-) create mode 100644 onnxruntime/test/python/quantization/test_symmetric_flag.py diff --git a/onnxruntime/python/tools/quantization/onnx_quantizer.py b/onnxruntime/python/tools/quantization/onnx_quantizer.py index c42398d6c5..98116c748d 100644 --- a/onnxruntime/python/tools/quantization/onnx_quantizer.py +++ b/onnxruntime/python/tools/quantization/onnx_quantizer.py @@ -16,14 +16,13 @@ from onnxruntime import SessionOptions, InferenceSession, GraphOptimizationLevel from .quant_utils import QuantizationMode, QuantizedValueType, QuantizedInitializer, QuantizedValue from .quant_utils import find_by_name, get_elem_index, get_mul_node, generate_identified_filename, attribute_to_kwarg, type_to_name -from .quant_utils import quantize_nparray, quantize_data, compute_scale_zp, get_qrange_for_qType +from .quant_utils import quantize_nparray, quantize_data, compute_scale_zp, get_qrange_for_qType, get_qmin_qmax_for_qType from .quant_utils import QuantType, onnx_domain, __producer__, __version__ from .registry import CreateOpQuantizer, CreateDefaultOpQuantizer from .onnx_model import ONNXModel - class ONNXQuantizer: def __init__(self, model, per_channel, reduce_range, mode, static, weight_qType, input_qType, tensors_range, nodes_to_quantize, nodes_to_exclude, op_types_to_quantize, extra_options={}): @@ -42,7 +41,8 @@ class ONNXQuantizer: self.fuse_dynamic_quant = False self.extra_options = extra_options if extra_options is not None else {} self.q_matmul_const_b_only = 'MatMulConstBOnly' in self.extra_options and self.extra_options['MatMulConstBOnly'] - self.is_weight_symmetric = 'WeightSymmetric' not in self.extra_options or self.extra_options['WeightSymmetric'] + self.is_weight_symmetric = True if 'WeightSymmetric' not in self.extra_options else self.extra_options['WeightSymmetric'] + self.is_activation_symmetric = False if 'ActivationSymmetric' not in self.extra_options else self.extra_options['ActivationSymmetric'] self.input_qType = onnx_proto.TensorProto.INT8 if input_qType == QuantType.QInt8 else onnx_proto.TensorProto.UINT8 self.weight_qType = onnx_proto.TensorProto.INT8 if weight_qType == QuantType.QInt8 else onnx_proto.TensorProto.UINT8 @@ -605,8 +605,8 @@ class ONNXQuantizer: # Update packed weight, zero point, and scale initializers weight_data = self.tensor_proto_to_array(weight) _, _, zero_point, scale, q_weight_data = quantize_data(weight_data.flatten().tolist(), - get_qrange_for_qType(qType, self.reduce_range and reduce_range), - qType, self.is_weight_symmetric) + qType, self.is_weight_symmetric, + self.reduce_range and reduce_range) q_weight_data = np.asarray(q_weight_data, dtype=onnx.mapping.TENSOR_TYPE_TO_NP_TYPE[qType]).reshape(weight.dims) q_weight_initializer = onnx.numpy_helper.from_array(q_weight_data, q_weight_name) @@ -641,9 +641,8 @@ class ONNXQuantizer: for i in range(channel_count): per_channel_data = weights.take(i, channel_axis) rmin, rmax, zero_point, scale, quantized_per_channel_data = quantize_data( - per_channel_data.flatten().tolist(), - get_qrange_for_qType(weight_qType, self.reduce_range and reduce_range), - weight_qType, self.is_weight_symmetric) + per_channel_data.flatten().tolist(), weight_qType, + self.is_weight_symmetric, self.reduce_range and reduce_range) rmin_list.append(rmin) rmax_list.append(rmax) zero_point_list.append(zero_point) @@ -735,15 +734,11 @@ class ONNXQuantizer: quantization_params = {} for tensor_name in self.tensors_range.keys(): rmin, rmax = self.tensors_range[tensor_name] + qmin, qmax = get_qmin_qmax_for_qType(self.input_qType) - # adjust rmin and rmax such that 0 is included in the range. This is required - # to make sure zero can be uniquely represented. - rmin = min(rmin, 0) - rmax = max(rmax, 0) - - quantization_params[tensor_name] = compute_scale_zp(rmin, rmax, self.input_qType, - get_qrange_for_qType(self.input_qType), - self.is_weight_symmetric) + quantization_params[tensor_name] = compute_scale_zp(rmin, rmax, + qmin, qmax, + self.is_activation_symmetric) return quantization_params diff --git a/onnxruntime/python/tools/quantization/quant_utils.py b/onnxruntime/python/tools/quantization/quant_utils.py index f52527dffc..24b10c9f13 100644 --- a/onnxruntime/python/tools/quantization/quant_utils.py +++ b/onnxruntime/python/tools/quantization/quant_utils.py @@ -109,29 +109,46 @@ def quantize_nparray(qType, arr, scale, zero_point, low=None, high=None): return arr_fp32.astype(dtype) -def compute_scale_zp(rmin, rmax, qType, quantize_range, symmetric): - if qType == onnx_proto.TensorProto.INT8: - if symmetric: - max_range = max(abs(rmin), abs(rmax)) - scale = (float(max_range) * 2) / quantize_range if max_range > 0 else 1.0 - zero_point = 0 - else: - max_range = float(rmax) - float(rmin) - scale = float(max_range) / quantize_range if max_range > 0 else 1.0 - zero_point = round((quantize_range / 2) - rmax / scale) - elif qType == onnx_proto.TensorProto.UINT8: - scale = (float(rmax) - rmin) / quantize_range if rmin != rmax else 1 - zero_point = round((0 - rmin) / scale) # round to nearest integer - else: - raise ValueError("Unexpected data type {} requested. Only INT8 and UINT8 are supported.".format(qType)) +def compute_scale_zp(rmin, rmax, qmin, qmax, symmetric=False): + ''' + Calculate the scale s and zero point z for the quantization relation + r = s(q-z), where r are the original values and q are the corresponding + quantized values. + + r and z are calculated such that every value within [rmin,rmax] has an + approximate representation within [qmin,qmax]. In addition, qmin <= z <= + qmax is enforced. If the symmetric flag is set to True, the interval + [rmin,rmax] is symmetrized to [-absmax, +absmax], where + absmax = max(abs(rmin), abs(rmax)). + + :parameter rmin: minimum value of r + :parameter rmax: maximum value of r + :parameter qmin: minimum value representable by the target quantization data type + :parameter qmax: maximum value representable by the target quantization data type + :return: zero and scale [z, s] + + ''' + + # Adjust rmin and rmax such that 0 is included in the range. This is + # required to make sure zero can be represented by the quantization data + # type (i.e. to make sure qmin <= zero_point <= qmax) + rmin = min(rmin, 0) + rmax = max(rmax, 0) + + if symmetric: + absmax = max(abs(rmin), abs(rmax)) + rmin = -absmax + rmax = +absmax + + scale = (rmax - rmin) / float(qmax-qmin) if rmax!=rmin else 1.0 + zero_point = round(qmin - rmin/scale) return [zero_point, scale] -def quantize_data(data, quantize_range, qType, symmetric=True): +def quantize_data(data, qType, symmetric, reduce_range=False): ''' :parameter data: data to quantize - :parameter quantize_range: list of data to weight pack. :parameter qType: data type to quantize to. Supported types UINT8 and INT8 :parameter symmetric: whether symmetric quantization is used or not. This is applied to INT8. :return: minimum, maximum, zero point, scale, and quantized weights @@ -146,14 +163,28 @@ def quantize_data(data, quantize_range, qType, symmetric=True): S: scale z: zero point ''' - rmin = min(min(data), 0.) - rmax = max(max(data), 0.) + rmin = min(data) + rmax = max(data) + qmin, qmax = get_qmin_qmax_for_qType(qType, reduce_range) - zero_point, scale = compute_scale_zp(rmin, rmax, qType, quantize_range, symmetric) + zero_point, scale = compute_scale_zp(rmin, rmax, qmin, qmax, symmetric) quantized_data = quantize_nparray(qType, numpy.asarray(data), scale, zero_point) return rmin, rmax, zero_point, scale, quantized_data +def get_qmin_qmax_for_qType(qType, reduce_range=False): + ''' + Return qmin and qmax, the minimum and maximum value representable by the given qType + :parameter qType: onnx.onnx_pb.TensorProto.UINT8 or onnx.onnx_pb.TensorProto.UINT8 + :return: qmin, qmax + ''' + if qType == onnx_proto.TensorProto.UINT8: + (qmin, qmax) = (0,127) if reduce_range else (0,255) + elif qType == onnx_proto.TensorProto.INT8: + (qmin, qmax) = (-64,64) if reduce_range else (-127,127) + else: + raise ValueError("Unexpected data type {} requested. Only INT8 and UINT8 are supported.".format(qType)) + return qmin, qmax def get_qrange_for_qType(qType, reduce_range=False): ''' @@ -161,13 +192,8 @@ def get_qrange_for_qType(qType, reduce_range=False): parameter qType: quantization type. return: quantization range. ''' - if qType == onnx_proto.TensorProto.UINT8: - return 127 if reduce_range else 255 - elif qType == onnx_proto.TensorProto.INT8: - return 128 if reduce_range else 254 # [-64, 64] for reduce_range, and [-127, 127] full_range. - else: - raise ValueError('unsupported quantization data type') - + qmin, qmax = get_qmin_qmax_for_qType(qType, reduce_range) + return qmax - qmin class QuantizedInitializer: ''' diff --git a/onnxruntime/python/tools/quantization/quantize.py b/onnxruntime/python/tools/quantization/quantize.py index f66d6e060f..5b88be1317 100644 --- a/onnxruntime/python/tools/quantization/quantize.py +++ b/onnxruntime/python/tools/quantization/quantize.py @@ -175,18 +175,17 @@ def quantize_static(model_input, List of nodes names to exclude. The nodes in this list will be excluded from quantization when it is not None. :param optimize_model: optimize model before quantization. - :parma use_external_data_format: option used for large size (>2GB) model. Set to False by default. + :param use_external_data_format: option used for large size (>2GB) model. Set to False by default. :param calibrate_method: Current calibration methods supported are MinMax and Entropy. Please use CalibrationMethod.MinMax or CalibrationMethod.Entropy as options. :param extra_options: key value pair dictionary for various options in different case. Current used: - extra.Sigmoid.nnapi = True (Default is False) + extra.Sigmoid.nnapi = True/False (Default is False) + ActivationSymmetric = True/False: symmetrize calibration data for activations (default is False). + WeightSymmetric = True/False: symmetrize calibration data for weights (default is True). ''' - if activation_type != QuantType.QUInt8: - raise ValueError("Static quantization only support uint8 for activation now.") - mode = QuantizationMode.QLinearOps if not op_types_to_quantize or len(op_types_to_quantize) == 0: diff --git a/onnxruntime/test/python/quantization/test_symmetric_flag.py b/onnxruntime/test/python/quantization/test_symmetric_flag.py new file mode 100644 index 0000000000..0b653815cf --- /dev/null +++ b/onnxruntime/test/python/quantization/test_symmetric_flag.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python +# coding: utf-8 +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +import unittest +import onnx +from onnxruntime import quantization +import numpy as np +from onnx import helper, TensorProto, numpy_helper + +class TestSymmetricFlag(unittest.TestCase): + + def setUp(self): + + # Set up symmetrically and asymmetrically disributed values for activations + self.symmetric_activations = [-1*np.ones([1, 2, 32, 32], dtype="float32"), + +1*np.ones([1, 2, 32, 32], dtype="float32")] + self.asymmetric_activations = [-1*np.ones([1, 2, 32, 32], dtype="float32"), + +2*np.ones([1, 2, 32, 32], dtype="float32")] + + # Set up symmetrically and asymmetrically disributed values for weights + self.symmetric_weights = np.concatenate((-1*np.ones([1, 1, 2, 2], dtype="float32"), + +1*np.ones([1, 1, 2, 2], dtype="float32")), axis = 1) + self.asymmetric_weights = np.concatenate((-1*np.ones([1, 1, 2, 2], dtype="float32"), + +2*np.ones([1, 1, 2, 2], dtype="float32")), axis = 1) + + + def perform_quantization(self, activations, weight, act_sym, wgt_sym): + + # One-layer convolution model + act = helper.make_tensor_value_info("ACT", TensorProto.FLOAT, activations[0].shape) + wgt = helper.make_tensor_value_info("WGT", TensorProto.FLOAT, weight.shape) + res = helper.make_tensor_value_info("RES", TensorProto.FLOAT, [None, None, None, None]) + wgt_init = numpy_helper.from_array(weight, "WGT") + conv_node = onnx.helper.make_node("Conv", ["ACT", "WGT"], ["RES"]) + graph = helper.make_graph([conv_node], "test", [act], [res], initializer=[wgt_init]) + model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 11)]) + onnx.save(model, "model.onnx") + + # Quantize model + class DummyDataReader(quantization.CalibrationDataReader): + def __init__(self): + self.iterator = ({"ACT": act} for act in activations) + def get_next(self): + return next(self.iterator, None) + quantization.quantize_static(model_input="model.onnx", + model_output="quantized-model.onnx", + calibration_data_reader=DummyDataReader(), + activation_type=quantization.QuantType.QInt8, + weight_type=quantization.QuantType.QInt8, + op_types_to_quantize=["Conv", "MatMul"], + extra_options = {"WeightSymmetric": wgt_sym, + "ActivationSymmetric": act_sym}) + + # Extract quantization parameters: scales and zero points for activations, weights, and results + model = onnx.load("quantized-model.onnx") + act_zp = [init for init in model.graph.initializer if init.name=="ACT_zero_point"][0].int32_data[0] + act_sc = [init for init in model.graph.initializer if init.name=="ACT_scale"][0].float_data[0] + wgt_zp = [init for init in model.graph.initializer if init.name=="WGT_zero_point"][0].int32_data[0] + wgt_sc = [init for init in model.graph.initializer if init.name=="WGT_scale"][0].float_data[0] + + # Return quantization parameters + return act_zp, act_sc, wgt_zp, wgt_sc + + def test_0(self): + + act_zp, act_sc, wgt_zp, wgt_sc = self.perform_quantization(self.asymmetric_activations, + self.asymmetric_weights, + act_sym = True, + wgt_sym = True) + + # Calibration activations are asymmetric, but activation + # symmetrization flag is set to True, hence expect activation zero + # point = 0 + self.assertEqual(act_zp, 0) + + # Weights are asymmetric, but weight symmetrization flag is set to + # True, hence expect weight zero point = 0 + self.assertEqual(wgt_zp, 0) + + def test_1(self): + + act_zp, act_sc, wgt_zp, wgt_sc = self.perform_quantization(self.asymmetric_activations, + self.asymmetric_weights, + act_sym = False, + wgt_sym = False) + + # Calibration activations are asymmetric, symmetrization flag not + # set, hence expect activation zero point != 0 + self.assertNotEqual(act_zp, 0) + + # Weights are asymmetric, weight symmetrization flag is set to + # False, hence expect weight zero point != 0 + self.assertNotEqual(wgt_zp, 0) + + def test_2(self): + + act_zp, act_sc, wgt_zp, wgt_sc = self.perform_quantization(self.symmetric_activations, + self.symmetric_weights, + act_sym = True, + wgt_sym = True) + + # Calibration activations are symmetric, hence expect activation + # zero point == 0 (regardless of flag) + self.assertEqual(act_zp, 0) + + # Weights are symmetric, hence expect weight + # zero point == 0 (regardless of flag) + self.assertEqual(wgt_zp, 0) + + def test_3(self): + + act_zp, act_sc, wgt_zp, wgt_sc = self.perform_quantization(self.symmetric_activations, + self.symmetric_weights, + act_sym = False, + wgt_sym = False) + + # Calibration activations are symmetric, hence expect activation + # zero point == 0 (regardless of flag) + self.assertEqual(act_zp, 0) + + # Weights are symmetric, hence expect weight + # zero point == 0 (regardless of flag) + self.assertEqual(wgt_zp, 0) + +if __name__ == '__main__': + + unittest.main()