saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-25 19:48:11 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Enable signed int8 data type for activations in static quantization (#7029)

Browse source 
* Add support for signed int8 static activation quantization. Make symmetrization in quantization switcheable
This commit is contained in:
SilvanK4t1qbit 2021-06-24 14:42:22 -07:00 committed by GitHub
parent e083d207cf
commit eb36258df4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 200 additions and 48 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

27
onnxruntime/python/tools/quantization/onnx_quantizer.py
View file

@ -16,14 +16,13 @@ from onnxruntime import SessionOptions, InferenceSession, GraphOptimizationLevel
from .quant_utils import QuantizationMode, QuantizedValueType, QuantizedInitializer, QuantizedValue
from .quant_utils import find_by_name, get_elem_index, get_mul_node, generate_identified_filename, attribute_to_kwarg, type_to_name
from .quant_utils import quantize_nparray, quantize_data, compute_scale_zp, get_qrange_for_qType
from .quant_utils import quantize_nparray, quantize_data, compute_scale_zp, get_qrange_for_qType, get_qmin_qmax_for_qType
from .quant_utils import QuantType, onnx_domain, __producer__, __version__
from .registry import CreateOpQuantizer, CreateDefaultOpQuantizer
from .onnx_model import ONNXModel
class ONNXQuantizer:
def __init__(self, model, per_channel, reduce_range, mode, static, weight_qType, input_qType, tensors_range,
nodes_to_quantize, nodes_to_exclude, op_types_to_quantize, extra_options={}):
@ -42,7 +41,8 @@ class ONNXQuantizer:
self.fuse_dynamic_quant = False
self.extra_options = extra_options if extra_options is not None else {}
self.q_matmul_const_b_only = 'MatMulConstBOnly' in self.extra_options and self.extra_options['MatMulConstBOnly']
self.is_weight_symmetric = 'WeightSymmetric' not in self.extra_options or self.extra_options['WeightSymmetric']
self.is_weight_symmetric = True if 'WeightSymmetric' not in self.extra_options else self.extra_options['WeightSymmetric']
self.is_activation_symmetric = False if 'ActivationSymmetric' not in self.extra_options else self.extra_options['ActivationSymmetric']
self.input_qType = onnx_proto.TensorProto.INT8 if input_qType == QuantType.QInt8 else onnx_proto.TensorProto.UINT8
self.weight_qType = onnx_proto.TensorProto.INT8 if weight_qType == QuantType.QInt8 else onnx_proto.TensorProto.UINT8
@ -605,8 +605,8 @@ class ONNXQuantizer:
# Update packed weight, zero point, and scale initializers
weight_data = self.tensor_proto_to_array(weight)
_, _, zero_point, scale, q_weight_data = quantize_data(weight_data.flatten().tolist(),
get_qrange_for_qType(qType, self.reduce_range and reduce_range),
qType, self.is_weight_symmetric)
qType, self.is_weight_symmetric,
self.reduce_range and reduce_range)
q_weight_data = np.asarray(q_weight_data, dtype=onnx.mapping.TENSOR_TYPE_TO_NP_TYPE[qType]).reshape(weight.dims)
q_weight_initializer = onnx.numpy_helper.from_array(q_weight_data, q_weight_name)
@ -641,9 +641,8 @@ class ONNXQuantizer:
for i in range(channel_count):
per_channel_data = weights.take(i, channel_axis)
rmin, rmax, zero_point, scale, quantized_per_channel_data = quantize_data(
per_channel_data.flatten().tolist(),
get_qrange_for_qType(weight_qType, self.reduce_range and reduce_range),
weight_qType, self.is_weight_symmetric)
per_channel_data.flatten().tolist(), weight_qType,
self.is_weight_symmetric, self.reduce_range and reduce_range)
rmin_list.append(rmin)
rmax_list.append(rmax)
zero_point_list.append(zero_point)
@ -735,15 +734,11 @@ class ONNXQuantizer:
quantization_params = {}
for tensor_name in self.tensors_range.keys():
rmin, rmax = self.tensors_range[tensor_name]
qmin, qmax = get_qmin_qmax_for_qType(self.input_qType)
# adjust rmin and rmax such that 0 is included in the range. This is required
# to make sure zero can be uniquely represented.
rmin = min(rmin, 0)
rmax = max(rmax, 0)
quantization_params[tensor_name] = compute_scale_zp(rmin, rmax, self.input_qType,
get_qrange_for_qType(self.input_qType),
self.is_weight_symmetric)
quantization_params[tensor_name] = compute_scale_zp(rmin, rmax,
qmin, qmax,
self.is_activation_symmetric)
return quantization_params

80
onnxruntime/python/tools/quantization/quant_utils.py
View file

@ -109,29 +109,46 @@ def quantize_nparray(qType, arr, scale, zero_point, low=None, high=None):
return arr_fp32.astype(dtype)
def compute_scale_zp(rmin, rmax, qType, quantize_range, symmetric):
if qType == onnx_proto.TensorProto.INT8:
if symmetric:
max_range = max(abs(rmin), abs(rmax))
scale = (float(max_range) * 2) / quantize_range if max_range > 0 else 1.0
zero_point = 0
else:
max_range = float(rmax) - float(rmin)
scale = float(max_range) / quantize_range if max_range > 0 else 1.0
zero_point = round((quantize_range / 2) - rmax / scale)
elif qType == onnx_proto.TensorProto.UINT8:
scale = (float(rmax) - rmin) / quantize_range if rmin != rmax else 1
zero_point = round((0 - rmin) / scale) # round to nearest integer
else:
raise ValueError("Unexpected data type {} requested. Only INT8 and UINT8 are supported.".format(qType))
def compute_scale_zp(rmin, rmax, qmin, qmax, symmetric=False):
'''
Calculate the scale s and zero point z for the quantization relation
r = s(q-z), where r are the original values and q are the corresponding
quantized values.
r and z are calculated such that every value within [rmin,rmax] has an
approximate representation within [qmin,qmax]. In addition, qmin <= z <=
qmax is enforced. If the symmetric flag is set to True, the interval
[rmin,rmax] is symmetrized to [-absmax, +absmax], where
absmax = max(abs(rmin), abs(rmax)).
:parameter rmin: minimum value of r
:parameter rmax: maximum value of r
:parameter qmin: minimum value representable by the target quantization data type
:parameter qmax: maximum value representable by the target quantization data type
:return: zero and scale [z, s]
'''
# Adjust rmin and rmax such that 0 is included in the range. This is
# required to make sure zero can be represented by the quantization data
# type (i.e. to make sure qmin <= zero_point <= qmax)
rmin = min(rmin, 0)
rmax = max(rmax, 0)
if symmetric:
absmax = max(abs(rmin), abs(rmax))
rmin = -absmax
rmax = +absmax
scale = (rmax - rmin) / float(qmax-qmin) if rmax!=rmin else 1.0
zero_point = round(qmin - rmin/scale)
return [zero_point, scale]
def quantize_data(data, quantize_range, qType, symmetric=True):
def quantize_data(data, qType, symmetric, reduce_range=False):
'''
:parameter data: data to quantize
:parameter quantize_range: list of data to weight pack.
:parameter qType: data type to quantize to. Supported types UINT8 and INT8
:parameter symmetric: whether symmetric quantization is used or not. This is applied to INT8.
:return: minimum, maximum, zero point, scale, and quantized weights
@ -146,14 +163,28 @@ def quantize_data(data, quantize_range, qType, symmetric=True):
S: scale
z: zero point
'''
rmin = min(min(data), 0.)
rmax = max(max(data), 0.)
rmin = min(data)
rmax = max(data)
qmin, qmax = get_qmin_qmax_for_qType(qType, reduce_range)
zero_point, scale = compute_scale_zp(rmin, rmax, qType, quantize_range, symmetric)
zero_point, scale = compute_scale_zp(rmin, rmax, qmin, qmax, symmetric)
quantized_data = quantize_nparray(qType, numpy.asarray(data), scale, zero_point)
return rmin, rmax, zero_point, scale, quantized_data
def get_qmin_qmax_for_qType(qType, reduce_range=False):
'''
Return qmin and qmax, the minimum and maximum value representable by the given qType
:parameter qType: onnx.onnx_pb.TensorProto.UINT8 or onnx.onnx_pb.TensorProto.UINT8
:return: qmin, qmax
'''
if qType == onnx_proto.TensorProto.UINT8:
(qmin, qmax) = (0,127) if reduce_range else (0,255)
elif qType == onnx_proto.TensorProto.INT8:
(qmin, qmax) = (-64,64) if reduce_range else (-127,127)
else:
raise ValueError("Unexpected data type {} requested. Only INT8 and UINT8 are supported.".format(qType))
return qmin, qmax
def get_qrange_for_qType(qType, reduce_range=False):
'''
@ -161,13 +192,8 @@ def get_qrange_for_qType(qType, reduce_range=False):
parameter qType: quantization type.
return: quantization range.
'''
if qType == onnx_proto.TensorProto.UINT8:
return 127 if reduce_range else 255
elif qType == onnx_proto.TensorProto.INT8:
return 128 if reduce_range else 254 # [-64, 64] for reduce_range, and [-127, 127] full_range.
else:
raise ValueError('unsupported quantization data type')
qmin, qmax = get_qmin_qmax_for_qType(qType, reduce_range)
return qmax - qmin
class QuantizedInitializer:
'''

9
onnxruntime/python/tools/quantization/quantize.py
View file

@ -175,18 +175,17 @@ def quantize_static(model_input,
List of nodes names to exclude. The nodes in this list will be excluded from quantization
when it is not None.
:param optimize_model: optimize model before quantization.
:parma use_external_data_format: option used for large size (>2GB) model. Set to False by default.
:param use_external_data_format: option used for large size (>2GB) model. Set to False by default.
:param calibrate_method:
Current calibration methods supported are MinMax and Entropy.
Please use CalibrationMethod.MinMax or CalibrationMethod.Entropy as options.
:param extra_options:
key value pair dictionary for various options in different case. Current used:
extra.Sigmoid.nnapi = True (Default is False)
extra.Sigmoid.nnapi = True/False (Default is False)
ActivationSymmetric = True/False: symmetrize calibration data for activations (default is False).
WeightSymmetric = True/False: symmetrize calibration data for weights (default is True).
'''
if activation_type != QuantType.QUInt8:
raise ValueError("Static quantization only support uint8 for activation now.")
mode = QuantizationMode.QLinearOps
if not op_types_to_quantize or len(op_types_to_quantize) == 0:

132
onnxruntime/test/python/quantization/test_symmetric_flag.py Normal file
View file

@ -0,0 +1,132 @@
#!/usr/bin/env python
# coding: utf-8
# -------------------------------------------------------------------------
# Copyright (c) Microsoft Corporation. All rights reserved.
# Licensed under the MIT License. See License.txt in the project root for
# license information.
# --------------------------------------------------------------------------
import unittest
import onnx
from onnxruntime import quantization
import numpy as np
from onnx import helper, TensorProto, numpy_helper
class TestSymmetricFlag(unittest.TestCase):
def setUp(self):
# Set up symmetrically and asymmetrically disributed values for activations
self.symmetric_activations = [-1*np.ones([1, 2, 32, 32], dtype="float32"),
+1*np.ones([1, 2, 32, 32], dtype="float32")]
self.asymmetric_activations = [-1*np.ones([1, 2, 32, 32], dtype="float32"),
+2*np.ones([1, 2, 32, 32], dtype="float32")]
# Set up symmetrically and asymmetrically disributed values for weights
self.symmetric_weights = np.concatenate((-1*np.ones([1, 1, 2, 2], dtype="float32"),
+1*np.ones([1, 1, 2, 2], dtype="float32")), axis = 1)
self.asymmetric_weights = np.concatenate((-1*np.ones([1, 1, 2, 2], dtype="float32"),
+2*np.ones([1, 1, 2, 2], dtype="float32")), axis = 1)
def perform_quantization(self, activations, weight, act_sym, wgt_sym):
# One-layer convolution model
act = helper.make_tensor_value_info("ACT", TensorProto.FLOAT, activations[0].shape)
wgt = helper.make_tensor_value_info("WGT", TensorProto.FLOAT, weight.shape)
res = helper.make_tensor_value_info("RES", TensorProto.FLOAT, [None, None, None, None])
wgt_init = numpy_helper.from_array(weight, "WGT")
conv_node = onnx.helper.make_node("Conv", ["ACT", "WGT"], ["RES"])
graph = helper.make_graph([conv_node], "test", [act], [res], initializer=[wgt_init])
model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 11)])
onnx.save(model, "model.onnx")
# Quantize model
class DummyDataReader(quantization.CalibrationDataReader):
def __init__(self):
self.iterator = ({"ACT": act} for act in activations)
def get_next(self):
return next(self.iterator, None)
quantization.quantize_static(model_input="model.onnx",
model_output="quantized-model.onnx",
calibration_data_reader=DummyDataReader(),
activation_type=quantization.QuantType.QInt8,
weight_type=quantization.QuantType.QInt8,
op_types_to_quantize=["Conv", "MatMul"],
extra_options = {"WeightSymmetric": wgt_sym,
"ActivationSymmetric": act_sym})
# Extract quantization parameters: scales and zero points for activations, weights, and results
model = onnx.load("quantized-model.onnx")
act_zp = [init for init in model.graph.initializer if init.name=="ACT_zero_point"][0].int32_data[0]
act_sc = [init for init in model.graph.initializer if init.name=="ACT_scale"][0].float_data[0]
wgt_zp = [init for init in model.graph.initializer if init.name=="WGT_zero_point"][0].int32_data[0]
wgt_sc = [init for init in model.graph.initializer if init.name=="WGT_scale"][0].float_data[0]
# Return quantization parameters
return act_zp, act_sc, wgt_zp, wgt_sc
def test_0(self):
act_zp, act_sc, wgt_zp, wgt_sc = self.perform_quantization(self.asymmetric_activations,
self.asymmetric_weights,
act_sym = True,
wgt_sym = True)
# Calibration activations are asymmetric, but activation
# symmetrization flag is set to True, hence expect activation zero
# point = 0
self.assertEqual(act_zp, 0)
# Weights are asymmetric, but weight symmetrization flag is set to
# True, hence expect weight zero point = 0
self.assertEqual(wgt_zp, 0)
def test_1(self):
act_zp, act_sc, wgt_zp, wgt_sc = self.perform_quantization(self.asymmetric_activations,
self.asymmetric_weights,
act_sym = False,
wgt_sym = False)
# Calibration activations are asymmetric, symmetrization flag not
# set, hence expect activation zero point != 0
self.assertNotEqual(act_zp, 0)
# Weights are asymmetric, weight symmetrization flag is set to
# False, hence expect weight zero point != 0
self.assertNotEqual(wgt_zp, 0)
def test_2(self):
act_zp, act_sc, wgt_zp, wgt_sc = self.perform_quantization(self.symmetric_activations,
self.symmetric_weights,
act_sym = True,
wgt_sym = True)
# Calibration activations are symmetric, hence expect activation
# zero point == 0 (regardless of flag)
self.assertEqual(act_zp, 0)
# Weights are symmetric, hence expect weight
# zero point == 0 (regardless of flag)
self.assertEqual(wgt_zp, 0)
def test_3(self):
act_zp, act_sc, wgt_zp, wgt_sc = self.perform_quantization(self.symmetric_activations,
self.symmetric_weights,
act_sym = False,
wgt_sym = False)
# Calibration activations are symmetric, hence expect activation
# zero point == 0 (regardless of flag)
self.assertEqual(act_zp, 0)
# Weights are symmetric, hence expect weight
# zero point == 0 (regardless of flag)
self.assertEqual(wgt_zp, 0)
if __name__ == '__main__':
unittest.main()