mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-12 17:57:38 +00:00
[Quantization] Tensor quant overrides and QNN EP quantization configuration (#18465)
### Description
#### 1. Adds `TensorQuantOverrides` extra option
Allows specifying a dictionary of tensor-level quantization overrides:
```
TensorQuantOverrides = dictionary :
Default is {}. Set tensor quantization overrides. The key is a tensor name and the value is a
list of dictionaries. For per-tensor quantization, the list contains a single dictionary. For
per-channel quantization, the list contains a dictionary for each channel in the tensor.
Each dictionary contains optional overrides with the following keys and values.
'quant_type' = QuantType : The tensor's quantization data type.
'scale' = Float : The scale value to use. Must also specify `zero_point` if set.
'zero_point' = Int : The zero-point value to use. Must also specify `scale` is set.
'symmetric' = Bool : If the tensor should use symmetric quantization. Invalid if also
set `scale` or `zero_point`.
'reduce_range' = Bool : If the quantization range should be reduced. Invalid if also
set `scale` or `zero_point`.
'rmax' = Float : Override the maximum real tensor value in calibration data.
Invalid if also set `scale` or `zero_point`.
'rmin' = Float : Override the minimum real tensor value in calibration data.
Invalid if also set `scale` or `zero_point`.
```
- All of the options are optional.
- Some combinations are invalid.
- Ex: `rmax` and `rmin` are unnecessary if the `zero_point` and `scale`
are also specified.
Example for per-tensor quantization overrides:
```Python3
extra_options = {
"TensorQuantOverrides": {
"SIG_OUT": [{"scale": 1.0, "zero_point": 127}],
"WGT": [{"quant_type": quantization.QuantType.QInt8, "symmetric": True, "reduce_range": True}],
"BIAS": [{"quant_type": quantization.QuantType.QInt8, "symmetric": True, "reduce_range": True}],
},
}
```
Example for per-channel quantization overrides (Conv weight and bias):
```Python3
extra_options = {
"TensorQuantOverrides": {
"WGT": [
{
"quant_type": quantization.QuantType.QUInt8,
"rmin": 0.0,
"rmax": 2.5,
"reduce_range": True,
},
{
"quant_type": quantization.QuantType.QUInt8,
"rmin": 0.2,
"rmax": 2.55,
"reduce_range": False,
},
],
"BIAS": [
{"zero_point": 0, "scale": 0.000621},
{"zero_point": 0, "scale": 0.23},
],
},
}
```
#### 2. Adds utilities to get the default QDQ configs for QNN EP
Added a `quantization.execution_providers.qnn.get_qnn_qdq_config` method
that inspects the model and returns suitable quantization
configurations.
Example usage:
```python3
from quantization import quantize, QuantType
from quantization.execution_providers.qnn import get_qnn_qdq_config
qnn_config = get_qnn_qdq_config(input_model_path,
data_reader,
activation_type=QuantType.QUInt16,
weight_type=QuantType.QUInt8)
quantize(input_model_path,
output_model_path,
qnn_config)
```
### Motivation and Context
Make it possible to create more QDQ models that run on QNN EP.
---------
Signed-off-by: adrianlizarraga <adlizarraga@microsoft.com>
This commit is contained in:
Adrian Lizarraga
GitHub
parent
01b5c78917
commit
e066fca777
13 changed files with 825 additions and 56 deletions
|
|
@ -453,6 +453,9 @@ file(GLOB onnxruntime_python_quantization_operators_src CONFIGURE_DEPENDS
|
|||
file(GLOB onnxruntime_python_quantization_cal_table_flatbuffers_src CONFIGURE_DEPENDS
|
||||
"${ONNXRUNTIME_ROOT}/python/tools/quantization/CalTableFlatBuffers/*.py"
|
||||
)
|
||||
file(GLOB onnxruntime_python_quantization_ep_qnn_src CONFIGURE_DEPENDS
|
||||
"${ONNXRUNTIME_ROOT}/python/tools/quantization/execution_providers/qnn/*.py"
|
||||
)
|
||||
file(GLOB onnxruntime_python_transformers_src CONFIGURE_DEPENDS
|
||||
"${ONNXRUNTIME_ROOT}/python/tools/transformers/*.py"
|
||||
)
|
||||
|
|
@ -547,6 +550,8 @@ add_custom_command(
|
|||
COMMAND ${CMAKE_COMMAND} -E make_directory $<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/quantization
|
||||
COMMAND ${CMAKE_COMMAND} -E make_directory $<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/quantization/operators
|
||||
COMMAND ${CMAKE_COMMAND} -E make_directory $<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/quantization/CalTableFlatBuffers
|
||||
COMMAND ${CMAKE_COMMAND} -E make_directory $<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/quantization/execution_providers
|
||||
COMMAND ${CMAKE_COMMAND} -E make_directory $<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/quantization/execution_providers/qnn
|
||||
COMMAND ${CMAKE_COMMAND} -E make_directory $<TARGET_FILE_DIR:${build_output_target}>/quantization
|
||||
COMMAND ${CMAKE_COMMAND} -E make_directory $<TARGET_FILE_DIR:${build_output_target}>/transformers
|
||||
COMMAND ${CMAKE_COMMAND} -E make_directory $<TARGET_FILE_DIR:${build_output_target}>/transformers/test_data/models
|
||||
|
|
@ -617,6 +622,9 @@ add_custom_command(
|
|||
COMMAND ${CMAKE_COMMAND} -E copy
|
||||
${onnxruntime_python_quantization_cal_table_flatbuffers_src}
|
||||
$<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/quantization/CalTableFlatBuffers/
|
||||
COMMAND ${CMAKE_COMMAND} -E copy
|
||||
${onnxruntime_python_quantization_ep_qnn_src}
|
||||
$<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/quantization/execution_providers/qnn/
|
||||
COMMAND ${CMAKE_COMMAND} -E copy
|
||||
${onnxruntime_python_transformers_src}
|
||||
$<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/transformers/
|
||||
|
|
|
|||
|
|
@ -0,0 +1 @@
|
|||
from .quant_config import get_qnn_qdq_config # noqa: F401
|
||||
|
|
@ -0,0 +1,84 @@
|
|||
# -------------------------------------------------------------------------
|
||||
# Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
# Licensed under the MIT License. See License.txt in the project root for
|
||||
# license information.
|
||||
# --------------------------------------------------------------------------
|
||||
from pathlib import Path
|
||||
|
||||
import onnx
|
||||
|
||||
from ...calibrate import CalibrationDataReader, CalibrationMethod
|
||||
from ...quant_utils import QuantType
|
||||
from ...quantize import StaticQuantConfig
|
||||
|
||||
Q16_TYPES = {QuantType.QInt16, QuantType.QUInt16}
|
||||
Q8_TYPES = {QuantType.QInt8, QuantType.QUInt8}
|
||||
OP_TYPES_TO_EXCLUDE = {"Cast"}
|
||||
|
||||
|
||||
def get_qnn_qdq_config(
|
||||
model_input: Path,
|
||||
calibration_data_reader: CalibrationDataReader,
|
||||
calibrate_method=CalibrationMethod.MinMax,
|
||||
activation_type=QuantType.QUInt8,
|
||||
weight_type=QuantType.QUInt8,
|
||||
per_channel=False,
|
||||
):
|
||||
if per_channel:
|
||||
raise ValueError("QNN EP does not yet support per-channel quantization.")
|
||||
|
||||
# Process model nodes to setup overrides.
|
||||
model = onnx.load_model(model_input)
|
||||
|
||||
op_types = set()
|
||||
tensor_quant_overrides = {}
|
||||
|
||||
name_to_initializer = {initializer.name: initializer for initializer in model.graph.initializer}
|
||||
|
||||
for node in model.graph.node:
|
||||
op_types.add(node.op_type)
|
||||
|
||||
if node.op_type == "MatMul" and activation_type in Q16_TYPES and weight_type in Q8_TYPES:
|
||||
weight_symmetric = weight_type == QuantType.QInt8
|
||||
|
||||
# Override initializers to use the weight_type
|
||||
for input_name in node.input:
|
||||
if input_name in name_to_initializer:
|
||||
tensor_quant_overrides[input_name] = [{"quant_type": weight_type, "symmetric": weight_symmetric}]
|
||||
elif node.op_type == "LayerNormalization" and activation_type in Q16_TYPES and weight_type in Q8_TYPES:
|
||||
weight_symmetric = weight_type == QuantType.QInt8
|
||||
|
||||
# Override initializers to use the weight_type. Don't override the bias input.
|
||||
for i in range(2):
|
||||
input_name = node.input[i]
|
||||
if input_name in name_to_initializer:
|
||||
tensor_quant_overrides[input_name] = [{"quant_type": weight_type, "symmetric": weight_symmetric}]
|
||||
elif node.op_type == "Sigmoid":
|
||||
if activation_type == QuantType.QUInt16:
|
||||
tensor_quant_overrides[node.output[0]] = [{"scale": 1.0 / 65536.0, "zero_point": 0}]
|
||||
elif activation_type == QuantType.QInt16:
|
||||
tensor_quant_overrides[node.output[0]] = [{"scale": 1.0 / 32768.0, "zero_point": 0}]
|
||||
elif node.op_type == "Tanh":
|
||||
if activation_type == QuantType.QUInt16:
|
||||
tensor_quant_overrides[node.output[0]] = [{"scale": 1.0 / 32768.0, "zero_point": 32768}]
|
||||
elif activation_type == QuantType.QInt16:
|
||||
tensor_quant_overrides[node.output[0]] = [{"scale": 1.0 / 32768.0, "zero_point": 0}]
|
||||
|
||||
extra_options = {
|
||||
"MinimumRealRange": 0.0001,
|
||||
"DedicatedQDQPair": False, # Let ORT optimizer duplicate DQ nodes
|
||||
"TensorQuantOverrides": tensor_quant_overrides,
|
||||
}
|
||||
|
||||
# TODO: Remove this extra option once ORT uses an ONNX version that supports 16-bit Q/DQ ops.
|
||||
if activation_type in Q16_TYPES or weight_type in Q16_TYPES:
|
||||
extra_options["UseQDQContribOps"] = True
|
||||
|
||||
return StaticQuantConfig(
|
||||
calibration_data_reader,
|
||||
calibrate_method=calibrate_method,
|
||||
activation_type=activation_type,
|
||||
weight_type=weight_type,
|
||||
op_types_to_quantize=list(op_types.difference(OP_TYPES_TO_EXCLUDE)),
|
||||
extra_options=extra_options,
|
||||
)
|
||||
|
|
@ -37,6 +37,7 @@ from .quant_utils import (
|
|||
model_has_infer_metadata,
|
||||
ms_domain,
|
||||
quantize_data,
|
||||
quantize_nparray,
|
||||
save_and_reload_model_with_shape_infer,
|
||||
tensor_proto_to_array,
|
||||
)
|
||||
|
|
@ -49,8 +50,8 @@ class QuantizationParams:
|
|||
for k, v in data.items():
|
||||
if not isinstance(k, str):
|
||||
raise TypeError(f"Keys must be strings not {type(k)}.")
|
||||
if not isinstance(v, (int, float, str)):
|
||||
raise TypeError(f"Values must be int, float, str not {type(v)}.")
|
||||
if not isinstance(v, (int, float, str, QuantType)):
|
||||
raise TypeError(f"Values must be int, float, str, or QuantType not {type(v)}.")
|
||||
self.data[k] = v
|
||||
|
||||
def __iter__(self):
|
||||
|
|
@ -148,6 +149,7 @@ class ONNXQuantizer:
|
|||
if self.mode not in QuantizationMode:
|
||||
raise ValueError(f"unsupported quantization mode {self.mode}")
|
||||
|
||||
self.tensor_quant_overrides = self._get_and_check_tensor_quant_overrides()
|
||||
self.quantization_params = self.calculate_quantization_params()
|
||||
|
||||
# QuantizeRange tensor name and zero tensor name for scale and zero point calculation.
|
||||
|
|
@ -167,6 +169,87 @@ class ONNXQuantizer:
|
|||
# to store specified scale and zeropoint instead of calculated value, tensor_name->(scale, zeropoint)
|
||||
self.used_scale_zp_map = {}
|
||||
|
||||
def _get_and_check_tensor_quant_overrides(self):
|
||||
"""
|
||||
Get tensor quantization overrides and check correctness.
|
||||
"""
|
||||
tensor_quant_overrides = self.extra_options.get("TensorQuantOverrides", {})
|
||||
|
||||
# Validate that compatible/valid overrides are provided.
|
||||
if tensor_quant_overrides:
|
||||
initializer_names = self.model.get_initializer_name_set()
|
||||
value_info_names = set(self.value_infos.keys())
|
||||
keys_unsupported_with_scale_zp = {"symmetric", "reduce_range", "rmax", "rmin"}
|
||||
|
||||
for tensor_name, quant_overrides_list in tensor_quant_overrides.items():
|
||||
if tensor_name not in initializer_names and tensor_name not in value_info_names:
|
||||
raise ValueError(f"Tensor '{tensor_name}' in TensorQuantOverrides is not present in the model")
|
||||
|
||||
if not isinstance(quant_overrides_list, list):
|
||||
raise ValueError(f"Tensor quantization overrides for '{tensor_name}' are not in a list")
|
||||
|
||||
is_initializer = tensor_name in initializer_names
|
||||
if not is_initializer and len(quant_overrides_list) > 1:
|
||||
raise ValueError(
|
||||
f"Tensor '{tensor_name}' has a list of per-channel overrides, but is not an initializer"
|
||||
)
|
||||
|
||||
quant_type = None
|
||||
for index, quant_overrides in enumerate(quant_overrides_list):
|
||||
if not isinstance(quant_overrides, dict):
|
||||
raise ValueError(
|
||||
f"Tensor quantization overrides at index {index} for '{tensor_name}' are not in a dict"
|
||||
)
|
||||
|
||||
# For per-channel quantization, all channels must use the same quantization type.
|
||||
# Therefore, if the user tries to override the quant_type for a channel, it must match in all
|
||||
# other channels.
|
||||
if index == 0:
|
||||
quant_type = quant_overrides.get("quant_type")
|
||||
elif quant_type != quant_overrides.get("quant_type"):
|
||||
raise ValueError(
|
||||
"Channel quantization types for tensor '{tensor_name}' do not match at index {index}."
|
||||
)
|
||||
|
||||
has_scale = "scale" in quant_overrides
|
||||
has_zero_point = "zero_point" in quant_overrides
|
||||
|
||||
if (has_scale and not has_zero_point) or (has_zero_point and not has_scale):
|
||||
raise ValueError(
|
||||
"Must provide both 'scale' and 'zero_point' if one of the overrides is provided"
|
||||
)
|
||||
|
||||
if has_scale:
|
||||
for key in keys_unsupported_with_scale_zp:
|
||||
if key in quant_overrides:
|
||||
raise ValueError(
|
||||
f"Tensor override option '{key}' is invalid with 'scale' and 'zero_point'"
|
||||
)
|
||||
|
||||
return tensor_quant_overrides
|
||||
|
||||
def get_per_tensor_quant_overrides(self, tensor_name):
|
||||
quant_overrides_list = self.tensor_quant_overrides.get(tensor_name, [{}])
|
||||
num_overrides = len(quant_overrides_list)
|
||||
if num_overrides > 1:
|
||||
raise ValueError(
|
||||
f"Expected tensor '{tensor_name}' to use per-tensor quantization overrides, "
|
||||
f"but found {num_overrides} per-channel overrides."
|
||||
)
|
||||
|
||||
return quant_overrides_list[0] if num_overrides > 0 else {}
|
||||
|
||||
def get_per_channel_quant_overrides(self, tensor_name, num_channels):
|
||||
quant_overrides_list = self.tensor_quant_overrides.get(tensor_name, [{} for i in range(num_channels)])
|
||||
|
||||
if len(quant_overrides_list) != num_channels:
|
||||
raise ValueError(
|
||||
f"Expected tensor '{tensor_name}' to have {num_channels} per-channel quantization overrides, "
|
||||
f"but found {len(quant_overrides_list)} instead."
|
||||
)
|
||||
|
||||
return quant_overrides_list
|
||||
|
||||
# routines for subgraph support
|
||||
def quantize_subgraph(self, subgraph, graph_key):
|
||||
"""
|
||||
|
|
@ -587,6 +670,8 @@ class ONNXQuantizer:
|
|||
parameter param_name: Name of the quantization parameter.
|
||||
return: result, scale_name, zero_point_name, scale_shape, zero_point_shape.
|
||||
"""
|
||||
zero_point_type = self.activation_qType
|
||||
|
||||
if use_scale is None or use_zeropoint is None:
|
||||
if self.quantization_params is None or param_name not in self.quantization_params:
|
||||
logging.info(f'Quantization parameters for tensor:"{param_name}" not specified')
|
||||
|
|
@ -595,21 +680,21 @@ class ONNXQuantizer:
|
|||
params = self.quantization_params[param_name]
|
||||
if not isinstance(params, QuantizationParams):
|
||||
raise TypeError(f"Unexpected type {type(params)} for {param_name!r}.")
|
||||
if params is None or len(params) != 2:
|
||||
if params is None or len(params) != 3:
|
||||
raise ValueError(
|
||||
"Quantization parameters should contain zero point and scale. "
|
||||
"Quantization parameters should contain zero point, scale, quant type. "
|
||||
f"Specified values for output {param_name}: {params}"
|
||||
)
|
||||
|
||||
zero_point_values = [params["zero_point"]]
|
||||
scale_values = [params["scale"]]
|
||||
zero_point_type = params["quant_type"]
|
||||
else:
|
||||
zero_point_values = [use_zeropoint]
|
||||
scale_values = [use_scale]
|
||||
|
||||
zero_point_shape = []
|
||||
zero_point_name = param_name + "_zero_point"
|
||||
zero_point_type = self.activation_qType
|
||||
scale_shape = []
|
||||
scale_name = param_name + "_scale"
|
||||
|
||||
|
|
@ -991,16 +1076,25 @@ class ONNXQuantizer:
|
|||
zp_name = weight.name + "_zero_point"
|
||||
scale_name = weight.name + "_scale"
|
||||
|
||||
# Update packed weight, zero point, and scale initializers
|
||||
# Quantize weight data. Use quantization overrides if provided by the user.
|
||||
weight_data = tensor_proto_to_array(weight)
|
||||
w_data = weight_data.flatten().tolist()
|
||||
_, _, zero_point, scale, q_weight_data = quantize_data(
|
||||
w_data,
|
||||
qType,
|
||||
self.is_weight_symmetric,
|
||||
self.reduce_range and reduce_range,
|
||||
self.min_real_range,
|
||||
)
|
||||
quant_overrides = self.get_per_tensor_quant_overrides(weight.name)
|
||||
if "quant_type" in quant_overrides:
|
||||
qType = quant_overrides["quant_type"].tensor_type # noqa: N806
|
||||
|
||||
if "scale" in quant_overrides and "zero_point" in quant_overrides:
|
||||
zero_point, scale = quant_overrides["zero_point"], quant_overrides["scale"]
|
||||
q_weight_data = quantize_nparray(qType, weight_data.flatten(), scale, zero_point)
|
||||
else:
|
||||
_, _, zero_point, scale, q_weight_data = quantize_data(
|
||||
weight_data.flatten().tolist(),
|
||||
qType,
|
||||
quant_overrides.get("symmetric", self.is_weight_symmetric),
|
||||
reduce_range=quant_overrides.get("reduce_range", self.reduce_range and reduce_range),
|
||||
min_real_range=self.min_real_range,
|
||||
rmin_override=quant_overrides.get("rmin"),
|
||||
rmax_override=quant_overrides.get("rmax"),
|
||||
)
|
||||
|
||||
if qType in {
|
||||
onnx.TensorProto.FLOAT8E4M3FN,
|
||||
|
|
@ -1076,23 +1170,43 @@ class ONNXQuantizer:
|
|||
|
||||
weights = tensor_proto_to_array(initializer)
|
||||
channel_count = weights.shape[channel_axis]
|
||||
rmin_list = []
|
||||
rmax_list = []
|
||||
quant_overrides_for_channels = self.get_per_channel_quant_overrides(weight_name, channel_count)
|
||||
|
||||
# If user provides per-channel quantization overrides, all channels must use the same quantization type.
|
||||
# So, just use the first channel's type.
|
||||
if "quant_type" in quant_overrides_for_channels[0]:
|
||||
weight_qType = quant_overrides_for_channels[0]["quant_type"].tensor_type # noqa: N806
|
||||
|
||||
zero_point_list = []
|
||||
scale_list = []
|
||||
quantized_per_channel_data_list = []
|
||||
for i in range(channel_count):
|
||||
per_channel_data = weights.take(i, channel_axis)
|
||||
rmin, rmax, zero_point, scale, quantized_per_channel_data = quantize_data(
|
||||
per_channel_data.flatten().tolist(),
|
||||
weight_qType,
|
||||
self.is_weight_symmetric
|
||||
or weight_qType in (onnx_proto.TensorProto.INT8, onnx_proto.TensorProto.FLOAT8E4M3FN),
|
||||
self.reduce_range and reduce_range,
|
||||
self.min_real_range,
|
||||
)
|
||||
rmin_list.append(rmin)
|
||||
rmax_list.append(rmax)
|
||||
channel_quant_overrides = quant_overrides_for_channels[i]
|
||||
|
||||
if "scale" in channel_quant_overrides and "zero_point" in channel_quant_overrides:
|
||||
zero_point, scale = channel_quant_overrides["zero_point"], channel_quant_overrides["scale"]
|
||||
quantized_per_channel_data = quantize_nparray(
|
||||
weight_qType, per_channel_data.flatten(), scale, zero_point
|
||||
)
|
||||
else:
|
||||
symmetric = channel_quant_overrides.get(
|
||||
"symmetric",
|
||||
(
|
||||
self.is_weight_symmetric
|
||||
or weight_qType in (onnx_proto.TensorProto.INT8, onnx_proto.TensorProto.FLOAT8E4M3FN)
|
||||
),
|
||||
)
|
||||
_, _, zero_point, scale, quantized_per_channel_data = quantize_data(
|
||||
per_channel_data.flatten().tolist(),
|
||||
weight_qType,
|
||||
symmetric,
|
||||
reduce_range=channel_quant_overrides.get("reduce_range", self.reduce_range and reduce_range),
|
||||
min_real_range=self.min_real_range,
|
||||
rmin_override=channel_quant_overrides.get("rmin"),
|
||||
rmax_override=channel_quant_overrides.get("rmax"),
|
||||
)
|
||||
|
||||
zero_point_list.append(zero_point)
|
||||
scale_list.append(scale)
|
||||
quantized_per_channel_data_list.append(quantized_per_channel_data)
|
||||
|
|
@ -1205,15 +1319,25 @@ class ONNXQuantizer:
|
|||
td = self.tensors_range[tensor_name]
|
||||
if not isinstance(td, TensorData):
|
||||
raise TypeError(f"Unexpected type {type(td)} for {tensor_name!r}.")
|
||||
if self.activation_qType == onnx.TensorProto.FLOAT8E4M3FN:
|
||||
zero, scale = compute_scale_zp_float8(self.activation_qType, td.avg_std[1])
|
||||
else:
|
||||
rmin, rmax = td.range_value
|
||||
qmin, qmax = get_qmin_qmax_for_qType(self.activation_qType, symmetric=self.is_activation_symmetric)
|
||||
|
||||
zero, scale = compute_scale_zp(
|
||||
rmin, rmax, qmin, qmax, self.is_activation_symmetric, self.min_real_range
|
||||
)
|
||||
quantization_params[tensor_name] = QuantizationParams(zero_point=zero, scale=scale)
|
||||
quant_overrides = self.get_per_tensor_quant_overrides(tensor_name)
|
||||
|
||||
quant_type = self.activation_qType
|
||||
if "quant_type" in quant_overrides:
|
||||
quant_type = quant_overrides["quant_type"].tensor_type
|
||||
|
||||
if "scale" in quant_overrides and "zero_point" in quant_overrides:
|
||||
zero, scale = quant_overrides["zero_point"], quant_overrides["scale"]
|
||||
elif quant_type == onnx.TensorProto.FLOAT8E4M3FN:
|
||||
zero, scale = compute_scale_zp_float8(quant_type, td.avg_std[1])
|
||||
else:
|
||||
rmin = quant_overrides.get("rmin", td.range_value[0])
|
||||
rmax = quant_overrides.get("rmax", td.range_value[1])
|
||||
symmetric = quant_overrides.get("symmetric", self.is_activation_symmetric)
|
||||
reduce_range = quant_overrides.get("reduce_range", False)
|
||||
qmin, qmax = get_qmin_qmax_for_qType(quant_type, reduce_range=reduce_range, symmetric=symmetric)
|
||||
zero, scale = compute_scale_zp(rmin, rmax, qmin, qmax, symmetric, self.min_real_range)
|
||||
|
||||
quantization_params[tensor_name] = QuantizationParams(zero_point=zero, scale=scale, quant_type=quant_type)
|
||||
|
||||
return quantization_params
|
||||
|
|
|
|||
|
|
@ -6,24 +6,32 @@
|
|||
from .qdq_base_operator import QDQOperatorBase
|
||||
|
||||
|
||||
class QDQInstanceNormalization(QDQOperatorBase):
|
||||
class QDQNormalization(QDQOperatorBase):
|
||||
def __init__(self, onnx_quantizer, onnx_node):
|
||||
super().__init__(onnx_quantizer, onnx_node)
|
||||
|
||||
def quantize(self):
|
||||
node = self.node
|
||||
assert node.op_type == "InstanceNormalization"
|
||||
assert node.op_type == "InstanceNormalization" or node.op_type == "LayerNormalization"
|
||||
|
||||
# Input
|
||||
self.quantizer.quantize_activation_tensor(node.input[0])
|
||||
if not self.disable_qdq_for_node_output:
|
||||
self.quantizer.quantize_activation_tensor(node.output[0])
|
||||
|
||||
# Scale
|
||||
if self.quantizer.is_per_channel():
|
||||
self.quantizer.quantize_weight_tensor_per_channel(node.input[1], axis=1)
|
||||
else:
|
||||
scale_is_initializer = self.quantizer.is_input_a_initializer(node.input[1])
|
||||
|
||||
if self.quantizer.is_per_channel() and scale_is_initializer:
|
||||
channel_axis = self.quantizer.qdq_op_type_per_channel_support_to_axis.get(node.op_type, 1)
|
||||
self.quantizer.quantize_weight_tensor_per_channel(node.input[1], axis=channel_axis)
|
||||
elif scale_is_initializer:
|
||||
self.quantizer.quantize_weight_tensor(node.input[1])
|
||||
else:
|
||||
self.quantizer.quantize_activation_tensor(node.input[1])
|
||||
|
||||
# Bias
|
||||
self.quantizer.quantize_bias_tensor(node.input[2], node.input[0], node.input[1])
|
||||
|
||||
# Output
|
||||
if not self.disable_qdq_for_node_output:
|
||||
for output_name in node.output:
|
||||
self.quantizer.quantize_activation_tensor(output_name)
|
||||
|
|
@ -85,11 +85,22 @@ class QLinearSoftmax(QuantOperatorBase):
|
|||
class QDQSoftmax(QDQOperatorBase):
|
||||
def quantize(self):
|
||||
super().quantize()
|
||||
symmetric = self.quantizer.is_activation_symmetric
|
||||
output_name = self.node.output[0]
|
||||
quant_overrides = self.quantizer.get_per_tensor_quant_overrides(output_name)
|
||||
|
||||
# Enforce Softmax range: 0.0 to 1.0
|
||||
rmin, rmax = 0.0, 1.0
|
||||
qmin, qmax = get_qmin_qmax_for_qType(self.quantizer.activation_qType, symmetric=symmetric)
|
||||
out_zero_point, out_scale = compute_scale_zp(rmin, rmax, qmin, qmax, symmetric=symmetric)
|
||||
quant_type = self.quantizer.activation_qType
|
||||
if "quant_type" in quant_overrides:
|
||||
quant_type = quant_overrides["quant_type"].tensor_type
|
||||
|
||||
self.quantizer.set_quant_scale_zp(self.node.output[0], (out_scale, out_zero_point))
|
||||
if "scale" in quant_overrides and "zero_point" in quant_overrides:
|
||||
out_zero_point, out_scale = quant_overrides["zero_point"], quant_overrides["scale"]
|
||||
else:
|
||||
# Unless overridden by the user, force Softmax to range from 0.0 to 1.0
|
||||
rmin = quant_overrides.get("rmin", 0.0)
|
||||
rmax = quant_overrides.get("rmax", 1.0)
|
||||
symmetric = quant_overrides.get("symmetric", self.quantizer.is_activation_symmetric)
|
||||
reduce_range = quant_overrides.get("reduce_range", False)
|
||||
qmin, qmax = get_qmin_qmax_for_qType(quant_type, reduce_range=reduce_range, symmetric=symmetric)
|
||||
out_zero_point, out_scale = compute_scale_zp(rmin, rmax, qmin, qmax, symmetric=symmetric)
|
||||
|
||||
self.quantizer.set_quant_scale_zp(output_name, (out_scale, out_zero_point))
|
||||
|
|
|
|||
|
|
@ -204,6 +204,17 @@ class QDQQuantizer(ONNXQuantizer):
|
|||
logging.warning(f"only support per-channel quantization on weight. Tensor: {tensor_name} is not quantized.")
|
||||
|
||||
def quantize_bias_tensor(self, bias_name, input_name, weight_name, beta=1.0):
|
||||
# If the user provided quantization overrides for this tensor, treat it as a regular weight.
|
||||
if self.tensor_quant_overrides.get(bias_name):
|
||||
logging.info(
|
||||
f"Quantizing bias tensor '{bias_name}' as a weight due to the presence of user-specified overrides"
|
||||
)
|
||||
if self.per_channel:
|
||||
self.quantize_weight_tensor_per_channel(bias_name, 0)
|
||||
else:
|
||||
self.quantize_weight_tensor(bias_name)
|
||||
return
|
||||
|
||||
weight = find_by_name(bias_name, self.model.initializer())
|
||||
if weight is not None:
|
||||
if weight.data_type == onnx_proto.TensorProto.FLOAT:
|
||||
|
|
|
|||
|
|
@ -260,13 +260,17 @@ def compute_scale_zp_float8(element_type, std):
|
|||
return [zero, scale]
|
||||
|
||||
|
||||
def quantize_data(data, qType, symmetric, reduce_range=False, min_real_range=None):
|
||||
def quantize_data(
|
||||
data, qType, symmetric, reduce_range=False, min_real_range=None, rmin_override=None, rmax_override=None
|
||||
):
|
||||
"""
|
||||
:param data: data to quantize
|
||||
:param qType: data type to quantize to. Supported types UINT8 and INT8
|
||||
:param symmetric: whether symmetric quantization is used or not. This is applied to INT8.
|
||||
:parameter reduce_range: True if the quantization range should be reduced. Defaults to False.
|
||||
:parameter min_real_range: Minimum floating-point range (i.e., rmax - rmin) to enforce. Defaults to None.
|
||||
:parameter rmin_override: The value of rmin to use if not None. Otherwise, uses min(data).
|
||||
:parameter rmax_override: The value of rmax to use if not None. Otherwise, uses max(data).
|
||||
:return: minimum, maximum, zero point, scale, and quantized weights
|
||||
|
||||
To pack weights, we compute a linear transformation
|
||||
|
|
@ -284,13 +288,19 @@ def quantize_data(data, qType, symmetric, reduce_range=False, min_real_range=Non
|
|||
- *S*: scale
|
||||
- *z*: zero point
|
||||
"""
|
||||
rmin = 0
|
||||
rmax = 0
|
||||
|
||||
if rmin_override is not None:
|
||||
rmin = rmin_override
|
||||
else:
|
||||
rmin = min(data) if len(data) else 0
|
||||
|
||||
if rmax_override is not None:
|
||||
rmax = rmax_override
|
||||
else:
|
||||
rmax = max(data) if len(data) else 0
|
||||
|
||||
zero_point = 0
|
||||
scale = 1.0
|
||||
if len(data):
|
||||
rmin = min(data)
|
||||
rmax = max(data)
|
||||
|
||||
if qType == TensorProto.FLOAT8E4M3FN:
|
||||
if reduce_range:
|
||||
|
|
|
|||
|
|
@ -155,6 +155,33 @@ class StaticQuantConfig(QuantConfig):
|
|||
SmoothQuantFolding = True/False :
|
||||
Default is True. It only works if SmoothQuant is True. If enabled, inserted Mul ops during
|
||||
SmoothQuant will be folded into the previous op if the previous op is foldable.
|
||||
UseQDQContribOps = True/False :
|
||||
Default is False. If enabled, the inserted QuantizeLinear and DequantizeLinear ops will have the
|
||||
`com.microsoft` domain, which forces use of ONNX Runtime's QuantizeLinear and DequantizeLinear
|
||||
contrib op implementations. The contrib op implementations may support features not standardized
|
||||
into the ONNX specification (e.g., 16-bit quantization types).
|
||||
MinimumRealRange = float|None :
|
||||
Default is None. If set to a floating-point value, the calculation of the quantization parameters
|
||||
(i.e., scale and zero point) will enforce a minimum range between rmin and rmax. If (rmax-rmin)
|
||||
is less than the specified minimum range, rmax will be set to rmin + MinimumRealRange. This is
|
||||
necessary for EPs like QNN that require a minimum floating-point range when determining
|
||||
quantization parameters.
|
||||
TensorQuantOverrides = dictionary :
|
||||
Default is {}. Set tensor quantization overrides. The key is a tensor name and the value is a
|
||||
list of dictionaries. For per-tensor quantization, the list contains a single dictionary. For
|
||||
per-channel quantization, the list contains a dictionary for each channel in the tensor.
|
||||
Each dictionary contains optional overrides with the following keys and values.
|
||||
'quant_type' = QuantType : The tensor's quantization data type.
|
||||
'scale' = Float : The scale value to use. Must also specify `zero_point` if set.
|
||||
'zero_point' = Int : The zero-point value to use. Must also specify `scale` is set.
|
||||
'symmetric' = Bool : If the tensor should use symmetric quantization. Invalid if also
|
||||
set `scale` or `zero_point`.
|
||||
'reduce_range' = Bool : If the quantization range should be reduced. Invalid if also
|
||||
set `scale` or `zero_point`.
|
||||
'rmax' = Float : Override the maximum real tensor value in calibration data.
|
||||
Invalid if also set `scale` or `zero_point`.
|
||||
'rmin' = Float : Override the minimum real tensor value in calibration data.
|
||||
Invalid if also set `scale` or `zero_point`.
|
||||
execution_provider : A enum indicates the Execution Provider such as: CPU, TRT, NNAPI, SNE, etc.
|
||||
Raises:
|
||||
ValueError: Raise ValueError if execution provider is unknown
|
||||
|
|
@ -376,6 +403,22 @@ def quantize_static(
|
|||
is less than the specified minimum range, rmax will be set to rmin + MinimumRealRange. This is
|
||||
necessary for EPs like QNN that require a minimum floating-point range when determining
|
||||
quantization parameters.
|
||||
TensorQuantOverrides = dictionary :
|
||||
Default is {}. Set tensor quantization overrides. The key is a tensor name and the value is a
|
||||
list of dictionaries. For per-tensor quantization, the list contains a single dictionary. For
|
||||
per-channel quantization, the list contains a dictionary for each channel in the tensor.
|
||||
Each dictionary contains optional overrides with the following keys and values.
|
||||
'quant_type' = QuantType : The tensor's quantization data type.
|
||||
'scale' = Float : The scale value to use. Must also specify `zero_point` if set.
|
||||
'zero_point' = Int : The zero-point value to use. Must also specify `scale` is set.
|
||||
'symmetric' = Bool : If the tensor should use symmetric quantization. Invalid if also
|
||||
set `scale` or `zero_point`.
|
||||
'reduce_range' = Bool : If the quantization range should be reduced. Invalid if also
|
||||
set `scale` or `zero_point`.
|
||||
'rmax' = Float : Override the maximum real tensor value in calibration data.
|
||||
Invalid if also set `scale` or `zero_point`.
|
||||
'rmin' = Float : Override the minimum real tensor value in calibration data.
|
||||
Invalid if also set `scale` or `zero_point`.
|
||||
"""
|
||||
if activation_type == QuantType.QFLOAT8E4M3FN or weight_type == QuantType.QFLOAT8E4M3FN:
|
||||
if calibrate_method != CalibrationMethod.Distribution:
|
||||
|
|
|
|||
|
|
@ -10,10 +10,10 @@ from .operators.embed_layernorm import EmbedLayerNormalizationQuant
|
|||
from .operators.gather import GatherQuant, QDQGather
|
||||
from .operators.gavgpool import QGlobalAveragePool
|
||||
from .operators.gemm import QDQGemm, QLinearGemm
|
||||
from .operators.instnorm import QDQInstanceNormalization
|
||||
from .operators.lstm import LSTMQuant
|
||||
from .operators.matmul import MatMulInteger, QDQMatMul, QLinearMatMul
|
||||
from .operators.maxpool import QDQMaxPool, QMaxPool
|
||||
from .operators.norm import QDQNormalization
|
||||
from .operators.pad import QPad
|
||||
from .operators.pooling import QLinearPool
|
||||
from .operators.qdq_base_operator import QDQOperatorBase
|
||||
|
|
@ -81,7 +81,8 @@ QDQRegistry = {
|
|||
"Gather": QDQGather,
|
||||
"Softmax": QDQSoftmax,
|
||||
"Where": QDQWhere,
|
||||
"InstanceNormalization": QDQInstanceNormalization,
|
||||
"InstanceNormalization": QDQNormalization,
|
||||
"LayerNormalization": QDQNormalization,
|
||||
}
|
||||
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,467 @@
|
|||
#!/usr/bin/env python
|
||||
# -------------------------------------------------------------------------
|
||||
# Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
# Licensed under the MIT License. See License.txt in the project root for
|
||||
# license information.
|
||||
# --------------------------------------------------------------------------
|
||||
|
||||
import struct
|
||||
import unittest
|
||||
|
||||
import numpy as np
|
||||
import onnx
|
||||
|
||||
from onnxruntime import quantization
|
||||
from onnxruntime.quantization.quant_utils import compute_scale_zp, get_qmin_qmax_for_qType
|
||||
|
||||
|
||||
class TestTensorQuantOverridesOption(unittest.TestCase):
|
||||
def setUp(self):
|
||||
self.activations = [
|
||||
np.array([[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]], dtype="float32"),
|
||||
]
|
||||
|
||||
self.weight = np.array([[[-1.0, -2.0], [1.0, 2.0]], [[-0.5, -1.5], [0.5, 1.5]]], dtype=np.float32)
|
||||
self.bias = np.array([0.0, 1.0], dtype=np.float32)
|
||||
self.default_act_qtype = onnx.TensorProto.UINT8
|
||||
self.default_wgt_qtype = onnx.TensorProto.UINT8
|
||||
self.default_wgt_qtype_per_channel = onnx.TensorProto.INT8
|
||||
self.default_bias_qtype = onnx.TensorProto.INT32
|
||||
|
||||
self.default_zp_scales = {
|
||||
"INP": (0, np.float32(0.0235294122248888)),
|
||||
"SIG_OUT": (0, np.float32(0.003911871928721666)),
|
||||
"WGT": (128, np.float32(0.01568627543747425)),
|
||||
"BIAS": (0, np.float32(0.0000613626980339177)), # zp == 0, scale = weight_scale * sig_out_scale
|
||||
"OUT": (0, np.float32(0.005075461231172085)),
|
||||
}
|
||||
self.default_zp_scales_per_channel = {
|
||||
"INP": (0, np.float32(0.0235294122248888)),
|
||||
"SIG_OUT": (0, np.float32(0.003911871928721666)),
|
||||
"WGT": ([0, 0], [np.float32(0.015748031437397003), np.float32(0.011811023578047752)]),
|
||||
"BIAS": ([0, 0], [np.float32(0.00006160428165458143), np.float32(0.00004620321124093607)]),
|
||||
"OUT": (0, np.float32(0.005075461231172085)),
|
||||
}
|
||||
|
||||
def perform_qdq_quantization(self, output_model_name, tensor_quant_overrides=None, per_channel=False):
|
||||
# (input)
|
||||
# |
|
||||
# Sigmoid
|
||||
# |
|
||||
# Conv
|
||||
# |
|
||||
# (output)
|
||||
|
||||
inp = onnx.helper.make_tensor_value_info("INP", onnx.TensorProto.FLOAT, self.activations[0].shape)
|
||||
sigmoid_node = onnx.helper.make_node("Sigmoid", ["INP"], ["SIG_OUT"])
|
||||
|
||||
out = onnx.helper.make_tensor_value_info("OUT", onnx.TensorProto.FLOAT, [None, None, None])
|
||||
wgt_init = onnx.numpy_helper.from_array(self.weight, "WGT")
|
||||
bias_init = onnx.numpy_helper.from_array(self.bias, "BIAS")
|
||||
conv_node = onnx.helper.make_node("Conv", ["SIG_OUT", "WGT", "BIAS"], ["OUT"])
|
||||
|
||||
graph = onnx.helper.make_graph(
|
||||
[sigmoid_node, conv_node], "test", [inp], [out], initializer=[wgt_init, bias_init]
|
||||
)
|
||||
model = onnx.helper.make_model(graph, opset_imports=[onnx.helper.make_opsetid("", 13)])
|
||||
onnx.save(model, "model.onnx")
|
||||
|
||||
# Quantize model
|
||||
class DummyDataReader(quantization.CalibrationDataReader):
|
||||
def __init__(self, activations):
|
||||
self.iterator = ({"INP": act} for act in activations)
|
||||
|
||||
def get_next(self):
|
||||
return next(self.iterator, None)
|
||||
|
||||
extra_options = {}
|
||||
if tensor_quant_overrides is not None:
|
||||
extra_options["TensorQuantOverrides"] = tensor_quant_overrides
|
||||
|
||||
quantization.quantize_static(
|
||||
model_input="model.onnx",
|
||||
model_output=output_model_name,
|
||||
calibration_data_reader=DummyDataReader(self.activations),
|
||||
quant_format=quantization.QuantFormat.QDQ,
|
||||
activation_type=self.default_act_qtype,
|
||||
weight_type=self.default_wgt_qtype,
|
||||
per_channel=per_channel,
|
||||
op_types_to_quantize=["Conv", "Sigmoid"],
|
||||
extra_options=extra_options,
|
||||
)
|
||||
|
||||
# Extract quantization parameters: scales and zero points for activations and weights.
|
||||
model = onnx.load(output_model_name)
|
||||
inp_zp = next(init for init in model.graph.initializer if init.name == "INP_zero_point")
|
||||
inp_sc = next(init for init in model.graph.initializer if init.name == "INP_scale")
|
||||
sig_out_zp = next(init for init in model.graph.initializer if init.name == "SIG_OUT_zero_point")
|
||||
sig_out_sc = next(init for init in model.graph.initializer if init.name == "SIG_OUT_scale")
|
||||
wgt_zp = next(init for init in model.graph.initializer if init.name == "WGT_zero_point")
|
||||
wgt_sc = next(init for init in model.graph.initializer if init.name == "WGT_scale")
|
||||
bias_zp = next(
|
||||
init
|
||||
for init in model.graph.initializer
|
||||
if init.name == "BIAS_quantized_zero_point" or init.name == "BIAS_zero_point"
|
||||
)
|
||||
bias_sc = next(
|
||||
init for init in model.graph.initializer if init.name == "BIAS_quantized_scale" or init.name == "BIAS_scale"
|
||||
)
|
||||
out_zp = next(init for init in model.graph.initializer if init.name == "OUT_zero_point")
|
||||
out_sc = next(init for init in model.graph.initializer if init.name == "OUT_scale")
|
||||
|
||||
# Return quantization parameters
|
||||
return inp_zp, inp_sc, sig_out_zp, sig_out_sc, wgt_zp, wgt_sc, bias_zp, bias_sc, out_zp, out_sc
|
||||
|
||||
def test_qdq_default(self):
|
||||
"""
|
||||
Test default behavior without specifying the TensorQuantOverrides option.
|
||||
"""
|
||||
(
|
||||
inp_zp,
|
||||
inp_sc,
|
||||
sig_out_zp,
|
||||
sig_out_sc,
|
||||
wgt_zp,
|
||||
wgt_sc,
|
||||
bias_zp,
|
||||
bias_sc,
|
||||
out_zp,
|
||||
out_sc,
|
||||
) = self.perform_qdq_quantization(
|
||||
"model_default_quant_overrides.onnx",
|
||||
tensor_quant_overrides=None, # default behavior
|
||||
)
|
||||
|
||||
# No overrides set. Expect default values
|
||||
self.assertEqual(inp_zp.int32_data[0], self.default_zp_scales["INP"][0])
|
||||
self.assertEqual(inp_zp.data_type, self.default_act_qtype)
|
||||
self.assertEqual(inp_sc.float_data[0], self.default_zp_scales["INP"][1])
|
||||
|
||||
self.assertEqual(sig_out_zp.int32_data[0], self.default_zp_scales["SIG_OUT"][0])
|
||||
self.assertEqual(sig_out_zp.data_type, self.default_act_qtype)
|
||||
self.assertEqual(sig_out_sc.float_data[0], self.default_zp_scales["SIG_OUT"][1])
|
||||
|
||||
self.assertEqual(wgt_zp.int32_data[0], self.default_zp_scales["WGT"][0])
|
||||
self.assertEqual(wgt_zp.data_type, self.default_wgt_qtype)
|
||||
self.assertEqual(wgt_sc.float_data[0], self.default_zp_scales["WGT"][1])
|
||||
|
||||
self.assertEqual(bias_zp.int32_data[0], self.default_zp_scales["BIAS"][0])
|
||||
self.assertEqual(bias_zp.data_type, self.default_bias_qtype)
|
||||
self.assertEqual(bias_sc.float_data[0], self.default_zp_scales["BIAS"][1])
|
||||
|
||||
self.assertEqual(out_zp.int32_data[0], self.default_zp_scales["OUT"][0])
|
||||
self.assertEqual(out_zp.data_type, self.default_act_qtype)
|
||||
self.assertEqual(out_sc.float_data[0], self.default_zp_scales["OUT"][1])
|
||||
|
||||
def test_qdq_default_per_channel(self):
|
||||
"""
|
||||
Test default per-channel behavior without specifying the TensorQuantOverrides option.
|
||||
"""
|
||||
(
|
||||
inp_zp,
|
||||
inp_sc,
|
||||
sig_out_zp,
|
||||
sig_out_sc,
|
||||
wgt_zp,
|
||||
wgt_sc,
|
||||
bias_zp,
|
||||
bias_sc,
|
||||
out_zp,
|
||||
out_sc,
|
||||
) = self.perform_qdq_quantization(
|
||||
"model_default_per_channel_quant_overrides.onnx",
|
||||
tensor_quant_overrides=None, # default behavior
|
||||
per_channel=True,
|
||||
)
|
||||
|
||||
# No overrides set. Expect default values
|
||||
self.assertEqual(inp_zp.int32_data[0], self.default_zp_scales["INP"][0])
|
||||
self.assertEqual(inp_zp.data_type, self.default_act_qtype)
|
||||
self.assertEqual(inp_sc.float_data[0], self.default_zp_scales["INP"][1])
|
||||
|
||||
self.assertEqual(sig_out_zp.int32_data[0], self.default_zp_scales["SIG_OUT"][0])
|
||||
self.assertEqual(sig_out_zp.data_type, self.default_act_qtype)
|
||||
self.assertEqual(sig_out_sc.float_data[0], self.default_zp_scales["SIG_OUT"][1])
|
||||
|
||||
self.assertEqual(wgt_zp.data_type, self.default_wgt_qtype_per_channel)
|
||||
for index, zp in enumerate(self.default_zp_scales_per_channel["WGT"][0]):
|
||||
self.assertEqual(wgt_zp.int32_data[index], zp)
|
||||
for index, scale in enumerate(self.default_zp_scales_per_channel["WGT"][1]):
|
||||
self.assertEqual(wgt_sc.float_data[index], scale)
|
||||
|
||||
self.assertEqual(bias_zp.data_type, self.default_bias_qtype)
|
||||
|
||||
num_bias_zps = len(self.default_zp_scales_per_channel["BIAS"][0])
|
||||
actual_bias_zps = struct.unpack(f"<{num_bias_zps}i", bias_zp.raw_data)
|
||||
for index, zp in enumerate(self.default_zp_scales_per_channel["BIAS"][0]):
|
||||
self.assertEqual(actual_bias_zps[index], zp)
|
||||
|
||||
num_bias_scales = len(self.default_zp_scales_per_channel["BIAS"][1])
|
||||
actual_bias_scales = struct.unpack(f"<{num_bias_scales}f", bias_sc.raw_data)
|
||||
for index, scale in enumerate(self.default_zp_scales_per_channel["BIAS"][1]):
|
||||
self.assertEqual(actual_bias_scales[index], scale)
|
||||
|
||||
self.assertEqual(out_zp.int32_data[0], self.default_zp_scales["OUT"][0])
|
||||
self.assertEqual(out_zp.data_type, self.default_act_qtype)
|
||||
self.assertEqual(out_sc.float_data[0], self.default_zp_scales["OUT"][1])
|
||||
|
||||
def test_qdq_overrides1(self):
|
||||
"""
|
||||
Test overriding:
|
||||
- scale/zp for Sigmoid output
|
||||
- quant_type, symmetric, reduce_range for Conv weight
|
||||
- quant_type, symmetric, reduce_range for Conv bias
|
||||
"""
|
||||
inp_zp, inp_sc, sig_out_zp, sig_out_sc, wgt_zp, wgt_sc, bias_zp, bias_sc, _, _ = self.perform_qdq_quantization(
|
||||
"model_quant_overrides1.onnx",
|
||||
tensor_quant_overrides={
|
||||
"SIG_OUT": [{"scale": 1.0, "zero_point": 127}],
|
||||
"WGT": [{"quant_type": quantization.QuantType.QInt8, "symmetric": True, "reduce_range": True}],
|
||||
"BIAS": [{"quant_type": quantization.QuantType.QInt8, "symmetric": True, "reduce_range": True}],
|
||||
},
|
||||
)
|
||||
|
||||
# Input should have same quant params
|
||||
self.assertEqual(inp_zp.int32_data[0], self.default_zp_scales["INP"][0])
|
||||
self.assertEqual(inp_zp.data_type, self.default_act_qtype)
|
||||
self.assertEqual(inp_sc.float_data[0], self.default_zp_scales["INP"][1])
|
||||
|
||||
# Sigmoid output should have overridden scale/zp
|
||||
self.assertEqual(sig_out_zp.int32_data[0], 127)
|
||||
self.assertEqual(sig_out_zp.data_type, self.default_act_qtype)
|
||||
self.assertEqual(sig_out_sc.float_data[0], np.float32(1.0))
|
||||
|
||||
# Weight should have different type, zero_point, and scale
|
||||
self.assertEqual(wgt_zp.data_type, quantization.QuantType.QInt8.tensor_type)
|
||||
|
||||
wgt_qmin, wgt_qmax = get_qmin_qmax_for_qType(wgt_zp.data_type, reduce_range=True, symmetric=True)
|
||||
wgt_rmin, wgt_rmax = np.min(self.weight), np.max(self.weight)
|
||||
new_wgt_zp, new_wgt_sc = compute_scale_zp(wgt_rmin, wgt_rmax, wgt_qmin, wgt_qmax, symmetric=True)
|
||||
self.assertEqual(wgt_zp.int32_data[0], new_wgt_zp)
|
||||
self.assertEqual(wgt_sc.float_data[0], np.float32(new_wgt_sc))
|
||||
|
||||
# Bias should now be treated as a weight and should have different type, zero_point, and scale
|
||||
self.assertEqual(bias_zp.data_type, quantization.QuantType.QInt8.tensor_type)
|
||||
|
||||
bias_qmin, bias_qmax = get_qmin_qmax_for_qType(bias_zp.data_type, reduce_range=True, symmetric=True)
|
||||
bias_rmin, bias_rmax = np.min(self.bias), np.max(self.bias)
|
||||
new_bias_zp, new_bias_sc = compute_scale_zp(bias_rmin, bias_rmax, bias_qmin, bias_qmax, symmetric=True)
|
||||
self.assertEqual(bias_zp.int32_data[0], new_bias_zp)
|
||||
self.assertEqual(bias_sc.float_data[0], np.float32(new_bias_sc))
|
||||
|
||||
def test_qdq_overrides2(self):
|
||||
"""
|
||||
Test overriding rmin/rmax for Sigmoid output.
|
||||
"""
|
||||
sigmoid_rmin, sigmoid_rmax = 0.0, 0.5
|
||||
inp_zp, inp_sc, sig_out_zp, sig_out_sc, _, _, _, _, _, _ = self.perform_qdq_quantization(
|
||||
"model_quant_overrides2.onnx",
|
||||
tensor_quant_overrides={"SIG_OUT": [{"rmin": sigmoid_rmin, "rmax": sigmoid_rmax}]},
|
||||
)
|
||||
|
||||
# Input should have same quant params
|
||||
self.assertEqual(inp_zp.int32_data[0], self.default_zp_scales["INP"][0])
|
||||
self.assertEqual(inp_zp.data_type, self.default_act_qtype)
|
||||
self.assertEqual(inp_sc.float_data[0], self.default_zp_scales["INP"][1])
|
||||
|
||||
# Sigmoid output should have different scale/zp due to overridden rmin/rmax
|
||||
self.assertEqual(sig_out_zp.data_type, self.default_act_qtype)
|
||||
|
||||
sigmoid_qmin, sigmoid_qmax = get_qmin_qmax_for_qType(sig_out_zp.data_type)
|
||||
new_sigmoid_zp, new_sigmoid_sc = compute_scale_zp(sigmoid_rmin, sigmoid_rmax, sigmoid_qmin, sigmoid_qmax)
|
||||
self.assertEqual(sig_out_zp.int32_data[0], new_sigmoid_zp)
|
||||
self.assertEqual(sig_out_sc.float_data[0], np.float32(new_sigmoid_sc))
|
||||
|
||||
def test_qdq_overrides3(self):
|
||||
"""
|
||||
Test overriding rmin and rmax for Conv weight
|
||||
"""
|
||||
wgt_rmin, wgt_rmax = 0.0, 1.0
|
||||
_, _, _, _, wgt_zp, wgt_sc, _, _, _, _ = self.perform_qdq_quantization(
|
||||
"model_quant_overrides3.onnx",
|
||||
tensor_quant_overrides={
|
||||
"WGT": [{"rmin": wgt_rmin, "rmax": wgt_rmax}],
|
||||
},
|
||||
)
|
||||
|
||||
# Weight should have different zero_point and scale
|
||||
self.assertEqual(wgt_zp.data_type, self.default_wgt_qtype)
|
||||
self.assertNotEqual(wgt_rmin, np.min(self.weight))
|
||||
self.assertNotEqual(wgt_rmax, np.max(self.weight))
|
||||
|
||||
wgt_qmin, wgt_qmax = get_qmin_qmax_for_qType(wgt_zp.data_type)
|
||||
new_wgt_zp, new_wgt_sc = compute_scale_zp(wgt_rmin, wgt_rmax, wgt_qmin, wgt_qmax)
|
||||
self.assertEqual(wgt_zp.int32_data[0], new_wgt_zp)
|
||||
self.assertEqual(wgt_sc.float_data[0], np.float32(new_wgt_sc))
|
||||
|
||||
def test_qdq_overrides4(self):
|
||||
"""
|
||||
Test overriding scale and zero_point for Conv weight
|
||||
"""
|
||||
wgt_zp_val, wgt_scale_val = 4, 0.5
|
||||
_, _, _, _, wgt_zp, wgt_sc, _, _, _, _ = self.perform_qdq_quantization(
|
||||
"model_quant_overrides4.onnx",
|
||||
tensor_quant_overrides={
|
||||
"WGT": [{"zero_point": wgt_zp_val, "scale": wgt_scale_val}],
|
||||
},
|
||||
)
|
||||
|
||||
# Weight should have have the expected zero_point and scale
|
||||
self.assertEqual(wgt_zp.data_type, self.default_wgt_qtype)
|
||||
self.assertEqual(wgt_zp.int32_data[0], wgt_zp_val)
|
||||
self.assertEqual(wgt_sc.float_data[0], np.float32(wgt_scale_val))
|
||||
|
||||
def test_qdq_overrides_per_channel1(self):
|
||||
"""
|
||||
Test per-channel overriding of scale/zero_point for Conv weight and bias.
|
||||
"""
|
||||
zp_vals, scale_vals = [2, 4], [0.5, 0.2]
|
||||
(
|
||||
_,
|
||||
_,
|
||||
_,
|
||||
_,
|
||||
wgt_zp,
|
||||
wgt_sc,
|
||||
bias_zp,
|
||||
bias_sc,
|
||||
_,
|
||||
_,
|
||||
) = self.perform_qdq_quantization(
|
||||
"model_per_channel_quant_overrides1.onnx",
|
||||
tensor_quant_overrides={
|
||||
"WGT": [
|
||||
{"zero_point": zp_vals[0], "scale": scale_vals[0]},
|
||||
{"zero_point": zp_vals[1], "scale": scale_vals[1]},
|
||||
],
|
||||
"BIAS": [
|
||||
{"zero_point": zp_vals[0], "scale": scale_vals[0]},
|
||||
{"zero_point": zp_vals[1], "scale": scale_vals[1]},
|
||||
],
|
||||
},
|
||||
per_channel=True,
|
||||
)
|
||||
|
||||
self.assertEqual(wgt_zp.data_type, self.default_wgt_qtype_per_channel)
|
||||
for index, zp in enumerate(zp_vals):
|
||||
self.assertEqual(wgt_zp.int32_data[index], zp)
|
||||
for index, scale in enumerate(scale_vals):
|
||||
self.assertEqual(wgt_sc.float_data[index], np.float32(scale))
|
||||
|
||||
# NOTE: Bias with overrides is treated as a weight.
|
||||
self.assertEqual(bias_zp.data_type, self.default_wgt_qtype_per_channel)
|
||||
for index, zp in enumerate(zp_vals):
|
||||
self.assertEqual(bias_zp.int32_data[index], zp)
|
||||
for index, scale in enumerate(scale_vals):
|
||||
self.assertEqual(bias_sc.float_data[index], np.float32(scale))
|
||||
|
||||
def test_qdq_overrides_per_channel2(self):
|
||||
"""
|
||||
Test per-channel overriding of rmin, rmax, reduce_range, and quant_type for Conv weight.
|
||||
"""
|
||||
rmin_vals = [0.0, 0.2]
|
||||
rmax_vals = [1.0, 0.8]
|
||||
quant_type = quantization.QuantType.QUInt8
|
||||
reduce_ranges = [True, False]
|
||||
(
|
||||
_,
|
||||
_,
|
||||
_,
|
||||
_,
|
||||
wgt_zp,
|
||||
wgt_sc,
|
||||
bias_zp,
|
||||
bias_sc,
|
||||
_,
|
||||
_,
|
||||
) = self.perform_qdq_quantization(
|
||||
"model_per_channel_quant_overrides2.onnx",
|
||||
tensor_quant_overrides={
|
||||
"WGT": [
|
||||
{
|
||||
"quant_type": quant_type,
|
||||
"rmin": rmin_vals[0],
|
||||
"rmax": rmax_vals[0],
|
||||
"reduce_range": reduce_ranges[0],
|
||||
},
|
||||
{
|
||||
"quant_type": quant_type,
|
||||
"rmin": rmin_vals[1],
|
||||
"rmax": rmax_vals[1],
|
||||
"reduce_range": reduce_ranges[1],
|
||||
},
|
||||
],
|
||||
},
|
||||
per_channel=True,
|
||||
)
|
||||
|
||||
self.assertEqual(wgt_zp.data_type, quant_type.tensor_type)
|
||||
for index, (zp, scale) in enumerate(zip(wgt_zp.int32_data, wgt_sc.float_data)):
|
||||
wgt_qmin, wgt_qmax = get_qmin_qmax_for_qType(wgt_zp.data_type, reduce_range=reduce_ranges[index])
|
||||
expected_zp, expected_scale = compute_scale_zp(rmin_vals[index], rmax_vals[index], wgt_qmin, wgt_qmax)
|
||||
self.assertEqual(zp, expected_zp)
|
||||
self.assertEqual(scale, np.float32(expected_scale))
|
||||
|
||||
def test_override_validation_nonexisting_tensor(self):
|
||||
"""
|
||||
Test that specifying a non-existing tensor should fail.
|
||||
"""
|
||||
with self.assertRaises(ValueError) as context:
|
||||
self.perform_qdq_quantization(
|
||||
"model_validation.onnx",
|
||||
tensor_quant_overrides={"NON_EXISTING": [{"rmin": 0.0, "rmax": 0.5}]},
|
||||
)
|
||||
|
||||
self.assertIn("is not present in the model", str(context.exception))
|
||||
|
||||
def test_override_validation_scale_missing_zp(self):
|
||||
"""
|
||||
Test that specifying a scale without zero_point should fail.
|
||||
"""
|
||||
with self.assertRaises(ValueError) as context:
|
||||
self.perform_qdq_quantization(
|
||||
"model_validation.onnx",
|
||||
tensor_quant_overrides={"SIG_OUT": [{"scale": 0.0}]},
|
||||
)
|
||||
|
||||
self.assertIn("Must provide both 'scale' and 'zero_point'", str(context.exception))
|
||||
|
||||
def test_override_validation_bad_combination(self):
|
||||
"""
|
||||
Test that specifying a scale/zero_point with rmax/rmin/symmetric/reduce_range should fail.
|
||||
"""
|
||||
with self.assertRaises(ValueError) as context:
|
||||
self.perform_qdq_quantization(
|
||||
"model_validation.onnx",
|
||||
tensor_quant_overrides={"SIG_OUT": [{"scale": 0.0, "zero_point": 0, "rmax": 10.0}]},
|
||||
)
|
||||
|
||||
self.assertIn("option 'rmax' is invalid with 'scale' and 'zero_point'", str(context.exception))
|
||||
|
||||
with self.assertRaises(ValueError) as context:
|
||||
self.perform_qdq_quantization(
|
||||
"model_validation.onnx",
|
||||
tensor_quant_overrides={"SIG_OUT": [{"scale": 0.0, "zero_point": 0, "rmin": 10.0}]},
|
||||
)
|
||||
|
||||
self.assertIn("option 'rmin' is invalid with 'scale' and 'zero_point'", str(context.exception))
|
||||
|
||||
with self.assertRaises(ValueError) as context:
|
||||
self.perform_qdq_quantization(
|
||||
"model_validation.onnx",
|
||||
tensor_quant_overrides={"SIG_OUT": [{"scale": 0.0, "zero_point": 0, "symmetric": True}]},
|
||||
)
|
||||
|
||||
self.assertIn("option 'symmetric' is invalid with 'scale' and 'zero_point'", str(context.exception))
|
||||
|
||||
with self.assertRaises(ValueError) as context:
|
||||
self.perform_qdq_quantization(
|
||||
"model_validation.onnx",
|
||||
tensor_quant_overrides={"SIG_OUT": [{"scale": 0.0, "zero_point": 0, "reduce_range": True}]},
|
||||
)
|
||||
|
||||
self.assertIn("option 'reduce_range' is invalid with 'scale' and 'zero_point'", str(context.exception))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
1
setup.py
1
setup.py
|
|
@ -408,6 +408,7 @@ packages = [
|
|||
"onnxruntime.quantization",
|
||||
"onnxruntime.quantization.operators",
|
||||
"onnxruntime.quantization.CalTableFlatBuffers",
|
||||
"onnxruntime.quantization.execution_providers.qnn",
|
||||
"onnxruntime.transformers",
|
||||
"onnxruntime.transformers.models.bart",
|
||||
"onnxruntime.transformers.models.bert",
|
||||
|
|
|
|||
Loading…
Reference in a new issue