diff --git a/cmake/onnxruntime_python.cmake b/cmake/onnxruntime_python.cmake index 345ef2b504..b93ccf77d5 100644 --- a/cmake/onnxruntime_python.cmake +++ b/cmake/onnxruntime_python.cmake @@ -453,6 +453,9 @@ file(GLOB onnxruntime_python_quantization_operators_src CONFIGURE_DEPENDS file(GLOB onnxruntime_python_quantization_cal_table_flatbuffers_src CONFIGURE_DEPENDS "${ONNXRUNTIME_ROOT}/python/tools/quantization/CalTableFlatBuffers/*.py" ) +file(GLOB onnxruntime_python_quantization_ep_qnn_src CONFIGURE_DEPENDS + "${ONNXRUNTIME_ROOT}/python/tools/quantization/execution_providers/qnn/*.py" +) file(GLOB onnxruntime_python_transformers_src CONFIGURE_DEPENDS "${ONNXRUNTIME_ROOT}/python/tools/transformers/*.py" ) @@ -547,6 +550,8 @@ add_custom_command( COMMAND ${CMAKE_COMMAND} -E make_directory $/onnxruntime/quantization COMMAND ${CMAKE_COMMAND} -E make_directory $/onnxruntime/quantization/operators COMMAND ${CMAKE_COMMAND} -E make_directory $/onnxruntime/quantization/CalTableFlatBuffers + COMMAND ${CMAKE_COMMAND} -E make_directory $/onnxruntime/quantization/execution_providers + COMMAND ${CMAKE_COMMAND} -E make_directory $/onnxruntime/quantization/execution_providers/qnn COMMAND ${CMAKE_COMMAND} -E make_directory $/quantization COMMAND ${CMAKE_COMMAND} -E make_directory $/transformers COMMAND ${CMAKE_COMMAND} -E make_directory $/transformers/test_data/models @@ -617,6 +622,9 @@ add_custom_command( COMMAND ${CMAKE_COMMAND} -E copy ${onnxruntime_python_quantization_cal_table_flatbuffers_src} $/onnxruntime/quantization/CalTableFlatBuffers/ + COMMAND ${CMAKE_COMMAND} -E copy + ${onnxruntime_python_quantization_ep_qnn_src} + $/onnxruntime/quantization/execution_providers/qnn/ COMMAND ${CMAKE_COMMAND} -E copy ${onnxruntime_python_transformers_src} $/onnxruntime/transformers/ diff --git a/onnxruntime/python/tools/quantization/execution_providers/__init__.py b/onnxruntime/python/tools/quantization/execution_providers/__init__.py new file mode 100644 index 0000000000..e69de29bb2 diff --git a/onnxruntime/python/tools/quantization/execution_providers/qnn/__init__.py b/onnxruntime/python/tools/quantization/execution_providers/qnn/__init__.py new file mode 100644 index 0000000000..c5f0b27f75 --- /dev/null +++ b/onnxruntime/python/tools/quantization/execution_providers/qnn/__init__.py @@ -0,0 +1 @@ +from .quant_config import get_qnn_qdq_config # noqa: F401 diff --git a/onnxruntime/python/tools/quantization/execution_providers/qnn/quant_config.py b/onnxruntime/python/tools/quantization/execution_providers/qnn/quant_config.py new file mode 100644 index 0000000000..eea3a04561 --- /dev/null +++ b/onnxruntime/python/tools/quantization/execution_providers/qnn/quant_config.py @@ -0,0 +1,84 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from pathlib import Path + +import onnx + +from ...calibrate import CalibrationDataReader, CalibrationMethod +from ...quant_utils import QuantType +from ...quantize import StaticQuantConfig + +Q16_TYPES = {QuantType.QInt16, QuantType.QUInt16} +Q8_TYPES = {QuantType.QInt8, QuantType.QUInt8} +OP_TYPES_TO_EXCLUDE = {"Cast"} + + +def get_qnn_qdq_config( + model_input: Path, + calibration_data_reader: CalibrationDataReader, + calibrate_method=CalibrationMethod.MinMax, + activation_type=QuantType.QUInt8, + weight_type=QuantType.QUInt8, + per_channel=False, +): + if per_channel: + raise ValueError("QNN EP does not yet support per-channel quantization.") + + # Process model nodes to setup overrides. + model = onnx.load_model(model_input) + + op_types = set() + tensor_quant_overrides = {} + + name_to_initializer = {initializer.name: initializer for initializer in model.graph.initializer} + + for node in model.graph.node: + op_types.add(node.op_type) + + if node.op_type == "MatMul" and activation_type in Q16_TYPES and weight_type in Q8_TYPES: + weight_symmetric = weight_type == QuantType.QInt8 + + # Override initializers to use the weight_type + for input_name in node.input: + if input_name in name_to_initializer: + tensor_quant_overrides[input_name] = [{"quant_type": weight_type, "symmetric": weight_symmetric}] + elif node.op_type == "LayerNormalization" and activation_type in Q16_TYPES and weight_type in Q8_TYPES: + weight_symmetric = weight_type == QuantType.QInt8 + + # Override initializers to use the weight_type. Don't override the bias input. + for i in range(2): + input_name = node.input[i] + if input_name in name_to_initializer: + tensor_quant_overrides[input_name] = [{"quant_type": weight_type, "symmetric": weight_symmetric}] + elif node.op_type == "Sigmoid": + if activation_type == QuantType.QUInt16: + tensor_quant_overrides[node.output[0]] = [{"scale": 1.0 / 65536.0, "zero_point": 0}] + elif activation_type == QuantType.QInt16: + tensor_quant_overrides[node.output[0]] = [{"scale": 1.0 / 32768.0, "zero_point": 0}] + elif node.op_type == "Tanh": + if activation_type == QuantType.QUInt16: + tensor_quant_overrides[node.output[0]] = [{"scale": 1.0 / 32768.0, "zero_point": 32768}] + elif activation_type == QuantType.QInt16: + tensor_quant_overrides[node.output[0]] = [{"scale": 1.0 / 32768.0, "zero_point": 0}] + + extra_options = { + "MinimumRealRange": 0.0001, + "DedicatedQDQPair": False, # Let ORT optimizer duplicate DQ nodes + "TensorQuantOverrides": tensor_quant_overrides, + } + + # TODO: Remove this extra option once ORT uses an ONNX version that supports 16-bit Q/DQ ops. + if activation_type in Q16_TYPES or weight_type in Q16_TYPES: + extra_options["UseQDQContribOps"] = True + + return StaticQuantConfig( + calibration_data_reader, + calibrate_method=calibrate_method, + activation_type=activation_type, + weight_type=weight_type, + op_types_to_quantize=list(op_types.difference(OP_TYPES_TO_EXCLUDE)), + extra_options=extra_options, + ) diff --git a/onnxruntime/python/tools/quantization/onnx_quantizer.py b/onnxruntime/python/tools/quantization/onnx_quantizer.py index c1c2248bc8..f6491f32d8 100644 --- a/onnxruntime/python/tools/quantization/onnx_quantizer.py +++ b/onnxruntime/python/tools/quantization/onnx_quantizer.py @@ -37,6 +37,7 @@ from .quant_utils import ( model_has_infer_metadata, ms_domain, quantize_data, + quantize_nparray, save_and_reload_model_with_shape_infer, tensor_proto_to_array, ) @@ -49,8 +50,8 @@ class QuantizationParams: for k, v in data.items(): if not isinstance(k, str): raise TypeError(f"Keys must be strings not {type(k)}.") - if not isinstance(v, (int, float, str)): - raise TypeError(f"Values must be int, float, str not {type(v)}.") + if not isinstance(v, (int, float, str, QuantType)): + raise TypeError(f"Values must be int, float, str, or QuantType not {type(v)}.") self.data[k] = v def __iter__(self): @@ -148,6 +149,7 @@ class ONNXQuantizer: if self.mode not in QuantizationMode: raise ValueError(f"unsupported quantization mode {self.mode}") + self.tensor_quant_overrides = self._get_and_check_tensor_quant_overrides() self.quantization_params = self.calculate_quantization_params() # QuantizeRange tensor name and zero tensor name for scale and zero point calculation. @@ -167,6 +169,87 @@ class ONNXQuantizer: # to store specified scale and zeropoint instead of calculated value, tensor_name->(scale, zeropoint) self.used_scale_zp_map = {} + def _get_and_check_tensor_quant_overrides(self): + """ + Get tensor quantization overrides and check correctness. + """ + tensor_quant_overrides = self.extra_options.get("TensorQuantOverrides", {}) + + # Validate that compatible/valid overrides are provided. + if tensor_quant_overrides: + initializer_names = self.model.get_initializer_name_set() + value_info_names = set(self.value_infos.keys()) + keys_unsupported_with_scale_zp = {"symmetric", "reduce_range", "rmax", "rmin"} + + for tensor_name, quant_overrides_list in tensor_quant_overrides.items(): + if tensor_name not in initializer_names and tensor_name not in value_info_names: + raise ValueError(f"Tensor '{tensor_name}' in TensorQuantOverrides is not present in the model") + + if not isinstance(quant_overrides_list, list): + raise ValueError(f"Tensor quantization overrides for '{tensor_name}' are not in a list") + + is_initializer = tensor_name in initializer_names + if not is_initializer and len(quant_overrides_list) > 1: + raise ValueError( + f"Tensor '{tensor_name}' has a list of per-channel overrides, but is not an initializer" + ) + + quant_type = None + for index, quant_overrides in enumerate(quant_overrides_list): + if not isinstance(quant_overrides, dict): + raise ValueError( + f"Tensor quantization overrides at index {index} for '{tensor_name}' are not in a dict" + ) + + # For per-channel quantization, all channels must use the same quantization type. + # Therefore, if the user tries to override the quant_type for a channel, it must match in all + # other channels. + if index == 0: + quant_type = quant_overrides.get("quant_type") + elif quant_type != quant_overrides.get("quant_type"): + raise ValueError( + "Channel quantization types for tensor '{tensor_name}' do not match at index {index}." + ) + + has_scale = "scale" in quant_overrides + has_zero_point = "zero_point" in quant_overrides + + if (has_scale and not has_zero_point) or (has_zero_point and not has_scale): + raise ValueError( + "Must provide both 'scale' and 'zero_point' if one of the overrides is provided" + ) + + if has_scale: + for key in keys_unsupported_with_scale_zp: + if key in quant_overrides: + raise ValueError( + f"Tensor override option '{key}' is invalid with 'scale' and 'zero_point'" + ) + + return tensor_quant_overrides + + def get_per_tensor_quant_overrides(self, tensor_name): + quant_overrides_list = self.tensor_quant_overrides.get(tensor_name, [{}]) + num_overrides = len(quant_overrides_list) + if num_overrides > 1: + raise ValueError( + f"Expected tensor '{tensor_name}' to use per-tensor quantization overrides, " + f"but found {num_overrides} per-channel overrides." + ) + + return quant_overrides_list[0] if num_overrides > 0 else {} + + def get_per_channel_quant_overrides(self, tensor_name, num_channels): + quant_overrides_list = self.tensor_quant_overrides.get(tensor_name, [{} for i in range(num_channels)]) + + if len(quant_overrides_list) != num_channels: + raise ValueError( + f"Expected tensor '{tensor_name}' to have {num_channels} per-channel quantization overrides, " + f"but found {len(quant_overrides_list)} instead." + ) + + return quant_overrides_list + # routines for subgraph support def quantize_subgraph(self, subgraph, graph_key): """ @@ -587,6 +670,8 @@ class ONNXQuantizer: parameter param_name: Name of the quantization parameter. return: result, scale_name, zero_point_name, scale_shape, zero_point_shape. """ + zero_point_type = self.activation_qType + if use_scale is None or use_zeropoint is None: if self.quantization_params is None or param_name not in self.quantization_params: logging.info(f'Quantization parameters for tensor:"{param_name}" not specified') @@ -595,21 +680,21 @@ class ONNXQuantizer: params = self.quantization_params[param_name] if not isinstance(params, QuantizationParams): raise TypeError(f"Unexpected type {type(params)} for {param_name!r}.") - if params is None or len(params) != 2: + if params is None or len(params) != 3: raise ValueError( - "Quantization parameters should contain zero point and scale. " + "Quantization parameters should contain zero point, scale, quant type. " f"Specified values for output {param_name}: {params}" ) zero_point_values = [params["zero_point"]] scale_values = [params["scale"]] + zero_point_type = params["quant_type"] else: zero_point_values = [use_zeropoint] scale_values = [use_scale] zero_point_shape = [] zero_point_name = param_name + "_zero_point" - zero_point_type = self.activation_qType scale_shape = [] scale_name = param_name + "_scale" @@ -991,16 +1076,25 @@ class ONNXQuantizer: zp_name = weight.name + "_zero_point" scale_name = weight.name + "_scale" - # Update packed weight, zero point, and scale initializers + # Quantize weight data. Use quantization overrides if provided by the user. weight_data = tensor_proto_to_array(weight) - w_data = weight_data.flatten().tolist() - _, _, zero_point, scale, q_weight_data = quantize_data( - w_data, - qType, - self.is_weight_symmetric, - self.reduce_range and reduce_range, - self.min_real_range, - ) + quant_overrides = self.get_per_tensor_quant_overrides(weight.name) + if "quant_type" in quant_overrides: + qType = quant_overrides["quant_type"].tensor_type # noqa: N806 + + if "scale" in quant_overrides and "zero_point" in quant_overrides: + zero_point, scale = quant_overrides["zero_point"], quant_overrides["scale"] + q_weight_data = quantize_nparray(qType, weight_data.flatten(), scale, zero_point) + else: + _, _, zero_point, scale, q_weight_data = quantize_data( + weight_data.flatten().tolist(), + qType, + quant_overrides.get("symmetric", self.is_weight_symmetric), + reduce_range=quant_overrides.get("reduce_range", self.reduce_range and reduce_range), + min_real_range=self.min_real_range, + rmin_override=quant_overrides.get("rmin"), + rmax_override=quant_overrides.get("rmax"), + ) if qType in { onnx.TensorProto.FLOAT8E4M3FN, @@ -1076,23 +1170,43 @@ class ONNXQuantizer: weights = tensor_proto_to_array(initializer) channel_count = weights.shape[channel_axis] - rmin_list = [] - rmax_list = [] + quant_overrides_for_channels = self.get_per_channel_quant_overrides(weight_name, channel_count) + + # If user provides per-channel quantization overrides, all channels must use the same quantization type. + # So, just use the first channel's type. + if "quant_type" in quant_overrides_for_channels[0]: + weight_qType = quant_overrides_for_channels[0]["quant_type"].tensor_type # noqa: N806 + zero_point_list = [] scale_list = [] quantized_per_channel_data_list = [] for i in range(channel_count): per_channel_data = weights.take(i, channel_axis) - rmin, rmax, zero_point, scale, quantized_per_channel_data = quantize_data( - per_channel_data.flatten().tolist(), - weight_qType, - self.is_weight_symmetric - or weight_qType in (onnx_proto.TensorProto.INT8, onnx_proto.TensorProto.FLOAT8E4M3FN), - self.reduce_range and reduce_range, - self.min_real_range, - ) - rmin_list.append(rmin) - rmax_list.append(rmax) + channel_quant_overrides = quant_overrides_for_channels[i] + + if "scale" in channel_quant_overrides and "zero_point" in channel_quant_overrides: + zero_point, scale = channel_quant_overrides["zero_point"], channel_quant_overrides["scale"] + quantized_per_channel_data = quantize_nparray( + weight_qType, per_channel_data.flatten(), scale, zero_point + ) + else: + symmetric = channel_quant_overrides.get( + "symmetric", + ( + self.is_weight_symmetric + or weight_qType in (onnx_proto.TensorProto.INT8, onnx_proto.TensorProto.FLOAT8E4M3FN) + ), + ) + _, _, zero_point, scale, quantized_per_channel_data = quantize_data( + per_channel_data.flatten().tolist(), + weight_qType, + symmetric, + reduce_range=channel_quant_overrides.get("reduce_range", self.reduce_range and reduce_range), + min_real_range=self.min_real_range, + rmin_override=channel_quant_overrides.get("rmin"), + rmax_override=channel_quant_overrides.get("rmax"), + ) + zero_point_list.append(zero_point) scale_list.append(scale) quantized_per_channel_data_list.append(quantized_per_channel_data) @@ -1205,15 +1319,25 @@ class ONNXQuantizer: td = self.tensors_range[tensor_name] if not isinstance(td, TensorData): raise TypeError(f"Unexpected type {type(td)} for {tensor_name!r}.") - if self.activation_qType == onnx.TensorProto.FLOAT8E4M3FN: - zero, scale = compute_scale_zp_float8(self.activation_qType, td.avg_std[1]) - else: - rmin, rmax = td.range_value - qmin, qmax = get_qmin_qmax_for_qType(self.activation_qType, symmetric=self.is_activation_symmetric) - zero, scale = compute_scale_zp( - rmin, rmax, qmin, qmax, self.is_activation_symmetric, self.min_real_range - ) - quantization_params[tensor_name] = QuantizationParams(zero_point=zero, scale=scale) + quant_overrides = self.get_per_tensor_quant_overrides(tensor_name) + + quant_type = self.activation_qType + if "quant_type" in quant_overrides: + quant_type = quant_overrides["quant_type"].tensor_type + + if "scale" in quant_overrides and "zero_point" in quant_overrides: + zero, scale = quant_overrides["zero_point"], quant_overrides["scale"] + elif quant_type == onnx.TensorProto.FLOAT8E4M3FN: + zero, scale = compute_scale_zp_float8(quant_type, td.avg_std[1]) + else: + rmin = quant_overrides.get("rmin", td.range_value[0]) + rmax = quant_overrides.get("rmax", td.range_value[1]) + symmetric = quant_overrides.get("symmetric", self.is_activation_symmetric) + reduce_range = quant_overrides.get("reduce_range", False) + qmin, qmax = get_qmin_qmax_for_qType(quant_type, reduce_range=reduce_range, symmetric=symmetric) + zero, scale = compute_scale_zp(rmin, rmax, qmin, qmax, symmetric, self.min_real_range) + + quantization_params[tensor_name] = QuantizationParams(zero_point=zero, scale=scale, quant_type=quant_type) return quantization_params diff --git a/onnxruntime/python/tools/quantization/operators/instnorm.py b/onnxruntime/python/tools/quantization/operators/norm.py similarity index 56% rename from onnxruntime/python/tools/quantization/operators/instnorm.py rename to onnxruntime/python/tools/quantization/operators/norm.py index ff3e992a42..e825fe6075 100644 --- a/onnxruntime/python/tools/quantization/operators/instnorm.py +++ b/onnxruntime/python/tools/quantization/operators/norm.py @@ -6,24 +6,32 @@ from .qdq_base_operator import QDQOperatorBase -class QDQInstanceNormalization(QDQOperatorBase): +class QDQNormalization(QDQOperatorBase): def __init__(self, onnx_quantizer, onnx_node): super().__init__(onnx_quantizer, onnx_node) def quantize(self): node = self.node - assert node.op_type == "InstanceNormalization" + assert node.op_type == "InstanceNormalization" or node.op_type == "LayerNormalization" # Input self.quantizer.quantize_activation_tensor(node.input[0]) - if not self.disable_qdq_for_node_output: - self.quantizer.quantize_activation_tensor(node.output[0]) # Scale - if self.quantizer.is_per_channel(): - self.quantizer.quantize_weight_tensor_per_channel(node.input[1], axis=1) - else: + scale_is_initializer = self.quantizer.is_input_a_initializer(node.input[1]) + + if self.quantizer.is_per_channel() and scale_is_initializer: + channel_axis = self.quantizer.qdq_op_type_per_channel_support_to_axis.get(node.op_type, 1) + self.quantizer.quantize_weight_tensor_per_channel(node.input[1], axis=channel_axis) + elif scale_is_initializer: self.quantizer.quantize_weight_tensor(node.input[1]) + else: + self.quantizer.quantize_activation_tensor(node.input[1]) # Bias self.quantizer.quantize_bias_tensor(node.input[2], node.input[0], node.input[1]) + + # Output + if not self.disable_qdq_for_node_output: + for output_name in node.output: + self.quantizer.quantize_activation_tensor(output_name) diff --git a/onnxruntime/python/tools/quantization/operators/softmax.py b/onnxruntime/python/tools/quantization/operators/softmax.py index bd09b05ddd..76c9054caa 100644 --- a/onnxruntime/python/tools/quantization/operators/softmax.py +++ b/onnxruntime/python/tools/quantization/operators/softmax.py @@ -85,11 +85,22 @@ class QLinearSoftmax(QuantOperatorBase): class QDQSoftmax(QDQOperatorBase): def quantize(self): super().quantize() - symmetric = self.quantizer.is_activation_symmetric + output_name = self.node.output[0] + quant_overrides = self.quantizer.get_per_tensor_quant_overrides(output_name) - # Enforce Softmax range: 0.0 to 1.0 - rmin, rmax = 0.0, 1.0 - qmin, qmax = get_qmin_qmax_for_qType(self.quantizer.activation_qType, symmetric=symmetric) - out_zero_point, out_scale = compute_scale_zp(rmin, rmax, qmin, qmax, symmetric=symmetric) + quant_type = self.quantizer.activation_qType + if "quant_type" in quant_overrides: + quant_type = quant_overrides["quant_type"].tensor_type - self.quantizer.set_quant_scale_zp(self.node.output[0], (out_scale, out_zero_point)) + if "scale" in quant_overrides and "zero_point" in quant_overrides: + out_zero_point, out_scale = quant_overrides["zero_point"], quant_overrides["scale"] + else: + # Unless overridden by the user, force Softmax to range from 0.0 to 1.0 + rmin = quant_overrides.get("rmin", 0.0) + rmax = quant_overrides.get("rmax", 1.0) + symmetric = quant_overrides.get("symmetric", self.quantizer.is_activation_symmetric) + reduce_range = quant_overrides.get("reduce_range", False) + qmin, qmax = get_qmin_qmax_for_qType(quant_type, reduce_range=reduce_range, symmetric=symmetric) + out_zero_point, out_scale = compute_scale_zp(rmin, rmax, qmin, qmax, symmetric=symmetric) + + self.quantizer.set_quant_scale_zp(output_name, (out_scale, out_zero_point)) diff --git a/onnxruntime/python/tools/quantization/qdq_quantizer.py b/onnxruntime/python/tools/quantization/qdq_quantizer.py index 5c97dd20cf..187555ff76 100644 --- a/onnxruntime/python/tools/quantization/qdq_quantizer.py +++ b/onnxruntime/python/tools/quantization/qdq_quantizer.py @@ -204,6 +204,17 @@ class QDQQuantizer(ONNXQuantizer): logging.warning(f"only support per-channel quantization on weight. Tensor: {tensor_name} is not quantized.") def quantize_bias_tensor(self, bias_name, input_name, weight_name, beta=1.0): + # If the user provided quantization overrides for this tensor, treat it as a regular weight. + if self.tensor_quant_overrides.get(bias_name): + logging.info( + f"Quantizing bias tensor '{bias_name}' as a weight due to the presence of user-specified overrides" + ) + if self.per_channel: + self.quantize_weight_tensor_per_channel(bias_name, 0) + else: + self.quantize_weight_tensor(bias_name) + return + weight = find_by_name(bias_name, self.model.initializer()) if weight is not None: if weight.data_type == onnx_proto.TensorProto.FLOAT: diff --git a/onnxruntime/python/tools/quantization/quant_utils.py b/onnxruntime/python/tools/quantization/quant_utils.py index 8825d78993..9acee9d8ab 100644 --- a/onnxruntime/python/tools/quantization/quant_utils.py +++ b/onnxruntime/python/tools/quantization/quant_utils.py @@ -260,13 +260,17 @@ def compute_scale_zp_float8(element_type, std): return [zero, scale] -def quantize_data(data, qType, symmetric, reduce_range=False, min_real_range=None): +def quantize_data( + data, qType, symmetric, reduce_range=False, min_real_range=None, rmin_override=None, rmax_override=None +): """ :param data: data to quantize :param qType: data type to quantize to. Supported types UINT8 and INT8 :param symmetric: whether symmetric quantization is used or not. This is applied to INT8. :parameter reduce_range: True if the quantization range should be reduced. Defaults to False. :parameter min_real_range: Minimum floating-point range (i.e., rmax - rmin) to enforce. Defaults to None. + :parameter rmin_override: The value of rmin to use if not None. Otherwise, uses min(data). + :parameter rmax_override: The value of rmax to use if not None. Otherwise, uses max(data). :return: minimum, maximum, zero point, scale, and quantized weights To pack weights, we compute a linear transformation @@ -284,13 +288,19 @@ def quantize_data(data, qType, symmetric, reduce_range=False, min_real_range=Non - *S*: scale - *z*: zero point """ - rmin = 0 - rmax = 0 + + if rmin_override is not None: + rmin = rmin_override + else: + rmin = min(data) if len(data) else 0 + + if rmax_override is not None: + rmax = rmax_override + else: + rmax = max(data) if len(data) else 0 + zero_point = 0 scale = 1.0 - if len(data): - rmin = min(data) - rmax = max(data) if qType == TensorProto.FLOAT8E4M3FN: if reduce_range: diff --git a/onnxruntime/python/tools/quantization/quantize.py b/onnxruntime/python/tools/quantization/quantize.py index c9e9a92e2a..aed46563c2 100644 --- a/onnxruntime/python/tools/quantization/quantize.py +++ b/onnxruntime/python/tools/quantization/quantize.py @@ -155,6 +155,33 @@ class StaticQuantConfig(QuantConfig): SmoothQuantFolding = True/False : Default is True. It only works if SmoothQuant is True. If enabled, inserted Mul ops during SmoothQuant will be folded into the previous op if the previous op is foldable. + UseQDQContribOps = True/False : + Default is False. If enabled, the inserted QuantizeLinear and DequantizeLinear ops will have the + `com.microsoft` domain, which forces use of ONNX Runtime's QuantizeLinear and DequantizeLinear + contrib op implementations. The contrib op implementations may support features not standardized + into the ONNX specification (e.g., 16-bit quantization types). + MinimumRealRange = float|None : + Default is None. If set to a floating-point value, the calculation of the quantization parameters + (i.e., scale and zero point) will enforce a minimum range between rmin and rmax. If (rmax-rmin) + is less than the specified minimum range, rmax will be set to rmin + MinimumRealRange. This is + necessary for EPs like QNN that require a minimum floating-point range when determining + quantization parameters. + TensorQuantOverrides = dictionary : + Default is {}. Set tensor quantization overrides. The key is a tensor name and the value is a + list of dictionaries. For per-tensor quantization, the list contains a single dictionary. For + per-channel quantization, the list contains a dictionary for each channel in the tensor. + Each dictionary contains optional overrides with the following keys and values. + 'quant_type' = QuantType : The tensor's quantization data type. + 'scale' = Float : The scale value to use. Must also specify `zero_point` if set. + 'zero_point' = Int : The zero-point value to use. Must also specify `scale` is set. + 'symmetric' = Bool : If the tensor should use symmetric quantization. Invalid if also + set `scale` or `zero_point`. + 'reduce_range' = Bool : If the quantization range should be reduced. Invalid if also + set `scale` or `zero_point`. + 'rmax' = Float : Override the maximum real tensor value in calibration data. + Invalid if also set `scale` or `zero_point`. + 'rmin' = Float : Override the minimum real tensor value in calibration data. + Invalid if also set `scale` or `zero_point`. execution_provider : A enum indicates the Execution Provider such as: CPU, TRT, NNAPI, SNE, etc. Raises: ValueError: Raise ValueError if execution provider is unknown @@ -376,6 +403,22 @@ def quantize_static( is less than the specified minimum range, rmax will be set to rmin + MinimumRealRange. This is necessary for EPs like QNN that require a minimum floating-point range when determining quantization parameters. + TensorQuantOverrides = dictionary : + Default is {}. Set tensor quantization overrides. The key is a tensor name and the value is a + list of dictionaries. For per-tensor quantization, the list contains a single dictionary. For + per-channel quantization, the list contains a dictionary for each channel in the tensor. + Each dictionary contains optional overrides with the following keys and values. + 'quant_type' = QuantType : The tensor's quantization data type. + 'scale' = Float : The scale value to use. Must also specify `zero_point` if set. + 'zero_point' = Int : The zero-point value to use. Must also specify `scale` is set. + 'symmetric' = Bool : If the tensor should use symmetric quantization. Invalid if also + set `scale` or `zero_point`. + 'reduce_range' = Bool : If the quantization range should be reduced. Invalid if also + set `scale` or `zero_point`. + 'rmax' = Float : Override the maximum real tensor value in calibration data. + Invalid if also set `scale` or `zero_point`. + 'rmin' = Float : Override the minimum real tensor value in calibration data. + Invalid if also set `scale` or `zero_point`. """ if activation_type == QuantType.QFLOAT8E4M3FN or weight_type == QuantType.QFLOAT8E4M3FN: if calibrate_method != CalibrationMethod.Distribution: diff --git a/onnxruntime/python/tools/quantization/registry.py b/onnxruntime/python/tools/quantization/registry.py index e8bcf9107c..a693f4192b 100644 --- a/onnxruntime/python/tools/quantization/registry.py +++ b/onnxruntime/python/tools/quantization/registry.py @@ -10,10 +10,10 @@ from .operators.embed_layernorm import EmbedLayerNormalizationQuant from .operators.gather import GatherQuant, QDQGather from .operators.gavgpool import QGlobalAveragePool from .operators.gemm import QDQGemm, QLinearGemm -from .operators.instnorm import QDQInstanceNormalization from .operators.lstm import LSTMQuant from .operators.matmul import MatMulInteger, QDQMatMul, QLinearMatMul from .operators.maxpool import QDQMaxPool, QMaxPool +from .operators.norm import QDQNormalization from .operators.pad import QPad from .operators.pooling import QLinearPool from .operators.qdq_base_operator import QDQOperatorBase @@ -81,7 +81,8 @@ QDQRegistry = { "Gather": QDQGather, "Softmax": QDQSoftmax, "Where": QDQWhere, - "InstanceNormalization": QDQInstanceNormalization, + "InstanceNormalization": QDQNormalization, + "LayerNormalization": QDQNormalization, } diff --git a/onnxruntime/test/python/quantization/test_tensor_quant_overrides_option.py b/onnxruntime/test/python/quantization/test_tensor_quant_overrides_option.py new file mode 100644 index 0000000000..770f292286 --- /dev/null +++ b/onnxruntime/test/python/quantization/test_tensor_quant_overrides_option.py @@ -0,0 +1,467 @@ +#!/usr/bin/env python +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- + +import struct +import unittest + +import numpy as np +import onnx + +from onnxruntime import quantization +from onnxruntime.quantization.quant_utils import compute_scale_zp, get_qmin_qmax_for_qType + + +class TestTensorQuantOverridesOption(unittest.TestCase): + def setUp(self): + self.activations = [ + np.array([[[1.0, 2.0, 3.0], [4.0, 5.0, 6.0]]], dtype="float32"), + ] + + self.weight = np.array([[[-1.0, -2.0], [1.0, 2.0]], [[-0.5, -1.5], [0.5, 1.5]]], dtype=np.float32) + self.bias = np.array([0.0, 1.0], dtype=np.float32) + self.default_act_qtype = onnx.TensorProto.UINT8 + self.default_wgt_qtype = onnx.TensorProto.UINT8 + self.default_wgt_qtype_per_channel = onnx.TensorProto.INT8 + self.default_bias_qtype = onnx.TensorProto.INT32 + + self.default_zp_scales = { + "INP": (0, np.float32(0.0235294122248888)), + "SIG_OUT": (0, np.float32(0.003911871928721666)), + "WGT": (128, np.float32(0.01568627543747425)), + "BIAS": (0, np.float32(0.0000613626980339177)), # zp == 0, scale = weight_scale * sig_out_scale + "OUT": (0, np.float32(0.005075461231172085)), + } + self.default_zp_scales_per_channel = { + "INP": (0, np.float32(0.0235294122248888)), + "SIG_OUT": (0, np.float32(0.003911871928721666)), + "WGT": ([0, 0], [np.float32(0.015748031437397003), np.float32(0.011811023578047752)]), + "BIAS": ([0, 0], [np.float32(0.00006160428165458143), np.float32(0.00004620321124093607)]), + "OUT": (0, np.float32(0.005075461231172085)), + } + + def perform_qdq_quantization(self, output_model_name, tensor_quant_overrides=None, per_channel=False): + # (input) + # | + # Sigmoid + # | + # Conv + # | + # (output) + + inp = onnx.helper.make_tensor_value_info("INP", onnx.TensorProto.FLOAT, self.activations[0].shape) + sigmoid_node = onnx.helper.make_node("Sigmoid", ["INP"], ["SIG_OUT"]) + + out = onnx.helper.make_tensor_value_info("OUT", onnx.TensorProto.FLOAT, [None, None, None]) + wgt_init = onnx.numpy_helper.from_array(self.weight, "WGT") + bias_init = onnx.numpy_helper.from_array(self.bias, "BIAS") + conv_node = onnx.helper.make_node("Conv", ["SIG_OUT", "WGT", "BIAS"], ["OUT"]) + + graph = onnx.helper.make_graph( + [sigmoid_node, conv_node], "test", [inp], [out], initializer=[wgt_init, bias_init] + ) + model = onnx.helper.make_model(graph, opset_imports=[onnx.helper.make_opsetid("", 13)]) + onnx.save(model, "model.onnx") + + # Quantize model + class DummyDataReader(quantization.CalibrationDataReader): + def __init__(self, activations): + self.iterator = ({"INP": act} for act in activations) + + def get_next(self): + return next(self.iterator, None) + + extra_options = {} + if tensor_quant_overrides is not None: + extra_options["TensorQuantOverrides"] = tensor_quant_overrides + + quantization.quantize_static( + model_input="model.onnx", + model_output=output_model_name, + calibration_data_reader=DummyDataReader(self.activations), + quant_format=quantization.QuantFormat.QDQ, + activation_type=self.default_act_qtype, + weight_type=self.default_wgt_qtype, + per_channel=per_channel, + op_types_to_quantize=["Conv", "Sigmoid"], + extra_options=extra_options, + ) + + # Extract quantization parameters: scales and zero points for activations and weights. + model = onnx.load(output_model_name) + inp_zp = next(init for init in model.graph.initializer if init.name == "INP_zero_point") + inp_sc = next(init for init in model.graph.initializer if init.name == "INP_scale") + sig_out_zp = next(init for init in model.graph.initializer if init.name == "SIG_OUT_zero_point") + sig_out_sc = next(init for init in model.graph.initializer if init.name == "SIG_OUT_scale") + wgt_zp = next(init for init in model.graph.initializer if init.name == "WGT_zero_point") + wgt_sc = next(init for init in model.graph.initializer if init.name == "WGT_scale") + bias_zp = next( + init + for init in model.graph.initializer + if init.name == "BIAS_quantized_zero_point" or init.name == "BIAS_zero_point" + ) + bias_sc = next( + init for init in model.graph.initializer if init.name == "BIAS_quantized_scale" or init.name == "BIAS_scale" + ) + out_zp = next(init for init in model.graph.initializer if init.name == "OUT_zero_point") + out_sc = next(init for init in model.graph.initializer if init.name == "OUT_scale") + + # Return quantization parameters + return inp_zp, inp_sc, sig_out_zp, sig_out_sc, wgt_zp, wgt_sc, bias_zp, bias_sc, out_zp, out_sc + + def test_qdq_default(self): + """ + Test default behavior without specifying the TensorQuantOverrides option. + """ + ( + inp_zp, + inp_sc, + sig_out_zp, + sig_out_sc, + wgt_zp, + wgt_sc, + bias_zp, + bias_sc, + out_zp, + out_sc, + ) = self.perform_qdq_quantization( + "model_default_quant_overrides.onnx", + tensor_quant_overrides=None, # default behavior + ) + + # No overrides set. Expect default values + self.assertEqual(inp_zp.int32_data[0], self.default_zp_scales["INP"][0]) + self.assertEqual(inp_zp.data_type, self.default_act_qtype) + self.assertEqual(inp_sc.float_data[0], self.default_zp_scales["INP"][1]) + + self.assertEqual(sig_out_zp.int32_data[0], self.default_zp_scales["SIG_OUT"][0]) + self.assertEqual(sig_out_zp.data_type, self.default_act_qtype) + self.assertEqual(sig_out_sc.float_data[0], self.default_zp_scales["SIG_OUT"][1]) + + self.assertEqual(wgt_zp.int32_data[0], self.default_zp_scales["WGT"][0]) + self.assertEqual(wgt_zp.data_type, self.default_wgt_qtype) + self.assertEqual(wgt_sc.float_data[0], self.default_zp_scales["WGT"][1]) + + self.assertEqual(bias_zp.int32_data[0], self.default_zp_scales["BIAS"][0]) + self.assertEqual(bias_zp.data_type, self.default_bias_qtype) + self.assertEqual(bias_sc.float_data[0], self.default_zp_scales["BIAS"][1]) + + self.assertEqual(out_zp.int32_data[0], self.default_zp_scales["OUT"][0]) + self.assertEqual(out_zp.data_type, self.default_act_qtype) + self.assertEqual(out_sc.float_data[0], self.default_zp_scales["OUT"][1]) + + def test_qdq_default_per_channel(self): + """ + Test default per-channel behavior without specifying the TensorQuantOverrides option. + """ + ( + inp_zp, + inp_sc, + sig_out_zp, + sig_out_sc, + wgt_zp, + wgt_sc, + bias_zp, + bias_sc, + out_zp, + out_sc, + ) = self.perform_qdq_quantization( + "model_default_per_channel_quant_overrides.onnx", + tensor_quant_overrides=None, # default behavior + per_channel=True, + ) + + # No overrides set. Expect default values + self.assertEqual(inp_zp.int32_data[0], self.default_zp_scales["INP"][0]) + self.assertEqual(inp_zp.data_type, self.default_act_qtype) + self.assertEqual(inp_sc.float_data[0], self.default_zp_scales["INP"][1]) + + self.assertEqual(sig_out_zp.int32_data[0], self.default_zp_scales["SIG_OUT"][0]) + self.assertEqual(sig_out_zp.data_type, self.default_act_qtype) + self.assertEqual(sig_out_sc.float_data[0], self.default_zp_scales["SIG_OUT"][1]) + + self.assertEqual(wgt_zp.data_type, self.default_wgt_qtype_per_channel) + for index, zp in enumerate(self.default_zp_scales_per_channel["WGT"][0]): + self.assertEqual(wgt_zp.int32_data[index], zp) + for index, scale in enumerate(self.default_zp_scales_per_channel["WGT"][1]): + self.assertEqual(wgt_sc.float_data[index], scale) + + self.assertEqual(bias_zp.data_type, self.default_bias_qtype) + + num_bias_zps = len(self.default_zp_scales_per_channel["BIAS"][0]) + actual_bias_zps = struct.unpack(f"<{num_bias_zps}i", bias_zp.raw_data) + for index, zp in enumerate(self.default_zp_scales_per_channel["BIAS"][0]): + self.assertEqual(actual_bias_zps[index], zp) + + num_bias_scales = len(self.default_zp_scales_per_channel["BIAS"][1]) + actual_bias_scales = struct.unpack(f"<{num_bias_scales}f", bias_sc.raw_data) + for index, scale in enumerate(self.default_zp_scales_per_channel["BIAS"][1]): + self.assertEqual(actual_bias_scales[index], scale) + + self.assertEqual(out_zp.int32_data[0], self.default_zp_scales["OUT"][0]) + self.assertEqual(out_zp.data_type, self.default_act_qtype) + self.assertEqual(out_sc.float_data[0], self.default_zp_scales["OUT"][1]) + + def test_qdq_overrides1(self): + """ + Test overriding: + - scale/zp for Sigmoid output + - quant_type, symmetric, reduce_range for Conv weight + - quant_type, symmetric, reduce_range for Conv bias + """ + inp_zp, inp_sc, sig_out_zp, sig_out_sc, wgt_zp, wgt_sc, bias_zp, bias_sc, _, _ = self.perform_qdq_quantization( + "model_quant_overrides1.onnx", + tensor_quant_overrides={ + "SIG_OUT": [{"scale": 1.0, "zero_point": 127}], + "WGT": [{"quant_type": quantization.QuantType.QInt8, "symmetric": True, "reduce_range": True}], + "BIAS": [{"quant_type": quantization.QuantType.QInt8, "symmetric": True, "reduce_range": True}], + }, + ) + + # Input should have same quant params + self.assertEqual(inp_zp.int32_data[0], self.default_zp_scales["INP"][0]) + self.assertEqual(inp_zp.data_type, self.default_act_qtype) + self.assertEqual(inp_sc.float_data[0], self.default_zp_scales["INP"][1]) + + # Sigmoid output should have overridden scale/zp + self.assertEqual(sig_out_zp.int32_data[0], 127) + self.assertEqual(sig_out_zp.data_type, self.default_act_qtype) + self.assertEqual(sig_out_sc.float_data[0], np.float32(1.0)) + + # Weight should have different type, zero_point, and scale + self.assertEqual(wgt_zp.data_type, quantization.QuantType.QInt8.tensor_type) + + wgt_qmin, wgt_qmax = get_qmin_qmax_for_qType(wgt_zp.data_type, reduce_range=True, symmetric=True) + wgt_rmin, wgt_rmax = np.min(self.weight), np.max(self.weight) + new_wgt_zp, new_wgt_sc = compute_scale_zp(wgt_rmin, wgt_rmax, wgt_qmin, wgt_qmax, symmetric=True) + self.assertEqual(wgt_zp.int32_data[0], new_wgt_zp) + self.assertEqual(wgt_sc.float_data[0], np.float32(new_wgt_sc)) + + # Bias should now be treated as a weight and should have different type, zero_point, and scale + self.assertEqual(bias_zp.data_type, quantization.QuantType.QInt8.tensor_type) + + bias_qmin, bias_qmax = get_qmin_qmax_for_qType(bias_zp.data_type, reduce_range=True, symmetric=True) + bias_rmin, bias_rmax = np.min(self.bias), np.max(self.bias) + new_bias_zp, new_bias_sc = compute_scale_zp(bias_rmin, bias_rmax, bias_qmin, bias_qmax, symmetric=True) + self.assertEqual(bias_zp.int32_data[0], new_bias_zp) + self.assertEqual(bias_sc.float_data[0], np.float32(new_bias_sc)) + + def test_qdq_overrides2(self): + """ + Test overriding rmin/rmax for Sigmoid output. + """ + sigmoid_rmin, sigmoid_rmax = 0.0, 0.5 + inp_zp, inp_sc, sig_out_zp, sig_out_sc, _, _, _, _, _, _ = self.perform_qdq_quantization( + "model_quant_overrides2.onnx", + tensor_quant_overrides={"SIG_OUT": [{"rmin": sigmoid_rmin, "rmax": sigmoid_rmax}]}, + ) + + # Input should have same quant params + self.assertEqual(inp_zp.int32_data[0], self.default_zp_scales["INP"][0]) + self.assertEqual(inp_zp.data_type, self.default_act_qtype) + self.assertEqual(inp_sc.float_data[0], self.default_zp_scales["INP"][1]) + + # Sigmoid output should have different scale/zp due to overridden rmin/rmax + self.assertEqual(sig_out_zp.data_type, self.default_act_qtype) + + sigmoid_qmin, sigmoid_qmax = get_qmin_qmax_for_qType(sig_out_zp.data_type) + new_sigmoid_zp, new_sigmoid_sc = compute_scale_zp(sigmoid_rmin, sigmoid_rmax, sigmoid_qmin, sigmoid_qmax) + self.assertEqual(sig_out_zp.int32_data[0], new_sigmoid_zp) + self.assertEqual(sig_out_sc.float_data[0], np.float32(new_sigmoid_sc)) + + def test_qdq_overrides3(self): + """ + Test overriding rmin and rmax for Conv weight + """ + wgt_rmin, wgt_rmax = 0.0, 1.0 + _, _, _, _, wgt_zp, wgt_sc, _, _, _, _ = self.perform_qdq_quantization( + "model_quant_overrides3.onnx", + tensor_quant_overrides={ + "WGT": [{"rmin": wgt_rmin, "rmax": wgt_rmax}], + }, + ) + + # Weight should have different zero_point and scale + self.assertEqual(wgt_zp.data_type, self.default_wgt_qtype) + self.assertNotEqual(wgt_rmin, np.min(self.weight)) + self.assertNotEqual(wgt_rmax, np.max(self.weight)) + + wgt_qmin, wgt_qmax = get_qmin_qmax_for_qType(wgt_zp.data_type) + new_wgt_zp, new_wgt_sc = compute_scale_zp(wgt_rmin, wgt_rmax, wgt_qmin, wgt_qmax) + self.assertEqual(wgt_zp.int32_data[0], new_wgt_zp) + self.assertEqual(wgt_sc.float_data[0], np.float32(new_wgt_sc)) + + def test_qdq_overrides4(self): + """ + Test overriding scale and zero_point for Conv weight + """ + wgt_zp_val, wgt_scale_val = 4, 0.5 + _, _, _, _, wgt_zp, wgt_sc, _, _, _, _ = self.perform_qdq_quantization( + "model_quant_overrides4.onnx", + tensor_quant_overrides={ + "WGT": [{"zero_point": wgt_zp_val, "scale": wgt_scale_val}], + }, + ) + + # Weight should have have the expected zero_point and scale + self.assertEqual(wgt_zp.data_type, self.default_wgt_qtype) + self.assertEqual(wgt_zp.int32_data[0], wgt_zp_val) + self.assertEqual(wgt_sc.float_data[0], np.float32(wgt_scale_val)) + + def test_qdq_overrides_per_channel1(self): + """ + Test per-channel overriding of scale/zero_point for Conv weight and bias. + """ + zp_vals, scale_vals = [2, 4], [0.5, 0.2] + ( + _, + _, + _, + _, + wgt_zp, + wgt_sc, + bias_zp, + bias_sc, + _, + _, + ) = self.perform_qdq_quantization( + "model_per_channel_quant_overrides1.onnx", + tensor_quant_overrides={ + "WGT": [ + {"zero_point": zp_vals[0], "scale": scale_vals[0]}, + {"zero_point": zp_vals[1], "scale": scale_vals[1]}, + ], + "BIAS": [ + {"zero_point": zp_vals[0], "scale": scale_vals[0]}, + {"zero_point": zp_vals[1], "scale": scale_vals[1]}, + ], + }, + per_channel=True, + ) + + self.assertEqual(wgt_zp.data_type, self.default_wgt_qtype_per_channel) + for index, zp in enumerate(zp_vals): + self.assertEqual(wgt_zp.int32_data[index], zp) + for index, scale in enumerate(scale_vals): + self.assertEqual(wgt_sc.float_data[index], np.float32(scale)) + + # NOTE: Bias with overrides is treated as a weight. + self.assertEqual(bias_zp.data_type, self.default_wgt_qtype_per_channel) + for index, zp in enumerate(zp_vals): + self.assertEqual(bias_zp.int32_data[index], zp) + for index, scale in enumerate(scale_vals): + self.assertEqual(bias_sc.float_data[index], np.float32(scale)) + + def test_qdq_overrides_per_channel2(self): + """ + Test per-channel overriding of rmin, rmax, reduce_range, and quant_type for Conv weight. + """ + rmin_vals = [0.0, 0.2] + rmax_vals = [1.0, 0.8] + quant_type = quantization.QuantType.QUInt8 + reduce_ranges = [True, False] + ( + _, + _, + _, + _, + wgt_zp, + wgt_sc, + bias_zp, + bias_sc, + _, + _, + ) = self.perform_qdq_quantization( + "model_per_channel_quant_overrides2.onnx", + tensor_quant_overrides={ + "WGT": [ + { + "quant_type": quant_type, + "rmin": rmin_vals[0], + "rmax": rmax_vals[0], + "reduce_range": reduce_ranges[0], + }, + { + "quant_type": quant_type, + "rmin": rmin_vals[1], + "rmax": rmax_vals[1], + "reduce_range": reduce_ranges[1], + }, + ], + }, + per_channel=True, + ) + + self.assertEqual(wgt_zp.data_type, quant_type.tensor_type) + for index, (zp, scale) in enumerate(zip(wgt_zp.int32_data, wgt_sc.float_data)): + wgt_qmin, wgt_qmax = get_qmin_qmax_for_qType(wgt_zp.data_type, reduce_range=reduce_ranges[index]) + expected_zp, expected_scale = compute_scale_zp(rmin_vals[index], rmax_vals[index], wgt_qmin, wgt_qmax) + self.assertEqual(zp, expected_zp) + self.assertEqual(scale, np.float32(expected_scale)) + + def test_override_validation_nonexisting_tensor(self): + """ + Test that specifying a non-existing tensor should fail. + """ + with self.assertRaises(ValueError) as context: + self.perform_qdq_quantization( + "model_validation.onnx", + tensor_quant_overrides={"NON_EXISTING": [{"rmin": 0.0, "rmax": 0.5}]}, + ) + + self.assertIn("is not present in the model", str(context.exception)) + + def test_override_validation_scale_missing_zp(self): + """ + Test that specifying a scale without zero_point should fail. + """ + with self.assertRaises(ValueError) as context: + self.perform_qdq_quantization( + "model_validation.onnx", + tensor_quant_overrides={"SIG_OUT": [{"scale": 0.0}]}, + ) + + self.assertIn("Must provide both 'scale' and 'zero_point'", str(context.exception)) + + def test_override_validation_bad_combination(self): + """ + Test that specifying a scale/zero_point with rmax/rmin/symmetric/reduce_range should fail. + """ + with self.assertRaises(ValueError) as context: + self.perform_qdq_quantization( + "model_validation.onnx", + tensor_quant_overrides={"SIG_OUT": [{"scale": 0.0, "zero_point": 0, "rmax": 10.0}]}, + ) + + self.assertIn("option 'rmax' is invalid with 'scale' and 'zero_point'", str(context.exception)) + + with self.assertRaises(ValueError) as context: + self.perform_qdq_quantization( + "model_validation.onnx", + tensor_quant_overrides={"SIG_OUT": [{"scale": 0.0, "zero_point": 0, "rmin": 10.0}]}, + ) + + self.assertIn("option 'rmin' is invalid with 'scale' and 'zero_point'", str(context.exception)) + + with self.assertRaises(ValueError) as context: + self.perform_qdq_quantization( + "model_validation.onnx", + tensor_quant_overrides={"SIG_OUT": [{"scale": 0.0, "zero_point": 0, "symmetric": True}]}, + ) + + self.assertIn("option 'symmetric' is invalid with 'scale' and 'zero_point'", str(context.exception)) + + with self.assertRaises(ValueError) as context: + self.perform_qdq_quantization( + "model_validation.onnx", + tensor_quant_overrides={"SIG_OUT": [{"scale": 0.0, "zero_point": 0, "reduce_range": True}]}, + ) + + self.assertIn("option 'reduce_range' is invalid with 'scale' and 'zero_point'", str(context.exception)) + + +if __name__ == "__main__": + unittest.main() diff --git a/setup.py b/setup.py index 798c8c4b28..2ede39915c 100644 --- a/setup.py +++ b/setup.py @@ -408,6 +408,7 @@ packages = [ "onnxruntime.quantization", "onnxruntime.quantization.operators", "onnxruntime.quantization.CalTableFlatBuffers", + "onnxruntime.quantization.execution_providers.qnn", "onnxruntime.transformers", "onnxruntime.transformers.models.bart", "onnxruntime.transformers.models.bert",