From 908a76d67504445f6561585b85b6ddd6e9e5c255 Mon Sep 17 00:00:00 2001 From: wejoncy Date: Tue, 9 Apr 2024 01:15:28 +0800 Subject: [PATCH] fix "4bit quantization scales and zeropoint tensor shape" (#19986) ### Description ### Motivation and Context --- .../quantization/matmul_4bits_quantizer.py | 19 ++++++++++++++++--- 1 file changed, 16 insertions(+), 3 deletions(-) diff --git a/onnxruntime/python/tools/quantization/matmul_4bits_quantizer.py b/onnxruntime/python/tools/quantization/matmul_4bits_quantizer.py index 3090296b77..11a830dc6d 100644 --- a/onnxruntime/python/tools/quantization/matmul_4bits_quantizer.py +++ b/onnxruntime/python/tools/quantization/matmul_4bits_quantizer.py @@ -322,6 +322,15 @@ class HQQWeightOnlyQuantizer: self.pack_on_row_fast_248bit(packed_torch, quant_weight_torch, self.config.bits) scales = scales_torch.cpu().numpy() zero_points = zero_points_torch.cpu().numpy() + # reshape to the predefined shape in MatmulNbits + scales = scales.reshape(-1) + zero_points = zero_points.reshape(-1) + rows, cols = b_array_torch.shape + block_size = self.config.block_size + blob_size = block_size // 2 + k_blocks = (rows + block_size - 1) // block_size + packed_torch = packed_torch.reshape(cols, k_blocks, blob_size) + b_quant = onnx.numpy_helper.from_array(packed_torch.cpu().numpy()) b_quant.name = b_pb.name + "_Q4" for input in bs_graph.input: @@ -647,8 +656,8 @@ set of 4b integers with a scaling factor and an optional offset. "--quant_method", default="default", type=str, - choices=["default", "hqq"], - help="the algorithm used to quantize weight", + choices=["default", "hqq", "rtn", "gptq"], + help="the algorithm used to quantize weight, \nrtn and gptq leverage Intel® Neural Compressor", ) parser.add_argument("--bits", default=4, type=int, help="the target bits to represent weight") parser.add_argument( @@ -659,7 +668,7 @@ set of 4b integers with a scaling factor and an optional offset. nargs="?", type=ort_convert_str_to_bool, choices=[True, False], - help="Indicate whether to quantize the model symmetrically", + help="Indicate whether to quantize the model symmetrically, symmetric is not supported by hqq", ) parser.add_argument( "--accuracy_level", @@ -695,6 +704,10 @@ if __name__ == "__main__": logger.error(f"file {output_model_path} already exists") raise Exception(f"file {output_model_path} already exists") + if args.symmetric and args.quant_method == "hqq": + logger.warning("symmetric is not supportted by hqq, will force to symmetric=False") + args.symmetric = False + model = onnx.load(input_model_path) if args.quant_method == "hqq": quant_config = HQQWeightOnlyQuantConfig(block_size=args.block_size, bits=args.bits)