fix "4bit quantization scales and zeropoint tensor shape" (#19986)

### Description
<!-- Describe your changes. -->



### Motivation and Context
<!-- - Why is this change required? What problem does it solve?
- If it fixes an open issue, please link to the issue here. -->
This commit is contained in:
wejoncy 2024-04-09 01:15:28 +08:00 committed by GitHub
parent 23d3afd4fe
commit 908a76d675
No known key found for this signature in database
GPG key ID: B5690EEEBB952194

View file

@ -322,6 +322,15 @@ class HQQWeightOnlyQuantizer:
self.pack_on_row_fast_248bit(packed_torch, quant_weight_torch, self.config.bits)
scales = scales_torch.cpu().numpy()
zero_points = zero_points_torch.cpu().numpy()
# reshape to the predefined shape in MatmulNbits
scales = scales.reshape(-1)
zero_points = zero_points.reshape(-1)
rows, cols = b_array_torch.shape
block_size = self.config.block_size
blob_size = block_size // 2
k_blocks = (rows + block_size - 1) // block_size
packed_torch = packed_torch.reshape(cols, k_blocks, blob_size)
b_quant = onnx.numpy_helper.from_array(packed_torch.cpu().numpy())
b_quant.name = b_pb.name + "_Q4"
for input in bs_graph.input:
@ -647,8 +656,8 @@ set of 4b integers with a scaling factor and an optional offset.
"--quant_method",
default="default",
type=str,
choices=["default", "hqq"],
help="the algorithm used to quantize weight",
choices=["default", "hqq", "rtn", "gptq"],
help="the algorithm used to quantize weight, \nrtn and gptq leverage Intel® Neural Compressor",
)
parser.add_argument("--bits", default=4, type=int, help="the target bits to represent weight")
parser.add_argument(
@ -659,7 +668,7 @@ set of 4b integers with a scaling factor and an optional offset.
nargs="?",
type=ort_convert_str_to_bool,
choices=[True, False],
help="Indicate whether to quantize the model symmetrically",
help="Indicate whether to quantize the model symmetrically, symmetric is not supported by hqq",
)
parser.add_argument(
"--accuracy_level",
@ -695,6 +704,10 @@ if __name__ == "__main__":
logger.error(f"file {output_model_path} already exists")
raise Exception(f"file {output_model_path} already exists")
if args.symmetric and args.quant_method == "hqq":
logger.warning("symmetric is not supportted by hqq, will force to symmetric=False")
args.symmetric = False
model = onnx.load(input_model_path)
if args.quant_method == "hqq":
quant_config = HQQWeightOnlyQuantConfig(block_size=args.block_size, bits=args.bits)