mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-25 19:48:11 +00:00
fix "4bit quantization scales and zeropoint tensor shape" (#19986)
### Description <!-- Describe your changes. --> ### Motivation and Context <!-- - Why is this change required? What problem does it solve? - If it fixes an open issue, please link to the issue here. -->
This commit is contained in:
parent
23d3afd4fe
commit
908a76d675
1 changed files with 16 additions and 3 deletions
|
|
@ -322,6 +322,15 @@ class HQQWeightOnlyQuantizer:
|
|||
self.pack_on_row_fast_248bit(packed_torch, quant_weight_torch, self.config.bits)
|
||||
scales = scales_torch.cpu().numpy()
|
||||
zero_points = zero_points_torch.cpu().numpy()
|
||||
# reshape to the predefined shape in MatmulNbits
|
||||
scales = scales.reshape(-1)
|
||||
zero_points = zero_points.reshape(-1)
|
||||
rows, cols = b_array_torch.shape
|
||||
block_size = self.config.block_size
|
||||
blob_size = block_size // 2
|
||||
k_blocks = (rows + block_size - 1) // block_size
|
||||
packed_torch = packed_torch.reshape(cols, k_blocks, blob_size)
|
||||
|
||||
b_quant = onnx.numpy_helper.from_array(packed_torch.cpu().numpy())
|
||||
b_quant.name = b_pb.name + "_Q4"
|
||||
for input in bs_graph.input:
|
||||
|
|
@ -647,8 +656,8 @@ set of 4b integers with a scaling factor and an optional offset.
|
|||
"--quant_method",
|
||||
default="default",
|
||||
type=str,
|
||||
choices=["default", "hqq"],
|
||||
help="the algorithm used to quantize weight",
|
||||
choices=["default", "hqq", "rtn", "gptq"],
|
||||
help="the algorithm used to quantize weight, \nrtn and gptq leverage Intel® Neural Compressor",
|
||||
)
|
||||
parser.add_argument("--bits", default=4, type=int, help="the target bits to represent weight")
|
||||
parser.add_argument(
|
||||
|
|
@ -659,7 +668,7 @@ set of 4b integers with a scaling factor and an optional offset.
|
|||
nargs="?",
|
||||
type=ort_convert_str_to_bool,
|
||||
choices=[True, False],
|
||||
help="Indicate whether to quantize the model symmetrically",
|
||||
help="Indicate whether to quantize the model symmetrically, symmetric is not supported by hqq",
|
||||
)
|
||||
parser.add_argument(
|
||||
"--accuracy_level",
|
||||
|
|
@ -695,6 +704,10 @@ if __name__ == "__main__":
|
|||
logger.error(f"file {output_model_path} already exists")
|
||||
raise Exception(f"file {output_model_path} already exists")
|
||||
|
||||
if args.symmetric and args.quant_method == "hqq":
|
||||
logger.warning("symmetric is not supportted by hqq, will force to symmetric=False")
|
||||
args.symmetric = False
|
||||
|
||||
model = onnx.load(input_model_path)
|
||||
if args.quant_method == "hqq":
|
||||
quant_config = HQQWeightOnlyQuantConfig(block_size=args.block_size, bits=args.bits)
|
||||
|
|
|
|||
Loading…
Reference in a new issue