saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-30 20:18:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix to avoid quantizing attention with varied q,k,v sizes (#9357)

Browse source 
* fix to avoid quantizing attention with varied q,k,v sizes

* updated the changes to address the comments
This commit is contained in:
Viswanath Boga 2021-10-13 16:25:34 -07:00 committed by GitHub
parent ba0cca96f0
commit 4771256be3
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 7 additions and 0 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
onnxruntime/python/tools/quantization/operators/attention.py
View file

@ -20,6 +20,13 @@ class AttentionQuant(QuantOperatorBase):
node = self.node
assert (node.op_type == "Attention")
# TODO This is a temporary fix to stop exporting QAttention with qkv_hidden_sizes
# attribute. This needs to be removed once the QAttention for varied q,k,v sizes
# is implemented
for attr in node.attribute:
if 'qkv_hidden_sizes' == attr.name:
return super().quantize()
(quantized_input_names, zero_point_names, scale_names, nodes) = \
self.quantizer.quantize_inputs(node, [0, 1], reduce_range=True, op_level_per_channel=True)
if quantized_input_names is None: