saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-22 19:23:30 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fix wrong percentile values returned during calibration (#10847)

Browse source 
* Use numpy.percentile to get the lookup value.

* Use 1.0 as float value rather than integer.

* Add missing cdf parameter for `np.percentile`.

* Use 100. instead of 1.0

* Remove print.

* Update from @yufenglee
This commit is contained in:
Funtowicz Morgan 2022-03-11 23:52:09 +01:00 committed by GitHub
parent 6ac3b8d46a
commit c4f73af234
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
1 changed files with 6 additions and 5 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

11
onnxruntime/python/tools/quantization/calibrate.py
View file

@ -599,7 +599,7 @@ class HistogramCollector(CalibrationDataCollector):
print("Number of tensors : {}".format(len(histogram_dict)))
print("Number of histogram bins : {}".format(self.num_bins))
print("Percentile : {}".format(percentile))
print("Percentile : ({},{})".format(100.0 - percentile, percentile))
for tensor, histogram in histogram_dict.items():
hist = histogram[0]
@ -607,11 +607,12 @@ class HistogramCollector(CalibrationDataCollector):
total = hist.sum()
cdf = np.cumsum(hist/total)
if self.symmetric:
idx_right = np.searchsorted(cdf, percentile/100)
thresholds_dict[tensor] = (-float(hist_edges[idx_right]), float(hist_edges[idx_right]))
idx_right = np.searchsorted(cdf, percentile / 100.0)
thresholds_dict[tensor] = (-float(hist_edges[idx_ringht]), float(hist_edges[idx_right]))
else:
idx_right = np.searchsorted(cdf, percentile/200)
idx_left = np.searchsorted(cdf, (1.0 - percentile/200))
percent_to_cut_one_side = (100.0 - percentile) / 200.0
idx_right = np.searchsorted(cdf, 1.0 - percent_to_cut_one_side)
idx_left = np.searchsorted(cdf, percent_to_cut_one_side)
thresholds_dict[tensor] = (float(hist_edges[idx_left]), float(hist_edges[idx_right]))
# Plot histogram for debug only