saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-30 20:18:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Fix a typo in quantization tools (#10940)

Browse source
This commit is contained in:
Ella Charlaix 2022-03-19 05:03:16 +01:00 committed by Chi Lo
parent 69e2d319ed
commit ec00d28a2f
1 changed files with 1 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
onnxruntime/python/tools/quantization/calibrate.py
View file

@ -608,7 +608,7 @@ class HistogramCollector(CalibrationDataCollector):
cdf = np.cumsum(hist/total) cdf = np.cumsum(hist/total)
if self.symmetric: if self.symmetric:
idx_right = np.searchsorted(cdf, percentile / 100.0) idx_right = np.searchsorted(cdf, percentile / 100.0)
thresholds_dict[tensor] = (-float(hist_edges[idx_ringht]), float(hist_edges[idx_right])) thresholds_dict[tensor] = (-float(hist_edges[idx_right]), float(hist_edges[idx_right]))
else: else:
percent_to_cut_one_side = (100.0 - percentile) / 200.0 percent_to_cut_one_side = (100.0 - percentile) / 200.0
idx_right = np.searchsorted(cdf, 1.0 - percent_to_cut_one_side) idx_right = np.searchsorted(cdf, 1.0 - percent_to_cut_one_side)