From c4f73af234efb8cbe3df3e4dd14c19313bc9b06a Mon Sep 17 00:00:00 2001 From: Funtowicz Morgan Date: Fri, 11 Mar 2022 23:52:09 +0100 Subject: [PATCH] Fix wrong percentile values returned during calibration (#10847) * Use numpy.percentile to get the lookup value. * Use 1.0 as float value rather than integer. * Add missing cdf parameter for `np.percentile`. * Use 100. instead of 1.0 * Remove print. * Update from @yufenglee --- onnxruntime/python/tools/quantization/calibrate.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/onnxruntime/python/tools/quantization/calibrate.py b/onnxruntime/python/tools/quantization/calibrate.py index 18e0accc4c..c559112028 100644 --- a/onnxruntime/python/tools/quantization/calibrate.py +++ b/onnxruntime/python/tools/quantization/calibrate.py @@ -599,7 +599,7 @@ class HistogramCollector(CalibrationDataCollector): print("Number of tensors : {}".format(len(histogram_dict))) print("Number of histogram bins : {}".format(self.num_bins)) - print("Percentile : {}".format(percentile)) + print("Percentile : ({},{})".format(100.0 - percentile, percentile)) for tensor, histogram in histogram_dict.items(): hist = histogram[0] @@ -607,11 +607,12 @@ class HistogramCollector(CalibrationDataCollector): total = hist.sum() cdf = np.cumsum(hist/total) if self.symmetric: - idx_right = np.searchsorted(cdf, percentile/100) - thresholds_dict[tensor] = (-float(hist_edges[idx_right]), float(hist_edges[idx_right])) + idx_right = np.searchsorted(cdf, percentile / 100.0) + thresholds_dict[tensor] = (-float(hist_edges[idx_ringht]), float(hist_edges[idx_right])) else: - idx_right = np.searchsorted(cdf, percentile/200) - idx_left = np.searchsorted(cdf, (1.0 - percentile/200)) + percent_to_cut_one_side = (100.0 - percentile) / 200.0 + idx_right = np.searchsorted(cdf, 1.0 - percent_to_cut_one_side) + idx_left = np.searchsorted(cdf, percent_to_cut_one_side) thresholds_dict[tensor] = (float(hist_edges[idx_left]), float(hist_edges[idx_right])) # Plot histogram for debug only