diff --git a/onnxruntime/python/tools/quantization/quant_utils.py b/onnxruntime/python/tools/quantization/quant_utils.py
index 7ec7185575..d7f287f3d7 100644
--- a/onnxruntime/python/tools/quantization/quant_utils.py
+++ b/onnxruntime/python/tools/quantization/quant_utils.py
@@ -119,8 +119,7 @@ def quantize_nparray(qType, arr, scale, zero_point, low=None, high=None):
 
 
 def compute_scale_zp(rmin, rmax, qmin, qmax, symmetric=False):
-    """
-    Calculate the scale s and zero point z for the quantization relation
+    """Calculate the scale s and zero point z for the quantization relation
     r = s(q-z), where r are the original values and q are the corresponding
     quantized values.
 
@@ -138,6 +137,9 @@ def compute_scale_zp(rmin, rmax, qmin, qmax, symmetric=False):
 
     """
 
+    if qmin > 0 or qmax < 0:
+        raise ValueError(f"qmin and qmax must meet requirement: qmin <= 0 <= qmax while qmin:{qmin}, qmmax:{qmax}")
+
     # Adjust rmin and rmax such that 0 is included in the range. This is
     # required to make sure zero can be represented by the quantization data
     # type (i.e. to make sure qmin <= zero_point <= qmax)
@@ -149,8 +151,12 @@ def compute_scale_zp(rmin, rmax, qmin, qmax, symmetric=False):
         rmin = -absmax
         rmax = +absmax
 
-    scale = (rmax - rmin) / float(qmax - qmin) if rmax != rmin else 1.0
-    zero_point = round(qmin - rmin / scale)
+    scale = (rmax - rmin) / float(qmax - qmin)
+    if scale < numpy.finfo(numpy.float32).tiny:
+        scale = 1.0
+        zero_point = 0
+    else:
+        zero_point = round(qmin - rmin / scale)
 
     return [zero_point, scale]
 
diff --git a/onnxruntime/test/python/quantization/test_quant_util.py b/onnxruntime/test/python/quantization/test_quant_util.py
new file mode 100644
index 0000000000..d7da0b8923
--- /dev/null
+++ b/onnxruntime/test/python/quantization/test_quant_util.py
@@ -0,0 +1,35 @@
+#!/usr/bin/env python
+# coding: utf-8
+# -------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License. See License.txt in the project root for
+# license information.
+# --------------------------------------------------------------------------
+
+import unittest
+
+import numpy
+
+from onnxruntime.quantization.quant_utils import compute_scale_zp
+
+
+class TestQuantUtil(unittest.TestCase):
+    def test_compute_scale_zp(self):
+        self.assertEqual(compute_scale_zp(0.0, 0.0, -127, 127, symmetric=True), [0, 1.0])
+        self.assertEqual(compute_scale_zp(1.0, -1.0, -127, 127, symmetric=True), [0, 1.0])
+        self.assertEqual(compute_scale_zp(0.0, 0.0, 0, 255, symmetric=True), [0, 1.0])
+        self.assertEqual(compute_scale_zp(1.0, -1.0, 0, 255, symmetric=True), [0, 1.0])
+
+        self.assertEqual(compute_scale_zp(-1.0, 2.0, -127, 127, symmetric=True), [0, 2.0 / 127])
+        self.assertEqual(compute_scale_zp(-1.0, 2.0, -127, 127, symmetric=False), [-42, 3.0 / 254])
+
+        self.assertEqual(compute_scale_zp(-1.0, 2.0, 0, 255, symmetric=True), [128, 4.0 / 255])
+        self.assertEqual(compute_scale_zp(-1.0, 2.0, 0, 255, symmetric=False), [85, 3.0 / 255])
+
+        tiny_float = numpy.float32(numpy.finfo(numpy.float32).tiny * 0.1)
+        self.assertEqual(compute_scale_zp(-tiny_float, tiny_float, 0, 255, symmetric=True), [0, 1.0])
+        self.assertEqual(compute_scale_zp(-tiny_float, 0.0, 0, 255, symmetric=False), [0, 1.0])
+
+
+if __name__ == "__main__":
+    unittest.main()