enable pipeline to run quantization tests (#6416)

* enable pipeline to run quantization tests
setup test pipeline for quantization

cmake/onnxruntime_python.cmake

@@ -194,6 +194,9 @@ file(GLOB onnxruntime_python_test_srcs CONFIGURE_DEPENDS
     "${ONNXRUNTIME_ROOT}/test/python/*.py"
     "${ORTTRAINING_SOURCE_DIR}/test/python/*.py"
 )
+file(GLOB onnxruntime_python_quantization_test_srcs CONFIGURE_DEPENDS
+    "${ONNXRUNTIME_ROOT}/test/python/quantization/*.py"
+)
 file(GLOB onnxruntime_python_checkpoint_test_srcs CONFIGURE_DEPENDS
     "${ORTTRAINING_SOURCE_DIR}/test/python/checkpoint/*.py"
 )
@@ -242,6 +245,7 @@ add_custom_command(
   COMMAND ${CMAKE_COMMAND} -E make_directory $<TARGET_FILE_DIR:${test_data_target}>/onnxruntime/quantization/operators
   COMMAND ${CMAKE_COMMAND} -E make_directory $<TARGET_FILE_DIR:${test_data_target}>/checkpoint
   COMMAND ${CMAKE_COMMAND} -E make_directory $<TARGET_FILE_DIR:${test_data_target}>/dhp_parallel
+  COMMAND ${CMAKE_COMMAND} -E make_directory $<TARGET_FILE_DIR:${test_data_target}>/quantization
   COMMAND ${CMAKE_COMMAND} -E copy
       ${ONNXRUNTIME_ROOT}/__init__.py
       $<TARGET_FILE_DIR:${test_data_target}>/onnxruntime/
@@ -257,6 +261,9 @@ add_custom_command(
   COMMAND ${CMAKE_COMMAND} -E copy
       ${onnxruntime_python_test_srcs}
       $<TARGET_FILE_DIR:${test_data_target}>
+  COMMAND ${CMAKE_COMMAND} -E copy
+      ${onnxruntime_python_quantization_test_srcs}
+      $<TARGET_FILE_DIR:${test_data_target}>/quantization/
   COMMAND ${CMAKE_COMMAND} -E copy
       ${onnxruntime_python_checkpoint_test_srcs}
       $<TARGET_FILE_DIR:${test_data_target}>/checkpoint/

onnxruntime/python/tools/quantization/E2E_example_model/object_detection/trt/yolov3/data_reader.py

@@ -1,5 +1,5 @@
 from onnxruntime.quantization import CalibrationDataReader
-from .preprocessing import yolov3_preprocess_func, yolov3_vision_preprocess_func
+from preprocessing import yolov3_preprocess_func, yolov3_vision_preprocess_func
 import onnxruntime
 from argparse import Namespace
 import os

onnxruntime/python/tools/quantization/E2E_example_model/object_detection/trt/yolov3/e2e_user_yolov3_example.py

@@ -1,5 +1,7 @@
 import os
-from onnxruntime.quantization import get_calibrator, YoloV3DataReader, YoloV3VisionDataReader, YoloV3Evaluator, YoloV3VisionEvaluator, generate_calibration_table, write_calibration_table
+from onnxruntime.quantization import get_calibrator, write_calibration_table, generate_calibration_table
+from data_reader import YoloV3DataReader, YoloV3VisionDataReader
+from evaluate import YoloV3Evaluator, YoloV3VisionEvaluator
 from dataset_utils import *
 
 

onnxruntime/python/tools/quantization/E2E_example_model/object_detection/trt/yolov3/evaluate.py

@@ -6,8 +6,8 @@
 # license information.
 # --------------------------------------------------------------------------
 
-from .calibrate import CalibrationDataReader, calibrate
 import onnxruntime
+from onnxruntime.quantization.calibrate import CalibrationDataReader, calibrate
 import numpy as np
 
 

onnxruntime/python/tools/quantization/__init__.py

@@ -3,5 +3,3 @@ from .quantize import QuantizationMode
 from .calibrate import CalibrationDataReader, calculate_calibration_data, get_calibrator, generate_calibration_table
 from .calibrate import calibrate
 from .quant_utils import QuantType, write_calibration_table
-from .evaluate import YoloV3Evaluator, YoloV3VisionEvaluator
-from .data_reader import YoloV3DataReader, YoloV3VisionDataReader

onnxruntime/test/python/quantization/read_me.txt

@@ -0,0 +1 @@
+Please name the test file with pattern test_*. It is the default module pattern unittest discover search with.

onnxruntime/test/python/quantization/test_calibration.py

@@ -75,7 +75,7 @@ class TestCalibrate(unittest.TestCase):
         matmul_node = onnx.helper.make_node('MatMul', ['D', 'E'], ['F'], name='MatMul')
         graph = helper.make_graph([conv_node, clip_node, matmul_node], 'test_graph_1', [A, B, E], [F])
 
-        model = helper.make_model(graph)
+        model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 13)])
         test_model_path = './test_model_1.onnx'
         onnx.save(model, test_model_path)
 
@@ -120,7 +120,7 @@ class TestCalibrate(unittest.TestCase):
                                             kernel_shape=[3, 3],
                                             pads=[1, 1, 1, 1])
         graph = helper.make_graph([conv_node_1, conv_node_2], 'test_graph_2', [G, H, J], [K])
-        model = helper.make_model(graph)
+        model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 13)])
         test_model_path = './test_model_2.onnx'
         onnx.save(model, test_model_path)
 
@@ -166,7 +166,7 @@ class TestCalibrate(unittest.TestCase):
         clip_node = onnx.helper.make_node('Clip', ['O'], ['P'], name='Clip')
         matmul_node = onnx.helper.make_node('MatMul', ['P', 'M'], ['Q'], name='MatMul')
         graph = helper.make_graph([relu_node, conv_node, clip_node, matmul_node], 'test_graph_3', [L, N], [Q])
-        model = helper.make_model(graph)
+        model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 13)])
         test_model_path = './test_model_3.onnx'
         onnx.save(model, test_model_path)
 
@@ -238,7 +238,7 @@ class TestCalibrate(unittest.TestCase):
         graph.initializer.add().CopyFrom(X5_weight)
         graph.initializer.add().CopyFrom(X5_bias)
 
-        model = helper.make_model(graph)
+        model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 13)])
         test_model_path = './test_model_4.onnx'
         onnx.save(model, test_model_path)
         data_reader = TestDataReaderSecond()
@@ -253,7 +253,7 @@ class TestCalibrate(unittest.TestCase):
         quantization_params_dict = calibrater.calculate_quantization_params(dict_for_quantization)
 
         #check the size of the quantization dictionary
-        self.assertEqual(len(quantization_params_dict), 11)
+        self.assertEqual(len(quantization_params_dict), 5)
 
         #check the computation of zp and scale
         for key, value in quantization_params_dict.items():

onnxruntime/test/python/quantization/test_qat.py

@@ -80,7 +80,7 @@ def generate_qat_model(model_names):
     graph.initializer.add().CopyFrom(input_weight_1)
     graph.initializer.add().CopyFrom(input_bias_1)
 
-    model_1 = onnx.helper.make_model(graph)
+    model_1 = onnx.helper.make_model(graph, opset_imports=[helper.make_opsetid("", 13)])
     model_1.ir_version = onnx.IR_VERSION
     onnx.save(model_1, model_names[0])
 
@@ -152,7 +152,7 @@ def generate_qat_model(model_names):
     graph.initializer.add().CopyFrom(conv_weight_1)
     graph.initializer.add().CopyFrom(conv_bias_1)
 
-    model_2 = onnx.helper.make_model(graph)
+    model_2 = onnx.helper.make_model(graph, opset_imports=[helper.make_opsetid("", 13)])
     model_2.ir_version = onnx.IR_VERSION
     onnx.save(model_2, model_names[1])
 
@@ -203,7 +203,7 @@ def generate_qat_support_model(model_names, test_initializers):
 
     model_1 = onnx.ModelProto()
     model_1.ir_version = onnx.IR_VERSION
-    model_1 = onnx.helper.make_model(graph)
+    model_1 = onnx.helper.make_model(graph, opset_imports=[helper.make_opsetid("", 13)])
     onnx.save(model_1, model_names[0])
 
     test_qat_support_models.extend([model_1])
@@ -244,7 +244,7 @@ def generate_qat_support_model(model_names, test_initializers):
 
     model_2 = onnx.ModelProto()
     model_2.ir_version = onnx.IR_VERSION
-    model_2 = onnx.helper.make_model(graph)
+    model_2 = onnx.helper.make_model(graph, opset_imports=[helper.make_opsetid("", 13)])
     onnx.save(model_1, model_names[1])
 
     test_qat_support_models.extend([model_2])

tools/ci_build/build.py

@@ -1467,6 +1467,8 @@ def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs):
 
             if onnx_test:
                 run_subprocess([sys.executable, 'onnxruntime_test_python_backend.py'], cwd=cwd, dll_path=dll_path)
+                run_subprocess([sys.executable, '-m', 'unittest', 'discover', '-s', 'quantization'],
+                               cwd=cwd, dll_path=dll_path)
 
                 if not args.disable_ml_ops:
                     run_subprocess([sys.executable, 'onnxruntime_test_python_backend_mlops.py'],