From c20965f9b2a241bf3c675f7f4dabb4890e5f3296 Mon Sep 17 00:00:00 2001 From: Yufeng Li Date: Mon, 25 Jan 2021 09:33:08 -0800 Subject: [PATCH] enable pipeline to run quantization tests (#6416) * enable pipeline to run quantization tests setup test pipeline for quantization --- cmake/onnxruntime_python.cmake | 7 +++++++ .../image_classification/{ => cpu}/ReadMe.txt | 0 .../{ => cpu}/mobilenetv2-7.onnx | Bin .../{ => cpu}/resnet50-v1-9.onnx | Bin .../image_classification/{ => cpu}/run.py | 0 .../{ => cpu}/test_images/daisy.jpg | Bin .../{ => cpu}/test_images/rose.jpg | Bin .../{ => cpu}/test_images/tulip.jpg | Bin .../trt/resnet50}/e2e_tensorrt_resnet_example.py | 0 .../annotations/instances_val2017_person.json | 0 .../trt/yolov3}/coco-object-categories-2017.json | 0 .../object_detection/trt/yolov3}/data_reader.py | 2 +- .../trt/yolov3}/e2e_user_yolov3_example.py | 4 +++- .../object_detection/trt/yolov3}/evaluate.py | 2 +- .../trt/yolov3}/onnx_coco_classes.txt | 0 .../object_detection/trt/yolov3}/preprocessing.py | 0 .../trt/yolov3}/trt_cal_table.fbs | 0 onnxruntime/python/tools/quantization/__init__.py | 2 -- onnxruntime/test/python/quantization/read_me.txt | 1 + .../python/quantization/test_calibration.py} | 10 +++++----- .../python/quantization/test_qat.py} | 8 ++++---- tools/ci_build/build.py | 2 ++ 22 files changed, 24 insertions(+), 14 deletions(-) rename onnxruntime/python/tools/quantization/E2E_example_model/image_classification/{ => cpu}/ReadMe.txt (100%) rename onnxruntime/python/tools/quantization/E2E_example_model/image_classification/{ => cpu}/mobilenetv2-7.onnx (100%) rename onnxruntime/python/tools/quantization/E2E_example_model/image_classification/{ => cpu}/resnet50-v1-9.onnx (100%) rename onnxruntime/python/tools/quantization/E2E_example_model/image_classification/{ => cpu}/run.py (100%) rename onnxruntime/python/tools/quantization/E2E_example_model/image_classification/{ => cpu}/test_images/daisy.jpg (100%) rename onnxruntime/python/tools/quantization/E2E_example_model/image_classification/{ => cpu}/test_images/rose.jpg (100%) rename onnxruntime/python/tools/quantization/E2E_example_model/image_classification/{ => cpu}/test_images/tulip.jpg (100%) rename onnxruntime/python/tools/quantization/E2E_example_model/{ => image_classification/trt/resnet50}/e2e_tensorrt_resnet_example.py (100%) rename onnxruntime/python/tools/quantization/E2E_example_model/{ => object_detection/trt/yolov3}/annotations/instances_val2017_person.json (100%) rename onnxruntime/python/tools/quantization/E2E_example_model/{ => object_detection/trt/yolov3}/coco-object-categories-2017.json (100%) rename onnxruntime/python/tools/quantization/{ => E2E_example_model/object_detection/trt/yolov3}/data_reader.py (99%) rename onnxruntime/python/tools/quantization/E2E_example_model/{ => object_detection/trt/yolov3}/e2e_user_yolov3_example.py (94%) rename onnxruntime/python/tools/quantization/{ => E2E_example_model/object_detection/trt/yolov3}/evaluate.py (99%) rename onnxruntime/python/tools/quantization/E2E_example_model/{ => object_detection/trt/yolov3}/onnx_coco_classes.txt (100%) rename onnxruntime/python/tools/quantization/{ => E2E_example_model/object_detection/trt/yolov3}/preprocessing.py (100%) rename onnxruntime/python/tools/quantization/E2E_example_model/{ => object_detection/trt/yolov3}/trt_cal_table.fbs (100%) create mode 100644 onnxruntime/test/python/quantization/read_me.txt rename onnxruntime/{python/tools/quantization/test/test_calibrate.py => test/python/quantization/test_calibration.py} (96%) rename onnxruntime/{python/tools/quantization/test/test_qat_support.py => test/python/quantization/test_qat.py} (97%) diff --git a/cmake/onnxruntime_python.cmake b/cmake/onnxruntime_python.cmake index 0f1eb949fb..fb950a379f 100644 --- a/cmake/onnxruntime_python.cmake +++ b/cmake/onnxruntime_python.cmake @@ -194,6 +194,9 @@ file(GLOB onnxruntime_python_test_srcs CONFIGURE_DEPENDS "${ONNXRUNTIME_ROOT}/test/python/*.py" "${ORTTRAINING_SOURCE_DIR}/test/python/*.py" ) +file(GLOB onnxruntime_python_quantization_test_srcs CONFIGURE_DEPENDS + "${ONNXRUNTIME_ROOT}/test/python/quantization/*.py" +) file(GLOB onnxruntime_python_checkpoint_test_srcs CONFIGURE_DEPENDS "${ORTTRAINING_SOURCE_DIR}/test/python/checkpoint/*.py" ) @@ -242,6 +245,7 @@ add_custom_command( COMMAND ${CMAKE_COMMAND} -E make_directory $/onnxruntime/quantization/operators COMMAND ${CMAKE_COMMAND} -E make_directory $/checkpoint COMMAND ${CMAKE_COMMAND} -E make_directory $/dhp_parallel + COMMAND ${CMAKE_COMMAND} -E make_directory $/quantization COMMAND ${CMAKE_COMMAND} -E copy ${ONNXRUNTIME_ROOT}/__init__.py $/onnxruntime/ @@ -257,6 +261,9 @@ add_custom_command( COMMAND ${CMAKE_COMMAND} -E copy ${onnxruntime_python_test_srcs} $ + COMMAND ${CMAKE_COMMAND} -E copy + ${onnxruntime_python_quantization_test_srcs} + $/quantization/ COMMAND ${CMAKE_COMMAND} -E copy ${onnxruntime_python_checkpoint_test_srcs} $/checkpoint/ diff --git a/onnxruntime/python/tools/quantization/E2E_example_model/image_classification/ReadMe.txt b/onnxruntime/python/tools/quantization/E2E_example_model/image_classification/cpu/ReadMe.txt similarity index 100% rename from onnxruntime/python/tools/quantization/E2E_example_model/image_classification/ReadMe.txt rename to onnxruntime/python/tools/quantization/E2E_example_model/image_classification/cpu/ReadMe.txt diff --git a/onnxruntime/python/tools/quantization/E2E_example_model/image_classification/mobilenetv2-7.onnx b/onnxruntime/python/tools/quantization/E2E_example_model/image_classification/cpu/mobilenetv2-7.onnx similarity index 100% rename from onnxruntime/python/tools/quantization/E2E_example_model/image_classification/mobilenetv2-7.onnx rename to onnxruntime/python/tools/quantization/E2E_example_model/image_classification/cpu/mobilenetv2-7.onnx diff --git a/onnxruntime/python/tools/quantization/E2E_example_model/image_classification/resnet50-v1-9.onnx b/onnxruntime/python/tools/quantization/E2E_example_model/image_classification/cpu/resnet50-v1-9.onnx similarity index 100% rename from onnxruntime/python/tools/quantization/E2E_example_model/image_classification/resnet50-v1-9.onnx rename to onnxruntime/python/tools/quantization/E2E_example_model/image_classification/cpu/resnet50-v1-9.onnx diff --git a/onnxruntime/python/tools/quantization/E2E_example_model/image_classification/run.py b/onnxruntime/python/tools/quantization/E2E_example_model/image_classification/cpu/run.py similarity index 100% rename from onnxruntime/python/tools/quantization/E2E_example_model/image_classification/run.py rename to onnxruntime/python/tools/quantization/E2E_example_model/image_classification/cpu/run.py diff --git a/onnxruntime/python/tools/quantization/E2E_example_model/image_classification/test_images/daisy.jpg b/onnxruntime/python/tools/quantization/E2E_example_model/image_classification/cpu/test_images/daisy.jpg similarity index 100% rename from onnxruntime/python/tools/quantization/E2E_example_model/image_classification/test_images/daisy.jpg rename to onnxruntime/python/tools/quantization/E2E_example_model/image_classification/cpu/test_images/daisy.jpg diff --git a/onnxruntime/python/tools/quantization/E2E_example_model/image_classification/test_images/rose.jpg b/onnxruntime/python/tools/quantization/E2E_example_model/image_classification/cpu/test_images/rose.jpg similarity index 100% rename from onnxruntime/python/tools/quantization/E2E_example_model/image_classification/test_images/rose.jpg rename to onnxruntime/python/tools/quantization/E2E_example_model/image_classification/cpu/test_images/rose.jpg diff --git a/onnxruntime/python/tools/quantization/E2E_example_model/image_classification/test_images/tulip.jpg b/onnxruntime/python/tools/quantization/E2E_example_model/image_classification/cpu/test_images/tulip.jpg similarity index 100% rename from onnxruntime/python/tools/quantization/E2E_example_model/image_classification/test_images/tulip.jpg rename to onnxruntime/python/tools/quantization/E2E_example_model/image_classification/cpu/test_images/tulip.jpg diff --git a/onnxruntime/python/tools/quantization/E2E_example_model/e2e_tensorrt_resnet_example.py b/onnxruntime/python/tools/quantization/E2E_example_model/image_classification/trt/resnet50/e2e_tensorrt_resnet_example.py similarity index 100% rename from onnxruntime/python/tools/quantization/E2E_example_model/e2e_tensorrt_resnet_example.py rename to onnxruntime/python/tools/quantization/E2E_example_model/image_classification/trt/resnet50/e2e_tensorrt_resnet_example.py diff --git a/onnxruntime/python/tools/quantization/E2E_example_model/annotations/instances_val2017_person.json b/onnxruntime/python/tools/quantization/E2E_example_model/object_detection/trt/yolov3/annotations/instances_val2017_person.json similarity index 100% rename from onnxruntime/python/tools/quantization/E2E_example_model/annotations/instances_val2017_person.json rename to onnxruntime/python/tools/quantization/E2E_example_model/object_detection/trt/yolov3/annotations/instances_val2017_person.json diff --git a/onnxruntime/python/tools/quantization/E2E_example_model/coco-object-categories-2017.json b/onnxruntime/python/tools/quantization/E2E_example_model/object_detection/trt/yolov3/coco-object-categories-2017.json similarity index 100% rename from onnxruntime/python/tools/quantization/E2E_example_model/coco-object-categories-2017.json rename to onnxruntime/python/tools/quantization/E2E_example_model/object_detection/trt/yolov3/coco-object-categories-2017.json diff --git a/onnxruntime/python/tools/quantization/data_reader.py b/onnxruntime/python/tools/quantization/E2E_example_model/object_detection/trt/yolov3/data_reader.py similarity index 99% rename from onnxruntime/python/tools/quantization/data_reader.py rename to onnxruntime/python/tools/quantization/E2E_example_model/object_detection/trt/yolov3/data_reader.py index c957482add..2adc698f36 100644 --- a/onnxruntime/python/tools/quantization/data_reader.py +++ b/onnxruntime/python/tools/quantization/E2E_example_model/object_detection/trt/yolov3/data_reader.py @@ -1,5 +1,5 @@ from onnxruntime.quantization import CalibrationDataReader -from .preprocessing import yolov3_preprocess_func, yolov3_vision_preprocess_func +from preprocessing import yolov3_preprocess_func, yolov3_vision_preprocess_func import onnxruntime from argparse import Namespace import os diff --git a/onnxruntime/python/tools/quantization/E2E_example_model/e2e_user_yolov3_example.py b/onnxruntime/python/tools/quantization/E2E_example_model/object_detection/trt/yolov3/e2e_user_yolov3_example.py similarity index 94% rename from onnxruntime/python/tools/quantization/E2E_example_model/e2e_user_yolov3_example.py rename to onnxruntime/python/tools/quantization/E2E_example_model/object_detection/trt/yolov3/e2e_user_yolov3_example.py index 4a43761da8..f968c5bfa1 100644 --- a/onnxruntime/python/tools/quantization/E2E_example_model/e2e_user_yolov3_example.py +++ b/onnxruntime/python/tools/quantization/E2E_example_model/object_detection/trt/yolov3/e2e_user_yolov3_example.py @@ -1,5 +1,7 @@ import os -from onnxruntime.quantization import get_calibrator, YoloV3DataReader, YoloV3VisionDataReader, YoloV3Evaluator, YoloV3VisionEvaluator, generate_calibration_table, write_calibration_table +from onnxruntime.quantization import get_calibrator, write_calibration_table, generate_calibration_table +from data_reader import YoloV3DataReader, YoloV3VisionDataReader +from evaluate import YoloV3Evaluator, YoloV3VisionEvaluator from dataset_utils import * diff --git a/onnxruntime/python/tools/quantization/evaluate.py b/onnxruntime/python/tools/quantization/E2E_example_model/object_detection/trt/yolov3/evaluate.py similarity index 99% rename from onnxruntime/python/tools/quantization/evaluate.py rename to onnxruntime/python/tools/quantization/E2E_example_model/object_detection/trt/yolov3/evaluate.py index 659f53cc1b..05d5c654f4 100644 --- a/onnxruntime/python/tools/quantization/evaluate.py +++ b/onnxruntime/python/tools/quantization/E2E_example_model/object_detection/trt/yolov3/evaluate.py @@ -6,8 +6,8 @@ # license information. # -------------------------------------------------------------------------- -from .calibrate import CalibrationDataReader, calibrate import onnxruntime +from onnxruntime.quantization.calibrate import CalibrationDataReader, calibrate import numpy as np diff --git a/onnxruntime/python/tools/quantization/E2E_example_model/onnx_coco_classes.txt b/onnxruntime/python/tools/quantization/E2E_example_model/object_detection/trt/yolov3/onnx_coco_classes.txt similarity index 100% rename from onnxruntime/python/tools/quantization/E2E_example_model/onnx_coco_classes.txt rename to onnxruntime/python/tools/quantization/E2E_example_model/object_detection/trt/yolov3/onnx_coco_classes.txt diff --git a/onnxruntime/python/tools/quantization/preprocessing.py b/onnxruntime/python/tools/quantization/E2E_example_model/object_detection/trt/yolov3/preprocessing.py similarity index 100% rename from onnxruntime/python/tools/quantization/preprocessing.py rename to onnxruntime/python/tools/quantization/E2E_example_model/object_detection/trt/yolov3/preprocessing.py diff --git a/onnxruntime/python/tools/quantization/E2E_example_model/trt_cal_table.fbs b/onnxruntime/python/tools/quantization/E2E_example_model/object_detection/trt/yolov3/trt_cal_table.fbs similarity index 100% rename from onnxruntime/python/tools/quantization/E2E_example_model/trt_cal_table.fbs rename to onnxruntime/python/tools/quantization/E2E_example_model/object_detection/trt/yolov3/trt_cal_table.fbs diff --git a/onnxruntime/python/tools/quantization/__init__.py b/onnxruntime/python/tools/quantization/__init__.py index 6debc324fe..e5816e79c7 100644 --- a/onnxruntime/python/tools/quantization/__init__.py +++ b/onnxruntime/python/tools/quantization/__init__.py @@ -3,5 +3,3 @@ from .quantize import QuantizationMode from .calibrate import CalibrationDataReader, calculate_calibration_data, get_calibrator, generate_calibration_table from .calibrate import calibrate from .quant_utils import QuantType, write_calibration_table -from .evaluate import YoloV3Evaluator, YoloV3VisionEvaluator -from .data_reader import YoloV3DataReader, YoloV3VisionDataReader diff --git a/onnxruntime/test/python/quantization/read_me.txt b/onnxruntime/test/python/quantization/read_me.txt new file mode 100644 index 0000000000..d8ebde00ee --- /dev/null +++ b/onnxruntime/test/python/quantization/read_me.txt @@ -0,0 +1 @@ +Please name the test file with pattern test_*. It is the default module pattern unittest discover search with. \ No newline at end of file diff --git a/onnxruntime/python/tools/quantization/test/test_calibrate.py b/onnxruntime/test/python/quantization/test_calibration.py similarity index 96% rename from onnxruntime/python/tools/quantization/test/test_calibrate.py rename to onnxruntime/test/python/quantization/test_calibration.py index bc41a502fd..b3b3af66a7 100644 --- a/onnxruntime/python/tools/quantization/test/test_calibrate.py +++ b/onnxruntime/test/python/quantization/test_calibration.py @@ -75,7 +75,7 @@ class TestCalibrate(unittest.TestCase): matmul_node = onnx.helper.make_node('MatMul', ['D', 'E'], ['F'], name='MatMul') graph = helper.make_graph([conv_node, clip_node, matmul_node], 'test_graph_1', [A, B, E], [F]) - model = helper.make_model(graph) + model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 13)]) test_model_path = './test_model_1.onnx' onnx.save(model, test_model_path) @@ -120,7 +120,7 @@ class TestCalibrate(unittest.TestCase): kernel_shape=[3, 3], pads=[1, 1, 1, 1]) graph = helper.make_graph([conv_node_1, conv_node_2], 'test_graph_2', [G, H, J], [K]) - model = helper.make_model(graph) + model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 13)]) test_model_path = './test_model_2.onnx' onnx.save(model, test_model_path) @@ -166,7 +166,7 @@ class TestCalibrate(unittest.TestCase): clip_node = onnx.helper.make_node('Clip', ['O'], ['P'], name='Clip') matmul_node = onnx.helper.make_node('MatMul', ['P', 'M'], ['Q'], name='MatMul') graph = helper.make_graph([relu_node, conv_node, clip_node, matmul_node], 'test_graph_3', [L, N], [Q]) - model = helper.make_model(graph) + model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 13)]) test_model_path = './test_model_3.onnx' onnx.save(model, test_model_path) @@ -238,7 +238,7 @@ class TestCalibrate(unittest.TestCase): graph.initializer.add().CopyFrom(X5_weight) graph.initializer.add().CopyFrom(X5_bias) - model = helper.make_model(graph) + model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 13)]) test_model_path = './test_model_4.onnx' onnx.save(model, test_model_path) data_reader = TestDataReaderSecond() @@ -253,7 +253,7 @@ class TestCalibrate(unittest.TestCase): quantization_params_dict = calibrater.calculate_quantization_params(dict_for_quantization) #check the size of the quantization dictionary - self.assertEqual(len(quantization_params_dict), 11) + self.assertEqual(len(quantization_params_dict), 5) #check the computation of zp and scale for key, value in quantization_params_dict.items(): diff --git a/onnxruntime/python/tools/quantization/test/test_qat_support.py b/onnxruntime/test/python/quantization/test_qat.py similarity index 97% rename from onnxruntime/python/tools/quantization/test/test_qat_support.py rename to onnxruntime/test/python/quantization/test_qat.py index e01fd23cd2..52ac1b1e92 100644 --- a/onnxruntime/python/tools/quantization/test/test_qat_support.py +++ b/onnxruntime/test/python/quantization/test_qat.py @@ -80,7 +80,7 @@ def generate_qat_model(model_names): graph.initializer.add().CopyFrom(input_weight_1) graph.initializer.add().CopyFrom(input_bias_1) - model_1 = onnx.helper.make_model(graph) + model_1 = onnx.helper.make_model(graph, opset_imports=[helper.make_opsetid("", 13)]) model_1.ir_version = onnx.IR_VERSION onnx.save(model_1, model_names[0]) @@ -152,7 +152,7 @@ def generate_qat_model(model_names): graph.initializer.add().CopyFrom(conv_weight_1) graph.initializer.add().CopyFrom(conv_bias_1) - model_2 = onnx.helper.make_model(graph) + model_2 = onnx.helper.make_model(graph, opset_imports=[helper.make_opsetid("", 13)]) model_2.ir_version = onnx.IR_VERSION onnx.save(model_2, model_names[1]) @@ -203,7 +203,7 @@ def generate_qat_support_model(model_names, test_initializers): model_1 = onnx.ModelProto() model_1.ir_version = onnx.IR_VERSION - model_1 = onnx.helper.make_model(graph) + model_1 = onnx.helper.make_model(graph, opset_imports=[helper.make_opsetid("", 13)]) onnx.save(model_1, model_names[0]) test_qat_support_models.extend([model_1]) @@ -244,7 +244,7 @@ def generate_qat_support_model(model_names, test_initializers): model_2 = onnx.ModelProto() model_2.ir_version = onnx.IR_VERSION - model_2 = onnx.helper.make_model(graph) + model_2 = onnx.helper.make_model(graph, opset_imports=[helper.make_opsetid("", 13)]) onnx.save(model_1, model_names[1]) test_qat_support_models.extend([model_2]) diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index 7713e1a3bb..a4fbb8f016 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -1467,6 +1467,8 @@ def run_onnxruntime_tests(args, source_dir, ctest_path, build_dir, configs): if onnx_test: run_subprocess([sys.executable, 'onnxruntime_test_python_backend.py'], cwd=cwd, dll_path=dll_path) + run_subprocess([sys.executable, '-m', 'unittest', 'discover', '-s', 'quantization'], + cwd=cwd, dll_path=dll_path) if not args.disable_ml_ops: run_subprocess([sys.executable, 'onnxruntime_test_python_backend_mlops.py'],