saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-29 20:14:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Quantization calibration refactor (#6893)

Browse source 
* Code refactor

* Modify code to tackle OOM when calibrating on larget dataset

* Fix mismatch issue when setting keepdims on ReduceMin/ReduceMax

* Add COCO val 2017 annotation

* Fix mismatch issue when setting keepdims on ReduceMin/ReduceMax

* Fix bug of "No module named:onnxruntime.quantization.CalTableFlatBuffers"

* Check and install flatbuffers module

* Add script to donwload coco dataset image and refactor example

* Fix bug of "No module
named:onnxruntime.quantization.CalTableFlatBuffers"

* Add CalTableFaltBuffers as module

* Remove annotation, user can download by themselves.

* Uncommet code

* Add back instances_val2017.json

* Make sure flatbuffers installed when ORT is installed

* Refactor code to call coco api

* Enable FP16 for example
This commit is contained in:
Chi Lo 2021-03-19 01:09:11 -07:00 committed by GitHub
parent 701e73b5b8
commit 8c3b59a026
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 365 additions and 163 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
cmake/onnxruntime_python.cmake
View file

@ -249,6 +249,9 @@ file(GLOB onnxruntime_python_quantization_src CONFIGURE_DEPENDS
file(GLOB onnxruntime_python_quantization_operators_src CONFIGURE_DEPENDS
"${ONNXRUNTIME_ROOT}/python/tools/quantization/operators/*.py"
)
file(GLOB onnxruntime_python_quantization_cal_table_flatbuffers_src CONFIGURE_DEPENDS
"${ONNXRUNTIME_ROOT}/python/tools/quantization/CalTableFlatBuffers/*.py"
)
file(GLOB onnxruntime_python_transformers_src CONFIGURE_DEPENDS
"${ONNXRUNTIME_ROOT}/python/tools/transformers/*.py"
)
@ -277,6 +280,7 @@ add_custom_command(
COMMAND ${CMAKE_COMMAND} -E make_directory $<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/transformers/longformer
COMMAND ${CMAKE_COMMAND} -E make_directory $<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/quantization
COMMAND ${CMAKE_COMMAND} -E make_directory $<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/quantization/operators
COMMAND ${CMAKE_COMMAND} -E make_directory $<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/quantization/CalTableFlatBuffers
COMMAND ${CMAKE_COMMAND} -E make_directory $<TARGET_FILE_DIR:${build_output_target}>/checkpoint
COMMAND ${CMAKE_COMMAND} -E make_directory $<TARGET_FILE_DIR:${build_output_target}>/dhp_parallel
COMMAND ${CMAKE_COMMAND} -E make_directory $<TARGET_FILE_DIR:${build_output_target}>/quantization
@ -322,6 +326,9 @@ add_custom_command(
COMMAND ${CMAKE_COMMAND} -E copy
${onnxruntime_python_quantization_operators_src}
$<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/quantization/operators/
COMMAND ${CMAKE_COMMAND} -E copy
${onnxruntime_python_quantization_cal_table_flatbuffers_src}
$<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/quantization/CalTableFlatBuffers/
COMMAND ${CMAKE_COMMAND} -E copy
${onnxruntime_python_transformers_src}
$<TARGET_FILE_DIR:${build_output_target}>/onnxruntime/transformers/

1
onnxruntime/python/tools/quantization/E2E_example_model/object_detection/trt/yolov3/annotations/instances_val2017.json Normal file
View file

File diff suppressed because one or more lines are too long

165
onnxruntime/python/tools/quantization/E2E_example_model/object_detection/trt/yolov3/coco_filter.py Normal file
View file

@ -0,0 +1,165 @@
import json
import os
from pathlib import Path
class CocoFilter():
""" Filters the COCO dataset
"""
def _process_info(self):
self.info = self.coco['info']
def _process_licenses(self):
self.licenses = self.coco['licenses']
def _process_categories(self):
self.categories = dict()
self.super_categories = dict()
self.category_set = set()
self.category_to_image_ids = dict()
for category in self.coco['categories']:
cat_id = category['id']
super_category = category['supercategory']
# Add category to categories dict
if cat_id not in self.categories:
self.categories[cat_id] = category
self.category_set.add(category['name'])
else:
print(f'ERROR: Skipping duplicate category id: {category}')
# Add category id to the super_categories dict
if super_category not in self.super_categories:
self.super_categories[super_category] = {cat_id}
else:
self.super_categories[super_category] |= {cat_id} # e.g. {1, 2, 3} |= {4} => {1, 2, 3, 4}
def _process_images(self):
self.images = dict()
for image in self.coco['images']:
image_id = image['id']
if image_id not in self.images:
self.images[image_id] = image
else:
print(f'ERROR: Skipping duplicate image id: {image}')
def _process_segmentations(self):
self.segmentations = dict()
for segmentation in self.coco['annotations']:
image_id = segmentation['image_id']
if image_id not in self.segmentations:
self.segmentations[image_id] = []
self.segmentations[image_id].append(segmentation)
def _filter_categories(self):
""" Find category ids matching args
Create mapping from original category id to new category id
Create new collection of categories
"""
if 'all' in self.filter_categories:
print("Filter all categories.")
self.filter_categories = self.category_set
missing_categories = set(self.filter_categories) - self.category_set
if len(missing_categories) > 0:
print(f'Did not find categories: {missing_categories}')
should_continue = input('Continue? (y/n) ').lower()
if should_continue != 'y' and should_continue != 'yes':
print('Quitting early.')
quit()
self.new_category_map = dict()
new_id = 1
for key, item in self.categories.items():
if item['name'] in self.filter_categories:
self.new_category_map[key] = new_id
new_id += 1
else:
print(item['name'])
self.new_categories = []
for original_cat_id, new_id in self.new_category_map.items():
new_category = dict(self.categories[original_cat_id])
new_category['id'] = int(new_id)
self.new_categories.append(new_category)
def _handle_images(self):
for image_id, segmentation_list in self.segmentations.items():
for segmentation in segmentation_list:
original_seg_cat = segmentation['category_id']
cat_id = original_seg_cat
if cat_id not in self.new_category_map.keys():
continue
if cat_id not in self.category_to_image_ids:
self.category_to_image_ids[cat_id] = set()
self.category_to_image_ids[cat_id].add(image_id)
for key, value in self.category_to_image_ids.items():
self.category_to_image_ids[key] = sorted(list(value))
import random
random.shuffle(self.category_to_image_ids[key])
self.calib_img_list = set()
for key, value in self.category_to_image_ids.items():
for i in value[:20]:
self.calib_img_list.add(i)
import requests
for id in self.calib_img_list:
im = self.images[id]
img_data = requests.get(im['coco_url']).content
with open(os.path.join(self.image_folder, 'calib', im['file_name']), 'wb') as handler:
handler.write(img_data)
def main(self, args):
# Open json
self.input_json_path = Path(args.input_json)
self.image_folder = Path(args.image_folder)
self.filter_categories = args.categories
# Verify input path exists
if not self.input_json_path.exists():
print('Input json path not found.')
print('Quitting early.')
quit()
# Load the json
print('Loading json file...')
with open(self.input_json_path) as json_file:
self.coco = json.load(json_file)
# Process the json
print('Processing input json...')
self._process_info()
self._process_licenses()
self._process_categories()
self._process_images()
self._process_segmentations()
# Filter to specific categories
print('Filtering...')
self._filter_categories()
self._handle_images()
if __name__ == "__main__":
import argparse
parser = argparse.ArgumentParser(description="Filter COCO JSON: "
"Filters a COCO Instances JSON file to only include specified categories. "
"This includes images. Does not modify 'info' or 'licenses'.")
parser.add_argument("-i", "--input_json", dest="input_json",
help="path to a json file in coco format")
parser.add_argument("-f", "--image_folder", dest="image_folder",
help="folder to save images")
parser.add_argument("-c", "--categories", nargs='+', dest="categories",
help="List of category names separated by spaces, e.g. -c person dog bicycle. If -c all, it includes all categories.")
args = parser.parse_args()
cf = CocoFilter()
cf.main(args)

43
onnxruntime/python/tools/quantization/E2E_example_model/object_detection/trt/yolov3/data_reader.py
View file

@ -1,5 +1,5 @@
from onnxruntime.quantization import CalibrationDataReader
from preprocessing import yolov3_preprocess_func, yolov3_preprocess_func_2, yolov3_variant_preprocess_func, yolov3_variant_preprocess_func_2, yolov3_variant_preprocess_func_3
from preprocessing import yolov3_preprocess_func, yolov3_preprocess_func_2, yolov3_variant_preprocess_func, yolov3_variant_preprocess_func_2
import onnxruntime
from argparse import Namespace
import os
@ -29,10 +29,18 @@ class ObejctDetectionDataReader(CalibrationDataReader):
self.stride = 1
self.batch_size = 1
self.enum_data_dicts = iter([])
self.input_name = None
self.get_input_name()
def get_batch_size(self):
return self.batch_size
def get_input_name(self):
if self.input_name:
return
session = onnxruntime.InferenceSession(self.model_path, providers=['CPUExecutionProvider'])
self.input_name = session.get_inputs()[0].name
class YoloV3DataReader(ObejctDetectionDataReader):
def __init__(self,
@ -45,7 +53,8 @@ class YoloV3DataReader(ObejctDetectionDataReader):
batch_size=1,
model_path='augmented_model.onnx',
is_evaluation=False,
annotations='./annotations/instances_val2017.json'):
annotations='./annotations/instances_val2017.json',
preprocess_func=yolov3_preprocess_func):
ObejctDetectionDataReader.__init__(self, model_path)
self.image_folder = calibration_image_folder
self.model_path = model_path
@ -59,18 +68,13 @@ class YoloV3DataReader(ObejctDetectionDataReader):
self.batch_size = batch_size
self.is_evaluation = is_evaluation
self.input_name = 'input_1'
# self.input_name = 'input_1'
self.img_name_to_img_id = parse_annotations(annotations)
self.preprocess_func = preprocess_func
def get_dataset_size(self):
return len(os.listdir(self.image_folder))
def get_input_name(self):
if self.input_name:
return
session = onnxruntime.InferenceSession(self.model_path, providers=['CPUExecutionProvider'])
self.input_name = session.get_inputs()[0].name
def get_next(self):
iter_data = next(self.enum_data_dicts, None)
if iter_data:
@ -93,10 +97,8 @@ class YoloV3DataReader(ObejctDetectionDataReader):
def load_serial(self):
width = self.width
height = self.width
nchw_data_list, filename_list, image_size_list = yolov3_preprocess_func_2(self.image_folder, height, width,
nchw_data_list, filename_list, image_size_list = preprocess_func(self.image_folder, height, width,
self.start_index, self.stride)
# nchw_data_list, filename_list, image_size_list = yolov3_preprocess_func(self.image_folder, height, width,
# self.start_index, self.stride)
input_name = self.input_name
print("Start from index %s ..." % (str(self.start_index)))
@ -129,7 +131,7 @@ class YoloV3DataReader(ObejctDetectionDataReader):
for index in range(0, stride, batch_size):
start_index = self.start_index + index
print("Load batch from index %s ..." % (str(start_index)))
nchw_data_list, filename_list, image_size_list = yolov3_preprocess_func(self.image_folder, height, width,
nchw_data_list, filename_list, image_size_list = preprocess_func(self.image_folder, height, width,
start_index, batch_size)
if nchw_data_list.size == 0:
@ -178,19 +180,18 @@ class YoloV3VariantDataReader(YoloV3DataReader):
batch_size=1,
model_path='augmented_model.onnx',
is_evaluation=False,
annotations='./annotations/instances_val2017.json'):
annotations='./annotations/instances_val2017.json',
preprocess_func=yolov3_variant_preprocess_func):
YoloV3DataReader.__init__(self, calibration_image_folder, width, height, start_index, end_index, stride,
batch_size, model_path, is_evaluation, annotations)
# self.input_name = '000_net'
self.input_name = 'images'
batch_size, model_path, is_evaluation, annotations, preprocess_func)
# # self.input_name = '000_net'
# self.input_name = 'images'
def load_serial(self):
width = self.width
height = self.height
input_name = self.input_name
# nchw_data_list, filename_list, image_size_list = yolov3_variant_preprocess_func_2(
# self.image_folder, height, width, self.start_index, self.stride)
nchw_data_list, filename_list, image_size_list = yolov3_variant_preprocess_func_3(
nchw_data_list, filename_list, image_size_list = self.preprocess_func(
self.image_folder, height, width, self.start_index, self.stride)
print("Start from index %s ..." % (str(self.start_index)))
@ -224,7 +225,7 @@ class YoloV3VariantDataReader(YoloV3DataReader):
for index in range(0, stride, batch_size):
start_index = self.start_index + index
print("Load batch from index %s ..." % (str(start_index)))
nchw_data_list, filename_list, image_size_list = yolov3_vision_preprocess_func(
nchw_data_list, filename_list, image_size_list = preprocess_func(
self.image_folder, height, width, start_index, batch_size)
if nchw_data_list.size == 0:

100
onnxruntime/python/tools/quantization/E2E_example_model/object_detection/trt/yolov3/e2e_user_yolov3_example.py
View file

@ -1,7 +1,8 @@
import os
from onnxruntime.quantization import create_calibrator, write_calibration_table, CalibrationMethod
from data_reader import YoloV3DataReader, YoloV3VariantDataReader
from evaluate import YoloV3Evaluator, YoloV3VariantEvaluator
from preprocessing import yolov3_preprocess_func, yolov3_preprocess_func_2, yolov3_variant_preprocess_func, yolov3_variant_preprocess_func_2
from evaluate import YoloV3Evaluator, YoloV3VariantEvaluator,YoloV3Variant2Evaluator, YoloV3Variant3Evaluator
def get_calibration_table(model_path, augmented_model_path, calibration_dataset):
@ -58,7 +59,6 @@ def get_prediction_evaluation(model_path, validation_dataset, providers):
result = evaluator.get_result()
annotations = './annotations/instances_val2017.json'
print(result)
evaluator.evaluate(result, annotations)
@ -72,35 +72,46 @@ def get_calibration_table_yolov3_variant(model_path, augmented_model_path, calib
'''
1. Use serial processing
We can use only one DataReader to do serial processing, however,
We can use only one data reader to do serial processing, however,
some machines don't have sufficient memory to hold all dataset images and all intermediate output.
So let multiple DataReader do handle different stride of dataset one by one.
So let multiple data readers to handle different stride of dataset one by one.
DataReader will use serial processing when batch_size is 1.
'''
width = 608
height = 608
total_data_size = len(os.listdir(calibration_dataset))
start_index = 0
stride = 25
stride = 20
batch_size = 1
for i in range(0, total_data_size, stride):
data_reader = YoloV3VariantDataReader(calibration_dataset,
width=608,
height=608,
width=width,
height=height,
start_index=start_index,
end_index=start_index + stride,
stride=stride,
batch_size=1,
batch_size=batch_size,
model_path=augmented_model_path)
calibrator.collect_data(data_reader)
start_index += stride
'''
2. Use batch processing (much faster)
Batch processing requires less memory for intermediate output, therefore let only one DataReader to handle dataset in batch.
However, if encountering OOM, we can make multiple DataReader to do the job just like serial processing does.
Batch processing requires less memory for intermediate output, therefore let only one data reader to handle dataset in batch.
However, if encountering OOM, we can make multiple data reader to do the job just like serial processing does.
DataReader will use batch processing when batch_size > 1.
'''
# data_reader = YoloV3VariantDataReader(calibration_dataset, width=608, height=608, stride=1000, batch_size=20, model_path=augmented_model_path)
# batch_size = 20
# stride=1000
# data_reader = YoloV3VariantDataReader(calibration_dataset,
# width=width,
# height=height,
# stride=stride,
# batch_size=batch_size,
# model_path=augmented_model_path)
# calibrator.collect_data(data_reader)
write_calibration_table(calibrator.compute_range())
@ -108,38 +119,71 @@ def get_calibration_table_yolov3_variant(model_path, augmented_model_path, calib
def get_prediction_evaluation_yolov3_variant(model_path, validation_dataset, providers):
data_reader = YoloV3VariantDataReader(validation_dataset,
width=608,
height=608,
stride=1000,
batch_size=1,
model_path=model_path,
is_evaluation=True)
evaluator = YoloV3VariantEvaluator(model_path, data_reader, width=608, height=608, providers=providers)
width = 608
height = 608
evaluator = YoloV3VariantEvaluator(model_path, None, width=width, height=height, providers=providers)
total_data_size = len(os.listdir(validation_dataset))
start_index = 0
stride=1000
batch_size = 1
for i in range(0, total_data_size, stride):
data_reader = YoloV3VariantDataReader(validation_dataset,
width=width,
height=height,
start_index=start_index,
end_index=start_index+stride,
stride=stride,
batch_size=batch_size,
model_path=model_path,
is_evaluation=True)
evaluator.set_data_reader(data_reader)
evaluator.predict()
start_index += stride
evaluator.predict()
result = evaluator.get_result()
annotations = './annotations/instances_val2017.json'
print(result)
evaluator.evaluate(result, annotations)
if __name__ == '__main__':
'''
TensorRT EP INT8 Inference on Yolov3 model.
The script is using subset of COCO 2017 Train images as calibration and COCO 2017 Val images as evaluation.
1. Please create workspace folders 'train2017/calib' and 'val2017'.
2. Download 2017 Val dataset: http://images.cocodataset.org/zips/val2017.zip
3. Download 2017 Val and Train annotations from http://images.cocodataset.org/annotations/annotations_trainval2017.zip
4. Run following script to download subset of COCO 2017 Train images and save them to 'train2017/calib':
python3 coco_filter.py -i annotations/instances_train2017.json -f train2017 -c all
(Reference and modify from https://github.com/immersive-limit/coco-manager)
5. Download Yolov3 model:
(i) ONNX model zoo yolov3: https://github.com/onnx/models/raw/master/vision/object_detection_segmentation/yolov3/model/yolov3-10.onnx
(ii) yolov3 variants: https://github.com/jkjung-avt/tensorrt_demos.git
'''
yolov3 = 'model zoo'
augmented_model_path = 'augmented_model.onnx'
calibration_dataset = './test2017'
calibration_dataset = './train2017/calib'
validation_dataset = './val2017'
is_onnx_model_zoo_yolov3 = False
if yolov3 == 'model zoo':
# ONNX Model Zoo yolov3
# TensorRT EP INT8 settings
os.environ["ORT_TENSORRT_FP16_ENABLE"] = "1" # Enable FP16 precision
os.environ["ORT_TENSORRT_INT8_ENABLE"] = "1" # Enable INT8 precision
os.environ["ORT_TENSORRT_INT8_CALIBRATION_TABLE_NAME"] = "calibration.flatbuffers" # Calibration table name
os.environ["ORT_TENSORRT_ENGINE_CACHE_ENABLE"] = "1" # Enable engine caching
execution_provider = ["TensorrtExecutionProvider"]
if is_onnx_model_zoo_yolov3:
model_path = 'yolov3.onnx'
get_calibration_table(model_path, augmented_model_path, calibration_dataset)
get_prediction_evaluation(model_path, validation_dataset, ["TensorrtExecutionProvider"])
get_prediction_evaluation(model_path, validation_dataset, execution_provider)
else:
# Yolov3 variants from here
# https://github.com/jkjung-avt/tensorrt_demos.git
model_path = 'yolov3-608.onnx'
get_calibration_table_yolov3_variant(model_path, augmented_model_path, calibration_dataset)
get_prediction_evaluation_yolov3_variant(model_path, validation_dataset, ["TensorrtExecutionProvider"])
get_prediction_evaluation_yolov3_variant(model_path, validation_dataset, execution_provider)

43
onnxruntime/python/tools/quantization/E2E_example_model/object_detection/trt/yolov3/evaluate.py
View file

@ -161,15 +161,6 @@ class YoloV3Evaluator:
# calling coco api
from pycocotools.coco import COCO
from pycocotools.cocoeval import COCOeval
import numpy as np
import skimage.io as io
import pylab
pylab.rcParams['figure.figsize'] = (10.0, 8.0)
annType = ['segm', 'bbox', 'keypoints']
annType = annType[1] #specify type here
prefix = 'person_keypoints' if annType == 'keypoints' else 'instances'
print('Running evaluation for *%s* results.' % (annType))
annFile = annotations
cocoGt = COCO(annFile)
@ -178,11 +169,9 @@ class YoloV3Evaluator:
cocoDt = cocoGt.loadRes(resFile)
imgIds = sorted(cocoGt.getImgIds())
imgIds = imgIds[0:100]
imgId = imgIds[np.random.randint(100)]
# running evaluation
cocoEval = COCOeval(cocoGt, cocoDt, annType)
cocoEval = COCOeval(cocoGt, cocoDt, 'bbox')
cocoEval.params.imgIds = imgIds
cocoEval.evaluate()
cocoEval.accumulate()
@ -491,36 +480,6 @@ class YoloV3Variant3Evaluator(YoloV3Evaluator):
YoloV3Evaluator.__init__(self, model_path, data_reader, width, height, providers,
ground_truth_object_class_file, onnx_object_class_file)
def set_bbox_prediction(self, bboxes, scores, image_height, image_width, image_id):
for i in range(bboxes.shape[0]):
bbox = bboxes[i]
bbox[0] *= self.width #x
bbox[1] *= self.height #y
bbox[2] *= self.width #w
bbox[3] *= self.height #h
img0_shape = (image_height, image_width)
img1_shape = (self.height, self.width)
bbox = self.xywh2xyxy(bbox)
bbox = self.scale_coords(img1_shape, bbox, img0_shape)
class_name = 'person'
if class_name in self.identical_class_map:
class_name = self.identical_class_map[class_name]
id = self.class_to_id[class_name]
bbox[2] = bbox[2] - bbox[0]
bbox[3] = bbox[3] - bbox[1]
self.prediction_result_list.append({
"image_id": int(image_id),
"category_id": int(id),
"bbox": list(bbox),
"score": scores[i][0]
})
def predict(self):
session = onnxruntime.InferenceSession(self.model_path, providers=self.providers)
outputs = []

61
onnxruntime/python/tools/quantization/E2E_example_model/object_detection/trt/yolov3/preprocessing.py
View file

@ -133,65 +133,6 @@ def yolov3_variant_preprocess_func(images_folder, height, width, start_index=0,
parameter size_limit: number of images to load. Default is 0 which means all images are picked.
return: list of matrices characterizing multiple images
'''
# this function is from yolo3.utils.letterbox_image
# https://github.com/qqwweee/keras-yolo3/blob/master/yolo3/utils.py
def letterbox_image(image, size):
'''resize image with unchanged aspect ratio using padding'''
iw, ih = image.size
w, h = size
scale = min(w/iw, h/ih)
nw = int(iw*scale)
nh = int(ih*scale)
image = image.resize((nw,nh), Image.BICUBIC)
new_image = Image.new('RGB', size, (128,128,128))
new_image.paste(image, ((w-nw)//2, (h-nh)//2))
return new_image
image_names = os.listdir(images_folder)
if start_index >= len(image_names):
return np.asanyarray([]), np.asanyarray([]), np.asanyarray([])
elif size_limit > 0 and len(image_names) >= size_limit:
end_index = start_index + size_limit
if end_index > len(image_names):
end_index = len(image_names)
batch_filenames = [image_names[i] for i in range(start_index, end_index)]
else:
batch_filenames = image_names
unconcatenated_batch_data = []
image_size_list = []
print(batch_filenames)
print("size: %s" % str(len(batch_filenames)))
for image_name in batch_filenames:
image_filepath = images_folder + '/' + image_name
img = Image.open(image_filepath)
model_image_size = (height, width)
boxed_image = letterbox_image(img, tuple(reversed(model_image_size)))
image_data = np.array(boxed_image, dtype='float32')
image_data /= 255.
image_data = np.transpose(image_data, [2, 0, 1])
image_data = np.expand_dims(image_data, 0)
unconcatenated_batch_data.append(image_data)
image_size_list.append((img.size[1], img.size[0])) # img.shape is h, w, c
# image_size_list.append(np.array([img.size[1], img.size[0]], dtype=np.float32).reshape(1, 2))
batch_data = np.concatenate(np.expand_dims(unconcatenated_batch_data, axis=0), axis=0)
return batch_data, batch_filenames, image_size_list
def yolov3_variant_preprocess_func_2(images_folder, height, width, start_index=0, size_limit=0):
'''
Loads a batch of images and preprocess them
parameter images_folder: path to folder storing images
parameter height: image height in pixels
parameter width: image width in pixels
parameter size_limit: number of images to load. Default is 0 which means all images are picked.
return: list of matrices characterizing multiple images
'''
# reference from here:
# https://github.com/jkjung-avt/tensorrt_demos/blob/3fb15c908b155d5edc1bf098c6b8c31886cd8e8d/utils/yolo.py#L60
@ -244,7 +185,7 @@ def yolov3_variant_preprocess_func_2(images_folder, height, width, start_index=0
# This is for special tuned yolov3 model
def yolov3_variant_preprocess_func_3(images_folder, height, width, start_index=0, size_limit=0):
def yolov3_variant_preprocess_func_2(images_folder, height, width, start_index=0, size_limit=0):
def letterbox(img, new_shape=(416, 416), color=(114, 114, 114), auto=True, scaleFill=False, scaleup=True):
# Resize image to a 32-pixel-multiple rectangle https://github.com/ultralytics/yolov3/issues/232
shape = img.shape[:2] # current shape [height, width]

56
onnxruntime/python/tools/quantization/calibrate.py
View file

@ -10,7 +10,7 @@ import os
import numpy as np
import onnx
import onnxruntime
from onnx import helper, TensorProto, ModelProto, shape_inference
from onnx import helper, TensorProto, ModelProto
from onnx import onnx_pb as onnx_proto
from six import string_types
from enum import Enum
@ -52,9 +52,6 @@ class CalibraterBase:
else:
raise ValueError('model should be either model path or onnx.ModelProto.')
# Apply shape inference on the model
self.model = onnx.shape_inference.infer_shapes(self.model)
self.op_types_to_calibrate = op_types_to_calibrate
self.augmented_model_path = augmented_model_path
@ -167,20 +164,34 @@ class MinMaxCalibrater(CalibraterBase):
for tensor in tensors:
# Get tensor's shape
dim = len(value_infos[tensor].type.tensor_type.shape.dim)
shape = (1,) if dim == 1 else list(1 for i in range(dim))
# When doing ReduceMax/ReduceMin, keep dimension if tensor contains dim with value of 0,
# for example:
# dim = [ dim_value: 0 ]
#
# otherwise, don't keep dimension.
#
dim = value_infos[tensor].type.tensor_type.shape.dim
keepdims = 0
shape = ()
for d in dim:
# A dimension can be either an integer value or a symbolic variable.
# Dimension with integer value and value of 0 is what we are looking for to keep dimension.
# Please see the def of TensorShapeProto https://github.com/onnx/onnx/blob/master/onnx/onnx.proto#L630
if d.WhichOneof('value') == 'dim_value' and d.dim_value == 0:
keepdims = 1
shape = (1,) if len(dim) == 1 else list(1 for i in range(len(dim)))
break
# Adding ReduceMin nodes
reduce_min_name = tensor + '_ReduceMin'
reduce_min_node = onnx.helper.make_node('ReduceMin', [tensor], [tensor + '_ReduceMin'], reduce_min_name)
reduce_min_node = onnx.helper.make_node('ReduceMin', [tensor], [tensor + '_ReduceMin'], reduce_min_name, keepdims=keepdims)
added_nodes.append(reduce_min_node)
added_outputs.append(helper.make_tensor_value_info(reduce_min_node.output[0], TensorProto.FLOAT, shape))
# Adding ReduceMax nodes
reduce_max_name = tensor + '_ReduceMax'
reduce_max_node = onnx.helper.make_node('ReduceMax', [tensor], [tensor + '_ReduceMax'], reduce_max_name)
reduce_max_node = onnx.helper.make_node('ReduceMax', [tensor], [tensor + '_ReduceMax'], reduce_max_name, keepdims=keepdims)
added_nodes.append(reduce_max_node)
added_outputs.append(helper.make_tensor_value_info(reduce_max_node.output[0], TensorProto.FLOAT, shape))
@ -200,6 +211,23 @@ class MinMaxCalibrater(CalibraterBase):
break
self.intermediate_outputs.append(self.infer_session.run(None, inputs))
if len(self.intermediate_outputs) == 0:
raise ValueError("No data is collected.")
self.compute_range()
self.clear_collected_data()
def merge_range(self, old_range, new_range):
if not old_range:
return new_range
for key, value in old_range.items():
min_value = min(value[0], new_range[key][0])
max_value = max(value[1], new_range[key][1])
new_range[key] = (min_value, max_value)
return new_range
def compute_range(self):
'''
Compute the min-max range of tensor
@ -207,7 +235,7 @@ class MinMaxCalibrater(CalibraterBase):
'''
if len(self.intermediate_outputs) == 0:
raise ValueError("No data is collected.")
return self.calibrate_tensors_range
output_names = [self.infer_session.get_outputs()[i].name for i in range(len(self.intermediate_outputs[0]))]
output_dicts_list = [
@ -239,7 +267,11 @@ class MinMaxCalibrater(CalibraterBase):
pairs.append(tuple([min_value, max_value]))
self.calibrate_tensors_range = dict(zip(calibrate_tensor_names, pairs))
new_calibrate_tensors_range = dict(zip(calibrate_tensor_names, pairs))
if self.calibrate_tensors_range:
self.calibrate_tensors_range = self.merge_range(self.calibrate_tensors_range, new_calibrate_tensors_range)
else:
self.calibrate_tensors_range = new_calibrate_tensors_range
return self.calibrate_tensors_range
@ -311,6 +343,8 @@ class EntropyCalibrater(CalibraterBase):
self.collector = HistogramCollector()
self.collector.collect(clean_merged_dict)
self.clear_collected_data()
def compute_range(self):
'''
Compute the min-max range of tensor

2
onnxruntime/python/tools/quantization/quant_utils.py
View file

@ -296,11 +296,11 @@ def generate_identified_filename(filename: Path, identifier: str) -> Path:
'''
return filename.parent.joinpath(filename.stem + identifier).with_suffix(filename.suffix)
def write_calibration_table(calibration_cache):
'''
Helper function to write calibration table to files.
'''
import json
import flatbuffers
import onnxruntime.quantization.CalTableFlatBuffers.TrtTable as TrtTable

48
onnxruntime/test/python/quantization/test_calibration.py
View file

@ -258,6 +258,54 @@ class TestCalibrate(unittest.TestCase):
for output_name in output_min_max_dict.keys():
self.assertEqual(output_min_max_dict[output_name], tensors_range[output_name])
def test_augment_graph_with_zero_value_dimension(self):
'''TEST_CONFIG_5'''
# Conv
# |
# Conv
# |
# Resize
G = helper.make_tensor_value_info('G', TensorProto.FLOAT, [1, 1, 5, 5])
H = helper.make_tensor_value_info('H', TensorProto.FLOAT, [1, 1, 3, 3])
J = helper.make_tensor_value_info('J', TensorProto.FLOAT, [1, 1, 3, 3])
M = helper.make_tensor_value_info('M', TensorProto.FLOAT, [0])
N = helper.make_tensor_value_info('N', TensorProto.FLOAT, [0])
O = helper.make_tensor_value_info('O', TensorProto.FLOAT, [1,1,5,5])
# O = helper.make_tensor_value_info('O', TensorProto.FLOAT, None)
conv_node_1 = onnx.helper.make_node('Conv', ['G', 'H'], ['I'],
name='Conv1',
kernel_shape=[3, 3],
pads=[1, 1, 1, 1])
conv_node_2 = onnx.helper.make_node('Conv', ['I', 'J'], ['K'],
name='Conv2',
kernel_shape=[3, 3],
pads=[1, 1, 1, 1])
resize_node_1 = onnx.helper.make_node('Resize', ['K', 'M', 'N'], ['O'],
name='Reize1')
graph = helper.make_graph([conv_node_1, conv_node_2, resize_node_1], 'test_graph_5', [G, H, J, M, N], [O])
model = helper.make_model(graph, opset_imports=[helper.make_opsetid("", 13)])
test_model_path = './test_model_5.onnx'
onnx.save(model, test_model_path)
augmented_model_path = './augmented_test_model_5.onnx'
calibrater = MinMaxCalibrater(test_model_path, [], augmented_model_path)
augmented_model = calibrater.get_augment_model()
augmented_model_node_names = [node.name for node in augmented_model.graph.node]
augmented_model_outputs = [output.name for output in augmented_model.graph.output]
added_node_names = ['I_ReduceMin', 'I_ReduceMax', 'K_ReduceMin', 'K_ReduceMax', 'O_ReduceMin', 'O_ReduceMax']
added_outputs = ['I_ReduceMin', 'I_ReduceMax', 'K_ReduceMin', 'K_ReduceMax', 'O_ReduceMin', 'O_ReduceMax']
# Original 3 nodes + added ReduceMin/Max nodes * 8
self.assertEqual(len(augmented_model_node_names), 19)
# Original 1 graph output + added outputs * 8
self.assertEqual(len(augmented_model_outputs), 17)
for name in added_node_names:
self.assertTrue(name in augmented_model_node_names)
for output in added_outputs:
self.assertTrue(output in augmented_model_outputs)
if __name__ == '__main__':
unittest.main()

1
requirements.txt
View file

@ -1,2 +1,3 @@
numpy >= 1.16.6
protobuf
flatbuffers

1
setup.py
View file

@ -227,6 +227,7 @@ packages = [
'onnxruntime.tools',
'onnxruntime.quantization',
'onnxruntime.quantization.operators',
'onnxruntime.quantization.CalTableFlatBuffers',
'onnxruntime.transformers',
'onnxruntime.transformers.longformer',
]