Add Bert Optimization Notebooks (#3204)

* Add notebooks for GPU and CPU inference of PyTorch BERT SQuAD model
* update bert_optimization.py: Do not add duplicated logger handler
* Add machineinfo.py to show machine configuration for notebook.
* Update bert performance test tool:
(1) Set OpenMP environment variable before importing onnxruntime.
(2) Use sub-process for each test
(3) Allow test multiple batch_size
(4) Add latency percentile
(5) Add warmup

onnxruntime/python/tools/bert/BertOnnxModel.py

@@ -941,7 +941,7 @@ class BertOnnxModel(OnnxModel):
                 normalize_node = onnx.helper.make_node('LayerNormalization',
                     inputs=[node.input[0], weight_input, bias_input],
                     outputs=[last_add_node.output[0]])
-                normalize_node.attribute.extend([onnx.helper.make_attribute("epsilon", add_weight)])
+                normalize_node.attribute.extend([onnx.helper.make_attribute("epsilon", float(add_weight))])
                 layernorm_nodes.extend([normalize_node])
 
         self.remove_nodes(nodes_to_remove)

onnxruntime/python/tools/bert/MachineInfo.py

@@ -0,0 +1,173 @@
+#-------------------------------------------------------------------------
+# Copyright (c) Microsoft Corporation.  All rights reserved.
+# Licensed under the MIT License.
+#--------------------------------------------------------------------------
+
+# It is used to dump machine information for Notebooks
+
+import argparse
+import logging
+from typing import List, Dict, Union, Tuple
+import cpuinfo
+import psutil
+import json
+import sys
+import platform
+from os import environ
+from py3nvml.py3nvml import nvmlInit, nvmlSystemGetDriverVersion, nvmlDeviceGetCount, nvmlDeviceGetHandleByIndex, \
+                            nvmlDeviceGetMemoryInfo, nvmlDeviceGetName, nvmlShutdown, NVMLError
+
+
+class MachineInfo():
+    """ Class encapsulating Machine Info logic. """
+    def __init__(self, silent=False, logger=None):
+        self.silent = silent
+
+        if logger is None:
+            logging.basicConfig(
+                format="%(asctime)s - %(name)s - %(levelname)s: %(message)s",
+                level=logging.INFO)
+            self.logger = logging.getLogger(__name__)
+        else:
+            self.logger = logger
+
+        self.machine_info = None
+        try:
+            self.machine_info = self.get_machine_info()
+        except Exception:
+            self.logger.exception("Exception in getting machine info.")
+            self.machine_info = None
+
+    def get_machine_info(self):
+        """Get machine info in metric format"""
+        gpu_info = self.get_gpu_info_by_nvml()
+
+        machine_info = {
+            "gpu": gpu_info,
+            "cpu": self.get_cpu_info(),
+            "memory": self.get_memory_info(),
+            "python": cpuinfo.get_cpu_info()["python_version"],  #sys.version,
+            "os": platform.platform(),
+            "onnxruntime": self.get_onnxruntime_info(),
+            "pytorch": self.get_pytorch_info(),
+            "tensorflow": self.get_tensorflow_info()
+        }
+        return machine_info
+
+    def get_memory_info(self) -> Dict:
+        """Get memory info"""
+        mem = psutil.virtual_memory()
+        return {"total": mem.total, "available": mem.available}
+
+    def get_cpu_info(self) -> Dict:
+        """Get CPU info"""
+        cpu_info = cpuinfo.get_cpu_info()
+        return {
+            "brand": cpu_info["brand"],
+            "cores": psutil.cpu_count(logical=False),
+            "logical_cores": psutil.cpu_count(logical=True),
+            "hz": cpu_info["hz_actual"],
+            "l2_cache": cpu_info["l2_cache_size"],
+            "l3_cache": cpu_info["l3_cache_size"],
+            "processor": platform.uname().processor
+        }
+
+    def get_gpu_info_by_nvml(self) -> Dict:
+        """Get GPU info using nvml"""
+        gpu_info_list = []
+        driver_version = None
+        try:
+            nvmlInit()
+            driver_version = nvmlSystemGetDriverVersion()
+            deviceCount = nvmlDeviceGetCount()
+            for i in range(deviceCount):
+                handle = nvmlDeviceGetHandleByIndex(i)
+                info = nvmlDeviceGetMemoryInfo(handle)
+                gpu_info = {}
+                gpu_info["memory_total"] = info.total
+                gpu_info["memory_available"] = info.free
+                gpu_info["name"] = nvmlDeviceGetName(handle)
+                gpu_info_list.append(gpu_info)
+            nvmlShutdown()
+        except NVMLError as error:
+            if not self.silent:
+                self.logger.error(
+                    "Error fetching GPU information using nvml: %s", error)
+            return None
+
+        result = {"driver_version": driver_version, "devices": gpu_info_list}
+
+        if 'CUDA_VISIBLE_DEVICES' in environ:
+            result["cuda_visible"] = environ['CUDA_VISIBLE_DEVICES']
+        return result
+
+    def get_onnxruntime_info(self) -> Dict:
+        try:
+            import onnxruntime
+            return {
+                "version":
+                onnxruntime.__version__,
+                "support_gpu":
+                'CUDAExecutionProvider' in
+                onnxruntime.get_available_providers()
+            }
+        except ImportError as error:
+            if not self.silent:
+                self.logger.exception(error)
+            return None
+        except Exception as exception:
+            if not self.silent:
+                self.logger.exception(exception, False)
+            return None
+
+    def get_pytorch_info(self) -> Dict:
+        try:
+            import torch
+            return {
+                "version": torch.__version__,
+                "support_gpu": torch.cuda.is_available()
+            }
+        except ImportError as error:
+            if not self.silent:
+                self.logger.exception(error)
+            return None
+        except Exception as exception:
+            if not self.silent:
+                self.logger.exception(exception, False)
+            return None
+
+    def get_tensorflow_info(self) -> Dict:
+        try:
+            import tensorflow as tf
+            return {
+                "version": tf.version.VERSION,
+                "git_version": tf.version.GIT_VERSION,
+                "support_gpu": tf.test.is_built_with_cuda()
+            }
+        except ImportError as error:
+            if not self.silent:
+                self.logger.exception(error)
+            return None
+        except ModuleNotFoundError as error:
+            if not self.silent:
+                self.logger.exception(error)
+            return None
+
+
+def parse_arguments():
+    parser = argparse.ArgumentParser()
+
+    parser.add_argument('--silent',
+                        required=False,
+                        action='store_true',
+                        help="Do not print error message")
+    parser.set_defaults(silent=False)
+
+    args = parser.parse_args()
+    return args
+
+
+if __name__ == '__main__':
+    args = parse_arguments()
+    machine = MachineInfo(args.silent)
+    print(json.dumps(machine.machine_info, indent=2))

onnxruntime/python/tools/bert/README.md

@@ -26,7 +26,7 @@ def export_onnx(args, model, output_path):
                       do_constant_folding=True,  # whether to execute constant folding for optimization
                       input_names = ["input_ids", "input_mask", "segment_ids"],
                       output_names = ["output"],
-                      dynamic_axes={'input_ids' : {0 : 'batch_size'},    # variable lenght axes
+                      dynamic_axes={'input_ids' : {0 : 'batch_size'},    # variable length axes
                                     'input_mask' : {0 : 'batch_size'},
                                     'segment_ids' : {0 : 'batch_size'},
                                     'output' : {0 : 'batch_size'}})
@@ -80,7 +80,8 @@ Most optimizations require exact match of a subgraph. That means this tool could
 Here is list of models that have been tested using this tool:
 - **BertForSequenceClassification** as in [transformers example](https://github.com/huggingface/transformers/blob/master/examples/run_glue.py) exported by PyTorch 1.2-1.4 using opset version 10 or 11.
 - **BertForQuestionAnswering** as in [transformers example](https://github.com/huggingface/transformers/blob/master/examples/run_squad.py) exported by PyTorch 1.2-1.4 using opset version 10 or 11.
-- **TFBertForSequenceClassification** as in [transformers example](https://github.com/huggingface/transformers/blob/master/examples/run_tf_glue.py) exported by keras2onnx 1.6.0.
+- **TFBertForSequenceClassification** as in [transformers example](https://github.com/huggingface/transformers/blob/master/examples/run_tf_glue.py) exported by keras2onnx installed from its master source.
+- **TFBertForQuestionAnswering** as in [transformers](https://github.com/huggingface/transformers/) exported by keras2onnx installed from its master source.
 
 If your model is not in the list, the optimized model might not work. You are welcome to update the scripts to support new models.
 

onnxruntime/python/tools/bert/bert_model_optimization.py

@@ -151,7 +151,11 @@ def main():
         log_handler.setFormatter(logging.Formatter('%(filename)20s: %(message)s'))
         logging_level = logging.INFO
     log_handler.setLevel(logging_level)
-    logger.addHandler(log_handler)
+
+    # Avoid duplicated handlers when runing this script in multiple cells of Jupyter Notebook.
+    if not logger.hasHandlers():
+        logger.addHandler(log_handler)
+
     logger.setLevel(logging_level)
 
     bert_model = optimize_model(args.input, args.model_type, args.gpu_only, args.num_heads, args.hidden_size, args.sequence_length, args.input_int32, args.float16)

onnxruntime/python/tools/bert/bert_perf_test.py

@@ -5,41 +5,59 @@
 
 # This tool measures the inference performance of onnxruntime or onnxruntime-gpu python package on Bert model.
 
+# The input model shall have exactly three inputs. The model is either fully optimized (with EmbedLayerNormalization node),
+# or with reasonable input names (one input name has 'mask' substring, another has 'token' or 'segment' substring).
+# See get_bert_inputs function in bert_test_data.py for more information.
+
+# Example command to run test on batch_size 1 and 2 for a model on GPU:
+#   python bert_perf_test.py --model bert.onnx --batch_size 1 2 --sequence_length 128 --use_gpu --samples 1000 --test_times 1
+
 import sys
 import argparse
 import os
-import onnxruntime
 from pathlib import Path
 import timeit
 import statistics
 import psutil
 import csv
 import numpy as np
+import random
 from datetime import datetime
+import multiprocessing
 from bert_test_data import get_bert_inputs, generate_test_data
 
-def create_session(model_path, use_gpu, use_openmp, graph_optimization_level, num_threads, wait_policy):
-    execution_providers = ['CPUExecutionProvider'] if not use_gpu else ['CUDAExecutionProvider', 'CPUExecutionProvider']
-    sess_options = onnxruntime.SessionOptions()
-    sess_options.execution_mode = onnxruntime.ExecutionMode.ORT_SEQUENTIAL
-    sess_options.graph_optimization_level = graph_optimization_level
-    if not use_openmp:
-        sess_options.intra_op_num_threads=num_threads
-        if "OMP_NUM_THREADS" in os.environ:
-            del os.environ["OMP_NUM_THREADS"]
-        if "OMP_WAIT_POLICY" in os.environ:
-            del os.environ["OMP_WAIT_POLICY"]
-    else:
-        sess_options.intra_op_num_threads=1
-        os.environ["OMP_NUM_THREADS"] = str(num_threads)
-        os.environ["OMP_WAIT_POLICY"] = wait_policy
+def create_session(model_path, use_gpu, intra_op_num_threads, graph_optimization_level=None):
+    # Import onnxruntime shall be after OpenMP environment variable setting.
+    # So we put the import in function to delay importing instead of top of this script.
+    import onnxruntime
+
+    if use_gpu and ('CUDAExecutionProvider' not in onnxruntime.get_available_providers()):
+        print("Warning: Please install onnxruntime-gpu package instead of onnxruntime, and use a machine with GPU for testing gpu performance.")
+    elif (not use_gpu) and ('CUDAExecutionProvider' in onnxruntime.get_available_providers()):
+        print("Warning: Please install onnxruntime package instead of onnxruntime-gpu to get best cpu performance.")
+
+    if intra_op_num_threads is None and graph_optimization_level is None:
+        session = onnxruntime.InferenceSession(model_path)
+    else:
+        execution_providers = ['CPUExecutionProvider'] if not use_gpu else ['CUDAExecutionProvider', 'CPUExecutionProvider']
+        sess_options = onnxruntime.SessionOptions()
+        sess_options.execution_mode = onnxruntime.ExecutionMode.ORT_SEQUENTIAL
+        if graph_optimization_level is None:
+            sess_options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
+        else:
+            sess_options.graph_optimization_level = graph_optimization_level
+        sess_options.intra_op_num_threads = intra_op_num_threads
+        session = onnxruntime.InferenceSession(model_path, sess_options, providers=execution_providers)
 
-    session = onnxruntime.InferenceSession(model_path, sess_options, providers=execution_providers)
     if use_gpu:
         assert 'CUDAExecutionProvider' in session.get_providers()
     return session
 
-def onnxruntime_inference(session, all_inputs, output_names):
+def onnxruntime_inference(session, all_inputs, output_names, warmup=True):
+    if warmup and len(all_inputs) > 0:
+        # Use a random input as warm up.
+        session.run(output_names, random.choice(all_inputs))
+
     results = []
     latency_list = []
     for test_case_id, inputs in enumerate(all_inputs):
@@ -76,68 +94,137 @@ def to_string(model_path, session, test_setting):
     option += ",{}".format(test_setting)
     return option
 
-def run_one_test(latency_results, model_path, all_inputs, batch_size, sequence_length, use_gpu, test_cases, test_times, use_openmp, contiguous, num_threads, wait_policy):
-    test_setting =  "batch_size={},sequence_length={},test_cases={},test_times={},contiguous={},use_gpu={}".format(batch_size,sequence_length,test_cases,test_times,contiguous,use_gpu)
+def setup_openmp_environ(omp_num_threads, omp_wait_policy):
+    if omp_num_threads is None:
+        if "OMP_NUM_THREADS" in os.environ:
+            del os.environ["OMP_NUM_THREADS"]
+    else:
+        os.environ["OMP_NUM_THREADS"] = str(omp_num_threads)
 
-    graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
-    session = create_session(model_path, use_gpu, use_openmp, graph_optimization_level, num_threads, wait_policy)
+    if omp_wait_policy is None:
+        if "OMP_WAIT_POLICY" in os.environ:
+            del os.environ["OMP_WAIT_POLICY"]
+    else:
+        assert omp_wait_policy in ["ACTIVE", "PASSIVE"]
+        os.environ["OMP_WAIT_POLICY"] = omp_wait_policy
+
+def run_one_test(perf_results, model_path, all_inputs, batch_size, sequence_length, use_gpu, test_cases, test_times, contiguous, intra_op_num_threads, omp_num_threads, omp_wait_policy, no_warmup, extra_latency):
+    # Environment variable shall be set before import onnxruntime.
+    setup_openmp_environ(omp_num_threads, omp_wait_policy)
+
+    test_setting = "batch_size={},sequence_length={},test_cases={},test_times={},contiguous={},use_gpu={},warmup={}".format(batch_size, sequence_length, test_cases, test_times, contiguous, use_gpu, not no_warmup)
+
+    session = create_session(model_path, use_gpu, intra_op_num_threads)
     output_names = [output.name for output in session.get_outputs()]
 
     key = to_string(model_path, session, test_setting)
+    if key in perf_results:
+        print("skip duplicated test:", key)
+        return
+
     print("Running test:", key)
 
     all_latency_list = []
     for i in range(test_times):
-        results, latency_list = onnxruntime_inference(session, all_inputs, output_names)
+        results, latency_list = onnxruntime_inference(session, all_inputs, output_names, not no_warmup)
         all_latency_list.extend(latency_list)
 
-    average_latency = statistics.mean(all_latency_list) * 1000
-    print("Average latency is {} ms".format(format(average_latency, '.2f')))
-    latency_results[key] = average_latency
+    # latency in miliseconds
+    latency_ms = np.array(all_latency_list) * 1000 + extra_latency
 
-def run_perf_tests(average_latency, model_path, batch_size, sequence_length, use_gpu, test_cases, test_times, seed, verbose, contiguous, input_ids, segment_ids, input_mask, all_inputs, run_all_settings):
-    if use_gpu or run_all_settings:
-        run_one_test(average_latency, model_path, all_inputs, batch_size, sequence_length, use_gpu, test_cases, test_times, use_openmp=False, contiguous=contiguous, num_threads=psutil.cpu_count(logical=True), wait_policy='ACTIVE')
+    average_latency = statistics.mean(latency_ms)
+    latency_50 = np.percentile(latency_ms, 50)
+    latency_75 = np.percentile(latency_ms, 75)
+    latency_90 = np.percentile(latency_ms, 90)
+    latency_95 = np.percentile(latency_ms, 95)
+    latency_99 = np.percentile(latency_ms, 99)
+    throughput = batch_size * (1000.0 / average_latency)
+
+    perf_results[key] = (average_latency, latency_50, latency_75, latency_90, latency_95, latency_99, throughput)
+
+    print("Average latency = {} ms, Throughput = {} QPS".format(format(average_latency, '.2f'), format(throughput, '.2f')))
+
+def launch_test(perf_results, model_path, all_inputs, batch_size, sequence_length, use_gpu, test_cases, test_times, contiguous, intra_op_num_threads, omp_num_threads, omp_wait_policy, no_warmup, extra_latency):
+    process = multiprocessing.Process(target=run_one_test, args=(perf_results, model_path, all_inputs, batch_size, sequence_length, use_gpu, test_cases, test_times, contiguous, intra_op_num_threads, omp_num_threads, omp_wait_policy, no_warmup, extra_latency))
+    process.start()
+    process.join()
+
+def run_perf_tests(perf_results, model_path, batch_size, sequence_length, use_gpu, test_cases, test_times, contiguous, all_inputs, test_all, no_warmup, extra_latency):
+    cpu_count = psutil.cpu_count(logical=False)
+    logical_cores = psutil.cpu_count(logical=True)
+
+    # Test a setting without any setting as baseline 1.
+    launch_test(perf_results, model_path, all_inputs, batch_size, sequence_length, use_gpu, test_cases, test_times, contiguous, None, None, None, no_warmup, extra_latency)
 
-    # onnxruntime-gpu package is not built with OpenMP, so skip openmp test for gpu.
     if not use_gpu:
-        run_one_test(average_latency, model_path, all_inputs, batch_size, sequence_length, use_gpu, test_cases, test_times, use_openmp=True, contiguous=contiguous, num_threads=psutil.cpu_count(logical=True), wait_policy='ACTIVE')
+        # For CPU: intra_op_num_threads = 1, omp_num_threads=None, omp_wait_policy=None
+        # Another setting without environment variable as baseline 2.
+        launch_test(perf_results, model_path, all_inputs, batch_size, sequence_length, use_gpu, test_cases, test_times, contiguous, 1, None, None, no_warmup, extra_latency)
+    else:
+        # For GPU, we test two more settings by default:
+        # (1) intra_op_num_threads = 1, omp_num_threads=cpu_count, omp_wait_policy=PASSIVE
+        # (2) intra_op_num_threads = logical_cores, omp_num_threads=1, omp_wait_policy=ACTIVE
+        launch_test(perf_results, model_path, all_inputs, batch_size, sequence_length, use_gpu, test_cases, test_times, contiguous, 1, cpu_count, 'PASSIVE', no_warmup, extra_latency)
         
-        if run_all_settings:
-            run_one_test(average_latency, model_path, all_inputs, batch_size, sequence_length, use_gpu, test_cases, test_times, use_openmp=True, contiguous=contiguous, num_threads=psutil.cpu_count(logical=True), wait_policy='PASSIVE')
+        launch_test(perf_results, model_path, all_inputs, batch_size, sequence_length, use_gpu, test_cases, test_times, contiguous, logical_cores, 1, 'ACTIVE', no_warmup, extra_latency)
 
-            if psutil.cpu_count(logical=True) != psutil.cpu_count(logical=False):
-                run_one_test(average_latency, model_path, all_inputs, batch_size, sequence_length, use_gpu, test_cases, test_times, use_openmp=True, contiguous=contiguous, num_threads=psutil.cpu_count(logical=False), wait_policy='ACTIVE')
-                run_one_test(average_latency, model_path, all_inputs, batch_size, sequence_length, use_gpu, test_cases, test_times, use_openmp=True, contiguous=contiguous, num_threads=psutil.cpu_count(logical=False), wait_policy='PASSIVE')
+    # GPU latency is not sensitive to these settings. No need to test many combinations.
+    # Skip remaining settings for GPU without --all flag.
+    if use_gpu and not test_all:
+        return
 
-def run_performance(average_latency, model_path, batch_size, sequence_length, use_gpu, test_cases, test_times, seed, verbose, run_all_settings):
+    candidates = list(set([1, logical_cores, cpu_count]))
+
+    for intra_op_num_threads in candidates:
+        for omp_num_threads in candidates:
+            # skip settings that are very slow
+            if intra_op_num_threads == 1 and omp_num_threads == 1 and logical_cores != 1:
+                continue
+
+            # When logical and physical cores are not the same, there are many combinations.
+            # Remove some settings are not good normally.
+            if logical_cores > cpu_count:
+                if omp_num_threads == logical_cores and intra_op_num_threads != 1:
+                    continue
+                if intra_op_num_threads == logical_cores and omp_num_threads != 1:
+                    continue
+
+            if not test_all:
+                if intra_op_num_threads != 1 and omp_num_threads != 1:
+                    continue
+
+            for omp_wait_policy in ['ACTIVE', 'PASSIVE']:
+                launch_test(perf_results, model_path, all_inputs, batch_size, sequence_length, use_gpu, test_cases, test_times, contiguous, intra_op_num_threads, omp_num_threads, omp_wait_policy, no_warmup, extra_latency)
+
+def run_performance(perf_results, model_path, batch_size, sequence_length, use_gpu, test_cases, test_times, seed, verbose, inclusive, test_all, no_warmup):
     # Try deduce input names from model.
     input_ids, segment_ids, input_mask = get_bert_inputs(model_path)
 
     # Do not generate random mask for performance test.
-    print("generating test data...")
+    print(f"Generating {test_cases} samples for batch_size={batch_size} sequence_length={sequence_length}")
     all_inputs = generate_test_data(batch_size, sequence_length, test_cases, seed, verbose, input_ids, segment_ids, input_mask, random_mask_length=False)
 
     contiguous = False
-    if run_all_settings:
-        run_perf_tests(average_latency, model_path, batch_size, sequence_length, use_gpu, test_cases, test_times, seed, verbose, contiguous, input_ids, segment_ids, input_mask, all_inputs, run_all_settings)
+    run_perf_tests(perf_results, model_path, batch_size, sequence_length, use_gpu, test_cases, test_times, contiguous, all_inputs, test_all, no_warmup, extra_latency=0)
+
+    # only test contiguous array when the --all flag is set.
+    if not test_all:
+        return
 
     # Convert inputs to contiguous array, which could improve inference performance
     all_inputs, contiguous_latency = get_contiguous_inputs(all_inputs)
     print("Extra latency for converting inputs to contiguous: {} ms".format(format(contiguous_latency, '.2f')))
 
     contiguous = True
-    run_perf_tests(average_latency, model_path, batch_size, sequence_length, use_gpu, test_cases, test_times, seed, verbose, contiguous, input_ids, segment_ids, input_mask, all_inputs, run_all_settings)
-
-    return contiguous_latency
+    run_perf_tests(perf_results, model_path, batch_size, sequence_length, use_gpu, test_cases, test_times, contiguous, all_inputs, test_all, no_warmup, extra_latency=contiguous_latency if inclusive else 0)
 
 def parse_arguments():
     parser = argparse.ArgumentParser()
     parser.add_argument('--model', required=True, type=str,
                         help="bert onnx model path")
 
-    parser.add_argument('--batch_size', required=True, type=int,
-                        help="batch size of input")
+    parser.add_argument('--batch_size', required=True, type=int, nargs="+",
+                        help="batch size of input. Allow one or multiple values in the range of [1, 128].")
 
     parser.add_argument('--sequence_length',  required=True, type=int,
                         help="maximum sequence length of input")
@@ -160,9 +247,12 @@ def parse_arguments():
     parser.add_argument('--inclusive', required=False, action='store_true', help="include the latency of converting array to contiguous")
     parser.set_defaults(inclusive=False)
 
-    parser.add_argument('--all', required=False, action='store_true', help="test all settings")
+    parser.add_argument('--all', required=False, action='store_true', help="test all candidate settings")
     parser.set_defaults(all=False)
 
+    parser.add_argument('--no_warmup', required=False, action='store_true', help="do not use one sample for warm-up.")
+    parser.set_defaults(no_warmup=False)
+
     args = parser.parse_args()
     return args
 
@@ -172,40 +262,38 @@ def main():
     if args.test_times == 0:
         args.test_times = max(1, int(1000 / args.samples))
 
-    if args.use_gpu and ('CUDAExecutionProvider' not in onnxruntime.get_available_providers()):
-        print("Please install onnxruntime-gpu package instead of onnxruntime, and use a machine with GPU for testing gpu performance.")
-        return
-    elif (not args.use_gpu) and ('CUDAExecutionProvider' in onnxruntime.get_available_providers()):
-        print("Warning: Please install onnxruntime package instead of onnxruntime-gpu to get best cpu performance.")
+    manager = multiprocessing.Manager()
+    perf_results = manager.dict()
 
-    average_latency = {}
-    contiguous_latency = run_performance(average_latency, args.model, args.batch_size, args.sequence_length, args.use_gpu, args.samples, args.test_times, args.seed, args.verbose, args.all)
+    batch_size_set = set(args.batch_size)
+    if not min(batch_size_set)>=1 and max(batch_size_set) <= 128:
+        raise Exception("batch_size not in range [1, 128]")
 
-    if average_latency is None:
-        return
+    for batch_size in batch_size_set:
+        run_performance(perf_results, args.model, batch_size, args.sequence_length, args.use_gpu, args.samples, args.test_times, args.seed, args.verbose, args.inclusive, args.all, args.no_warmup)
 
-    summary_file = os.path.join(Path(args.model).parent, "perf_results_{}_B{}_S{}_{}.txt".format('GPU' if args.use_gpu else 'CPU', args.batch_size, args.sequence_length, datetime.now().strftime("%Y%m%d-%H%M%S")))
+    # Sort the results so that the first one has smallest latency.
+    sorted_results = sorted(perf_results.items(), reverse=False, key=lambda x: x[1])
+ 
+    summary_file = os.path.join(Path(args.model).parent, "perf_results_{}_B{}_S{}_{}.txt".format('GPU' if args.use_gpu else 'CPU', "-".join([str(x) for x in sorted(list(batch_size_set))]), args.sequence_length, datetime.now().strftime("%Y%m%d-%H%M%S")))
     with open(summary_file, 'w+', newline='') as tsv_file:
         tsv_writer = csv.writer(tsv_file, delimiter='\t', lineterminator='\n')
         headers = None
-        for key, latency in average_latency.items():
+        for (key, perf_result) in sorted_results:
             params = key.split(',')
             if headers is None:
-                headers = ["Latency(ms)", "Throughput(QPS)"]
+                headers = ["Latency(ms)", "Latency_P50", "Latency_P75", "Latency_P90", "Latency_P95", "Latency_P99", "Throughput(QPS)"]
                 headers.extend([x.split('=')[0] for x in params])
                 tsv_writer.writerow(headers)
 
-            # include the extra latency of array conversion if required.
-            if args.inclusive and 'contiguous=True' in params:
-                latency += contiguous_latency
-
-            throughput = args.batch_size * (1000 / latency)
-            values = [format(latency, '.2f'), format(throughput, '.2f')]
-
+            values = [format(x, '.2f') for x in perf_result]
             values.extend([x.split('=')[1] for x in params])
             tsv_writer.writerow(values)
 
     print("Test summary is saved to", summary_file)
 
 if __name__ == "__main__":
+    # work around for AnaConda Jupyter. See https://stackoverflow.com/questions/45720153/python-multiprocessing-error-attributeerror-module-main-has-no-attribute
+    __spec__ = None
+    
     main()

onnxruntime/python/tools/bert/compare_bert_results.py

@@ -9,7 +9,6 @@ import sys
 import argparse
 import numpy as np
 import os
-import onnxruntime
 import random
 from pathlib import Path
 import statistics
@@ -21,12 +20,22 @@ import timeit
 from datetime import datetime
 from onnx import ModelProto, TensorProto, numpy_helper
 from OnnxModel import OnnxModel
-from bert_model_optimization import optimize_by_onnxruntime
 from bert_test_data import get_bert_inputs, generate_test_data, output_test_data
-from bert_perf_test import create_session, onnxruntime_inference
+from bert_perf_test import create_session, onnxruntime_inference, setup_openmp_environ
+
+def run_model(model_path, all_inputs, use_gpu, use_openmp, disable_optimization):
+    # Import onnxruntime shall be after OpenMP environment variable setting.
+    # So we put import here to delay importing.
+    import onnxruntime
+
+    graph_optimization_level = None
+    if disable_optimization:
+        graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_DISABLE_ALL
+
+    intra_op_num_threads =  1 if use_openmp else psutil.cpu_count(logical=False)
+
+    session = create_session(model_path, use_gpu, intra_op_num_threads, graph_optimization_level)
 
-def run_model(baseline_model, all_inputs, use_gpu, use_openmp, graph_optimization_level):
-    session = create_session(baseline_model, use_gpu, use_openmp, graph_optimization_level, num_threads=psutil.cpu_count(logical=True), wait_policy='ACTIVE')
     output_names = [output.name for output in session.get_outputs()]
     results, latency_list = onnxruntime_inference(session, all_inputs, output_names)
     return results, latency_list, output_names
@@ -39,20 +48,22 @@ def compare(baseline_results, treatment_results, verbose, rtol=1e-3, atol=1e-4):
     for test_case_id, results in enumerate(baseline_results):
         case_passed = True
         for i in range(len(results)):
-            treatment_first_output = treatment_results[test_case_id][0].tolist()
-            rel_diff = np.amax(np.abs((treatment_results[test_case_id][0] - results[0]) / results[0]))
-            abs_diff = np.amax(np.abs(treatment_results[test_case_id][0] - results[0]))
+            treatment_output = treatment_results[test_case_id][i]
+            rel_diff = np.amax(np.abs((treatment_output - results[i]) / results[i]))
+            abs_diff = np.amax(np.abs(treatment_output - results[i]))
             max_rel_diff = max(max_rel_diff, rel_diff)
             max_abs_diff = max(max_abs_diff, abs_diff)
-            if verbose:
-                print("case {} output {}".format(test_case_id, i))
-                print("baseline={}\ntreatment={}".format(results[0].tolist(), treatment_first_output))
-                print("rel_diff={} abs_diff={}".format(rel_diff, abs_diff))
-            if not np.allclose(results[0].tolist(), treatment_first_output, rtol=rtol, atol=atol):
+            if not np.allclose(results[i].tolist(), treatment_output.tolist(), rtol=rtol, atol=atol):
                 if case_passed:
                     case_passed = False
                     diff_count += 1
 
+                    if verbose:
+                        print("case {} output {}".format(test_case_id, i))
+                        print("baseline={}\ntreatment={}".format(results[i].tolist(), treatment_output))
+                        print("rel_diff={} abs_diff={}".format(rel_diff, abs_diff))
+            
+
     if diff_count == 0:
         print("100% passed for {} random inputs given thresholds (rtol={}, atol={}).".format(len(baseline_results), rtol, atol))
     else:
@@ -69,15 +80,21 @@ def run_test(baseline_model, optimized_model, output_dir, batch_size, sequence_l
     # Use random mask length for accuracy test. It might introduce slight inflation in latency reported in this script.
     all_inputs = generate_test_data(batch_size, sequence_length, test_cases, seed, verbose, input_ids, segment_ids, input_mask, random_mask_length=True)
 
-    baseline_results, baseline_latency, output_names = run_model(baseline_model, all_inputs, use_gpu, use_openmp, onnxruntime.GraphOptimizationLevel.ORT_DISABLE_ALL)
+    # OpenMP environment variables must be set before the very first "import onnxruntime"
+    if use_openmp:
+        setup_openmp_environ(omp_num_threads=psutil.cpu_count(logical=False), omp_wait_policy='ACTIVE')
+    else:
+        setup_openmp_environ(omp_num_threads=1, omp_wait_policy='ACTIVE')
+
+    baseline_results, baseline_latency, output_names = run_model(baseline_model, all_inputs, use_gpu, use_openmp, disable_optimization=True)
     if verbose:
-        print("baseline average latency: {} ms".format(statistics.mean(baseline_latency) * 1000))
+        print("baseline average latency (all optimizations disabled): {} ms".format(statistics.mean(baseline_latency) * 1000))
 
     if output_dir is not None:
         for i, inputs in enumerate(all_inputs):
             output_test_data(output_dir, i, inputs)
 
-    treatment_results, treatment_latency, treatment_output_names = run_model(optimized_model, all_inputs, use_gpu, use_openmp, onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL)
+    treatment_results, treatment_latency, treatment_output_names = run_model(optimized_model, all_inputs, use_gpu, use_openmp, disable_optimization=False)
     if verbose:
         print("treatment average latency: {} ms".format(statistics.mean(treatment_latency) * 1000))
 
@@ -87,10 +104,10 @@ def run_test(baseline_model, optimized_model, output_dir, batch_size, sequence_l
 def parse_arguments():
     parser = argparse.ArgumentParser()
     parser.add_argument('--baseline_model', required=True, type=str,
-                        help="baseline onnx model path")
+                        help="baseline onnx model path.")
 
-    parser.add_argument('--optimized_model', required=False, type=str, default=None,
-                        help="optimized model for the baseline model. They shall have same inputs. If it is None, an optimized model will be generated using OnnxRuntime.")
+    parser.add_argument('--optimized_model', required=True, type=str, default=None,
+                        help="path of the optimized model. It shall have same inputs as the baseline model.")
 
     parser.add_argument('--output_dir', required=False, type=str, default=None,
                         help="output test data path. If not specified, test data will not be saved.")
@@ -116,8 +133,8 @@ def parse_arguments():
     parser.add_argument('--use_gpu',  required=False, action='store_true', help="use GPU")
     parser.set_defaults(use_gpu=False)
 
-    parser.add_argument('--no_openmp', required=False, action='store_true', help="do not use openmp")
-    parser.set_defaults(no_openmp=False)
+    parser.add_argument('--openmp', required=False, action='store_true', help="use openmp")
+    parser.set_defaults(openmp=False)
 
     parser.add_argument('--verbose', required=False, action='store_true', help="print verbose information")
     parser.set_defaults(verbose=False)
@@ -128,11 +145,6 @@ def parse_arguments():
 def main():
     args = parse_arguments()
 
-    optimized_model = optimize_by_onnxruntime(args.baseline_model, args.use_gpu) if (args.optimized_model is None) else args.optimized_model
-
-    if args.use_gpu and ('CUDAExecutionProvider' not in onnxruntime.get_available_providers()):
-        print("Please install onnxruntime-gpu package instead of onnxruntime, and use a machine with GPU for testing gpu.")
-
     if args.output_dir is not None:
         # create the output directory if not existed
         path = Path(args.output_dir)
@@ -140,14 +152,14 @@ def main():
 
     run_test(
         args.baseline_model,
-        optimized_model,
+        args.optimized_model,
         args.output_dir,
         args.batch_size,
         args.sequence_length,
         args.use_gpu,
         args.samples,
         args.seed,
-        not args.no_openmp,
+        args.openmp,
         args.verbose,
         args.rtol,
         args.atol)