diff --git a/onnxruntime/python/tools/bert/BertOnnxModel.py b/onnxruntime/python/tools/bert/BertOnnxModel.py index 516c55c089..8d7cb450f2 100644 --- a/onnxruntime/python/tools/bert/BertOnnxModel.py +++ b/onnxruntime/python/tools/bert/BertOnnxModel.py @@ -941,7 +941,7 @@ class BertOnnxModel(OnnxModel): normalize_node = onnx.helper.make_node('LayerNormalization', inputs=[node.input[0], weight_input, bias_input], outputs=[last_add_node.output[0]]) - normalize_node.attribute.extend([onnx.helper.make_attribute("epsilon", add_weight)]) + normalize_node.attribute.extend([onnx.helper.make_attribute("epsilon", float(add_weight))]) layernorm_nodes.extend([normalize_node]) self.remove_nodes(nodes_to_remove) diff --git a/onnxruntime/python/tools/bert/MachineInfo.py b/onnxruntime/python/tools/bert/MachineInfo.py new file mode 100644 index 0000000000..81bc1e6835 --- /dev/null +++ b/onnxruntime/python/tools/bert/MachineInfo.py @@ -0,0 +1,173 @@ +#------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. +#-------------------------------------------------------------------------- + +# It is used to dump machine information for Notebooks + +import argparse +import logging +from typing import List, Dict, Union, Tuple +import cpuinfo +import psutil +import json +import sys +import platform +from os import environ +from py3nvml.py3nvml import nvmlInit, nvmlSystemGetDriverVersion, nvmlDeviceGetCount, nvmlDeviceGetHandleByIndex, \ + nvmlDeviceGetMemoryInfo, nvmlDeviceGetName, nvmlShutdown, NVMLError + + +class MachineInfo(): + """ Class encapsulating Machine Info logic. """ + def __init__(self, silent=False, logger=None): + self.silent = silent + + if logger is None: + logging.basicConfig( + format="%(asctime)s - %(name)s - %(levelname)s: %(message)s", + level=logging.INFO) + self.logger = logging.getLogger(__name__) + else: + self.logger = logger + + self.machine_info = None + try: + self.machine_info = self.get_machine_info() + except Exception: + self.logger.exception("Exception in getting machine info.") + self.machine_info = None + + def get_machine_info(self): + """Get machine info in metric format""" + gpu_info = self.get_gpu_info_by_nvml() + + machine_info = { + "gpu": gpu_info, + "cpu": self.get_cpu_info(), + "memory": self.get_memory_info(), + "python": cpuinfo.get_cpu_info()["python_version"], #sys.version, + "os": platform.platform(), + "onnxruntime": self.get_onnxruntime_info(), + "pytorch": self.get_pytorch_info(), + "tensorflow": self.get_tensorflow_info() + } + return machine_info + + def get_memory_info(self) -> Dict: + """Get memory info""" + mem = psutil.virtual_memory() + return {"total": mem.total, "available": mem.available} + + def get_cpu_info(self) -> Dict: + """Get CPU info""" + cpu_info = cpuinfo.get_cpu_info() + return { + "brand": cpu_info["brand"], + "cores": psutil.cpu_count(logical=False), + "logical_cores": psutil.cpu_count(logical=True), + "hz": cpu_info["hz_actual"], + "l2_cache": cpu_info["l2_cache_size"], + "l3_cache": cpu_info["l3_cache_size"], + "processor": platform.uname().processor + } + + def get_gpu_info_by_nvml(self) -> Dict: + """Get GPU info using nvml""" + gpu_info_list = [] + driver_version = None + try: + nvmlInit() + driver_version = nvmlSystemGetDriverVersion() + deviceCount = nvmlDeviceGetCount() + for i in range(deviceCount): + handle = nvmlDeviceGetHandleByIndex(i) + info = nvmlDeviceGetMemoryInfo(handle) + gpu_info = {} + gpu_info["memory_total"] = info.total + gpu_info["memory_available"] = info.free + gpu_info["name"] = nvmlDeviceGetName(handle) + gpu_info_list.append(gpu_info) + nvmlShutdown() + except NVMLError as error: + if not self.silent: + self.logger.error( + "Error fetching GPU information using nvml: %s", error) + return None + + result = {"driver_version": driver_version, "devices": gpu_info_list} + + if 'CUDA_VISIBLE_DEVICES' in environ: + result["cuda_visible"] = environ['CUDA_VISIBLE_DEVICES'] + return result + + def get_onnxruntime_info(self) -> Dict: + try: + import onnxruntime + return { + "version": + onnxruntime.__version__, + "support_gpu": + 'CUDAExecutionProvider' in + onnxruntime.get_available_providers() + } + except ImportError as error: + if not self.silent: + self.logger.exception(error) + return None + except Exception as exception: + if not self.silent: + self.logger.exception(exception, False) + return None + + def get_pytorch_info(self) -> Dict: + try: + import torch + return { + "version": torch.__version__, + "support_gpu": torch.cuda.is_available() + } + except ImportError as error: + if not self.silent: + self.logger.exception(error) + return None + except Exception as exception: + if not self.silent: + self.logger.exception(exception, False) + return None + + def get_tensorflow_info(self) -> Dict: + try: + import tensorflow as tf + return { + "version": tf.version.VERSION, + "git_version": tf.version.GIT_VERSION, + "support_gpu": tf.test.is_built_with_cuda() + } + except ImportError as error: + if not self.silent: + self.logger.exception(error) + return None + except ModuleNotFoundError as error: + if not self.silent: + self.logger.exception(error) + return None + + +def parse_arguments(): + parser = argparse.ArgumentParser() + + parser.add_argument('--silent', + required=False, + action='store_true', + help="Do not print error message") + parser.set_defaults(silent=False) + + args = parser.parse_args() + return args + + +if __name__ == '__main__': + args = parse_arguments() + machine = MachineInfo(args.silent) + print(json.dumps(machine.machine_info, indent=2)) diff --git a/onnxruntime/python/tools/bert/README.md b/onnxruntime/python/tools/bert/README.md index 56bad9081e..ab6859aa7a 100644 --- a/onnxruntime/python/tools/bert/README.md +++ b/onnxruntime/python/tools/bert/README.md @@ -26,7 +26,7 @@ def export_onnx(args, model, output_path): do_constant_folding=True, # whether to execute constant folding for optimization input_names = ["input_ids", "input_mask", "segment_ids"], output_names = ["output"], - dynamic_axes={'input_ids' : {0 : 'batch_size'}, # variable lenght axes + dynamic_axes={'input_ids' : {0 : 'batch_size'}, # variable length axes 'input_mask' : {0 : 'batch_size'}, 'segment_ids' : {0 : 'batch_size'}, 'output' : {0 : 'batch_size'}}) @@ -80,7 +80,8 @@ Most optimizations require exact match of a subgraph. That means this tool could Here is list of models that have been tested using this tool: - **BertForSequenceClassification** as in [transformers example](https://github.com/huggingface/transformers/blob/master/examples/run_glue.py) exported by PyTorch 1.2-1.4 using opset version 10 or 11. - **BertForQuestionAnswering** as in [transformers example](https://github.com/huggingface/transformers/blob/master/examples/run_squad.py) exported by PyTorch 1.2-1.4 using opset version 10 or 11. -- **TFBertForSequenceClassification** as in [transformers example](https://github.com/huggingface/transformers/blob/master/examples/run_tf_glue.py) exported by keras2onnx 1.6.0. +- **TFBertForSequenceClassification** as in [transformers example](https://github.com/huggingface/transformers/blob/master/examples/run_tf_glue.py) exported by keras2onnx installed from its master source. +- **TFBertForQuestionAnswering** as in [transformers](https://github.com/huggingface/transformers/) exported by keras2onnx installed from its master source. If your model is not in the list, the optimized model might not work. You are welcome to update the scripts to support new models. diff --git a/onnxruntime/python/tools/bert/bert_model_optimization.py b/onnxruntime/python/tools/bert/bert_model_optimization.py index b2db4776d3..4800d42f4b 100644 --- a/onnxruntime/python/tools/bert/bert_model_optimization.py +++ b/onnxruntime/python/tools/bert/bert_model_optimization.py @@ -151,7 +151,11 @@ def main(): log_handler.setFormatter(logging.Formatter('%(filename)20s: %(message)s')) logging_level = logging.INFO log_handler.setLevel(logging_level) - logger.addHandler(log_handler) + + # Avoid duplicated handlers when runing this script in multiple cells of Jupyter Notebook. + if not logger.hasHandlers(): + logger.addHandler(log_handler) + logger.setLevel(logging_level) bert_model = optimize_model(args.input, args.model_type, args.gpu_only, args.num_heads, args.hidden_size, args.sequence_length, args.input_int32, args.float16) diff --git a/onnxruntime/python/tools/bert/bert_perf_test.py b/onnxruntime/python/tools/bert/bert_perf_test.py index 71b77b8817..4c8273c6b7 100644 --- a/onnxruntime/python/tools/bert/bert_perf_test.py +++ b/onnxruntime/python/tools/bert/bert_perf_test.py @@ -5,41 +5,59 @@ # This tool measures the inference performance of onnxruntime or onnxruntime-gpu python package on Bert model. +# The input model shall have exactly three inputs. The model is either fully optimized (with EmbedLayerNormalization node), +# or with reasonable input names (one input name has 'mask' substring, another has 'token' or 'segment' substring). +# See get_bert_inputs function in bert_test_data.py for more information. + +# Example command to run test on batch_size 1 and 2 for a model on GPU: +# python bert_perf_test.py --model bert.onnx --batch_size 1 2 --sequence_length 128 --use_gpu --samples 1000 --test_times 1 + import sys import argparse import os -import onnxruntime from pathlib import Path import timeit import statistics import psutil import csv import numpy as np +import random from datetime import datetime +import multiprocessing from bert_test_data import get_bert_inputs, generate_test_data -def create_session(model_path, use_gpu, use_openmp, graph_optimization_level, num_threads, wait_policy): - execution_providers = ['CPUExecutionProvider'] if not use_gpu else ['CUDAExecutionProvider', 'CPUExecutionProvider'] - sess_options = onnxruntime.SessionOptions() - sess_options.execution_mode = onnxruntime.ExecutionMode.ORT_SEQUENTIAL - sess_options.graph_optimization_level = graph_optimization_level - if not use_openmp: - sess_options.intra_op_num_threads=num_threads - if "OMP_NUM_THREADS" in os.environ: - del os.environ["OMP_NUM_THREADS"] - if "OMP_WAIT_POLICY" in os.environ: - del os.environ["OMP_WAIT_POLICY"] - else: - sess_options.intra_op_num_threads=1 - os.environ["OMP_NUM_THREADS"] = str(num_threads) - os.environ["OMP_WAIT_POLICY"] = wait_policy +def create_session(model_path, use_gpu, intra_op_num_threads, graph_optimization_level=None): + # Import onnxruntime shall be after OpenMP environment variable setting. + # So we put the import in function to delay importing instead of top of this script. + import onnxruntime + + if use_gpu and ('CUDAExecutionProvider' not in onnxruntime.get_available_providers()): + print("Warning: Please install onnxruntime-gpu package instead of onnxruntime, and use a machine with GPU for testing gpu performance.") + elif (not use_gpu) and ('CUDAExecutionProvider' in onnxruntime.get_available_providers()): + print("Warning: Please install onnxruntime package instead of onnxruntime-gpu to get best cpu performance.") + + if intra_op_num_threads is None and graph_optimization_level is None: + session = onnxruntime.InferenceSession(model_path) + else: + execution_providers = ['CPUExecutionProvider'] if not use_gpu else ['CUDAExecutionProvider', 'CPUExecutionProvider'] + sess_options = onnxruntime.SessionOptions() + sess_options.execution_mode = onnxruntime.ExecutionMode.ORT_SEQUENTIAL + if graph_optimization_level is None: + sess_options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL + else: + sess_options.graph_optimization_level = graph_optimization_level + sess_options.intra_op_num_threads = intra_op_num_threads + session = onnxruntime.InferenceSession(model_path, sess_options, providers=execution_providers) - session = onnxruntime.InferenceSession(model_path, sess_options, providers=execution_providers) if use_gpu: assert 'CUDAExecutionProvider' in session.get_providers() return session -def onnxruntime_inference(session, all_inputs, output_names): +def onnxruntime_inference(session, all_inputs, output_names, warmup=True): + if warmup and len(all_inputs) > 0: + # Use a random input as warm up. + session.run(output_names, random.choice(all_inputs)) + results = [] latency_list = [] for test_case_id, inputs in enumerate(all_inputs): @@ -76,68 +94,137 @@ def to_string(model_path, session, test_setting): option += ",{}".format(test_setting) return option -def run_one_test(latency_results, model_path, all_inputs, batch_size, sequence_length, use_gpu, test_cases, test_times, use_openmp, contiguous, num_threads, wait_policy): - test_setting = "batch_size={},sequence_length={},test_cases={},test_times={},contiguous={},use_gpu={}".format(batch_size,sequence_length,test_cases,test_times,contiguous,use_gpu) +def setup_openmp_environ(omp_num_threads, omp_wait_policy): + if omp_num_threads is None: + if "OMP_NUM_THREADS" in os.environ: + del os.environ["OMP_NUM_THREADS"] + else: + os.environ["OMP_NUM_THREADS"] = str(omp_num_threads) - graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL - session = create_session(model_path, use_gpu, use_openmp, graph_optimization_level, num_threads, wait_policy) + if omp_wait_policy is None: + if "OMP_WAIT_POLICY" in os.environ: + del os.environ["OMP_WAIT_POLICY"] + else: + assert omp_wait_policy in ["ACTIVE", "PASSIVE"] + os.environ["OMP_WAIT_POLICY"] = omp_wait_policy + +def run_one_test(perf_results, model_path, all_inputs, batch_size, sequence_length, use_gpu, test_cases, test_times, contiguous, intra_op_num_threads, omp_num_threads, omp_wait_policy, no_warmup, extra_latency): + # Environment variable shall be set before import onnxruntime. + setup_openmp_environ(omp_num_threads, omp_wait_policy) + + test_setting = "batch_size={},sequence_length={},test_cases={},test_times={},contiguous={},use_gpu={},warmup={}".format(batch_size, sequence_length, test_cases, test_times, contiguous, use_gpu, not no_warmup) + + session = create_session(model_path, use_gpu, intra_op_num_threads) output_names = [output.name for output in session.get_outputs()] key = to_string(model_path, session, test_setting) + if key in perf_results: + print("skip duplicated test:", key) + return + print("Running test:", key) all_latency_list = [] for i in range(test_times): - results, latency_list = onnxruntime_inference(session, all_inputs, output_names) + results, latency_list = onnxruntime_inference(session, all_inputs, output_names, not no_warmup) all_latency_list.extend(latency_list) - average_latency = statistics.mean(all_latency_list) * 1000 - print("Average latency is {} ms".format(format(average_latency, '.2f'))) - latency_results[key] = average_latency + # latency in miliseconds + latency_ms = np.array(all_latency_list) * 1000 + extra_latency -def run_perf_tests(average_latency, model_path, batch_size, sequence_length, use_gpu, test_cases, test_times, seed, verbose, contiguous, input_ids, segment_ids, input_mask, all_inputs, run_all_settings): - if use_gpu or run_all_settings: - run_one_test(average_latency, model_path, all_inputs, batch_size, sequence_length, use_gpu, test_cases, test_times, use_openmp=False, contiguous=contiguous, num_threads=psutil.cpu_count(logical=True), wait_policy='ACTIVE') + average_latency = statistics.mean(latency_ms) + latency_50 = np.percentile(latency_ms, 50) + latency_75 = np.percentile(latency_ms, 75) + latency_90 = np.percentile(latency_ms, 90) + latency_95 = np.percentile(latency_ms, 95) + latency_99 = np.percentile(latency_ms, 99) + throughput = batch_size * (1000.0 / average_latency) + + perf_results[key] = (average_latency, latency_50, latency_75, latency_90, latency_95, latency_99, throughput) + + print("Average latency = {} ms, Throughput = {} QPS".format(format(average_latency, '.2f'), format(throughput, '.2f'))) + +def launch_test(perf_results, model_path, all_inputs, batch_size, sequence_length, use_gpu, test_cases, test_times, contiguous, intra_op_num_threads, omp_num_threads, omp_wait_policy, no_warmup, extra_latency): + process = multiprocessing.Process(target=run_one_test, args=(perf_results, model_path, all_inputs, batch_size, sequence_length, use_gpu, test_cases, test_times, contiguous, intra_op_num_threads, omp_num_threads, omp_wait_policy, no_warmup, extra_latency)) + process.start() + process.join() + +def run_perf_tests(perf_results, model_path, batch_size, sequence_length, use_gpu, test_cases, test_times, contiguous, all_inputs, test_all, no_warmup, extra_latency): + cpu_count = psutil.cpu_count(logical=False) + logical_cores = psutil.cpu_count(logical=True) + + # Test a setting without any setting as baseline 1. + launch_test(perf_results, model_path, all_inputs, batch_size, sequence_length, use_gpu, test_cases, test_times, contiguous, None, None, None, no_warmup, extra_latency) - # onnxruntime-gpu package is not built with OpenMP, so skip openmp test for gpu. if not use_gpu: - run_one_test(average_latency, model_path, all_inputs, batch_size, sequence_length, use_gpu, test_cases, test_times, use_openmp=True, contiguous=contiguous, num_threads=psutil.cpu_count(logical=True), wait_policy='ACTIVE') + # For CPU: intra_op_num_threads = 1, omp_num_threads=None, omp_wait_policy=None + # Another setting without environment variable as baseline 2. + launch_test(perf_results, model_path, all_inputs, batch_size, sequence_length, use_gpu, test_cases, test_times, contiguous, 1, None, None, no_warmup, extra_latency) + else: + # For GPU, we test two more settings by default: + # (1) intra_op_num_threads = 1, omp_num_threads=cpu_count, omp_wait_policy=PASSIVE + # (2) intra_op_num_threads = logical_cores, omp_num_threads=1, omp_wait_policy=ACTIVE + launch_test(perf_results, model_path, all_inputs, batch_size, sequence_length, use_gpu, test_cases, test_times, contiguous, 1, cpu_count, 'PASSIVE', no_warmup, extra_latency) - if run_all_settings: - run_one_test(average_latency, model_path, all_inputs, batch_size, sequence_length, use_gpu, test_cases, test_times, use_openmp=True, contiguous=contiguous, num_threads=psutil.cpu_count(logical=True), wait_policy='PASSIVE') + launch_test(perf_results, model_path, all_inputs, batch_size, sequence_length, use_gpu, test_cases, test_times, contiguous, logical_cores, 1, 'ACTIVE', no_warmup, extra_latency) - if psutil.cpu_count(logical=True) != psutil.cpu_count(logical=False): - run_one_test(average_latency, model_path, all_inputs, batch_size, sequence_length, use_gpu, test_cases, test_times, use_openmp=True, contiguous=contiguous, num_threads=psutil.cpu_count(logical=False), wait_policy='ACTIVE') - run_one_test(average_latency, model_path, all_inputs, batch_size, sequence_length, use_gpu, test_cases, test_times, use_openmp=True, contiguous=contiguous, num_threads=psutil.cpu_count(logical=False), wait_policy='PASSIVE') + # GPU latency is not sensitive to these settings. No need to test many combinations. + # Skip remaining settings for GPU without --all flag. + if use_gpu and not test_all: + return -def run_performance(average_latency, model_path, batch_size, sequence_length, use_gpu, test_cases, test_times, seed, verbose, run_all_settings): + candidates = list(set([1, logical_cores, cpu_count])) + + for intra_op_num_threads in candidates: + for omp_num_threads in candidates: + # skip settings that are very slow + if intra_op_num_threads == 1 and omp_num_threads == 1 and logical_cores != 1: + continue + + # When logical and physical cores are not the same, there are many combinations. + # Remove some settings are not good normally. + if logical_cores > cpu_count: + if omp_num_threads == logical_cores and intra_op_num_threads != 1: + continue + if intra_op_num_threads == logical_cores and omp_num_threads != 1: + continue + + if not test_all: + if intra_op_num_threads != 1 and omp_num_threads != 1: + continue + + for omp_wait_policy in ['ACTIVE', 'PASSIVE']: + launch_test(perf_results, model_path, all_inputs, batch_size, sequence_length, use_gpu, test_cases, test_times, contiguous, intra_op_num_threads, omp_num_threads, omp_wait_policy, no_warmup, extra_latency) + +def run_performance(perf_results, model_path, batch_size, sequence_length, use_gpu, test_cases, test_times, seed, verbose, inclusive, test_all, no_warmup): # Try deduce input names from model. input_ids, segment_ids, input_mask = get_bert_inputs(model_path) # Do not generate random mask for performance test. - print("generating test data...") + print(f"Generating {test_cases} samples for batch_size={batch_size} sequence_length={sequence_length}") all_inputs = generate_test_data(batch_size, sequence_length, test_cases, seed, verbose, input_ids, segment_ids, input_mask, random_mask_length=False) contiguous = False - if run_all_settings: - run_perf_tests(average_latency, model_path, batch_size, sequence_length, use_gpu, test_cases, test_times, seed, verbose, contiguous, input_ids, segment_ids, input_mask, all_inputs, run_all_settings) + run_perf_tests(perf_results, model_path, batch_size, sequence_length, use_gpu, test_cases, test_times, contiguous, all_inputs, test_all, no_warmup, extra_latency=0) + + # only test contiguous array when the --all flag is set. + if not test_all: + return # Convert inputs to contiguous array, which could improve inference performance all_inputs, contiguous_latency = get_contiguous_inputs(all_inputs) print("Extra latency for converting inputs to contiguous: {} ms".format(format(contiguous_latency, '.2f'))) contiguous = True - run_perf_tests(average_latency, model_path, batch_size, sequence_length, use_gpu, test_cases, test_times, seed, verbose, contiguous, input_ids, segment_ids, input_mask, all_inputs, run_all_settings) - - return contiguous_latency + run_perf_tests(perf_results, model_path, batch_size, sequence_length, use_gpu, test_cases, test_times, contiguous, all_inputs, test_all, no_warmup, extra_latency=contiguous_latency if inclusive else 0) def parse_arguments(): parser = argparse.ArgumentParser() parser.add_argument('--model', required=True, type=str, help="bert onnx model path") - parser.add_argument('--batch_size', required=True, type=int, - help="batch size of input") + parser.add_argument('--batch_size', required=True, type=int, nargs="+", + help="batch size of input. Allow one or multiple values in the range of [1, 128].") parser.add_argument('--sequence_length', required=True, type=int, help="maximum sequence length of input") @@ -160,9 +247,12 @@ def parse_arguments(): parser.add_argument('--inclusive', required=False, action='store_true', help="include the latency of converting array to contiguous") parser.set_defaults(inclusive=False) - parser.add_argument('--all', required=False, action='store_true', help="test all settings") + parser.add_argument('--all', required=False, action='store_true', help="test all candidate settings") parser.set_defaults(all=False) + parser.add_argument('--no_warmup', required=False, action='store_true', help="do not use one sample for warm-up.") + parser.set_defaults(no_warmup=False) + args = parser.parse_args() return args @@ -172,40 +262,38 @@ def main(): if args.test_times == 0: args.test_times = max(1, int(1000 / args.samples)) - if args.use_gpu and ('CUDAExecutionProvider' not in onnxruntime.get_available_providers()): - print("Please install onnxruntime-gpu package instead of onnxruntime, and use a machine with GPU for testing gpu performance.") - return - elif (not args.use_gpu) and ('CUDAExecutionProvider' in onnxruntime.get_available_providers()): - print("Warning: Please install onnxruntime package instead of onnxruntime-gpu to get best cpu performance.") + manager = multiprocessing.Manager() + perf_results = manager.dict() - average_latency = {} - contiguous_latency = run_performance(average_latency, args.model, args.batch_size, args.sequence_length, args.use_gpu, args.samples, args.test_times, args.seed, args.verbose, args.all) + batch_size_set = set(args.batch_size) + if not min(batch_size_set)>=1 and max(batch_size_set) <= 128: + raise Exception("batch_size not in range [1, 128]") - if average_latency is None: - return + for batch_size in batch_size_set: + run_performance(perf_results, args.model, batch_size, args.sequence_length, args.use_gpu, args.samples, args.test_times, args.seed, args.verbose, args.inclusive, args.all, args.no_warmup) - summary_file = os.path.join(Path(args.model).parent, "perf_results_{}_B{}_S{}_{}.txt".format('GPU' if args.use_gpu else 'CPU', args.batch_size, args.sequence_length, datetime.now().strftime("%Y%m%d-%H%M%S"))) + # Sort the results so that the first one has smallest latency. + sorted_results = sorted(perf_results.items(), reverse=False, key=lambda x: x[1]) + + summary_file = os.path.join(Path(args.model).parent, "perf_results_{}_B{}_S{}_{}.txt".format('GPU' if args.use_gpu else 'CPU', "-".join([str(x) for x in sorted(list(batch_size_set))]), args.sequence_length, datetime.now().strftime("%Y%m%d-%H%M%S"))) with open(summary_file, 'w+', newline='') as tsv_file: tsv_writer = csv.writer(tsv_file, delimiter='\t', lineterminator='\n') headers = None - for key, latency in average_latency.items(): + for (key, perf_result) in sorted_results: params = key.split(',') if headers is None: - headers = ["Latency(ms)", "Throughput(QPS)"] + headers = ["Latency(ms)", "Latency_P50", "Latency_P75", "Latency_P90", "Latency_P95", "Latency_P99", "Throughput(QPS)"] headers.extend([x.split('=')[0] for x in params]) tsv_writer.writerow(headers) - # include the extra latency of array conversion if required. - if args.inclusive and 'contiguous=True' in params: - latency += contiguous_latency - - throughput = args.batch_size * (1000 / latency) - values = [format(latency, '.2f'), format(throughput, '.2f')] - + values = [format(x, '.2f') for x in perf_result] values.extend([x.split('=')[1] for x in params]) tsv_writer.writerow(values) print("Test summary is saved to", summary_file) if __name__ == "__main__": + # work around for AnaConda Jupyter. See https://stackoverflow.com/questions/45720153/python-multiprocessing-error-attributeerror-module-main-has-no-attribute + __spec__ = None + main() diff --git a/onnxruntime/python/tools/bert/compare_bert_results.py b/onnxruntime/python/tools/bert/compare_bert_results.py index 9bac647a4d..4a1a7a734a 100644 --- a/onnxruntime/python/tools/bert/compare_bert_results.py +++ b/onnxruntime/python/tools/bert/compare_bert_results.py @@ -9,7 +9,6 @@ import sys import argparse import numpy as np import os -import onnxruntime import random from pathlib import Path import statistics @@ -21,12 +20,22 @@ import timeit from datetime import datetime from onnx import ModelProto, TensorProto, numpy_helper from OnnxModel import OnnxModel -from bert_model_optimization import optimize_by_onnxruntime from bert_test_data import get_bert_inputs, generate_test_data, output_test_data -from bert_perf_test import create_session, onnxruntime_inference +from bert_perf_test import create_session, onnxruntime_inference, setup_openmp_environ + +def run_model(model_path, all_inputs, use_gpu, use_openmp, disable_optimization): + # Import onnxruntime shall be after OpenMP environment variable setting. + # So we put import here to delay importing. + import onnxruntime + + graph_optimization_level = None + if disable_optimization: + graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_DISABLE_ALL + + intra_op_num_threads = 1 if use_openmp else psutil.cpu_count(logical=False) + + session = create_session(model_path, use_gpu, intra_op_num_threads, graph_optimization_level) -def run_model(baseline_model, all_inputs, use_gpu, use_openmp, graph_optimization_level): - session = create_session(baseline_model, use_gpu, use_openmp, graph_optimization_level, num_threads=psutil.cpu_count(logical=True), wait_policy='ACTIVE') output_names = [output.name for output in session.get_outputs()] results, latency_list = onnxruntime_inference(session, all_inputs, output_names) return results, latency_list, output_names @@ -39,20 +48,22 @@ def compare(baseline_results, treatment_results, verbose, rtol=1e-3, atol=1e-4): for test_case_id, results in enumerate(baseline_results): case_passed = True for i in range(len(results)): - treatment_first_output = treatment_results[test_case_id][0].tolist() - rel_diff = np.amax(np.abs((treatment_results[test_case_id][0] - results[0]) / results[0])) - abs_diff = np.amax(np.abs(treatment_results[test_case_id][0] - results[0])) + treatment_output = treatment_results[test_case_id][i] + rel_diff = np.amax(np.abs((treatment_output - results[i]) / results[i])) + abs_diff = np.amax(np.abs(treatment_output - results[i])) max_rel_diff = max(max_rel_diff, rel_diff) max_abs_diff = max(max_abs_diff, abs_diff) - if verbose: - print("case {} output {}".format(test_case_id, i)) - print("baseline={}\ntreatment={}".format(results[0].tolist(), treatment_first_output)) - print("rel_diff={} abs_diff={}".format(rel_diff, abs_diff)) - if not np.allclose(results[0].tolist(), treatment_first_output, rtol=rtol, atol=atol): + if not np.allclose(results[i].tolist(), treatment_output.tolist(), rtol=rtol, atol=atol): if case_passed: case_passed = False diff_count += 1 + if verbose: + print("case {} output {}".format(test_case_id, i)) + print("baseline={}\ntreatment={}".format(results[i].tolist(), treatment_output)) + print("rel_diff={} abs_diff={}".format(rel_diff, abs_diff)) + + if diff_count == 0: print("100% passed for {} random inputs given thresholds (rtol={}, atol={}).".format(len(baseline_results), rtol, atol)) else: @@ -69,15 +80,21 @@ def run_test(baseline_model, optimized_model, output_dir, batch_size, sequence_l # Use random mask length for accuracy test. It might introduce slight inflation in latency reported in this script. all_inputs = generate_test_data(batch_size, sequence_length, test_cases, seed, verbose, input_ids, segment_ids, input_mask, random_mask_length=True) - baseline_results, baseline_latency, output_names = run_model(baseline_model, all_inputs, use_gpu, use_openmp, onnxruntime.GraphOptimizationLevel.ORT_DISABLE_ALL) + # OpenMP environment variables must be set before the very first "import onnxruntime" + if use_openmp: + setup_openmp_environ(omp_num_threads=psutil.cpu_count(logical=False), omp_wait_policy='ACTIVE') + else: + setup_openmp_environ(omp_num_threads=1, omp_wait_policy='ACTIVE') + + baseline_results, baseline_latency, output_names = run_model(baseline_model, all_inputs, use_gpu, use_openmp, disable_optimization=True) if verbose: - print("baseline average latency: {} ms".format(statistics.mean(baseline_latency) * 1000)) + print("baseline average latency (all optimizations disabled): {} ms".format(statistics.mean(baseline_latency) * 1000)) if output_dir is not None: for i, inputs in enumerate(all_inputs): output_test_data(output_dir, i, inputs) - treatment_results, treatment_latency, treatment_output_names = run_model(optimized_model, all_inputs, use_gpu, use_openmp, onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL) + treatment_results, treatment_latency, treatment_output_names = run_model(optimized_model, all_inputs, use_gpu, use_openmp, disable_optimization=False) if verbose: print("treatment average latency: {} ms".format(statistics.mean(treatment_latency) * 1000)) @@ -87,10 +104,10 @@ def run_test(baseline_model, optimized_model, output_dir, batch_size, sequence_l def parse_arguments(): parser = argparse.ArgumentParser() parser.add_argument('--baseline_model', required=True, type=str, - help="baseline onnx model path") + help="baseline onnx model path.") - parser.add_argument('--optimized_model', required=False, type=str, default=None, - help="optimized model for the baseline model. They shall have same inputs. If it is None, an optimized model will be generated using OnnxRuntime.") + parser.add_argument('--optimized_model', required=True, type=str, default=None, + help="path of the optimized model. It shall have same inputs as the baseline model.") parser.add_argument('--output_dir', required=False, type=str, default=None, help="output test data path. If not specified, test data will not be saved.") @@ -116,8 +133,8 @@ def parse_arguments(): parser.add_argument('--use_gpu', required=False, action='store_true', help="use GPU") parser.set_defaults(use_gpu=False) - parser.add_argument('--no_openmp', required=False, action='store_true', help="do not use openmp") - parser.set_defaults(no_openmp=False) + parser.add_argument('--openmp', required=False, action='store_true', help="use openmp") + parser.set_defaults(openmp=False) parser.add_argument('--verbose', required=False, action='store_true', help="print verbose information") parser.set_defaults(verbose=False) @@ -128,11 +145,6 @@ def parse_arguments(): def main(): args = parse_arguments() - optimized_model = optimize_by_onnxruntime(args.baseline_model, args.use_gpu) if (args.optimized_model is None) else args.optimized_model - - if args.use_gpu and ('CUDAExecutionProvider' not in onnxruntime.get_available_providers()): - print("Please install onnxruntime-gpu package instead of onnxruntime, and use a machine with GPU for testing gpu.") - if args.output_dir is not None: # create the output directory if not existed path = Path(args.output_dir) @@ -140,14 +152,14 @@ def main(): run_test( args.baseline_model, - optimized_model, + args.optimized_model, args.output_dir, args.batch_size, args.sequence_length, args.use_gpu, args.samples, args.seed, - not args.no_openmp, + args.openmp, args.verbose, args.rtol, args.atol) diff --git a/onnxruntime/python/tools/bert/notebooks/PyTorch_Bert-Squad_OnnxRuntime_CPU.ipynb b/onnxruntime/python/tools/bert/notebooks/PyTorch_Bert-Squad_OnnxRuntime_CPU.ipynb new file mode 100644 index 0000000000..aaff0445f9 --- /dev/null +++ b/onnxruntime/python/tools/bert/notebooks/PyTorch_Bert-Squad_OnnxRuntime_CPU.ipynb @@ -0,0 +1,1152 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Copyright (c) Microsoft Corporation. All rights reserved. \n", + "Licensed under the MIT License." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Inference PyTorch Bert Model with ONNX Runtime on CPU" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this tutorial, you'll be introduced to how to load a Bert model from PyTorch, convert it to ONNX, and inference it for high performance using ONNX Runtime. In the following sections, we are going to use the Bert model trained with Stanford Question Answering Dataset (SQuAD) dataset as an example. Bert SQuAD model is used in question answering scenarios, where the answer to every question is a segment of text, or span, from the corresponding reading passage, or the question might be unanswerable.\n", + "\n", + "This notebook is for CPU inference. For GPU inferenece, please look at another notebook [Inference PyTorch Bert Model with ONNX Runtime on GPU](PyTorch_Bert-Squad_OnnxRuntime_GPU.ipynb)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 0. Prerequisites ##\n", + "\n", + "If you have Jupyter Notebook, you may directly run this notebook. We will use pip to install or upgrade [PyTorch](https://pytorch.org/), [OnnxRuntime](https://microsoft.github.io/onnxruntime/) and other required packages.\n", + "\n", + "Otherwise, you can setup a new environment. First, we install [AnaConda](https://www.anaconda.com/distribution/). Then open an AnaConda prompt window and run the following commands:\n", + "\n", + "```console\n", + "conda create -n cpu_env python=3.6\n", + "conda activate cpu_env\n", + "conda install pytorch torchvision cpuonly -c pytorch\n", + "pip install onnxruntime\n", + "conda install jupyter\n", + "jupyter notebook\n", + "```\n", + "The last command will launch Jupyter Notebook and we can open this notebook in browser to continue." + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": { + "collapsed": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Looking in links: https://download.pytorch.org/whl/torch_stable.html\n", + "Requirement already up-to-date: torch==1.4.0+cpu in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (1.4.0+cpu)\n", + "Requirement already up-to-date: torchvision==0.5.0+cpu in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (0.5.0+cpu)\n", + "Requirement already satisfied, skipping upgrade: six in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from torchvision==0.5.0+cpu) (1.14.0)\n", + "Requirement already satisfied, skipping upgrade: numpy in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from torchvision==0.5.0+cpu) (1.18.1)\n", + "Requirement already satisfied, skipping upgrade: pillow>=4.1.1 in c:\\users\\tianl\\appdata\\roaming\\python\\python36\\site-packages (from torchvision==0.5.0+cpu) (7.0.0)\n", + "Requirement already up-to-date: onnxruntime in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (1.2.0)\n", + "Requirement already satisfied, skipping upgrade: onnx>=1.2.3 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from onnxruntime) (1.6.0)\n", + "Requirement already satisfied, skipping upgrade: numpy>=1.18.0 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from onnxruntime) (1.18.1)\n", + "Requirement already satisfied, skipping upgrade: protobuf in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from onnx>=1.2.3->onnxruntime) (3.11.3)\n", + "Requirement already satisfied, skipping upgrade: six in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from onnx>=1.2.3->onnxruntime) (1.14.0)\n", + "Requirement already satisfied, skipping upgrade: typing-extensions>=3.6.2.1 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from onnx>=1.2.3->onnxruntime) (3.7.4.1)\n", + "Requirement already satisfied, skipping upgrade: setuptools in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from protobuf->onnx>=1.2.3->onnxruntime) (45.2.0.post20200210)\n", + "Requirement already satisfied: transformers==2.5.1 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (2.5.1)\n", + "Requirement already satisfied: tqdm>=4.27 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from transformers==2.5.1) (4.43.0)\n", + "Requirement already satisfied: regex!=2019.12.17 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from transformers==2.5.1) (2020.2.20)\n", + "Requirement already satisfied: boto3 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from transformers==2.5.1) (1.12.11)\n", + "Requirement already satisfied: sentencepiece in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from transformers==2.5.1) (0.1.85)\n", + "Requirement already satisfied: sacremoses in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from transformers==2.5.1) (0.0.38)\n", + "Requirement already satisfied: tokenizers==0.5.2 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from transformers==2.5.1) (0.5.2)\n", + "Requirement already satisfied: requests in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from transformers==2.5.1) (2.23.0)\n", + "Requirement already satisfied: numpy in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from transformers==2.5.1) (1.18.1)\n", + "Requirement already satisfied: filelock in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from transformers==2.5.1) (3.0.12)\n", + "Requirement already satisfied: s3transfer<0.4.0,>=0.3.0 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from boto3->transformers==2.5.1) (0.3.3)\n", + "Requirement already satisfied: jmespath<1.0.0,>=0.7.1 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from boto3->transformers==2.5.1) (0.9.5)\n", + "Requirement already satisfied: botocore<1.16.0,>=1.15.11 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from boto3->transformers==2.5.1) (1.15.11)\n", + "Requirement already satisfied: click in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from sacremoses->transformers==2.5.1) (7.0)\n", + "Requirement already satisfied: six in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from sacremoses->transformers==2.5.1) (1.14.0)\n", + "Requirement already satisfied: joblib in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from sacremoses->transformers==2.5.1) (0.14.1)\n", + "Requirement already satisfied: idna<3,>=2.5 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from requests->transformers==2.5.1) (2.9)\n", + "Requirement already satisfied: chardet<4,>=3.0.2 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from requests->transformers==2.5.1) (3.0.4)\n", + "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from requests->transformers==2.5.1) (1.25.8)\n", + "Requirement already satisfied: certifi>=2017.4.17 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from requests->transformers==2.5.1) (2019.11.28)\n", + "Requirement already satisfied: python-dateutil<3.0.0,>=2.1 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from botocore<1.16.0,>=1.15.11->boto3->transformers==2.5.1) (2.8.1)\n", + "Requirement already satisfied: docutils<0.16,>=0.10 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from botocore<1.16.0,>=1.15.11->boto3->transformers==2.5.1) (0.15.2)\n", + "Requirement already satisfied: wget in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (3.2)\n", + "Requirement already satisfied: psutil in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (5.7.0)\n", + "Requirement already satisfied: onnx in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (1.6.0)\n", + "Requirement already satisfied: pytz in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (2019.3)\n", + "Requirement already satisfied: pandas in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (1.0.1)\n", + "Requirement already satisfied: py-cpuinfo in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (5.0.0)\n", + "Requirement already satisfied: py3nvml in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (0.2.5)\n", + "Requirement already satisfied: typing-extensions>=3.6.2.1 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from onnx) (3.7.4.1)\n", + "Requirement already satisfied: protobuf in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from onnx) (3.11.3)\n", + "Requirement already satisfied: numpy in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from onnx) (1.18.1)\n", + "Requirement already satisfied: six in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from onnx) (1.14.0)\n", + "Requirement already satisfied: python-dateutil>=2.6.1 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from pandas) (2.8.1)\n", + "Requirement already satisfied: xmltodict in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from py3nvml) (0.12.0)\n", + "Requirement already satisfied: setuptools in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from protobuf->onnx) (45.2.0.post20200210)\n" + ] + } + ], + "source": [ + "# Install or upgrade PyTorch 1.4.0 and OnnxRuntime for CPU-only.\n", + "import sys\n", + "!{sys.executable} -m pip install --upgrade torch==1.4.0+cpu torchvision==0.5.0+cpu -f https://download.pytorch.org/whl/torch_stable.html\n", + "!{sys.executable} -m pip install --upgrade onnxruntime\n", + "\n", + "# Install other packages used in this notebook.\n", + "!{sys.executable} -m pip install transformers==2.5.1\n", + "!{sys.executable} -m pip install wget psutil onnx pytz pandas py-cpuinfo py3nvml netron" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Load Pretrained Bert model ##" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We begin by downloading the SQuAD data file and store them in the specified location." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "cache_dir = \"./squad\"\n", + "if not os.path.exists(cache_dir):\n", + " os.makedirs(cache_dir)\n", + "\n", + "predict_file_url = \"https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json\"\n", + "predict_file = os.path.join(cache_dir, \"dev-v1.1.json\")\n", + "if not os.path.exists(predict_file):\n", + " import wget\n", + " print(\"Start downloading predict file.\")\n", + " wget.download(predict_file_url, predict_file)\n", + " print(\"Predict file downloaded.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Specify some model configuration variables and constant." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# For fine tuned large model, the model name is \"bert-large-uncased-whole-word-masking-finetuned-squad\". Here we use bert-base for demo.\n", + "model_name_or_path = \"bert-base-cased\"\n", + "max_seq_length = 128\n", + "doc_stride = 128\n", + "max_query_length = 64\n", + "\n", + "# Enable overwrite to export onnx model and download latest script each time when running this notebook.\n", + "enable_overwrite = True\n", + "\n", + "# Total samples to inference. It shall be large enough to get stable latency measurement.\n", + "total_samples = 100" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Start to load model from pretrained. This step could take a few minutes. " + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████████████████████████████████████████████████████████████████████████████| 48/48 [00:03<00:00, 12.65it/s]\n", + "convert squad examples to features: 100%|███████████████████████████████████████████| 100/100 [00:00<00:00, 145.99it/s]\n", + "add example index and unique id: 100%|███████████████████████████████████████████████████████| 100/100 [00:00\n", + "\n", + "For CPU, optimized graph is slightly different: FastGelu is replaced by BiasGelu." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [], + "source": [ + "import netron\n", + "\n", + "# Change it to True if want to view the optimized model in browser.\n", + "enable_netron = False\n", + "if enable_netron:\n", + " # If you encounter error \"access a socket in a way forbidden by its access permissions\", install Netron as standalone application instead.\n", + " netron.start(optimized_model_path)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Model Results Comparison Tool\n", + "\n", + "If your BERT model has three inputs, a script compare_bert_results.py can be used to do a quick verification. The tool will generate some fake input data, and compare results from both the original and optimized models. If outputs are all close, it is safe to use the optimized model.\n", + "\n", + "Example of verifying models:" + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "100% passed for 100 random inputs given thresholds (rtol=0.001, atol=0.0001).\n", + "maximum absolute difference=5.0961971282958984e-06\n", + "maximum relative difference=0.003811897709965706\n" + ] + } + ], + "source": [ + "%run ./bert_scripts/compare_bert_results.py --baseline_model $export_model_path --optimized_model $optimized_model_path --batch_size 1 --sequence_length 128 --samples 100" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Performance Test Tool\n", + "\n", + "This tool measures performance of BERT model inference using OnnxRuntime Python API.\n", + "\n", + "The following command will create 100 samples of batch_size 1 and sequence length 128 to run inference, then calculate performance numbers like average latency and throughput etc. You can increase number of samples (recommended 1000) to get more stable result." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generating 100 samples for batch_size=1 sequence_length=128\n", + "Extra latency for converting inputs to contiguous: 0.00 ms\n", + "Test summary is saved to onnx\\perf_results_CPU_B1_S128_20200313-001048.txt\n" + ] + } + ], + "source": [ + "%run ./bert_scripts/bert_perf_test.py --model $optimized_model_path --batch_size 1 --sequence_length 128 --samples 100 --test_times 1 --inclusive --all" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's load the summary file and take a look." + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "./onnx\\perf_results_CPU_B1_S128_20200313-001048.txt\n", + "The best setting is: NO openmp; use contiguous array\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Latency(ms)Latency_P75Latency_P90Latency_P99Throughput(QPS)intra_op_num_threadsOMP_NUM_THREADSOMP_WAIT_POLICYcontiguous
074.6876.7483.7589.2613.39121PASSIVETrue
174.7076.5785.8689.3013.39121ACTIVETrue
275.2378.1386.4988.9913.29121ACTIVEFalse
375.6678.3687.6095.1713.22121PASSIVEFalse
476.5678.7391.02101.5813.06112PASSIVEFalse
576.7178.9991.7198.7013.04112PASSIVETrue
679.0880.8785.98121.3612.65112ACTIVETrue
780.3082.2696.97122.5512.451True
888.4391.29111.84119.9911.3116PASSIVEFalse
989.0392.76110.57121.0611.2316PASSIVETrue
1094.6094.41101.17110.3910.5766ACTIVETrue
1195.3294.32102.61119.7910.4966ACTIVEFalse
12100.75107.69112.40119.289.9316ACTIVETrue
13101.89108.09113.35118.929.8116ACTIVEFalse
14104.49105.83107.46109.169.5761ACTIVETrue
15104.73106.29108.70109.339.5561PASSIVEFalse
16104.84106.26107.87109.739.5461PASSIVETrue
17105.21106.93109.11110.329.5161ACTIVEFalse
18107.31108.37111.24115.969.3266PASSIVEFalse
19107.98111.01119.32129.669.26112ACTIVEFalse
20108.65110.69112.22113.699.2066PASSIVETrue
21109.86110.35116.37127.009.101False
22111.36110.36125.15157.798.980True
23119.46119.96135.38171.818.370False
\n", + "
" + ], + "text/plain": [ + " Latency(ms) Latency_P75 Latency_P90 Latency_P99 Throughput(QPS) \\\n", + "0 74.68 76.74 83.75 89.26 13.39 \n", + "1 74.70 76.57 85.86 89.30 13.39 \n", + "2 75.23 78.13 86.49 88.99 13.29 \n", + "3 75.66 78.36 87.60 95.17 13.22 \n", + "4 76.56 78.73 91.02 101.58 13.06 \n", + "5 76.71 78.99 91.71 98.70 13.04 \n", + "6 79.08 80.87 85.98 121.36 12.65 \n", + "7 80.30 82.26 96.97 122.55 12.45 \n", + "8 88.43 91.29 111.84 119.99 11.31 \n", + "9 89.03 92.76 110.57 121.06 11.23 \n", + "10 94.60 94.41 101.17 110.39 10.57 \n", + "11 95.32 94.32 102.61 119.79 10.49 \n", + "12 100.75 107.69 112.40 119.28 9.93 \n", + "13 101.89 108.09 113.35 118.92 9.81 \n", + "14 104.49 105.83 107.46 109.16 9.57 \n", + "15 104.73 106.29 108.70 109.33 9.55 \n", + "16 104.84 106.26 107.87 109.73 9.54 \n", + "17 105.21 106.93 109.11 110.32 9.51 \n", + "18 107.31 108.37 111.24 115.96 9.32 \n", + "19 107.98 111.01 119.32 129.66 9.26 \n", + "20 108.65 110.69 112.22 113.69 9.20 \n", + "21 109.86 110.35 116.37 127.00 9.10 \n", + "22 111.36 110.36 125.15 157.79 8.98 \n", + "23 119.46 119.96 135.38 171.81 8.37 \n", + "\n", + " intra_op_num_threads OMP_NUM_THREADS OMP_WAIT_POLICY contiguous \n", + "0 12 1 PASSIVE True \n", + "1 12 1 ACTIVE True \n", + "2 12 1 ACTIVE False \n", + "3 12 1 PASSIVE False \n", + "4 1 12 PASSIVE False \n", + "5 1 12 PASSIVE True \n", + "6 1 12 ACTIVE True \n", + "7 1 True \n", + "8 1 6 PASSIVE False \n", + "9 1 6 PASSIVE True \n", + "10 6 6 ACTIVE True \n", + "11 6 6 ACTIVE False \n", + "12 1 6 ACTIVE True \n", + "13 1 6 ACTIVE False \n", + "14 6 1 ACTIVE True \n", + "15 6 1 PASSIVE False \n", + "16 6 1 PASSIVE True \n", + "17 6 1 ACTIVE False \n", + "18 6 6 PASSIVE False \n", + "19 1 12 ACTIVE False \n", + "20 6 6 PASSIVE True \n", + "21 1 False \n", + "22 0 True \n", + "23 0 False " + ] + }, + "execution_count": 15, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import os\n", + "import glob \n", + "import pandas\n", + "latest_result_file = max(glob.glob(\"./onnx/perf_results_*.txt\"), key=os.path.getmtime)\n", + "result_data = pandas.read_table(latest_result_file, converters={'OMP_NUM_THREADS': str, 'OMP_WAIT_POLICY':str})\n", + "print(latest_result_file)\n", + "print(\"The best setting is: {} openmp; {} contiguous array\".format('use' if result_data['intra_op_num_threads'].iloc[0] == 1 else 'NO', 'use' if result_data['contiguous'].iloc[0] else 'NO'))\n", + "# Remove some columns that have same values for all rows.\n", + "columns_to_remove = ['model', 'graph_optimization_level', 'batch_size', 'sequence_length', 'test_cases', 'test_times', 'use_gpu', 'warmup']\n", + "# Hide some latency percentile columns to fit screen width.\n", + "columns_to_remove.extend(['Latency_P50', 'Latency_P95'])\n", + "result_data.drop(columns_to_remove, axis=1, inplace=True)\n", + "result_data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 6. Additional Info\n", + "\n", + "Note that running Jupyter Notebook has slight impact on performance result since Jupyter Notebook is using system resources like CPU and memory etc. It is recommended to close Jupyter Notebook and other applications, then run the performance test tool in a console to get more accurate performance numbers.\n", + "\n", + "[OnnxRuntime C API](https://github.com/microsoft/onnxruntime/blob/master/docs/C_API.md) could get slightly better performance than python API. If you use C API in inference, you can use OnnxRuntime_Perf_Test.exe built from source to measure performance instead.\n", + "\n", + "Here is the machine configuration that generated the above results. The machine has GPU but not used in CPU inference.\n", + "You might get slower or faster result based on your hardware." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"gpu\": {\n", + " \"driver_version\": \"441.22\",\n", + " \"devices\": [\n", + " {\n", + " \"memory_total\": 8589934592,\n", + " \"memory_available\": 6741864448,\n", + " \"name\": \"GeForce GTX 1070\"\n", + " }\n", + " ]\n", + " },\n", + " \"cpu\": {\n", + " \"brand\": \"Intel(R) Core(TM) i7-8700 CPU @ 3.20GHz\",\n", + " \"cores\": 6,\n", + " \"logical_cores\": 12,\n", + " \"hz\": \"3.1920 GHz\",\n", + " \"l2_cache\": \"1536 KB\",\n", + " \"l3_cache\": \"12288 KB\",\n", + " \"processor\": \"Intel64 Family 6 Model 158 Stepping 10, GenuineIntel\"\n", + " },\n", + " \"memory\": {\n", + " \"total\": 16971259904,\n", + " \"available\": 2581991424\n", + " },\n", + " \"python\": \"3.6.10.final.0 (64 bit)\",\n", + " \"os\": \"Windows-10-10.0.18362-SP0\",\n", + " \"onnxruntime\": {\n", + " \"version\": \"1.2.0\",\n", + " \"support_gpu\": false\n", + " },\n", + " \"pytorch\": {\n", + " \"version\": \"1.4.0+cpu\",\n", + " \"support_gpu\": false\n", + " },\n", + " \"tensorflow\": {\n", + " \"version\": \"2.1.0\",\n", + " \"git_version\": \"v2.1.0-rc2-17-ge5bf8de410\",\n", + " \"support_gpu\": true\n", + " }\n", + "}\n" + ] + } + ], + "source": [ + "%run ./bert_scripts/MachineInfo.py --silent" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "cpu_env", + "language": "python", + "name": "cpu_env" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.10" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/onnxruntime/python/tools/bert/notebooks/PyTorch_Bert-Squad_OnnxRuntime_GPU.ipynb b/onnxruntime/python/tools/bert/notebooks/PyTorch_Bert-Squad_OnnxRuntime_GPU.ipynb new file mode 100644 index 0000000000..d9ac643786 --- /dev/null +++ b/onnxruntime/python/tools/bert/notebooks/PyTorch_Bert-Squad_OnnxRuntime_GPU.ipynb @@ -0,0 +1,1801 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Copyright (c) Microsoft Corporation. All rights reserved. \n", + "Licensed under the MIT License." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Inference PyTorch Bert Model with ONNX Runtime on GPU" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this tutorial, you'll be introduced to how to load a Bert model from PyTorch, convert it to ONNX, and inference it for high performance using ONNX Runtime and NVIDIA GPU. In the following sections, we are going to use the Bert model trained with Stanford Question Answering Dataset (SQuAD) dataset as an example. Bert SQuAD model is used in question answering scenarios, where the answer to every question is a segment of text, or span, from the corresponding reading passage, or the question might be unanswerable.\n", + "\n", + "This notebook is for CPU inference. For GPU inference, please look at another notebook [Inference PyTorch Bert Model with ONNX Runtime on CPU](PyTorch_Bert-Squad_OnnxRuntime_CPU.ipynb)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 0. Prerequisites ##\n", + "It requires your machine to have a GPU, and a python environment with [PyTorch](https://pytorch.org/) and [OnnxRuntime](https://microsoft.github.io/onnxruntime/) installed before running this notebook.\n", + "\n", + "#### GPU Environment Setup using AnaConda\n", + "\n", + "First, we install [AnaConda](https://www.anaconda.com/distribution/) in a target machine and open an AnaConda prompt window when it is done. Then run the following commands to create a conda environment. This notebook is tested with PyTorch 1.4 and OnnxRuntime 1.2.0.\n", + "\n", + "```console\n", + "conda create -n gpu_env python=3.6\n", + "conda activate gpu_env\n", + "conda install pytorch torchvision cudatoolkit=10.1 -c pytorch\n", + "pip install onnxruntime-gpu\n", + "pip install transformers==2.5.1\n", + "pip install wget psutil onnx pytz pandas py-cpuinfo py3nvml netron\n", + "conda install jupyter\n", + "jupyter notebook\n", + "```\n", + "\n", + "Onnxruntime-gpu need specified version of CUDA and cuDNN. You can find the corresponding version in [release note](https://github.com/microsoft/onnxruntime/releases). If the version is different from above cudatoolkit version, you have to install them separately, and add their bin directories to PATH environment variable (See [CUDA and cuDNN Path](#CUDA-and-cuDNN-Path) below)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 1. Load Pretrained Bert model ##" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We begin by downloading the SQuAD data file and store them in the specified location. " + ] + }, + { + "cell_type": "code", + "execution_count": 1, + "metadata": {}, + "outputs": [], + "source": [ + "import os\n", + "\n", + "cache_dir = \"./squad\"\n", + "if not os.path.exists(cache_dir):\n", + " os.makedirs(cache_dir)\n", + "\n", + "predict_file_url = \"https://rajpurkar.github.io/SQuAD-explorer/dataset/dev-v1.1.json\"\n", + "predict_file = os.path.join(cache_dir, \"dev-v1.1.json\")\n", + "if not os.path.exists(predict_file):\n", + " import wget\n", + " print(\"Start downloading predict file.\")\n", + " wget.download(predict_file_url, predict_file)\n", + " print(\"Predict file downloaded.\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's first define some constant variables." + ] + }, + { + "cell_type": "code", + "execution_count": 2, + "metadata": {}, + "outputs": [], + "source": [ + "# Whether allow overwriting existing ONNX model and download the latest script from GitHub\n", + "enable_overwrite = True\n", + "\n", + "# Total samples to inference, so that we can get average latency\n", + "total_samples = 1000\n", + "\n", + "# ONNX opset version: 10 or 11\n", + "opset_version=11" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Specify some model configuration variables." + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [], + "source": [ + "# For fine-tuned large model, the model name is \"bert-large-uncased-whole-word-masking-finetuned-squad\". Here we use bert-base for demo.\n", + "model_name_or_path = \"bert-base-cased\"\n", + "max_seq_length = 128\n", + "doc_stride = 128\n", + "max_query_length = 64" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Start to load model from pretrained. This step could take a few minutes. " + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "100%|██████████████████████████████████████████████████████████████████████████████████| 48/48 [00:04<00:00, 10.72it/s]\n", + "convert squad examples to features: 100%|█████████████████████████████████████████| 1000/1000 [00:08<00:00, 123.84it/s]\n", + "add example index and unique id: 100%|█████████████████████████████████████████| 1000/1000 [00:00<00:00, 500274.81it/s]\n" + ] + } + ], + "source": [ + "# The following code is adapted from HuggingFace transformers\n", + "# https://github.com/huggingface/transformers/blob/master/examples/run_squad.py\n", + "\n", + "from transformers import (BertConfig, BertForQuestionAnswering, BertTokenizer)\n", + "\n", + "# Load pretrained model and tokenizer\n", + "config_class, model_class, tokenizer_class = (BertConfig, BertForQuestionAnswering, BertTokenizer)\n", + "config = config_class.from_pretrained(model_name_or_path, cache_dir=cache_dir)\n", + "tokenizer = tokenizer_class.from_pretrained(model_name_or_path, do_lower_case=True, cache_dir=cache_dir)\n", + "model = model_class.from_pretrained(model_name_or_path,\n", + " from_tf=False,\n", + " config=config,\n", + " cache_dir=cache_dir)\n", + "# load some examples\n", + "from transformers.data.processors.squad import SquadV1Processor\n", + "\n", + "processor = SquadV1Processor()\n", + "examples = processor.get_dev_examples(None, filename=predict_file)\n", + "\n", + "from transformers import squad_convert_examples_to_features\n", + "features, dataset = squad_convert_examples_to_features( \n", + " examples=examples[:total_samples], # convert enough examples for this notebook\n", + " tokenizer=tokenizer,\n", + " max_seq_length=max_seq_length,\n", + " doc_stride=doc_stride,\n", + " max_query_length=max_query_length,\n", + " is_training=False,\n", + " return_dataset='pt'\n", + " )" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2. Export the loaded model ##\n", + "Once the model is loaded, we can export the loaded PyTorch model to ONNX." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Model exported at ./onnx\\bert-base-cased-squad_opset11.onnx\n" + ] + } + ], + "source": [ + "output_dir = \"./onnx\"\n", + "if not os.path.exists(output_dir):\n", + " os.makedirs(output_dir) \n", + "export_model_path = os.path.join(output_dir, 'bert-base-cased-squad_opset{}.onnx'.format(opset_version))\n", + "\n", + "import torch\n", + "use_gpu = torch.cuda.is_available()\n", + "device = torch.device(\"cuda\" if use_gpu else \"cpu\")\n", + "\n", + "# Get the first example data to run the model and export it to ONNX\n", + "data = dataset[0]\n", + "inputs = {\n", + " 'input_ids': data[0].to(device).reshape(1, max_seq_length),\n", + " 'attention_mask': data[1].to(device).reshape(1, max_seq_length),\n", + " 'token_type_ids': data[2].to(device).reshape(1, max_seq_length)\n", + "}\n", + "\n", + "# Set model to inference mode, which is required before exporting the model because some operators behave differently in \n", + "# inference and training mode.\n", + "model.eval()\n", + "model.to(device)\n", + "\n", + "if enable_overwrite or not os.path.exists(export_model_path):\n", + " with torch.no_grad():\n", + " symbolic_names = {0: 'batch_size', 1: 'max_seq_len'}\n", + " torch.onnx.export(model, # model being run\n", + " args=tuple(inputs.values()), # model input (or a tuple for multiple inputs)\n", + " f=export_model_path, # where to save the model (can be a file or file-like object)\n", + " opset_version=opset_version, # the ONNX version to export the model to\n", + " do_constant_folding=True, # whether to execute constant folding for optimization\n", + " input_names=['input_ids', # the model's input names\n", + " 'input_mask', \n", + " 'segment_ids'],\n", + " output_names=['start', 'end'], # the model's output names\n", + " dynamic_axes={'input_ids': symbolic_names, # variable length axes\n", + " 'input_mask' : symbolic_names,\n", + " 'segment_ids' : symbolic_names,\n", + " 'start' : symbolic_names,\n", + " 'end' : symbolic_names})\n", + " print(\"Model exported at \", export_model_path)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 3. PyTorch Inference ##\n", + "Use PyTorch to evaluate an example input for comparison purpose." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "PyTorch cuda Inference time = 16.54 ms\n" + ] + } + ], + "source": [ + "import time\n", + "\n", + "# Measure the latency. It is not accurate using Jupyter Notebook, it is recommended to use standalone python script.\n", + "latency = []\n", + "with torch.no_grad():\n", + " for i in range(total_samples):\n", + " data = dataset[i]\n", + " inputs = {\n", + " 'input_ids': data[0].to(device).reshape(1, max_seq_length),\n", + " 'attention_mask': data[1].to(device).reshape(1, max_seq_length),\n", + " 'token_type_ids': data[2].to(device).reshape(1, max_seq_length)\n", + " }\n", + " start = time.time()\n", + " outputs = model(**inputs)\n", + " latency.append(time.time() - start)\n", + "print(\"PyTorch {} Inference time = {} ms\".format(device.type, format(sum(latency) * 1000 / len(latency), '.2f')))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 4. Inference ONNX Model with ONNX Runtime ##\n", + "\n", + "### CUDA and cuDNN Path\n", + "onnxruntime-gpu has dependency on [CUDA](https://developer.nvidia.com/cuda-downloads) and [cuDNN](https://developer.nvidia.com/cudnn):\n", + "\n", + "* [onnxruntime-gpu v1.2.0](https://github.com/microsoft/onnxruntime/releases/tag/v1.2.0) requires CUDA Runtime 10.1.243 and CUDNN 7.6.5.32.\n", + "* [onnxruntime-gpu v1.0.0](https://github.com/microsoft/onnxruntime/releases/tag/v1.0.0) ~ v1.1.2 requires CUDA Runtime 10.0 and CUDNN 7.6.\n", + "\n", + "During installing PyTorch 1.4, we installed cudatoolkit 10.1.243 in this conda environment. That shall be good for onnxruntime-gpu 1.2.0 in Jupyter Notebook.\n", + "\n", + "If you use onnxruntime-gpu 1.0.0 ~ 1.1.2, you will have to install CUDA and CUDNN, then add them to path like the following:" + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [], + "source": [ + "# Change to True when onnxruntime (like onnxruntime-gpu 1.0.0 ~ 1.1.2) cannot be imported.\n", + "add_cuda_path = False\n", + "\n", + "if add_cuda_path:\n", + " # Add path of CUDA 10.0 and CUDNN 7.6 for onnxruntime-gpu 1.0.0 ~ 1.1.2\n", + " cuda_dir = 'D:/NVidia/CUDA/v10.0/bin'\n", + " cudnn_dir = 'D:/NVidia/CUDA/v10.0/bin'\n", + " if not (os.path.exists(cuda_dir) and os.path.exists(cudnn_dir)):\n", + " raise ValueError(\"Please specify correct path for CUDA and cuDNN. Otherwise onnxruntime cannot be imported.\")\n", + " else:\n", + " if cuda_dir == cudnn_dir:\n", + " os.environ[\"PATH\"] = cuda_dir + ';' + os.environ[\"PATH\"]\n", + " else:\n", + " os.environ[\"PATH\"] = cuda_dir + ';' + cudnn_dir + ';' + os.environ[\"PATH\"]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### OpenMP Environment Variable\n", + "\n", + "OpenMP environment variables are optional for GPU inference of standard Bert model. It has little performance impact on Bert model since most nodes are executed in GPU. \n", + "\n", + "You can find the best setting based on [Performance Test Tool](#Performance-Test-Tool) result in later part of this notebook.\n", + "\n", + "**Attention: Setting environment variables shall be done before importing onnxruntime**. Otherwise, they might not take effect." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "# Optional. You can change them according to Performance Test Tool result.\n", + "#os.environ[\"OMP_NUM_THREADS\"] = '1'\n", + "#os.environ[\"OMP_WAIT_POLICY\"] = 'PASSIVE'" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Now we are ready to inference the model with ONNX Runtime." + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "OnnxRuntime gpu Inference time = 3.72 ms\n" + ] + } + ], + "source": [ + "import psutil\n", + "import onnxruntime\n", + "import numpy\n", + "\n", + "assert 'CUDAExecutionProvider' in onnxruntime.get_available_providers()\n", + "device_name = 'gpu'\n", + "\n", + "sess_options = onnxruntime.SessionOptions()\n", + "\n", + "# Optional: store the optimized graph and view it using Netron to verify that model is fully optimized.\n", + "# Note that this will increase session creation time so enable it for debugging only.\n", + "sess_options.optimized_model_filepath = os.path.join(output_dir, \"optimized_model_{}.onnx\".format(device_name))\n", + "\n", + "# Please change the value according to best setting in Performance Test Tool result.\n", + "sess_options.intra_op_num_threads=psutil.cpu_count(logical=True)\n", + "\n", + "session = onnxruntime.InferenceSession(export_model_path, sess_options)\n", + "\n", + "latency = []\n", + "for i in range(total_samples):\n", + " data = dataset[i]\n", + " # Use contiguous array as input might improve performance\n", + " ort_inputs = {\n", + " 'input_ids': numpy.ascontiguousarray(data[0].cpu().reshape(1, max_seq_length).numpy()),\n", + " 'input_mask': numpy.ascontiguousarray(data[1].cpu().reshape(1, max_seq_length).numpy()),\n", + " 'segment_ids': numpy.ascontiguousarray(data[2].cpu().reshape(1, max_seq_length).numpy())\n", + " }\n", + " start = time.time()\n", + " ort_outputs = session.run(None, ort_inputs)\n", + " latency.append(time.time() - start)\n", + " \n", + "print(\"OnnxRuntime {} Inference time = {} ms\".format(device_name, format(sum(latency) * 1000 / len(latency), '.2f')))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can compare the output of PyTorch and ONNX Runtime. We can see some results are not close. It is because ONNX Runtime uses some approximation in CUDA optimization. Based on our evaluation on SQuAD data set, F1 score is on par for models before and after optimization." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "***** Verifying correctness *****\n", + "PyTorch and ONNX Runtime output 0 are close: True\n", + "maximum_diff=0.005700558423995972 average_diff=0.0005014391499571502\n", + "PyTorch and ONNX Runtime output 1 are close: True\n", + "maximum_diff=0.0023995935916900635 average_diff=0.00045171938836574554\n" + ] + } + ], + "source": [ + "print(\"***** Verifying correctness *****\")\n", + "for i in range(2): \n", + " print('PyTorch and ONNX Runtime output {} are close:'.format(i), numpy.allclose(ort_outputs[i], outputs[i].cpu(), rtol=1e-02, atol=1e-02))\n", + " diff = ort_outputs[i] - outputs[i].cpu().numpy()\n", + " max_diff = numpy.max(numpy.abs(diff))\n", + " avg_diff = numpy.average(numpy.abs(diff))\n", + " print(f'maximum_diff={max_diff} average_diff={avg_diff}')" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Inference with Actual Sequence Length\n", + "Note that ONNX model is exported using dynamic length axis. It is recommended to use actual sequence input without padding instead of fixed length input for best performance. Let's see how it can be applied to this model.\n", + "\n", + "From an example input below, we can see zero padding at the end of each sequence." + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "data": { + "text/plain": [ + "{'input_ids': tensor([[ 101, 1293, 1242, 2557, 1127, 1226, 1104, 1103, 3613, 16429,\n", + " 5235, 136, 102, 3613, 16429, 5988, 170, 107, 1353, 1671,\n", + " 1992, 1342, 107, 5235, 117, 1107, 1134, 1473, 3683, 3538,\n", + " 1125, 170, 1476, 118, 1248, 2595, 4086, 1714, 1104, 2965,\n", + " 15897, 1104, 3613, 16429, 119, 1473, 3683, 3538, 3222, 1149,\n", + " 2551, 1168, 23759, 1116, 1121, 1506, 1103, 10280, 2231, 1111,\n", + " 1103, 1714, 16355, 119, 102, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0]],\n", + " device='cuda:0'),\n", + " 'attention_mask': tensor([[1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", + " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", + " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0]], device='cuda:0'),\n", + " 'token_type_ids': tensor([[0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", + " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1,\n", + " 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0,\n", + " 0, 0, 0, 0, 0, 0, 0, 0]], device='cuda:0')}" + ] + }, + "execution_count": 11, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "# An example input (we can see padding). From attention_mask, we can deduce the actual length.\n", + "inputs" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "The original sequence length is 128. After removing paddings, the sequence length is reduced. Input with smaller sequence length need less computation, thus we can see there is improvement on inference latency. " + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Average length 101\n", + "OnnxRuntime gpu Inference time with actual sequence length = 3.44 ms\n" + ] + } + ], + "source": [ + "import statistics\n", + "\n", + "latency = []\n", + "lengths = []\n", + "for i in range(total_samples):\n", + " data = dataset[i]\n", + " # Instead of using fixed length (128), we can use actual sequence length (less than 128), which helps to get better performance.\n", + " actual_sequence_length = sum(data[1].numpy())\n", + " lengths.append(actual_sequence_length)\n", + " opt_inputs = {\n", + " 'input_ids': data[0].numpy()[:actual_sequence_length].reshape(1, actual_sequence_length),\n", + " 'input_mask': data[1].numpy()[:actual_sequence_length].reshape(1, actual_sequence_length),\n", + " 'segment_ids': data[2].numpy()[:actual_sequence_length].reshape(1, actual_sequence_length)\n", + " }\n", + " start = time.time()\n", + " opt_outputs = session.run(None, opt_inputs)\n", + " latency.append(time.time() - start)\n", + "print(\"Average length\", statistics.mean(lengths))\n", + "print(\"OnnxRuntime {} Inference time with actual sequence length = {} ms\".format(device_name, format(sum(latency) * 1000 / len(latency), '.2f')))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's compare the output and see whether the results are close.\n", + "\n", + "**Note**: Need end-to-end evaluation on performance and accuracy if you use this strategy." + ] + }, + { + "cell_type": "code", + "execution_count": 13, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "***** Comparing results with/without paddings *****\n", + "Output 0 are close: True\n", + "Output 1 are close: True\n" + ] + } + ], + "source": [ + "print(\"***** Comparing results with/without paddings *****\")\n", + "for i in range(2):\n", + " print('Output {} are close:'.format(i), numpy.allclose(opt_outputs[i], ort_outputs[i][:,:len(opt_outputs[i][0])], rtol=1e-03, atol=1e-03))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 5. Offline Optimization and Test Tools\n", + "\n", + "It is recommended to download the [OnnxRuntime Python Tools for BERT](https://github.com/microsoft/onnxruntime/tree/master/onnxruntime/python/tools/bert), and try them on the exported ONNX models. It could help verify whether the model is fully optimized, and get performance test results.\n", + "\n", + "### Download OnnxRuntime Python Tools for Bert\n", + "You may copy the whole [directory](https://github.com/microsoft/onnxruntime/tree/master/onnxruntime/python/tools/bert) to a sub-directory named bert_scripts for this notebook. The list of script files might need update if import error happens when you run some script." + ] + }, + { + "cell_type": "code", + "execution_count": 14, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "100% [..............................................................................] 15310 / 15310Downloaded bert_perf_test.py\n", + "100% [................................................................................] 9571 / 9571Downloaded bert_test_data.py\n", + "100% [................................................................................] 7272 / 7272Downloaded compare_bert_results.py\n", + "100% [..............................................................................] 44905 / 44905Downloaded BertOnnxModel.py\n", + "100% [..............................................................................] 21565 / 21565Downloaded BertOnnxModelKeras.py\n", + "100% [..............................................................................] 26114 / 26114Downloaded BertOnnxModelTF.py\n", + "100% [..............................................................................] 22773 / 22773Downloaded OnnxModel.py\n", + "100% [................................................................................] 7795 / 7795Downloaded bert_model_optimization.py\n", + "100% [................................................................................] 5885 / 5885Downloaded MachineInfo.py\n" + ] + } + ], + "source": [ + "import os\n", + "import wget\n", + "\n", + "url_prfix = \"https://raw.githubusercontent.com/microsoft/onnxruntime/master/onnxruntime/python/tools/bert/\"\n", + "script_files = ['bert_perf_test.py', 'bert_test_data.py', 'compare_bert_results.py', 'BertOnnxModel.py', 'BertOnnxModelKeras.py', 'BertOnnxModelTF.py', 'OnnxModel.py', 'bert_model_optimization.py', 'MachineInfo.py']\n", + "\n", + "script_dir = './bert_scripts'\n", + "if not os.path.exists(script_dir):\n", + " os.makedirs(script_dir)\n", + "\n", + "for filename in script_files:\n", + " target_file = os.path.join(script_dir, filename)\n", + " if enable_overwrite and os.path.exists(target_file):\n", + " os.remove(target_file)\n", + " if not os.path.exists(target_file):\n", + " wget.download(url_prfix + filename, target_file)\n", + " print(\"Downloaded\", filename)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### BERT Optimization Script\n", + "\n", + "Sometime, some optimization of OnnxRuntime cannot be applied to a Bert model due to different reasons:\n", + "* A new subgraph pattern is exported, which is not covered by the onnxruntime version users are using. For example, Gelu from PyTorch 1.4 is not fused by OnnxRuntime 1.1.2 (Note: it is covered in OnnxRuntime v1.2.0).\n", + "* The exported model uses dynamic axis. That impacts shape inference. Without enough shape information, some optimization cannot be applied due to the constraint on the input shape.\n", + "* Some optimization are not supported by OnnxRuntime, but it is feasible in offline script. Like changing input tensor type from int64 to int32 to avoid extra Cast nodes, or converting model to float16 to achieve better performance in V100 or T4 GPU.\n", + "\n", + "We have python script **bert_model_optimization.py**, which is flexible in graph pattern matching and model conversions to tackle these problems.\n", + "\n", + "In below example, we can see that the tool provide an extra optimization - SkipLayerNormalization and bias (Add) are not fused in OnnxRuntime due to shape inference.\n", + "\n", + "The tool will tell whether a model is fully optimized or not. If not, that means you might need change the script to handle some new subgraph patern." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Float32 Model\n", + "Let us optimize the ONNX model using the script. The first example will output model with float32 to store weights. This is the choice for most GPUs without Tensor Core.\n", + "\n", + "If your GPU (like V100 or T4) has Tensor Core, jump to [Float16 Model](#6.-Model-Optimization-with-Float16) section since that will give you better performance than Float32 model." + ] + }, + { + "cell_type": "code", + "execution_count": 15, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "bert_model_optimization.py: Save optimized model by onnxruntime to ./onnx\\bert-base-cased-squad_opset11_ort_gpu.onnx\n", + "bert_model_optimization.py: Use OnnxRuntime to optimize and save the optimized model to ./onnx\\bert-base-cased-squad_opset11_ort_gpu.onnx\n", + " BertOnnxModel.py: Fused LayerNormalization count: 0\n", + " BertOnnxModel.py: Fused Reshape count:0\n", + " BertOnnxModel.py: Fused SkipLayerNormalization count: 24\n", + " BertOnnxModel.py: Fused Attention count:0\n", + " BertOnnxModel.py: skip embed layer fusion since mask input is not found\n", + " BertOnnxModel.py: Fused SkipLayerNormalization with Bias count:24\n", + " BertOnnxModel.py: opset verion: 11\n", + " OnnxModel.py: Output model to ./onnx/bert-base-cased-squad_opt_gpu_fp32.onnx\n", + " BertOnnxModel.py: EmbedLayer=1, Attention=12, Gelu=12, LayerNormalization=24, Succesful=True\n", + "bert_model_optimization.py: The output model is fully optimized.\n" + ] + } + ], + "source": [ + "GPU_OPTION = '--gpu_only' if use_gpu else ''\n", + "optimized_fp32_model_path = './onnx/bert-base-cased-squad_opt_{}_fp32.onnx'.format('gpu' if use_gpu else 'cpu')\n", + "%run ./bert_scripts/bert_model_optimization.py --input $export_model_path --output $optimized_fp32_model_path $GPU_OPTION --input_int32" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Optimized Graph\n", + "We can open the optimized model using [Netron](https://github.com/lutzroeder/netron) to visualize.\n", + "\n", + "The graph is like the following:\n", + "\n", + "\n", + "Sometime, optimized graph is slightly different. For example, FastGelu is replaced by BiasGelu for CPU inference; When the option --input_int32 is used, Cast nodes for inputs are removed." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [], + "source": [ + "import netron\n", + "\n", + "# change it to True if want to view the optimized model in browser\n", + "enable_netron = False\n", + "if enable_netron:\n", + " # If you encounter error \"access a socket in a way forbidden by its access permissions\", install Netron as standalone application instead.\n", + " netron.start(optimized_fp32_model_path)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Performance Test Tool\n", + "\n", + "The following will create 1000 random inputs of batch_size 1 and sequence length 128, then measure the average latency and throughput numbers.\n", + "\n", + "Note that the test uses fixed sequence length. If you use [dynamic sequence length](#Inference-with-Actual-Sequence-Length), actual performance depends on the distribution of sequence length.\n", + "\n", + "Note that this tool measures performance of inference using OnnxRuntime Python API. " + ] + }, + { + "cell_type": "code", + "execution_count": 17, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generating 1000 samples for batch_size=1 sequence_length=128\n", + "Extra latency for converting inputs to contiguous: 0.01 ms\n", + "Test summary is saved to onnx\\perf_results_GPU_B1_S128_20200313-152329.txt\n" + ] + } + ], + "source": [ + "GPU_OPTION = '--use_gpu' if use_gpu else ''\n", + "\n", + "%run ./bert_scripts/bert_perf_test.py --model $optimized_fp32_model_path --batch_size 1 --sequence_length 128 --samples 1000 --test_times 1 --inclusive --all $GPU_OPTION" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Let's load the summary file and take a look. Note that blank value in OMP_NUM_THREADS or OMP_WAIT_POLICY means the environment variable does not exist." + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Float32 model perf results from ./onnx\\perf_results_GPU_B1_S128_20200313-152329.txt\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Latency(ms)Latency_P50Latency_P75Latency_P90Latency_P95Latency_P99Throughput(QPS)intra_op_num_threadsOMP_NUM_THREADSOMP_WAIT_POLICYcontiguouswarmup
03.433.423.443.463.473.58291.5961ACTIVEFalseTrue
13.433.413.433.453.463.54291.5861PASSIVEFalseTrue
23.433.413.433.453.473.81291.2766ACTIVEFalseTrue
33.443.423.443.453.473.57290.920FalseTrue
43.443.433.453.463.473.55290.4616ACTIVETrueTrue
53.443.433.453.463.483.94290.3116PASSIVEFalseTrue
63.453.433.453.473.493.66290.2361ACTIVETrueTrue
73.453.433.443.463.483.56289.9016PASSIVETrueTrue
83.453.433.453.473.494.07289.7466PASSIVETrueTrue
93.463.453.463.483.503.58289.1716ACTIVEFalseTrue
103.463.453.463.483.503.68289.0166ACTIVETrueTrue
113.463.443.453.473.493.84288.9866PASSIVEFalseTrue
123.483.463.483.493.513.67287.4761PASSIVETrueTrue
133.533.523.533.553.583.80282.930TrueTrue
\n", + "
" + ], + "text/plain": [ + " Latency(ms) Latency_P50 Latency_P75 Latency_P90 Latency_P95 \\\n", + "0 3.43 3.42 3.44 3.46 3.47 \n", + "1 3.43 3.41 3.43 3.45 3.46 \n", + "2 3.43 3.41 3.43 3.45 3.47 \n", + "3 3.44 3.42 3.44 3.45 3.47 \n", + "4 3.44 3.43 3.45 3.46 3.47 \n", + "5 3.44 3.43 3.45 3.46 3.48 \n", + "6 3.45 3.43 3.45 3.47 3.49 \n", + "7 3.45 3.43 3.44 3.46 3.48 \n", + "8 3.45 3.43 3.45 3.47 3.49 \n", + "9 3.46 3.45 3.46 3.48 3.50 \n", + "10 3.46 3.45 3.46 3.48 3.50 \n", + "11 3.46 3.44 3.45 3.47 3.49 \n", + "12 3.48 3.46 3.48 3.49 3.51 \n", + "13 3.53 3.52 3.53 3.55 3.58 \n", + "\n", + " Latency_P99 Throughput(QPS) intra_op_num_threads OMP_NUM_THREADS \\\n", + "0 3.58 291.59 6 1 \n", + "1 3.54 291.58 6 1 \n", + "2 3.81 291.27 6 6 \n", + "3 3.57 290.92 0 \n", + "4 3.55 290.46 1 6 \n", + "5 3.94 290.31 1 6 \n", + "6 3.66 290.23 6 1 \n", + "7 3.56 289.90 1 6 \n", + "8 4.07 289.74 6 6 \n", + "9 3.58 289.17 1 6 \n", + "10 3.68 289.01 6 6 \n", + "11 3.84 288.98 6 6 \n", + "12 3.67 287.47 6 1 \n", + "13 3.80 282.93 0 \n", + "\n", + " OMP_WAIT_POLICY contiguous warmup \n", + "0 ACTIVE False True \n", + "1 PASSIVE False True \n", + "2 ACTIVE False True \n", + "3 False True \n", + "4 ACTIVE True True \n", + "5 PASSIVE False True \n", + "6 ACTIVE True True \n", + "7 PASSIVE True True \n", + "8 PASSIVE True True \n", + "9 ACTIVE False True \n", + "10 ACTIVE True True \n", + "11 PASSIVE False True \n", + "12 PASSIVE True True \n", + "13 True True " + ] + }, + "execution_count": 18, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "environment import os\n", + "import glob \n", + "import pandas\n", + "latest_result_file = max(glob.glob(\"./onnx/perf_results_GPU_B1_S128_*.txt\"), key=os.path.getmtime)\n", + "result_data = pandas.read_table(latest_result_file, converters={'OMP_NUM_THREADS': str, 'OMP_WAIT_POLICY':str})\n", + "print(\"Float32 model perf results from\", latest_result_file)\n", + "# Remove some columns that have same values for all rows.\n", + "columns_to_remove = ['model', 'graph_optimization_level', 'batch_size', 'sequence_length', 'test_cases', 'test_times', 'use_gpu']\n", + "result_data.drop(columns_to_remove, axis=1, inplace=True)\n", + "result_data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "From above result, we can see that latency is very close for different settings. The default setting (intra_op_num_threads=0, OMP_NUM_THREADS and OMP_WAIT_POLICY does not exist) performs almost on par with the best setting. \n", + "\n", + "### Model Results Comparison Tool\n", + "\n", + "When a BERT model is optimized, some approximation is used in calculation. If your BERT model has three inputs, a script compare_bert_results.py can be used to do a quick verification. The tool will generate some fake input data, and compare the inference outputs of the original and optimized models. If outputs are all close, it is safe to use the optimized model.\n", + "\n", + "For GPU inference, the absolute or relative difference is larger than those numbers of CPU inference. Note that slight difference in output will not impact final result. We did end-to-end evaluation using SQuAD data set using a fine-tuned squad model, and F1 score is almost the same before/after optimization." + ] + }, + { + "cell_type": "code", + "execution_count": 19, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "1 out of 100 results not passed for thresholds (rtol=0.01, atol=0.01).\n", + "maximum absolute difference=0.033498868346214294\n", + "maximum relative difference=20.89048957824707\n" + ] + } + ], + "source": [ + "%run ./bert_scripts/compare_bert_results.py --baseline_model $export_model_path --optimized_model $optimized_fp32_model_path --batch_size 1 --sequence_length 128 --samples 100 --rtol 0.01 --atol 0.01 $GPU_OPTION" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 6. Model Optimization with Float16\n", + "\n", + "The bert_model_optimization.py script have an option **--float16** to convert model to use float16 to store weights. After the conversion, it could be faster to run in GPU with tensor cores like V100 or T4.\n", + "\n", + "Let's run tools to measure the performance on V100. The results show significant performance improvement: latency is about 3.4 ms for float32 model, and 1.8 ms for float16 model." + ] + }, + { + "cell_type": "code", + "execution_count": 20, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "bert_model_optimization.py: Save optimized model by onnxruntime to ./onnx\\bert-base-cased-squad_opset11_ort_gpu.onnx\n", + "bert_model_optimization.py: Use OnnxRuntime to optimize and save the optimized model to ./onnx\\bert-base-cased-squad_opset11_ort_gpu.onnx\n", + " BertOnnxModel.py: Fused LayerNormalization count: 0\n", + " BertOnnxModel.py: Fused Reshape count:0\n", + " BertOnnxModel.py: Fused SkipLayerNormalization count: 24\n", + " BertOnnxModel.py: Fused Attention count:0\n", + " BertOnnxModel.py: skip embed layer fusion since mask input is not found\n", + " BertOnnxModel.py: Fused SkipLayerNormalization with Bias count:24\n", + " BertOnnxModel.py: opset verion: 11\n", + " OnnxModel.py: Output model to ./onnx/bert-base-cased-squad_opt_gpu_fp16.onnx\n", + " BertOnnxModel.py: EmbedLayer=1, Attention=12, Gelu=12, LayerNormalization=24, Succesful=True\n", + "bert_model_optimization.py: The output model is fully optimized.\n" + ] + } + ], + "source": [ + "GPU_OPTION = '--gpu_only' if use_gpu else ''\n", + "optimized_fp16_model_path = './onnx/bert-base-cased-squad_opt_{}_fp16.onnx'.format('gpu' if use_gpu else 'cpu')\n", + "%run ./bert_scripts/bert_model_optimization.py --input $export_model_path --output $optimized_fp16_model_path $GPU_OPTION --float16 --input_int32" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generating 1000 samples for batch_size=1 sequence_length=128\n", + "Extra latency for converting inputs to contiguous: 0.00 ms\n", + "Test summary is saved to onnx\\perf_results_GPU_B1_S128_20200313-152527.txt\n" + ] + } + ], + "source": [ + "GPU_OPTION = '--use_gpu' if use_gpu else ''\n", + "%run ./bert_scripts/bert_perf_test.py --model $optimized_fp16_model_path --batch_size 1 --sequence_length 128 --samples 1000 --test_times 1 --inclusive --all $GPU_OPTION" + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Float32 model perf results from ./onnx\\perf_results_GPU_B1_S128_20200313-152527.txt\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Latency(ms)Latency_P50Latency_P75Latency_P90Latency_P95Latency_P99Throughput(QPS)intra_op_num_threadsOMP_NUM_THREADSOMP_WAIT_POLICYcontiguouswarmup
01.841.831.831.851.872.00544.9461PASSIVEFalseTrue
11.851.821.831.851.882.11541.890FalseTrue
21.851.831.841.871.902.12541.8461ACTIVEFalseTrue
31.851.841.851.871.892.12541.6416ACTIVETrueTrue
41.851.831.841.861.892.09541.610TrueTrue
51.851.831.841.861.902.07541.0066ACTIVEFalseTrue
61.851.841.841.861.892.18540.8116PASSIVETrueTrue
71.851.831.841.861.892.08540.3861ACTIVETrueTrue
81.851.831.841.851.882.16540.0116ACTIVEFalseTrue
91.861.831.841.871.902.14538.6716PASSIVEFalseTrue
101.861.841.851.871.902.24538.1066PASSIVETrueTrue
111.861.841.851.871.892.29537.3561PASSIVETrueTrue
121.881.851.861.881.932.31533.3266ACTIVETrueTrue
131.881.841.871.962.112.47530.8766PASSIVEFalseTrue
\n", + "
" + ], + "text/plain": [ + " Latency(ms) Latency_P50 Latency_P75 Latency_P90 Latency_P95 \\\n", + "0 1.84 1.83 1.83 1.85 1.87 \n", + "1 1.85 1.82 1.83 1.85 1.88 \n", + "2 1.85 1.83 1.84 1.87 1.90 \n", + "3 1.85 1.84 1.85 1.87 1.89 \n", + "4 1.85 1.83 1.84 1.86 1.89 \n", + "5 1.85 1.83 1.84 1.86 1.90 \n", + "6 1.85 1.84 1.84 1.86 1.89 \n", + "7 1.85 1.83 1.84 1.86 1.89 \n", + "8 1.85 1.83 1.84 1.85 1.88 \n", + "9 1.86 1.83 1.84 1.87 1.90 \n", + "10 1.86 1.84 1.85 1.87 1.90 \n", + "11 1.86 1.84 1.85 1.87 1.89 \n", + "12 1.88 1.85 1.86 1.88 1.93 \n", + "13 1.88 1.84 1.87 1.96 2.11 \n", + "\n", + " Latency_P99 Throughput(QPS) intra_op_num_threads OMP_NUM_THREADS \\\n", + "0 2.00 544.94 6 1 \n", + "1 2.11 541.89 0 \n", + "2 2.12 541.84 6 1 \n", + "3 2.12 541.64 1 6 \n", + "4 2.09 541.61 0 \n", + "5 2.07 541.00 6 6 \n", + "6 2.18 540.81 1 6 \n", + "7 2.08 540.38 6 1 \n", + "8 2.16 540.01 1 6 \n", + "9 2.14 538.67 1 6 \n", + "10 2.24 538.10 6 6 \n", + "11 2.29 537.35 6 1 \n", + "12 2.31 533.32 6 6 \n", + "13 2.47 530.87 6 6 \n", + "\n", + " OMP_WAIT_POLICY contiguous warmup \n", + "0 PASSIVE False True \n", + "1 False True \n", + "2 ACTIVE False True \n", + "3 ACTIVE True True \n", + "4 True True \n", + "5 ACTIVE False True \n", + "6 PASSIVE True True \n", + "7 ACTIVE True True \n", + "8 ACTIVE False True \n", + "9 PASSIVE False True \n", + "10 PASSIVE True True \n", + "11 PASSIVE True True \n", + "12 ACTIVE True True \n", + "13 PASSIVE False True " + ] + }, + "execution_count": 22, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import os\n", + "import glob \n", + "import pandas\n", + "latest_result_file = max(glob.glob(\"./onnx/perf_results_GPU_B1_S128_*.txt\"), key=os.path.getmtime)\n", + "result_data = pandas.read_table(latest_result_file, converters={'OMP_NUM_THREADS': str, 'OMP_WAIT_POLICY':str})\n", + "print(\"Float32 model perf results from\", latest_result_file)\n", + "# Remove some columns that have same values for all rows.\n", + "columns_to_remove = ['model', 'graph_optimization_level', 'batch_size', 'sequence_length', 'test_cases', 'test_times', 'use_gpu']\n", + "result_data.drop(columns_to_remove, axis=1, inplace=True)\n", + "result_data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### Throughput Tuning\n", + "\n", + "Some application need best throughput under some constraint on latency. This can be done by testing performance of different batch sizes. The tool could help on this.\n", + "\n", + "Here is an example that check the performance of multiple batch sizes (1, 2, 4, 8, 16, 32 and 64) using default settings." + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Generating 1000 samples for batch_size=32 sequence_length=128\n", + "Generating 1000 samples for batch_size=1 sequence_length=128\n", + "Generating 1000 samples for batch_size=2 sequence_length=128\n", + "Generating 1000 samples for batch_size=64 sequence_length=128\n", + "Generating 1000 samples for batch_size=4 sequence_length=128\n", + "Generating 1000 samples for batch_size=8 sequence_length=128\n", + "Generating 1000 samples for batch_size=16 sequence_length=128\n", + "Test summary is saved to onnx\\perf_results_GPU_B1-2-4-8-16-32-64_S128_20200313-153014.txt\n" + ] + } + ], + "source": [ + "GPU_OPTION = '--use_gpu' if use_gpu else ''\n", + "%run ./bert_scripts/bert_perf_test.py --model $optimized_fp16_model_path --batch_size 1 2 4 8 16 32 64 --sequence_length 128 --samples 1000 --test_times 1 --inclusive $GPU_OPTION" + ] + }, + { + "cell_type": "code", + "execution_count": 24, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Float16 model summary from ./onnx\\perf_results_GPU_B1-2-4-8-16-32-64_S128_20200313-153014.txt\n" + ] + }, + { + "data": { + "text/html": [ + "
\n", + "\n", + "\n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + " \n", + "
Latency(ms)Latency_P50Latency_P75Latency_P90Latency_P95Latency_P99Throughput(QPS)batch_size
01.841.821.841.851.872.09544.951
32.192.182.192.202.222.40914.562
72.892.872.892.902.923.041383.384
114.794.764.784.804.845.191671.428
128.168.158.178.208.228.351960.2116
1514.6914.6614.7014.7514.8014.972178.4932
1828.4328.4228.5028.6128.6828.782251.3664
\n", + "
" + ], + "text/plain": [ + " Latency(ms) Latency_P50 Latency_P75 Latency_P90 Latency_P95 \\\n", + "0 1.84 1.82 1.84 1.85 1.87 \n", + "3 2.19 2.18 2.19 2.20 2.22 \n", + "7 2.89 2.87 2.89 2.90 2.92 \n", + "11 4.79 4.76 4.78 4.80 4.84 \n", + "12 8.16 8.15 8.17 8.20 8.22 \n", + "15 14.69 14.66 14.70 14.75 14.80 \n", + "18 28.43 28.42 28.50 28.61 28.68 \n", + "\n", + " Latency_P99 Throughput(QPS) batch_size \n", + "0 2.09 544.95 1 \n", + "3 2.40 914.56 2 \n", + "7 3.04 1383.38 4 \n", + "11 5.19 1671.42 8 \n", + "12 8.35 1960.21 16 \n", + "15 14.97 2178.49 32 \n", + "18 28.78 2251.36 64 " + ] + }, + "execution_count": 24, + "metadata": {}, + "output_type": "execute_result" + } + ], + "source": [ + "import os\n", + "import glob \n", + "import pandas\n", + "latest_result_file = max(glob.glob(\"./onnx/perf_results_*.txt\"), key=os.path.getmtime)\n", + "result_data = pandas.read_table(latest_result_file, converters={'OMP_NUM_THREADS': str, 'OMP_WAIT_POLICY':str})\n", + "print(\"Float16 model summary from\", latest_result_file)\n", + "columns_to_remove = ['model', 'graph_optimization_level', 'test_cases', 'test_times', 'use_gpu', 'warmup', 'sequence_length']\n", + "\n", + "# Set it True to see all rows. Here we only see one setting of intra_op_num_threads==0 and no OMP environment variables\n", + "show_all_rows = False\n", + "if not show_all_rows:\n", + " result_data = result_data.loc[result_data['intra_op_num_threads']==0]\n", + " columns_to_remove.extend(['intra_op_num_threads', 'OMP_NUM_THREADS', 'OMP_WAIT_POLICY', 'contiguous'])\n", + "result_data.drop(columns_to_remove, axis=1, inplace=True)\n", + "result_data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 7. Additional Info\n", + "\n", + "Note that running Jupyter Notebook has slight impact on performance result since Jupyter Notebook is using system resources like CPU etc. You can close Jupyter Notebook and other applications, then run the performance test in a console to get more accurate performance numbers.\n", + "\n", + "[OnnxRuntime C API](https://github.com/microsoft/onnxruntime/blob/master/docs/C_API.md) could get slightly better performance than python API. If you use C API in inference, you can use OnnxRuntime_Perf_Test.exe built from source to measure performance instead.\n", + "\n", + "Here is the machine configuration that generated the above results. You might get slower or faster result according to your hardware." + ] + }, + { + "cell_type": "code", + "execution_count": 25, + "metadata": { + "scrolled": true + }, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"gpu\": {\n", + " \"driver_version\": \"419.69\",\n", + " \"devices\": [\n", + " {\n", + " \"memory_total\": 17048272896,\n", + " \"memory_available\": 12573147136,\n", + " \"name\": \"Tesla V100-PCIE-16GB\"\n", + " }\n", + " ]\n", + " },\n", + " \"cpu\": {\n", + " \"brand\": \"Intel(R) Xeon(R) CPU E5-2690 v4 @ 2.60GHz\",\n", + " \"cores\": 6,\n", + " \"logical_cores\": 6,\n", + " \"hz\": \"2.5940 GHz\",\n", + " \"l2_cache\": \"64\",\n", + " \"l3_cache\": \"0 KB\",\n", + " \"processor\": \"Intel64 Family 6 Model 79 Stepping 1, GenuineIntel\"\n", + " },\n", + " \"memory\": {\n", + " \"total\": 120258613248,\n", + " \"available\": 104351649792\n", + " },\n", + " \"python\": \"3.6.10.final.0 (64 bit)\",\n", + " \"os\": \"Windows-10-10.0.14393-SP0\",\n", + " \"onnxruntime\": {\n", + " \"version\": \"1.2.0\",\n", + " \"support_gpu\": true\n", + " },\n", + " \"pytorch\": {\n", + " \"version\": \"1.4.0\",\n", + " \"support_gpu\": true\n", + " },\n", + " \"tensorflow\": null\n", + "}\n" + ] + } + ], + "source": [ + "%run ./bert_scripts/MachineInfo.py --silent" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "torch14_gpu", + "language": "python", + "name": "torch14_gpu" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.6.10" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} diff --git a/onnxruntime/python/tools/bert/notebooks/images/optimized_bert_gpu.png b/onnxruntime/python/tools/bert/notebooks/images/optimized_bert_gpu.png new file mode 100644 index 0000000000..5113d1ed38 Binary files /dev/null and b/onnxruntime/python/tools/bert/notebooks/images/optimized_bert_gpu.png differ