mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-24 19:43:35 +00:00
Add Benchmark of GPT2 CPU inference (#3351)
* Add benchmark script and notebook for GPT2 * Update Reshape fusion for GPT2 model * Add opt_level option for bert_model_optimization to disable onnxruntime by setting --opt_level 0 * Fix keras optimization
This commit is contained in:
Tianlei Wu
GitHub
parent
ace741680d
commit
ecbacd7d79
5 changed files with 1155 additions and 13 deletions
|
|
@ -596,10 +596,22 @@ class BertOnnxModel(OnnxModel):
|
|||
if len(concat_node.input) == 3 and self.get_initializer(concat_node.input[2]) is None:
|
||||
path2 = self.match_parent_path(concat_node, ['Unsqueeze', 'Mul', 'Gather', 'Shape'], [2, 0, 0, 0],
|
||||
output_name_to_node)
|
||||
if path2 is None:
|
||||
path2 = self.match_parent_path(
|
||||
concat_node, ['Unsqueeze', 'Mul', 'Squeeze', 'Slice', 'Shape'], [2, 0, 0, 0, 0],
|
||||
output_name_to_node) # GPT2 exported by PyTorch 1.4 with opset_version=11
|
||||
if path2 is None:
|
||||
continue
|
||||
|
||||
path3 = self.match_parent_path(concat_node, ['Unsqueeze', 'Mul', 'Gather', 'Shape'], [2, 0, 1, 0],
|
||||
output_name_to_node)
|
||||
if path2 is None or path3 is None:
|
||||
continue
|
||||
if path3 is None:
|
||||
path3 = self.match_parent_path(
|
||||
concat_node, ['Unsqueeze', 'Mul', 'Squeeze', 'Slice', 'Shape'], [2, 0, 1, 0, 0],
|
||||
output_name_to_node) # GPT2 exported by PyTorch 1.4 with opset_version=11
|
||||
if path3 is None:
|
||||
continue
|
||||
|
||||
shape_nodes.extend([path2[-1], path3[-1]])
|
||||
shape.append(-1)
|
||||
elif (len(concat_node.input) > 2):
|
||||
|
|
@ -613,11 +625,18 @@ class BertOnnxModel(OnnxModel):
|
|||
shape.append(concat_value)
|
||||
|
||||
if len(concat_node.input) == 4 and self.get_initializer(concat_node.input[3]) is None:
|
||||
if -1 in shape:
|
||||
continue
|
||||
|
||||
path2 = self.match_parent_path(concat_node, ['Unsqueeze', 'Div', 'Gather', 'Shape'], [3, 0, 0, 0],
|
||||
output_name_to_node)
|
||||
if path2 is None:
|
||||
path2 = self.match_parent_path(
|
||||
concat_node, ['Unsqueeze', 'Div', 'Squeeze', 'Slice', 'Shape'], [3, 0, 0, 0, 0],
|
||||
output_name_to_node) # GPT2 exported by PyTorch 1.4 with opset_version=11
|
||||
if path2 is None:
|
||||
continue
|
||||
shape_nodes.extend([path2[-1]])
|
||||
if path2 is None or -1 in shape:
|
||||
continue
|
||||
shape.append(-1)
|
||||
elif (len(concat_node.input) > 3):
|
||||
concat_3 = self.get_initializer(concat_node.input[3])
|
||||
|
|
@ -1078,6 +1097,6 @@ class BertOnnxModel(OnnxModel):
|
|||
layer_norm = op_count['LayerNormalization'] + op_count['SkipLayerNormalization']
|
||||
is_optimized = (embed > 0) and (attention > 0) and (attention == gelu) and (layer_norm >= 2 * attention)
|
||||
logger.info(
|
||||
f"EmbedLayer={embed}, Attention={attention}, Gelu={gelu}, LayerNormalization={layer_norm}, Succesful={is_optimized}"
|
||||
f"EmbedLayer={embed}, Attention={attention}, Gelu={gelu}, LayerNormalization={layer_norm}, Successful={is_optimized}"
|
||||
)
|
||||
return is_optimized
|
||||
|
|
|
|||
|
|
@ -174,7 +174,7 @@ class BertOnnxModelKeras(BertOnnxModelTF):
|
|||
reshape_nodes = self.get_nodes_by_op_type("Reshape")
|
||||
for reshape_node in reshape_nodes:
|
||||
parent = self.get_parent(reshape_node, 0)
|
||||
if parent is None or parent.op_type == "Reshape":
|
||||
if parent is not None and parent.op_type == "Reshape":
|
||||
reshape_node.input[0] = parent.input[0]
|
||||
count += 1
|
||||
|
||||
|
|
@ -346,7 +346,7 @@ class BertOnnxModelKeras(BertOnnxModelTF):
|
|||
for skiplayernorm_node in skiplayernorm_nodes:
|
||||
path = self.match_parent_path(
|
||||
skiplayernorm_node,
|
||||
['Add', 'Reshape', 'MatMul', 'Reshape', 'Gelu', 'Add', 'Reshape', 'MatMul', 'Reshape', 'SkipLayerNormalization']
|
||||
['Add', 'Reshape', 'MatMul', 'Reshape', 'Gelu', 'Add', 'Reshape', 'MatMul', 'Reshape', 'SkipLayerNormalization'],
|
||||
[None, 0, 0, 0, 0, 0, 0, 0, 0, 0]) # yapf: disable
|
||||
if path is None:
|
||||
continue
|
||||
|
|
|
|||
240
onnxruntime/python/tools/bert/benchmark_gpt2.py
Normal file
240
onnxruntime/python/tools/bert/benchmark_gpt2.py
Normal file
|
|
@ -0,0 +1,240 @@
|
|||
# -------------------------------------------------------------------------
|
||||
# Copyright (c) Microsoft Corporation. All rights reserved.
|
||||
# Licensed under the MIT License. See License.txt in the project root for
|
||||
# license information.
|
||||
# --------------------------------------------------------------------------
|
||||
import os
|
||||
import sys
|
||||
import numpy
|
||||
import time
|
||||
import psutil
|
||||
import argparse
|
||||
import logging
|
||||
import torch
|
||||
from transformers import GPT2Model, GPT2Tokenizer
|
||||
|
||||
logger = logging.getLogger('')
|
||||
|
||||
|
||||
def dump_environment():
|
||||
if "OMP_NUM_THREADS" in os.environ:
|
||||
logger.info("OMP_NUM_THREADS={}".format(os.environ["OMP_NUM_THREADS"]))
|
||||
else:
|
||||
logger.info("no environment variable of OMP_NUM_THREADS")
|
||||
|
||||
if "OMP_WAIT_POLICY" in os.environ:
|
||||
logger.info("OMP_WAIT_POLICY={}".format(os.environ["OMP_WAIT_POLICY"]))
|
||||
else:
|
||||
logger.info("no environment variable of OMP_WAIT_POLICY")
|
||||
|
||||
|
||||
def setup_environment(use_openmp=False):
|
||||
# ATTENTION: these environment variables must be set before importing onnxruntime.
|
||||
if use_openmp:
|
||||
os.environ["OMP_NUM_THREADS"] = str(psutil.cpu_count(logical=True))
|
||||
else:
|
||||
os.environ["OMP_NUM_THREADS"] = '1'
|
||||
|
||||
os.environ["OMP_WAIT_POLICY"] = 'ACTIVE'
|
||||
dump_environment()
|
||||
|
||||
|
||||
def pytorch_inference(model, input_ids, past=None, total_runs=100):
|
||||
latency = []
|
||||
with torch.no_grad():
|
||||
for _ in range(total_runs):
|
||||
start = time.time()
|
||||
outputs = model(
|
||||
input_ids=input_ids,
|
||||
past=past) #attention_mask=inputs['attention_mask'], token_type_ids=inputs['token_type_ids'],
|
||||
latency.append(time.time() - start)
|
||||
|
||||
logger.info("PyTorch Inference time = {} ms".format(format(sum(latency) * 1000 / len(latency), '.2f')))
|
||||
return outputs
|
||||
|
||||
|
||||
def onnxruntime_inference(ort_session, input_ids, past=None, total_runs=100):
|
||||
# Use contiguous array as input might improve performance.
|
||||
# You can check the results from performance test tool to see whether you need it.
|
||||
ort_inputs = {'input_ids': numpy.ascontiguousarray(input_ids.cpu().numpy())}
|
||||
|
||||
if past is not None:
|
||||
for i, past_i in enumerate(past):
|
||||
ort_inputs[f'past_{i}'] = numpy.ascontiguousarray(past[i].cpu().numpy())
|
||||
|
||||
latency = []
|
||||
for _ in range(total_runs):
|
||||
start = time.time()
|
||||
ort_outputs = ort_session.run(None, ort_inputs)
|
||||
latency.append(time.time() - start)
|
||||
|
||||
logger.info("OnnxRuntime Inference time = {} ms".format(format(sum(latency) * 1000 / len(latency), '.2f')))
|
||||
|
||||
return ort_outputs
|
||||
|
||||
|
||||
def inference(model, ort_session, input_ids, past=None, total_runs=100, verify_outputs=True):
|
||||
outputs = pytorch_inference(model, input_ids, past, total_runs)
|
||||
ort_outputs = onnxruntime_inference(ort_session, input_ids, past, total_runs)
|
||||
if verify_outputs:
|
||||
logger.info('PyTorch and OnnxRuntime output 0 (last_state) are close:'.format(0),
|
||||
numpy.allclose(ort_outputs[0], outputs[0].cpu(), rtol=1e-05, atol=1e-04))
|
||||
|
||||
for layer in range(model.config.n_layer):
|
||||
logger.info('PyTorch and OnnxRuntime layer {} state (present_{}) are close:'.format(layer, layer),
|
||||
numpy.allclose(ort_outputs[1 + layer], outputs[1][layer].cpu(), rtol=1e-05, atol=1e-04))
|
||||
|
||||
|
||||
def parse_arguments():
|
||||
parser = argparse.ArgumentParser()
|
||||
|
||||
parser.add_argument('--cache_dir', required=True, type=str, help="cache directory")
|
||||
|
||||
parser.add_argument('--output_dir', required=True, type=str, help="output onnx model directory")
|
||||
|
||||
parser.add_argument('--total_runs', required=False, type=int, help="total runs", default=100)
|
||||
|
||||
parser.add_argument('--enable_past_input', required=False, action='store_true')
|
||||
parser.set_defaults(enable_past_input=False)
|
||||
|
||||
parser.add_argument('--enable_optimization', required=False, action='store_true')
|
||||
parser.set_defaults(enable_optimization=False)
|
||||
|
||||
parser.add_argument('--verify_outputs', required=False, action='store_true')
|
||||
parser.set_defaults(verify_outputs=False)
|
||||
|
||||
parser.add_argument('--use_openmp', required=False, action='store_true')
|
||||
parser.set_defaults(use_openmp=False)
|
||||
|
||||
parser.add_argument('--verbose', required=False, action='store_true')
|
||||
parser.set_defaults(verbose=False)
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
return args
|
||||
|
||||
|
||||
def setup_logger(verbose=True):
|
||||
# output logging to stdout
|
||||
log_handler = logging.StreamHandler(sys.stdout)
|
||||
if verbose:
|
||||
log_handler.setFormatter(logging.Formatter('[%(filename)s:%(lineno)s - %(funcName)20s()] %(message)s'))
|
||||
logging_level = logging.DEBUG
|
||||
else:
|
||||
log_handler.setFormatter(logging.Formatter('%(filename)20s: %(message)s'))
|
||||
logging_level = logging.INFO
|
||||
log_handler.setLevel(logging_level)
|
||||
|
||||
# Avoid duplicated handlers when runing this script in multiple cells of Jupyter Notebook.
|
||||
if not logger.hasHandlers():
|
||||
logger.addHandler(log_handler)
|
||||
|
||||
logger.setLevel(logging_level)
|
||||
|
||||
|
||||
def main():
|
||||
args = parse_arguments()
|
||||
setup_logger(args.verbose)
|
||||
dump_environment()
|
||||
|
||||
enable_past_input = args.enable_past_input
|
||||
|
||||
cache_dir = args.cache_dir
|
||||
if not os.path.exists(cache_dir):
|
||||
os.makedirs(cache_dir)
|
||||
|
||||
output_dir = args.output_dir
|
||||
if not os.path.exists(output_dir):
|
||||
os.makedirs(output_dir)
|
||||
|
||||
model_class, tokenizer_class, model_name_or_path = (GPT2Model, GPT2Tokenizer, 'gpt2')
|
||||
tokenizer = tokenizer_class.from_pretrained(model_name_or_path, cache_dir=cache_dir)
|
||||
model = model_class.from_pretrained(model_name_or_path, cache_dir=cache_dir)
|
||||
model.eval().cpu()
|
||||
|
||||
inputs = tokenizer.encode_plus("Here is an example input for GPT2 model",
|
||||
add_special_tokens=True,
|
||||
return_tensors='pt')
|
||||
input_ids = inputs['input_ids']
|
||||
outputs = model(input_ids=input_ids, past=None)
|
||||
|
||||
num_layer = model.config.n_layer
|
||||
present_names = [f'present_{i}' for i in range(num_layer)]
|
||||
output_names = ["last_state"] + present_names
|
||||
|
||||
input_names = ['input_ids']
|
||||
dynamic_axes = {'input_ids': {0: 'batch_size', 1: 'seq_len'}, 'last_state': {0: 'batch_size', 1: 'seq_len'}}
|
||||
for name in present_names:
|
||||
dynamic_axes[name] = {1: 'batch_size', 3: 'seq_len'}
|
||||
|
||||
if enable_past_input:
|
||||
past_names = [f'past_{i}' for i in range(num_layer)]
|
||||
input_names = ['input_ids'] + past_names
|
||||
dummy_past = [torch.zeros(list(outputs[1][0].shape)) for _ in range(num_layer)]
|
||||
for name in past_names:
|
||||
dynamic_axes[name] = {1: 'batch_size', 3: 'seq_len'}
|
||||
export_inputs = (inputs['input_ids'], tuple(dummy_past))
|
||||
else:
|
||||
export_inputs = (inputs['input_ids'])
|
||||
|
||||
export_model_path = os.path.join(output_dir, 'gpt2_past{}.onnx'.format(int(enable_past_input)))
|
||||
|
||||
torch.onnx.export(model,
|
||||
args=export_inputs,
|
||||
f=export_model_path,
|
||||
input_names=input_names,
|
||||
output_names=output_names,
|
||||
dynamic_axes=dynamic_axes,
|
||||
opset_version=11,
|
||||
do_constant_folding=True,
|
||||
verbose=False)
|
||||
|
||||
# Let's run performance test on PyTorch before updating environment variable.
|
||||
past = dummy_past if enable_past_input else None
|
||||
outputs = pytorch_inference(model, input_ids, past, total_runs=args.total_runs)
|
||||
|
||||
# setup environment variables before importing onnxruntime.
|
||||
setup_environment(args.use_openmp)
|
||||
import onnxruntime
|
||||
|
||||
onnx_model_path = export_model_path
|
||||
if args.enable_optimization:
|
||||
from bert_model_optimization import optimize_model
|
||||
m = optimize_model(export_model_path,
|
||||
model_type='bert',
|
||||
gpu_only=False,
|
||||
num_heads=12,
|
||||
hidden_size=768,
|
||||
sequence_length=64,
|
||||
input_int32=False,
|
||||
float16=False,
|
||||
opt_level=0)
|
||||
onnx_model_path = os.path.join(output_dir, 'gpt2_past{}_optimized.onnx'.format(int(enable_past_input)))
|
||||
m.save_model_to_file(onnx_model_path)
|
||||
|
||||
if 'CUDAExecutionProvider' in onnxruntime.get_available_providers():
|
||||
logger.warning(
|
||||
"onnxruntime-gpu is not built with OpenMP. You might try onnxruntime package to test CPU inference.")
|
||||
|
||||
sess_options = onnxruntime.SessionOptions()
|
||||
|
||||
if args.use_openmp:
|
||||
sess_options.intra_op_num_threads = 1
|
||||
else:
|
||||
sess_options.intra_op_num_threads = psutil.cpu_count(logical=True)
|
||||
logger.info(f"session option: intra_op_num_threads={sess_options.intra_op_num_threads}")
|
||||
|
||||
logger.info(f"Start inferencing onnx model: {onnx_model_path}")
|
||||
session = onnxruntime.InferenceSession(onnx_model_path, sess_options, providers=['CPUExecutionProvider'])
|
||||
|
||||
ort_outputs = onnxruntime_inference(session, input_ids, past, args.total_runs)
|
||||
if args.verify_outputs:
|
||||
logger.info('PyTorch and OnnxRuntime output 0 (last_state) are close:'.format(0),
|
||||
numpy.allclose(ort_outputs[0], outputs[0].cpu(), rtol=1e-05, atol=1e-04))
|
||||
|
||||
for layer in range(model.config.n_layer):
|
||||
logger.info('PyTorch and OnnxRuntime layer {} state (present_{}) are close:'.format(layer, layer),
|
||||
numpy.allclose(ort_outputs[1 + layer], outputs[1][layer].cpu(), rtol=1e-05, atol=1e-04))
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
|
|
@ -44,7 +44,7 @@ MODEL_CLASSES = {
|
|||
}
|
||||
|
||||
|
||||
def optimize_by_onnxruntime(onnx_model_path, use_gpu, optimized_model_path=None):
|
||||
def optimize_by_onnxruntime(onnx_model_path, use_gpu, optimized_model_path=None, opt_level=99):
|
||||
"""
|
||||
Use onnxruntime package to optimize model. It could support models exported by PyTorch.
|
||||
|
||||
|
|
@ -63,7 +63,13 @@ def optimize_by_onnxruntime(onnx_model_path, use_gpu, optimized_model_path=None)
|
|||
return onnx_model_path
|
||||
|
||||
sess_options = onnxruntime.SessionOptions()
|
||||
sess_options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_EXTENDED
|
||||
if opt_level == 1:
|
||||
sess_options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_BASIC
|
||||
elif opt_level == 2:
|
||||
sess_options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_EXTENDED
|
||||
else:
|
||||
assert opt_level == 99
|
||||
sess_options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL
|
||||
|
||||
if optimized_model_path is None:
|
||||
path_prefix = onnx_model_path[:-5] #remove .onnx suffix
|
||||
|
|
@ -131,17 +137,32 @@ def parse_arguments():
|
|||
parser.add_argument('--verbose', required=False, action='store_true')
|
||||
parser.set_defaults(verbose=False)
|
||||
|
||||
parser.add_argument('--opt_level',
|
||||
required=False,
|
||||
type=int,
|
||||
choices=[0, 1, 2, 99],
|
||||
default=99,
|
||||
help="onnxruntime optimization level. 0 will disable onnxruntime.")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
return args
|
||||
|
||||
|
||||
def optimize_model(input, model_type, gpu_only, num_heads, hidden_size, sequence_length, input_int32, float16):
|
||||
def optimize_model(input,
|
||||
model_type,
|
||||
gpu_only,
|
||||
num_heads,
|
||||
hidden_size,
|
||||
sequence_length,
|
||||
input_int32,
|
||||
float16,
|
||||
opt_level=99):
|
||||
(optimizer_class, producer, run_onnxruntime) = MODEL_CLASSES[model_type]
|
||||
|
||||
input_model_path = input
|
||||
if run_onnxruntime:
|
||||
input_model_path = optimize_by_onnxruntime(input_model_path, gpu_only)
|
||||
if run_onnxruntime and opt_level > 0:
|
||||
input_model_path = optimize_by_onnxruntime(input_model_path, gpu_only, opt_level=opt_level)
|
||||
logger.info("Use OnnxRuntime to optimize and save the optimized model to {}".format(input_model_path))
|
||||
|
||||
model = ModelProto()
|
||||
|
|
@ -179,7 +200,7 @@ def main():
|
|||
logger.setLevel(logging_level)
|
||||
|
||||
bert_model = optimize_model(args.input, args.model_type, args.gpu_only, args.num_heads, args.hidden_size,
|
||||
args.sequence_length, args.input_int32, args.float16)
|
||||
args.sequence_length, args.input_int32, args.float16, args.opt_level)
|
||||
|
||||
bert_model.save_model_to_file(args.output)
|
||||
|
||||
|
|
|
|||
|
|
@ -0,0 +1,862 @@
|
|||
{
|
||||
"cells": [
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Copyright (c) Microsoft Corporation. All rights reserved. \n",
|
||||
"Licensed under the MIT License."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"# Inference PyTorch GPT2 Model with ONNX Runtime on CPU\n",
|
||||
"\n",
|
||||
"In this tutorial, you'll be introduced to how to load a GPT2 model from PyTorch, convert it to ONNX, and inference it using ONNX Runtime."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Prerequisites ##\n",
|
||||
"\n",
|
||||
"If you have Jupyter Notebook, you may directly run this notebook. We will use pip to install or upgrade [PyTorch](https://pytorch.org/), [OnnxRuntime](https://microsoft.github.io/onnxruntime/) and other required packages.\n",
|
||||
"\n",
|
||||
"Otherwise, you can setup a new environment. First, we install [AnaConda](https://www.anaconda.com/distribution/). Then open an AnaConda prompt window and run the following commands:\n",
|
||||
"\n",
|
||||
"```console\n",
|
||||
"conda create -n cpu_env python=3.6\n",
|
||||
"conda activate cpu_env\n",
|
||||
"\n",
|
||||
"conda install pytorch torchvision cpuonly -c pytorch\n",
|
||||
"pip install onnxruntime\n",
|
||||
"pip install transformers==2.5.1\n",
|
||||
"pip install onnx psutil pytz pandas py-cpuinfo py3nvml netron\n",
|
||||
"\n",
|
||||
"conda install jupyter\n",
|
||||
"jupyter notebook\n",
|
||||
"```\n",
|
||||
"The last command will launch Jupyter Notebook and we can open this notebook in browser to continue."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 1,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Enable pass state in input.\n",
|
||||
"enable_past_input = False"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 16,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"\n",
|
||||
"cache_dir = \"./gpt2\"\n",
|
||||
"if not os.path.exists(cache_dir):\n",
|
||||
" os.makedirs(cache_dir)\n",
|
||||
"\n",
|
||||
"output_dir = './gpt2_onnx'\n",
|
||||
"if not os.path.exists(output_dir):\n",
|
||||
" os.makedirs(output_dir)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Benchmark ##\n",
|
||||
"\n",
|
||||
"You will need git clone the onnxruntime repository like\n",
|
||||
"```console\n",
|
||||
"git clone https://github.com/microsoft/onnxruntime.git\n",
|
||||
"```\n",
|
||||
"Then update the bert_tools_dir according to the path in your machine."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 3,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
" benchmark_gpt2.py: no environment variable of OMP_NUM_THREADS\n",
|
||||
" benchmark_gpt2.py: no environment variable of OMP_WAIT_POLICY\n",
|
||||
"tokenization_utils.py: loading file https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-vocab.json from cache at ./gpt2\\f2808208f9bec2320371a9f5f891c184ae0b674ef866b79c58177067d15732dd.1512018be4ba4e8726e41b9145129dc30651ea4fec86aa61f4b9f40bf94eac71\n",
|
||||
"tokenization_utils.py: loading file https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-merges.txt from cache at ./gpt2\\d629f792e430b3c76a1291bb2766b0a047e36fae0588f9dbc1ae51decdff691b.70bec105b4158ed9a1747fea67a43f5dee97855c64d62b6ec3742f4cfdb5feda\n",
|
||||
"configuration_utils.py: loading configuration file https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-config.json from cache at ./gpt2\\4be02c5697d91738003fb1685c9872f284166aa32e061576bbe6aaeb95649fcf.699bbd1c449e9861456f359d6daa51bd523ac085b4b531ab0aad5a55d091e942\n",
|
||||
"configuration_utils.py: Model config GPT2Config {\n",
|
||||
" \"architectures\": [\n",
|
||||
" \"GPT2LMHeadModel\"\n",
|
||||
" ],\n",
|
||||
" \"attn_pdrop\": 0.1,\n",
|
||||
" \"bos_token_id\": null,\n",
|
||||
" \"do_sample\": false,\n",
|
||||
" \"embd_pdrop\": 0.1,\n",
|
||||
" \"eos_token_ids\": null,\n",
|
||||
" \"finetuning_task\": null,\n",
|
||||
" \"id2label\": {\n",
|
||||
" \"0\": \"LABEL_0\",\n",
|
||||
" \"1\": \"LABEL_1\"\n",
|
||||
" },\n",
|
||||
" \"initializer_range\": 0.02,\n",
|
||||
" \"is_decoder\": false,\n",
|
||||
" \"label2id\": {\n",
|
||||
" \"LABEL_0\": 0,\n",
|
||||
" \"LABEL_1\": 1\n",
|
||||
" },\n",
|
||||
" \"layer_norm_epsilon\": 1e-05,\n",
|
||||
" \"length_penalty\": 1.0,\n",
|
||||
" \"max_length\": 20,\n",
|
||||
" \"model_type\": \"gpt2\",\n",
|
||||
" \"n_ctx\": 1024,\n",
|
||||
" \"n_embd\": 768,\n",
|
||||
" \"n_head\": 12,\n",
|
||||
" \"n_layer\": 12,\n",
|
||||
" \"n_positions\": 1024,\n",
|
||||
" \"num_beams\": 1,\n",
|
||||
" \"num_labels\": 2,\n",
|
||||
" \"num_return_sequences\": 1,\n",
|
||||
" \"output_attentions\": false,\n",
|
||||
" \"output_hidden_states\": false,\n",
|
||||
" \"output_past\": true,\n",
|
||||
" \"pad_token_id\": null,\n",
|
||||
" \"pruned_heads\": {},\n",
|
||||
" \"repetition_penalty\": 1.0,\n",
|
||||
" \"resid_pdrop\": 0.1,\n",
|
||||
" \"summary_activation\": null,\n",
|
||||
" \"summary_first_dropout\": 0.1,\n",
|
||||
" \"summary_proj_to_labels\": true,\n",
|
||||
" \"summary_type\": \"cls_index\",\n",
|
||||
" \"summary_use_proj\": true,\n",
|
||||
" \"temperature\": 1.0,\n",
|
||||
" \"top_k\": 50,\n",
|
||||
" \"top_p\": 1.0,\n",
|
||||
" \"torchscript\": false,\n",
|
||||
" \"use_bfloat16\": false,\n",
|
||||
" \"vocab_size\": 50257\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
" modeling_utils.py: loading weights file https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-pytorch_model.bin from cache at ./gpt2\\4295d67f022061768f4adc386234dbdb781c814c39662dd1662221c309962c55.778cf36f5c4e5d94c8cd9cefcf2a580c8643570eb327f0d4a1f007fab2acbdf1\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"D:\\Anaconda3\\envs\\cpu_env\\lib\\site-packages\\transformers\\modeling_gpt2.py:143: TracerWarning: Converting a tensor to a Python float might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n",
|
||||
" w = w / math.sqrt(v.size(-1))\n",
|
||||
"D:\\Anaconda3\\envs\\cpu_env\\lib\\site-packages\\transformers\\modeling_gpt2.py:145: TracerWarning: Converting a tensor to a Python index might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n",
|
||||
" b = self.bias[:, :, ns - nd : ns, :ns]\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
" benchmark_gpt2.py: PyTorch Inference time = 36.77 ms\n",
|
||||
" benchmark_gpt2.py: OMP_NUM_THREADS=1\n",
|
||||
" benchmark_gpt2.py: OMP_WAIT_POLICY=ACTIVE\n",
|
||||
" BertOnnxModel.py: Fused LayerNormalization count: 25\n",
|
||||
" BertOnnxModel.py: Fused Gelu (FastGelu fits better) count: 12\n",
|
||||
" BertOnnxModel.py: Fused Reshape count:48\n",
|
||||
" BertOnnxModel.py: Fused SkipLayerNormalization count: 1\n",
|
||||
" OnnxModel.py: Removed unused constant nodes: 422\n",
|
||||
" BertOnnxModel.py: Fused Attention count:0\n",
|
||||
" BertOnnxModel.py: skip embed layer fusion since mask input is not found\n",
|
||||
" BertOnnxModel.py: opset verion: 11\n",
|
||||
" OnnxModel.py: Output model to ./gpt2_onnx\\gpt2_past0_optimized.onnx\n",
|
||||
" benchmark_gpt2.py: session option: intra_op_num_threads=12\n",
|
||||
" benchmark_gpt2.py: Start inferencing onnx model: ./gpt2_onnx\\gpt2_past0_optimized.onnx\n",
|
||||
" benchmark_gpt2.py: OnnxRuntime Inference time = 31.60 ms\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Assume you have git clone the repository of onnxruntime from github.\n",
|
||||
"bert_tools_dir = r'D:\\Git\\onnxruntime\\onnxruntime\\python\\tools\\bert'\n",
|
||||
"benchmark_script = os.path.join(bert_tools_dir, 'benchmark_gpt2.py')\n",
|
||||
"\n",
|
||||
"if enable_past_input:\n",
|
||||
" %run $benchmark_script --cache_dir $cache_dir --output_dir $output_dir --enable_optimization --enable_past_input\n",
|
||||
"else:\n",
|
||||
" %run $benchmark_script --cache_dir $cache_dir --output_dir $output_dir --enable_optimization"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"If you only need the benchmark results. You can skip the remaining parts.\n",
|
||||
"\n",
|
||||
"In the following, we will introduce the benchmark script.\n",
|
||||
"\n",
|
||||
"### Load pretrained model"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"tokenization_utils.py: loading file https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-vocab.json from cache at ./gpt2\\f2808208f9bec2320371a9f5f891c184ae0b674ef866b79c58177067d15732dd.1512018be4ba4e8726e41b9145129dc30651ea4fec86aa61f4b9f40bf94eac71\n",
|
||||
"tokenization_utils.py: loading file https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-merges.txt from cache at ./gpt2\\d629f792e430b3c76a1291bb2766b0a047e36fae0588f9dbc1ae51decdff691b.70bec105b4158ed9a1747fea67a43f5dee97855c64d62b6ec3742f4cfdb5feda\n",
|
||||
"configuration_utils.py: loading configuration file https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-config.json from cache at ./gpt2\\4be02c5697d91738003fb1685c9872f284166aa32e061576bbe6aaeb95649fcf.699bbd1c449e9861456f359d6daa51bd523ac085b4b531ab0aad5a55d091e942\n",
|
||||
"configuration_utils.py: Model config GPT2Config {\n",
|
||||
" \"architectures\": [\n",
|
||||
" \"GPT2LMHeadModel\"\n",
|
||||
" ],\n",
|
||||
" \"attn_pdrop\": 0.1,\n",
|
||||
" \"bos_token_id\": null,\n",
|
||||
" \"do_sample\": false,\n",
|
||||
" \"embd_pdrop\": 0.1,\n",
|
||||
" \"eos_token_ids\": null,\n",
|
||||
" \"finetuning_task\": null,\n",
|
||||
" \"id2label\": {\n",
|
||||
" \"0\": \"LABEL_0\",\n",
|
||||
" \"1\": \"LABEL_1\"\n",
|
||||
" },\n",
|
||||
" \"initializer_range\": 0.02,\n",
|
||||
" \"is_decoder\": false,\n",
|
||||
" \"label2id\": {\n",
|
||||
" \"LABEL_0\": 0,\n",
|
||||
" \"LABEL_1\": 1\n",
|
||||
" },\n",
|
||||
" \"layer_norm_epsilon\": 1e-05,\n",
|
||||
" \"length_penalty\": 1.0,\n",
|
||||
" \"max_length\": 20,\n",
|
||||
" \"model_type\": \"gpt2\",\n",
|
||||
" \"n_ctx\": 1024,\n",
|
||||
" \"n_embd\": 768,\n",
|
||||
" \"n_head\": 12,\n",
|
||||
" \"n_layer\": 12,\n",
|
||||
" \"n_positions\": 1024,\n",
|
||||
" \"num_beams\": 1,\n",
|
||||
" \"num_labels\": 2,\n",
|
||||
" \"num_return_sequences\": 1,\n",
|
||||
" \"output_attentions\": false,\n",
|
||||
" \"output_hidden_states\": false,\n",
|
||||
" \"output_past\": true,\n",
|
||||
" \"pad_token_id\": null,\n",
|
||||
" \"pruned_heads\": {},\n",
|
||||
" \"repetition_penalty\": 1.0,\n",
|
||||
" \"resid_pdrop\": 0.1,\n",
|
||||
" \"summary_activation\": null,\n",
|
||||
" \"summary_first_dropout\": 0.1,\n",
|
||||
" \"summary_proj_to_labels\": true,\n",
|
||||
" \"summary_type\": \"cls_index\",\n",
|
||||
" \"summary_use_proj\": true,\n",
|
||||
" \"temperature\": 1.0,\n",
|
||||
" \"top_k\": 50,\n",
|
||||
" \"top_p\": 1.0,\n",
|
||||
" \"torchscript\": false,\n",
|
||||
" \"use_bfloat16\": false,\n",
|
||||
" \"vocab_size\": 50257\n",
|
||||
"}\n",
|
||||
"\n",
|
||||
" modeling_utils.py: loading weights file https://s3.amazonaws.com/models.huggingface.co/bert/gpt2-pytorch_model.bin from cache at ./gpt2\\4295d67f022061768f4adc386234dbdb781c814c39662dd1662221c309962c55.778cf36f5c4e5d94c8cd9cefcf2a580c8643570eb327f0d4a1f007fab2acbdf1\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"GPT2Model(\n",
|
||||
" (wte): Embedding(50257, 768)\n",
|
||||
" (wpe): Embedding(1024, 768)\n",
|
||||
" (drop): Dropout(p=0.1, inplace=False)\n",
|
||||
" (h): ModuleList(\n",
|
||||
" (0): Block(\n",
|
||||
" (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
|
||||
" (attn): Attention(\n",
|
||||
" (c_attn): Conv1D()\n",
|
||||
" (c_proj): Conv1D()\n",
|
||||
" (attn_dropout): Dropout(p=0.1, inplace=False)\n",
|
||||
" (resid_dropout): Dropout(p=0.1, inplace=False)\n",
|
||||
" )\n",
|
||||
" (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
|
||||
" (mlp): MLP(\n",
|
||||
" (c_fc): Conv1D()\n",
|
||||
" (c_proj): Conv1D()\n",
|
||||
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
||||
" )\n",
|
||||
" )\n",
|
||||
" (1): Block(\n",
|
||||
" (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
|
||||
" (attn): Attention(\n",
|
||||
" (c_attn): Conv1D()\n",
|
||||
" (c_proj): Conv1D()\n",
|
||||
" (attn_dropout): Dropout(p=0.1, inplace=False)\n",
|
||||
" (resid_dropout): Dropout(p=0.1, inplace=False)\n",
|
||||
" )\n",
|
||||
" (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
|
||||
" (mlp): MLP(\n",
|
||||
" (c_fc): Conv1D()\n",
|
||||
" (c_proj): Conv1D()\n",
|
||||
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
||||
" )\n",
|
||||
" )\n",
|
||||
" (2): Block(\n",
|
||||
" (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
|
||||
" (attn): Attention(\n",
|
||||
" (c_attn): Conv1D()\n",
|
||||
" (c_proj): Conv1D()\n",
|
||||
" (attn_dropout): Dropout(p=0.1, inplace=False)\n",
|
||||
" (resid_dropout): Dropout(p=0.1, inplace=False)\n",
|
||||
" )\n",
|
||||
" (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
|
||||
" (mlp): MLP(\n",
|
||||
" (c_fc): Conv1D()\n",
|
||||
" (c_proj): Conv1D()\n",
|
||||
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
||||
" )\n",
|
||||
" )\n",
|
||||
" (3): Block(\n",
|
||||
" (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
|
||||
" (attn): Attention(\n",
|
||||
" (c_attn): Conv1D()\n",
|
||||
" (c_proj): Conv1D()\n",
|
||||
" (attn_dropout): Dropout(p=0.1, inplace=False)\n",
|
||||
" (resid_dropout): Dropout(p=0.1, inplace=False)\n",
|
||||
" )\n",
|
||||
" (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
|
||||
" (mlp): MLP(\n",
|
||||
" (c_fc): Conv1D()\n",
|
||||
" (c_proj): Conv1D()\n",
|
||||
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
||||
" )\n",
|
||||
" )\n",
|
||||
" (4): Block(\n",
|
||||
" (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
|
||||
" (attn): Attention(\n",
|
||||
" (c_attn): Conv1D()\n",
|
||||
" (c_proj): Conv1D()\n",
|
||||
" (attn_dropout): Dropout(p=0.1, inplace=False)\n",
|
||||
" (resid_dropout): Dropout(p=0.1, inplace=False)\n",
|
||||
" )\n",
|
||||
" (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
|
||||
" (mlp): MLP(\n",
|
||||
" (c_fc): Conv1D()\n",
|
||||
" (c_proj): Conv1D()\n",
|
||||
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
||||
" )\n",
|
||||
" )\n",
|
||||
" (5): Block(\n",
|
||||
" (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
|
||||
" (attn): Attention(\n",
|
||||
" (c_attn): Conv1D()\n",
|
||||
" (c_proj): Conv1D()\n",
|
||||
" (attn_dropout): Dropout(p=0.1, inplace=False)\n",
|
||||
" (resid_dropout): Dropout(p=0.1, inplace=False)\n",
|
||||
" )\n",
|
||||
" (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
|
||||
" (mlp): MLP(\n",
|
||||
" (c_fc): Conv1D()\n",
|
||||
" (c_proj): Conv1D()\n",
|
||||
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
||||
" )\n",
|
||||
" )\n",
|
||||
" (6): Block(\n",
|
||||
" (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
|
||||
" (attn): Attention(\n",
|
||||
" (c_attn): Conv1D()\n",
|
||||
" (c_proj): Conv1D()\n",
|
||||
" (attn_dropout): Dropout(p=0.1, inplace=False)\n",
|
||||
" (resid_dropout): Dropout(p=0.1, inplace=False)\n",
|
||||
" )\n",
|
||||
" (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
|
||||
" (mlp): MLP(\n",
|
||||
" (c_fc): Conv1D()\n",
|
||||
" (c_proj): Conv1D()\n",
|
||||
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
||||
" )\n",
|
||||
" )\n",
|
||||
" (7): Block(\n",
|
||||
" (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
|
||||
" (attn): Attention(\n",
|
||||
" (c_attn): Conv1D()\n",
|
||||
" (c_proj): Conv1D()\n",
|
||||
" (attn_dropout): Dropout(p=0.1, inplace=False)\n",
|
||||
" (resid_dropout): Dropout(p=0.1, inplace=False)\n",
|
||||
" )\n",
|
||||
" (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
|
||||
" (mlp): MLP(\n",
|
||||
" (c_fc): Conv1D()\n",
|
||||
" (c_proj): Conv1D()\n",
|
||||
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
||||
" )\n",
|
||||
" )\n",
|
||||
" (8): Block(\n",
|
||||
" (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
|
||||
" (attn): Attention(\n",
|
||||
" (c_attn): Conv1D()\n",
|
||||
" (c_proj): Conv1D()\n",
|
||||
" (attn_dropout): Dropout(p=0.1, inplace=False)\n",
|
||||
" (resid_dropout): Dropout(p=0.1, inplace=False)\n",
|
||||
" )\n",
|
||||
" (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
|
||||
" (mlp): MLP(\n",
|
||||
" (c_fc): Conv1D()\n",
|
||||
" (c_proj): Conv1D()\n",
|
||||
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
||||
" )\n",
|
||||
" )\n",
|
||||
" (9): Block(\n",
|
||||
" (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
|
||||
" (attn): Attention(\n",
|
||||
" (c_attn): Conv1D()\n",
|
||||
" (c_proj): Conv1D()\n",
|
||||
" (attn_dropout): Dropout(p=0.1, inplace=False)\n",
|
||||
" (resid_dropout): Dropout(p=0.1, inplace=False)\n",
|
||||
" )\n",
|
||||
" (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
|
||||
" (mlp): MLP(\n",
|
||||
" (c_fc): Conv1D()\n",
|
||||
" (c_proj): Conv1D()\n",
|
||||
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
||||
" )\n",
|
||||
" )\n",
|
||||
" (10): Block(\n",
|
||||
" (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
|
||||
" (attn): Attention(\n",
|
||||
" (c_attn): Conv1D()\n",
|
||||
" (c_proj): Conv1D()\n",
|
||||
" (attn_dropout): Dropout(p=0.1, inplace=False)\n",
|
||||
" (resid_dropout): Dropout(p=0.1, inplace=False)\n",
|
||||
" )\n",
|
||||
" (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
|
||||
" (mlp): MLP(\n",
|
||||
" (c_fc): Conv1D()\n",
|
||||
" (c_proj): Conv1D()\n",
|
||||
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
||||
" )\n",
|
||||
" )\n",
|
||||
" (11): Block(\n",
|
||||
" (ln_1): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
|
||||
" (attn): Attention(\n",
|
||||
" (c_attn): Conv1D()\n",
|
||||
" (c_proj): Conv1D()\n",
|
||||
" (attn_dropout): Dropout(p=0.1, inplace=False)\n",
|
||||
" (resid_dropout): Dropout(p=0.1, inplace=False)\n",
|
||||
" )\n",
|
||||
" (ln_2): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
|
||||
" (mlp): MLP(\n",
|
||||
" (c_fc): Conv1D()\n",
|
||||
" (c_proj): Conv1D()\n",
|
||||
" (dropout): Dropout(p=0.1, inplace=False)\n",
|
||||
" )\n",
|
||||
" )\n",
|
||||
" )\n",
|
||||
" (ln_f): LayerNorm((768,), eps=1e-05, elementwise_affine=True)\n",
|
||||
")"
|
||||
]
|
||||
},
|
||||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"from transformers import GPT2Model, GPT2Tokenizer\n",
|
||||
"model_class, tokenizer_class, model_name_or_path = (GPT2Model, GPT2Tokenizer, 'gpt2')\n",
|
||||
"tokenizer = tokenizer_class.from_pretrained(model_name_or_path, cache_dir=cache_dir)\n",
|
||||
"model = model_class.from_pretrained(model_name_or_path, cache_dir=cache_dir)\n",
|
||||
"model.eval().cpu()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import numpy\n",
|
||||
"import time\n",
|
||||
"\n",
|
||||
"def pytorch_inference(model, input_ids, past=None, total_runs = 100):\n",
|
||||
" latency = []\n",
|
||||
" with torch.no_grad():\n",
|
||||
" for _ in range(total_runs):\n",
|
||||
" start = time.time()\n",
|
||||
" outputs = model(input_ids=input_ids, past=past)\n",
|
||||
" latency.append(time.time() - start)\n",
|
||||
" \n",
|
||||
" if total_runs > 1:\n",
|
||||
" print(\"PyTorch Inference time = {} ms\".format(format(sum(latency) * 1000 / len(latency), '.2f')))\n",
|
||||
" \n",
|
||||
" return outputs\n",
|
||||
" \n",
|
||||
"def onnxruntime_inference(ort_session, input_ids, past=None, total_runs=100): \n",
|
||||
" # Use contiguous array as input might improve performance.\n",
|
||||
" # You can check the results from performance test tool to see whether you need it.\n",
|
||||
" ort_inputs = {\n",
|
||||
" 'input_ids': numpy.ascontiguousarray(input_ids.cpu().numpy())\n",
|
||||
" }\n",
|
||||
" \n",
|
||||
" if past is not None:\n",
|
||||
" for i, past_i in enumerate(past):\n",
|
||||
" ort_inputs[f'past_{i}'] = numpy.ascontiguousarray(past[i].cpu().numpy())\n",
|
||||
" \n",
|
||||
" latency = []\n",
|
||||
" for _ in range(total_runs):\n",
|
||||
" start = time.time()\n",
|
||||
" ort_outputs = ort_session.run(None, ort_inputs)\n",
|
||||
" latency.append(time.time() - start)\n",
|
||||
" \n",
|
||||
" if total_runs > 1:\n",
|
||||
" print(\"OnnxRuntime Inference time = {} ms\".format(format(sum(latency) * 1000 / len(latency), '.2f')))\n",
|
||||
" \n",
|
||||
" return ort_outputs\n",
|
||||
"\n",
|
||||
"def inference(model, ort_session, input_ids, past=None, total_runs=100, verify_outputs=True):\n",
|
||||
" outputs = pytorch_inference(model, input_ids, past, total_runs)\n",
|
||||
" ort_outputs = onnxruntime_inference(ort_session, input_ids, past, total_runs)\n",
|
||||
" if verify_outputs:\n",
|
||||
" print('PyTorch and OnnxRuntime output 0 (last_state) are close:'.format(0), numpy.allclose(ort_outputs[0], outputs[0].cpu(), rtol=1e-05, atol=1e-04))\n",
|
||||
" \n",
|
||||
" for layer in range(model.config.n_layer):\n",
|
||||
" print('PyTorch and OnnxRuntime layer {} state (present_{}) are close:'.format(layer, layer), numpy.allclose(ort_outputs[1 + layer], outputs[1][layer].cpu(), rtol=1e-05, atol=1e-04)) "
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import torch\n",
|
||||
"import os\n",
|
||||
"\n",
|
||||
"inputs = tokenizer.encode_plus(\"Here is an example input for GPT2 model\", add_special_tokens=True, return_tensors='pt')\n",
|
||||
"input_ids = inputs['input_ids']\n",
|
||||
"\n",
|
||||
"# run without past so that we can know the shape of past from output.\n",
|
||||
"outputs = model(input_ids=input_ids, past=None)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"num_layer = model.config.n_layer \n",
|
||||
"present_names = [f'present_{i}' for i in range(num_layer)]\n",
|
||||
"output_names = [\"last_state\"] + present_names\n",
|
||||
"\n",
|
||||
"input_names = ['input_ids']\n",
|
||||
"dynamic_axes= {'input_ids': {0: 'batch_size', 1: 'seq_len'},\n",
|
||||
" #'token_type_ids' : {0: 'batch_size', 1: 'seq_len'},\n",
|
||||
" #'attention_mask' : {0: 'batch_size', 1: 'seq_len'},\n",
|
||||
" 'last_state' : {0: 'batch_size', 1: 'seq_len'}\n",
|
||||
" }\n",
|
||||
"for name in present_names:\n",
|
||||
" dynamic_axes[name] = {1: 'batch_size', 3: 'seq_len'}\n",
|
||||
" \n",
|
||||
"if enable_past_input:\n",
|
||||
" past_names = [f'past_{i}' for i in range(num_layer)]\n",
|
||||
" input_names = ['input_ids'] + past_names #+ ['token_type_ids', 'attention_mask']\n",
|
||||
" dummy_past = [torch.zeros(list(outputs[1][0].shape)) for _ in range(num_layer)]\n",
|
||||
" for name in past_names:\n",
|
||||
" dynamic_axes[name] = {1: 'batch_size', 3: 'seq_len'}\n",
|
||||
" export_inputs = (inputs['input_ids'], tuple(dummy_past)) #, inputs['token_type_ids'], inputs['attention_mask'])\n",
|
||||
"else:\n",
|
||||
" export_inputs = (inputs['input_ids'])\n",
|
||||
"\n",
|
||||
"export_model_path = os.path.join(output_dir, 'gpt2_past{}.onnx'.format(int(enable_past_input)))\n",
|
||||
"\n",
|
||||
"torch.onnx.export(model,\n",
|
||||
" args=export_inputs,\n",
|
||||
" f=export_model_path,\n",
|
||||
" input_names=input_names,\n",
|
||||
" output_names=output_names,\n",
|
||||
" dynamic_axes=dynamic_axes,\n",
|
||||
" opset_version=11,\n",
|
||||
" do_constant_folding = True,\n",
|
||||
" verbose=False)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Inference with ONNX Runtime\n",
|
||||
"\n",
|
||||
"### OpenMP Environment Variable\n",
|
||||
"\n",
|
||||
"OpenMP environment variables are very important for CPU inference of GPT2 model. It has large performance impact on GPT2 model so you might need set it carefully according to benchmark script.\n",
|
||||
"\n",
|
||||
"Setting environment variables shall be done before importing onnxruntime. Otherwise, they might not take effect."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"import psutil\n",
|
||||
"\n",
|
||||
"# You may change the settings in this cell according to Performance Test Tool result.\n",
|
||||
"use_openmp = False\n",
|
||||
"\n",
|
||||
"# ATTENTION: these environment variables must be set before importing onnxruntime.\n",
|
||||
"if use_openmp:\n",
|
||||
" os.environ[\"OMP_NUM_THREADS\"] = str(psutil.cpu_count(logical=True))\n",
|
||||
"else:\n",
|
||||
" os.environ[\"OMP_NUM_THREADS\"] = '1'\n",
|
||||
"\n",
|
||||
"os.environ[\"OMP_WAIT_POLICY\"] = 'ACTIVE'"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"PyTorch Inference time = 37.81 ms\n",
|
||||
"OnnxRuntime Inference time = 31.81 ms\n",
|
||||
"PyTorch and OnnxRuntime output 0 (last_state) are close: True\n",
|
||||
"PyTorch and OnnxRuntime layer 0 state (present_0) are close: True\n",
|
||||
"PyTorch and OnnxRuntime layer 1 state (present_1) are close: True\n",
|
||||
"PyTorch and OnnxRuntime layer 2 state (present_2) are close: True\n",
|
||||
"PyTorch and OnnxRuntime layer 3 state (present_3) are close: True\n",
|
||||
"PyTorch and OnnxRuntime layer 4 state (present_4) are close: True\n",
|
||||
"PyTorch and OnnxRuntime layer 5 state (present_5) are close: True\n",
|
||||
"PyTorch and OnnxRuntime layer 6 state (present_6) are close: True\n",
|
||||
"PyTorch and OnnxRuntime layer 7 state (present_7) are close: True\n",
|
||||
"PyTorch and OnnxRuntime layer 8 state (present_8) are close: True\n",
|
||||
"PyTorch and OnnxRuntime layer 9 state (present_9) are close: True\n",
|
||||
"PyTorch and OnnxRuntime layer 10 state (present_10) are close: True\n",
|
||||
"PyTorch and OnnxRuntime layer 11 state (present_11) are close: True\n",
|
||||
"Wall time: 6.97 s\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import onnxruntime\n",
|
||||
"import numpy\n",
|
||||
"\n",
|
||||
"# Print warning if user uses onnxruntime-gpu instead of onnxruntime package.\n",
|
||||
"if 'CUDAExecutionProvider' in onnxruntime.get_available_providers():\n",
|
||||
" print(\"warning: onnxruntime-gpu is not built with OpenMP. You might try onnxruntime package to test CPU inference.\")\n",
|
||||
"\n",
|
||||
"sess_options = onnxruntime.SessionOptions()\n",
|
||||
"\n",
|
||||
"# Optional: store the optimized graph and view it using Netron to verify that model is fully optimized.\n",
|
||||
"# Note that this will increase session creation time, so it is for debugging only.\n",
|
||||
"#sess_options.optimized_model_filepath = os.path.join(output_dir, \"optimized_model_cpu.onnx\")\n",
|
||||
" \n",
|
||||
"if use_openmp:\n",
|
||||
" sess_options.intra_op_num_threads=1\n",
|
||||
"else:\n",
|
||||
" sess_options.intra_op_num_threads=psutil.cpu_count(logical=True)\n",
|
||||
"\n",
|
||||
"# Specify providers when you use onnxruntime-gpu for CPU inference.\n",
|
||||
"session = onnxruntime.InferenceSession(export_model_path, sess_options, providers=['CPUExecutionProvider'])\n",
|
||||
"\n",
|
||||
"# Compare PyTorch and OnnxRuntime inference performance and results\n",
|
||||
"%time inference(model, session, input_ids, past=dummy_past if enable_past_input else None)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"335"
|
||||
]
|
||||
},
|
||||
"execution_count": 10,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import gc\n",
|
||||
"del session\n",
|
||||
"gc.collect()"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"optimized_model = os.path.join(output_dir, 'gpt2_past{}_optimized.onnx'.format(int(enable_past_input)))"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"bert_opt_script = os.path.join(bert_tools_dir, 'bert_model_optimization.py')"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 13,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
" BertOnnxModel.py: Fused LayerNormalization count: 25\n",
|
||||
" BertOnnxModel.py: Fused Gelu (FastGelu fits better) count: 12\n",
|
||||
" BertOnnxModel.py: Fused Reshape count:48\n",
|
||||
" BertOnnxModel.py: Fused SkipLayerNormalization count: 1\n",
|
||||
" OnnxModel.py: Removed unused constant nodes: 422\n",
|
||||
" BertOnnxModel.py: Fused Attention count:0\n",
|
||||
" BertOnnxModel.py: skip embed layer fusion since mask input is not found\n",
|
||||
" BertOnnxModel.py: opset verion: 11\n",
|
||||
" OnnxModel.py: Output model to ./gpt2_onnx\\gpt2_past0_optimized.onnx\n",
|
||||
" BertOnnxModel.py: EmbedLayer=0, Attention=0, Gelu=12, LayerNormalization=25, Successful=False\n",
|
||||
"bert_model_optimization.py: The output model is not fully optimized. It might not be usable.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Local directory corresponding to https://github.com/microsoft/onnxruntime/tree/master/onnxruntime/python/tools/bert/\n",
|
||||
"%run $bert_opt_script --input $export_model_path --output $optimized_model --opt_level 0"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 14,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"PyTorch Inference time = 38.72 ms\n",
|
||||
"OnnxRuntime Inference time = 29.58 ms\n",
|
||||
"Wall time: 6.83 s\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"session = onnxruntime.InferenceSession(optimized_model, sess_options, providers=['CPUExecutionProvider'])\n",
|
||||
"\n",
|
||||
"%time inference(model, session, input_ids, past=dummy_past if enable_past_input else None, verify_outputs=False)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Additional Info\n",
|
||||
"\n",
|
||||
"Note that running Jupyter Notebook has slight impact on performance result since Jupyter Notebook is using system resources like CPU and memory etc. It is recommended to close Jupyter Notebook and other applications, then run the benchmark script in a console to get more accurate performance numbers.\n",
|
||||
"\n",
|
||||
"[OnnxRuntime C API](https://github.com/microsoft/onnxruntime/blob/master/docs/C_API.md) could get slightly better performance than python API. If you use C API in inference, you can use OnnxRuntime_Perf_Test.exe built from source to measure performance instead.\n",
|
||||
"\n",
|
||||
"Here is the machine configuration that generated the above results. The machine has GPU but not used in CPU inference.\n",
|
||||
"You might get slower or faster result based on your hardware."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 15,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"{\n",
|
||||
" \"gpu\": {\n",
|
||||
" \"driver_version\": \"441.22\",\n",
|
||||
" \"devices\": [\n",
|
||||
" {\n",
|
||||
" \"memory_total\": 8589934592,\n",
|
||||
" \"memory_available\": 6569947136,\n",
|
||||
" \"name\": \"GeForce GTX 1070\"\n",
|
||||
" }\n",
|
||||
" ]\n",
|
||||
" },\n",
|
||||
" \"cpu\": {\n",
|
||||
" \"brand\": \"Intel(R) Core(TM) i7-8700 CPU @ 3.20GHz\",\n",
|
||||
" \"cores\": 6,\n",
|
||||
" \"logical_cores\": 12,\n",
|
||||
" \"hz\": \"3.1920 GHz\",\n",
|
||||
" \"l2_cache\": \"1536 KB\",\n",
|
||||
" \"l3_cache\": \"12288 KB\",\n",
|
||||
" \"processor\": \"Intel64 Family 6 Model 158 Stepping 10, GenuineIntel\"\n",
|
||||
" },\n",
|
||||
" \"memory\": {\n",
|
||||
" \"total\": 16971259904,\n",
|
||||
" \"available\": 2854060032\n",
|
||||
" },\n",
|
||||
" \"python\": \"3.6.10.final.0 (64 bit)\",\n",
|
||||
" \"os\": \"Windows-10-10.0.18362-SP0\",\n",
|
||||
" \"onnxruntime\": {\n",
|
||||
" \"version\": \"1.2.0\",\n",
|
||||
" \"support_gpu\": false\n",
|
||||
" },\n",
|
||||
" \"pytorch\": {\n",
|
||||
" \"version\": \"1.4.0+cpu\",\n",
|
||||
" \"support_gpu\": false\n",
|
||||
" },\n",
|
||||
" \"tensorflow\": {\n",
|
||||
" \"version\": \"2.1.0\",\n",
|
||||
" \"git_version\": \"v2.1.0-rc2-17-ge5bf8de410\",\n",
|
||||
" \"support_gpu\": true\n",
|
||||
" }\n",
|
||||
"}\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"machine_info_script = os.path.join(bert_tools_dir, 'MachineInfo.py')\n",
|
||||
"%run $machine_info_script --silent"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "cpu_env",
|
||||
"language": "python",
|
||||
"name": "cpu_env"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
"name": "ipython",
|
||||
"version": 3
|
||||
},
|
||||
"file_extension": ".py",
|
||||
"mimetype": "text/x-python",
|
||||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.6.10"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
}
|
||||
Loading…
Reference in a new issue