From 769c11f217035d229d87ae83c0a285a3ea9df1ae Mon Sep 17 00:00:00 2001 From: Tianlei Wu Date: Sat, 16 May 2020 11:13:34 -0700 Subject: [PATCH] Update doc for transformers tools (#3963) * update readme for onnxruntime-tools package * update license section in benchmark --- .../python/tools/transformers/README.md | 135 ++++++++---------- .../python/tools/transformers/benchmark.py | 33 ++++- 2 files changed, 82 insertions(+), 86 deletions(-) diff --git a/onnxruntime/python/tools/transformers/README.md b/onnxruntime/python/tools/transformers/README.md index cf6a318c38..6c1a521ce4 100644 --- a/onnxruntime/python/tools/transformers/README.md +++ b/onnxruntime/python/tools/transformers/README.md @@ -1,51 +1,42 @@ -# BERT Model Optimization Tool Overview +# Transformer Model Optimization Tool Overview -This tool showcases how to fuse a BERT ONNX model either exported from PyTorch or converted from TensorFlow, and generates an optimized model to run faster with OnnxRuntime. +ONNX Runtime automatically applies most optimizations while loading a transformer model. Some of the latest optimizations that have not yet been integrated into ONNX Runtime are available in this tool that tunes models for the best performance. -Note that OnnxRuntime can fuse the Bert ONNX model exported from PyTorch automatically. You don't need this tool to fuse the model. It is only required for Bert Model converted from Tensorflow. +This tool can help in the following senarios: +* Model is exported by tf2onnx or keras2onnx, and ONNX Runtime does not have graph optimization for them right now. +* Convert model to use float16 to boost performance using mixed precision on GPUs with Tensor Cores (like V100 or T4). +* Model has inputs with dynamic axis, which blocks some optimizations to be applied in ONNX Runtime due to shape inference. +* Disable or enable some fusions to see its impact on performance or accuracy. -## Export a BERT model from PyTorch -For example, after using https://github.com/huggingface/transformers/tree/master/examples/run_glue.py to train a BERT model in PyTorch 1.3, you can use the following function to export ONNX model. +## Installation +First you need install onnxruntime or onnxruntime-gpu package for CPU or GPU inference. To use onnxruntime-gpu, it is required to install CUDA and cuDNN and add their bin directories to PATH environment variable. -Please specify do_constant_folding=True. That's required for this tool. - -```python -def export_onnx(args, model, output_path): - model.eval() # set the model to inference mode - device = torch.device("cpu") - model.to(device) - dummy_input0 = torch.LongTensor(args.eval_batch_size, args.max_seq_length).fill_(1).to(device) - dummy_input1 = torch.LongTensor(args.eval_batch_size, args.max_seq_length).fill_(1).to(device) - dummy_input2 = torch.LongTensor(args.eval_batch_size, args.max_seq_length).fill_(0).to(device) - dummy_input = (dummy_input0, dummy_input1, dummy_input2) - torch.onnx.export(model, # model being run - dummy_input, # model input (or a tuple for multiple inputs) - output_path, # where to save the model (can be a file or file-like object) - export_params=True, # store the trained parameter weights inside the model file - opset_version=10, # the ONNX version to export the model to - do_constant_folding=True, # whether to execute constant folding for optimization - input_names = ["input_ids", "input_mask", "segment_ids"], - output_names = ["output"], - dynamic_axes={'input_ids' : {0 : 'batch_size'}, # variable length axes - 'input_mask' : {0 : 'batch_size'}, - 'segment_ids' : {0 : 'batch_size'}, - 'output' : {0 : 'batch_size'}}) +This tool can be installed using pip as follows: +```console +pip install onnxruntime-tools ``` -## Convert a BERT model from Tensorflow - -The tf2onnx and keras2onnx tools can be used to convert model that trained by Tensorflow. - -For Keras2onnx, please refere to its [example script](https://github.com/onnx/keras-onnx/blob/master/applications/nightly_build/test_transformers.py). - -For tf2onnx, please refer to this notebook: https://github.com/onnx/tensorflow-onnx/blob/master/tutorials/BertTutorial.ipynb - - -## Model Optimization - -Example of using the script optimizer.py to convert a BERT-large model to run in V100 GPU: +After it is installed, you can use command like the following to optimize model: ```console -python optimizer.py --input original_model.onnx --output optimized_model_gpu.onnx --num_heads 16 --hidden_size 1024 --input_int32 --float16 +python -m onnxruntime_tools.transformers.optimizer --input gpt2.onnx --output gpt2_opt.onnx --model_type gpt2 +``` + +If you want to use the latest script, you can get script files from [here](https://github.com/microsoft/onnxruntime/tree/master/onnxruntime/python/tools/transformers/). Then run it like the following: +```console +python optimizer.py --input gpt2.onnx --output gpt2_opt.onnx --model_type gpt2 +``` + +## Export a transformer model to ONNX +PyTorch could export model to ONNX. The tf2onnx and keras2onnx tools can be used to convert model that trained by Tensorflow. +Huggingface transformers has a [notebook](https://github.com/huggingface/transformers/blob/master/notebooks/04-onnx-export.ipynb) shows an example of exporting a pretrained model to ONNX. +For Keras2onnx, please refere to its [example script](https://github.com/onnx/keras-onnx/blob/master/applications/nightly_build/test_transformers.py). +For tf2onnx, please refer to [its BERT tutorial](https://github.com/onnx/tensorflow-onnx/blob/master/tutorials/BertTutorial.ipynb). + +## Model Optimizer + +Example of using the script optimizer.py to optimize a BERT-large model to run in V100 GPU: +```console +python -m onnxruntime_tools.transformers.optimizer --input bert_large.onnx --output bert_large_fp16.onnx --num_heads 16 --hidden_size 1024 --float16 ``` ### Options @@ -55,7 +46,7 @@ See below for description of some options of optimizer.py: - **input**: input model path - **output**: output model path - **model_type**: (*defaul: bert*) - There are 4 model types: *bert* (exported by PyTorch), *bert_tf* (BERT exported by tf2onnx), *bert_keras* (BERT exported by keras2onnx) and *gpt2* (exported by PyTorch) respectively. + There are 4 model types: *bert* (exported by PyTorch), *gpt2* (exported by PyTorch), and *bert_tf* (BERT exported by tf2onnx), *bert_keras* (BERT exported by keras2onnx) respectively. - **num_heads**: (*default: 12*) Number of attention heads. BERT-base and BERT-large has 12 and 16 respectively. - **hidden_size**: (*default: 768*) @@ -69,9 +60,9 @@ See below for description of some options of optimizer.py: ### Supported Models -Right now, this tool assumes input model has 3 inputs for input IDs, segment IDs, and attention mask. A model with less or addtional inputs might not be optimized. +Right now, this tool assumes input model has 3 inputs for input IDs, segment IDs, and attention mask. A model with less or addtional inputs might not be fully optimized. -Most optimizations require exact match of a subgraph. That means this tool could only support similar models with such subgraphs. Any layout change in subgraph might cause optimization not working. Note that different training or export tool (including different versions) might get different graph layouts. +Most optimizations require exact match of a subgraph. Any layout change in subgraph might cause some optimization not working. Note that different versions of training or export tool might lead to different graph layouts. Here is list of models from [Huggingface Transformers](https://github.com/huggingface/transformers/) that have been tested using this tool: - **BertForSequenceClassification** as in [transformers example](https://github.com/huggingface/transformers/blob/master/examples/run_glue.py) exported by PyTorch 1.2-1.4 using opset version 10 or 11. @@ -82,53 +73,39 @@ Here is list of models from [Huggingface Transformers](https://github.com/huggin - **GPT2LMHeadModel** exported by PyTorch 1.4 using opset version 10 or 11. If your model is not in the list, the optimized model might not work. You are welcome to update the scripts to support new models. +For GPT2 models, current optimization does not support past state (both inputs and outputs). You need disable it in transformers by setting enable_cache=False during exporting. + +## Benchmark + +The benchmark script requires PyTorch be installed. + +You can run benchmark script to see the inference speed of OnnxRuntime. Here is an example to run benchmark on a pretrained model bert-base-cased on GPU. + +```console +python -m onnxruntime_tools.transformers.benchmark -m bert-base-cased -g +``` + ## Model Verification -When a BERT model is optimized, some optimization uses approximation in calculation so the output might be slightly different. It is recommended to use your evaluation set to measure the precision and recall. We expect the accuracy shall be on par after optimization. +If your model has three inputs (like input_ids, token_type_ids and attention_mask), a script compare_bert_results.py can be used to do a quick verification. The tool will generate some fake input data, and compare results from both the original and optimized models. If outputs are all close, it is safe to use the optimized model. -If your BERT model has three inputs, a script compare_bert_results.py can be used to do a quick verification. The tool will generate some fake input data, and compare results from both the original and optimized models. If outputs are all close, it is safe to use the optimized model. - -Example of verifying models optimized for CPU and GPU: +Example of verifying models optimized for CPU: ```console -pip install onnxruntime -python compare_bert_results.py --baseline_model original_model.onnx --optimized_model optimized_model_cpu.onnx --batch_size 1 --sequence_length 128 --samples 100 - -pip uninstall onnxruntime -pip install onnxruntime-gpu -python compare_bert_results.py --baseline_model original_model.onnx --optimized_model optimized_model_gpu.onnx --batch_size 1 --sequence_length 128 --samples 100 --use_gpu +python -m onnxruntime_tools.transformers.compare_bert_results --baseline_model original_model.onnx --optimized_model optimized_model_cpu.onnx --batch_size 1 --sequence_length 128 --samples 100 ``` -To use onnxruntime-gpu, it is required to install CUDA and cuDNN and add their bin directories to PATH environment variable. +For GPU, please append --use_gpu to the command. -## Performance Test (Python) +## Performance Test -bert_perf_test.py can be used to check the model inference performance of python API. Below are examples: +bert_perf_test.py can be used to check the model inference performance. Below are examples: ```console -pip install onnxruntime -python bert_perf_test.py --model optimized_model_cpu.onnx --batch_size 1 --sequence_length 128 --samples 100 --test_times 10 --inclusive - -pip uninstall onnxruntime -pip install onnxruntime-gpu -python bert_perf_test.py --model optimized_model_gpu.onnx --batch_size 1 --sequence_length 128 --samples 100 --test_times 10 --use_gpu --inclusive +python -m onnxruntime_tools.transformers.bert_perf_test --model optimized_model_cpu.onnx --batch_size 1 --sequence_length 128 --samples 100 --test_times 10 --inclusive ``` +For GPU, please append --use_gpu to the command. + After test is finished, a file like perf_results_CPU_B1_S128_.txt or perf_results_GPU_B1_S128_.txt will be output to the model directory. -## Performance Test (C API) - -First, we need generate some test data. Please make sure there is no sub-directories on the directory of onnx model. - -Here is an example: -```console -python bert_test_data.py --model bert.onnx --batch_size 1 --sequence_length 32 --samples 100 --output_dir . -``` - -You can go to root of this git repository, and build onnxruntime_perf_test.exe from source to test performance of C API. Example commands in Windows: -```console -build.bat --config RelWithDebInfo --enable_lto --use_openmp --build_shared_lib --parallel --cmake_generator "Visual Studio 16 2019" -Set OMP_NUM_THREADS=%NUMBER_OF_PROCESSORS% -Set OMP_WAIT_POLICY=PASSIVE -build\Windows\RelWithDebInfo\RelWithDebInfo\onnxruntime_perf_test.exe -e cpu -r 100 -s -o 2 bert.onnx output.txt -``` diff --git a/onnxruntime/python/tools/transformers/benchmark.py b/onnxruntime/python/tools/transformers/benchmark.py index 500b21751a..8b8d1bef93 100644 --- a/onnxruntime/python/tools/transformers/benchmark.py +++ b/onnxruntime/python/tools/transformers/benchmark.py @@ -1,8 +1,24 @@ -#------------------------------------------------------------------------- # Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. -#-------------------------------------------------------------------------- +# Copyright 2018 The HuggingFace Inc. team. +# Copyright (c) 2018, NVIDIA CORPORATION. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + """ Benchmarking the inference of pretrained transformer models + PyTorch/TorchScript benchmark is based on https://github.com/huggingface/transformers/blob/master/examples/benchmarks.py. + + This will automatically export a pretrained model to ONNX, and do optimization (optional) + Example commands: Export all models to ONNX, optimize and validate them: python benchmark.py -b 0 -o -v -i 1 2 3 @@ -40,11 +56,14 @@ MODELS = { "bert-base-cased": (["input_ids", "attention_mask", "token_type_ids"], 11, "bert"), "distilbert-base-uncased": (["input_ids", "attention_mask"], 11, "bert"), "roberta-base": (["input_ids", "attention_mask"], 11, "bert"), - "gpt2": (["input_ids"], 11, "gpt2"), # no past state - "distilgpt2": (["input_ids"], 11, "gpt2"), # no past state + + # No past state inputs for GPT models. You might set enable_cache=False in modeling_gpt2.py (need installing transformers from source) to disable past state output. + "gpt2": (["input_ids"], 11, "gpt2"), + "distilgpt2": (["input_ids"], 11, "gpt2"), "openai-gpt": (["input_ids"], 11, "gpt2"), - # Models uses Einsum, which need opset version 12 and PyTorch 1.5.0 or above. + # Models uses Einsum, which need opset version 12 and PyTorch 1.5.0 or above. + # Currently OnnxRuntime lacks cuda op for Einsum. GPU inference will be very slow. "albert-base-v2": (["input_ids"], 12, "bert"), "xlnet-base-cased": (["input_ids"], 12, "bert"), } @@ -451,7 +470,7 @@ def parse_arguments(): required=False, nargs="+", type=str, - default=list(MODELS.keys()), + default=["bert-base-cased", "roberta-base", "gpt2"], choices=list(MODELS.keys()), help="Pre-trained models in the list: " + ", ".join(MODELS.keys()))