diff --git a/onnxruntime/python/tools/transformers/notebooks/Inference_GPT2_with_OnnxRuntime_on_CPU.ipynb b/onnxruntime/python/tools/transformers/notebooks/Inference_GPT2_with_OnnxRuntime_on_CPU.ipynb index 2a97343fd7..5635992e15 100644 --- a/onnxruntime/python/tools/transformers/notebooks/Inference_GPT2_with_OnnxRuntime_on_CPU.ipynb +++ b/onnxruntime/python/tools/transformers/notebooks/Inference_GPT2_with_OnnxRuntime_on_CPU.ipynb @@ -14,9 +14,7 @@ "source": [ "# Inference PyTorch GPT2 Model with ONNX Runtime on CPU\n", "\n", - "In this tutorial, you'll be introduced to how to load a GPT2 model from PyTorch, convert it to ONNX, and inference it using ONNX Runtime.\n", - "\n", - "**Note: this work is still in progresss. Need install ort_nightly package before onnxruntime 1.3.0 is ready. The performance number of ort_nightly does not reflect the final result for onnxruntime 1.3.0. **" + "In this tutorial, you'll be introduced to how to load a GPT2 model from PyTorch, convert it to ONNX, and inference it using ONNX Runtime using IO Binding. Note that past state is used to get better performance." ] }, { @@ -30,14 +28,8 @@ "Otherwise, you can setup a new environment. First, we install [AnaConda](https://www.anaconda.com/distribution/). Then open an AnaConda prompt window and run the following commands:\n", "\n", "```console\n", - "conda create -n cpu_env python=3.6\n", + "conda create -n cpu_env python=3.8\n", "conda activate cpu_env\n", - "\n", - "conda install pytorch torchvision cpuonly -c pytorch\n", - "pip install onnxruntime\n", - "pip install transformers==2.5.1\n", - "pip install onnx psutil pytz pandas py-cpuinfo py3nvml netron\n", - "\n", "conda install jupyter\n", "jupyter notebook\n", "```\n", @@ -46,29 +38,541 @@ }, { "cell_type": "code", - "execution_count": null, + "execution_count": 1, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Looking in links: https://download.pytorch.org/whl/torch_stable.html\n", + "Requirement already up-to-date: torch==1.6.0+cpu in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (1.6.0+cpu)\n", + "Requirement already up-to-date: torchvision==0.7.0+cpu in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (0.7.0+cpu)\n", + "Requirement already satisfied, skipping upgrade: future in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from torch==1.6.0+cpu) (0.18.2)\n", + "Requirement already satisfied, skipping upgrade: numpy in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from torch==1.6.0+cpu) (1.18.1)\n", + "Requirement already satisfied, skipping upgrade: pillow>=4.1.1 in c:\\users\\tianl\\appdata\\roaming\\python\\python36\\site-packages (from torchvision==0.7.0+cpu) (7.0.0)\n", + "Requirement already up-to-date: onnxruntime==1.4.0 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (1.4.0)\n", + "Requirement already satisfied, skipping upgrade: numpy>=1.16.6 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from onnxruntime==1.4.0) (1.18.1)\n", + "Requirement already satisfied, skipping upgrade: protobuf in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from onnxruntime==1.4.0) (3.11.3)\n", + "Requirement already satisfied, skipping upgrade: setuptools in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from protobuf->onnxruntime==1.4.0) (45.2.0.post20200210)\n", + "Requirement already satisfied, skipping upgrade: six>=1.9 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from protobuf->onnxruntime==1.4.0) (1.14.0)\n", + "Requirement already up-to-date: onnxruntime-tools in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (1.4.2)\n", + "Requirement already satisfied, skipping upgrade: py-cpuinfo in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from onnxruntime-tools) (5.0.0)\n", + "Requirement already satisfied, skipping upgrade: numpy in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from onnxruntime-tools) (1.18.1)\n", + "Requirement already satisfied, skipping upgrade: packaging in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from onnxruntime-tools) (20.1)\n", + "Requirement already satisfied, skipping upgrade: onnx in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from onnxruntime-tools) (1.7.0)\n", + "Requirement already satisfied, skipping upgrade: coloredlogs in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from onnxruntime-tools) (14.0)\n", + "Requirement already satisfied, skipping upgrade: py3nvml in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from onnxruntime-tools) (0.2.5)\n", + "Requirement already satisfied, skipping upgrade: psutil in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from onnxruntime-tools) (5.7.0)\n", + "Requirement already satisfied, skipping upgrade: pyparsing>=2.0.2 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from packaging->onnxruntime-tools) (2.4.6)\n", + "Requirement already satisfied, skipping upgrade: six in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from packaging->onnxruntime-tools) (1.14.0)\n", + "Requirement already satisfied, skipping upgrade: protobuf in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from onnx->onnxruntime-tools) (3.11.3)\n", + "Requirement already satisfied, skipping upgrade: typing-extensions>=3.6.2.1 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from onnx->onnxruntime-tools) (3.7.4.1)\n", + "Requirement already satisfied, skipping upgrade: humanfriendly>=7.1 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from coloredlogs->onnxruntime-tools) (8.1)\n", + "Requirement already satisfied, skipping upgrade: xmltodict in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from py3nvml->onnxruntime-tools) (0.12.0)\n", + "Requirement already satisfied, skipping upgrade: setuptools in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from protobuf->onnx->onnxruntime-tools) (45.2.0.post20200210)\n", + "Requirement already satisfied, skipping upgrade: pyreadline; sys_platform == \"win32\" in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from humanfriendly>=7.1->coloredlogs->onnxruntime-tools) (2.1)\n", + "Requirement already satisfied: transformers==3.0.2 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (3.0.2)\n", + "Requirement already satisfied: sentencepiece!=0.1.92 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from transformers==3.0.2) (0.1.85)\n", + "Requirement already satisfied: packaging in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from transformers==3.0.2) (20.1)\n", + "Requirement already satisfied: sacremoses in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from transformers==3.0.2) (0.0.38)\n", + "Requirement already satisfied: tokenizers==0.8.1.rc1 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from transformers==3.0.2) (0.8.1rc1)\n", + "Requirement already satisfied: tqdm>=4.27 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from transformers==3.0.2) (4.43.0)\n", + "Requirement already satisfied: requests in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from transformers==3.0.2) (2.23.0)\n", + "Requirement already satisfied: numpy in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from transformers==3.0.2) (1.18.1)\n", + "Requirement already satisfied: regex!=2019.12.17 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from transformers==3.0.2) (2020.2.20)\n", + "Requirement already satisfied: dataclasses; python_version < \"3.7\" in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from transformers==3.0.2) (0.7)\n", + "Requirement already satisfied: filelock in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from transformers==3.0.2) (3.0.12)\n", + "Requirement already satisfied: six in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from packaging->transformers==3.0.2) (1.14.0)\n", + "Requirement already satisfied: pyparsing>=2.0.2 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from packaging->transformers==3.0.2) (2.4.6)\n", + "Requirement already satisfied: joblib in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from sacremoses->transformers==3.0.2) (0.14.1)\n", + "Requirement already satisfied: click in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from sacremoses->transformers==3.0.2) (7.0)\n", + "Requirement already satisfied: certifi>=2017.4.17 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from requests->transformers==3.0.2) (2020.4.5.1)\n", + "Requirement already satisfied: chardet<4,>=3.0.2 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from requests->transformers==3.0.2) (3.0.4)\n", + "Requirement already satisfied: idna<3,>=2.5 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from requests->transformers==3.0.2) (2.9)\n", + "Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from requests->transformers==3.0.2) (1.25.8)\n", + "Requirement already satisfied: onnx in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (1.7.0)\n", + "Requirement already satisfied: psutil in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (5.7.0)\n", + "Requirement already satisfied: pytz in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (2019.3)\n", + "Requirement already satisfied: pandas in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (1.0.1)\n", + "Requirement already satisfied: py-cpuinfo in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (5.0.0)\n", + "Requirement already satisfied: py3nvml in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (0.2.5)\n", + "Requirement already satisfied: netron in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (3.9.6)\n", + "Requirement already satisfied: protobuf in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from onnx) (3.11.3)\n", + "Requirement already satisfied: numpy in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from onnx) (1.18.1)\n", + "Requirement already satisfied: typing-extensions>=3.6.2.1 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from onnx) (3.7.4.1)\n", + "Requirement already satisfied: six in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from onnx) (1.14.0)\n", + "Requirement already satisfied: python-dateutil>=2.6.1 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from pandas) (2.8.1)\n", + "Requirement already satisfied: xmltodict in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from py3nvml) (0.12.0)\n", + "Requirement already satisfied: setuptools in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from protobuf->onnx) (45.2.0.post20200210)\n" + ] + } + ], "source": [ - "# Enable pass state in input.\n", - "enable_past_input = False" + "# Install PyTorch 1.6.0 and OnnxRuntime 1.4.0 for CPU-only.\n", + "import sys\n", + "if sys.platform == 'darwin': # Mac\n", + " !{sys.executable} -m pip install --upgrade torch torchvision\n", + "else:\n", + " !{sys.executable} -m pip install --upgrade torch==1.6.0+cpu torchvision==0.7.0+cpu -f https://download.pytorch.org/whl/torch_stable.html\n", + "!{sys.executable} -m pip install --upgrade onnxruntime==1.4.0\n", + "!{sys.executable} -m pip install --upgrade onnxruntime-tools\n", + "\n", + "# Install other packages used in this notebook.\n", + "!{sys.executable} -m pip install transformers==3.0.2\n", + "!{sys.executable} -m pip install onnx psutil pytz pandas py-cpuinfo py3nvml netron" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 2, "metadata": {}, "outputs": [], "source": [ "import os\n", "\n", - "cache_dir = \"./gpt2\"\n", + "# Create a cache directory to store pretrained model.\n", + "cache_dir = os.path.join(\".\", \"cache_models\")\n", "if not os.path.exists(cache_dir):\n", - " os.makedirs(cache_dir)\n", + " os.makedirs(cache_dir)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Convert GPT2 model from PyTorch to ONNX ##\n", "\n", - "output_dir = './gpt2_onnx'\n", - "if not os.path.exists(output_dir):\n", - " os.makedirs(output_dir)" + "We have a script [convert_to_onnx.py](https://github.com/microsoft/onnxruntime/blob/master/onnxruntime/python/tools/transformers/convert_to_onnx.py) that could help you to convert GPT2 with past state to ONNX. \n", + "\n", + "The script accepts a pretrained model name or path of a checkpoint directory as input, and converts the model to ONNX. It also verifies that the ONNX model could generate same input as the pytorch model. The usage is like \n", + "```\n", + "python -m onnxruntime_tools.transformers.convert_to_onnx -m model_name_or_path --output gpt2.onnx -o -p fp32|fp16|int8\n", + "```\n", + "The -p option can be used to choose the precision: fp32 (float32), fp16 (mixed precision) or int8 (quantization). The -o option will generate optimized model, which is required for fp16 or int8.\n", + "\n", + "Here we use a pretrained model as example:" + ] + }, + { + "cell_type": "code", + "execution_count": 3, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Some weights of MyGPT2LMHeadModel were not initialized from the model checkpoint at gpt2 and are newly initialized: ['h.0.attn.masked_bias', 'h.1.attn.masked_bias', 'h.2.attn.masked_bias', 'h.3.attn.masked_bias', 'h.4.attn.masked_bias', 'h.5.attn.masked_bias', 'h.6.attn.masked_bias', 'h.7.attn.masked_bias', 'h.8.attn.masked_bias', 'h.9.attn.masked_bias', 'h.10.attn.masked_bias', 'h.11.attn.masked_bias', 'lm_head.weight']\n", + "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n" + ] + }, + { + "name": "stdout", + "output_type": "stream", + "text": [ + "GPT2Config {\n", + " \"activation_function\": \"gelu_new\",\n", + " \"architectures\": [\n", + " \"GPT2LMHeadModel\"\n", + " ],\n", + " \"attn_pdrop\": 0.1,\n", + " \"bos_token_id\": 50256,\n", + " \"embd_pdrop\": 0.1,\n", + " \"eos_token_id\": 50256,\n", + " \"initializer_range\": 0.02,\n", + " \"layer_norm_epsilon\": 1e-05,\n", + " \"model_type\": \"gpt2\",\n", + " \"n_ctx\": 1024,\n", + " \"n_embd\": 768,\n", + " \"n_head\": 12,\n", + " \"n_layer\": 12,\n", + " \"n_positions\": 1024,\n", + " \"resid_pdrop\": 0.1,\n", + " \"summary_activation\": null,\n", + " \"summary_first_dropout\": 0.1,\n", + " \"summary_proj_to_labels\": true,\n", + " \"summary_type\": \"cls_index\",\n", + " \"summary_use_proj\": true,\n", + " \"task_specific_params\": {\n", + " \"text-generation\": {\n", + " \"do_sample\": true,\n", + " \"max_length\": 50\n", + " }\n", + " },\n", + " \"vocab_size\": 50257\n", + "}\n", + "\n" + ] + } + ], + "source": [ + "from onnxruntime_tools.transformers.gpt2_helper import Gpt2Helper, MyGPT2LMHeadModel\n", + "from transformers import AutoConfig\n", + "import torch\n", + "\n", + "model_name_or_path = \"gpt2\"\n", + "config = AutoConfig.from_pretrained(model_name_or_path, cache_dir=cache_dir)\n", + "model = MyGPT2LMHeadModel.from_pretrained(model_name_or_path, config=config, cache_dir=cache_dir)\n", + "device = torch.device(\"cpu\")\n", + "model.eval().to(device)\n", + "\n", + "print(model.config)\n", + "\n", + "num_attention_heads = model.config.n_head\n", + "hidden_size = model.config.n_embd\n", + "num_layer = model.config.n_layer" + ] + }, + { + "cell_type": "code", + "execution_count": 4, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "D:\\Anaconda3\\envs\\cpu_env\\lib\\site-packages\\transformers\\modeling_gpt2.py:445: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n", + " assert batch_size > 0, \"batch_size has to be defined and > 0\"\n", + "D:\\Anaconda3\\envs\\cpu_env\\lib\\site-packages\\transformers\\modeling_gpt2.py:149: TracerWarning: Converting a tensor to a Python float might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n", + " w = w / (float(v.size(-1)) ** 0.5)\n", + "D:\\Anaconda3\\envs\\cpu_env\\lib\\site-packages\\transformers\\modeling_gpt2.py:151: TracerWarning: Converting a tensor to a Python index might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n", + " mask = self.bias[:, :, ns - nd : ns, :ns]\n" + ] + } + ], + "source": [ + "onnx_model_path = \"gpt2.onnx\"\n", + "Gpt2Helper.export_onnx(model, device, onnx_model_path) # add parameter use_external_data_format=True when model size > 2 GB" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## PyTorch Inference using Huggingface Transformers##\n", + "\n", + "In the following, we will use an example input to get the output from PyTorch for comparison purpose.\n", + "For the first inference, there is no any past state. We can prepare empty state for input." + ] + }, + { + "cell_type": "code", + "execution_count": 5, + "metadata": {}, + "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Some weights of GPT2LMHeadModel were not initialized from the model checkpoint at gpt2 and are newly initialized: ['h.0.attn.masked_bias', 'h.1.attn.masked_bias', 'h.2.attn.masked_bias', 'h.3.attn.masked_bias', 'h.4.attn.masked_bias', 'h.5.attn.masked_bias', 'h.6.attn.masked_bias', 'h.7.attn.masked_bias', 'h.8.attn.masked_bias', 'h.9.attn.masked_bias', 'h.10.attn.masked_bias', 'h.11.attn.masked_bias', 'lm_head.weight']\n", + "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n" + ] + } + ], + "source": [ + "from transformers import AutoTokenizer\n", + "\n", + "EXAMPLE_Text = ['ONNX runtime is', 'here is an example of gpt2 model']\n", + "\n", + "def get_tokenizer(model_name_or_path, cache_dir):\n", + " tokenizer = AutoTokenizer.from_pretrained(model_name_or_path, cache_dir=cache_dir)\n", + " tokenizer.padding_side = \"left\"\n", + " tokenizer.pad_token = tokenizer.eos_token\n", + " #okenizer.add_special_tokens({'pad_token': '[PAD]'})\n", + " return tokenizer\n", + "\n", + "def get_example_inputs(prompt_text=EXAMPLE_Text, verbose=False): \n", + " tokenizer = get_tokenizer(model_name_or_path, cache_dir)\n", + " encodings_dict = tokenizer.batch_encode_plus(prompt_text, pad_to_max_length=True)\n", + "\n", + " input_ids = torch.tensor(encodings_dict['input_ids'], dtype=torch.int64)\n", + " attention_mask = torch.tensor(encodings_dict['attention_mask'], dtype=torch.float32)\n", + " position_ids = (attention_mask.long().cumsum(-1) - 1)\n", + " position_ids.masked_fill_(position_ids < 0, 0)\n", + "\n", + " #Empty Past State for generating first word\n", + " empty_past = []\n", + " batch_size = input_ids.size(0)\n", + " sequence_length = input_ids.size(1)\n", + " past_shape = [2, batch_size, num_attention_heads, 0, hidden_size // num_attention_heads]\n", + " for i in range(num_layer):\n", + " empty_past.append(torch.empty(past_shape).type(torch.float32).to(device))\n", + " \n", + " if verbose:\n", + " print(\"input_ids\", input_ids)\n", + " print(\"attention_mask\", attention_mask)\n", + " print(\"position_ids\", position_ids)\n", + " \n", + " return input_ids, attention_mask, position_ids, empty_past\n", + "\n", + "\n", + "from transformers import GPT2LMHeadModel\n", + "torch_model = GPT2LMHeadModel.from_pretrained(model_name_or_path, config=config, cache_dir=cache_dir)\n", + "device = torch.device(\"cpu\")\n", + "torch_model.eval().to(device)\n", + "\n", + "input_ids, attention_mask, position_ids, empty_past = get_example_inputs()\n", + "with torch.no_grad():\n", + " torch_output = torch_model(input_ids, past=empty_past, attention_mask=attention_mask, position_ids=position_ids)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## ONNX Runtime Inference ##\n", + "\n", + "We can use ONNX Runtime to inference. The inputs are dictionary with name and numpy array as value, and the output is list of numpy array. Note that both input and output are in CPU. When you run the inference in GPU, it will involve data copy between CPU and GPU for input and output.\n", + "\n", + "Let's create an inference session for ONNX Runtime given the exported ONNX model, and see the output." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [], + "source": [ + "import onnxruntime\n", + "import numpy\n", + "\n", + "input_ids, attention_mask, position_ids, empty_past = get_example_inputs()\n", + "\n", + "onnx_model_path = \"gpt2.onnx\"\n", + "session = onnxruntime.InferenceSession(onnx_model_path)\n", + "ort_inputs = {'input_ids': numpy.ascontiguousarray(input_ids.cpu().numpy()),\n", + " 'attention_mask' : numpy.ascontiguousarray(attention_mask.cpu().numpy()),\n", + " 'position_ids': numpy.ascontiguousarray(position_ids.cpu().numpy())\n", + " }\n", + "for i, past_i in enumerate(empty_past):\n", + " ort_inputs[f'past_{i}'] = numpy.ascontiguousarray(past_i.cpu().numpy())\n", + "ort_outputs = session.run(None, ort_inputs)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can compare the outputs from PyTorch and ONNX Runtime. Logits are very close (max difference is 1E-4)." + ] + }, + { + "cell_type": "code", + "execution_count": 7, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "max logits diff (ignored padding) tensor(0.0001)\n" + ] + } + ], + "source": [ + "logits_masked_diff = (torch_output[0] - ort_outputs[0]) * attention_mask.unsqueeze(2)\n", + "max_logits_diff = logits_masked_diff.abs().max()\n", + "print(\"max logits diff (ignored padding)\", max_logits_diff)\n", + "\n", + "#past_diff = [(torch_output[1][i] - ort_outputs[i + 1]).abs().max() for i in range(num_layer)]\n", + "#print(\"past state diff for each layer\", past_diff)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## ONNX Runtime Inference with IO Binding ##\n", + "\n", + "To avoid data copy for input and output, ONNX Runtime also supports IO Binding. User could provide some buffer for input and outputs. For GPU inference, the buffer can be in GPU to reduce memory copy between CPU and GPU. This is helpful for high performance inference in GPU. For GPT-2, IO Binding might help the performance when batch size or (past) sequence length is large." + ] + }, + { + "cell_type": "code", + "execution_count": 8, + "metadata": {}, + "outputs": [], + "source": [ + "def inference_with_io_binding(session, config, input_ids, position_ids, attention_mask, past):\n", + " output_shapes = Gpt2Helper.get_output_shapes(batch_size=input_ids.size(0),\n", + " past_sequence_length=past[0].size(3),\n", + " sequence_length=input_ids.size(1),\n", + " config=config)\n", + " output_buffers = Gpt2Helper.get_output_buffers(output_shapes, device)\n", + "\n", + " io_binding = Gpt2Helper.prepare_io_binding(session, input_ids, position_ids, attention_mask, past,\n", + " output_buffers, output_shapes)\n", + " session.run_with_iobinding(io_binding)\n", + "\n", + " outputs = Gpt2Helper.get_outputs_from_io_binding_buffer(session, output_buffers, output_shapes,\n", + " return_numpy=False)\n", + " return outputs" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "We can see that the result is exactly same with/without IO Binding:" + ] + }, + { + "cell_type": "code", + "execution_count": 9, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "IO Binding result is good\n" + ] + } + ], + "source": [ + "input_ids, attention_mask, position_ids, empty_past = get_example_inputs()\n", + "outputs = inference_with_io_binding(session, config, input_ids, position_ids, attention_mask, empty_past)\n", + "for i in range(len(outputs)):\n", + " assert torch.eq(outputs[i], torch.from_numpy(ort_outputs[i])).all()\n", + "print(\"IO Binding result is good\")" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Batch Text Generation ##\n", + "\n", + "Here is an example for text generation using ONNX Runtime or PyTorch. For ONNX Runtime, IO Binding is used for better performance." + ] + }, + { + "cell_type": "code", + "execution_count": 10, + "metadata": {}, + "outputs": [], + "source": [ + "def test_generation(tokenizer, input_text, use_onnxruntime=True):\n", + " print(\"Text generation using\", \"OnnxRuntime\" if use_onnxruntime else \"PyTorch\", \"...\")\n", + " eos_token_id = tokenizer.eos_token_id\n", + " \n", + " input_ids, attention_mask, position_ids, past = get_example_inputs(input_text)\n", + " batch_size = input_ids.size(0)\n", + "\n", + " has_eos = torch.zeros(batch_size, dtype=torch.bool)\n", + "\n", + " all_token_ids = input_ids.clone()\n", + "\n", + " num_tokens_to_produce = 30\n", + " for step in range(num_tokens_to_produce):\n", + " if use_onnxruntime:\n", + " outputs = inference_with_io_binding(session, config, input_ids, position_ids, attention_mask, past)\n", + " else:\n", + " outputs = torch_model(input_ids, attention_mask=attention_mask, position_ids=position_ids, past=past) \n", + "\n", + " next_token_logits = outputs[0][:, -1, :]\n", + " # Greedy approach is used here. You can easily extend it to use beam search and sampling to pick next tokens.\n", + " next_tokens = torch.argmax(next_token_logits, dim=-1)\n", + "\n", + " has_eos = has_eos | (next_tokens == eos_token_id)\n", + " tokens_to_add = next_tokens.masked_fill(has_eos, eos_token_id)\n", + " all_token_ids = torch.cat([all_token_ids, tokens_to_add.unsqueeze(-1)], dim=-1)\n", + "\n", + " # Update input_ids, attention_mask, position_ids and past\n", + " input_ids = tokens_to_add.clone().detach().reshape([batch_size, 1]).to(device) \n", + " position_ids = (position_ids[:,-1] + 1).reshape(batch_size,1)\n", + " attention_mask = torch.cat([attention_mask, torch.ones([batch_size, 1]).type_as(attention_mask)], 1).to(device) \n", + "\n", + " past = []\n", + " if not use_onnxruntime:\n", + " past = list(outputs[1]) # past in torch output is tuple\n", + " else:\n", + " for i in range(num_layer):\n", + " past_i = torch.from_numpy(outputs[i + 1]) if isinstance(outputs[i + 1], numpy.ndarray) else outputs[i + 1].clone().detach()\n", + " past.append(past_i.to(device))\n", + "\n", + " if torch.all(has_eos):\n", + " break\n", + "\n", + " for i, output in enumerate(all_token_ids):\n", + " print(f\"Example {i}:\", tokenizer.decode(output, skip_special_tokens=True))" + ] + }, + { + "cell_type": "code", + "execution_count": 11, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Text generation using OnnxRuntime ...\n", + "Example 0: ONNX runtime is not supported.\n", + "\n", + "The following is a list of the supported languages:\n", + "\n", + "English\n", + "\n", + "French\n", + "\n", + "German\n", + "\n", + "Italian\n", + "\n", + "Japanese\n", + "Example 1: here is an example of gpt2 model.\n", + "\n", + "The gpt2 model is a simple, but powerful, way to generate a GPT2-like data structure. It is a\n" + ] + } + ], + "source": [ + "tokenizer = get_tokenizer(model_name_or_path, cache_dir)\n", + "input_text = EXAMPLE_Text\n", + "test_generation(tokenizer, input_text, use_onnxruntime=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Next, we use PyTorch to run again and we can see that the result is exactly same." + ] + }, + { + "cell_type": "code", + "execution_count": 12, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Text generation using PyTorch ...\n", + "Example 0: ONNX runtime is not supported.\n", + "\n", + "The following is a list of the supported languages:\n", + "\n", + "English\n", + "\n", + "French\n", + "\n", + "German\n", + "\n", + "Italian\n", + "\n", + "Japanese\n", + "Example 1: here is an example of gpt2 model.\n", + "\n", + "The gpt2 model is a simple, but powerful, way to generate a GPT2-like data structure. It is a\n" + ] + } + ], + "source": [ + "test_generation(tokenizer, input_text, use_onnxruntime=False)" ] }, { @@ -76,332 +580,221 @@ "metadata": {}, "source": [ "## Benchmark ##\n", - "\n", - "You will need git clone the onnxruntime repository like\n", - "```console\n", - "git clone https://github.com/microsoft/onnxruntime.git\n", - "```\n", - "Then update the bert_tools_dir according to the path in your machine." + "There is a tool benchmark_gpt2.py, which can be used to measure the performance of GPT-2 by PyTorch, ONNX Runtime without/with IO Binding." ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 13, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "ATen/Parallel:\n", + "\tat::get_num_threads() : 12\n", + "\tat::get_num_interop_threads() : 6\n", + "OpenMP 2019\n", + "\tomp_get_max_threads() : 12\n", + "Intel(R) Math Kernel Library Version 2020.0.0 Product Build 20191125 for Intel(R) 64 architecture applications\n", + "\tmkl_get_max_threads() : 12\n", + "Intel(R) MKL-DNN v1.5.0 (Git Hash e2ac1fac44c5078ca927cb9b90e1b3066a0b2ed0)\n", + "std::thread::hardware_concurrency() : 12\n", + "Environment variables:\n", + "\tOMP_NUM_THREADS : [not set]\n", + "\tMKL_NUM_THREADS : [not set]\n", + "ATen parallel backend: OpenMP\n", + "\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2020-08-14 00:40:03.220131: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library cudart64_101.dll\n", + "Arguments:Namespace(batch_sizes=[1], cache_dir='.\\\\cache_models', include_copy_output_latency=False, model_class='GPT2LMHeadModel', model_name='gpt2', onnx_dir='.\\\\onnx_models', optimize_onnx=True, past_sequence_lengths=[8, 16, 32, 64, 128, 256], precision=, result_csv=None, test_times=100, thread_num=-1, torchscript=False, use_gpu=False, validate_onnx=False, verbose=False)\n", + "PyTorch Version:1.6.0+cpu\n", + "Transformers Version:3.0.2\n", + "Onnxruntime Version:1.4.0\n", + "Some weights of MyGPT2LMHeadModel were not initialized from the model checkpoint at gpt2 and are newly initialized: ['h.0.attn.masked_bias', 'h.1.attn.masked_bias', 'h.2.attn.masked_bias', 'h.3.attn.masked_bias', 'h.4.attn.masked_bias', 'h.5.attn.masked_bias', 'h.6.attn.masked_bias', 'h.7.attn.masked_bias', 'h.8.attn.masked_bias', 'h.9.attn.masked_bias', 'h.10.attn.masked_bias', 'h.11.attn.masked_bias', 'lm_head.weight']\n", + "You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n", + "Shapes: input_ids=torch.Size([1, 1]) past=torch.Size([2, 1, 12, 1, 64]) output=torch.Size([1, 1, 50257]) present=torch.Size([2, 1, 12, 2, 64])\n", + "D:\\Anaconda3\\envs\\cpu_env\\lib\\site-packages\\transformers\\modeling_gpt2.py:445: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n", + " assert batch_size > 0, \"batch_size has to be defined and > 0\"\n", + "D:\\Anaconda3\\envs\\cpu_env\\lib\\site-packages\\transformers\\modeling_gpt2.py:149: TracerWarning: Converting a tensor to a Python float might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n", + " w = w / (float(v.size(-1)) ** 0.5)\n", + "D:\\Anaconda3\\envs\\cpu_env\\lib\\site-packages\\transformers\\modeling_gpt2.py:151: TracerWarning: Converting a tensor to a Python index might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n", + " mask = self.bias[:, :, ns - nd : ns, :ns]\n", + "Fused LayerNormalization count: 25\n", + "Fused FastGelu count: 12\n", + "Fused Attention(with past) count: 12\n", + "Graph pruned: 0 inputs, 0 outputs and 813 nodes are removed\n", + "Graph pruned: 0 inputs, 0 outputs and 360 nodes are removed\n", + "postprocess: remove Reshape count:48\n", + "Fused FastGelu(add bias) count: 12\n", + "opset verion: 11\n", + "Output model to .\\onnx_models\\gpt2_past_fp32.onnx\n", + "batch_size=1, past_sequence_length=8, torch_latency=41.62, ort_latency=31.69, ort_io_latency=31.39\n", + "batch_size=1, past_sequence_length=16, torch_latency=42.37, ort_latency=31.67, ort_io_latency=31.36\n", + "batch_size=1, past_sequence_length=32, torch_latency=43.34, ort_latency=32.20, ort_io_latency=32.25\n", + "batch_size=1, past_sequence_length=64, torch_latency=44.01, ort_latency=34.90, ort_io_latency=31.53\n", + "batch_size=1, past_sequence_length=128, torch_latency=47.43, ort_latency=39.52, ort_io_latency=33.05\n", + "batch_size=1, past_sequence_length=256, torch_latency=54.88, ort_latency=45.20, ort_io_latency=38.01\n", + "Results are saved to file benchmark_result_20200814-004123.csv\n" + ] + } + ], "source": [ - "# Assume you have git clone the repository of onnxruntime from github.\n", - "bert_tools_dir = r'D:\\Git\\onnxruntime\\onnxruntime\\python\\tools\\bert'\n", - "benchmark_script = os.path.join(bert_tools_dir, 'benchmark_gpt2.py')\n", - "\n", - "if enable_past_input:\n", - " %run $benchmark_script --model_type gpt2 --cache_dir $cache_dir --output_dir $output_dir --enable_optimization --enable_past_input\n", - "else:\n", - " %run $benchmark_script --model_type gpt2 --cache_dir $cache_dir --output_dir $output_dir --enable_optimization" + "!{sys.executable} -m onnxruntime_tools.transformers.benchmark_gpt2 -m gpt2 -o" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "If you only need the benchmark results. You can skip the remaining parts.\n", - "\n", - "In the following, we will introduce the benchmark script.\n", - "\n", - "### Load pretrained model" + "### Test Environment ###\n", + "The following is the hardware of the test machine, and software version:" ] }, { "cell_type": "code", - "execution_count": null, + "execution_count": 14, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "{\n", + " \"gpu\": {\n", + " \"driver_version\": \"442.23\",\n", + " \"devices\": [\n", + " {\n", + " \"memory_total\": 8589934592,\n", + " \"memory_available\": 5534052352,\n", + " \"name\": \"GeForce GTX 1070\"\n", + " }\n", + " ]\n", + " },\n", + " \"cpu\": {\n", + " \"brand\": \"Intel(R) Core(TM) i7-8700 CPU @ 3.20GHz\",\n", + " \"cores\": 6,\n", + " \"logical_cores\": 12,\n", + " \"hz\": \"3.1920 GHz\",\n", + " \"l2_cache\": \"1536 KB\",\n", + " \"flags\": [\n", + " \"3dnow\",\n", + " \"3dnowprefetch\",\n", + " \"abm\",\n", + " \"acpi\",\n", + " \"adx\",\n", + " \"aes\",\n", + " \"apic\",\n", + " \"avx\",\n", + " \"avx2\",\n", + " \"bmi1\",\n", + " \"bmi2\",\n", + " \"clflush\",\n", + " \"clflushopt\",\n", + " \"cmov\",\n", + " \"cx16\",\n", + " \"cx8\",\n", + " \"de\",\n", + " \"dtes64\",\n", + " \"dts\",\n", + " \"erms\",\n", + " \"est\",\n", + " \"f16c\",\n", + " \"fma\",\n", + " \"fpu\",\n", + " \"fxsr\",\n", + " \"hle\",\n", + " \"ht\",\n", + " \"hypervisor\",\n", + " \"ia64\",\n", + " \"invpcid\",\n", + " \"lahf_lm\",\n", + " \"mca\",\n", + " \"mce\",\n", + " \"mmx\",\n", + " \"movbe\",\n", + " \"mpx\",\n", + " \"msr\",\n", + " \"mtrr\",\n", + " \"osxsave\",\n", + " \"pae\",\n", + " \"pat\",\n", + " \"pbe\",\n", + " \"pcid\",\n", + " \"pclmulqdq\",\n", + " \"pdcm\",\n", + " \"pge\",\n", + " \"pni\",\n", + " \"popcnt\",\n", + " \"pse\",\n", + " \"pse36\",\n", + " \"rdrnd\",\n", + " \"rdseed\",\n", + " \"rtm\",\n", + " \"sep\",\n", + " \"serial\",\n", + " \"sgx\",\n", + " \"sgx_lc\",\n", + " \"smap\",\n", + " \"smep\",\n", + " \"ss\",\n", + " \"sse\",\n", + " \"sse2\",\n", + " \"sse4_1\",\n", + " \"sse4_2\",\n", + " \"ssse3\",\n", + " \"tm\",\n", + " \"tm2\",\n", + " \"tsc\",\n", + " \"vme\",\n", + " \"x2apic\",\n", + " \"xsave\",\n", + " \"xtpr\"\n", + " ],\n", + " \"processor\": \"Intel64 Family 6 Model 158 Stepping 10, GenuineIntel\"\n", + " },\n", + " \"memory\": {\n", + " \"total\": 16971276288,\n", + " \"available\": 3952107520\n", + " },\n", + " \"python\": \"3.6.10.final.0 (64 bit)\",\n", + " \"os\": \"Windows-10-10.0.19041-SP0\",\n", + " \"onnxruntime\": {\n", + " \"version\": \"1.4.0\",\n", + " \"support_gpu\": false\n", + " },\n", + " \"onnxruntime_tools\": {\n", + " \"version\": \"1.4.2\"\n", + " },\n", + " \"pytorch\": {\n", + " \"version\": \"1.6.0+cpu\",\n", + " \"support_gpu\": false,\n", + " \"cuda\": null\n", + " },\n", + " \"tensorflow\": {\n", + " \"version\": \"2.3.0\",\n", + " \"git_version\": \"v2.3.0-rc2-23-gb36436b087\",\n", + " \"support_gpu\": true\n", + " }\n", + "}\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "2020-08-14 00:42:57.062514: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library cudart64_101.dll\n" + ] + } + ], "source": [ - "from transformers import GPT2Model, GPT2Tokenizer\n", - "model_class, tokenizer_class, model_name_or_path = (GPT2Model, GPT2Tokenizer, 'gpt2')\n", - "tokenizer = tokenizer_class.from_pretrained(model_name_or_path, cache_dir=cache_dir)\n", - "model = model_class.from_pretrained(model_name_or_path, cache_dir=cache_dir)\n", - "model.eval().cpu()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import numpy\n", - "import time\n", - "\n", - "def pytorch_inference(model, input_ids, past=None, total_runs = 100):\n", - " latency = []\n", - " with torch.no_grad():\n", - " for _ in range(total_runs):\n", - " start = time.time()\n", - " outputs = model(input_ids=input_ids, past=past)\n", - " latency.append(time.time() - start)\n", - " \n", - " if total_runs > 1:\n", - " print(\"PyTorch Inference time = {} ms\".format(format(sum(latency) * 1000 / len(latency), '.2f')))\n", - " \n", - " return outputs\n", - " \n", - "def onnxruntime_inference(ort_session, input_ids, past=None, total_runs=100): \n", - " # Use contiguous array as input might improve performance.\n", - " # You can check the results from performance test tool to see whether you need it.\n", - " ort_inputs = {\n", - " 'input_ids': numpy.ascontiguousarray(input_ids.cpu().numpy())\n", - " }\n", - " \n", - " if past is not None:\n", - " for i, past_i in enumerate(past):\n", - " ort_inputs[f'past_{i}'] = numpy.ascontiguousarray(past[i].cpu().numpy())\n", - " \n", - " latency = []\n", - " for _ in range(total_runs):\n", - " start = time.time()\n", - " ort_outputs = ort_session.run(None, ort_inputs)\n", - " latency.append(time.time() - start)\n", - " \n", - " if total_runs > 1:\n", - " print(\"OnnxRuntime Inference time = {} ms\".format(format(sum(latency) * 1000 / len(latency), '.2f')))\n", - " \n", - " return ort_outputs\n", - "\n", - "def inference(model, ort_session, input_ids, past=None, total_runs=100, verify_outputs=True):\n", - " outputs = pytorch_inference(model, input_ids, past, total_runs)\n", - " ort_outputs = onnxruntime_inference(ort_session, input_ids, past, total_runs)\n", - " if verify_outputs:\n", - " print('PyTorch and OnnxRuntime output 0 (last_state) are close:'.format(0), numpy.allclose(ort_outputs[0], outputs[0].cpu(), rtol=1e-05, atol=1e-04))\n", - "\n", - " if enable_past_input:\n", - " for layer in range(model.config.n_layer):\n", - " print('PyTorch and OnnxRuntime layer {} state (present_{}) are close:'.format(layer, layer), numpy.allclose(ort_outputs[1 + layer], outputs[1][layer].cpu(), rtol=1e-05, atol=1e-04)) " - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import torch\n", - "import os\n", - "\n", - "inputs = tokenizer.encode_plus(\"Here is an example input for GPT2 model\", add_special_tokens=True, return_tensors='pt')\n", - "input_ids = inputs['input_ids']\n", - "\n", - "# run without past so that we can know the shape of past from output.\n", - "outputs = model(input_ids=input_ids, past=None)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "num_layer = model.config.n_layer \n", - "present_names = [f'present_{i}' for i in range(num_layer)]\n", - "output_names = [\"last_state\"] + present_names\n", - "\n", - "input_names = ['input_ids']\n", - "dynamic_axes= {'input_ids': {0: 'batch_size', 1: 'seq_len'},\n", - " #'token_type_ids' : {0: 'batch_size', 1: 'seq_len'},\n", - " #'attention_mask' : {0: 'batch_size', 1: 'seq_len'},\n", - " 'last_state' : {0: 'batch_size', 1: 'seq_len'}\n", - " }\n", - "for name in present_names:\n", - " dynamic_axes[name] = {1: 'batch_size', 3: 'seq_len'}\n", - " \n", - "if enable_past_input:\n", - " past_names = [f'past_{i}' for i in range(num_layer)]\n", - " input_names = ['input_ids'] + past_names #+ ['token_type_ids', 'attention_mask']\n", - " dummy_past = [torch.zeros(list(outputs[1][0].shape)) for _ in range(num_layer)]\n", - " for name in past_names:\n", - " dynamic_axes[name] = {1: 'batch_size', 3: 'seq_len'}\n", - " export_inputs = (inputs['input_ids'], tuple(dummy_past)) #, inputs['token_type_ids'], inputs['attention_mask'])\n", - "else:\n", - " export_inputs = (inputs['input_ids'])\n", - "\n", - "export_model_path = os.path.join(output_dir, 'gpt2_past{}.onnx'.format(int(enable_past_input)))\n", - "\n", - "torch.onnx.export(model,\n", - " args=export_inputs,\n", - " f=export_model_path,\n", - " input_names=input_names,\n", - " output_names=output_names,\n", - " dynamic_axes=dynamic_axes,\n", - " opset_version=11,\n", - " do_constant_folding = True,\n", - " verbose=False)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "def remove_past_outputs(export_model_path, output_model_path):\n", - " from onnx import ModelProto\n", - " from OnnxModel import OnnxModel\n", - "\n", - " model = ModelProto()\n", - " with open(export_model_path, \"rb\") as f:\n", - " model.ParseFromString(f.read())\n", - " bert_model = OnnxModel(model)\n", - "\n", - " # remove past state outputs and only keep the first output.\n", - " keep_output_names = [bert_model.model.graph.output[0].name]\n", - " logger.info(f\"Prune graph to keep the first output and drop past state outputs:{keep_output_names}\")\n", - " bert_model.prune_graph(keep_output_names)\n", - "\n", - " bert_model.save_model_to_file(output_model_path)\n", - " \n", - "if enable_past_input:\n", - " onnx_model_path = export_model_path\n", - "else:\n", - " onnx_model_path = os.path.join(output_dir, 'gpt2_past{}_out1.onnx'.format(int(enable_past_input)))\n", - " remove_past_outputs(export_model_path, onnx_model_path)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Inference with ONNX Runtime\n", - "\n", - "### OpenMP Environment Variable\n", - "\n", - "OpenMP environment variables are very important for CPU inference of GPT2 model. It has large performance impact on GPT2 model so you might need set it carefully according to benchmark script.\n", - "\n", - "Setting environment variables shall be done before importing onnxruntime. Otherwise, they might not take effect." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import psutil\n", - "\n", - "# You may change the settings in this cell according to Performance Test Tool result.\n", - "use_openmp = True\n", - "\n", - "# ATTENTION: these environment variables must be set before importing onnxruntime.\n", - "if use_openmp:\n", - " os.environ[\"OMP_NUM_THREADS\"] = str(psutil.cpu_count(logical=True))\n", - "else:\n", - " os.environ[\"OMP_NUM_THREADS\"] = '1'\n", - "\n", - "os.environ[\"OMP_WAIT_POLICY\"] = 'ACTIVE'" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import onnxruntime\n", - "import numpy\n", - "\n", - "# Print warning if user uses onnxruntime-gpu instead of onnxruntime package.\n", - "if 'CUDAExecutionProvider' in onnxruntime.get_available_providers():\n", - " print(\"warning: onnxruntime-gpu is not built with OpenMP. You might try onnxruntime package to test CPU inference.\")\n", - "\n", - "sess_options = onnxruntime.SessionOptions()\n", - "\n", - "# Optional: store the optimized graph and view it using Netron to verify that model is fully optimized.\n", - "# Note that this will increase session creation time, so it is for debugging only.\n", - "#sess_options.optimized_model_filepath = os.path.join(output_dir, \"optimized_model_cpu.onnx\")\n", - " \n", - "if use_openmp:\n", - " sess_options.intra_op_num_threads=1\n", - "else:\n", - " sess_options.intra_op_num_threads=psutil.cpu_count(logical=True)\n", - "\n", - "# Specify providers when you use onnxruntime-gpu for CPU inference.\n", - "session = onnxruntime.InferenceSession(onnx_model_path, sess_options, providers=['CPUExecutionProvider'])\n", - "\n", - "# Compare PyTorch and OnnxRuntime inference performance and results\n", - "%time inference(model, session, input_ids, past=dummy_past if enable_past_input else None)" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "import gc\n", - "del session\n", - "gc.collect()" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "optimized_model = os.path.join(output_dir, 'gpt2_past{}_optimized.onnx'.format(int(enable_past_input)))" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "bert_opt_script = os.path.join(bert_tools_dir, 'optimizer.py')" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "# Local directory corresponding to https://github.com/microsoft/onnxruntime/tree/master/onnxruntime/python/tools/transformers/\n", - "%run $bert_opt_script --model_type gpt2 --input $onnx_model_path --output $optimized_model --opt_level 0" - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "session = onnxruntime.InferenceSession(optimized_model, sess_options, providers=['CPUExecutionProvider'])\n", - "\n", - "%time inference(model, session, input_ids, past=dummy_past if enable_past_input else None, verify_outputs=False)" - ] - }, - { - "cell_type": "markdown", - "metadata": {}, - "source": [ - "## Additional Info\n", - "\n", - "Note that running Jupyter Notebook has slight impact on performance result since Jupyter Notebook is using system resources like CPU and memory etc. It is recommended to close Jupyter Notebook and other applications, then run the benchmark script in a console to get more accurate performance numbers.\n", - "\n", - "[OnnxRuntime C API](https://github.com/microsoft/onnxruntime/blob/master/docs/C_API.md) could get slightly better performance than python API. If you use C API in inference, you can use OnnxRuntime_Perf_Test.exe built from source to measure performance instead.\n", - "\n", - "Here is the machine configuration that generated the above results. The machine has GPU but not used in CPU inference.\n", - "You might get slower or faster result based on your hardware." - ] - }, - { - "cell_type": "code", - "execution_count": null, - "metadata": {}, - "outputs": [], - "source": [ - "machine_info_script = os.path.join(bert_tools_dir, 'MachineInfo.py')\n", - "%run $machine_info_script --silent" + "!{sys.executable} -m onnxruntime_tools.transformers.machine_info --silent" ] } ],