saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-28 20:11:22 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Update Bert Notebooks for ORT 1.3.0 (#4274)

Browse source 
* update keras notebook
* re-run pytorch bert notebook
This commit is contained in:
Tianlei Wu 2020-06-19 14:02:16 -07:00 committed by GitHub
parent 466511c1c3
commit e08181f74d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 571 additions and 1093 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1125
onnxruntime/python/tools/transformers/notebooks/PyTorch_Bert-Squad_OnnxRuntime_GPU.ipynb
View file

File diff suppressed because it is too large Load diff

539
onnxruntime/python/tools/transformers/notebooks/Tensorflow_Keras_Bert-Squad_OnnxRuntime_CPU.ipynb
View file

@ -34,7 +34,6 @@
"```console\n",
"conda create -n cpu_env python=3.6\n",
"conda activate cpu_env\n",
"\n",
"conda install -c anaconda ipykernel\n",
"conda install -c conda-forge ipywidgets\n",
"python -m ipykernel install --user --name=cpu_env\n",
@ -54,17 +53,20 @@
"outputs": [],
"source": [
"import sys\n",
"\n",
"!{sys.executable} -m pip install --quiet --upgrade tensorflow==2.1.0\n",
" \n",
"!{sys.executable} -m pip install --quiet --upgrade tensorflow==2.2.0\n",
"!{sys.executable} -m pip install --quiet --upgrade onnxruntime\n",
"\n",
"# Install keras2onnx from source, since the latest package (1.6.0) does not support bert models from tensorflow 2.1 currently.\n",
"!{sys.executable} -m pip install --quiet git+https://github.com/microsoft/onnxconverter-common\n",
"!{sys.executable} -m pip install --quiet git+https://github.com/onnx/keras-onnx\n",
" \n",
"# Install other packages used in this notebook. \n",
"!{sys.executable} -m pip install --quiet transformers==2.5.1\n",
"!{sys.executable} -m pip install --quiet wget psutil onnx pytz pandas py-cpuinfo py3nvml"
"!{sys.executable} -m pip install --quiet --upgrade onnxruntime-tools\n",
"!{sys.executable} -m pip install --quiet --upgrade keras2onnx\n",
"!{sys.executable} -m pip install --quiet transformers==2.11.0\n",
"!{sys.executable} -m pip install --quiet wget pandas"
]
},
{
"cell_type": "markdown",
"metadata": {},
"source": [
"Let's define some constants:"
]
},
{
@ -74,56 +76,28 @@
"outputs": [],
"source": [
"# Whether allow overwrite existing script or model.\n",
"enable_overwrite = True\n",
"enable_overwrite = False\n",
"\n",
"# Number of runs to get average latency.\n",
"total_runs = 100"
"total_runs = 100\n",
"\n",
"# Max sequence length for the export model\n",
"max_sequence_length = 512"
]
},
{
"cell_type": "code",
"execution_count": 3,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
"100% [..............................................................................] 15310 / 15310Downloaded bert_perf_test.py\n",
"100% [................................................................................] 9571 / 9571Downloaded bert_test_data.py\n",
"100% [................................................................................] 7272 / 7272Downloaded compare_bert_results.py\n",
"100% [..............................................................................] 44905 / 44905Downloaded BertOnnxModel.py\n",
"100% [..............................................................................] 21565 / 21565Downloaded BertOnnxModelKeras.py\n",
"100% [..............................................................................] 26114 / 26114Downloaded BertOnnxModelTF.py\n",
"100% [..............................................................................] 22773 / 22773Downloaded OnnxModel.py\n",
"100% [................................................................................] 7795 / 7795Downloaded optimizer.py\n"
]
}
],
"source": [
"import os\n",
"import wget\n",
"\n",
"cache_dir = \"./squad\"\n",
"output_dir = \"./output\"\n",
"script_dir = './bert_scripts'\n",
"\n",
"for directory in [cache_dir, output_dir, script_dir]:\n",
" if not os.path.exists(directory):\n",
" os.makedirs(directory)\n",
"\n",
"# Download scripts for BERT optimization.\n",
"url_prfix = \"https://raw.githubusercontent.com/microsoft/onnxruntime/rel-1.3.0/onnxruntime/python/tools/bert/\"\n",
"script_files = ['bert_perf_test.py', 'bert_test_data.py', 'compare_bert_results.py', 'BertOnnxModel.py', 'BertOnnxModelKeras.py', 'BertOnnxModelTF.py', 'Gpt2OnnxModel.py', 'OnnxModel.py', 'optimizer.py']\n",
"\n",
"for filename in script_files:\n",
" target_file = os.path.join(script_dir, filename)\n",
" if enable_overwrite and os.path.exists(target_file):\n",
" os.remove(target_file)\n",
" if not os.path.exists(target_file):\n",
" wget.download(url_prfix + filename, target_file)\n",
" print(\"Downloaded\", filename)"
]
"outputs": [],
"source": [
"import os\n",
"cache_dir = './cached_models'\n",
"output_dir = './onnx_models'\n",
"for directory in [cache_dir, output_dir]:\n",
" if not os.path.exists(directory):\n",
" os.makedirs(directory)"
]
},
{
"cell_type": "markdown",
@ -136,7 +110,7 @@
"cell_type": "markdown",
"metadata": {},
"source": [
"Start to load fine-tuned model. This step take a few minutes to download the model (1.3 GB) for the first time."
"Start to load fine-tuned model. This step take a few minutes to download the model for the first time."
]
},
{
@ -149,11 +123,14 @@
"source": [
"from transformers import (TFBertForQuestionAnswering, BertTokenizer)\n",
"\n",
"model_name_or_path = 'bert-large-uncased-whole-word-masking-finetuned-squad'\n",
"#model_name_or_path = 'bert-large-uncased-whole-word-masking-finetuned-squad'\n",
"model_name_or_path = \"bert-base-cased\"\n",
"\n",
"# Load model and tokenizer\n",
"tokenizer = BertTokenizer.from_pretrained(model_name_or_path, do_lower_case=True, cache_dir=cache_dir)\n",
"model = TFBertForQuestionAnswering.from_pretrained(model_name_or_path, cache_dir=cache_dir)"
"model = TFBertForQuestionAnswering.from_pretrained(model_name_or_path, cache_dir=cache_dir)\n",
"# Needed this to export onnx model with multiple inputs with TF 2.2\n",
"model._saved_model_inputs_spec = None"
]
},
{
@ -174,15 +151,18 @@
"name": "stdout",
"output_type": "stream",
"text": [
"The answer is: a performance - focused inference engine for on ##nx models\n"
"The answer is: [CLS] what is on ##nx run ##time ? [SEP] on ##nx run ##time is a performance - focused in ##ference engine for on ##nx models\n"
]
}
],
"source": [
"import tensorflow as tf\n",
"import numpy\n",
"question, text = \"What is ONNX Runtime?\", \"ONNX Runtime is a performance-focused inference engine for ONNX models.\"\n",
"inputs = tokenizer.encode_plus(question, text, add_special_tokens=True, return_tensors='tf')\n",
"\n",
"question, text = \"What is ONNX Runtime?\", \"ONNX Runtime is a performance-focused inference engine for ONNX models.\"\n",
"# Pad to max length is needed. Otherwise, position embedding might be truncated by constant folding.\n",
"inputs = tokenizer.encode_plus(question, text, add_special_tokens=True, return_tensors='tf',\n",
" max_length=max_sequence_length, pad_to_max_length=True)\n",
"start_scores, end_scores = model(inputs)\n",
"\n",
"num_tokens = len(inputs[\"input_ids\"][0])\n",
@ -199,7 +179,7 @@
"name": "stdout",
"output_type": "stream",
"text": [
"Tensorflow Inference time for sequence length 26 = 227.06 ms\n"
"Tensorflow Inference time for sequence length 512 = 94.62 ms\n"
]
}
],
@ -218,36 +198,21 @@
"source": [
"## 3. Export model to ONNX using Keras2onnx\n",
"\n",
"Now we use Keras2onnx to export the model to ONNX format. It takes about 18 minutes for the large model."
"Now we use Keras2onnx to export the model to ONNX format. It takes about 3 minutes for bert-base, or 18 minutes for bert-large model."
]
},
{
"cell_type": "code",
"execution_count": 7,
"metadata": {},
"outputs": [
{
"name": "stderr",
"output_type": "stream",
"text": [
"The node number after optimization: 5257 -> 3836\n"
]
},
{
"name": "stdout",
"output_type": "stream",
"text": [
"Keras2onnx run time = 1052.26 s\n"
]
}
],
"outputs": [],
"source": [
"import keras2onnx\n",
"from keras2onnx.proto import keras\n",
"\n",
"output_model_path = os.path.join(output_dir, 'keras_{}.onnx'.format(model_name_or_path))\n",
"\n",
"if enable_overwrite or not os.path.exists(output_model_path):\n",
" model.predict(inputs)\n",
" start = time.time()\n",
" onnx_model = keras2onnx.convert_keras(model, model.name)\n",
" keras2onnx.save_model(onnx_model, output_model_path)\n",
@ -281,15 +246,8 @@
"import os\n",
"import psutil\n",
"\n",
"# You may change the settings in this cell according to Performance Test Tool result after running the whole notebook.\n",
"use_openmp = True\n",
"\n",
"# ATTENTION: these environment variables must be set before importing onnxruntime.\n",
"if use_openmp:\n",
" os.environ[\"OMP_NUM_THREADS\"] = str(psutil.cpu_count(logical=True))\n",
"else:\n",
" os.environ[\"OMP_NUM_THREADS\"] = '1'\n",
"\n",
"os.environ[\"OMP_NUM_THREADS\"] = str(psutil.cpu_count(logical=True))\n",
"os.environ[\"OMP_WAIT_POLICY\"] = 'ACTIVE'"
]
},
@ -309,7 +267,7 @@
"name": "stdout",
"output_type": "stream",
"text": [
"ONNX Runtime cpu inference time for sequence length 26 (model not optimized): 170.70 ms\n"
"ONNX Runtime cpu inference time for sequence length 512 (model not optimized): 630.54 ms\n"
]
}
],
@ -318,23 +276,17 @@
"import onnxruntime\n",
"import numpy\n",
"\n",
"# User might use onnxruntime-gpu for CPU inference.\n",
"if use_openmp and 'CUDAExecutionProvider' in onnxruntime.get_available_providers():\n",
" print(\"warning: onnxruntime-gpu is not built with OpenMP. You might try onnxruntime package.\")\n",
" \n",
"sess_options = onnxruntime.SessionOptions()\n",
"\n",
"# The following settings enables OpenMP, which is required to get best performance for CPU inference of Bert models.\n",
"if use_openmp:\n",
" sess_options.intra_op_num_threads=1\n",
"else:\n",
" sess_options.intra_op_num_threads=psutil.cpu_count(logical=True)\n",
"# intra_op_num_threads=1 can be used to enable OpenMP in OnnxRuntime 1.2.0.\n",
"# For OnnxRuntime 1.3.0 or later, this does not have effect unless you are using onnxruntime-gpu package.\n",
"sess_options.intra_op_num_threads=1\n",
"\n",
"# Providers is optional. Only needed when you use onnxruntime-gpu for CPU inference.\n",
"session = onnxruntime.InferenceSession(output_model_path, sess_options, providers=['CPUExecutionProvider'])\n",
"\n",
"# Use contiguous array as input could improve performance.\n",
"inputs_onnx = {k_: numpy.ascontiguousarray(v_.numpy()) for k_, v_ in inputs.items()}\n",
"batch_size = 1\n",
"inputs_onnx = {k_: numpy.repeat(v_, batch_size, axis=0) for k_, v_ in inputs.items()}\n",
"\n",
"# Warm up with one run.\n",
"results = session.run(None, inputs_onnx)\n",
@ -361,8 +313,8 @@
"WARNING:tensorflow:From <ipython-input-10-453158d8869f>:2: _EagerTensorBase.cpu (from tensorflow.python.framework.ops) is deprecated and will be removed in a future version.\n",
"Instructions for updating:\n",
"Use tf.identity instead.\n",
"start_scores are close: True\n",
"end_scores are close: True\n"
"start_scores are close: False\n",
"end_scores are close: False\n"
]
}
],
@ -396,38 +348,14 @@
"cell_type": "code",
"execution_count": 11,
"metadata": {},
"outputs": [
{
"name": "stdout",
"output_type": "stream",
"text": [
" BertOnnxModelTF.py: Fused LayerNormalization count: 49\n",
"BertOnnxModelKeras.py: Fused Gelu count:24\n",
"BertOnnxModelKeras.py: start processing embedding layer...\n",
"BertOnnxModelKeras.py: Found word embedding. name:tf_bert_for_question_answering/bert/embeddings/Gather/resource:0, shape:(30522, 1024)\n",
"BertOnnxModelKeras.py: Found word embedding. name:tf_bert_for_question_answering/bert/embeddings/position_embeddings/embedding_lookup/413066:0, shape:(512, 1024)\n",
"BertOnnxModelKeras.py: Found segment embedding. name:tf_bert_for_question_answering/bert/embeddings/token_type_embeddings/embedding_lookup/413071:0, shape:(2, 1024)\n",
"BertOnnxModelKeras.py: Create Embedding node\n",
" OnnxModel.py: Graph pruned: 0 inputs, 0 outputs and 9 nodes are removed\n",
"BertOnnxModelKeras.py: Fused mask\n",
"BertOnnxModelKeras.py: Skip consequent Reshape count: 24\n",
" BertOnnxModel.py: Fused Reshape count:0\n",
" BertOnnxModel.py: Fused SkipLayerNormalization count: 48\n",
"BertOnnxModelKeras.py: Fused Attention count:24\n",
" BertOnnxModel.py: Fused SkipLayerNormalization with Bias count:24\n",
"BertOnnxModelKeras.py: Remove 96 Reshape nodes.\n",
" OnnxModel.py: Graph pruned: 0 inputs, 0 outputs and 2160 nodes are removed\n",
" BertOnnxModel.py: opset verion: 11\n",
" OnnxModel.py: Output model to ./output\\keras_bert_large_opt_cpu.onnx\n",
" BertOnnxModel.py: EmbedLayer=1, Attention=24, Gelu=24, LayerNormalization=48, Succesful=True\n",
"optimizer.py: The output model is fully optimized.\n"
]
}
],
"outputs": [],
"source": [
"optimized_model_path = os.path.join(output_dir, 'keras_bert_large_opt_cpu.onnx')\n",
"optimized_model_path = os.path.join(output_dir, 'keras_{}_opt_cpu.onnx'.format(model_name_or_path))\n",
"\n",
"%run bert_scripts/optimizer.py --input $output_model_path --output $optimized_model_path --model_type bert_keras --num_heads 16 --hidden_size 1024"
"from onnxruntime_tools import optimizer\n",
"optimized_model = optimizer.optimize_model(output_model_path, model_type='bert_keras', num_heads=12, hidden_size=768)\n",
"optimized_model.use_dynamic_axes()\n",
"optimized_model.save_model_to_file(optimized_model_path)"
]
},
{
@ -446,7 +374,7 @@
"name": "stdout",
"output_type": "stream",
"text": [
"ONNX Runtime cpu inference time on optimized model: 133.35 ms\n"
"ONNX Runtime cpu inference time on optimized model: 369.18 ms\n"
]
}
],
@ -474,8 +402,8 @@
"output_type": "stream",
"text": [
"***** Verifying correctness (before and after optimization) *****\n",
"start_scores are close: True\n",
"end_scores are close: True\n"
"start_scores are close: False\n",
"end_scores are close: False\n"
]
}
],
@ -506,14 +434,14 @@
"output_type": "stream",
"text": [
"100% passed for 10 random inputs given thresholds (rtol=0.001, atol=0.0001).\n",
"maximum absolute difference=2.3484230041503906e-05\n",
"maximum relative difference=0.00013404049968812615\n"
"maximum absolute difference=1.2461096048355103e-06\n",
"maximum relative difference=0.006510902661830187\n"
]
}
],
"source": [
"# The base model is exported using sequence length 26\n",
"%run ./bert_scripts/compare_bert_results.py --baseline_model $output_model_path --optimized_model $optimized_model_path --batch_size 1 --sequence_length 26 --samples 10"
"# The baseline model is exported using max sequence length, and no dynamic axes\n",
"!{sys.executable} -m onnxruntime_tools.transformers.compare_bert_results --baseline_model $output_model_path --optimized_model $optimized_model_path --batch_size 1 --sequence_length $max_sequence_length --samples 10"
]
},
{
@ -524,9 +452,7 @@
"\n",
"This tool measures performance of BERT model inference using OnnxRuntime Python API.\n",
"\n",
"The following command will create 100 samples of batch_size 1 and sequence length 128 to run inference, then calculate performance numbers like average latency and throughput etc. \n",
"\n",
"It takes about 20 minutes to run this test. You can remove --all to reduce number of settings in the test."
"The following command will create 100 samples of batch_size 1 and sequence length 128 to run inference, then calculate performance numbers like average latency and throughput etc."
]
},
{
@ -538,14 +464,18 @@
"name": "stdout",
"output_type": "stream",
"text": [
"Running test: model=keras_bert-base-cased_opt_cpu.onnx,graph_optimization_level=ENABLE_ALL,intra_op_num_threads=1,OMP_NUM_THREADS=12,OMP_WAIT_POLICY=ACTIVE,batch_size=1,sequence_length=128,test_cases=100,test_times=1,contiguous=None,use_gpu=False,warmup=True\n",
"Average latency = 99.01 ms, Throughput = 10.10 QPS\n",
"test setting TestSetting(batch_size=1, sequence_length=128, test_cases=100, test_times=1, contiguous=None, use_gpu=False, warmup=True, omp_num_threads=12, omp_wait_policy='ACTIVE', intra_op_num_threads=1, seed=3, verbose=False, inclusive=False, extra_latency=True)\n",
"Generating 100 samples for batch_size=1 sequence_length=128\n",
"Extra latency for converting inputs to contiguous: 0.04 ms\n",
"Test summary is saved to output\\perf_results_CPU_B1_S128_20200319-141051.txt\n"
"Test summary is saved to onnx_models\\perf_results_CPU_B1_S128_20200617-210258.txt\n"
]
}
],
"source": [
"%run ./bert_scripts/bert_perf_test.py --model $optimized_model_path --batch_size 1 --sequence_length 128 --samples 100 --test_times 1 --inclusive --all"
"THREAD_SETTING = '--intra_op_num_threads 1 --omp_num_threads {} --omp_wait_policy ACTIVE'.format(psutil.cpu_count(logical=True))\n",
"\n",
"!{sys.executable} -m onnxruntime_tools.transformers.bert_perf_test --model $optimized_model_path --batch_size 1 --sequence_length 128 --samples 100 --test_times 1 --inclusive $THREAD_SETTING\n"
]
},
{
@ -564,8 +494,7 @@
"name": "stdout",
"output_type": "stream",
"text": [
"./output\\perf_results_CPU_B1_S128_20200319-141051.txt\n",
"The best setting is: use openmp; NO contiguous array\n"
"./onnx_models\\perf_results_CPU_B1_S128_20200617-210258.txt\n"
]
},
{
@ -603,298 +532,22 @@
" <th>0</th>\n",
" <td>1</td>\n",
" <td>12</td>\n",
" <td>PASSIVE</td>\n",
" <td>False</td>\n",
" <td>254.20</td>\n",
" <td>277.38</td>\n",
" <td>3.93</td>\n",
" </tr>\n",
" <tr>\n",
" <th>1</th>\n",
" <td>1</td>\n",
" <td>12</td>\n",
" <td>PASSIVE</td>\n",
" <td>True</td>\n",
" <td>255.47</td>\n",
" <td>283.56</td>\n",
" <td>3.91</td>\n",
" </tr>\n",
" <tr>\n",
" <th>2</th>\n",
" <td>1</td>\n",
" <td>12</td>\n",
" <td>ACTIVE</td>\n",
" <td>False</td>\n",
" <td>274.95</td>\n",
" <td>334.49</td>\n",
" <td>3.64</td>\n",
" </tr>\n",
" <tr>\n",
" <th>3</th>\n",
" <td>12</td>\n",
" <td>1</td>\n",
" <td>ACTIVE</td>\n",
" <td>True</td>\n",
" <td>278.91</td>\n",
" <td>294.30</td>\n",
" <td>3.59</td>\n",
" </tr>\n",
" <tr>\n",
" <th>4</th>\n",
" <td>1</td>\n",
" <td>12</td>\n",
" <td>ACTIVE</td>\n",
" <td>True</td>\n",
" <td>280.97</td>\n",
" <td>351.41</td>\n",
" <td>3.56</td>\n",
" </tr>\n",
" <tr>\n",
" <th>5</th>\n",
" <td>12</td>\n",
" <td>1</td>\n",
" <td>ACTIVE</td>\n",
" <td>False</td>\n",
" <td>281.89</td>\n",
" <td>296.11</td>\n",
" <td>3.55</td>\n",
" </tr>\n",
" <tr>\n",
" <th>6</th>\n",
" <td>12</td>\n",
" <td>1</td>\n",
" <td>PASSIVE</td>\n",
" <td>True</td>\n",
" <td>282.75</td>\n",
" <td>313.10</td>\n",
" <td>3.54</td>\n",
" </tr>\n",
" <tr>\n",
" <th>7</th>\n",
" <td>1</td>\n",
" <td></td>\n",
" <td></td>\n",
" <td>False</td>\n",
" <td>284.61</td>\n",
" <td>356.98</td>\n",
" <td>3.51</td>\n",
" </tr>\n",
" <tr>\n",
" <th>8</th>\n",
" <td>1</td>\n",
" <td></td>\n",
" <td></td>\n",
" <td>True</td>\n",
" <td>292.11</td>\n",
" <td>361.00</td>\n",
" <td>3.42</td>\n",
" </tr>\n",
" <tr>\n",
" <th>9</th>\n",
" <td>1</td>\n",
" <td>6</td>\n",
" <td>PASSIVE</td>\n",
" <td>False</td>\n",
" <td>292.28</td>\n",
" <td>346.47</td>\n",
" <td>3.42</td>\n",
" </tr>\n",
" <tr>\n",
" <th>10</th>\n",
" <td>12</td>\n",
" <td>1</td>\n",
" <td>PASSIVE</td>\n",
" <td>False</td>\n",
" <td>292.66</td>\n",
" <td>346.73</td>\n",
" <td>3.42</td>\n",
" </tr>\n",
" <tr>\n",
" <th>11</th>\n",
" <td>1</td>\n",
" <td>6</td>\n",
" <td>PASSIVE</td>\n",
" <td>True</td>\n",
" <td>310.60</td>\n",
" <td>443.16</td>\n",
" <td>3.22</td>\n",
" </tr>\n",
" <tr>\n",
" <th>12</th>\n",
" <td>1</td>\n",
" <td>6</td>\n",
" <td>ACTIVE</td>\n",
" <td>True</td>\n",
" <td>338.61</td>\n",
" <td>402.03</td>\n",
" <td>2.95</td>\n",
" </tr>\n",
" <tr>\n",
" <th>13</th>\n",
" <td>6</td>\n",
" <td>1</td>\n",
" <td>PASSIVE</td>\n",
" <td>True</td>\n",
" <td>362.84</td>\n",
" <td>378.08</td>\n",
" <td>2.76</td>\n",
" </tr>\n",
" <tr>\n",
" <th>14</th>\n",
" <td>6</td>\n",
" <td>1</td>\n",
" <td>ACTIVE</td>\n",
" <td>True</td>\n",
" <td>362.96</td>\n",
" <td>372.77</td>\n",
" <td>2.76</td>\n",
" </tr>\n",
" <tr>\n",
" <th>15</th>\n",
" <td>6</td>\n",
" <td>1</td>\n",
" <td>ACTIVE</td>\n",
" <td>False</td>\n",
" <td>363.42</td>\n",
" <td>392.19</td>\n",
" <td>2.75</td>\n",
" </tr>\n",
" <tr>\n",
" <th>16</th>\n",
" <td>6</td>\n",
" <td>1</td>\n",
" <td>PASSIVE</td>\n",
" <td>False</td>\n",
" <td>363.76</td>\n",
" <td>385.01</td>\n",
" <td>2.75</td>\n",
" </tr>\n",
" <tr>\n",
" <th>17</th>\n",
" <td>1</td>\n",
" <td>6</td>\n",
" <td>ACTIVE</td>\n",
" <td>False</td>\n",
" <td>367.37</td>\n",
" <td>423.17</td>\n",
" <td>2.72</td>\n",
" </tr>\n",
" <tr>\n",
" <th>18</th>\n",
" <td>6</td>\n",
" <td>6</td>\n",
" <td>ACTIVE</td>\n",
" <td>False</td>\n",
" <td>372.56</td>\n",
" <td>484.90</td>\n",
" <td>2.68</td>\n",
" </tr>\n",
" <tr>\n",
" <th>19</th>\n",
" <td>6</td>\n",
" <td>6</td>\n",
" <td>PASSIVE</td>\n",
" <td>True</td>\n",
" <td>383.58</td>\n",
" <td>408.13</td>\n",
" <td>2.61</td>\n",
" </tr>\n",
" <tr>\n",
" <th>20</th>\n",
" <td>6</td>\n",
" <td>6</td>\n",
" <td>PASSIVE</td>\n",
" <td>False</td>\n",
" <td>384.07</td>\n",
" <td>393.32</td>\n",
" <td>2.60</td>\n",
" </tr>\n",
" <tr>\n",
" <th>21</th>\n",
" <td>6</td>\n",
" <td>6</td>\n",
" <td>ACTIVE</td>\n",
" <td>True</td>\n",
" <td>388.30</td>\n",
" <td>647.23</td>\n",
" <td>2.58</td>\n",
" </tr>\n",
" <tr>\n",
" <th>22</th>\n",
" <td>0</td>\n",
" <td></td>\n",
" <td></td>\n",
" <td>True</td>\n",
" <td>423.20</td>\n",
" <td>465.31</td>\n",
" <td>2.36</td>\n",
" </tr>\n",
" <tr>\n",
" <th>23</th>\n",
" <td>0</td>\n",
" <td></td>\n",
" <td></td>\n",
" <td>False</td>\n",
" <td>448.80</td>\n",
" <td>550.75</td>\n",
" <td>2.23</td>\n",
" <td>None</td>\n",
" <td>99.01</td>\n",
" <td>130.11</td>\n",
" <td>10.1</td>\n",
" </tr>\n",
" </tbody>\n",
"</table>\n",
"</div>"
],
"text/plain": [
" intra_op_num_threads OMP_NUM_THREADS OMP_WAIT_POLICY contiguous \\\n",
"0 1 12 PASSIVE False \n",
"1 1 12 PASSIVE True \n",
"2 1 12 ACTIVE False \n",
"3 12 1 ACTIVE True \n",
"4 1 12 ACTIVE True \n",
"5 12 1 ACTIVE False \n",
"6 12 1 PASSIVE True \n",
"7 1 False \n",
"8 1 True \n",
"9 1 6 PASSIVE False \n",
"10 12 1 PASSIVE False \n",
"11 1 6 PASSIVE True \n",
"12 1 6 ACTIVE True \n",
"13 6 1 PASSIVE True \n",
"14 6 1 ACTIVE True \n",
"15 6 1 ACTIVE False \n",
"16 6 1 PASSIVE False \n",
"17 1 6 ACTIVE False \n",
"18 6 6 ACTIVE False \n",
"19 6 6 PASSIVE True \n",
"20 6 6 PASSIVE False \n",
"21 6 6 ACTIVE True \n",
"22 0 True \n",
"23 0 False \n",
" intra_op_num_threads OMP_NUM_THREADS OMP_WAIT_POLICY contiguous \\\n",
"0 1 12 ACTIVE None \n",
"\n",
" Latency(ms) Latency_P99 Throughput(QPS) \n",
"0 254.20 277.38 3.93 \n",
"1 255.47 283.56 3.91 \n",
"2 274.95 334.49 3.64 \n",
"3 278.91 294.30 3.59 \n",
"4 280.97 351.41 3.56 \n",
"5 281.89 296.11 3.55 \n",
"6 282.75 313.10 3.54 \n",
"7 284.61 356.98 3.51 \n",
"8 292.11 361.00 3.42 \n",
"9 292.28 346.47 3.42 \n",
"10 292.66 346.73 3.42 \n",
"11 310.60 443.16 3.22 \n",
"12 338.61 402.03 2.95 \n",
"13 362.84 378.08 2.76 \n",
"14 362.96 372.77 2.76 \n",
"15 363.42 392.19 2.75 \n",
"16 363.76 385.01 2.75 \n",
"17 367.37 423.17 2.72 \n",
"18 372.56 484.90 2.68 \n",
"19 383.58 408.13 2.61 \n",
"20 384.07 393.32 2.60 \n",
"21 388.30 647.23 2.58 \n",
"22 423.20 465.31 2.36 \n",
"23 448.80 550.75 2.23 "
" Latency(ms) Latency_P99 Throughput(QPS) \n",
"0 99.01 130.11 10.1 "
]
},
"execution_count": 16,
@ -909,7 +562,6 @@
"latest_result_file = max(glob.glob(os.path.join(output_dir, \"perf_results_*.txt\")), key=os.path.getmtime)\n",
"result_data = pandas.read_table(latest_result_file, converters={'OMP_NUM_THREADS': str, 'OMP_WAIT_POLICY':str})\n",
"print(latest_result_file)\n",
"print(\"The best setting is: {} openmp; {} contiguous array\".format('use' if result_data['intra_op_num_threads'].iloc[0] == 1 else 'NO', 'use' if result_data['contiguous'].iloc[0] else 'NO'))\n",
"\n",
"result_data.drop(['model', 'graph_optimization_level', 'batch_size', 'sequence_length', 'test_cases', 'test_times', 'use_gpu', 'warmup'], axis=1, inplace=True)\n",
"result_data.drop(['Latency_P50', 'Latency_P75', 'Latency_P90', 'Latency_P95'], axis=1, inplace=True)\n",
@ -925,7 +577,9 @@
"source": [
"## 6. Additional Info\n",
"\n",
"Note that running Jupyter Notebook has slight impact on performance result since Jupyter Notebook is using system resources like CPU and memory etc. It is recommended to close Jupyter Notebook and other applications, then run the performance test tool in a console to get more accurate performance numbers.\n",
"Note that running Jupyter Notebook has impact on performance result since Jupyter Notebook is using system resources like CPU and memory etc. It is recommended to close Jupyter Notebook and other applications, then run the performance test tool in a console to get more accurate performance numbers.\n",
"\n",
"We have a [benchmark script](https://github.com/microsoft/onnxruntime/blob/master/onnxruntime/python/tools/transformers/run_benchmark.sh). It is recommended to use it to measure inference speed of OnnxRuntime.\n",
"\n",
"[OnnxRuntime C API](https://github.com/microsoft/onnxruntime/blob/master/docs/C_API.md) could get slightly better performance than python API. If you use C API in inference, you can use OnnxRuntime_Perf_Test.exe built from source to measure performance instead.\n",
"\n",
@ -944,11 +598,11 @@
"text": [
"{\n",
" \"gpu\": {\n",
" \"driver_version\": \"441.22\",\n",
" \"driver_version\": \"442.23\",\n",
" \"devices\": [\n",
" {\n",
" \"memory_total\": 8589934592,\n",
" \"memory_available\": 611880960,\n",
" \"memory_available\": 1643134976,\n",
" \"name\": \"GeForce GTX 1070\"\n",
" }\n",
" ]\n",
@ -964,29 +618,36 @@
" },\n",
" \"memory\": {\n",
" \"total\": 16971259904,\n",
" \"available\": 6245142528\n",
" \"available\": 3282817024\n",
" },\n",
" \"python\": \"3.6.10.final.0 (64 bit)\",\n",
" \"os\": \"Windows-10-10.0.18362-SP0\",\n",
" \"onnxruntime\": {\n",
" \"version\": \"1.2.0\",\n",
" \"version\": \"1.3.0\",\n",
" \"support_gpu\": false\n",
" },\n",
" \"pytorch\": {\n",
" \"version\": \"1.4.0+cpu\",\n",
" \"version\": \"1.5.0+cpu\",\n",
" \"support_gpu\": false\n",
" },\n",
" \"tensorflow\": {\n",
" \"version\": \"2.1.0\",\n",
" \"git_version\": \"v2.1.0-rc2-17-ge5bf8de410\",\n",
" \"version\": \"2.2.0\",\n",
" \"git_version\": \"v2.2.0-rc4-8-g2b96f3662b\",\n",
" \"support_gpu\": true\n",
" }\n",
"}\n"
]
},
{
"name": "stderr",
"output_type": "stream",
"text": [
"2020-06-17 21:03:03.409601: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library cudart64_101.dll\n"
]
}
],
"source": [
"%run ./bert_scripts/MachineInfo.py --silent"
"!{sys.executable} -m onnxruntime_tools.transformers.machine_info --silent"
]
}
],