mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-27 20:02:15 +00:00
[NupharEP] update tutorial with GPT-2 (#2677)
This commit is contained in:
KeDengMS
GitHub
parent
b38b36a941
commit
c767e264c5
1 changed files with 142 additions and 34 deletions
|
|
@ -24,7 +24,7 @@
|
|||
"2. Create and run inference on a simple ONNX model, and understand how ***compilation*** works in Nuphar.\n",
|
||||
"3. Create and run inference on a model using ***LSTM***, run symbolic shape inference, edit LSTM ops to Scan, and check Nuphar speedup.\n",
|
||||
"4. ***Quantize*** the LSTM model and check speedup in Nuphar (CPU with AVX2 support is required).\n",
|
||||
"5. Working on real models from onnx model zoo: ***BERT squad*** and ***Bidirectional Attention Flow ([BiDAF](https://arxiv.org/pdf/1611.01603))***.\n",
|
||||
"5. Working on real models from onnx model zoo: ***BERT squad***, ***GPT-2*** and ***Bidirectional Attention Flow ([BiDAF](https://arxiv.org/pdf/1611.01603))***.\n",
|
||||
"6. ***Ahead-Of-Time (AOT) compilation*** to save just-in-time compilation cost on model load.\n",
|
||||
"7. Performance tuning for single thread inference.\n"
|
||||
]
|
||||
|
|
@ -216,8 +216,8 @@
|
|||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Fusion speed-up 434.50%\n",
|
||||
" Baseline: 0.716 s, Current: 0.134 s\n"
|
||||
"Fusion speed-up 436.78%\n",
|
||||
" Baseline: 0.725 s, Current: 0.135 s\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
|
@ -339,8 +339,8 @@
|
|||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Nuphar Scan speed-up 7.68%\n",
|
||||
" Baseline: 3.037 s, Current: 2.821 s\n"
|
||||
"Nuphar Scan speed-up 3.37%\n",
|
||||
" Baseline: 3.099 s, Current: 2.998 s\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
|
@ -444,8 +444,8 @@
|
|||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Quantization speed-up 278.52%\n",
|
||||
" Baseline: 2.821 s, Current: 0.745 s\n"
|
||||
"Quantization speed-up 299.66%\n",
|
||||
" Baseline: 2.998 s, Current: 0.750 s\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
|
@ -575,8 +575,8 @@
|
|||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Nuphar BERT squad speed-up 65.18%\n",
|
||||
" Baseline: 5.023 s, Current: 3.041 s\n"
|
||||
"Nuphar BERT squad speed-up 60.13%\n",
|
||||
" Baseline: 4.928 s, Current: 3.077 s\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
|
@ -589,6 +589,112 @@
|
|||
"print_speedup('Nuphar BERT squad', end_baseline - start_baseline, end_nuphar - start_nuphar)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"### GPT-2 with fixed batch size\n",
|
||||
"GPT-2 is a language model using Generative Pre-Trained Transformer for text generation. With Nuphar, we may fuse and compile the model to accelerate inference on CPU.\n",
|
||||
"\n",
|
||||
"#### Download model and test data"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 21,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# download GPT-2 model\n",
|
||||
"cwd = os.getcwd()\n",
|
||||
"model_url = 'https://onnxzoo.blob.core.windows.net/models/opset_10/GPT2/GPT-2.tar.gz'\n",
|
||||
"model_local = os.path.join(cwd, 'GPT-2.tar.gz')\n",
|
||||
"if not os.path.exists(model_local):\n",
|
||||
" urllib.request.urlretrieve(model_url, model_local)\n",
|
||||
"with tarfile.open(model_local, 'r') as f:\n",
|
||||
" f.extractall(cwd)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"#### Change batch dimension to fixed value, and run symbolic shape inference\n",
|
||||
"The GPT-2 model from model zoo has a symbolic batch dimension. By replacing it with a fixed value, compiler would be able to generate better code."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 22,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"model_dir = os.path.join(cwd, 'GPT2')\n",
|
||||
"model = os.path.join(model_dir, 'model.onnx')\n",
|
||||
"\n",
|
||||
"# edit batch dimension from symbolic to int value for better codegen\n",
|
||||
"mp = onnx.load(model)\n",
|
||||
"mp.graph.input[0].type.tensor_type.shape.dim[0].dim_value = 1\n",
|
||||
"onnx.save(mp, model)\n",
|
||||
"\n",
|
||||
"model_with_shape_inference = os.path.join(model_dir, 'model_shaped.onnx')\n",
|
||||
"\n",
|
||||
"# run symbolic shape inference\n",
|
||||
"SymbolicShapeInference.infer_shapes(model, model_with_shape_inference, auto_merge=True)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"#### Run inference and compare accuracy/performance to CPU provider"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 23,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Nuphar GPT-2 speed-up 13.48%\n",
|
||||
" Baseline: 2.535 s, Current: 2.234 s\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"sess_options = onnxruntime.SessionOptions()\n",
|
||||
"sess_options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL\n",
|
||||
"sess_baseline = onnxruntime.InferenceSession(model, sess_options)\n",
|
||||
"sess_baseline.set_providers(['CPUExecutionProvider'])\n",
|
||||
"\n",
|
||||
"# load test data, note the tensor proto name in data does not match model, so override it in feed\n",
|
||||
"input_name = [i.name for i in sess_baseline.get_inputs()][0] # This model only has one input\n",
|
||||
"test_data_dir = os.path.join(model_dir, 'test_data_set_0')\n",
|
||||
"tp = onnx.load_tensor(os.path.join(test_data_dir, 'input_0.pb'))\n",
|
||||
"feed = {input_name:numpy_helper.to_array(tp).reshape(1,-1)} # the test data missed batch dimension\n",
|
||||
"output_baseline = sess_baseline.run([], feed)\n",
|
||||
"\n",
|
||||
"repeats = 100\n",
|
||||
"start_baseline = timer()\n",
|
||||
"for i in range(repeats):\n",
|
||||
" output = sess_baseline.run([], feed)\n",
|
||||
"end_baseline = timer()\n",
|
||||
"\n",
|
||||
"sess = onnxruntime.InferenceSession(model_with_shape_inference)\n",
|
||||
"output = sess.run([], feed)\n",
|
||||
"assert all([np.allclose(o, ob, atol=1e-4) for o, ob in zip(output, output_baseline)])\n",
|
||||
"\n",
|
||||
"start_nuphar = timer()\n",
|
||||
"for i in range(repeats):\n",
|
||||
" output = sess.run([], feed)\n",
|
||||
"end_nuphar = timer()\n",
|
||||
"\n",
|
||||
"print_speedup('Nuphar GPT-2', end_baseline - start_baseline, end_nuphar - start_nuphar)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
|
|
@ -602,7 +708,7 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 21,
|
||||
"execution_count": 24,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
|
|
@ -625,7 +731,7 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 22,
|
||||
"execution_count": 25,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
|
|
@ -648,7 +754,7 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 23,
|
||||
"execution_count": 26,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
|
|
@ -657,7 +763,7 @@
|
|||
"\"with 4:51 left in regulation , carolina got the ball on their own 24 - yard line with a chance to mount a game - winning drive , and soon faced 3rd - and - 9 . on the next play , miller stripped the ball away from newton , and after several players dove for it , it took a long bounce backwards and was recovered by ward , who returned it five yards to the panthers 4 - yard line . although several players dove into the pile to attempt to recover it , newton did not and his lack of aggression later earned him heavy criticism . meanwhile , denver ' s offense was kept out of the end zone for three plays , but a holding penalty on cornerback josh norman gave the broncos a new set of downs . then anderson scored on a 2 - yard touchdown run and manning completed a pass to bennie fowler for a 2 - point conversion , giving denver a 24 – 10 lead with 3:08 left and essentially putting the game away . carolina had two more drives , but failed to get a first down on each one .\""
|
||||
]
|
||||
},
|
||||
"execution_count": 23,
|
||||
"execution_count": 26,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
|
|
@ -675,7 +781,7 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 24,
|
||||
"execution_count": 27,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
|
|
@ -684,7 +790,7 @@
|
|||
"'who recovered the strip ball ?'"
|
||||
]
|
||||
},
|
||||
"execution_count": 24,
|
||||
"execution_count": 27,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
|
|
@ -702,7 +808,7 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 25,
|
||||
"execution_count": 28,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
|
|
@ -711,7 +817,7 @@
|
|||
"'ward'"
|
||||
]
|
||||
},
|
||||
"execution_count": 25,
|
||||
"execution_count": 28,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
|
|
@ -729,7 +835,7 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 26,
|
||||
"execution_count": 29,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
|
|
@ -758,15 +864,15 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 27,
|
||||
"execution_count": 30,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Nuphar quantized BiDAF speed-up 45.63%\n",
|
||||
" Baseline: 0.305 s, Current: 0.209 s\n"
|
||||
"Nuphar quantized BiDAF speed-up 47.77%\n",
|
||||
" Baseline: 1.564 s, Current: 1.058 s\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
|
@ -801,16 +907,16 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 28,
|
||||
"execution_count": 31,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"text/plain": [
|
||||
"'JIT took 4.655 seconds'"
|
||||
"'JIT took 4.721 seconds'"
|
||||
]
|
||||
},
|
||||
"execution_count": 28,
|
||||
"execution_count": 31,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
|
|
@ -824,8 +930,10 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 29,
|
||||
"metadata": {},
|
||||
"execution_count": 32,
|
||||
"metadata": {
|
||||
"scrolled": true
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# create a folder for JIT cache\n",
|
||||
|
|
@ -849,7 +957,7 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 30,
|
||||
"execution_count": 33,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
|
|
@ -858,7 +966,7 @@
|
|||
"['jit.so']"
|
||||
]
|
||||
},
|
||||
"execution_count": 30,
|
||||
"execution_count": 33,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
|
|
@ -880,15 +988,15 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 31,
|
||||
"execution_count": 34,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"AOT speed-up 967.73%\n",
|
||||
" Baseline: 4.655 s, Current: 0.436 s\n"
|
||||
"AOT speed-up 764.62%\n",
|
||||
" Baseline: 4.721 s, Current: 0.546 s\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
|
@ -912,15 +1020,15 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 32,
|
||||
"execution_count": 35,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Single thread perf w/o parallel schedule speed-up 2.83%\n",
|
||||
" Baseline: 0.315 s, Current: 0.307 s\n"
|
||||
"Single thread perf w/o parallel schedule speed-up 1.05%\n",
|
||||
" Baseline: 1.542 s, Current: 1.526 s\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
|
|
|||
Loading…
Reference in a new issue