From c767e264c52c3bac2c319b630d37f541f4d2a677 Mon Sep 17 00:00:00 2001 From: KeDengMS Date: Mon, 16 Dec 2019 17:57:34 -0800 Subject: [PATCH] [NupharEP] update tutorial with GPT-2 (#2677) --- .../onnxruntime-nuphar-tutorial.ipynb | 176 ++++++++++++++---- 1 file changed, 142 insertions(+), 34 deletions(-) diff --git a/docs/python/notebooks/onnxruntime-nuphar-tutorial.ipynb b/docs/python/notebooks/onnxruntime-nuphar-tutorial.ipynb index c7179ea967..2515dfed0b 100644 --- a/docs/python/notebooks/onnxruntime-nuphar-tutorial.ipynb +++ b/docs/python/notebooks/onnxruntime-nuphar-tutorial.ipynb @@ -24,7 +24,7 @@ "2. Create and run inference on a simple ONNX model, and understand how ***compilation*** works in Nuphar.\n", "3. Create and run inference on a model using ***LSTM***, run symbolic shape inference, edit LSTM ops to Scan, and check Nuphar speedup.\n", "4. ***Quantize*** the LSTM model and check speedup in Nuphar (CPU with AVX2 support is required).\n", - "5. Working on real models from onnx model zoo: ***BERT squad*** and ***Bidirectional Attention Flow ([BiDAF](https://arxiv.org/pdf/1611.01603))***.\n", + "5. Working on real models from onnx model zoo: ***BERT squad***, ***GPT-2*** and ***Bidirectional Attention Flow ([BiDAF](https://arxiv.org/pdf/1611.01603))***.\n", "6. ***Ahead-Of-Time (AOT) compilation*** to save just-in-time compilation cost on model load.\n", "7. Performance tuning for single thread inference.\n" ] @@ -216,8 +216,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "Fusion speed-up 434.50%\n", - " Baseline: 0.716 s, Current: 0.134 s\n" + "Fusion speed-up 436.78%\n", + " Baseline: 0.725 s, Current: 0.135 s\n" ] } ], @@ -339,8 +339,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "Nuphar Scan speed-up 7.68%\n", - " Baseline: 3.037 s, Current: 2.821 s\n" + "Nuphar Scan speed-up 3.37%\n", + " Baseline: 3.099 s, Current: 2.998 s\n" ] } ], @@ -444,8 +444,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "Quantization speed-up 278.52%\n", - " Baseline: 2.821 s, Current: 0.745 s\n" + "Quantization speed-up 299.66%\n", + " Baseline: 2.998 s, Current: 0.750 s\n" ] } ], @@ -575,8 +575,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "Nuphar BERT squad speed-up 65.18%\n", - " Baseline: 5.023 s, Current: 3.041 s\n" + "Nuphar BERT squad speed-up 60.13%\n", + " Baseline: 4.928 s, Current: 3.077 s\n" ] } ], @@ -589,6 +589,112 @@ "print_speedup('Nuphar BERT squad', end_baseline - start_baseline, end_nuphar - start_nuphar)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### GPT-2 with fixed batch size\n", + "GPT-2 is a language model using Generative Pre-Trained Transformer for text generation. With Nuphar, we may fuse and compile the model to accelerate inference on CPU.\n", + "\n", + "#### Download model and test data" + ] + }, + { + "cell_type": "code", + "execution_count": 21, + "metadata": {}, + "outputs": [], + "source": [ + "# download GPT-2 model\n", + "cwd = os.getcwd()\n", + "model_url = 'https://onnxzoo.blob.core.windows.net/models/opset_10/GPT2/GPT-2.tar.gz'\n", + "model_local = os.path.join(cwd, 'GPT-2.tar.gz')\n", + "if not os.path.exists(model_local):\n", + " urllib.request.urlretrieve(model_url, model_local)\n", + "with tarfile.open(model_local, 'r') as f:\n", + " f.extractall(cwd)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Change batch dimension to fixed value, and run symbolic shape inference\n", + "The GPT-2 model from model zoo has a symbolic batch dimension. By replacing it with a fixed value, compiler would be able to generate better code." + ] + }, + { + "cell_type": "code", + "execution_count": 22, + "metadata": {}, + "outputs": [], + "source": [ + "model_dir = os.path.join(cwd, 'GPT2')\n", + "model = os.path.join(model_dir, 'model.onnx')\n", + "\n", + "# edit batch dimension from symbolic to int value for better codegen\n", + "mp = onnx.load(model)\n", + "mp.graph.input[0].type.tensor_type.shape.dim[0].dim_value = 1\n", + "onnx.save(mp, model)\n", + "\n", + "model_with_shape_inference = os.path.join(model_dir, 'model_shaped.onnx')\n", + "\n", + "# run symbolic shape inference\n", + "SymbolicShapeInference.infer_shapes(model, model_with_shape_inference, auto_merge=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Run inference and compare accuracy/performance to CPU provider" + ] + }, + { + "cell_type": "code", + "execution_count": 23, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Nuphar GPT-2 speed-up 13.48%\n", + " Baseline: 2.535 s, Current: 2.234 s\n" + ] + } + ], + "source": [ + "sess_options = onnxruntime.SessionOptions()\n", + "sess_options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL\n", + "sess_baseline = onnxruntime.InferenceSession(model, sess_options)\n", + "sess_baseline.set_providers(['CPUExecutionProvider'])\n", + "\n", + "# load test data, note the tensor proto name in data does not match model, so override it in feed\n", + "input_name = [i.name for i in sess_baseline.get_inputs()][0] # This model only has one input\n", + "test_data_dir = os.path.join(model_dir, 'test_data_set_0')\n", + "tp = onnx.load_tensor(os.path.join(test_data_dir, 'input_0.pb'))\n", + "feed = {input_name:numpy_helper.to_array(tp).reshape(1,-1)} # the test data missed batch dimension\n", + "output_baseline = sess_baseline.run([], feed)\n", + "\n", + "repeats = 100\n", + "start_baseline = timer()\n", + "for i in range(repeats):\n", + " output = sess_baseline.run([], feed)\n", + "end_baseline = timer()\n", + "\n", + "sess = onnxruntime.InferenceSession(model_with_shape_inference)\n", + "output = sess.run([], feed)\n", + "assert all([np.allclose(o, ob, atol=1e-4) for o, ob in zip(output, output_baseline)])\n", + "\n", + "start_nuphar = timer()\n", + "for i in range(repeats):\n", + " output = sess.run([], feed)\n", + "end_nuphar = timer()\n", + "\n", + "print_speedup('Nuphar GPT-2', end_baseline - start_baseline, end_nuphar - start_nuphar)" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -602,7 +708,7 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 24, "metadata": {}, "outputs": [], "source": [ @@ -625,7 +731,7 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 25, "metadata": {}, "outputs": [], "source": [ @@ -648,7 +754,7 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 26, "metadata": {}, "outputs": [ { @@ -657,7 +763,7 @@ "\"with 4:51 left in regulation , carolina got the ball on their own 24 - yard line with a chance to mount a game - winning drive , and soon faced 3rd - and - 9 . on the next play , miller stripped the ball away from newton , and after several players dove for it , it took a long bounce backwards and was recovered by ward , who returned it five yards to the panthers 4 - yard line . although several players dove into the pile to attempt to recover it , newton did not and his lack of aggression later earned him heavy criticism . meanwhile , denver ' s offense was kept out of the end zone for three plays , but a holding penalty on cornerback josh norman gave the broncos a new set of downs . then anderson scored on a 2 - yard touchdown run and manning completed a pass to bennie fowler for a 2 - point conversion , giving denver a 24 – 10 lead with 3:08 left and essentially putting the game away . carolina had two more drives , but failed to get a first down on each one .\"" ] }, - "execution_count": 23, + "execution_count": 26, "metadata": {}, "output_type": "execute_result" } @@ -675,7 +781,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 27, "metadata": {}, "outputs": [ { @@ -684,7 +790,7 @@ "'who recovered the strip ball ?'" ] }, - "execution_count": 24, + "execution_count": 27, "metadata": {}, "output_type": "execute_result" } @@ -702,7 +808,7 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 28, "metadata": {}, "outputs": [ { @@ -711,7 +817,7 @@ "'ward'" ] }, - "execution_count": 25, + "execution_count": 28, "metadata": {}, "output_type": "execute_result" } @@ -729,7 +835,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 29, "metadata": {}, "outputs": [], "source": [ @@ -758,15 +864,15 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 30, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Nuphar quantized BiDAF speed-up 45.63%\n", - " Baseline: 0.305 s, Current: 0.209 s\n" + "Nuphar quantized BiDAF speed-up 47.77%\n", + " Baseline: 1.564 s, Current: 1.058 s\n" ] } ], @@ -801,16 +907,16 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 31, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "'JIT took 4.655 seconds'" + "'JIT took 4.721 seconds'" ] }, - "execution_count": 28, + "execution_count": 31, "metadata": {}, "output_type": "execute_result" } @@ -824,8 +930,10 @@ }, { "cell_type": "code", - "execution_count": 29, - "metadata": {}, + "execution_count": 32, + "metadata": { + "scrolled": true + }, "outputs": [], "source": [ "# create a folder for JIT cache\n", @@ -849,7 +957,7 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 33, "metadata": {}, "outputs": [ { @@ -858,7 +966,7 @@ "['jit.so']" ] }, - "execution_count": 30, + "execution_count": 33, "metadata": {}, "output_type": "execute_result" } @@ -880,15 +988,15 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 34, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "AOT speed-up 967.73%\n", - " Baseline: 4.655 s, Current: 0.436 s\n" + "AOT speed-up 764.62%\n", + " Baseline: 4.721 s, Current: 0.546 s\n" ] } ], @@ -912,15 +1020,15 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 35, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Single thread perf w/o parallel schedule speed-up 2.83%\n", - " Baseline: 0.315 s, Current: 0.307 s\n" + "Single thread perf w/o parallel schedule speed-up 1.05%\n", + " Baseline: 1.542 s, Current: 1.526 s\n" ] } ],