diff --git a/docs/python/notebooks/onnxruntime-nuphar-tutorial.ipynb b/docs/python/notebooks/onnxruntime-nuphar-tutorial.ipynb index 2515dfed0b..2872f35781 100644 --- a/docs/python/notebooks/onnxruntime-nuphar-tutorial.ipynb +++ b/docs/python/notebooks/onnxruntime-nuphar-tutorial.ipynb @@ -46,6 +46,7 @@ "outputs": [], "source": [ "import cpufeature\n", + "import hashlib\n", "import numpy as np\n", "import onnx\n", "from onnx import helper, numpy_helper\n", @@ -67,7 +68,20 @@ "\n", "def print_speedup(name, delta_baseline, delta):\n", " print(\"{} speed-up {:.2f}%\".format(name, 100*(delta_baseline/delta - 1)))\n", - " print(\" Baseline: {:.3f} s, Current: {:.3f} s\".format(delta_baseline, delta))" + " print(\" Baseline: {:.3f} s, Current: {:.3f} s\".format(delta_baseline, delta))\n", + "\n", + "def create_cache_dir(cache_dir):\n", + " # remove any stale cache files\n", + " if os.path.exists(cache_dir):\n", + " shutil.rmtree(cache_dir)\n", + " os.makedirs(cache_dir, exist_ok=True)\n", + "\n", + "def md5(file_name):\n", + " hash_md5 = hashlib.md5()\n", + " with open(file_name, \"rb\") as f:\n", + " for chunk in iter(lambda: f.read(4096), b\"\"):\n", + " hash_md5.update(chunk)\n", + " return hash_md5.hexdigest()" ] }, { @@ -86,7 +100,7 @@ "import onnxruntime\n", "from onnxruntime.nuphar.model_editor import convert_to_scan_model\n", "from onnxruntime.nuphar.model_quantizer import convert_matmul_model\n", - "from onnxruntime.nuphar.rnn_benchmark import generate_model\n", + "from onnxruntime.nuphar.rnn_benchmark import generate_model, perf_test\n", "from onnxruntime.nuphar.symbolic_shape_infer import SymbolicShapeInference" ] }, @@ -216,8 +230,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "Fusion speed-up 436.78%\n", - " Baseline: 0.725 s, Current: 0.135 s\n" + "Fusion speed-up 445.44%\n", + " Baseline: 0.732 s, Current: 0.134 s\n" ] } ], @@ -226,18 +240,18 @@ "batch = 16\n", "input_data = np.random.rand(seq, batch, dim).astype(np.float32)\n", "sess = onnxruntime.InferenceSession(simple_model_name)\n", - "feed = {X:input_data}\n", - "output = sess.run([], feed)\n", + "simple_feed = {X:input_data}\n", + "simple_output = sess.run([], simple_feed)\n", "np_output = ((((input_data + input_data) * input_data) + input_data) * input_data) + input_data\n", - "assert np.allclose(output[0], np_output)\n", + "assert np.allclose(simple_output[0], np_output)\n", "\n", - "repeats = 100\n", + "simple_repeats = 100\n", "start_ort = timer()\n", - "for i in range(repeats):\n", - " output = sess.run([], feed)\n", + "for i in range(simple_repeats):\n", + " sess.run([], simple_feed)\n", "end_ort = timer()\n", "start_np = timer()\n", - "for i in range(repeats):\n", + "for i in range(simple_repeats):\n", " np_output = ((((input_data + input_data) * input_data) + input_data) * input_data) + input_data\n", "end_np = timer()\n", "print_speedup('Fusion', end_np - start_np, end_ort - start_ort)" @@ -271,7 +285,9 @@ "collapsed": true }, "source": [ - "**IMPORTANT**: Nuphar generates code before knowing shapes of input data, unlike other execution providers that do runtime shape inference. Thus, shape inference information is critical for compiler optimizations in Nuphar. To do that, we run symbolic shape inference on the model. Symbolic shape inference is based on the ONNX shape inference, and enhanced by sympy to better handle Shape/ConstantOfShape/etc. ops using symbolic computation." + "**IMPORTANT**: Nuphar generates code before knowing shapes of input data, unlike other execution providers that do runtime shape inference. Thus, shape inference information is critical for compiler optimizations in Nuphar. To do that, we run symbolic shape inference on the model. Symbolic shape inference is based on the ONNX shape inference, and enhanced by sympy to better handle Shape/ConstantOfShape/etc. ops using symbolic computation.\n", + "\n", + "**IMPORTANT**: When running multi-threaded inference, Nuphar currently uses TVM's parallel schedule with has its own thread pool that's compatible with OpenMP and MKLML. The TVM thread pool has not been integrated with ONNX runtime thread pool, so intra_op_num_threads won't control it. Please make sure the build is with OpenMP or MKLML, and use OMP_NUM_THREADS to control thread pool." ] }, { @@ -298,12 +314,11 @@ "metadata": {}, "outputs": [], "source": [ - "sess_baseline = onnxruntime.InferenceSession(lstm_model)\n", - "sess_baseline.set_providers(['CPUExecutionProvider']) # default provider in this container is Nuphar, this overrides to CPU EP\n", + "sess_baseline = onnxruntime.InferenceSession(lstm_model, providers=['CPUExecutionProvider'])\n", "seq = 128\n", "input_data = np.random.rand(seq, 1, input_dim).astype(np.float32)\n", - "feed = {sess_baseline.get_inputs()[0].name:input_data}\n", - "output = sess_baseline.run([], feed)" + "lstm_feed = {sess_baseline.get_inputs()[0].name:input_data}\n", + "lstm_output = sess_baseline.run([], lstm_feed)" ] }, { @@ -319,8 +334,8 @@ "metadata": {}, "outputs": [], "source": [ - "scan_model = 'Scan_LSTMx4.onnx'\n", - "convert_to_scan_model(lstm_model, scan_model)" + "lstm_scan_model = 'Scan_LSTMx4.onnx'\n", + "convert_to_scan_model(lstm_model, lstm_scan_model)" ] }, { @@ -339,28 +354,28 @@ "name": "stdout", "output_type": "stream", "text": [ - "Nuphar Scan speed-up 3.37%\n", - " Baseline: 3.099 s, Current: 2.998 s\n" + "Nuphar Scan speed-up 9.22%\n", + " Baseline: 3.070 s, Current: 2.811 s\n" ] } ], "source": [ - "sess_nuphar = onnxruntime.InferenceSession(scan_model)\n", - "output_nuphar = sess_nuphar.run([], feed)\n", - "assert np.allclose(output[0], output_nuphar[0])\n", + "sess_nuphar = onnxruntime.InferenceSession(lstm_scan_model)\n", + "output_nuphar = sess_nuphar.run([], lstm_feed)\n", + "assert np.allclose(lstm_output[0], output_nuphar[0])\n", "\n", - "repeats = 10\n", - "start_baseline = timer()\n", - "for i in range(repeats):\n", - " output = sess_baseline.run([], feed)\n", - "end_baseline = timer()\n", + "lstm_repeats = 10\n", + "start_lstm_baseline = timer()\n", + "for i in range(lstm_repeats):\n", + " sess_baseline.run([], lstm_feed)\n", + "end_lstm_baseline = timer()\n", "\n", "start_nuphar = timer()\n", - "for i in range(repeats):\n", - " output = sess_nuphar.run([], feed)\n", + "for i in range(lstm_repeats):\n", + " sess_nuphar.run([], lstm_feed)\n", "end_nuphar = timer()\n", "\n", - "print_speedup('Nuphar Scan', end_baseline - start_baseline, end_nuphar - start_nuphar)" + "print_speedup('Nuphar Scan', end_lstm_baseline - start_lstm_baseline, end_nuphar - start_nuphar)" ] }, { @@ -406,8 +421,8 @@ "metadata": {}, "outputs": [], "source": [ - "quantized_model = 'Scan_LSTMx4_int8.onnx'\n", - "convert_matmul_model(scan_model, quantized_model)" + "lstm_quantized_model = 'Scan_LSTMx4_int8.onnx'\n", + "convert_matmul_model(lstm_scan_model, lstm_quantized_model)" ] }, { @@ -423,9 +438,9 @@ "metadata": {}, "outputs": [], "source": [ - "sess_quantized = onnxruntime.InferenceSession(quantized_model)\n", - "output_quantized = sess_quantized.run([], feed)\n", - "assert np.allclose(output[0], output_quantized[0], rtol=1e-3, atol=1e-3)" + "sess_quantized = onnxruntime.InferenceSession(lstm_quantized_model)\n", + "output_quantized = sess_quantized.run([], lstm_feed)\n", + "assert np.allclose(lstm_output[0], output_quantized[0], rtol=1e-3, atol=1e-3)" ] }, { @@ -444,27 +459,64 @@ "name": "stdout", "output_type": "stream", "text": [ - "Quantization speed-up 299.66%\n", - " Baseline: 2.998 s, Current: 0.750 s\n" + "Quantization speed-up 722.57%\n", + " Baseline: 2.811 s, Current: 0.342 s\n" ] } ], "source": [ "start_quantized = timer()\n", - "for i in range(repeats):\n", - " output = sess_quantized.run([], feed)\n", + "for i in range(lstm_repeats):\n", + " sess_quantized.run([], lstm_feed)\n", "end_quantized = timer()\n", "\n", "print_speedup('Quantization', end_nuphar - start_nuphar, end_quantized - start_quantized)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "To check RNN quantization performance, please use rnn_benchmark.perf_test." + ] + }, + { + "cell_type": "code", + "execution_count": 16, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "perf_rnn (with default threads) lstm_i80_h512_bi_l6_no_batch.onnx: run for 348 iterations, top 5 avg 28.368 ms\n", + "perf_scan (with 4 threads) lstm_i80_h512_bi_l6_no_batch_scan.onnx: run for 373 iterations, top 5 avg 25.819 ms\n", + "perf_int8 (with 4 threads) lstm_i80_h512_bi_l6_no_batch_int8.onnx: run for 778 iterations, top 5 avg 12.534 ms\n", + "Nuphar Quantization speed up speed-up 126.33%\n", + " Baseline: 0.028 s, Current: 0.013 s\n" + ] + } + ], + "source": [ + "rnn_type = 'lstm' # could be 'lstm', 'gru' or 'rnn'\n", + "num_threads = cpufeature.CPUFeature['num_physical_cores'] # no hyper thread\n", + "input_dim = 80 # size of input dimension\n", + "hidden_dim = 512 # size of hidden dimension in cell\n", + "bidirectional = True # specify RNN being bidirectional\n", + "layers = 6 # number of stacked RNN layers\n", + "seq_len = 40 # length of sequence\n", + "batch_size = 1 # size of batch\n", + "original_ms, scan_ms, int8_ms = perf_test(rnn_type, num_threads, input_dim, hidden_dim, bidirectional, layers, seq_len, batch_size)\n", + "print_speedup('Nuphar Quantization speed up', original_ms / 1000, int8_ms / 1000)" + ] + }, { "cell_type": "markdown", "metadata": {}, "source": [ "## 5. Working on real models\n", "\n", - "### BERT Squad\n", + "### 5.1 BERT Squad\n", "\n", "BERT (Bidirectional Encoder Representations from Transformers) applies Transformers to language modelling. With Nuphar, we may fuse and compile the model to accelerate inference on CPU.\n", "\n", @@ -473,17 +525,17 @@ }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 17, "metadata": {}, "outputs": [], "source": [ "# download BERT squad model\n", "cwd = os.getcwd()\n", - "model_url = 'https://onnxzoo.blob.core.windows.net/models/opset_10/bert_squad/download_sample_10.tar.gz'\n", - "model_local = os.path.join(cwd, 'download_sample_10.tar.gz')\n", - "if not os.path.exists(model_local):\n", - " urllib.request.urlretrieve(model_url, model_local)\n", - "with tarfile.open(model_local, 'r') as f:\n", + "bert_model_url = 'https://onnxzoo.blob.core.windows.net/models/opset_10/bert_squad/download_sample_10.tar.gz'\n", + "bert_model_local = os.path.join(cwd, 'download_sample_10.tar.gz')\n", + "if not os.path.exists(bert_model_local):\n", + " urllib.request.urlretrieve(bert_model_url, bert_model_local)\n", + "with tarfile.open(bert_model_local, 'r') as f:\n", " f.extractall(cwd)" ] }, @@ -497,16 +549,16 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 18, "metadata": {}, "outputs": [], "source": [ - "model_dir = os.path.join(cwd, 'download_sample_10')\n", - "model = os.path.join(model_dir, 'bertsquad10.onnx')\n", - "model_with_shape_inference = os.path.join(model_dir, 'bertsquad10_shaped.onnx')\n", + "bert_model_dir = os.path.join(cwd, 'download_sample_10')\n", + "bert_model = os.path.join(bert_model_dir, 'bertsquad10.onnx')\n", + "bert_model_with_shape_inference = os.path.join(bert_model_dir, 'bertsquad10_shaped.onnx')\n", "\n", "# run symbolic shape inference\n", - "SymbolicShapeInference.infer_shapes(model, model_with_shape_inference, auto_merge=True, int_max=100000)" + "SymbolicShapeInference.infer_shapes(bert_model, bert_model_with_shape_inference, auto_merge=True, int_max=100000)" ] }, { @@ -518,26 +570,25 @@ }, { "cell_type": "code", - "execution_count": 18, + "execution_count": 19, "metadata": {}, "outputs": [], "source": [ "sess_options = onnxruntime.SessionOptions()\n", "sess_options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL\n", - "sess_baseline = onnxruntime.InferenceSession(model, sess_options)\n", - "sess_baseline.set_providers(['CPUExecutionProvider'])\n", + "sess_baseline = onnxruntime.InferenceSession(bert_model, sess_options=sess_options, providers=['CPUExecutionProvider'])\n", "\n", "# load test data\n", - "test_data_dir = os.path.join(model_dir, 'test_data_set_1')\n", + "test_data_dir = os.path.join(bert_model_dir, 'test_data_set_1')\n", "tps = [onnx.load_tensor(os.path.join(test_data_dir, 'input_{}.pb'.format(i))) for i in range(len(sess_baseline.get_inputs()))]\n", - "feed = {tp.name:numpy_helper.to_array(tp) for tp in tps}\n", - "output_baseline = sess_baseline.run([], feed)\n", + "bert_feed = {tp.name:numpy_helper.to_array(tp) for tp in tps}\n", + "bert_output_baseline = sess_baseline.run([], bert_feed)\n", "\n", - "repeats = 20\n", - "start_baseline = timer()\n", - "for i in range(repeats):\n", - " output = sess_baseline.run([], feed)\n", - "end_baseline = timer()" + "bert_repeats = 20\n", + "start_bert_baseline = timer()\n", + "for i in range(bert_repeats):\n", + " sess_baseline.run([], bert_feed)\n", + "end_bert_baseline = timer()" ] }, { @@ -550,13 +601,13 @@ }, { "cell_type": "code", - "execution_count": 19, + "execution_count": 20, "metadata": {}, "outputs": [], "source": [ - "sess = onnxruntime.InferenceSession(model_with_shape_inference)\n", - "output = sess.run([], feed)\n", - "assert all([np.allclose(o, ob, atol=1e-4) for o, ob in zip(output, output_baseline)])" + "sess = onnxruntime.InferenceSession(bert_model_with_shape_inference)\n", + "output = sess.run([], bert_feed)\n", + "assert all([np.allclose(o, ob, atol=1e-4) for o, ob in zip(output, bert_output_baseline)])" ] }, { @@ -568,32 +619,32 @@ }, { "cell_type": "code", - "execution_count": 20, + "execution_count": 21, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Nuphar BERT squad speed-up 60.13%\n", - " Baseline: 4.928 s, Current: 3.077 s\n" + "Nuphar BERT squad speed-up 66.70%\n", + " Baseline: 5.093 s, Current: 3.055 s\n" ] } ], "source": [ "start_nuphar = timer()\n", - "for i in range(repeats):\n", - " output = sess.run([], feed)\n", + "for i in range(bert_repeats):\n", + " sess.run([], bert_feed)\n", "end_nuphar = timer()\n", "\n", - "print_speedup('Nuphar BERT squad', end_baseline - start_baseline, end_nuphar - start_nuphar)" + "print_speedup('Nuphar BERT squad', end_bert_baseline - start_bert_baseline, end_nuphar - start_nuphar)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### GPT-2 with fixed batch size\n", + "### 5.2 GPT-2 with fixed batch size\n", "GPT-2 is a language model using Generative Pre-Trained Transformer for text generation. With Nuphar, we may fuse and compile the model to accelerate inference on CPU.\n", "\n", "#### Download model and test data" @@ -601,17 +652,17 @@ }, { "cell_type": "code", - "execution_count": 21, + "execution_count": 22, "metadata": {}, "outputs": [], "source": [ "# download GPT-2 model\n", "cwd = os.getcwd()\n", - "model_url = 'https://onnxzoo.blob.core.windows.net/models/opset_10/GPT2/GPT-2.tar.gz'\n", - "model_local = os.path.join(cwd, 'GPT-2.tar.gz')\n", - "if not os.path.exists(model_local):\n", - " urllib.request.urlretrieve(model_url, model_local)\n", - "with tarfile.open(model_local, 'r') as f:\n", + "gpt2_model_url = 'https://onnxzoo.blob.core.windows.net/models/opset_10/GPT2/GPT-2.tar.gz'\n", + "gpt2_model_local = os.path.join(cwd, 'GPT-2.tar.gz')\n", + "if not os.path.exists(gpt2_model_local):\n", + " urllib.request.urlretrieve(gpt2_model_url, gpt2_model_local)\n", + "with tarfile.open(gpt2_model_local, 'r') as f:\n", " f.extractall(cwd)" ] }, @@ -625,22 +676,22 @@ }, { "cell_type": "code", - "execution_count": 22, + "execution_count": 23, "metadata": {}, "outputs": [], "source": [ - "model_dir = os.path.join(cwd, 'GPT2')\n", - "model = os.path.join(model_dir, 'model.onnx')\n", + "gpt2_model_dir = os.path.join(cwd, 'GPT2')\n", + "gpt2_model = os.path.join(gpt2_model_dir, 'model.onnx')\n", "\n", "# edit batch dimension from symbolic to int value for better codegen\n", - "mp = onnx.load(model)\n", + "mp = onnx.load(gpt2_model)\n", "mp.graph.input[0].type.tensor_type.shape.dim[0].dim_value = 1\n", - "onnx.save(mp, model)\n", + "onnx.save(mp, gpt2_model)\n", "\n", - "model_with_shape_inference = os.path.join(model_dir, 'model_shaped.onnx')\n", + "gpt2_model_with_shape_inference = os.path.join(gpt2_model_dir, 'model_shaped.onnx')\n", "\n", "# run symbolic shape inference\n", - "SymbolicShapeInference.infer_shapes(model, model_with_shape_inference, auto_merge=True)" + "SymbolicShapeInference.infer_shapes(gpt2_model, gpt2_model_with_shape_inference, auto_merge=True)" ] }, { @@ -652,54 +703,53 @@ }, { "cell_type": "code", - "execution_count": 23, + "execution_count": 24, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Nuphar GPT-2 speed-up 13.48%\n", - " Baseline: 2.535 s, Current: 2.234 s\n" + "Nuphar GPT-2 speed-up 16.15%\n", + " Baseline: 2.671 s, Current: 2.299 s\n" ] } ], "source": [ "sess_options = onnxruntime.SessionOptions()\n", "sess_options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL\n", - "sess_baseline = onnxruntime.InferenceSession(model, sess_options)\n", - "sess_baseline.set_providers(['CPUExecutionProvider'])\n", + "sess_baseline = onnxruntime.InferenceSession(gpt2_model, sess_options=sess_options, providers=['CPUExecutionProvider'])\n", "\n", "# load test data, note the tensor proto name in data does not match model, so override it in feed\n", "input_name = [i.name for i in sess_baseline.get_inputs()][0] # This model only has one input\n", - "test_data_dir = os.path.join(model_dir, 'test_data_set_0')\n", + "test_data_dir = os.path.join(gpt2_model_dir, 'test_data_set_0')\n", "tp = onnx.load_tensor(os.path.join(test_data_dir, 'input_0.pb'))\n", - "feed = {input_name:numpy_helper.to_array(tp).reshape(1,-1)} # the test data missed batch dimension\n", - "output_baseline = sess_baseline.run([], feed)\n", + "gpt2_feed = {input_name:numpy_helper.to_array(tp).reshape(1,-1)} # the test data missed batch dimension\n", + "gpt2_output_baseline = sess_baseline.run([], gpt2_feed)\n", "\n", - "repeats = 100\n", - "start_baseline = timer()\n", - "for i in range(repeats):\n", - " output = sess_baseline.run([], feed)\n", - "end_baseline = timer()\n", + "gpt2_repeats = 100\n", + "start_gpt2_baseline = timer()\n", + "for i in range(gpt2_repeats):\n", + " sess_baseline.run([], gpt2_feed)\n", + "end_gpt2_baseline = timer()\n", "\n", - "sess = onnxruntime.InferenceSession(model_with_shape_inference)\n", - "output = sess.run([], feed)\n", - "assert all([np.allclose(o, ob, atol=1e-4) for o, ob in zip(output, output_baseline)])\n", + "sess = onnxruntime.InferenceSession(gpt2_model_with_shape_inference)\n", + "output = sess.run([], gpt2_feed)\n", + "assert all([np.allclose(o, ob, atol=1e-4) for o, ob in zip(output, gpt2_output_baseline)])\n", "\n", "start_nuphar = timer()\n", - "for i in range(repeats):\n", - " output = sess.run([], feed)\n", + "for i in range(gpt2_repeats):\n", + " output = sess.run([], gpt2_feed)\n", "end_nuphar = timer()\n", "\n", - "print_speedup('Nuphar GPT-2', end_baseline - start_baseline, end_nuphar - start_nuphar)" + "print_speedup('Nuphar GPT-2', end_gpt2_baseline - start_gpt2_baseline, end_nuphar - start_nuphar)" ] }, { "cell_type": "markdown", "metadata": {}, "source": [ - "### BiDAF with quantization\n", + "### 5.3 BiDAF with quantization\n", "\n", "BiDAF is a machine comprehension model that uses LSTMs. The inputs to this model are paragraphs of contexts and queries, and the outputs are start/end indices of words in the contexts that answers the queries.\n", "\n", @@ -708,7 +758,7 @@ }, { "cell_type": "code", - "execution_count": 24, + "execution_count": 25, "metadata": {}, "outputs": [], "source": [ @@ -731,18 +781,18 @@ }, { "cell_type": "code", - "execution_count": 25, + "execution_count": 26, "metadata": {}, "outputs": [], "source": [ - "bidaf = os.path.join(cwd, 'bidaf', 'bidaf.onnx')\n", - "sess_baseline = onnxruntime.InferenceSession(bidaf)\n", - "sess_baseline.set_providers(['CPUExecutionProvider'])\n", + "bidaf_dir = os.path.join(cwd, 'bidaf')\n", + "bidaf = os.path.join(bidaf_dir, 'bidaf.onnx')\n", + "sess_baseline = onnxruntime.InferenceSession(bidaf, providers=['CPUExecutionProvider'])\n", "# load test data\n", "test_data_dir = os.path.join(cwd, 'bidaf', 'test_data_set_3')\n", "tps = [onnx.load_tensor(os.path.join(test_data_dir, 'input_{}.pb'.format(i))) for i in range(len(sess_baseline.get_inputs()))]\n", - "feed = {tp.name:numpy_helper.to_array(tp) for tp in tps}\n", - "output_baseline = sess_baseline.run([], feed)" + "bidaf_feed = {tp.name:numpy_helper.to_array(tp) for tp in tps}\n", + "bidaf_output_baseline = sess_baseline.run([], bidaf_feed)" ] }, { @@ -754,7 +804,7 @@ }, { "cell_type": "code", - "execution_count": 26, + "execution_count": 27, "metadata": {}, "outputs": [ { @@ -763,13 +813,13 @@ "\"with 4:51 left in regulation , carolina got the ball on their own 24 - yard line with a chance to mount a game - winning drive , and soon faced 3rd - and - 9 . on the next play , miller stripped the ball away from newton , and after several players dove for it , it took a long bounce backwards and was recovered by ward , who returned it five yards to the panthers 4 - yard line . although several players dove into the pile to attempt to recover it , newton did not and his lack of aggression later earned him heavy criticism . meanwhile , denver ' s offense was kept out of the end zone for three plays , but a holding penalty on cornerback josh norman gave the broncos a new set of downs . then anderson scored on a 2 - yard touchdown run and manning completed a pass to bennie fowler for a 2 - point conversion , giving denver a 24 – 10 lead with 3:08 left and essentially putting the game away . carolina had two more drives , but failed to get a first down on each one .\"" ] }, - "execution_count": 26, + "execution_count": 27, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "' '.join(list(feed['context_word'].reshape(-1)))" + "' '.join(list(bidaf_feed['context_word'].reshape(-1)))" ] }, { @@ -781,7 +831,7 @@ }, { "cell_type": "code", - "execution_count": 27, + "execution_count": 28, "metadata": {}, "outputs": [ { @@ -790,13 +840,13 @@ "'who recovered the strip ball ?'" ] }, - "execution_count": 27, + "execution_count": 28, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "' '.join(list(feed['query_word'].reshape(-1)))" + "' '.join(list(bidaf_feed['query_word'].reshape(-1)))" ] }, { @@ -808,7 +858,7 @@ }, { "cell_type": "code", - "execution_count": 28, + "execution_count": 29, "metadata": {}, "outputs": [ { @@ -817,13 +867,13 @@ "'ward'" ] }, - "execution_count": 28, + "execution_count": 29, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "' '.join(list(feed['context_word'][output_baseline[0][0]:output_baseline[1][0]+1].reshape(-1)))" + "' '.join(list(bidaf_feed['context_word'][bidaf_output_baseline[0][0]:bidaf_output_baseline[1][0]+1].reshape(-1)))" ] }, { @@ -835,7 +885,7 @@ }, { "cell_type": "code", - "execution_count": 29, + "execution_count": 30, "metadata": {}, "outputs": [], "source": [ @@ -851,8 +901,8 @@ "\n", "# inference and verify accuracy\n", "sess = onnxruntime.InferenceSession(bidaf_converted)\n", - "output = sess.run([], feed)\n", - "assert all([np.allclose(o, ob) for o, ob in zip(output, output_baseline)])" + "output = sess.run([], bidaf_feed)\n", + "assert all([np.allclose(o, ob) for o, ob in zip(output, bidaf_output_baseline)])" ] }, { @@ -864,30 +914,31 @@ }, { "cell_type": "code", - "execution_count": 30, + "execution_count": 31, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Nuphar quantized BiDAF speed-up 47.77%\n", - " Baseline: 1.564 s, Current: 1.058 s\n" + "Nuphar quantized BiDAF speed-up 43.79%\n", + " Baseline: 1.517 s, Current: 1.055 s\n" ] } ], "source": [ - "start_baseline = timer()\n", - "for i in range(repeats):\n", - " output = sess_baseline.run([], feed)\n", - "end_baseline = timer()\n", + "bidaf_repeats = 100\n", + "start_bidaf_baseline = timer()\n", + "for i in range(bidaf_repeats):\n", + " sess_baseline.run([], bidaf_feed)\n", + "end_bidaf_baseline = timer()\n", "\n", "start_nuphar = timer()\n", - "for i in range(repeats):\n", - " output = sess.run([], feed)\n", + "for i in range(bidaf_repeats):\n", + " sess.run([], bidaf_feed)\n", "end_nuphar = timer()\n", "\n", - "print_speedup('Nuphar quantized BiDAF', end_baseline - start_baseline, end_nuphar - start_nuphar)" + "print_speedup('Nuphar quantized BiDAF', end_bidaf_baseline - start_bidaf_baseline, end_nuphar - start_nuphar)" ] }, { @@ -907,16 +958,16 @@ }, { "cell_type": "code", - "execution_count": 31, + "execution_count": 32, "metadata": {}, "outputs": [ { "data": { "text/plain": [ - "'JIT took 4.721 seconds'" + "'JIT took 5.749 seconds'" ] }, - "execution_count": 31, + "execution_count": 32, "metadata": {}, "output_type": "execute_result" } @@ -930,20 +981,16 @@ }, { "cell_type": "code", - "execution_count": 32, + "execution_count": 33, "metadata": { "scrolled": true }, "outputs": [], "source": [ - "# create a folder for JIT cache\n", - "cache_dir = os.path.join(cwd, 'bidaf_cache')\n", - "# remove any stale cache files\n", - "if os.path.exists(cache_dir):\n", - " shutil.rmtree(cache_dir)\n", - "os.makedirs(cache_dir, exist_ok=True)\n", "# use settings to enable JIT cache\n", - "settings = 'nuphar_cache_path:{}'.format(cache_dir)\n", + "bidaf_cache_dir = os.path.join(bidaf_dir, 'cache')\n", + "create_cache_dir(bidaf_cache_dir)\n", + "settings = 'nuphar_cache_path:{}'.format(bidaf_cache_dir)\n", "onnxruntime.capi._pybind_state.set_nuphar_settings(settings)\n", "sess = onnxruntime.InferenceSession(bidaf_converted)" ] @@ -957,7 +1004,7 @@ }, { "cell_type": "code", - "execution_count": 33, + "execution_count": 34, "metadata": {}, "outputs": [ { @@ -966,17 +1013,16 @@ "['jit.so']" ] }, - "execution_count": 33, + "execution_count": 34, "metadata": {}, "output_type": "execute_result" } ], "source": [ - "cache_versioned_dir = os.path.join(cache_dir, os.listdir(cache_dir)[0])\n", + "bidaf_cache_versioned_dir = os.path.join(bidaf_cache_dir, os.listdir(bidaf_cache_dir)[0])\n", "# use onnxruntime.nuphar.create_shared module to create dll\n", - "onnxruntime_dir = os.path.split(os.path.abspath(onnxruntime.__file__))[0]\n", - "subprocess.run([sys.executable, '-m', 'onnxruntime.nuphar.create_shared', '--input_dir', cache_versioned_dir], check=True)\n", - "os.listdir(cache_versioned_dir)" + "subprocess.run([sys.executable, '-m', 'onnxruntime.nuphar.create_shared', '--input_dir', bidaf_cache_versioned_dir], check=True)\n", + "os.listdir(bidaf_cache_versioned_dir)" ] }, { @@ -988,28 +1034,95 @@ }, { "cell_type": "code", - "execution_count": 34, + "execution_count": 35, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "AOT speed-up 764.62%\n", - " Baseline: 4.721 s, Current: 0.546 s\n" + "AOT speed-up 694.06%\n", + " Baseline: 5.749 s, Current: 0.724 s\n" ] } ], "source": [ "start_aot = timer()\n", "# NOTE: Nuphar settings string is not sticky. It needs to be reset before creating InferenceSession\n", - "settings = 'nuphar_cache_path:{}'.format(cache_dir)\n", + "settings = 'nuphar_cache_path:{}'.format(bidaf_cache_dir)\n", "onnxruntime.capi._pybind_state.set_nuphar_settings(settings)\n", "sess = onnxruntime.InferenceSession(bidaf_converted)\n", "end_aot = timer()\n", "print_speedup('AOT', end_jit - start_jit, end_aot - start_aot)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Moreover, Nuphar AOT also supports:\n", + "* Generate JIT cache with AVX/AVX2/AVX-512 and build a AOT dll including support for all these CPUs, which makes deployment easier when targeting different CPUs in one package.\n", + "* Bake model checksum into AOT dll to validate model with given AOT dll." + ] + }, + { + "cell_type": "code", + "execution_count": 36, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Scan_LSTMx4_int8.onnx in avx2 speed-up 790.69%\n", + " Baseline: 3.070 s, Current: 0.345 s\n", + "Scan_LSTMx4_int8.onnx in avx speed-up 361.68%\n", + " Baseline: 3.070 s, Current: 0.665 s\n" + ] + } + ], + "source": [ + "# create object files for different CPUs\n", + "cache_dir = os.path.join(os.getcwd(), 'lstm_cache')\n", + "model_name = lstm_quantized_model\n", + "model_checksum = md5(model_name)\n", + "repeats = lstm_repeats\n", + "feed = lstm_feed\n", + "time_baseline = end_lstm_baseline - start_lstm_baseline\n", + "multi_isa_so = 'avx_avx2_avx512.so'\n", + "\n", + "create_cache_dir(cache_dir)\n", + "settings = 'nuphar_cache_path:{}'.format(cache_dir)\n", + "for isa in ['avx512', 'avx2', 'avx']:\n", + " settings_with_isa = settings + ', nuphar_codegen_target:' + isa\n", + " onnxruntime.capi._pybind_state.set_nuphar_settings(settings_with_isa)\n", + " sess = onnxruntime.InferenceSession(model_name)\n", + " cache_versioned_dir = os.path.join(cache_dir, os.listdir(cache_dir)[0])\n", + "\n", + "# link object files to AOT dll\n", + "subprocess.run([sys.executable, '-m', 'onnxruntime.nuphar.create_shared', '--input_dir', cache_versioned_dir, '--input_model', model_name, '--output_name', multi_isa_so], check=True)\n", + "\n", + "# now load the model with AOT dll\n", + "# NOTE: when nuphar_codegen_target is not set, it defaults to current CPU ISA\n", + "settings = 'nuphar_cache_path:{}, nuphar_cache_so_name:{}, nuphar_cache_model_checksum:{}, nuphar_cache_force_no_jit:on'.format(cache_dir, multi_isa_so, model_checksum)\n", + "onnxruntime.capi._pybind_state.set_nuphar_settings(settings)\n", + "sess = onnxruntime.InferenceSession(model_name)\n", + "\n", + "# force to a different ISA which is a subset of current CPU\n", + "# NOTE: if an incompatible ISA is used, exception on invalid instructions would be thrown\n", + "for valid_isa in ['avx2', 'avx']:\n", + " settings_with_isa = 'nuphar_cache_path:{}, nuphar_cache_so_name:{}, nuphar_cache_model_checksum:{}, nuphar_codegen_target:{}, nuphar_cache_force_no_jit:on'.format(cache_dir, multi_isa_so, model_checksum, valid_isa)\n", + " onnxruntime.capi._pybind_state.set_nuphar_settings(settings_with_isa)\n", + " sess = onnxruntime.InferenceSession(model_name)\n", + "\n", + " start_nuphar = timer()\n", + " for i in range(repeats):\n", + " sess.run([], feed)\n", + " end_nuphar = timer()\n", + "\n", + " print_speedup('{} in {}'.format(model_name, valid_isa), time_baseline, end_nuphar - start_nuphar)" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -1020,15 +1133,15 @@ }, { "cell_type": "code", - "execution_count": 35, + "execution_count": 37, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Single thread perf w/o parallel schedule speed-up 1.05%\n", - " Baseline: 1.542 s, Current: 1.526 s\n" + "Single thread perf w/o parallel schedule speed-up 1.49%\n", + " Baseline: 1.540 s, Current: 1.518 s\n" ] } ], @@ -1039,8 +1152,8 @@ "\n", "sess = onnxruntime.InferenceSession(bidaf_converted)\n", "start_baseline = timer()\n", - "for i in range(repeats):\n", - " output_baseline = sess_baseline.run([], feed)\n", + "for i in range(bidaf_repeats):\n", + " sess_baseline.run([], bidaf_feed)\n", "end_baseline = timer()\n", "\n", "# use NUPHAR_PARALLEL_MIN_WORKLOADS=0 to turn off parallel schedule, using settings string\n", @@ -1050,10 +1163,12 @@ "sess = onnxruntime.InferenceSession(bidaf_converted)\n", "\n", "start = timer()\n", - "for i in range(repeats):\n", - " output = sess_baseline.run([], feed)\n", + "for i in range(bidaf_repeats):\n", + " sess_baseline.run([], bidaf_feed)\n", "end = timer()\n", - "print_speedup('Single thread perf w/o parallel schedule', end_baseline - start_baseline, end - start)" + "print_speedup('Single thread perf w/o parallel schedule', end_baseline - start_baseline, end - start)\n", + "\n", + "del os.environ['OMP_NUM_THREADS']" ] } ],