From 412b0d0831a0c8da16ac0304313f6942717aabe6 Mon Sep 17 00:00:00 2001 From: Tianlei Wu Date: Tue, 15 Aug 2023 00:55:03 -0700 Subject: [PATCH] Update BERT and GPT-2 optimization notebooks for CPU EP (#17057) The notebooks are not up to update. (1) Update BERT and GPT-2 optimization notebooks for CPU EP with latest PyTorch and ONNX Runtime. (2) Add links to quantization example ### Motivation and Context https://github.com/microsoft/onnxruntime/issues/16515 --- ...ference_GPT2_with_OnnxRuntime_on_CPU.ipynb | 178 +++--- .../PyTorch_Bert-Squad_OnnxRuntime_CPU.ipynb | 524 +++++++++--------- 2 files changed, 370 insertions(+), 332 deletions(-) diff --git a/onnxruntime/python/tools/transformers/notebooks/Inference_GPT2_with_OnnxRuntime_on_CPU.ipynb b/onnxruntime/python/tools/transformers/notebooks/Inference_GPT2_with_OnnxRuntime_on_CPU.ipynb index e92e8bb0af..5e81e754e1 100644 --- a/onnxruntime/python/tools/transformers/notebooks/Inference_GPT2_with_OnnxRuntime_on_CPU.ipynb +++ b/onnxruntime/python/tools/transformers/notebooks/Inference_GPT2_with_OnnxRuntime_on_CPU.ipynb @@ -28,10 +28,11 @@ "Otherwise, you can setup a new environment. First, we install [AnaConda](https://www.anaconda.com/distribution/). Then open an AnaConda prompt window and run the following commands:\n", "\n", "```console\n", - "conda create -n cpu_env python=3.8\n", + "conda create -n cpu_env python=3.10\n", "conda activate cpu_env\n", "pip install jupyterlab\n", "conda install ipykernel\n", + "conda install -c conda-forge ipywidgets\n", "ipython kernel install --user --name cpu_env\n", "jupyter-lab\n", "```\n", @@ -44,23 +45,14 @@ "metadata": {}, "outputs": [], "source": [ - "# Install CPU-only PyTorch 1.10.1 and OnnxRuntime 1.12.0 packages, and other packages used in this notebook.\n", + "# Please refer to https://pytorch.org to install CPU-only PyTorch\n", "import sys\n", - "\n", - "if sys.platform == \"darwin\": # Mac\n", - " !{sys.executable} -m pip install torch==1.10.1 torchvision==0.11.2 torchaudio==0.10.1 >pip_output.txt\n", + "if sys.platform in [\"darwin\", \"win32\"]: # Mac or Windows\n", + " !{sys.executable} -m pip install torch -q\n", "else:\n", - " !{sys.executable} -m pip install torch==1.10.1+cpu torchvision==0.11.2+cpu torchaudio==0.10.1 -f https://download.pytorch.org/whl/torch_stable.html --no-warn-script-location >pip_output.txt\n", + " !{sys.executable} -m pip install install torch --index-url https://download.pytorch.org/whl/cpu -q\n", "\n", - "!{sys.executable} -m pip install flatbuffers >>pip_output.txt\n", - "\n", - "# This notebook requires onnxruntime 1.12.0 or later, use ort-nightly package until 1.12 is released.\n", - "# Please do not install both onnxruntime and ort-nightly at the same time.\n", - "#!{sys.executable} -m pip install onnxruntime==1.12.0\n", - "!{sys.executable} -m pip install -i https://test.pypi.org/simple/ ort-nightly >>pip_output.txt\n", - "\n", - "# Install other packages used in this notebook.\n", - "!{sys.executable} -m pip install transformers==4.18.0 onnx==1.11.0 psutil pytz pandas py-cpuinfo py3nvml netron coloredlogs ipywidgets --no-warn-script-location >>pip_output.txt" + "!{sys.executable} -m pip install onnxruntime transformers==4.18 onnx psutil pandas py-cpuinfo py3nvml netron coloredlogs --no-warn-script-location -q" ] }, { @@ -71,6 +63,9 @@ "source": [ "import os\n", "\n", + "if sys.platform in [\"win32\"]:\n", + " os.environ[\"HF_HUB_DISABLE_SYMLINKS_WARNING\"] = \"1\"\n", + "\n", "# Create a cache directory to store pretrained model.\n", "cache_dir = os.path.join(\".\", \"cache_models\")\n", "if not os.path.exists(cache_dir):\n", @@ -147,16 +142,7 @@ } ], "source": [ - "from packaging import version\n", - "from onnxruntime import __version__ as ort_version\n", - "\n", - "if version.parse(ort_version) >= version.parse(\"1.12.0\"):\n", - " from onnxruntime.transformers.models.gpt2.gpt2_helper import Gpt2Helper, MyGPT2LMHeadModel\n", - "else:\n", - " from onnxruntime.transformers.gpt2_helper import Gpt2Helper, MyGPT2LMHeadModel\n", - "\n", - " raise RuntimeError(\"Please install onnxruntime 1.12.0 or later to run this notebook\")\n", - "\n", + "from onnxruntime.transformers.models.gpt2.gpt2_helper import Gpt2Helper, MyGPT2LMHeadModel\n", "from transformers import AutoConfig\n", "import torch\n", "\n", @@ -188,7 +174,35 @@ "metadata": {}, "outputs": [], "source": [ - "!{sys.executable} -m onnxruntime.transformers.models.gpt2.convert_to_onnx -m $model_name_or_path --output $onnx_model_path -o -p fp32 --use_int32_inputs -t 10>export_output.txt 2>&1" + "!{sys.executable} -m onnxruntime.transformers.models.gpt2.convert_to_onnx -m $model_name_or_path --output $onnx_model_path -o -p fp32 -t 10 >export_output.txt 2>&1" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Please pay attention to the optimized operators in the output. Counters of EmbedLayerNormalization, Attention, FastGelu and SkipLayerNormalization shall be positive for fully optimized GPT-2 model." + ] + }, + { + "cell_type": "code", + "execution_count": 6, + "metadata": {}, + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Optimized operators:{'EmbedLayerNormalization': 1, 'Attention': 12, 'MultiHeadAttention': 0, 'Gelu': 0, 'FastGelu': 12, 'BiasGelu': 0, 'GemmFastGelu': 0, 'LayerNormalization': 0, 'SkipLayerNormalization': 24, 'QOrderedAttention': 0, 'QOrderedGelu': 0, 'QOrderedLayerNormalization': 0, 'QOrderedMatMul': 0}\n", + "\n" + ] + } + ], + "source": [ + "file = open(\"export_output.txt\", \"r\")\n", + "for line in file.readlines():\n", + " if \"Optimized operators\" in line:\n", + " print(line)" ] }, { @@ -203,7 +217,7 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 7, "metadata": {}, "outputs": [ { @@ -268,7 +282,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 8, "metadata": {}, "outputs": [], "source": [ @@ -291,7 +305,7 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 9, "metadata": {}, "outputs": [], "source": [ @@ -300,7 +314,7 @@ "\n", "input_ids, attention_mask, position_ids, empty_past = get_example_inputs()\n", "\n", - "session = onnxruntime.InferenceSession(onnx_model_path)\n", + "session = onnxruntime.InferenceSession(onnx_model_path, providers=[\"CPUExecutionProvider\"])\n", "ort_inputs = {\n", " \"input_ids\": numpy.ascontiguousarray(input_ids.cpu().numpy()),\n", " \"attention_mask\": numpy.ascontiguousarray(attention_mask.cpu().numpy()),\n", @@ -320,7 +334,7 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 10, "metadata": {}, "outputs": [ { @@ -348,7 +362,7 @@ }, { "cell_type": "code", - "execution_count": 10, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ @@ -386,7 +400,7 @@ }, { "cell_type": "code", - "execution_count": 11, + "execution_count": 12, "metadata": {}, "outputs": [ { @@ -416,7 +430,7 @@ }, { "cell_type": "code", - "execution_count": 12, + "execution_count": 13, "metadata": {}, "outputs": [], "source": [ @@ -476,7 +490,7 @@ }, { "cell_type": "code", - "execution_count": 13, + "execution_count": 14, "metadata": {}, "outputs": [ { @@ -508,7 +522,7 @@ }, { "cell_type": "code", - "execution_count": 14, + "execution_count": 15, "metadata": {}, "outputs": [ { @@ -539,35 +553,35 @@ }, { "cell_type": "code", - "execution_count": 15, + "execution_count": 16, "metadata": { "tags": [] }, "outputs": [], "source": [ - "!{sys.executable} -m onnxruntime.transformers.models.gpt2.benchmark_gpt2 -m gpt2 -o >benchmark_output.txt 2>&1" + "!{sys.executable} -m onnxruntime.transformers.models.gpt2.benchmark_gpt2 -m gpt2 -o -b 1 ----sequence_lengths 1 --past_sequence_lengths 128 >benchmark_output.txt 2>&1" ] }, { "cell_type": "code", - "execution_count": 16, + "execution_count": 17, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "batch_size=1, sequence_length=1, past_sequence_length=8, torch_latency=37.48, onnxruntime_latency=24.77, onnxruntime_io_binding_latency=24.65\n", + "batch_size=1, sequence_length=1, past_sequence_length=8, onnxruntime_latency=21.33 \n", "\n", - "batch_size=1, sequence_length=1, past_sequence_length=16, torch_latency=37.30, onnxruntime_latency=24.95, onnxruntime_io_binding_latency=24.62\n", + "batch_size=1, sequence_length=1, past_sequence_length=16, onnxruntime_latency=22.88 \n", "\n", - "batch_size=1, sequence_length=1, past_sequence_length=32, torch_latency=37.88, onnxruntime_latency=25.19, onnxruntime_io_binding_latency=22.05\n", + "batch_size=1, sequence_length=1, past_sequence_length=32, onnxruntime_latency=22.81 \n", "\n", - "batch_size=1, sequence_length=1, past_sequence_length=64, torch_latency=42.60, onnxruntime_latency=25.64, onnxruntime_io_binding_latency=25.08\n", + "batch_size=1, sequence_length=1, past_sequence_length=64, onnxruntime_latency=24.01 \n", "\n", - "batch_size=1, sequence_length=1, past_sequence_length=128, torch_latency=45.89, onnxruntime_latency=27.66, onnxruntime_io_binding_latency=25.71\n", + "batch_size=1, sequence_length=1, past_sequence_length=128, onnxruntime_latency=22.87 \n", "\n", - "batch_size=1, sequence_length=1, past_sequence_length=256, torch_latency=52.47, onnxruntime_latency=32.24, onnxruntime_io_binding_latency=25.46\n", + "batch_size=1, sequence_length=1, past_sequence_length=256, onnxruntime_latency=25.30 \n", "\n" ] } @@ -579,6 +593,31 @@ " print(line)" ] }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## ONNX Model for Generation ##\n", + "There is a tool [convert_generation.py](https://github.com/microsoft/onnxruntime/blob/main/onnxruntime/python/tools/transformers/convert_generation.py), which can convert GPT-2 model to enable Beam Search, Greedy Search or Sampling in one ONNX model." + ] + }, + { + "cell_type": "code", + "execution_count": 18, + "metadata": {}, + "outputs": [], + "source": [ + "!{sys.executable} -m onnxruntime.transformers.convert_generation -m gpt2 --output gpt2_beam_search.onnx >convert_generation_output.txt 2>&1" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## Quantization ##\n", + "For large language model, we can quantize the model to int8 to run efficiently in CPU. See examples of [GPT quantization](https://github.com/microsoft/onnxruntime-inference-examples/tree/main/quantization/language_model/gpt2) and [LLaMA quantization](https://github.com/microsoft/onnxruntime-inference-examples/tree/main/quantization/language_model/llama)" + ] + }, { "cell_type": "markdown", "metadata": {}, @@ -589,7 +628,7 @@ }, { "cell_type": "code", - "execution_count": 17, + "execution_count": 19, "metadata": {}, "outputs": [ { @@ -598,12 +637,12 @@ "text": [ "{\n", " \"gpu\": {\n", - " \"driver_version\": \"471.11\",\n", + " \"driver_version\": \"472.88\",\n", " \"devices\": [\n", " {\n", - " \"memory_total\": 8589934592,\n", - " \"memory_available\": 7099449344,\n", - " \"name\": \"NVIDIA GeForce GTX 1070\"\n", + " \"memory_total\": 12884901888,\n", + " \"memory_available\": 12732858368,\n", + " \"name\": \"NVIDIA GeForce RTX 3060\"\n", " }\n", " ]\n", " },\n", @@ -613,38 +652,45 @@ " \"logical_cores\": 12,\n", " \"hz\": \"3192000000,0\",\n", " \"l2_cache\": 1572864,\n", - " \"flags\": \"3dnow,3dnowprefetch,abm,acpi,adx,aes,apic,avx,avx2,bmi1,bmi2,clflush,clflushopt,cmov,cx16,cx8,de,dtes64,dts,erms,est,f16c,fma,fpu,fxsr,hle,ht,hypervisor,ia64,invpcid,lahf_lm,mca,mce,mmx,movbe,mpx,msr,mtrr,osxsave,pae,pat,pbe,pcid,pclmulqdq,pdcm,pge,pni,popcnt,pse,pse36,rdrnd,rdseed,rtm,sep,serial,sgx,sgx_lc,smap,smep,ss,sse,sse2,sse4_1,sse4_2,ssse3,tm,tm2,tsc,tscdeadline,vme,x2apic,xsave,xtpr\",\n", + " \"flags\": \"3dnow,3dnowprefetch,abm,acpi,adx,aes,apic,avx,avx2,bmi1,bmi2,clflush,clflushopt,cmov,cx16,cx8,de,dtes64,dts,erms,est,f16c,fma,fpu,fxsr,hle,ht,hypervisor,ia64,invpcid,lahf_lm,mca,mce,mmx,monitor,movbe,mpx,msr,mtrr,osxsave,pae,pat,pbe,pcid,pclmulqdq,pdcm,pge,pni,popcnt,pse,pse36,rdrnd,rdseed,rtm,sep,serial,sgx,sgx_lc,smap,smep,ss,sse,sse2,sse4_1,sse4_2,ssse3,tm,tm2,tsc,tscdeadline,vme,x2apic,xsave,xtpr\",\n", " \"processor\": \"Intel64 Family 6 Model 158 Stepping 10, GenuineIntel\"\n", " },\n", " \"memory\": {\n", " \"total\": 16977195008,\n", - " \"available\": 7204651008\n", + " \"available\": 9550102528\n", " },\n", - " \"os\": \"Windows-10-10.0.22000-SP0\",\n", - " \"python\": \"3.8.13.final.0 (64 bit)\",\n", + " \"os\": \"Windows-10-10.0.22621-SP0\",\n", + " \"python\": \"3.10.12.final.0 (64 bit)\",\n", " \"packages\": {\n", - " \"sympy\": \"1.5.1\",\n", - " \"transformers\": \"4.18.0\",\n", - " \"protobuf\": \"3.20.1\",\n", - " \"flatbuffers\": \"2.0\",\n", - " \"numpy\": \"1.22.3\",\n", - " \"ort-nightly\": \"1.12.0.dev20220428004\",\n", - " \"onnx\": \"1.11.0\",\n", - " \"torch\": \"1.10.1+cpu\",\n", - " \"onnxconverter-common\": \"1.9.0\"\n", + " \"flatbuffers\": \"23.5.26\",\n", + " \"numpy\": \"1.25.2\",\n", + " \"onnx\": \"1.14.0\",\n", + " \"onnxruntime\": \"1.15.1\",\n", + " \"protobuf\": \"4.23.4\",\n", + " \"sympy\": \"1.12\",\n", + " \"torch\": \"2.0.1\",\n", + " \"transformers\": \"4.18.0\"\n", " },\n", " \"onnxruntime\": {\n", - " \"version\": \"1.12.0\",\n", + " \"version\": \"1.15.1\",\n", " \"support_gpu\": false\n", " },\n", " \"pytorch\": {\n", - " \"version\": \"1.10.1+cpu\",\n", + " \"version\": \"2.0.1+cpu\",\n", " \"support_gpu\": false,\n", " \"cuda\": null\n", " },\n", " \"tensorflow\": null\n", "}\n" ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "f:\\anaconda3\\envs\\cpu_env\\lib\\site-packages\\onnxruntime\\transformers\\machine_info.py:127: DeprecationWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html\n", + " import pkg_resources\n" + ] } ], "source": [ @@ -668,7 +714,7 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.8.13" + "version": "3.10.12" } }, "nbformat": 4, diff --git a/onnxruntime/python/tools/transformers/notebooks/PyTorch_Bert-Squad_OnnxRuntime_CPU.ipynb b/onnxruntime/python/tools/transformers/notebooks/PyTorch_Bert-Squad_OnnxRuntime_CPU.ipynb index d6bd3884ba..997ff7e3ff 100644 --- a/onnxruntime/python/tools/transformers/notebooks/PyTorch_Bert-Squad_OnnxRuntime_CPU.ipynb +++ b/onnxruntime/python/tools/transformers/notebooks/PyTorch_Bert-Squad_OnnxRuntime_CPU.ipynb @@ -35,10 +35,13 @@ "Otherwise, you can setup a new environment. First, we install [AnaConda](https://www.anaconda.com/distribution/). Then open an AnaConda prompt window and run the following commands:\n", "\n", "```console\n", - "conda create -n cpu_env python=3.6\n", + "conda create -n cpu_env python=3.10\n", "conda activate cpu_env\n", - "conda install jupyter\n", - "jupyter notebook\n", + "pip install jupyterlab\n", + "conda install ipykernel\n", + "conda install -c conda-forge ipywidgets\n", + "ipython kernel install --user --name cpu_env\n", + "jupyter-lababook\n", "```\n", "The last command will launch Jupyter Notebook and we can open this notebook in browser to continue." ] @@ -49,21 +52,15 @@ "metadata": {}, "outputs": [], "source": [ + "# Please refer to https://pytorch.org to install CPU-only PyTorch\n", "import sys\n", + "if sys.platform in [\"darwin\", \"win32\"]: # Mac or Windows\n", + " !{sys.executable} -m pip install torch -q\n", + "else:\n", + " !{sys.executable} -m pip install install torch --index-url https://download.pytorch.org/whl/cpu -q\n", "\n", - "run_install = False # Only need install once\n", - "if run_install:\n", - " if sys.platform in ['linux', 'win32']: # Linux or Windows\n", - " !{sys.executable} -m pip install --upgrade torch torchvision torchaudio\n", - " else: # Mac\n", - " !{sys.executable} -m pip install torch==1.9.0+cpu torchvision==0.10.0+cpu torchaudio==0.9.0 -f https://download.pytorch.org/whl/torch_stable.html\n", - "\n", - " !{sys.executable} -m pip install onnxruntime==1.8.1 onnx==1.9.0 onnxconverter_common==1.8.1\n", - "\n", - " # Install other packages used in this notebook.\n", - " !{sys.executable} -m pip install transformers==4.8.2\n", - " !{sys.executable} -m pip install psutil pytz pandas py-cpuinfo py3nvml\n", - " !{sys.executable} -m pip install wget netron" + "# Install other packages used in this notebook.\n", + "!{sys.executable} -m pip install onnxruntime onnx transformers==4.18 psutil pandas py-cpuinfo py3nvml wget netron -q" ] }, { @@ -84,11 +81,20 @@ "cell_type": "code", "execution_count": 2, "metadata": {}, - "outputs": [], + "outputs": [ + { + "name": "stdout", + "output_type": "stream", + "text": [ + "Start downloading predict file.\n", + "100% [..........................................................................] 4854279 / 4854279Predict file downloaded.\n" + ] + } + ], "source": [ "import os\n", "\n", - "cache_dir = os.path.join(\"..\", \"cache_models\")\n", + "cache_dir = os.path.join(\".\", \"cache_models\")\n", "if not os.path.exists(cache_dir):\n", " os.makedirs(cache_dir)\n", "\n", @@ -139,18 +145,74 @@ "execution_count": 4, "metadata": {}, "outputs": [ + { + "data": { + "application/vnd.jupyter.widget-view+json": { + "model_id": "2d13e6ef5b5840d8b58930b908b1a0c8", + "version_major": 2, + "version_minor": 0 + }, + "text/plain": [ + "Downloading: 0%| | 0.00/570 [00:00\n", " \n", " 0\n", - " 54.26\n", - " 56.05\n", - " 60.32\n", - " 109.21\n", - " 18.43\n", + " 52.26\n", + " 52.92\n", + " 60.94\n", + " 88.51\n", + " 19.14\n", " 12\n", " \n", " \n", " 1\n", - " 55.80\n", - " 56.74\n", - " 59.67\n", - " 73.62\n", - " 17.92\n", + " 52.67\n", + " 52.73\n", + " 54.06\n", + " 60.48\n", + " 18.99\n", " 11\n", " \n", " \n", " 2\n", - " 56.19\n", - " 61.29\n", - " 71.69\n", - " 80.15\n", - " 17.80\n", - " 6\n", - " \n", - " \n", - " 3\n", - " 57.66\n", - " 58.50\n", - " 61.96\n", - " 65.12\n", - " 17.34\n", - " 9\n", - " \n", - " \n", - " 4\n", - " 59.90\n", - " 59.72\n", - " 65.16\n", - " 116.16\n", - " 16.70\n", - " 5\n", - " \n", - " \n", - " 5\n", - " 62.84\n", - " 67.05\n", - " 69.07\n", - " 75.99\n", - " 15.91\n", - " 8\n", - " \n", - " \n", - " 6\n", - " 63.72\n", - " 64.17\n", - " 69.44\n", - " 73.10\n", - " 15.69\n", - " 4\n", - " \n", - " \n", - " 7\n", - " 65.31\n", - " 65.35\n", - " 80.70\n", - " 177.94\n", - " 15.31\n", + " 54.57\n", + " 55.05\n", + " 56.52\n", + " 59.62\n", + " 18.32\n", " 10\n", " \n", " \n", - " 8\n", - " 69.29\n", - " 69.04\n", - " 70.68\n", - " 85.03\n", - " 14.43\n", + " 3\n", + " 58.62\n", + " 65.86\n", + " 71.94\n", + " 79.38\n", + " 17.06\n", + " 6\n", + " \n", + " \n", + " 4\n", + " 59.80\n", + " 61.09\n", + " 65.72\n", + " 72.79\n", + " 16.72\n", + " 8\n", + " \n", + " \n", + " 5\n", + " 63.12\n", + " 61.92\n", + " 77.47\n", + " 129.26\n", + " 15.84\n", + " 9\n", + " \n", + " \n", + " 6\n", + " 64.72\n", + " 65.94\n", + " 68.54\n", + " 75.00\n", + " 15.45\n", " 7\n", " \n", " \n", + " 7\n", + " 73.72\n", + " 80.43\n", + " 86.39\n", + " 92.95\n", + " 13.56\n", + " 5\n", + " \n", + " \n", + " 8\n", + " 88.76\n", + " 96.30\n", + " 101.13\n", + " 109.78\n", + " 11.27\n", + " 4\n", + " \n", + " \n", " 9\n", - " 82.44\n", - " 83.20\n", - " 89.64\n", - " 98.80\n", - " 12.13\n", + " 91.14\n", + " 99.14\n", + " 104.41\n", + " 110.69\n", + " 10.97\n", " 3\n", " \n", " \n", " 10\n", - " 119.64\n", - " 119.07\n", - " 122.62\n", - " 135.67\n", - " 8.36\n", + " 119.80\n", + " 119.78\n", + " 123.66\n", + " 130.64\n", + " 8.35\n", " 2\n", " \n", " \n", " 11\n", - " 223.21\n", - " 223.22\n", - " 226.83\n", - " 249.08\n", - " 4.48\n", + " 225.06\n", + " 227.11\n", + " 229.45\n", + " 252.40\n", + " 4.44\n", " 1\n", " \n", " \n", @@ -729,29 +781,29 @@ ], "text/plain": [ " Latency(ms) Latency_P75 Latency_P90 Latency_P99 Throughput(QPS) \\\n", - "0 54.26 56.05 60.32 109.21 18.43 \n", - "1 55.80 56.74 59.67 73.62 17.92 \n", - "2 56.19 61.29 71.69 80.15 17.80 \n", - "3 57.66 58.50 61.96 65.12 17.34 \n", - "4 59.90 59.72 65.16 116.16 16.70 \n", - "5 62.84 67.05 69.07 75.99 15.91 \n", - "6 63.72 64.17 69.44 73.10 15.69 \n", - "7 65.31 65.35 80.70 177.94 15.31 \n", - "8 69.29 69.04 70.68 85.03 14.43 \n", - "9 82.44 83.20 89.64 98.80 12.13 \n", - "10 119.64 119.07 122.62 135.67 8.36 \n", - "11 223.21 223.22 226.83 249.08 4.48 \n", + "0 52.26 52.92 60.94 88.51 19.14 \n", + "1 52.67 52.73 54.06 60.48 18.99 \n", + "2 54.57 55.05 56.52 59.62 18.32 \n", + "3 58.62 65.86 71.94 79.38 17.06 \n", + "4 59.80 61.09 65.72 72.79 16.72 \n", + "5 63.12 61.92 77.47 129.26 15.84 \n", + "6 64.72 65.94 68.54 75.00 15.45 \n", + "7 73.72 80.43 86.39 92.95 13.56 \n", + "8 88.76 96.30 101.13 109.78 11.27 \n", + "9 91.14 99.14 104.41 110.69 10.97 \n", + "10 119.80 119.78 123.66 130.64 8.35 \n", + "11 225.06 227.11 229.45 252.40 4.44 \n", "\n", " intra_op_num_threads \n", "0 12 \n", "1 11 \n", - "2 6 \n", - "3 9 \n", - "4 5 \n", - "5 8 \n", - "6 4 \n", - "7 10 \n", - "8 7 \n", + "2 10 \n", + "3 6 \n", + "4 8 \n", + "5 9 \n", + "6 7 \n", + "7 5 \n", + "8 4 \n", "9 3 \n", "10 2 \n", "11 1 " @@ -782,7 +834,16 @@ "cell_type": "markdown", "metadata": {}, "source": [ - "## 6. Additional Info\n", + "## 6. Quantization\n", + "\n", + "Please see example in [BERT quantization notebook](https://github.com/microsoft/onnxruntime-inference-examples/blob/main/quantization/notebooks/bert/Bert-GLUE_OnnxRuntime_quantization.ipynb)." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 7. Additional Info\n", "\n", "Note that running Jupyter Notebook has slight impact on performance result since Jupyter Notebook is using system resources like CPU and memory etc. It is recommended to close Jupyter Notebook and other applications, then run the performance test tool in a console to get more accurate performance numbers.\n", "\n", @@ -805,12 +866,12 @@ "text": [ "{\n", " \"gpu\": {\n", - " \"driver_version\": \"470.14\",\n", + " \"driver_version\": \"472.88\",\n", " \"devices\": [\n", " {\n", - " \"memory_total\": 8589934592,\n", - " \"memory_available\": 6782619648,\n", - " \"name\": \"NVIDIA GeForce GTX 1070\"\n", + " \"memory_total\": 12884901888,\n", + " \"memory_available\": 12732858368,\n", + " \"name\": \"NVIDIA GeForce RTX 3060\"\n", " }\n", " ]\n", " },\n", @@ -818,106 +879,37 @@ " \"brand\": \"Intel(R) Core(TM) i7-8700 CPU @ 3.20GHz\",\n", " \"cores\": 6,\n", " \"logical_cores\": 12,\n", - " \"hz\": \"3.1920 GHz\",\n", - " \"l2_cache\": \"1536 KB\",\n", - " \"flags\": [\n", - " \"3dnow\",\n", - " \"3dnowprefetch\",\n", - " \"abm\",\n", - " \"acpi\",\n", - " \"adx\",\n", - " \"aes\",\n", - " \"apic\",\n", - " \"avx\",\n", - " \"avx2\",\n", - " \"bmi1\",\n", - " \"bmi2\",\n", - " \"clflush\",\n", - " \"clflushopt\",\n", - " \"cmov\",\n", - " \"cx16\",\n", - " \"cx8\",\n", - " \"de\",\n", - " \"dtes64\",\n", - " \"dts\",\n", - " \"erms\",\n", - " \"est\",\n", - " \"f16c\",\n", - " \"fma\",\n", - " \"fpu\",\n", - " \"fxsr\",\n", - " \"hle\",\n", - " \"ht\",\n", - " \"hypervisor\",\n", - " \"ia64\",\n", - " \"invpcid\",\n", - " \"lahf_lm\",\n", - " \"mca\",\n", - " \"mce\",\n", - " \"mmx\",\n", - " \"movbe\",\n", - " \"mpx\",\n", - " \"msr\",\n", - " \"mtrr\",\n", - " \"osxsave\",\n", - " \"pae\",\n", - " \"pat\",\n", - " \"pbe\",\n", - " \"pcid\",\n", - " \"pclmulqdq\",\n", - " \"pdcm\",\n", - " \"pge\",\n", - " \"pni\",\n", - " \"popcnt\",\n", - " \"pse\",\n", - " \"pse36\",\n", - " \"rdrnd\",\n", - " \"rdseed\",\n", - " \"rtm\",\n", - " \"sep\",\n", - " \"serial\",\n", - " \"sgx\",\n", - " \"sgx_lc\",\n", - " \"smap\",\n", - " \"smep\",\n", - " \"ss\",\n", - " \"sse\",\n", - " \"sse2\",\n", - " \"sse4_1\",\n", - " \"sse4_2\",\n", - " \"ssse3\",\n", - " \"tm\",\n", - " \"tm2\",\n", - " \"tsc\",\n", - " \"tscdeadline\",\n", - " \"vme\",\n", - " \"x2apic\",\n", - " \"xsave\",\n", - " \"xtpr\"\n", - " ],\n", + " \"hz\": \"3192000000,0\",\n", + " \"l2_cache\": 1572864,\n", + " \"flags\": \"3dnow,3dnowprefetch,abm,acpi,adx,aes,apic,avx,avx2,bmi1,bmi2,clflush,clflushopt,cmov,cx16,cx8,de,dtes64,dts,erms,est,f16c,fma,fpu,fxsr,hle,ht,hypervisor,ia64,invpcid,lahf_lm,mca,mce,mmx,monitor,movbe,mpx,msr,mtrr,osxsave,pae,pat,pbe,pcid,pclmulqdq,pdcm,pge,pni,popcnt,pse,pse36,rdrnd,rdseed,rtm,sep,serial,sgx,sgx_lc,smap,smep,ss,sse,sse2,sse4_1,sse4_2,ssse3,tm,tm2,tsc,tscdeadline,vme,x2apic,xsave,xtpr\",\n", " \"processor\": \"Intel64 Family 6 Model 158 Stepping 10, GenuineIntel\"\n", " },\n", " \"memory\": {\n", " \"total\": 16977195008,\n", - " \"available\": 6085459968\n", + " \"available\": 5000630272\n", + " },\n", + " \"os\": \"Windows-10-10.0.22621-SP0\",\n", + " \"python\": \"3.10.12.final.0 (64 bit)\",\n", + " \"packages\": {\n", + " \"flatbuffers\": \"23.5.26\",\n", + " \"numpy\": \"1.25.2\",\n", + " \"onnx\": \"1.14.0\",\n", + " \"onnxruntime\": \"1.15.1\",\n", + " \"protobuf\": \"4.23.4\",\n", + " \"sympy\": \"1.12\",\n", + " \"torch\": \"2.0.1\",\n", + " \"transformers\": \"4.18.0\"\n", " },\n", - " \"python\": \"3.6.10.final.0 (64 bit)\",\n", - " \"os\": \"Windows-10-10.0.21390-SP0\",\n", " \"onnxruntime\": {\n", - " \"version\": \"1.8.1\",\n", + " \"version\": \"1.15.1\",\n", " \"support_gpu\": false\n", " },\n", - " \"onnxruntime_tools\": null,\n", " \"pytorch\": {\n", - " \"version\": \"1.9.0+cpu\",\n", + " \"version\": \"2.0.1+cpu\",\n", " \"support_gpu\": false,\n", " \"cuda\": null\n", " },\n", - " \"tensorflow\": {\n", - " \"version\": \"2.3.0\",\n", - " \"git_version\": \"v2.3.0-rc2-23-gb36436b087\",\n", - " \"support_gpu\": true\n", - " }\n", + " \"tensorflow\": null\n", "}\n" ] }, @@ -925,8 +917,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "2021-07-13 14:41:45.376756: W tensorflow/stream_executor/platform/default/dso_loader.cc:59] Could not load dynamic library 'cudart64_101.dll'; dlerror: cudart64_101.dll not found\n", - "2021-07-13 14:41:45.376780: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n" + "f:\\anaconda3\\envs\\cpu_env\\lib\\site-packages\\onnxruntime\\transformers\\machine_info.py:127: DeprecationWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html\n", + " import pkg_resources\n" ] } ], @@ -951,9 +943,9 @@ "name": "python", "nbconvert_exporter": "python", "pygments_lexer": "ipython3", - "version": "3.6.10" + "version": "3.10.12" } }, "nbformat": 4, - "nbformat_minor": 2 + "nbformat_minor": 4 }