mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-23 19:32:23 +00:00
Update BERT and GPT-2 optimization notebooks for CPU EP (#17057)
The notebooks are not up to update. (1) Update BERT and GPT-2 optimization notebooks for CPU EP with latest PyTorch and ONNX Runtime. (2) Add links to quantization example ### Motivation and Context https://github.com/microsoft/onnxruntime/issues/16515
This commit is contained in:
Tianlei Wu
GitHub
parent
abf9765d73
commit
412b0d0831
2 changed files with 370 additions and 332 deletions
|
|
@ -28,10 +28,11 @@
|
|||
"Otherwise, you can setup a new environment. First, we install [AnaConda](https://www.anaconda.com/distribution/). Then open an AnaConda prompt window and run the following commands:\n",
|
||||
"\n",
|
||||
"```console\n",
|
||||
"conda create -n cpu_env python=3.8\n",
|
||||
"conda create -n cpu_env python=3.10\n",
|
||||
"conda activate cpu_env\n",
|
||||
"pip install jupyterlab\n",
|
||||
"conda install ipykernel\n",
|
||||
"conda install -c conda-forge ipywidgets\n",
|
||||
"ipython kernel install --user --name cpu_env\n",
|
||||
"jupyter-lab\n",
|
||||
"```\n",
|
||||
|
|
@ -44,23 +45,14 @@
|
|||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Install CPU-only PyTorch 1.10.1 and OnnxRuntime 1.12.0 packages, and other packages used in this notebook.\n",
|
||||
"# Please refer to https://pytorch.org to install CPU-only PyTorch\n",
|
||||
"import sys\n",
|
||||
"\n",
|
||||
"if sys.platform == \"darwin\": # Mac\n",
|
||||
" !{sys.executable} -m pip install torch==1.10.1 torchvision==0.11.2 torchaudio==0.10.1 >pip_output.txt\n",
|
||||
"if sys.platform in [\"darwin\", \"win32\"]: # Mac or Windows\n",
|
||||
" !{sys.executable} -m pip install torch -q\n",
|
||||
"else:\n",
|
||||
" !{sys.executable} -m pip install torch==1.10.1+cpu torchvision==0.11.2+cpu torchaudio==0.10.1 -f https://download.pytorch.org/whl/torch_stable.html --no-warn-script-location >pip_output.txt\n",
|
||||
" !{sys.executable} -m pip install install torch --index-url https://download.pytorch.org/whl/cpu -q\n",
|
||||
"\n",
|
||||
"!{sys.executable} -m pip install flatbuffers >>pip_output.txt\n",
|
||||
"\n",
|
||||
"# This notebook requires onnxruntime 1.12.0 or later, use ort-nightly package until 1.12 is released.\n",
|
||||
"# Please do not install both onnxruntime and ort-nightly at the same time.\n",
|
||||
"#!{sys.executable} -m pip install onnxruntime==1.12.0\n",
|
||||
"!{sys.executable} -m pip install -i https://test.pypi.org/simple/ ort-nightly >>pip_output.txt\n",
|
||||
"\n",
|
||||
"# Install other packages used in this notebook.\n",
|
||||
"!{sys.executable} -m pip install transformers==4.18.0 onnx==1.11.0 psutil pytz pandas py-cpuinfo py3nvml netron coloredlogs ipywidgets --no-warn-script-location >>pip_output.txt"
|
||||
"!{sys.executable} -m pip install onnxruntime transformers==4.18 onnx psutil pandas py-cpuinfo py3nvml netron coloredlogs --no-warn-script-location -q"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -71,6 +63,9 @@
|
|||
"source": [
|
||||
"import os\n",
|
||||
"\n",
|
||||
"if sys.platform in [\"win32\"]:\n",
|
||||
" os.environ[\"HF_HUB_DISABLE_SYMLINKS_WARNING\"] = \"1\"\n",
|
||||
"\n",
|
||||
"# Create a cache directory to store pretrained model.\n",
|
||||
"cache_dir = os.path.join(\".\", \"cache_models\")\n",
|
||||
"if not os.path.exists(cache_dir):\n",
|
||||
|
|
@ -147,16 +142,7 @@
|
|||
}
|
||||
],
|
||||
"source": [
|
||||
"from packaging import version\n",
|
||||
"from onnxruntime import __version__ as ort_version\n",
|
||||
"\n",
|
||||
"if version.parse(ort_version) >= version.parse(\"1.12.0\"):\n",
|
||||
" from onnxruntime.transformers.models.gpt2.gpt2_helper import Gpt2Helper, MyGPT2LMHeadModel\n",
|
||||
"else:\n",
|
||||
" from onnxruntime.transformers.gpt2_helper import Gpt2Helper, MyGPT2LMHeadModel\n",
|
||||
"\n",
|
||||
" raise RuntimeError(\"Please install onnxruntime 1.12.0 or later to run this notebook\")\n",
|
||||
"\n",
|
||||
"from onnxruntime.transformers.models.gpt2.gpt2_helper import Gpt2Helper, MyGPT2LMHeadModel\n",
|
||||
"from transformers import AutoConfig\n",
|
||||
"import torch\n",
|
||||
"\n",
|
||||
|
|
@ -188,7 +174,35 @@
|
|||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!{sys.executable} -m onnxruntime.transformers.models.gpt2.convert_to_onnx -m $model_name_or_path --output $onnx_model_path -o -p fp32 --use_int32_inputs -t 10>export_output.txt 2>&1"
|
||||
"!{sys.executable} -m onnxruntime.transformers.models.gpt2.convert_to_onnx -m $model_name_or_path --output $onnx_model_path -o -p fp32 -t 10 >export_output.txt 2>&1"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"Please pay attention to the optimized operators in the output. Counters of EmbedLayerNormalization, Attention, FastGelu and SkipLayerNormalization shall be positive for fully optimized GPT-2 model."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Optimized operators:{'EmbedLayerNormalization': 1, 'Attention': 12, 'MultiHeadAttention': 0, 'Gelu': 0, 'FastGelu': 12, 'BiasGelu': 0, 'GemmFastGelu': 0, 'LayerNormalization': 0, 'SkipLayerNormalization': 24, 'QOrderedAttention': 0, 'QOrderedGelu': 0, 'QOrderedLayerNormalization': 0, 'QOrderedMatMul': 0}\n",
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"file = open(\"export_output.txt\", \"r\")\n",
|
||||
"for line in file.readlines():\n",
|
||||
" if \"Optimized operators\" in line:\n",
|
||||
" print(line)"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -203,7 +217,7 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 6,
|
||||
"execution_count": 7,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
|
|
@ -268,7 +282,7 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 7,
|
||||
"execution_count": 8,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
|
|
@ -291,7 +305,7 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 8,
|
||||
"execution_count": 9,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
|
|
@ -300,7 +314,7 @@
|
|||
"\n",
|
||||
"input_ids, attention_mask, position_ids, empty_past = get_example_inputs()\n",
|
||||
"\n",
|
||||
"session = onnxruntime.InferenceSession(onnx_model_path)\n",
|
||||
"session = onnxruntime.InferenceSession(onnx_model_path, providers=[\"CPUExecutionProvider\"])\n",
|
||||
"ort_inputs = {\n",
|
||||
" \"input_ids\": numpy.ascontiguousarray(input_ids.cpu().numpy()),\n",
|
||||
" \"attention_mask\": numpy.ascontiguousarray(attention_mask.cpu().numpy()),\n",
|
||||
|
|
@ -320,7 +334,7 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 9,
|
||||
"execution_count": 10,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
|
|
@ -348,7 +362,7 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 10,
|
||||
"execution_count": 11,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
|
|
@ -386,7 +400,7 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"execution_count": 12,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
|
|
@ -416,7 +430,7 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 12,
|
||||
"execution_count": 13,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
|
|
@ -476,7 +490,7 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 13,
|
||||
"execution_count": 14,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
|
|
@ -508,7 +522,7 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 14,
|
||||
"execution_count": 15,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
|
|
@ -539,35 +553,35 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 15,
|
||||
"execution_count": 16,
|
||||
"metadata": {
|
||||
"tags": []
|
||||
},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!{sys.executable} -m onnxruntime.transformers.models.gpt2.benchmark_gpt2 -m gpt2 -o >benchmark_output.txt 2>&1"
|
||||
"!{sys.executable} -m onnxruntime.transformers.models.gpt2.benchmark_gpt2 -m gpt2 -o -b 1 ----sequence_lengths 1 --past_sequence_lengths 128 >benchmark_output.txt 2>&1"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 16,
|
||||
"execution_count": 17,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"batch_size=1, sequence_length=1, past_sequence_length=8, torch_latency=37.48, onnxruntime_latency=24.77, onnxruntime_io_binding_latency=24.65\n",
|
||||
"batch_size=1, sequence_length=1, past_sequence_length=8, onnxruntime_latency=21.33 \n",
|
||||
"\n",
|
||||
"batch_size=1, sequence_length=1, past_sequence_length=16, torch_latency=37.30, onnxruntime_latency=24.95, onnxruntime_io_binding_latency=24.62\n",
|
||||
"batch_size=1, sequence_length=1, past_sequence_length=16, onnxruntime_latency=22.88 \n",
|
||||
"\n",
|
||||
"batch_size=1, sequence_length=1, past_sequence_length=32, torch_latency=37.88, onnxruntime_latency=25.19, onnxruntime_io_binding_latency=22.05\n",
|
||||
"batch_size=1, sequence_length=1, past_sequence_length=32, onnxruntime_latency=22.81 \n",
|
||||
"\n",
|
||||
"batch_size=1, sequence_length=1, past_sequence_length=64, torch_latency=42.60, onnxruntime_latency=25.64, onnxruntime_io_binding_latency=25.08\n",
|
||||
"batch_size=1, sequence_length=1, past_sequence_length=64, onnxruntime_latency=24.01 \n",
|
||||
"\n",
|
||||
"batch_size=1, sequence_length=1, past_sequence_length=128, torch_latency=45.89, onnxruntime_latency=27.66, onnxruntime_io_binding_latency=25.71\n",
|
||||
"batch_size=1, sequence_length=1, past_sequence_length=128, onnxruntime_latency=22.87 \n",
|
||||
"\n",
|
||||
"batch_size=1, sequence_length=1, past_sequence_length=256, torch_latency=52.47, onnxruntime_latency=32.24, onnxruntime_io_binding_latency=25.46\n",
|
||||
"batch_size=1, sequence_length=1, past_sequence_length=256, onnxruntime_latency=25.30 \n",
|
||||
"\n"
|
||||
]
|
||||
}
|
||||
|
|
@ -579,6 +593,31 @@
|
|||
" print(line)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## ONNX Model for Generation ##\n",
|
||||
"There is a tool [convert_generation.py](https://github.com/microsoft/onnxruntime/blob/main/onnxruntime/python/tools/transformers/convert_generation.py), which can convert GPT-2 model to enable Beam Search, Greedy Search or Sampling in one ONNX model."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 18,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"!{sys.executable} -m onnxruntime.transformers.convert_generation -m gpt2 --output gpt2_beam_search.onnx >convert_generation_output.txt 2>&1"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## Quantization ##\n",
|
||||
"For large language model, we can quantize the model to int8 to run efficiently in CPU. See examples of [GPT quantization](https://github.com/microsoft/onnxruntime-inference-examples/tree/main/quantization/language_model/gpt2) and [LLaMA quantization](https://github.com/microsoft/onnxruntime-inference-examples/tree/main/quantization/language_model/llama)"
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
|
|
@ -589,7 +628,7 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 17,
|
||||
"execution_count": 19,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
|
|
@ -598,12 +637,12 @@
|
|||
"text": [
|
||||
"{\n",
|
||||
" \"gpu\": {\n",
|
||||
" \"driver_version\": \"471.11\",\n",
|
||||
" \"driver_version\": \"472.88\",\n",
|
||||
" \"devices\": [\n",
|
||||
" {\n",
|
||||
" \"memory_total\": 8589934592,\n",
|
||||
" \"memory_available\": 7099449344,\n",
|
||||
" \"name\": \"NVIDIA GeForce GTX 1070\"\n",
|
||||
" \"memory_total\": 12884901888,\n",
|
||||
" \"memory_available\": 12732858368,\n",
|
||||
" \"name\": \"NVIDIA GeForce RTX 3060\"\n",
|
||||
" }\n",
|
||||
" ]\n",
|
||||
" },\n",
|
||||
|
|
@ -613,38 +652,45 @@
|
|||
" \"logical_cores\": 12,\n",
|
||||
" \"hz\": \"3192000000,0\",\n",
|
||||
" \"l2_cache\": 1572864,\n",
|
||||
" \"flags\": \"3dnow,3dnowprefetch,abm,acpi,adx,aes,apic,avx,avx2,bmi1,bmi2,clflush,clflushopt,cmov,cx16,cx8,de,dtes64,dts,erms,est,f16c,fma,fpu,fxsr,hle,ht,hypervisor,ia64,invpcid,lahf_lm,mca,mce,mmx,movbe,mpx,msr,mtrr,osxsave,pae,pat,pbe,pcid,pclmulqdq,pdcm,pge,pni,popcnt,pse,pse36,rdrnd,rdseed,rtm,sep,serial,sgx,sgx_lc,smap,smep,ss,sse,sse2,sse4_1,sse4_2,ssse3,tm,tm2,tsc,tscdeadline,vme,x2apic,xsave,xtpr\",\n",
|
||||
" \"flags\": \"3dnow,3dnowprefetch,abm,acpi,adx,aes,apic,avx,avx2,bmi1,bmi2,clflush,clflushopt,cmov,cx16,cx8,de,dtes64,dts,erms,est,f16c,fma,fpu,fxsr,hle,ht,hypervisor,ia64,invpcid,lahf_lm,mca,mce,mmx,monitor,movbe,mpx,msr,mtrr,osxsave,pae,pat,pbe,pcid,pclmulqdq,pdcm,pge,pni,popcnt,pse,pse36,rdrnd,rdseed,rtm,sep,serial,sgx,sgx_lc,smap,smep,ss,sse,sse2,sse4_1,sse4_2,ssse3,tm,tm2,tsc,tscdeadline,vme,x2apic,xsave,xtpr\",\n",
|
||||
" \"processor\": \"Intel64 Family 6 Model 158 Stepping 10, GenuineIntel\"\n",
|
||||
" },\n",
|
||||
" \"memory\": {\n",
|
||||
" \"total\": 16977195008,\n",
|
||||
" \"available\": 7204651008\n",
|
||||
" \"available\": 9550102528\n",
|
||||
" },\n",
|
||||
" \"os\": \"Windows-10-10.0.22000-SP0\",\n",
|
||||
" \"python\": \"3.8.13.final.0 (64 bit)\",\n",
|
||||
" \"os\": \"Windows-10-10.0.22621-SP0\",\n",
|
||||
" \"python\": \"3.10.12.final.0 (64 bit)\",\n",
|
||||
" \"packages\": {\n",
|
||||
" \"sympy\": \"1.5.1\",\n",
|
||||
" \"transformers\": \"4.18.0\",\n",
|
||||
" \"protobuf\": \"3.20.1\",\n",
|
||||
" \"flatbuffers\": \"2.0\",\n",
|
||||
" \"numpy\": \"1.22.3\",\n",
|
||||
" \"ort-nightly\": \"1.12.0.dev20220428004\",\n",
|
||||
" \"onnx\": \"1.11.0\",\n",
|
||||
" \"torch\": \"1.10.1+cpu\",\n",
|
||||
" \"onnxconverter-common\": \"1.9.0\"\n",
|
||||
" \"flatbuffers\": \"23.5.26\",\n",
|
||||
" \"numpy\": \"1.25.2\",\n",
|
||||
" \"onnx\": \"1.14.0\",\n",
|
||||
" \"onnxruntime\": \"1.15.1\",\n",
|
||||
" \"protobuf\": \"4.23.4\",\n",
|
||||
" \"sympy\": \"1.12\",\n",
|
||||
" \"torch\": \"2.0.1\",\n",
|
||||
" \"transformers\": \"4.18.0\"\n",
|
||||
" },\n",
|
||||
" \"onnxruntime\": {\n",
|
||||
" \"version\": \"1.12.0\",\n",
|
||||
" \"version\": \"1.15.1\",\n",
|
||||
" \"support_gpu\": false\n",
|
||||
" },\n",
|
||||
" \"pytorch\": {\n",
|
||||
" \"version\": \"1.10.1+cpu\",\n",
|
||||
" \"version\": \"2.0.1+cpu\",\n",
|
||||
" \"support_gpu\": false,\n",
|
||||
" \"cuda\": null\n",
|
||||
" },\n",
|
||||
" \"tensorflow\": null\n",
|
||||
"}\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"f:\\anaconda3\\envs\\cpu_env\\lib\\site-packages\\onnxruntime\\transformers\\machine_info.py:127: DeprecationWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html\n",
|
||||
" import pkg_resources\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
|
|
@ -668,7 +714,7 @@
|
|||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.8.13"
|
||||
"version": "3.10.12"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
|
|
|
|||
|
|
@ -35,10 +35,13 @@
|
|||
"Otherwise, you can setup a new environment. First, we install [AnaConda](https://www.anaconda.com/distribution/). Then open an AnaConda prompt window and run the following commands:\n",
|
||||
"\n",
|
||||
"```console\n",
|
||||
"conda create -n cpu_env python=3.6\n",
|
||||
"conda create -n cpu_env python=3.10\n",
|
||||
"conda activate cpu_env\n",
|
||||
"conda install jupyter\n",
|
||||
"jupyter notebook\n",
|
||||
"pip install jupyterlab\n",
|
||||
"conda install ipykernel\n",
|
||||
"conda install -c conda-forge ipywidgets\n",
|
||||
"ipython kernel install --user --name cpu_env\n",
|
||||
"jupyter-lababook\n",
|
||||
"```\n",
|
||||
"The last command will launch Jupyter Notebook and we can open this notebook in browser to continue."
|
||||
]
|
||||
|
|
@ -49,21 +52,15 @@
|
|||
"metadata": {},
|
||||
"outputs": [],
|
||||
"source": [
|
||||
"# Please refer to https://pytorch.org to install CPU-only PyTorch\n",
|
||||
"import sys\n",
|
||||
"if sys.platform in [\"darwin\", \"win32\"]: # Mac or Windows\n",
|
||||
" !{sys.executable} -m pip install torch -q\n",
|
||||
"else:\n",
|
||||
" !{sys.executable} -m pip install install torch --index-url https://download.pytorch.org/whl/cpu -q\n",
|
||||
"\n",
|
||||
"run_install = False # Only need install once\n",
|
||||
"if run_install:\n",
|
||||
" if sys.platform in ['linux', 'win32']: # Linux or Windows\n",
|
||||
" !{sys.executable} -m pip install --upgrade torch torchvision torchaudio\n",
|
||||
" else: # Mac\n",
|
||||
" !{sys.executable} -m pip install torch==1.9.0+cpu torchvision==0.10.0+cpu torchaudio==0.9.0 -f https://download.pytorch.org/whl/torch_stable.html\n",
|
||||
"\n",
|
||||
" !{sys.executable} -m pip install onnxruntime==1.8.1 onnx==1.9.0 onnxconverter_common==1.8.1\n",
|
||||
"\n",
|
||||
" # Install other packages used in this notebook.\n",
|
||||
" !{sys.executable} -m pip install transformers==4.8.2\n",
|
||||
" !{sys.executable} -m pip install psutil pytz pandas py-cpuinfo py3nvml\n",
|
||||
" !{sys.executable} -m pip install wget netron"
|
||||
"# Install other packages used in this notebook.\n",
|
||||
"!{sys.executable} -m pip install onnxruntime onnx transformers==4.18 psutil pandas py-cpuinfo py3nvml wget netron -q"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -84,11 +81,20 @@
|
|||
"cell_type": "code",
|
||||
"execution_count": 2,
|
||||
"metadata": {},
|
||||
"outputs": [],
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Start downloading predict file.\n",
|
||||
"100% [..........................................................................] 4854279 / 4854279Predict file downloaded.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"import os\n",
|
||||
"\n",
|
||||
"cache_dir = os.path.join(\"..\", \"cache_models\")\n",
|
||||
"cache_dir = os.path.join(\".\", \"cache_models\")\n",
|
||||
"if not os.path.exists(cache_dir):\n",
|
||||
" os.makedirs(cache_dir)\n",
|
||||
"\n",
|
||||
|
|
@ -139,18 +145,74 @@
|
|||
"execution_count": 4,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "2d13e6ef5b5840d8b58930b908b1a0c8",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Downloading: 0%| | 0.00/570 [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "74cc5552b28c4ca4957b04e03a8d8db1",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Downloading: 0%| | 0.00/208k [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "f9f3c5c1a8494d048c0ead297812907c",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Downloading: 0%| | 0.00/29.0 [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"data": {
|
||||
"application/vnd.jupyter.widget-view+json": {
|
||||
"model_id": "377f46f6e3e64e628514eec9a01d54e2",
|
||||
"version_major": 2,
|
||||
"version_minor": 0
|
||||
},
|
||||
"text/plain": [
|
||||
"Downloading: 0%| | 0.00/416M [00:00<?, ?B/s]"
|
||||
]
|
||||
},
|
||||
"metadata": {},
|
||||
"output_type": "display_data"
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Some weights of the model checkpoint at bert-base-cased were not used when initializing BertForQuestionAnswering: ['cls.seq_relationship.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.predictions.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.dense.weight']\n",
|
||||
"Some weights of the model checkpoint at bert-base-cased were not used when initializing BertForQuestionAnswering: ['cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.transform.LayerNorm.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.decoder.weight', 'cls.predictions.bias', 'cls.seq_relationship.bias', 'cls.seq_relationship.weight']\n",
|
||||
"- This IS expected if you are initializing BertForQuestionAnswering from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPreTraining model).\n",
|
||||
"- This IS NOT expected if you are initializing BertForQuestionAnswering from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
|
||||
"Some weights of BertForQuestionAnswering were not initialized from the model checkpoint at bert-base-cased and are newly initialized: ['qa_outputs.weight', 'qa_outputs.bias']\n",
|
||||
"You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
|
||||
"100%|███████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 48/48 [00:03<00:00, 12.15it/s]\n",
|
||||
"convert squad examples to features: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 100/100 [00:00<00:00, 135.87it/s]\n",
|
||||
"add example index and unique id: 100%|████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████████| 100/100 [00:00<00:00, 100031.10it/s]\n"
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████| 48/48 [00:04<00:00, 11.48it/s]\n",
|
||||
"convert squad examples to features: 100%|███████████████████████████████████████████| 100/100 [00:00<00:00, 129.66it/s]\n",
|
||||
"add example index and unique id: 100%|███████████████████████████████████████████████████████| 100/100 [00:00<?, ?it/s]\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
|
@ -199,24 +261,20 @@
|
|||
"execution_count": 5,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"d:\\git\\transformers\\src\\transformers\\modeling_utils.py:2074: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n",
|
||||
" input_tensor.shape[chunk_dim] == tensor_shape for input_tensor in input_tensors\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"============== Diagnostic Run torch.onnx.export version 2.0.1+cpu ==============\n",
|
||||
"verbose: False, log level: Level.ERROR\n",
|
||||
"======================= 0 NONE 0 NOTE 0 WARNING 0 ERROR ========================\n",
|
||||
"\n",
|
||||
"Model exported at ..\\onnx_models\\bert-base-cased-squad.onnx\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"output_dir = os.path.join(\"..\", \"onnx_models\")\n",
|
||||
"output_dir = os.path.join(\".\", \"onnx_models\")\n",
|
||||
"if not os.path.exists(output_dir):\n",
|
||||
" os.makedirs(output_dir) \n",
|
||||
"export_model_path = os.path.join(output_dir, 'bert-base-cased-squad.onnx')\n",
|
||||
|
|
@ -274,7 +332,7 @@
|
|||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"PyTorch cpu Inference time = 119.80 ms\n"
|
||||
"PyTorch cpu Inference time = 178.87 ms\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
|
@ -322,7 +380,7 @@
|
|||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"OnnxRuntime cpu Inference time = 72.46 ms\n"
|
||||
"OnnxRuntime cpu Inference time = 84.11 ms\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
|
@ -336,10 +394,6 @@
|
|||
"# Note that this will increase session creation time, so it is for debugging only.\n",
|
||||
"sess_options.optimized_model_filepath = os.path.join(output_dir, \"optimized_model_cpu.onnx\")\n",
|
||||
"\n",
|
||||
"# For OnnxRuntime 1.7.0 or later, you can set intra_op_num_threads to set thread number like\n",
|
||||
"# sess_options.intra_op_num_threads=4\n",
|
||||
"# Here we use the default value which is a good choice in most cases.\n",
|
||||
"\n",
|
||||
"# Specify providers when you use onnxruntime-gpu for CPU inference.\n",
|
||||
"session = onnxruntime.InferenceSession(export_model_path, sess_options, providers=['CPUExecutionProvider'])\n",
|
||||
"\n",
|
||||
|
|
@ -423,29 +477,27 @@
|
|||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
" apply: Fused LayerNormalization count: 25\n",
|
||||
" apply: Fused Gelu count: 12\n",
|
||||
"adjust_reshape_and_expand: Removed Reshape and Expand count: 0\n",
|
||||
" apply: Fused SkipLayerNormalization count: 24\n",
|
||||
" apply: Fused Attention count: 12\n",
|
||||
" prune_graph: Graph pruned: 0 inputs, 0 outputs and 5 nodes are removed\n",
|
||||
" apply: Fused EmbedLayerNormalization(with mask) count: 1\n",
|
||||
" prune_graph: Graph pruned: 0 inputs, 0 outputs and 3 nodes are removed\n",
|
||||
" prune_graph: Graph pruned: 0 inputs, 0 outputs and 0 nodes are removed\n",
|
||||
" apply: Fused BiasGelu count: 12\n",
|
||||
" apply: Fused SkipLayerNormalization(add bias) count: 24\n",
|
||||
" apply: Fused LayerNormalization: 25\n",
|
||||
" apply: Fused Gelu: 12\n",
|
||||
" apply: Fused SkipLayerNormalization: 24\n",
|
||||
" apply: Fused Attention: 12\n",
|
||||
" prune_graph: Removed 5 nodes\n",
|
||||
" apply: Fused EmbedLayerNormalization(with mask): 1\n",
|
||||
" prune_graph: Removed 10 nodes\n",
|
||||
" apply: Fused BiasGelu: 12\n",
|
||||
" apply: Fused SkipLayerNormalization(add bias): 24\n",
|
||||
" optimize: opset version: 11\n",
|
||||
"get_fused_operator_statistics: Optimized operators:{'EmbedLayerNormalization': 1, 'Attention': 12, 'MultiHeadAttention': 0, 'Gelu': 0, 'FastGelu': 0, 'BiasGelu': 12, 'GemmFastGelu': 0, 'LayerNormalization': 0, 'SkipLayerNormalization': 24, 'QOrderedAttention': 0, 'QOrderedGelu': 0, 'QOrderedLayerNormalization': 0, 'QOrderedMatMul': 0}\n",
|
||||
" main: The model has been fully optimized.\n",
|
||||
" save_model_to_file: Sort graphs in topological order\n",
|
||||
" save_model_to_file: Output model to ..\\onnx_models\\bert-base-cased-squad_opt_cpu.onnx\n",
|
||||
"get_fused_operator_statistics: Optimized operators:{'EmbedLayerNormalization': 1, 'Attention': 12, 'Gelu': 0, 'FastGelu': 0, 'BiasGelu': 12, 'LayerNormalization': 0, 'SkipLayerNormalization': 24}\n",
|
||||
" main: The model has been fully optimized.\n"
|
||||
" save_model_to_file: Model saved to ..\\onnx_models\\bert-base-cased-squad_opt_cpu.onnx\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"optimized_model_path = os.path.join(output_dir, 'bert-base-cased-squad_opt_cpu.onnx')\n",
|
||||
"\n",
|
||||
"!{sys.executable} -m onnxruntime.transformers.optimizer --input $export_model_path --output $optimized_model_path --model_type bert --num_heads 12 --hidden_size 768"
|
||||
"!{sys.executable} -m onnxruntime.transformers.optimizer --input $export_model_path --output $optimized_model_path --model_type bert"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -503,8 +555,8 @@
|
|||
"output_type": "stream",
|
||||
"text": [
|
||||
"100% passed for 100 random inputs given thresholds (rtol=0.001, atol=0.0001).\n",
|
||||
"maximum absolute difference=4.604458808898926e-06\n",
|
||||
"maximum relative difference=0.006278202868998051\n"
|
||||
"maximum absolute difference=2.771615982055664e-06\n",
|
||||
"maximum relative difference=0.017596205696463585\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
|
@ -533,32 +585,32 @@
|
|||
"output_type": "stream",
|
||||
"text": [
|
||||
"Running test: model=bert-base-cased-squad_opt_cpu.onnx,graph_optimization_level=ENABLE_ALL,intra_op_num_threads=12,batch_size=1,sequence_length=128,test_cases=100,test_times=1,use_gpu=False\n",
|
||||
"Average latency = 54.26 ms, Throughput = 18.43 QPS\n",
|
||||
"Average latency = 52.26 ms, Throughput = 19.14 QPS\n",
|
||||
"Running test: model=bert-base-cased-squad_opt_cpu.onnx,graph_optimization_level=ENABLE_ALL,intra_op_num_threads=11,batch_size=1,sequence_length=128,test_cases=100,test_times=1,use_gpu=False\n",
|
||||
"Average latency = 55.80 ms, Throughput = 17.92 QPS\n",
|
||||
"Average latency = 52.67 ms, Throughput = 18.99 QPS\n",
|
||||
"Running test: model=bert-base-cased-squad_opt_cpu.onnx,graph_optimization_level=ENABLE_ALL,intra_op_num_threads=10,batch_size=1,sequence_length=128,test_cases=100,test_times=1,use_gpu=False\n",
|
||||
"Average latency = 65.31 ms, Throughput = 15.31 QPS\n",
|
||||
"Average latency = 54.57 ms, Throughput = 18.32 QPS\n",
|
||||
"Running test: model=bert-base-cased-squad_opt_cpu.onnx,graph_optimization_level=ENABLE_ALL,intra_op_num_threads=9,batch_size=1,sequence_length=128,test_cases=100,test_times=1,use_gpu=False\n",
|
||||
"Average latency = 57.66 ms, Throughput = 17.34 QPS\n",
|
||||
"Average latency = 63.12 ms, Throughput = 15.84 QPS\n",
|
||||
"Running test: model=bert-base-cased-squad_opt_cpu.onnx,graph_optimization_level=ENABLE_ALL,intra_op_num_threads=8,batch_size=1,sequence_length=128,test_cases=100,test_times=1,use_gpu=False\n",
|
||||
"Average latency = 62.84 ms, Throughput = 15.91 QPS\n",
|
||||
"Average latency = 59.80 ms, Throughput = 16.72 QPS\n",
|
||||
"Running test: model=bert-base-cased-squad_opt_cpu.onnx,graph_optimization_level=ENABLE_ALL,intra_op_num_threads=7,batch_size=1,sequence_length=128,test_cases=100,test_times=1,use_gpu=False\n",
|
||||
"Average latency = 69.29 ms, Throughput = 14.43 QPS\n",
|
||||
"Average latency = 64.72 ms, Throughput = 15.45 QPS\n",
|
||||
"Running test: model=bert-base-cased-squad_opt_cpu.onnx,graph_optimization_level=ENABLE_ALL,intra_op_num_threads=6,batch_size=1,sequence_length=128,test_cases=100,test_times=1,use_gpu=False\n",
|
||||
"Average latency = 56.19 ms, Throughput = 17.80 QPS\n",
|
||||
"Average latency = 58.62 ms, Throughput = 17.06 QPS\n",
|
||||
"Running test: model=bert-base-cased-squad_opt_cpu.onnx,graph_optimization_level=ENABLE_ALL,intra_op_num_threads=5,batch_size=1,sequence_length=128,test_cases=100,test_times=1,use_gpu=False\n",
|
||||
"Average latency = 59.90 ms, Throughput = 16.70 QPS\n",
|
||||
"Average latency = 73.72 ms, Throughput = 13.56 QPS\n",
|
||||
"Running test: model=bert-base-cased-squad_opt_cpu.onnx,graph_optimization_level=ENABLE_ALL,intra_op_num_threads=4,batch_size=1,sequence_length=128,test_cases=100,test_times=1,use_gpu=False\n",
|
||||
"Average latency = 63.72 ms, Throughput = 15.69 QPS\n",
|
||||
"Average latency = 88.76 ms, Throughput = 11.27 QPS\n",
|
||||
"Running test: model=bert-base-cased-squad_opt_cpu.onnx,graph_optimization_level=ENABLE_ALL,intra_op_num_threads=3,batch_size=1,sequence_length=128,test_cases=100,test_times=1,use_gpu=False\n",
|
||||
"Average latency = 82.44 ms, Throughput = 12.13 QPS\n",
|
||||
"Average latency = 91.14 ms, Throughput = 10.97 QPS\n",
|
||||
"Running test: model=bert-base-cased-squad_opt_cpu.onnx,graph_optimization_level=ENABLE_ALL,intra_op_num_threads=2,batch_size=1,sequence_length=128,test_cases=100,test_times=1,use_gpu=False\n",
|
||||
"Average latency = 119.64 ms, Throughput = 8.36 QPS\n",
|
||||
"Average latency = 119.80 ms, Throughput = 8.35 QPS\n",
|
||||
"Running test: model=bert-base-cased-squad_opt_cpu.onnx,graph_optimization_level=ENABLE_ALL,intra_op_num_threads=1,batch_size=1,sequence_length=128,test_cases=100,test_times=1,use_gpu=False\n",
|
||||
"Average latency = 223.21 ms, Throughput = 4.48 QPS\n",
|
||||
"test setting TestSetting(batch_size=1, sequence_length=128, test_cases=100, test_times=1, use_gpu=False, intra_op_num_threads=None, seed=3, verbose=False)\n",
|
||||
"Average latency = 225.06 ms, Throughput = 4.44 QPS\n",
|
||||
"test setting TestSetting(batch_size=1, sequence_length=128, test_cases=100, test_times=1, use_gpu=False, use_io_binding=False, provider=None, intra_op_num_threads=None, seed=3, verbose=False, log_severity=2)\n",
|
||||
"Generating 100 samples for batch_size=1 sequence_length=128\n",
|
||||
"Test summary is saved to ..\\onnx_models\\perf_results_CPU_B1_S128_20210713-144140.txt\n"
|
||||
"Test summary is saved to ..\\onnx_models\\perf_results_CPU_B1_S128_20230805-202517.txt\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
|
@ -582,7 +634,7 @@
|
|||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"..\\onnx_models\\perf_results_CPU_B1_S128_20210713-144140.txt\n"
|
||||
"..\\onnx_models\\perf_results_CPU_B1_S128_20230805-202517.txt\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -617,110 +669,110 @@
|
|||
" <tbody>\n",
|
||||
" <tr>\n",
|
||||
" <th>0</th>\n",
|
||||
" <td>54.26</td>\n",
|
||||
" <td>56.05</td>\n",
|
||||
" <td>60.32</td>\n",
|
||||
" <td>109.21</td>\n",
|
||||
" <td>18.43</td>\n",
|
||||
" <td>52.26</td>\n",
|
||||
" <td>52.92</td>\n",
|
||||
" <td>60.94</td>\n",
|
||||
" <td>88.51</td>\n",
|
||||
" <td>19.14</td>\n",
|
||||
" <td>12</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1</th>\n",
|
||||
" <td>55.80</td>\n",
|
||||
" <td>56.74</td>\n",
|
||||
" <td>59.67</td>\n",
|
||||
" <td>73.62</td>\n",
|
||||
" <td>17.92</td>\n",
|
||||
" <td>52.67</td>\n",
|
||||
" <td>52.73</td>\n",
|
||||
" <td>54.06</td>\n",
|
||||
" <td>60.48</td>\n",
|
||||
" <td>18.99</td>\n",
|
||||
" <td>11</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2</th>\n",
|
||||
" <td>56.19</td>\n",
|
||||
" <td>61.29</td>\n",
|
||||
" <td>71.69</td>\n",
|
||||
" <td>80.15</td>\n",
|
||||
" <td>17.80</td>\n",
|
||||
" <td>6</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>3</th>\n",
|
||||
" <td>57.66</td>\n",
|
||||
" <td>58.50</td>\n",
|
||||
" <td>61.96</td>\n",
|
||||
" <td>65.12</td>\n",
|
||||
" <td>17.34</td>\n",
|
||||
" <td>9</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>4</th>\n",
|
||||
" <td>59.90</td>\n",
|
||||
" <td>59.72</td>\n",
|
||||
" <td>65.16</td>\n",
|
||||
" <td>116.16</td>\n",
|
||||
" <td>16.70</td>\n",
|
||||
" <td>5</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>5</th>\n",
|
||||
" <td>62.84</td>\n",
|
||||
" <td>67.05</td>\n",
|
||||
" <td>69.07</td>\n",
|
||||
" <td>75.99</td>\n",
|
||||
" <td>15.91</td>\n",
|
||||
" <td>8</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>6</th>\n",
|
||||
" <td>63.72</td>\n",
|
||||
" <td>64.17</td>\n",
|
||||
" <td>69.44</td>\n",
|
||||
" <td>73.10</td>\n",
|
||||
" <td>15.69</td>\n",
|
||||
" <td>4</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>7</th>\n",
|
||||
" <td>65.31</td>\n",
|
||||
" <td>65.35</td>\n",
|
||||
" <td>80.70</td>\n",
|
||||
" <td>177.94</td>\n",
|
||||
" <td>15.31</td>\n",
|
||||
" <td>54.57</td>\n",
|
||||
" <td>55.05</td>\n",
|
||||
" <td>56.52</td>\n",
|
||||
" <td>59.62</td>\n",
|
||||
" <td>18.32</td>\n",
|
||||
" <td>10</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>8</th>\n",
|
||||
" <td>69.29</td>\n",
|
||||
" <td>69.04</td>\n",
|
||||
" <td>70.68</td>\n",
|
||||
" <td>85.03</td>\n",
|
||||
" <td>14.43</td>\n",
|
||||
" <th>3</th>\n",
|
||||
" <td>58.62</td>\n",
|
||||
" <td>65.86</td>\n",
|
||||
" <td>71.94</td>\n",
|
||||
" <td>79.38</td>\n",
|
||||
" <td>17.06</td>\n",
|
||||
" <td>6</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>4</th>\n",
|
||||
" <td>59.80</td>\n",
|
||||
" <td>61.09</td>\n",
|
||||
" <td>65.72</td>\n",
|
||||
" <td>72.79</td>\n",
|
||||
" <td>16.72</td>\n",
|
||||
" <td>8</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>5</th>\n",
|
||||
" <td>63.12</td>\n",
|
||||
" <td>61.92</td>\n",
|
||||
" <td>77.47</td>\n",
|
||||
" <td>129.26</td>\n",
|
||||
" <td>15.84</td>\n",
|
||||
" <td>9</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>6</th>\n",
|
||||
" <td>64.72</td>\n",
|
||||
" <td>65.94</td>\n",
|
||||
" <td>68.54</td>\n",
|
||||
" <td>75.00</td>\n",
|
||||
" <td>15.45</td>\n",
|
||||
" <td>7</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>7</th>\n",
|
||||
" <td>73.72</td>\n",
|
||||
" <td>80.43</td>\n",
|
||||
" <td>86.39</td>\n",
|
||||
" <td>92.95</td>\n",
|
||||
" <td>13.56</td>\n",
|
||||
" <td>5</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>8</th>\n",
|
||||
" <td>88.76</td>\n",
|
||||
" <td>96.30</td>\n",
|
||||
" <td>101.13</td>\n",
|
||||
" <td>109.78</td>\n",
|
||||
" <td>11.27</td>\n",
|
||||
" <td>4</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>9</th>\n",
|
||||
" <td>82.44</td>\n",
|
||||
" <td>83.20</td>\n",
|
||||
" <td>89.64</td>\n",
|
||||
" <td>98.80</td>\n",
|
||||
" <td>12.13</td>\n",
|
||||
" <td>91.14</td>\n",
|
||||
" <td>99.14</td>\n",
|
||||
" <td>104.41</td>\n",
|
||||
" <td>110.69</td>\n",
|
||||
" <td>10.97</td>\n",
|
||||
" <td>3</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>10</th>\n",
|
||||
" <td>119.64</td>\n",
|
||||
" <td>119.07</td>\n",
|
||||
" <td>122.62</td>\n",
|
||||
" <td>135.67</td>\n",
|
||||
" <td>8.36</td>\n",
|
||||
" <td>119.80</td>\n",
|
||||
" <td>119.78</td>\n",
|
||||
" <td>123.66</td>\n",
|
||||
" <td>130.64</td>\n",
|
||||
" <td>8.35</td>\n",
|
||||
" <td>2</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>11</th>\n",
|
||||
" <td>223.21</td>\n",
|
||||
" <td>223.22</td>\n",
|
||||
" <td>226.83</td>\n",
|
||||
" <td>249.08</td>\n",
|
||||
" <td>4.48</td>\n",
|
||||
" <td>225.06</td>\n",
|
||||
" <td>227.11</td>\n",
|
||||
" <td>229.45</td>\n",
|
||||
" <td>252.40</td>\n",
|
||||
" <td>4.44</td>\n",
|
||||
" <td>1</td>\n",
|
||||
" </tr>\n",
|
||||
" </tbody>\n",
|
||||
|
|
@ -729,29 +781,29 @@
|
|||
],
|
||||
"text/plain": [
|
||||
" Latency(ms) Latency_P75 Latency_P90 Latency_P99 Throughput(QPS) \\\n",
|
||||
"0 54.26 56.05 60.32 109.21 18.43 \n",
|
||||
"1 55.80 56.74 59.67 73.62 17.92 \n",
|
||||
"2 56.19 61.29 71.69 80.15 17.80 \n",
|
||||
"3 57.66 58.50 61.96 65.12 17.34 \n",
|
||||
"4 59.90 59.72 65.16 116.16 16.70 \n",
|
||||
"5 62.84 67.05 69.07 75.99 15.91 \n",
|
||||
"6 63.72 64.17 69.44 73.10 15.69 \n",
|
||||
"7 65.31 65.35 80.70 177.94 15.31 \n",
|
||||
"8 69.29 69.04 70.68 85.03 14.43 \n",
|
||||
"9 82.44 83.20 89.64 98.80 12.13 \n",
|
||||
"10 119.64 119.07 122.62 135.67 8.36 \n",
|
||||
"11 223.21 223.22 226.83 249.08 4.48 \n",
|
||||
"0 52.26 52.92 60.94 88.51 19.14 \n",
|
||||
"1 52.67 52.73 54.06 60.48 18.99 \n",
|
||||
"2 54.57 55.05 56.52 59.62 18.32 \n",
|
||||
"3 58.62 65.86 71.94 79.38 17.06 \n",
|
||||
"4 59.80 61.09 65.72 72.79 16.72 \n",
|
||||
"5 63.12 61.92 77.47 129.26 15.84 \n",
|
||||
"6 64.72 65.94 68.54 75.00 15.45 \n",
|
||||
"7 73.72 80.43 86.39 92.95 13.56 \n",
|
||||
"8 88.76 96.30 101.13 109.78 11.27 \n",
|
||||
"9 91.14 99.14 104.41 110.69 10.97 \n",
|
||||
"10 119.80 119.78 123.66 130.64 8.35 \n",
|
||||
"11 225.06 227.11 229.45 252.40 4.44 \n",
|
||||
"\n",
|
||||
" intra_op_num_threads \n",
|
||||
"0 12 \n",
|
||||
"1 11 \n",
|
||||
"2 6 \n",
|
||||
"3 9 \n",
|
||||
"4 5 \n",
|
||||
"5 8 \n",
|
||||
"6 4 \n",
|
||||
"7 10 \n",
|
||||
"8 7 \n",
|
||||
"2 10 \n",
|
||||
"3 6 \n",
|
||||
"4 8 \n",
|
||||
"5 9 \n",
|
||||
"6 7 \n",
|
||||
"7 5 \n",
|
||||
"8 4 \n",
|
||||
"9 3 \n",
|
||||
"10 2 \n",
|
||||
"11 1 "
|
||||
|
|
@ -782,7 +834,16 @@
|
|||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 6. Additional Info\n",
|
||||
"## 6. Quantization\n",
|
||||
"\n",
|
||||
"Please see example in [BERT quantization notebook](https://github.com/microsoft/onnxruntime-inference-examples/blob/main/quantization/notebooks/bert/Bert-GLUE_OnnxRuntime_quantization.ipynb)."
|
||||
]
|
||||
},
|
||||
{
|
||||
"cell_type": "markdown",
|
||||
"metadata": {},
|
||||
"source": [
|
||||
"## 7. Additional Info\n",
|
||||
"\n",
|
||||
"Note that running Jupyter Notebook has slight impact on performance result since Jupyter Notebook is using system resources like CPU and memory etc. It is recommended to close Jupyter Notebook and other applications, then run the performance test tool in a console to get more accurate performance numbers.\n",
|
||||
"\n",
|
||||
|
|
@ -805,12 +866,12 @@
|
|||
"text": [
|
||||
"{\n",
|
||||
" \"gpu\": {\n",
|
||||
" \"driver_version\": \"470.14\",\n",
|
||||
" \"driver_version\": \"472.88\",\n",
|
||||
" \"devices\": [\n",
|
||||
" {\n",
|
||||
" \"memory_total\": 8589934592,\n",
|
||||
" \"memory_available\": 6782619648,\n",
|
||||
" \"name\": \"NVIDIA GeForce GTX 1070\"\n",
|
||||
" \"memory_total\": 12884901888,\n",
|
||||
" \"memory_available\": 12732858368,\n",
|
||||
" \"name\": \"NVIDIA GeForce RTX 3060\"\n",
|
||||
" }\n",
|
||||
" ]\n",
|
||||
" },\n",
|
||||
|
|
@ -818,106 +879,37 @@
|
|||
" \"brand\": \"Intel(R) Core(TM) i7-8700 CPU @ 3.20GHz\",\n",
|
||||
" \"cores\": 6,\n",
|
||||
" \"logical_cores\": 12,\n",
|
||||
" \"hz\": \"3.1920 GHz\",\n",
|
||||
" \"l2_cache\": \"1536 KB\",\n",
|
||||
" \"flags\": [\n",
|
||||
" \"3dnow\",\n",
|
||||
" \"3dnowprefetch\",\n",
|
||||
" \"abm\",\n",
|
||||
" \"acpi\",\n",
|
||||
" \"adx\",\n",
|
||||
" \"aes\",\n",
|
||||
" \"apic\",\n",
|
||||
" \"avx\",\n",
|
||||
" \"avx2\",\n",
|
||||
" \"bmi1\",\n",
|
||||
" \"bmi2\",\n",
|
||||
" \"clflush\",\n",
|
||||
" \"clflushopt\",\n",
|
||||
" \"cmov\",\n",
|
||||
" \"cx16\",\n",
|
||||
" \"cx8\",\n",
|
||||
" \"de\",\n",
|
||||
" \"dtes64\",\n",
|
||||
" \"dts\",\n",
|
||||
" \"erms\",\n",
|
||||
" \"est\",\n",
|
||||
" \"f16c\",\n",
|
||||
" \"fma\",\n",
|
||||
" \"fpu\",\n",
|
||||
" \"fxsr\",\n",
|
||||
" \"hle\",\n",
|
||||
" \"ht\",\n",
|
||||
" \"hypervisor\",\n",
|
||||
" \"ia64\",\n",
|
||||
" \"invpcid\",\n",
|
||||
" \"lahf_lm\",\n",
|
||||
" \"mca\",\n",
|
||||
" \"mce\",\n",
|
||||
" \"mmx\",\n",
|
||||
" \"movbe\",\n",
|
||||
" \"mpx\",\n",
|
||||
" \"msr\",\n",
|
||||
" \"mtrr\",\n",
|
||||
" \"osxsave\",\n",
|
||||
" \"pae\",\n",
|
||||
" \"pat\",\n",
|
||||
" \"pbe\",\n",
|
||||
" \"pcid\",\n",
|
||||
" \"pclmulqdq\",\n",
|
||||
" \"pdcm\",\n",
|
||||
" \"pge\",\n",
|
||||
" \"pni\",\n",
|
||||
" \"popcnt\",\n",
|
||||
" \"pse\",\n",
|
||||
" \"pse36\",\n",
|
||||
" \"rdrnd\",\n",
|
||||
" \"rdseed\",\n",
|
||||
" \"rtm\",\n",
|
||||
" \"sep\",\n",
|
||||
" \"serial\",\n",
|
||||
" \"sgx\",\n",
|
||||
" \"sgx_lc\",\n",
|
||||
" \"smap\",\n",
|
||||
" \"smep\",\n",
|
||||
" \"ss\",\n",
|
||||
" \"sse\",\n",
|
||||
" \"sse2\",\n",
|
||||
" \"sse4_1\",\n",
|
||||
" \"sse4_2\",\n",
|
||||
" \"ssse3\",\n",
|
||||
" \"tm\",\n",
|
||||
" \"tm2\",\n",
|
||||
" \"tsc\",\n",
|
||||
" \"tscdeadline\",\n",
|
||||
" \"vme\",\n",
|
||||
" \"x2apic\",\n",
|
||||
" \"xsave\",\n",
|
||||
" \"xtpr\"\n",
|
||||
" ],\n",
|
||||
" \"hz\": \"3192000000,0\",\n",
|
||||
" \"l2_cache\": 1572864,\n",
|
||||
" \"flags\": \"3dnow,3dnowprefetch,abm,acpi,adx,aes,apic,avx,avx2,bmi1,bmi2,clflush,clflushopt,cmov,cx16,cx8,de,dtes64,dts,erms,est,f16c,fma,fpu,fxsr,hle,ht,hypervisor,ia64,invpcid,lahf_lm,mca,mce,mmx,monitor,movbe,mpx,msr,mtrr,osxsave,pae,pat,pbe,pcid,pclmulqdq,pdcm,pge,pni,popcnt,pse,pse36,rdrnd,rdseed,rtm,sep,serial,sgx,sgx_lc,smap,smep,ss,sse,sse2,sse4_1,sse4_2,ssse3,tm,tm2,tsc,tscdeadline,vme,x2apic,xsave,xtpr\",\n",
|
||||
" \"processor\": \"Intel64 Family 6 Model 158 Stepping 10, GenuineIntel\"\n",
|
||||
" },\n",
|
||||
" \"memory\": {\n",
|
||||
" \"total\": 16977195008,\n",
|
||||
" \"available\": 6085459968\n",
|
||||
" \"available\": 5000630272\n",
|
||||
" },\n",
|
||||
" \"os\": \"Windows-10-10.0.22621-SP0\",\n",
|
||||
" \"python\": \"3.10.12.final.0 (64 bit)\",\n",
|
||||
" \"packages\": {\n",
|
||||
" \"flatbuffers\": \"23.5.26\",\n",
|
||||
" \"numpy\": \"1.25.2\",\n",
|
||||
" \"onnx\": \"1.14.0\",\n",
|
||||
" \"onnxruntime\": \"1.15.1\",\n",
|
||||
" \"protobuf\": \"4.23.4\",\n",
|
||||
" \"sympy\": \"1.12\",\n",
|
||||
" \"torch\": \"2.0.1\",\n",
|
||||
" \"transformers\": \"4.18.0\"\n",
|
||||
" },\n",
|
||||
" \"python\": \"3.6.10.final.0 (64 bit)\",\n",
|
||||
" \"os\": \"Windows-10-10.0.21390-SP0\",\n",
|
||||
" \"onnxruntime\": {\n",
|
||||
" \"version\": \"1.8.1\",\n",
|
||||
" \"version\": \"1.15.1\",\n",
|
||||
" \"support_gpu\": false\n",
|
||||
" },\n",
|
||||
" \"onnxruntime_tools\": null,\n",
|
||||
" \"pytorch\": {\n",
|
||||
" \"version\": \"1.9.0+cpu\",\n",
|
||||
" \"version\": \"2.0.1+cpu\",\n",
|
||||
" \"support_gpu\": false,\n",
|
||||
" \"cuda\": null\n",
|
||||
" },\n",
|
||||
" \"tensorflow\": {\n",
|
||||
" \"version\": \"2.3.0\",\n",
|
||||
" \"git_version\": \"v2.3.0-rc2-23-gb36436b087\",\n",
|
||||
" \"support_gpu\": true\n",
|
||||
" }\n",
|
||||
" \"tensorflow\": null\n",
|
||||
"}\n"
|
||||
]
|
||||
},
|
||||
|
|
@ -925,8 +917,8 @@
|
|||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"2021-07-13 14:41:45.376756: W tensorflow/stream_executor/platform/default/dso_loader.cc:59] Could not load dynamic library 'cudart64_101.dll'; dlerror: cudart64_101.dll not found\n",
|
||||
"2021-07-13 14:41:45.376780: I tensorflow/stream_executor/cuda/cudart_stub.cc:29] Ignore above cudart dlerror if you do not have a GPU set up on your machine.\n"
|
||||
"f:\\anaconda3\\envs\\cpu_env\\lib\\site-packages\\onnxruntime\\transformers\\machine_info.py:127: DeprecationWarning: pkg_resources is deprecated as an API. See https://setuptools.pypa.io/en/latest/pkg_resources.html\n",
|
||||
" import pkg_resources\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
|
@ -951,9 +943,9 @@
|
|||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.6.10"
|
||||
"version": "3.10.12"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
"nbformat_minor": 2
|
||||
"nbformat_minor": 4
|
||||
}
|
||||
|
|
|
|||
Loading…
Reference in a new issue