mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-30 20:18:08 +00:00
update bert quantization notebook (#7137)
This commit is contained in:
Yufeng Li
GitHub
parent
c9b29fbd06
commit
3771e0bf10
1 changed files with 114 additions and 121 deletions
|
|
@ -33,7 +33,7 @@
|
|||
"Otherwise, you can setup a new environment. First, install [AnaConda](https://www.anaconda.com/distribution/). Then open an AnaConda prompt window and run the following commands:\n",
|
||||
"\n",
|
||||
"```console\n",
|
||||
"conda create -n cpu_env python=3.6\n",
|
||||
"conda create -n cpu_env python=3.8\n",
|
||||
"conda activate cpu_env\n",
|
||||
"conda install jupyter\n",
|
||||
"jupyter notebook\n",
|
||||
|
|
@ -46,7 +46,7 @@
|
|||
"metadata": {},
|
||||
"source": [
|
||||
"### 0.1 Install packages\n",
|
||||
"Let's install nessasary packages to start the tutorial. We will install PyTorch 1.6, OnnxRuntime 1.5.1, latest ONNX, OnnxRuntime-tools, transformers, and sklearn."
|
||||
"Let's install nessasary packages to start the tutorial. We will install PyTorch 1.8, OnnxRuntime 1.7, latest ONNX, OnnxRuntime-tools, transformers, and sklearn."
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -61,89 +61,71 @@
|
|||
"output_type": "stream",
|
||||
"text": [
|
||||
"Looking in links: https://download.pytorch.org/whl/torch_stable.html\n",
|
||||
"Requirement already up-to-date: torch==1.6.0+cpu in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (1.6.0+cpu)\n",
|
||||
"Requirement already up-to-date: torchvision==0.7.0+cpu in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (0.7.0+cpu)\n",
|
||||
"Requirement already satisfied, skipping upgrade: numpy in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (from torch==1.6.0+cpu) (1.18.1)\n",
|
||||
"Requirement already satisfied, skipping upgrade: future in /home/yufeng/.local/lib/python3.6/site-packages (from torch==1.6.0+cpu) (0.18.2)\n",
|
||||
"Requirement already satisfied, skipping upgrade: pillow>=4.1.1 in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from torchvision==0.7.0+cpu) (7.0.0)\n",
|
||||
"Collecting onnxruntime==1.5.1\n",
|
||||
" Using cached onnxruntime-1.5.1-cp36-cp36m-manylinux2014_x86_64.whl (3.8 MB)\n",
|
||||
"Requirement already satisfied, skipping upgrade: numpy>=1.16.6 in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (from onnxruntime==1.5.1) (1.18.1)\n",
|
||||
"Requirement already satisfied, skipping upgrade: protobuf in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from onnxruntime==1.5.1) (3.9.1)\n",
|
||||
"Requirement already satisfied, skipping upgrade: six>=1.9 in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from protobuf->onnxruntime==1.5.1) (1.12.0)\n",
|
||||
"Requirement already satisfied, skipping upgrade: setuptools in /home/yufeng/.local/lib/python3.6/site-packages (from protobuf->onnxruntime==1.5.1) (41.0.1)\n",
|
||||
"Installing collected packages: onnxruntime\n",
|
||||
" Attempting uninstall: onnxruntime\n",
|
||||
" Found existing installation: onnxruntime 1.4.0\n",
|
||||
" Uninstalling onnxruntime-1.4.0:\n",
|
||||
" Successfully uninstalled onnxruntime-1.4.0\n",
|
||||
"Successfully installed onnxruntime-1.5.1\n",
|
||||
"Requirement already up-to-date: onnxruntime-tools in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (1.5.1)\n",
|
||||
"Requirement already satisfied, skipping upgrade: py-cpuinfo in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (from onnxruntime-tools) (5.0.0)\n",
|
||||
"Requirement already satisfied, skipping upgrade: packaging in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from onnxruntime-tools) (19.1)\n",
|
||||
"Requirement already satisfied, skipping upgrade: onnx in /home/yufeng/project/onnx (from onnxruntime-tools) (1.6.0)\n",
|
||||
"Requirement already satisfied, skipping upgrade: coloredlogs in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (from onnxruntime-tools) (14.0)\n",
|
||||
"Requirement already satisfied, skipping upgrade: py3nvml in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (from onnxruntime-tools) (0.2.6)\n",
|
||||
"Requirement already satisfied, skipping upgrade: psutil in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from onnxruntime-tools) (5.6.3)\n",
|
||||
"Requirement already satisfied, skipping upgrade: numpy in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (from onnxruntime-tools) (1.18.1)\n",
|
||||
"Requirement already satisfied, skipping upgrade: six in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from packaging->onnxruntime-tools) (1.12.0)\n",
|
||||
"Requirement already satisfied, skipping upgrade: attrs in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from packaging->onnxruntime-tools) (19.1.0)\n",
|
||||
"Requirement already satisfied, skipping upgrade: pyparsing>=2.0.2 in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from packaging->onnxruntime-tools) (2.4.2)\n",
|
||||
"Requirement already satisfied, skipping upgrade: protobuf in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from onnx->onnxruntime-tools) (3.9.1)\n",
|
||||
"Requirement already satisfied, skipping upgrade: typing-extensions>=3.6.2.1 in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages/typing_extensions-3.7.4-py3.6.egg (from onnx->onnxruntime-tools) (3.7.4)\n",
|
||||
"Requirement already satisfied, skipping upgrade: humanfriendly>=7.1 in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (from coloredlogs->onnxruntime-tools) (8.2)\n",
|
||||
"Requirement already satisfied, skipping upgrade: xmltodict in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (from py3nvml->onnxruntime-tools) (0.12.0)\n",
|
||||
"Requirement already satisfied, skipping upgrade: setuptools in /home/yufeng/.local/lib/python3.6/site-packages (from protobuf->onnx->onnxruntime-tools) (41.0.1)\n",
|
||||
"Requirement already up-to-date: transformers in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (3.3.1)\n",
|
||||
"Requirement already satisfied, skipping upgrade: tokenizers==0.8.1.rc2 in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (from transformers) (0.8.1rc2)\n",
|
||||
"Requirement already satisfied, skipping upgrade: tqdm>=4.27 in /home/yufeng/.local/lib/python3.6/site-packages (from transformers) (4.45.0)\n",
|
||||
"Requirement already satisfied, skipping upgrade: dataclasses; python_version < \"3.7\" in /home/yufeng/.local/lib/python3.6/site-packages (from transformers) (0.7)\n",
|
||||
"Requirement already satisfied, skipping upgrade: sentencepiece!=0.1.92 in /home/yufeng/.local/lib/python3.6/site-packages (from transformers) (0.1.85)\n",
|
||||
"Requirement already satisfied, skipping upgrade: sacremoses in /home/yufeng/.local/lib/python3.6/site-packages (from transformers) (0.0.38)\n",
|
||||
"Requirement already satisfied, skipping upgrade: regex!=2019.12.17 in /home/yufeng/.local/lib/python3.6/site-packages (from transformers) (2020.4.4)\n",
|
||||
"Requirement already satisfied, skipping upgrade: requests in /home/yufeng/.local/lib/python3.6/site-packages (from transformers) (2.23.0)\n",
|
||||
"Requirement already satisfied, skipping upgrade: numpy in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (from transformers) (1.18.1)\n",
|
||||
"Requirement already satisfied, skipping upgrade: packaging in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from transformers) (19.1)\n",
|
||||
"Requirement already satisfied, skipping upgrade: filelock in /home/yufeng/.local/lib/python3.6/site-packages (from transformers) (3.0.12)\n",
|
||||
"Requirement already satisfied, skipping upgrade: six in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from sacremoses->transformers) (1.12.0)\n",
|
||||
"Requirement already satisfied, skipping upgrade: joblib in /home/yufeng/.local/lib/python3.6/site-packages (from sacremoses->transformers) (0.14.1)\n",
|
||||
"Requirement already satisfied, skipping upgrade: click in /home/yufeng/.local/lib/python3.6/site-packages (from sacremoses->transformers) (7.1.1)\n",
|
||||
"Requirement already satisfied, skipping upgrade: chardet<4,>=3.0.2 in /home/yufeng/.local/lib/python3.6/site-packages (from requests->transformers) (3.0.4)\n",
|
||||
"Requirement already satisfied, skipping upgrade: idna<3,>=2.5 in /home/yufeng/.local/lib/python3.6/site-packages (from requests->transformers) (2.9)\n",
|
||||
"Requirement already satisfied, skipping upgrade: certifi>=2017.4.17 in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from requests->transformers) (2019.6.16)\n",
|
||||
"Requirement already satisfied, skipping upgrade: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /home/yufeng/.local/lib/python3.6/site-packages (from requests->transformers) (1.25.8)\n",
|
||||
"Requirement already satisfied, skipping upgrade: attrs in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from packaging->transformers) (19.1.0)\n",
|
||||
"Requirement already satisfied, skipping upgrade: pyparsing>=2.0.2 in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from packaging->transformers) (2.4.2)\n",
|
||||
"Requirement already satisfied: onnx in /home/yufeng/project/onnx (1.6.0)\n",
|
||||
"Requirement already satisfied: sklearn in /home/yufeng/.local/lib/python3.6/site-packages (0.0)\n",
|
||||
"Requirement already satisfied: protobuf in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from onnx) (3.9.1)\n",
|
||||
"Requirement already satisfied: numpy in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (from onnx) (1.18.1)\n",
|
||||
"Requirement already satisfied: six in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from onnx) (1.12.0)\n",
|
||||
"Requirement already satisfied: typing-extensions>=3.6.2.1 in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages/typing_extensions-3.7.4-py3.6.egg (from onnx) (3.7.4)\n",
|
||||
"Requirement already satisfied: scikit-learn in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (from sklearn) (0.23.1)\n",
|
||||
"Requirement already satisfied: setuptools in /home/yufeng/.local/lib/python3.6/site-packages (from protobuf->onnx) (41.0.1)\n",
|
||||
"Requirement already satisfied: joblib>=0.11 in /home/yufeng/.local/lib/python3.6/site-packages (from scikit-learn->sklearn) (0.14.1)\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Requirement already satisfied: threadpoolctl>=2.0.0 in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (from scikit-learn->sklearn) (2.1.0)\r\n",
|
||||
"Requirement already satisfied: scipy>=0.19.1 in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (from scikit-learn->sklearn) (1.5.1)\r\n"
|
||||
"Requirement already up-to-date: install in /home/yufeng/anaconda3/lib/python3.8/site-packages (1.3.4)\n",
|
||||
"Requirement already up-to-date: torch==1.8.1+cpu in /home/yufeng/anaconda3/lib/python3.8/site-packages (1.8.1+cpu)\n",
|
||||
"Requirement already up-to-date: torchvision==0.9.1+cpu in /home/yufeng/anaconda3/lib/python3.8/site-packages (0.9.1+cpu)\n",
|
||||
"Requirement already up-to-date: torchaudio===0.8.1 in /home/yufeng/anaconda3/lib/python3.8/site-packages (0.8.1)\n",
|
||||
"Requirement already satisfied, skipping upgrade: numpy in /home/yufeng/anaconda3/lib/python3.8/site-packages (from torch==1.8.1+cpu) (1.19.2)\n",
|
||||
"Requirement already satisfied, skipping upgrade: typing-extensions in /home/yufeng/anaconda3/lib/python3.8/site-packages (from torch==1.8.1+cpu) (3.7.4.3)\n",
|
||||
"Requirement already satisfied, skipping upgrade: pillow>=4.1.1 in /home/yufeng/anaconda3/lib/python3.8/site-packages (from torchvision==0.9.1+cpu) (8.0.1)\n",
|
||||
"Requirement already up-to-date: onnxruntime==1.7.0 in /home/yufeng/anaconda3/lib/python3.8/site-packages (1.7.0)\n",
|
||||
"Requirement already satisfied, skipping upgrade: numpy>=1.16.6 in /home/yufeng/anaconda3/lib/python3.8/site-packages (from onnxruntime==1.7.0) (1.19.2)\n",
|
||||
"Requirement already satisfied, skipping upgrade: protobuf in /home/yufeng/anaconda3/lib/python3.8/site-packages (from onnxruntime==1.7.0) (3.15.6)\n",
|
||||
"Requirement already satisfied, skipping upgrade: six>=1.9 in /home/yufeng/anaconda3/lib/python3.8/site-packages (from protobuf->onnxruntime==1.7.0) (1.15.0)\n",
|
||||
"Requirement already up-to-date: onnxruntime-tools in /home/yufeng/anaconda3/lib/python3.8/site-packages (1.6.0)\n",
|
||||
"Requirement already satisfied, skipping upgrade: psutil in /home/yufeng/anaconda3/lib/python3.8/site-packages (from onnxruntime-tools) (5.7.2)\n",
|
||||
"Requirement already satisfied, skipping upgrade: numpy in /home/yufeng/anaconda3/lib/python3.8/site-packages (from onnxruntime-tools) (1.19.2)\n",
|
||||
"Requirement already satisfied, skipping upgrade: packaging in /home/yufeng/anaconda3/lib/python3.8/site-packages (from onnxruntime-tools) (20.4)\n",
|
||||
"Requirement already satisfied, skipping upgrade: coloredlogs in /home/yufeng/anaconda3/lib/python3.8/site-packages (from onnxruntime-tools) (15.0)\n",
|
||||
"Requirement already satisfied, skipping upgrade: onnx in /home/yufeng/anaconda3/lib/python3.8/site-packages (from onnxruntime-tools) (1.8.1)\n",
|
||||
"Requirement already satisfied, skipping upgrade: py-cpuinfo in /home/yufeng/anaconda3/lib/python3.8/site-packages (from onnxruntime-tools) (7.0.0)\n",
|
||||
"Requirement already satisfied, skipping upgrade: py3nvml in /home/yufeng/anaconda3/lib/python3.8/site-packages (from onnxruntime-tools) (0.2.6)\n",
|
||||
"Requirement already satisfied, skipping upgrade: pyparsing>=2.0.2 in /home/yufeng/anaconda3/lib/python3.8/site-packages (from packaging->onnxruntime-tools) (2.4.7)\n",
|
||||
"Requirement already satisfied, skipping upgrade: six in /home/yufeng/anaconda3/lib/python3.8/site-packages (from packaging->onnxruntime-tools) (1.15.0)\n",
|
||||
"Requirement already satisfied, skipping upgrade: humanfriendly>=9.1 in /home/yufeng/anaconda3/lib/python3.8/site-packages (from coloredlogs->onnxruntime-tools) (9.1)\n",
|
||||
"Requirement already satisfied, skipping upgrade: typing-extensions>=3.6.2.1 in /home/yufeng/anaconda3/lib/python3.8/site-packages (from onnx->onnxruntime-tools) (3.7.4.3)\n",
|
||||
"Requirement already satisfied, skipping upgrade: protobuf in /home/yufeng/anaconda3/lib/python3.8/site-packages (from onnx->onnxruntime-tools) (3.15.6)\n",
|
||||
"Requirement already satisfied, skipping upgrade: xmltodict in /home/yufeng/anaconda3/lib/python3.8/site-packages (from py3nvml->onnxruntime-tools) (0.12.0)\n",
|
||||
"Requirement already up-to-date: transformers in /home/yufeng/anaconda3/lib/python3.8/site-packages (4.4.2)\n",
|
||||
"Requirement already satisfied, skipping upgrade: tqdm>=4.27 in /home/yufeng/anaconda3/lib/python3.8/site-packages (from transformers) (4.50.2)\n",
|
||||
"Requirement already satisfied, skipping upgrade: tokenizers<0.11,>=0.10.1 in /home/yufeng/anaconda3/lib/python3.8/site-packages (from transformers) (0.10.1)\n",
|
||||
"Requirement already satisfied, skipping upgrade: regex!=2019.12.17 in /home/yufeng/anaconda3/lib/python3.8/site-packages (from transformers) (2020.10.15)\n",
|
||||
"Requirement already satisfied, skipping upgrade: numpy>=1.17 in /home/yufeng/anaconda3/lib/python3.8/site-packages (from transformers) (1.19.2)\n",
|
||||
"Requirement already satisfied, skipping upgrade: packaging in /home/yufeng/anaconda3/lib/python3.8/site-packages (from transformers) (20.4)\n",
|
||||
"Requirement already satisfied, skipping upgrade: filelock in /home/yufeng/anaconda3/lib/python3.8/site-packages (from transformers) (3.0.12)\n",
|
||||
"Requirement already satisfied, skipping upgrade: sacremoses in /home/yufeng/anaconda3/lib/python3.8/site-packages (from transformers) (0.0.43)\n",
|
||||
"Requirement already satisfied, skipping upgrade: requests in /home/yufeng/anaconda3/lib/python3.8/site-packages (from transformers) (2.24.0)\n",
|
||||
"Requirement already satisfied, skipping upgrade: pyparsing>=2.0.2 in /home/yufeng/anaconda3/lib/python3.8/site-packages (from packaging->transformers) (2.4.7)\n",
|
||||
"Requirement already satisfied, skipping upgrade: six in /home/yufeng/anaconda3/lib/python3.8/site-packages (from packaging->transformers) (1.15.0)\n",
|
||||
"Requirement already satisfied, skipping upgrade: click in /home/yufeng/anaconda3/lib/python3.8/site-packages (from sacremoses->transformers) (7.1.2)\n",
|
||||
"Requirement already satisfied, skipping upgrade: joblib in /home/yufeng/anaconda3/lib/python3.8/site-packages (from sacremoses->transformers) (0.17.0)\n",
|
||||
"Requirement already satisfied, skipping upgrade: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /home/yufeng/anaconda3/lib/python3.8/site-packages (from requests->transformers) (1.25.11)\n",
|
||||
"Requirement already satisfied, skipping upgrade: chardet<4,>=3.0.2 in /home/yufeng/anaconda3/lib/python3.8/site-packages (from requests->transformers) (3.0.4)\n",
|
||||
"Requirement already satisfied, skipping upgrade: idna<3,>=2.5 in /home/yufeng/anaconda3/lib/python3.8/site-packages (from requests->transformers) (2.10)\n",
|
||||
"Requirement already satisfied, skipping upgrade: certifi>=2017.4.17 in /home/yufeng/anaconda3/lib/python3.8/site-packages (from requests->transformers) (2020.6.20)\n",
|
||||
"Requirement already up-to-date: onnx in /home/yufeng/anaconda3/lib/python3.8/site-packages (1.8.1)\n",
|
||||
"Requirement already up-to-date: sklearn in /home/yufeng/anaconda3/lib/python3.8/site-packages (0.0)\n",
|
||||
"Requirement already satisfied, skipping upgrade: numpy>=1.16.6 in /home/yufeng/anaconda3/lib/python3.8/site-packages (from onnx) (1.19.2)\n",
|
||||
"Requirement already satisfied, skipping upgrade: six in /home/yufeng/anaconda3/lib/python3.8/site-packages (from onnx) (1.15.0)\n",
|
||||
"Requirement already satisfied, skipping upgrade: protobuf in /home/yufeng/anaconda3/lib/python3.8/site-packages (from onnx) (3.15.6)\n",
|
||||
"Requirement already satisfied, skipping upgrade: typing-extensions>=3.6.2.1 in /home/yufeng/anaconda3/lib/python3.8/site-packages (from onnx) (3.7.4.3)\n",
|
||||
"Requirement already satisfied, skipping upgrade: scikit-learn in /home/yufeng/anaconda3/lib/python3.8/site-packages (from sklearn) (0.23.2)\n",
|
||||
"Requirement already satisfied, skipping upgrade: joblib>=0.11 in /home/yufeng/anaconda3/lib/python3.8/site-packages (from scikit-learn->sklearn) (0.17.0)\n",
|
||||
"Requirement already satisfied, skipping upgrade: threadpoolctl>=2.0.0 in /home/yufeng/anaconda3/lib/python3.8/site-packages (from scikit-learn->sklearn) (2.1.0)\n",
|
||||
"Requirement already satisfied, skipping upgrade: scipy>=0.19.1 in /home/yufeng/anaconda3/lib/python3.8/site-packages (from scikit-learn->sklearn) (1.5.2)\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Install or upgrade PyTorch 1.6.0 and OnnxRuntime 1.5.1 for CPU-only.\n",
|
||||
"# Install or upgrade PyTorch 1.8.0 and OnnxRuntime 1.7.0 for CPU-only.\n",
|
||||
"import sys\n",
|
||||
"!{sys.executable} -m pip install --upgrade torch==1.6.0+cpu torchvision==0.7.0+cpu -f https://download.pytorch.org/whl/torch_stable.html\n",
|
||||
"!{sys.executable} -m pip install --upgrade onnxruntime==1.5.1\n",
|
||||
"!{sys.executable} -m pip install --upgrade install torch==1.8.1+cpu torchvision==0.9.1+cpu torchaudio===0.8.1 -f https://download.pytorch.org/whl/torch_stable.html\n",
|
||||
"!{sys.executable} -m pip install --upgrade onnxruntime==1.7.0\n",
|
||||
"!{sys.executable} -m pip install --upgrade onnxruntime-tools\n",
|
||||
"\n",
|
||||
"# Install other packages used in this notebook.\n",
|
||||
"!{sys.executable} -m pip install --upgrade transformers\n",
|
||||
"!{sys.executable} -m pip install onnx sklearn"
|
||||
"!{sys.executable} -m pip install --upgrade onnx sklearn"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -172,26 +154,26 @@
|
|||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"--2020-10-06 22:44:47-- https://raw.githubusercontent.com/huggingface/transformers/f98ef14d161d7bcdc9808b5ec399981481411cc1/utils/download_glue_data.py\n",
|
||||
"Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.188.133\n",
|
||||
"Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.188.133|:443... connected.\n",
|
||||
"--2021-03-25 20:48:36-- https://raw.githubusercontent.com/huggingface/transformers/f98ef14d161d7bcdc9808b5ec399981481411cc1/utils/download_glue_data.py\n",
|
||||
"Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.109.133, 185.199.108.133, 185.199.111.133, ...\n",
|
||||
"Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.109.133|:443... connected.\n",
|
||||
"HTTP request sent, awaiting response... 200 OK\n",
|
||||
"Length: 8209 (8.0K) [text/plain]\n",
|
||||
"Saving to: ‘download_glue_data.py.2’\n",
|
||||
"\n",
|
||||
"download_glue_data. 100%[===================>] 8.02K --.-KB/s in 0s \n",
|
||||
"\n",
|
||||
"2020-10-06 22:44:47 (26.9 MB/s) - ‘download_glue_data.py.2’ saved [8209/8209]\n",
|
||||
"2021-03-25 20:48:36 (79.6 MB/s) - ‘download_glue_data.py.2’ saved [8209/8209]\n",
|
||||
"\n",
|
||||
"Processing MRPC...\n",
|
||||
"Local MRPC data not specified, downloading data from https://dl.fbaipublicfiles.com/senteval/senteval_data/msr_paraphrase_train.txt\n",
|
||||
"\tCompleted!\n",
|
||||
"cached_dev_bert-base-uncased_128_mrpc msr_paraphrase_test.txt\t train.tsv\n",
|
||||
"dev_ids.tsv\t\t\t msr_paraphrase_train.txt\n",
|
||||
"dev.tsv\t\t\t\t test.tsv\n",
|
||||
"dev.tsv\t\t\t\t msr_paraphrase_train.txt\n",
|
||||
"dev_ids.tsv\t\t\t test.tsv\n",
|
||||
" % Total % Received % Xferd Average Speed Time Time Time Current\n",
|
||||
" Dload Upload Total Spent Left Speed\n",
|
||||
"100 386M 100 386M 0 0 32.8M 0 0:00:11 0:00:11 --:--:-- 35.3M\n",
|
||||
"100 386M 100 386M 0 0 224M 0 0:00:01 0:00:01 --:--:-- 224M\n",
|
||||
"Archive: MPRC.zip\n"
|
||||
]
|
||||
}
|
||||
|
|
@ -245,7 +227,7 @@
|
|||
"text": [
|
||||
" % Total % Received % Xferd Average Speed Time Time Time Current\n",
|
||||
" Dload Upload Total Spent Left Speed\n",
|
||||
"100 386M 100 386M 0 0 77.7M 0 0:00:04 0:00:04 --:--:-- 83.1M\n",
|
||||
"100 386M 100 386M 0 0 190M 0 0:00:02 0:00:02 --:--:-- 190M\n",
|
||||
"Archive: MPRC.zip\n"
|
||||
]
|
||||
}
|
||||
|
|
@ -287,7 +269,15 @@
|
|||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"1.6.0+cpu\n"
|
||||
"1.8.1+cpu\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/home/yufeng/anaconda3/lib/python3.8/site-packages/transformers/data/processors/glue.py:175: FutureWarning: This processor will be removed from the library soon, preprocessing should be handled with the 🤗 Datasets library. You can have a look at this example script for pointers: https://github.com/huggingface/transformers/blob/master/examples/text-classification/run_glue.py\n",
|
||||
" warnings.warn(DEPRECATION_WARNING.format(\"processor\"), FutureWarning)\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
|
@ -434,8 +424,7 @@
|
|||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"\r",
|
||||
"Evaluating: 0%| | 0/408 [00:00<?, ?it/s]"
|
||||
"Evaluating: 0%| | 1/408 [00:00<00:49, 8.28it/s]"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -449,8 +438,14 @@
|
|||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Evaluating: 100%|██████████| 408/408 [00:33<00:00, 12.12it/s]\n",
|
||||
"Evaluating: 0%| | 0/408 [00:00<?, ?it/s]"
|
||||
"Evaluating: 100%|██████████| 408/408 [00:52<00:00, 7.85it/s]\n",
|
||||
"/home/yufeng/anaconda3/lib/python3.8/site-packages/transformers/data/metrics/__init__.py:66: FutureWarning: This metric will be removed from the library soon, metrics should be handled with the 🤗 Datasets library. You can have a look at this example script for pointers: https://github.com/huggingface/transformers/blob/master/examples/text-classification/run_glue.py\n",
|
||||
" warnings.warn(DEPRECATION_WARNING, FutureWarning)\n",
|
||||
"/home/yufeng/anaconda3/lib/python3.8/site-packages/transformers/data/metrics/__init__.py:42: FutureWarning: This metric will be removed from the library soon, metrics should be handled with the 🤗 Datasets library. You can have a look at this example script for pointers: https://github.com/huggingface/transformers/blob/master/examples/text-classification/run_glue.py\n",
|
||||
" warnings.warn(DEPRECATION_WARNING, FutureWarning)\n",
|
||||
"/home/yufeng/anaconda3/lib/python3.8/site-packages/transformers/data/metrics/__init__.py:36: FutureWarning: This metric will be removed from the library soon, metrics should be handled with the 🤗 Datasets library. You can have a look at this example script for pointers: https://github.com/huggingface/transformers/blob/master/examples/text-classification/run_glue.py\n",
|
||||
" warnings.warn(DEPRECATION_WARNING, FutureWarning)\n",
|
||||
"Evaluating: 0%| | 2/408 [00:00<00:28, 14.33it/s]"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -458,7 +453,7 @@
|
|||
"output_type": "stream",
|
||||
"text": [
|
||||
"{'acc': 0.8602941176470589, 'f1': 0.9018932874354562, 'acc_and_f1': 0.8810937025412575}\n",
|
||||
"Evaluate total time (seconds): 33.8\n",
|
||||
"Evaluate total time (seconds): 52.0\n",
|
||||
"Evaluating PyTorch quantization accuracy and performance:\n"
|
||||
]
|
||||
},
|
||||
|
|
@ -466,15 +461,15 @@
|
|||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Evaluating: 100%|██████████| 408/408 [00:25<00:00, 15.74it/s]"
|
||||
"Evaluating: 100%|██████████| 408/408 [00:26<00:00, 15.61it/s]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"{'acc': 0.8578431372549019, 'f1': 0.8993055555555555, 'acc_and_f1': 0.8785743464052287}\n",
|
||||
"Evaluate total time (seconds): 26.0\n"
|
||||
"{'acc': 0.8504901960784313, 'f1': 0.8942807625649914, 'acc_and_f1': 0.8723854793217114}\n",
|
||||
"Evaluate total time (seconds): 26.2\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -669,10 +664,8 @@
|
|||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"/home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages/transformers/modeling_bert.py:191: TracerWarning: Converting a tensor to a Python index might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n",
|
||||
" position_ids = self.position_ids[:, :seq_length]\n",
|
||||
"/home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages/transformers/modeling_utils.py:1645: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n",
|
||||
" input_tensor.shape == tensor_shape for input_tensor in input_tensors\n"
|
||||
"/home/yufeng/anaconda3/lib/python3.8/site-packages/transformers/modeling_utils.py:1791: TracerWarning: Converting a tensor to a Python boolean might cause the trace to be incorrect. We can't record the data flow of Python values, so this value will be treated as a constant in the future. This means that the trace might not generalize to other inputs!\n",
|
||||
" assert all(\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
|
@ -741,8 +734,8 @@
|
|||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"ONNX full precision model size (MB): 417.6690320968628\n",
|
||||
"ONNX quantized model size (MB): 106.49767780303955\n"
|
||||
"ONNX full precision model size (MB): 417.66954708099365\n",
|
||||
"ONNX quantized model size (MB): 104.80786514282227\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
|
@ -774,14 +767,15 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 11,
|
||||
"execution_count": 9,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Evaluating: 0%| | 2/408 [00:00<00:33, 11.96it/s]"
|
||||
"\r",
|
||||
"Evaluating: 0%| | 0/408 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -795,8 +789,8 @@
|
|||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Evaluating: 100%|██████████| 408/408 [00:21<00:00, 18.74it/s]\n",
|
||||
"Evaluating: 1%| | 3/408 [00:00<00:19, 20.46it/s]"
|
||||
"Evaluating: 100%|██████████| 408/408 [00:40<00:00, 10.04it/s]\n",
|
||||
"Evaluating: 0%| | 0/408 [00:00<?, ?it/s]"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -804,7 +798,7 @@
|
|||
"output_type": "stream",
|
||||
"text": [
|
||||
"{'acc': 0.8602941176470589, 'f1': 0.9018932874354562, 'acc_and_f1': 0.8810937025412575}\n",
|
||||
"Evaluate total time (seconds): 22.5\n",
|
||||
"Evaluate total time (seconds): 41.0\n",
|
||||
"Evaluating ONNXRuntime quantization accuracy and performance:\n"
|
||||
]
|
||||
},
|
||||
|
|
@ -812,15 +806,15 @@
|
|||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Evaluating: 100%|██████████| 408/408 [00:19<00:00, 21.07it/s]"
|
||||
"Evaluating: 100%|██████████| 408/408 [00:18<00:00, 21.84it/s]"
|
||||
]
|
||||
},
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"{'acc': 0.8578431372549019, 'f1': 0.902027027027027, 'acc_and_f1': 0.8799350821409644}\n",
|
||||
"Evaluate total time (seconds): 19.5\n"
|
||||
"{'acc': 0.8529411764705882, 'f1': 0.8986486486486487, 'acc_and_f1': 0.8757949125596185}\n",
|
||||
"Evaluate total time (seconds): 18.8\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -836,7 +830,6 @@
|
|||
"\n",
|
||||
" sess_options = onnxruntime.SessionOptions()\n",
|
||||
" sess_options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_ALL\n",
|
||||
" sess_options.intra_op_num_threads=1\n",
|
||||
" session = onnxruntime.InferenceSession(model_path, sess_options)\n",
|
||||
"\n",
|
||||
" # Loop to handle MNLI double evaluation (matched, mis-matched)\n",
|
||||
|
|
@ -926,39 +919,39 @@
|
|||
"In this tutorial, we demonstrated how to quantize a fine-tuned BERT model for MPRC task on GLUE data set. Let's summarize the main metrics of quantization.\n",
|
||||
"\n",
|
||||
"### Model Size\n",
|
||||
"PyTorch quantizes torch.nn.Linear modules only and reduce the model from 438 MB to 181 MB. OnnxRuntime quantizes not only Linear(MatMul), but also the embedding layer. It achieves almost the ideal model size reduction with quantization.\n",
|
||||
"PyTorch quantizes torch.nn.Linear modules only and reduce the model from 438 MB to 173 MB. OnnxRuntime quantizes not only Linear(MatMul), but also the embedding layer. It achieves almost the ideal model size reduction with quantization.\n",
|
||||
"\n",
|
||||
"| Engine | Full Precision(MB) | Quantized(MB) |\n",
|
||||
"| --- | --- | --- |\n",
|
||||
"| PyTorch 1.6 | 417.7 | 173.1 |\n",
|
||||
"| ORT 1.5 | 417.7 | 106.5 |\n",
|
||||
"| PyTorch 1.8 | 417.7 | 173.1 |\n",
|
||||
"| ORT 1.7 | 417.7 | 104.5 |\n",
|
||||
"\n",
|
||||
"### Accuracy\n",
|
||||
"OnnxRuntime achieves a little bit better accuracy and F1 score, even though it has small model size.\n",
|
||||
"\n",
|
||||
"| Metrics | Full Precision | PyTorch 1.6 Quantization | ORT 1.5 Quantization |\n",
|
||||
"| Metrics | Full Precision | PyTorch 1.8 Quantization | ORT 1.7 Quantization |\n",
|
||||
"| --- | --- | --- | --- |\n",
|
||||
"| Accuracy | 0.86029 | 0.85784 | 0.85784 |\n",
|
||||
"| F1 score | 0.90189 | 0.89931 | 0.90203 |\n",
|
||||
"| Acc and F1 | 0.88109 | 0.87857 | 0.87994 |\n",
|
||||
"| Accuracy | 0.86029 | 0.85049 | 0.85294 |\n",
|
||||
"| F1 score | 0.90189 | 0.89428 | 0.89865 |\n",
|
||||
"| Acc and F1 | 0.88109 | 0.87239 | 0.87579 |\n",
|
||||
"\n",
|
||||
"### Performance\n",
|
||||
"\n",
|
||||
"The evaluation data set has 408 sample. Table below shows the performance on my machine with Intel(R) Xeon(R) E5-1650 v4@3.60GHz CPU. Comparing with PyTorch full precision, PyTorch quantization achieves ~1.33x speedup, and ORT quantization achieves ~1.73x speedup. And ORT quantization can achieve ~1.33x speedup, comparing with PyTorch quantization. \n",
|
||||
"The evaluation data set has 408 sample. Table below shows the performance on **Azure VM: Standard E4ds_v4 (4 vcpus, 32 GiB memory)**. Comparing with PyTorch full precision, PyTorch quantization achieves ~2x speedup, and ORT quantization achieves ~1.73x speedup. And ORT quantization can achieve ~2.77x speedup, comparing with PyTorch quantization. \n",
|
||||
"You can run the [benchmark.py](https://github.com/microsoft/onnxruntime/blob/master/onnxruntime/python/tools/transformers/benchmark.py) for comparison on more models.\n",
|
||||
"\n",
|
||||
"|Engine | Full Precision Latency(s) | Quantized(s) |\n",
|
||||
"| --- | --- | --- |\n",
|
||||
"| PyTorch 1.6 | 33.8 | 22.5 |\n",
|
||||
"| ORT 1.5 | 26.0 | 19.5 |"
|
||||
"| PyTorch 1.8 | 52.0 | 26.2 |\n",
|
||||
"| ORT 1.7 | 41.0 | 18.8 |"
|
||||
]
|
||||
}
|
||||
],
|
||||
"metadata": {
|
||||
"kernelspec": {
|
||||
"display_name": "Python 3.6.10 64-bit ('pytorch': virtualenv)",
|
||||
"display_name": "base",
|
||||
"language": "python",
|
||||
"name": "python361064bitpytorchvirtualenv3d87422a0f2a46ceb32788376e2c7e04"
|
||||
"name": "base"
|
||||
},
|
||||
"language_info": {
|
||||
"codemirror_mode": {
|
||||
|
|
@ -970,7 +963,7 @@
|
|||
"name": "python",
|
||||
"nbconvert_exporter": "python",
|
||||
"pygments_lexer": "ipython3",
|
||||
"version": "3.6.10"
|
||||
"version": "3.8.5"
|
||||
}
|
||||
},
|
||||
"nbformat": 4,
|
||||
|
|
|
|||
Loading…
Reference in a new issue