diff --git a/onnxruntime/python/tools/quantization/notebooks/Bert-GLUE_OnnxRuntime_quantization.ipynb b/onnxruntime/python/tools/quantization/notebooks/Bert-GLUE_OnnxRuntime_quantization.ipynb index e5a5c343eb..c4e89e4d93 100644 --- a/onnxruntime/python/tools/quantization/notebooks/Bert-GLUE_OnnxRuntime_quantization.ipynb +++ b/onnxruntime/python/tools/quantization/notebooks/Bert-GLUE_OnnxRuntime_quantization.ipynb @@ -46,12 +46,12 @@ "metadata": {}, "source": [ "### 0.1 Install packages\n", - "Let's install nessasary packages to start the tutorial. We will install PyTorch 1.5, OnnxRuntime 1.4.0, latest ONNX, OnnxRuntime-tools, transformers, and sklearn." + "Let's install nessasary packages to start the tutorial. We will install PyTorch 1.6, OnnxRuntime 1.4.0, latest ONNX, OnnxRuntime-tools, transformers, and sklearn." ] }, { "cell_type": "code", - "execution_count": 1, + "execution_count": 15, "metadata": { "scrolled": true }, @@ -61,14 +61,14 @@ "output_type": "stream", "text": [ "Looking in links: https://download.pytorch.org/whl/torch_stable.html\n", - "Requirement already up-to-date: torch==1.5.0+cpu in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (1.5.0+cpu)\n", - "Requirement already up-to-date: torchvision==0.6.0+cpu in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (0.6.0+cpu)\n", - "Requirement already satisfied, skipping upgrade: numpy in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (from torch==1.5.0+cpu) (1.18.1)\n", - "Requirement already satisfied, skipping upgrade: future in /home/yufeng/.local/lib/python3.6/site-packages (from torch==1.5.0+cpu) (0.18.2)\n", - "Requirement already satisfied, skipping upgrade: pillow>=4.1.1 in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from torchvision==0.6.0+cpu) (7.0.0)\n", + "Requirement already up-to-date: torch==1.6.0+cpu in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (1.6.0+cpu)\n", + "Requirement already up-to-date: torchvision==0.7.0+cpu in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (0.7.0+cpu)\n", + "Requirement already satisfied, skipping upgrade: future in /home/yufeng/.local/lib/python3.6/site-packages (from torch==1.6.0+cpu) (0.18.2)\n", + "Requirement already satisfied, skipping upgrade: numpy in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (from torch==1.6.0+cpu) (1.18.1)\n", + "Requirement already satisfied, skipping upgrade: pillow>=4.1.1 in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from torchvision==0.7.0+cpu) (7.0.0)\n", "Requirement already up-to-date: onnxruntime==1.4.0 in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (1.4.0)\n", - "Requirement already satisfied, skipping upgrade: numpy>=1.16.6 in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (from onnxruntime==1.4.0) (1.18.1)\n", "Requirement already satisfied, skipping upgrade: protobuf in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from onnxruntime==1.4.0) (3.9.1)\n", + "Requirement already satisfied, skipping upgrade: numpy>=1.16.6 in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (from onnxruntime==1.4.0) (1.18.1)\n", "Requirement already satisfied, skipping upgrade: six>=1.9 in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from protobuf->onnxruntime==1.4.0) (1.12.0)\n", "Requirement already satisfied, skipping upgrade: setuptools in /home/yufeng/.local/lib/python3.6/site-packages (from protobuf->onnxruntime==1.4.0) (41.0.1)\n", "Looking in indexes: https://test.pypi.org/simple/\n", @@ -77,36 +77,36 @@ "Requirement already satisfied: protobuf in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from ort-nightly) (3.9.1)\n", "Requirement already satisfied: six>=1.9 in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from protobuf->ort-nightly) (1.12.0)\n", "Requirement already satisfied: setuptools in /home/yufeng/.local/lib/python3.6/site-packages (from protobuf->ort-nightly) (41.0.1)\n", - "Requirement already up-to-date: onnxruntime-tools in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (1.4.0)\n", - "Requirement already satisfied, skipping upgrade: coloredlogs in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (from onnxruntime-tools) (14.0)\n", + "Requirement already up-to-date: onnxruntime-tools in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (1.4.2)\n", "Requirement already satisfied, skipping upgrade: psutil in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from onnxruntime-tools) (5.6.3)\n", - "Requirement already satisfied, skipping upgrade: numpy in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (from onnxruntime-tools) (1.18.1)\n", - "Requirement already satisfied, skipping upgrade: packaging in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from onnxruntime-tools) (19.1)\n", "Requirement already satisfied, skipping upgrade: py-cpuinfo in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (from onnxruntime-tools) (5.0.0)\n", "Requirement already satisfied, skipping upgrade: py3nvml in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (from onnxruntime-tools) (0.2.6)\n", + "Requirement already satisfied, skipping upgrade: packaging in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from onnxruntime-tools) (19.1)\n", + "Requirement already satisfied, skipping upgrade: coloredlogs in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (from onnxruntime-tools) (14.0)\n", "Requirement already satisfied, skipping upgrade: onnx in /home/yufeng/project/onnx (from onnxruntime-tools) (1.6.0)\n", - "Requirement already satisfied, skipping upgrade: humanfriendly>=7.1 in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (from coloredlogs->onnxruntime-tools) (8.2)\n", - "Requirement already satisfied, skipping upgrade: six in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from packaging->onnxruntime-tools) (1.12.0)\n", - "Requirement already satisfied, skipping upgrade: attrs in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from packaging->onnxruntime-tools) (19.1.0)\n", - "Requirement already satisfied, skipping upgrade: pyparsing>=2.0.2 in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from packaging->onnxruntime-tools) (2.4.2)\n", + "Requirement already satisfied, skipping upgrade: numpy in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (from onnxruntime-tools) (1.18.1)\n", "Requirement already satisfied, skipping upgrade: xmltodict in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (from py3nvml->onnxruntime-tools) (0.12.0)\n", + "Requirement already satisfied, skipping upgrade: attrs in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from packaging->onnxruntime-tools) (19.1.0)\n", + "Requirement already satisfied, skipping upgrade: six in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from packaging->onnxruntime-tools) (1.12.0)\n", + "Requirement already satisfied, skipping upgrade: pyparsing>=2.0.2 in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from packaging->onnxruntime-tools) (2.4.2)\n", + "Requirement already satisfied, skipping upgrade: humanfriendly>=7.1 in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (from coloredlogs->onnxruntime-tools) (8.2)\n", "Requirement already satisfied, skipping upgrade: protobuf in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from onnx->onnxruntime-tools) (3.9.1)\n", "Requirement already satisfied, skipping upgrade: typing-extensions>=3.6.2.1 in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages/typing_extensions-3.7.4-py3.6.egg (from onnx->onnxruntime-tools) (3.7.4)\n", "Requirement already satisfied, skipping upgrade: setuptools in /home/yufeng/.local/lib/python3.6/site-packages (from protobuf->onnx->onnxruntime-tools) (41.0.1)\n", "Requirement already up-to-date: transformers in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (3.0.2)\n", - "Requirement already satisfied, skipping upgrade: dataclasses; python_version < \"3.7\" in /home/yufeng/.local/lib/python3.6/site-packages (from transformers) (0.7)\n", - "Requirement already satisfied, skipping upgrade: packaging in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from transformers) (19.1)\n", "Requirement already satisfied, skipping upgrade: regex!=2019.12.17 in /home/yufeng/.local/lib/python3.6/site-packages (from transformers) (2020.4.4)\n", "Requirement already satisfied, skipping upgrade: sentencepiece!=0.1.92 in /home/yufeng/.local/lib/python3.6/site-packages (from transformers) (0.1.85)\n", - "Requirement already satisfied, skipping upgrade: requests in /home/yufeng/.local/lib/python3.6/site-packages (from transformers) (2.23.0)\n", - "Requirement already satisfied, skipping upgrade: tokenizers==0.8.1.rc1 in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (from transformers) (0.8.1rc1)\n", - "Requirement already satisfied, skipping upgrade: filelock in /home/yufeng/.local/lib/python3.6/site-packages (from transformers) (3.0.12)\n", - "Requirement already satisfied, skipping upgrade: sacremoses in /home/yufeng/.local/lib/python3.6/site-packages (from transformers) (0.0.38)\n", - "Requirement already satisfied, skipping upgrade: tqdm>=4.27 in /home/yufeng/.local/lib/python3.6/site-packages (from transformers) (4.45.0)\n", "Requirement already satisfied, skipping upgrade: numpy in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (from transformers) (1.18.1)\n", + "Requirement already satisfied, skipping upgrade: tokenizers==0.8.1.rc1 in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (from transformers) (0.8.1rc1)\n", + "Requirement already satisfied, skipping upgrade: dataclasses; python_version < \"3.7\" in /home/yufeng/.local/lib/python3.6/site-packages (from transformers) (0.7)\n", + "Requirement already satisfied, skipping upgrade: packaging in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from transformers) (19.1)\n", + "Requirement already satisfied, skipping upgrade: tqdm>=4.27 in /home/yufeng/.local/lib/python3.6/site-packages (from transformers) (4.45.0)\n", + "Requirement already satisfied, skipping upgrade: requests in /home/yufeng/.local/lib/python3.6/site-packages (from transformers) (2.23.0)\n", + "Requirement already satisfied, skipping upgrade: sacremoses in /home/yufeng/.local/lib/python3.6/site-packages (from transformers) (0.0.38)\n", + "Requirement already satisfied, skipping upgrade: filelock in /home/yufeng/.local/lib/python3.6/site-packages (from transformers) (3.0.12)\n", "Requirement already satisfied, skipping upgrade: six in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from packaging->transformers) (1.12.0)\n", - "Requirement already satisfied, skipping upgrade: pyparsing>=2.0.2 in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from packaging->transformers) (2.4.2)\n", "Requirement already satisfied, skipping upgrade: attrs in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from packaging->transformers) (19.1.0)\n", + "Requirement already satisfied, skipping upgrade: pyparsing>=2.0.2 in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from packaging->transformers) (2.4.2)\n", "Requirement already satisfied, skipping upgrade: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /home/yufeng/.local/lib/python3.6/site-packages (from requests->transformers) (1.25.8)\n", "Requirement already satisfied, skipping upgrade: idna<3,>=2.5 in /home/yufeng/.local/lib/python3.6/site-packages (from requests->transformers) (2.9)\n", "Requirement already satisfied, skipping upgrade: chardet<4,>=3.0.2 in /home/yufeng/.local/lib/python3.6/site-packages (from requests->transformers) (3.0.4)\n", @@ -128,9 +128,9 @@ } ], "source": [ - "# Install or upgrade PyTorch 1.5.0 and OnnxRuntime 1.4.0 for CPU-only.\n", + "# Install or upgrade PyTorch 1.6.0 and OnnxRuntime 1.4.0 for CPU-only.\n", "import sys\n", - "!{sys.executable} -m pip install --upgrade torch==1.5.0+cpu torchvision==0.6.0+cpu -f https://download.pytorch.org/whl/torch_stable.html\n", + "!{sys.executable} -m pip install --upgrade torch==1.6.0+cpu torchvision==0.7.0+cpu -f https://download.pytorch.org/whl/torch_stable.html\n", "!{sys.executable} -m pip install --upgrade onnxruntime==1.4.0\n", "!{sys.executable} -m pip install -i https://test.pypi.org/simple/ ort-nightly\n", "!{sys.executable} -m pip install --upgrade onnxruntime-tools\n", @@ -157,7 +157,7 @@ }, { "cell_type": "code", - "execution_count": 2, + "execution_count": 4, "metadata": { "scrolled": true }, @@ -166,16 +166,16 @@ "name": "stdout", "output_type": "stream", "text": [ - "--2020-07-20 13:24:10-- https://raw.githubusercontent.com/huggingface/transformers/f98ef14d161d7bcdc9808b5ec399981481411cc1/utils/download_glue_data.py\n", - "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.40.133\n", - "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.40.133|:443... connected.\n", + "--2020-08-17 14:51:59-- https://raw.githubusercontent.com/huggingface/transformers/f98ef14d161d7bcdc9808b5ec399981481411cc1/utils/download_glue_data.py\n", + "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.188.133\n", + "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.188.133|:443... connected.\n", "HTTP request sent, awaiting response... 200 OK\n", "Length: 8209 (8.0K) [text/plain]\n", "Saving to: ‘download_glue_data.py’\n", "\n", "download_glue_data. 100%[===================>] 8.02K --.-KB/s in 0s \n", "\n", - "2020-07-20 13:24:11 (21.9 MB/s) - ‘download_glue_data.py’ saved [8209/8209]\n", + "2020-08-17 14:51:59 (21.6 MB/s) - ‘download_glue_data.py’ saved [8209/8209]\n", "\n", "Processing MRPC...\n", "Local MRPC data not specified, downloading data from https://dl.fbaipublicfiles.com/senteval/senteval_data/msr_paraphrase_train.txt\n", @@ -184,7 +184,7 @@ "dev.tsv msr_paraphrase_train.txt train.tsv\n", " % Total % Received % Xferd Average Speed Time Time Time Current\n", " Dload Upload Total Spent Left Speed\n", - "100 386M 100 386M 0 0 18.7M 0 0:00:20 0:00:20 --:--:-- 17.7M\n", + "100 386M 100 386M 0 0 31.7M 0 0:00:12 0:00:12 --:--:-- 34.6M\n", "Archive: MPRC.zip\n", " creating: MRPC/\n", " extracting: MRPC/added_tokens.json \n", @@ -237,7 +237,7 @@ }, { "cell_type": "code", - "execution_count": 3, + "execution_count": 5, "metadata": {}, "outputs": [ { @@ -281,14 +281,14 @@ }, { "cell_type": "code", - "execution_count": 4, + "execution_count": 6, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "1.5.0+cpu\n" + "1.6.0+cpu\n" ] } ], @@ -385,15 +385,15 @@ }, { "cell_type": "code", - "execution_count": 5, + "execution_count": 8, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "Size (MB): 437.982584\n", - "Size (MB): 181.443657\n" + "Size (MB): 417.72576904296875\n", + "Size (MB): 173.0903205871582\n" ] } ], @@ -411,7 +411,7 @@ "\n", "def print_size_of_model(model):\n", " torch.save(model.state_dict(), \"temp.p\")\n", - " print('Size (MB):', os.path.getsize(\"temp.p\")/1e6)\n", + " print('Size (MB):', os.path.getsize(\"temp.p\")/(1024*1024))\n", " os.remove('temp.p')\n", "\n", "print_size_of_model(model)\n", @@ -428,9 +428,16 @@ }, { "cell_type": "code", - "execution_count": 6, + "execution_count": 10, "metadata": {}, "outputs": [ + { + "name": "stderr", + "output_type": "stream", + "text": [ + "Evaluating: 0%| | 1/408 [00:00<00:45, 9.04it/s]" + ] + }, { "name": "stdout", "output_type": "stream", @@ -442,8 +449,8 @@ "name": "stderr", "output_type": "stream", "text": [ - "Evaluating: 100%|██████████| 408/408 [00:32<00:00, 12.56it/s]\n", - "Evaluating: 0%| | 2/408 [00:00<00:24, 16.58it/s]" + "Evaluating: 100%|██████████| 408/408 [00:32<00:00, 12.46it/s]\n", + "Evaluating: 0%| | 2/408 [00:00<00:25, 16.06it/s]" ] }, { @@ -451,7 +458,7 @@ "output_type": "stream", "text": [ "{'acc': 0.8602941176470589, 'f1': 0.9018932874354562, 'acc_and_f1': 0.8810937025412575}\n", - "Evaluate total time (seconds): 32.9\n", + "Evaluate total time (seconds): 32.8\n", "Evaluating PyTorch quantization accuracy and performance:\n" ] }, @@ -459,15 +466,22 @@ "name": "stderr", "output_type": "stream", "text": [ - "Evaluating: 100%|██████████| 408/408 [00:24<00:00, 16.44it/s]\n" + "Evaluating: 100%|██████████| 408/408 [00:25<00:00, 16.00it/s]" ] }, { "name": "stdout", "output_type": "stream", "text": [ - "{'acc': 0.8455882352941176, 'f1': 0.8934010152284263, 'acc_and_f1': 0.869494625261272}\n", - "Evaluate total time (seconds): 24.8\n" + "{'acc': 0.8578431372549019, 'f1': 0.8993055555555555, 'acc_and_f1': 0.8785743464052287}\n", + "Evaluate total time (seconds): 25.5\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "\n" ] } ], @@ -648,7 +662,7 @@ }, { "cell_type": "code", - "execution_count": 7, + "execution_count": 11, "metadata": {}, "outputs": [], "source": [ @@ -709,15 +723,15 @@ }, { "cell_type": "code", - "execution_count": 8, + "execution_count": 13, "metadata": {}, "outputs": [ { "name": "stdout", "output_type": "stream", "text": [ - "ONNX full precision model size (MB): 437.954025\n", - "ONNX quantized model size (MB): 111.666214\n" + "ONNX full precision model size (MB): 417.6655149459839\n", + "ONNX quantized model size (MB): 106.49320030212402\n" ] } ], @@ -736,8 +750,8 @@ "\n", "quantize_onnx_model('bert.opt.onnx', 'bert.opt.quant.onnx')\n", "\n", - "print('ONNX full precision model size (MB):', os.path.getsize(\"bert.opt.onnx\")/1e6)\n", - "print('ONNX quantized model size (MB):', os.path.getsize(\"bert.opt.quant.onnx\")/1e6)" + "print('ONNX full precision model size (MB):', os.path.getsize(\"bert.opt.onnx\")/(1024*1024))\n", + "print('ONNX quantized model size (MB):', os.path.getsize(\"bert.opt.quant.onnx\")/(1024*1024))" ] }, { @@ -751,14 +765,15 @@ }, { "cell_type": "code", - "execution_count": 9, + "execution_count": 14, "metadata": {}, "outputs": [ { "name": "stderr", "output_type": "stream", "text": [ - "Evaluating: 0%| | 1/408 [00:00<00:44, 9.20it/s]" + "\r", + "Evaluating: 0%| | 0/408 [00:00