mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-30 20:18:08 +00:00
Update notebook for mac since onnxruntime 1.3 or 1.4 in mac does not have openmp (#4732)
This commit is contained in:
Tianlei Wu
GitHub
parent
37c45c3d6b
commit
9c729d1719
1 changed files with 227 additions and 230 deletions
|
|
@ -53,67 +53,68 @@
|
|||
"output_type": "stream",
|
||||
"text": [
|
||||
"Looking in links: https://download.pytorch.org/whl/torch_stable.html\n",
|
||||
"Requirement already up-to-date: torch==1.5.0+cpu in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (1.5.0+cpu)\n",
|
||||
"Requirement already up-to-date: torchvision==0.6.0+cpu in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (0.6.0+cpu)\n",
|
||||
"Requirement already satisfied, skipping upgrade: numpy in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from torch==1.5.0+cpu) (1.18.1)\n",
|
||||
"Requirement already satisfied, skipping upgrade: future in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from torch==1.5.0+cpu) (0.18.2)\n",
|
||||
"Requirement already satisfied, skipping upgrade: pillow>=4.1.1 in c:\\users\\tianl\\appdata\\roaming\\python\\python36\\site-packages (from torchvision==0.6.0+cpu) (7.0.0)\n",
|
||||
"Requirement already up-to-date: onnxruntime==1.3.0 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (1.3.0)\n",
|
||||
"Requirement already satisfied, skipping upgrade: onnx>=1.2.3 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from onnxruntime==1.3.0) (1.7.0)\n",
|
||||
"Requirement already satisfied, skipping upgrade: numpy>=1.16.6 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from onnxruntime==1.3.0) (1.18.1)\n",
|
||||
"Requirement already satisfied, skipping upgrade: protobuf in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from onnxruntime==1.3.0) (3.11.3)\n",
|
||||
"Requirement already satisfied, skipping upgrade: typing-extensions>=3.6.2.1 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from onnx>=1.2.3->onnxruntime==1.3.0) (3.7.4.1)\n",
|
||||
"Requirement already satisfied, skipping upgrade: six in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from onnx>=1.2.3->onnxruntime==1.3.0) (1.14.0)\n",
|
||||
"Requirement already satisfied, skipping upgrade: setuptools in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from protobuf->onnxruntime==1.3.0) (45.2.0.post20200210)\n",
|
||||
"Requirement already up-to-date: onnxruntime-tools in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (1.3.0.1007)\n",
|
||||
"Requirement already satisfied, skipping upgrade: numpy in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from onnxruntime-tools) (1.18.1)\n",
|
||||
"Requirement already satisfied, skipping upgrade: py3nvml in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from onnxruntime-tools) (0.2.5)\n",
|
||||
"Requirement already satisfied, skipping upgrade: packaging in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from onnxruntime-tools) (20.1)\n",
|
||||
"Requirement already satisfied, skipping upgrade: onnx in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from onnxruntime-tools) (1.7.0)\n",
|
||||
"Requirement already satisfied, skipping upgrade: psutil in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from onnxruntime-tools) (5.7.0)\n",
|
||||
"Requirement already up-to-date: torch==1.6.0+cpu in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (1.6.0+cpu)\n",
|
||||
"Requirement already up-to-date: torchvision==0.7.0+cpu in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (0.7.0+cpu)\n",
|
||||
"Requirement already satisfied, skipping upgrade: numpy in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from torch==1.6.0+cpu) (1.18.1)\n",
|
||||
"Requirement already satisfied, skipping upgrade: future in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from torch==1.6.0+cpu) (0.18.2)\n",
|
||||
"Requirement already satisfied, skipping upgrade: pillow>=4.1.1 in c:\\users\\tianl\\appdata\\roaming\\python\\python36\\site-packages (from torchvision==0.7.0+cpu) (7.0.0)\n",
|
||||
"Requirement already up-to-date: onnxruntime==1.4.0 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (1.4.0)\n",
|
||||
"Requirement already satisfied, skipping upgrade: numpy>=1.16.6 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from onnxruntime==1.4.0) (1.18.1)\n",
|
||||
"Requirement already satisfied, skipping upgrade: protobuf in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from onnxruntime==1.4.0) (3.11.3)\n",
|
||||
"Requirement already satisfied, skipping upgrade: six>=1.9 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from protobuf->onnxruntime==1.4.0) (1.14.0)\n",
|
||||
"Requirement already satisfied, skipping upgrade: setuptools in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from protobuf->onnxruntime==1.4.0) (45.2.0.post20200210)\n",
|
||||
"Requirement already up-to-date: onnxruntime-tools in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (1.4.1)\n",
|
||||
"Requirement already satisfied, skipping upgrade: py-cpuinfo in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from onnxruntime-tools) (5.0.0)\n",
|
||||
"Requirement already satisfied, skipping upgrade: packaging in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from onnxruntime-tools) (20.1)\n",
|
||||
"Requirement already satisfied, skipping upgrade: coloredlogs in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from onnxruntime-tools) (14.0)\n",
|
||||
"Requirement already satisfied, skipping upgrade: xmltodict in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from py3nvml->onnxruntime-tools) (0.12.0)\n",
|
||||
"Requirement already satisfied, skipping upgrade: py3nvml in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from onnxruntime-tools) (0.2.5)\n",
|
||||
"Requirement already satisfied, skipping upgrade: numpy in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from onnxruntime-tools) (1.18.1)\n",
|
||||
"Requirement already satisfied, skipping upgrade: psutil in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from onnxruntime-tools) (5.7.0)\n",
|
||||
"Requirement already satisfied, skipping upgrade: onnx in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from onnxruntime-tools) (1.7.0)\n",
|
||||
"Requirement already satisfied, skipping upgrade: pyparsing>=2.0.2 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from packaging->onnxruntime-tools) (2.4.6)\n",
|
||||
"Requirement already satisfied, skipping upgrade: six in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from packaging->onnxruntime-tools) (1.14.0)\n",
|
||||
"Requirement already satisfied, skipping upgrade: humanfriendly>=7.1 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from coloredlogs->onnxruntime-tools) (8.1)\n",
|
||||
"Requirement already satisfied, skipping upgrade: xmltodict in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from py3nvml->onnxruntime-tools) (0.12.0)\n",
|
||||
"Requirement already satisfied, skipping upgrade: protobuf in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from onnx->onnxruntime-tools) (3.11.3)\n",
|
||||
"Requirement already satisfied, skipping upgrade: typing-extensions>=3.6.2.1 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from onnx->onnxruntime-tools) (3.7.4.1)\n",
|
||||
"Requirement already satisfied, skipping upgrade: humanfriendly>=7.1 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from coloredlogs->onnxruntime-tools) (8.1)\n",
|
||||
"Requirement already satisfied, skipping upgrade: setuptools in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from protobuf->onnx->onnxruntime-tools) (45.2.0.post20200210)\n",
|
||||
"Requirement already satisfied, skipping upgrade: pyreadline; sys_platform == \"win32\" in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from humanfriendly>=7.1->coloredlogs->onnxruntime-tools) (2.1)\n",
|
||||
"Requirement already satisfied: transformers==2.11.0 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (2.11.0)\n",
|
||||
"Requirement already satisfied: regex!=2019.12.17 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from transformers==2.11.0) (2020.2.20)\n",
|
||||
"Requirement already satisfied: dataclasses; python_version < \"3.7\" in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from transformers==2.11.0) (0.7)\n",
|
||||
"Requirement already satisfied: filelock in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from transformers==2.11.0) (3.0.12)\n",
|
||||
"Requirement already satisfied: requests in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from transformers==2.11.0) (2.23.0)\n",
|
||||
"Requirement already satisfied: sacremoses in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from transformers==2.11.0) (0.0.38)\n",
|
||||
"Requirement already satisfied: packaging in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from transformers==2.11.0) (20.1)\n",
|
||||
"Requirement already satisfied: sentencepiece in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from transformers==2.11.0) (0.1.85)\n",
|
||||
"Requirement already satisfied: tqdm>=4.27 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from transformers==2.11.0) (4.43.0)\n",
|
||||
"Requirement already satisfied: tokenizers==0.7.0 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from transformers==2.11.0) (0.7.0)\n",
|
||||
"Requirement already satisfied: numpy in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from transformers==2.11.0) (1.18.1)\n",
|
||||
"Requirement already satisfied: chardet<4,>=3.0.2 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from requests->transformers==2.11.0) (3.0.4)\n",
|
||||
"Requirement already satisfied: idna<3,>=2.5 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from requests->transformers==2.11.0) (2.9)\n",
|
||||
"Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from requests->transformers==2.11.0) (1.25.8)\n",
|
||||
"Requirement already satisfied: certifi>=2017.4.17 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from requests->transformers==2.11.0) (2020.4.5.1)\n",
|
||||
"Requirement already satisfied: six in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from sacremoses->transformers==2.11.0) (1.14.0)\n",
|
||||
"Requirement already satisfied: joblib in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from sacremoses->transformers==2.11.0) (0.14.1)\n",
|
||||
"Requirement already satisfied: click in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from sacremoses->transformers==2.11.0) (7.0)\n",
|
||||
"Requirement already satisfied: pyparsing>=2.0.2 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from packaging->transformers==2.11.0) (2.4.6)\n",
|
||||
"Requirement already satisfied, skipping upgrade: setuptools in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from protobuf->onnx->onnxruntime-tools) (45.2.0.post20200210)\n",
|
||||
"Requirement already satisfied: transformers==3.0.2 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (3.0.2)\n",
|
||||
"Requirement already satisfied: packaging in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from transformers==3.0.2) (20.1)\n",
|
||||
"Requirement already satisfied: sentencepiece!=0.1.92 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from transformers==3.0.2) (0.1.85)\n",
|
||||
"Requirement already satisfied: sacremoses in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from transformers==3.0.2) (0.0.38)\n",
|
||||
"Requirement already satisfied: dataclasses; python_version < \"3.7\" in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from transformers==3.0.2) (0.7)\n",
|
||||
"Requirement already satisfied: requests in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from transformers==3.0.2) (2.23.0)\n",
|
||||
"Requirement already satisfied: tokenizers==0.8.1.rc1 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from transformers==3.0.2) (0.8.1rc1)\n",
|
||||
"Requirement already satisfied: tqdm>=4.27 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from transformers==3.0.2) (4.43.0)\n",
|
||||
"Requirement already satisfied: regex!=2019.12.17 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from transformers==3.0.2) (2020.2.20)\n",
|
||||
"Requirement already satisfied: filelock in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from transformers==3.0.2) (3.0.12)\n",
|
||||
"Requirement already satisfied: numpy in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from transformers==3.0.2) (1.18.1)\n",
|
||||
"Requirement already satisfied: pyparsing>=2.0.2 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from packaging->transformers==3.0.2) (2.4.6)\n",
|
||||
"Requirement already satisfied: six in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from packaging->transformers==3.0.2) (1.14.0)\n",
|
||||
"Requirement already satisfied: click in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from sacremoses->transformers==3.0.2) (7.0)\n",
|
||||
"Requirement already satisfied: joblib in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from sacremoses->transformers==3.0.2) (0.14.1)\n",
|
||||
"Requirement already satisfied: certifi>=2017.4.17 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from requests->transformers==3.0.2) (2020.4.5.1)\n",
|
||||
"Requirement already satisfied: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from requests->transformers==3.0.2) (1.25.8)\n",
|
||||
"Requirement already satisfied: idna<3,>=2.5 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from requests->transformers==3.0.2) (2.9)\n",
|
||||
"Requirement already satisfied: chardet<4,>=3.0.2 in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (from requests->transformers==3.0.2) (3.0.4)\n",
|
||||
"Requirement already satisfied: wget in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (3.2)\n",
|
||||
"Requirement already satisfied: netron in d:\\anaconda3\\envs\\cpu_env\\lib\\site-packages (3.9.6)\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"# Install or upgrade PyTorch 1.5.0 and OnnxRuntime 1.3.0 for CPU-only.\n",
|
||||
"# Install PyTorch 1.6.0 and OnnxRuntime 1.4.0 for CPU-only.\n",
|
||||
"import sys\n",
|
||||
"!{sys.executable} -m pip install --upgrade torch==1.5.0+cpu torchvision==0.6.0+cpu -f https://download.pytorch.org/whl/torch_stable.html\n",
|
||||
"!{sys.executable} -m pip install --upgrade onnxruntime==1.3.0\n",
|
||||
"if sys.platform == 'darwin': # Mac\n",
|
||||
" !{sys.executable} -m pip install --upgrade torch torchvision\n",
|
||||
"else:\n",
|
||||
" !{sys.executable} -m pip install --upgrade torch==1.6.0+cpu torchvision==0.7.0+cpu -f https://download.pytorch.org/whl/torch_stable.html\n",
|
||||
"!{sys.executable} -m pip install --upgrade onnxruntime==1.4.0\n",
|
||||
"!{sys.executable} -m pip install --upgrade onnxruntime-tools\n",
|
||||
"\n",
|
||||
"# Install other packages used in this notebook.\n",
|
||||
"!{sys.executable} -m pip install transformers==2.11.0\n",
|
||||
"!{sys.executable} -m pip install transformers==3.0.2\n",
|
||||
"!{sys.executable} -m pip install wget netron"
|
||||
]
|
||||
},
|
||||
|
|
@ -139,7 +140,7 @@
|
|||
"source": [
|
||||
"import os\n",
|
||||
"\n",
|
||||
"cache_dir = \"./squad\"\n",
|
||||
"cache_dir = os.path.join(\"..\", \"cache_models\")\n",
|
||||
"if not os.path.exists(cache_dir):\n",
|
||||
" os.makedirs(cache_dir)\n",
|
||||
"\n",
|
||||
|
|
@ -194,9 +195,14 @@
|
|||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████| 48/48 [00:04<00:00, 10.47it/s]\n",
|
||||
"convert squad examples to features: 100%|████████████████████████████████████████████| 100/100 [00:01<00:00, 91.16it/s]\n",
|
||||
"add example index and unique id: 100%|███████████████████████████████████████████| 100/100 [00:00<00:00, 100007.25it/s]\n"
|
||||
"Some weights of the model checkpoint at bert-base-cased were not used when initializing BertForQuestionAnswering: ['cls.predictions.bias', 'cls.predictions.transform.dense.weight', 'cls.predictions.transform.dense.bias', 'cls.predictions.decoder.weight', 'cls.seq_relationship.weight', 'cls.seq_relationship.bias', 'cls.predictions.transform.LayerNorm.weight', 'cls.predictions.transform.LayerNorm.bias']\n",
|
||||
"- This IS expected if you are initializing BertForQuestionAnswering from the checkpoint of a model trained on another task or with another architecture (e.g. initializing a BertForSequenceClassification model from a BertForPretraining model).\n",
|
||||
"- This IS NOT expected if you are initializing BertForQuestionAnswering from the checkpoint of a model that you expect to be exactly identical (initializing a BertForSequenceClassification model from a BertForSequenceClassification model).\n",
|
||||
"Some weights of BertForQuestionAnswering were not initialized from the model checkpoint at bert-base-cased and are newly initialized: ['qa_outputs.weight', 'qa_outputs.bias']\n",
|
||||
"You should probably TRAIN this model on a down-stream task to be able to use it for predictions and inference.\n",
|
||||
"100%|██████████████████████████████████████████████████████████████████████████████████| 48/48 [00:03<00:00, 12.87it/s]\n",
|
||||
"convert squad examples to features: 100%|███████████████████████████████████████████| 100/100 [00:00<00:00, 131.41it/s]\n",
|
||||
"add example index and unique id: 100%|████████████████████████████████████████████| 100/100 [00:00<00:00, 96776.74it/s]\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
|
@ -249,12 +255,12 @@
|
|||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Model exported at ./onnx\\bert-base-cased-squad.onnx\n"
|
||||
"Model exported at ..\\onnx_models\\bert-base-cased-squad.onnx\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"output_dir = \"./onnx\"\n",
|
||||
"output_dir = os.path.join(\"..\", \"onnx_models\")\n",
|
||||
"if not os.path.exists(output_dir):\n",
|
||||
" os.makedirs(output_dir) \n",
|
||||
"export_model_path = os.path.join(output_dir, 'bert-base-cased-squad.onnx')\n",
|
||||
|
|
@ -312,7 +318,7 @@
|
|||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"PyTorch cpu Inference time = 198.99 ms\n"
|
||||
"PyTorch cpu Inference time = 144.73 ms\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
|
@ -379,7 +385,7 @@
|
|||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"OnnxRuntime cpu Inference time = 176.96 ms\n"
|
||||
"OnnxRuntime cpu Inference time = 88.55 ms\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
|
@ -397,10 +403,9 @@
|
|||
"# Note that this will increase session creation time, so it is for debugging only.\n",
|
||||
"sess_options.optimized_model_filepath = os.path.join(output_dir, \"optimized_model_cpu.onnx\")\n",
|
||||
"\n",
|
||||
"# intra_op_num_threads is needed for OnnxRuntime 1.2.0.\n",
|
||||
"# For OnnxRuntime 1.3.0 or later, this does not have effect unless you are using onnxruntime-gpu package.\n",
|
||||
"sess_options.intra_op_num_threads=1\n",
|
||||
"\n",
|
||||
"# For OnnxRuntime 1.2.0, you might need set intra_op_num_threads to 1 to enable OpenMP\n",
|
||||
"# sess_options.intra_op_num_threads=1\n",
|
||||
"# For OnnxRuntime 1.3.0 or later, it is recommended to use the default setting so you need not set it.\n",
|
||||
"\n",
|
||||
"# Specify providers when you use onnxruntime-gpu for CPU inference.\n",
|
||||
"session = onnxruntime.InferenceSession(export_model_path, sess_options, providers=['CPUExecutionProvider'])\n",
|
||||
|
|
@ -408,7 +413,6 @@
|
|||
"latency = []\n",
|
||||
"for i in range(total_samples):\n",
|
||||
" data = dataset[i]\n",
|
||||
" # TODO: use IO Binding (see https://github.com/microsoft/onnxruntime/pull/4206) to improve performance.\n",
|
||||
" ort_inputs = {\n",
|
||||
" 'input_ids': data[0].cpu().reshape(1, max_seq_length).numpy(),\n",
|
||||
" 'input_mask': data[1].cpu().reshape(1, max_seq_length).numpy(),\n",
|
||||
|
|
@ -486,31 +490,25 @@
|
|||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"optimize_by_onnxruntime: Save optimized model by onnxruntime to ./onnx\\bert-base-cased-squad_o1_cpu.onnx\n",
|
||||
" apply: Fused LayerNormalization count: 25\n",
|
||||
" apply: Fused LayerNormalization count: 0\n",
|
||||
" apply: Fused Gelu count: 12\n",
|
||||
" apply: Fused FastGelu count: 0\n",
|
||||
" apply: Fused Reshape count: 0\n",
|
||||
" apply: Fused SkipLayerNormalization count: 25\n",
|
||||
" apply: Fused Attention count: 12\n",
|
||||
" prune_graph: Graph pruned: 0 inputs, 0 outputs and 5 nodes are removed\n",
|
||||
" apply: Fused EmbedLayerNormalization(with mask) count: 1\n",
|
||||
" prune_graph: Graph pruned: 0 inputs, 0 outputs and 12 nodes are removed\n",
|
||||
" prune_graph: Graph pruned: 0 inputs, 0 outputs and 0 nodes are removed\n",
|
||||
" apply: Fused FastGelu(add bias) count: 0\n",
|
||||
" apply: Fused BiasGelu count: 12\n",
|
||||
" apply: Fused SkipLayerNormalization(add bias) count: 24\n",
|
||||
" optimize: opset verion: 11\n",
|
||||
" save_model_to_file: Output model to ./onnx/bert-base-cased-squad_opt_cpu.onnx\n",
|
||||
" save_model_to_file: Output model to ..\\onnx_models\\bert-base-cased-squad_opt_cpu.onnx\n",
|
||||
"get_fused_operator_statistics: Optimized operators:{'EmbedLayerNormalization': 1, 'Attention': 12, 'Gelu': 0, 'FastGelu': 0, 'BiasGelu': 12, 'LayerNormalization': 0, 'SkipLayerNormalization': 24}\n",
|
||||
" is_fully_optimized: EmbedLayer=1, Attention=12, Gelu=12, LayerNormalization=24, Successful=True\n",
|
||||
" main: The output model is fully optimized.\n"
|
||||
" main: The model has been fully optimized.\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"optimized_model_path = './onnx/bert-base-cased-squad_opt_cpu.onnx'\n",
|
||||
"optimized_model_path = os.path.join(output_dir, 'bert-base-cased-squad_opt_cpu.onnx')\n",
|
||||
"\n",
|
||||
"!{sys.executable} -m onnxruntime_tools.optimizer_cli --input $export_model_path --output $optimized_model_path --model_type bert --num_heads 12 --hidden_size 768"
|
||||
]
|
||||
|
|
@ -535,7 +533,7 @@
|
|||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Serving './onnx/bert-base-cased-squad_opt_cpu.onnx' at http://localhost:8080\n"
|
||||
"Serving '..\\onnx_models\\bert-base-cased-squad_opt_cpu.onnx' at http://localhost:8080\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
|
@ -570,8 +568,8 @@
|
|||
"output_type": "stream",
|
||||
"text": [
|
||||
"100% passed for 100 random inputs given thresholds (rtol=0.001, atol=0.0001).\n",
|
||||
"maximum absolute difference=3.46451997756958e-06\n",
|
||||
"maximum relative difference=0.03302651643753052\n"
|
||||
"maximum absolute difference=5.930662155151367e-06\n",
|
||||
"maximum relative difference=0.021568937227129936\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
|
@ -592,45 +590,45 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 16,
|
||||
"execution_count": 13,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"Running test: model=bert-base-cased-squad_opt_cpu.onnx,graph_optimization_level=ENABLE_ALL,intra_op_num_threads=1,OMP_NUM_THREADS=,OMP_WAIT_POLICY=,batch_size=1,sequence_length=128,test_cases=100,test_times=1,contiguous=False,use_gpu=False,warmup=True\n",
|
||||
"Average latency = 102.17 ms, Throughput = 9.79 QPS\n",
|
||||
"Running test: model=bert-base-cased-squad_opt_cpu.onnx,graph_optimization_level=ENABLE_ALL,intra_op_num_threads=1,OMP_NUM_THREADS=,OMP_WAIT_POLICY=PASSIVE,batch_size=1,sequence_length=128,test_cases=100,test_times=1,contiguous=False,use_gpu=False,warmup=True\n",
|
||||
"Average latency = 91.39 ms, Throughput = 10.94 QPS\n",
|
||||
"Running test: model=bert-base-cased-squad_opt_cpu.onnx,graph_optimization_level=ENABLE_ALL,intra_op_num_threads=1,OMP_NUM_THREADS=,OMP_WAIT_POLICY=ACTIVE,batch_size=1,sequence_length=128,test_cases=100,test_times=1,contiguous=False,use_gpu=False,warmup=True\n",
|
||||
"Average latency = 105.09 ms, Throughput = 9.52 QPS\n",
|
||||
"Running test: model=bert-base-cased-squad_opt_cpu.onnx,graph_optimization_level=ENABLE_ALL,intra_op_num_threads=1,OMP_NUM_THREADS=1,OMP_WAIT_POLICY=,batch_size=1,sequence_length=128,test_cases=100,test_times=1,contiguous=False,use_gpu=False,warmup=True\n",
|
||||
"Average latency = 291.53 ms, Throughput = 3.43 QPS\n",
|
||||
"Running test: model=bert-base-cased-squad_opt_cpu.onnx,graph_optimization_level=ENABLE_ALL,intra_op_num_threads=1,OMP_NUM_THREADS=1,OMP_WAIT_POLICY=PASSIVE,batch_size=1,sequence_length=128,test_cases=100,test_times=1,contiguous=False,use_gpu=False,warmup=True\n",
|
||||
"Average latency = 306.75 ms, Throughput = 3.26 QPS\n",
|
||||
"Running test: model=bert-base-cased-squad_opt_cpu.onnx,graph_optimization_level=ENABLE_ALL,intra_op_num_threads=1,OMP_NUM_THREADS=1,OMP_WAIT_POLICY=ACTIVE,batch_size=1,sequence_length=128,test_cases=100,test_times=1,contiguous=False,use_gpu=False,warmup=True\n",
|
||||
"Average latency = 289.82 ms, Throughput = 3.45 QPS\n",
|
||||
"Running test: model=bert-base-cased-squad_opt_cpu.onnx,graph_optimization_level=ENABLE_ALL,intra_op_num_threads=1,OMP_NUM_THREADS=12,OMP_WAIT_POLICY=,batch_size=1,sequence_length=128,test_cases=100,test_times=1,contiguous=False,use_gpu=False,warmup=True\n",
|
||||
"Average latency = 101.56 ms, Throughput = 9.85 QPS\n",
|
||||
"Running test: model=bert-base-cased-squad_opt_cpu.onnx,graph_optimization_level=ENABLE_ALL,intra_op_num_threads=1,OMP_NUM_THREADS=12,OMP_WAIT_POLICY=PASSIVE,batch_size=1,sequence_length=128,test_cases=100,test_times=1,contiguous=False,use_gpu=False,warmup=True\n",
|
||||
"Average latency = 84.73 ms, Throughput = 11.80 QPS\n",
|
||||
"Running test: model=bert-base-cased-squad_opt_cpu.onnx,graph_optimization_level=ENABLE_ALL,intra_op_num_threads=1,OMP_NUM_THREADS=12,OMP_WAIT_POLICY=ACTIVE,batch_size=1,sequence_length=128,test_cases=100,test_times=1,contiguous=False,use_gpu=False,warmup=True\n",
|
||||
"Average latency = 101.53 ms, Throughput = 9.85 QPS\n",
|
||||
"Running test: model=bert-base-cased-squad_opt_cpu.onnx,graph_optimization_level=ENABLE_ALL,intra_op_num_threads=1,OMP_NUM_THREADS=6,OMP_WAIT_POLICY=,batch_size=1,sequence_length=128,test_cases=100,test_times=1,contiguous=False,use_gpu=False,warmup=True\n",
|
||||
"Average latency = 100.54 ms, Throughput = 9.95 QPS\n",
|
||||
"Running test: model=bert-base-cased-squad_opt_cpu.onnx,graph_optimization_level=ENABLE_ALL,intra_op_num_threads=1,OMP_NUM_THREADS=6,OMP_WAIT_POLICY=PASSIVE,batch_size=1,sequence_length=128,test_cases=100,test_times=1,contiguous=False,use_gpu=False,warmup=True\n",
|
||||
"Average latency = 111.82 ms, Throughput = 8.94 QPS\n",
|
||||
"Running test: model=bert-base-cased-squad_opt_cpu.onnx,graph_optimization_level=ENABLE_ALL,intra_op_num_threads=1,OMP_NUM_THREADS=6,OMP_WAIT_POLICY=ACTIVE,batch_size=1,sequence_length=128,test_cases=100,test_times=1,contiguous=False,use_gpu=False,warmup=True\n",
|
||||
"Average latency = 101.14 ms, Throughput = 9.89 QPS\n",
|
||||
"test setting TestSetting(use_gpu=False, batch_size=1, sequence_length=128, test_cases=100, test_times=1, omp_num_threads=None, omp_wait_policy=None, intra_op_num_threads=1, seed=3, verbose=False, contiguous=False, inclusive=True, extra_latency=0, warmup=True)\n",
|
||||
"Running test: model=bert-base-cased-squad_opt_cpu.onnx,graph_optimization_level=ENABLE_ALL,intra_op_num_threads=0,OMP_NUM_THREADS=,OMP_WAIT_POLICY=,batch_size=1,sequence_length=128,test_cases=100,test_times=1,contiguous=None,use_gpu=False,warmup=True\n",
|
||||
"Average latency = 80.08 ms, Throughput = 12.49 QPS\n",
|
||||
"Running test: model=bert-base-cased-squad_opt_cpu.onnx,graph_optimization_level=ENABLE_ALL,intra_op_num_threads=1,OMP_NUM_THREADS=,OMP_WAIT_POLICY=,batch_size=1,sequence_length=128,test_cases=100,test_times=1,contiguous=None,use_gpu=False,warmup=True\n",
|
||||
"Average latency = 78.56 ms, Throughput = 12.73 QPS\n",
|
||||
"Running test: model=bert-base-cased-squad_opt_cpu.onnx,graph_optimization_level=ENABLE_ALL,intra_op_num_threads=1,OMP_NUM_THREADS=12,OMP_WAIT_POLICY=ACTIVE,batch_size=1,sequence_length=128,test_cases=100,test_times=1,contiguous=None,use_gpu=False,warmup=True\n",
|
||||
"Average latency = 77.78 ms, Throughput = 12.86 QPS\n",
|
||||
"Running test: model=bert-base-cased-squad_opt_cpu.onnx,graph_optimization_level=ENABLE_ALL,intra_op_num_threads=1,OMP_NUM_THREADS=12,OMP_WAIT_POLICY=PASSIVE,batch_size=1,sequence_length=128,test_cases=100,test_times=1,contiguous=None,use_gpu=False,warmup=True\n",
|
||||
"Average latency = 73.52 ms, Throughput = 13.60 QPS\n",
|
||||
"Running test: model=bert-base-cased-squad_opt_cpu.onnx,graph_optimization_level=ENABLE_ALL,intra_op_num_threads=1,OMP_NUM_THREADS=6,OMP_WAIT_POLICY=ACTIVE,batch_size=1,sequence_length=128,test_cases=100,test_times=1,contiguous=None,use_gpu=False,warmup=True\n",
|
||||
"Average latency = 95.36 ms, Throughput = 10.49 QPS\n",
|
||||
"Running test: model=bert-base-cased-squad_opt_cpu.onnx,graph_optimization_level=ENABLE_ALL,intra_op_num_threads=1,OMP_NUM_THREADS=6,OMP_WAIT_POLICY=PASSIVE,batch_size=1,sequence_length=128,test_cases=100,test_times=1,contiguous=None,use_gpu=False,warmup=True\n",
|
||||
"Average latency = 78.49 ms, Throughput = 12.74 QPS\n",
|
||||
"Running test: model=bert-base-cased-squad_opt_cpu.onnx,graph_optimization_level=ENABLE_ALL,intra_op_num_threads=12,OMP_NUM_THREADS=1,OMP_WAIT_POLICY=ACTIVE,batch_size=1,sequence_length=128,test_cases=100,test_times=1,contiguous=None,use_gpu=False,warmup=True\n",
|
||||
"Average latency = 288.71 ms, Throughput = 3.46 QPS\n",
|
||||
"Running test: model=bert-base-cased-squad_opt_cpu.onnx,graph_optimization_level=ENABLE_ALL,intra_op_num_threads=12,OMP_NUM_THREADS=1,OMP_WAIT_POLICY=PASSIVE,batch_size=1,sequence_length=128,test_cases=100,test_times=1,contiguous=None,use_gpu=False,warmup=True\n",
|
||||
"Average latency = 288.61 ms, Throughput = 3.46 QPS\n",
|
||||
"Running test: model=bert-base-cased-squad_opt_cpu.onnx,graph_optimization_level=ENABLE_ALL,intra_op_num_threads=6,OMP_NUM_THREADS=1,OMP_WAIT_POLICY=ACTIVE,batch_size=1,sequence_length=128,test_cases=100,test_times=1,contiguous=None,use_gpu=False,warmup=True\n",
|
||||
"Average latency = 288.97 ms, Throughput = 3.46 QPS\n",
|
||||
"Running test: model=bert-base-cased-squad_opt_cpu.onnx,graph_optimization_level=ENABLE_ALL,intra_op_num_threads=6,OMP_NUM_THREADS=1,OMP_WAIT_POLICY=PASSIVE,batch_size=1,sequence_length=128,test_cases=100,test_times=1,contiguous=None,use_gpu=False,warmup=True\n",
|
||||
"Average latency = 288.37 ms, Throughput = 3.47 QPS\n",
|
||||
"Running test: model=bert-base-cased-squad_opt_cpu.onnx,graph_optimization_level=ENABLE_ALL,intra_op_num_threads=6,OMP_NUM_THREADS=6,OMP_WAIT_POLICY=ACTIVE,batch_size=1,sequence_length=128,test_cases=100,test_times=1,contiguous=None,use_gpu=False,warmup=True\n",
|
||||
"Average latency = 95.49 ms, Throughput = 10.47 QPS\n",
|
||||
"Running test: model=bert-base-cased-squad_opt_cpu.onnx,graph_optimization_level=ENABLE_ALL,intra_op_num_threads=6,OMP_NUM_THREADS=6,OMP_WAIT_POLICY=PASSIVE,batch_size=1,sequence_length=128,test_cases=100,test_times=1,contiguous=None,use_gpu=False,warmup=True\n",
|
||||
"Average latency = 79.17 ms, Throughput = 12.63 QPS\n",
|
||||
"test setting TestSetting(batch_size=1, sequence_length=128, test_cases=100, test_times=1, contiguous=None, use_gpu=False, warmup=True, omp_num_threads=None, omp_wait_policy=None, intra_op_num_threads=None, seed=3, verbose=False, inclusive=False, extra_latency=True)\n",
|
||||
"Generating 100 samples for batch_size=1 sequence_length=128\n",
|
||||
"Test summary is saved to onnx\\perf_results_CPU_B1_S128_20200612-115010.txt\n"
|
||||
"Test summary is saved to ..\\onnx_models\\perf_results_CPU_B1_S128_20200806-173045.txt\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
"source": [
|
||||
"!{sys.executable} -m onnxruntime_tools.transformers.bert_perf_test --model $optimized_model_path --batch_size 1 --sequence_length 128 --samples 100 --test_times 1 --intra_op_num_threads 1 --inclusive --all"
|
||||
"!{sys.executable} -m onnxruntime_tools.transformers.bert_perf_test --model $optimized_model_path --batch_size 1 --sequence_length 128 --samples 100 --test_times 1 --inclusive --all"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -642,15 +640,14 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 17,
|
||||
"execution_count": 14,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
"name": "stdout",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"./onnx\\perf_results_CPU_B1_S128_20200612-115010.txt\n",
|
||||
"The best setting is: use openmp; NO contiguous array\n"
|
||||
"..\\onnx_models\\perf_results_CPU_B1_S128_20200806-173045.txt\n"
|
||||
]
|
||||
},
|
||||
{
|
||||
|
|
@ -688,147 +685,147 @@
|
|||
" <tbody>\n",
|
||||
" <tr>\n",
|
||||
" <th>0</th>\n",
|
||||
" <td>84.73</td>\n",
|
||||
" <td>87.19</td>\n",
|
||||
" <td>91.51</td>\n",
|
||||
" <td>95.46</td>\n",
|
||||
" <td>11.80</td>\n",
|
||||
" <td>73.52</td>\n",
|
||||
" <td>75.78</td>\n",
|
||||
" <td>78.21</td>\n",
|
||||
" <td>89.29</td>\n",
|
||||
" <td>13.60</td>\n",
|
||||
" <td>1</td>\n",
|
||||
" <td>12</td>\n",
|
||||
" <td>PASSIVE</td>\n",
|
||||
" <td>False</td>\n",
|
||||
" <td>None</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>1</th>\n",
|
||||
" <td>91.39</td>\n",
|
||||
" <td>94.92</td>\n",
|
||||
" <td>101.16</td>\n",
|
||||
" <td>119.06</td>\n",
|
||||
" <td>10.94</td>\n",
|
||||
" <td>77.78</td>\n",
|
||||
" <td>82.35</td>\n",
|
||||
" <td>87.02</td>\n",
|
||||
" <td>104.54</td>\n",
|
||||
" <td>12.86</td>\n",
|
||||
" <td>1</td>\n",
|
||||
" <td></td>\n",
|
||||
" <td>PASSIVE</td>\n",
|
||||
" <td>False</td>\n",
|
||||
" <td>12</td>\n",
|
||||
" <td>ACTIVE</td>\n",
|
||||
" <td>None</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>2</th>\n",
|
||||
" <td>100.54</td>\n",
|
||||
" <td>101.63</td>\n",
|
||||
" <td>103.12</td>\n",
|
||||
" <td>109.20</td>\n",
|
||||
" <td>9.95</td>\n",
|
||||
" <td>78.49</td>\n",
|
||||
" <td>80.92</td>\n",
|
||||
" <td>85.77</td>\n",
|
||||
" <td>98.98</td>\n",
|
||||
" <td>12.74</td>\n",
|
||||
" <td>1</td>\n",
|
||||
" <td>6</td>\n",
|
||||
" <td></td>\n",
|
||||
" <td>False</td>\n",
|
||||
" <td>PASSIVE</td>\n",
|
||||
" <td>None</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>3</th>\n",
|
||||
" <td>101.14</td>\n",
|
||||
" <td>102.05</td>\n",
|
||||
" <td>106.15</td>\n",
|
||||
" <td>112.17</td>\n",
|
||||
" <td>9.89</td>\n",
|
||||
" <td>78.56</td>\n",
|
||||
" <td>82.29</td>\n",
|
||||
" <td>93.46</td>\n",
|
||||
" <td>108.73</td>\n",
|
||||
" <td>12.73</td>\n",
|
||||
" <td>1</td>\n",
|
||||
" <td>6</td>\n",
|
||||
" <td>ACTIVE</td>\n",
|
||||
" <td>False</td>\n",
|
||||
" <td></td>\n",
|
||||
" <td></td>\n",
|
||||
" <td>None</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>4</th>\n",
|
||||
" <td>101.53</td>\n",
|
||||
" <td>102.30</td>\n",
|
||||
" <td>105.58</td>\n",
|
||||
" <td>113.53</td>\n",
|
||||
" <td>9.85</td>\n",
|
||||
" <td>1</td>\n",
|
||||
" <td>12</td>\n",
|
||||
" <td>ACTIVE</td>\n",
|
||||
" <td>False</td>\n",
|
||||
" <td>79.17</td>\n",
|
||||
" <td>82.02</td>\n",
|
||||
" <td>87.60</td>\n",
|
||||
" <td>99.55</td>\n",
|
||||
" <td>12.63</td>\n",
|
||||
" <td>6</td>\n",
|
||||
" <td>6</td>\n",
|
||||
" <td>PASSIVE</td>\n",
|
||||
" <td>None</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>5</th>\n",
|
||||
" <td>101.56</td>\n",
|
||||
" <td>102.54</td>\n",
|
||||
" <td>104.08</td>\n",
|
||||
" <td>106.72</td>\n",
|
||||
" <td>9.85</td>\n",
|
||||
" <td>1</td>\n",
|
||||
" <td>12</td>\n",
|
||||
" <td>80.08</td>\n",
|
||||
" <td>83.18</td>\n",
|
||||
" <td>95.60</td>\n",
|
||||
" <td>107.72</td>\n",
|
||||
" <td>12.49</td>\n",
|
||||
" <td>0</td>\n",
|
||||
" <td></td>\n",
|
||||
" <td>False</td>\n",
|
||||
" <td></td>\n",
|
||||
" <td>None</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>6</th>\n",
|
||||
" <td>102.17</td>\n",
|
||||
" <td>104.58</td>\n",
|
||||
" <td>106.50</td>\n",
|
||||
" <td>111.18</td>\n",
|
||||
" <td>9.79</td>\n",
|
||||
" <td>95.36</td>\n",
|
||||
" <td>101.25</td>\n",
|
||||
" <td>103.61</td>\n",
|
||||
" <td>105.15</td>\n",
|
||||
" <td>10.49</td>\n",
|
||||
" <td>1</td>\n",
|
||||
" <td></td>\n",
|
||||
" <td></td>\n",
|
||||
" <td>False</td>\n",
|
||||
" <td>6</td>\n",
|
||||
" <td>ACTIVE</td>\n",
|
||||
" <td>None</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>7</th>\n",
|
||||
" <td>105.09</td>\n",
|
||||
" <td>107.26</td>\n",
|
||||
" <td>112.70</td>\n",
|
||||
" <td>123.89</td>\n",
|
||||
" <td>9.52</td>\n",
|
||||
" <td>1</td>\n",
|
||||
" <td></td>\n",
|
||||
" <td>95.49</td>\n",
|
||||
" <td>101.50</td>\n",
|
||||
" <td>102.66</td>\n",
|
||||
" <td>104.82</td>\n",
|
||||
" <td>10.47</td>\n",
|
||||
" <td>6</td>\n",
|
||||
" <td>6</td>\n",
|
||||
" <td>ACTIVE</td>\n",
|
||||
" <td>False</td>\n",
|
||||
" <td>None</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>8</th>\n",
|
||||
" <td>111.82</td>\n",
|
||||
" <td>113.28</td>\n",
|
||||
" <td>116.62</td>\n",
|
||||
" <td>119.12</td>\n",
|
||||
" <td>8.94</td>\n",
|
||||
" <td>1</td>\n",
|
||||
" <td>288.37</td>\n",
|
||||
" <td>290.48</td>\n",
|
||||
" <td>295.37</td>\n",
|
||||
" <td>308.91</td>\n",
|
||||
" <td>3.47</td>\n",
|
||||
" <td>6</td>\n",
|
||||
" <td>1</td>\n",
|
||||
" <td>PASSIVE</td>\n",
|
||||
" <td>False</td>\n",
|
||||
" <td>None</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>9</th>\n",
|
||||
" <td>289.82</td>\n",
|
||||
" <td>294.33</td>\n",
|
||||
" <td>300.76</td>\n",
|
||||
" <td>333.44</td>\n",
|
||||
" <td>3.45</td>\n",
|
||||
" <td>288.61</td>\n",
|
||||
" <td>291.10</td>\n",
|
||||
" <td>295.78</td>\n",
|
||||
" <td>301.52</td>\n",
|
||||
" <td>3.46</td>\n",
|
||||
" <td>12</td>\n",
|
||||
" <td>1</td>\n",
|
||||
" <td>1</td>\n",
|
||||
" <td>ACTIVE</td>\n",
|
||||
" <td>False</td>\n",
|
||||
" <td>PASSIVE</td>\n",
|
||||
" <td>None</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>10</th>\n",
|
||||
" <td>291.53</td>\n",
|
||||
" <td>298.85</td>\n",
|
||||
" <td>312.26</td>\n",
|
||||
" <td>339.63</td>\n",
|
||||
" <td>3.43</td>\n",
|
||||
" <td>288.71</td>\n",
|
||||
" <td>292.64</td>\n",
|
||||
" <td>298.28</td>\n",
|
||||
" <td>305.92</td>\n",
|
||||
" <td>3.46</td>\n",
|
||||
" <td>12</td>\n",
|
||||
" <td>1</td>\n",
|
||||
" <td>1</td>\n",
|
||||
" <td></td>\n",
|
||||
" <td>False</td>\n",
|
||||
" <td>ACTIVE</td>\n",
|
||||
" <td>None</td>\n",
|
||||
" </tr>\n",
|
||||
" <tr>\n",
|
||||
" <th>11</th>\n",
|
||||
" <td>306.75</td>\n",
|
||||
" <td>314.65</td>\n",
|
||||
" <td>335.34</td>\n",
|
||||
" <td>422.32</td>\n",
|
||||
" <td>3.26</td>\n",
|
||||
" <td>288.97</td>\n",
|
||||
" <td>291.18</td>\n",
|
||||
" <td>297.68</td>\n",
|
||||
" <td>309.30</td>\n",
|
||||
" <td>3.46</td>\n",
|
||||
" <td>6</td>\n",
|
||||
" <td>1</td>\n",
|
||||
" <td>1</td>\n",
|
||||
" <td>PASSIVE</td>\n",
|
||||
" <td>False</td>\n",
|
||||
" <td>ACTIVE</td>\n",
|
||||
" <td>None</td>\n",
|
||||
" </tr>\n",
|
||||
" </tbody>\n",
|
||||
"</table>\n",
|
||||
|
|
@ -836,35 +833,35 @@
|
|||
],
|
||||
"text/plain": [
|
||||
" Latency(ms) Latency_P75 Latency_P90 Latency_P99 Throughput(QPS) \\\n",
|
||||
"0 84.73 87.19 91.51 95.46 11.80 \n",
|
||||
"1 91.39 94.92 101.16 119.06 10.94 \n",
|
||||
"2 100.54 101.63 103.12 109.20 9.95 \n",
|
||||
"3 101.14 102.05 106.15 112.17 9.89 \n",
|
||||
"4 101.53 102.30 105.58 113.53 9.85 \n",
|
||||
"5 101.56 102.54 104.08 106.72 9.85 \n",
|
||||
"6 102.17 104.58 106.50 111.18 9.79 \n",
|
||||
"7 105.09 107.26 112.70 123.89 9.52 \n",
|
||||
"8 111.82 113.28 116.62 119.12 8.94 \n",
|
||||
"9 289.82 294.33 300.76 333.44 3.45 \n",
|
||||
"10 291.53 298.85 312.26 339.63 3.43 \n",
|
||||
"11 306.75 314.65 335.34 422.32 3.26 \n",
|
||||
"0 73.52 75.78 78.21 89.29 13.60 \n",
|
||||
"1 77.78 82.35 87.02 104.54 12.86 \n",
|
||||
"2 78.49 80.92 85.77 98.98 12.74 \n",
|
||||
"3 78.56 82.29 93.46 108.73 12.73 \n",
|
||||
"4 79.17 82.02 87.60 99.55 12.63 \n",
|
||||
"5 80.08 83.18 95.60 107.72 12.49 \n",
|
||||
"6 95.36 101.25 103.61 105.15 10.49 \n",
|
||||
"7 95.49 101.50 102.66 104.82 10.47 \n",
|
||||
"8 288.37 290.48 295.37 308.91 3.47 \n",
|
||||
"9 288.61 291.10 295.78 301.52 3.46 \n",
|
||||
"10 288.71 292.64 298.28 305.92 3.46 \n",
|
||||
"11 288.97 291.18 297.68 309.30 3.46 \n",
|
||||
"\n",
|
||||
" intra_op_num_threads OMP_NUM_THREADS OMP_WAIT_POLICY contiguous \n",
|
||||
"0 1 12 PASSIVE False \n",
|
||||
"1 1 PASSIVE False \n",
|
||||
"2 1 6 False \n",
|
||||
"3 1 6 ACTIVE False \n",
|
||||
"4 1 12 ACTIVE False \n",
|
||||
"5 1 12 False \n",
|
||||
"6 1 False \n",
|
||||
"7 1 ACTIVE False \n",
|
||||
"8 1 6 PASSIVE False \n",
|
||||
"9 1 1 ACTIVE False \n",
|
||||
"10 1 1 False \n",
|
||||
"11 1 1 PASSIVE False "
|
||||
" intra_op_num_threads OMP_NUM_THREADS OMP_WAIT_POLICY contiguous \n",
|
||||
"0 1 12 PASSIVE None \n",
|
||||
"1 1 12 ACTIVE None \n",
|
||||
"2 1 6 PASSIVE None \n",
|
||||
"3 1 None \n",
|
||||
"4 6 6 PASSIVE None \n",
|
||||
"5 0 None \n",
|
||||
"6 1 6 ACTIVE None \n",
|
||||
"7 6 6 ACTIVE None \n",
|
||||
"8 6 1 PASSIVE None \n",
|
||||
"9 12 1 PASSIVE None \n",
|
||||
"10 12 1 ACTIVE None \n",
|
||||
"11 6 1 ACTIVE None "
|
||||
]
|
||||
},
|
||||
"execution_count": 17,
|
||||
"execution_count": 14,
|
||||
"metadata": {},
|
||||
"output_type": "execute_result"
|
||||
}
|
||||
|
|
@ -873,7 +870,7 @@
|
|||
"import os\n",
|
||||
"import glob \n",
|
||||
"import pandas\n",
|
||||
"latest_result_file = max(glob.glob(\"./onnx/perf_results_*.txt\"), key=os.path.getmtime)\n",
|
||||
"latest_result_file = max(glob.glob(os.path.join(output_dir, \"perf_results_*.txt\")), key=os.path.getmtime)\n",
|
||||
"result_data = pandas.read_table(latest_result_file, converters={'OMP_NUM_THREADS': str, 'OMP_WAIT_POLICY':str})\n",
|
||||
"print(latest_result_file)\n",
|
||||
"\n",
|
||||
|
|
@ -903,7 +900,7 @@
|
|||
},
|
||||
{
|
||||
"cell_type": "code",
|
||||
"execution_count": 18,
|
||||
"execution_count": 15,
|
||||
"metadata": {},
|
||||
"outputs": [
|
||||
{
|
||||
|
|
@ -916,7 +913,7 @@
|
|||
" \"devices\": [\n",
|
||||
" {\n",
|
||||
" \"memory_total\": 8589934592,\n",
|
||||
" \"memory_available\": 8480882688,\n",
|
||||
" \"memory_available\": 6997721088,\n",
|
||||
" \"name\": \"GeForce GTX 1070\"\n",
|
||||
" }\n",
|
||||
" ]\n",
|
||||
|
|
@ -931,22 +928,22 @@
|
|||
" \"processor\": \"Intel64 Family 6 Model 158 Stepping 10, GenuineIntel\"\n",
|
||||
" },\n",
|
||||
" \"memory\": {\n",
|
||||
" \"total\": 16971259904,\n",
|
||||
" \"available\": 6604914688\n",
|
||||
" \"total\": 16971276288,\n",
|
||||
" \"available\": 4723568640\n",
|
||||
" },\n",
|
||||
" \"python\": \"3.6.10.final.0 (64 bit)\",\n",
|
||||
" \"os\": \"Windows-10-10.0.18362-SP0\",\n",
|
||||
" \"os\": \"Windows-10-10.0.19041-SP0\",\n",
|
||||
" \"onnxruntime\": {\n",
|
||||
" \"version\": \"1.3.0\",\n",
|
||||
" \"version\": \"1.4.0\",\n",
|
||||
" \"support_gpu\": false\n",
|
||||
" },\n",
|
||||
" \"pytorch\": {\n",
|
||||
" \"version\": \"1.5.0+cpu\",\n",
|
||||
" \"version\": \"1.6.0+cpu\",\n",
|
||||
" \"support_gpu\": false\n",
|
||||
" },\n",
|
||||
" \"tensorflow\": {\n",
|
||||
" \"version\": \"2.1.0\",\n",
|
||||
" \"git_version\": \"v2.1.0-rc2-17-ge5bf8de410\",\n",
|
||||
" \"version\": \"2.3.0\",\n",
|
||||
" \"git_version\": \"v2.3.0-rc2-23-gb36436b087\",\n",
|
||||
" \"support_gpu\": true\n",
|
||||
" }\n",
|
||||
"}\n"
|
||||
|
|
@ -956,7 +953,7 @@
|
|||
"name": "stderr",
|
||||
"output_type": "stream",
|
||||
"text": [
|
||||
"2020-06-12 11:50:22.559054: I tensorflow/stream_executor/platform/default/dso_loader.cc:44] Successfully opened dynamic library cudart64_101.dll\n"
|
||||
"2020-08-06 17:30:50.400838: I tensorflow/stream_executor/platform/default/dso_loader.cc:48] Successfully opened dynamic library cudart64_101.dll\n"
|
||||
]
|
||||
}
|
||||
],
|
||||
|
|
|
|||
Loading…
Reference in a new issue