From 782c6c24b2d52b2fcf4fcbb129567f30f9649a04 Mon Sep 17 00:00:00 2001 From: Tianlei Wu Date: Thu, 14 May 2020 15:32:59 -0700 Subject: [PATCH] Rename bert to transformers (#3946) * rename folder bert to transformers * rename bert_model_optimization.py to optimizer.py * update URL links in notebooks --- .../test_data_set_0/input_0.pb | Bin 47 -> 0 bytes .../test_data_set_0/output_0.pb | 1 - .../test_data_set_0/input_0.pb | Bin 99 -> 0 bytes .../test_data_set_0/output_0.pb | 3 - .../test_data_set_0/output_1.pb | 2 - .../test_data_set_0/input_1.pb | Bin 100 -> 0 bytes .../test_data_set_0/input_2.pb | Bin 101 -> 0 bytes .../test_data_set_0/input_1.pb | Bin 52 -> 0 bytes .../test_data_set_0/input_2.pb | Bin 52 -> 0 bytes .../{bert => transformers}/BertOnnxModel.py | 0 .../BertOnnxModelKeras.py | 0 .../{bert => transformers}/BertOnnxModelTF.py | 0 .../{bert => transformers}/Gpt2OnnxModel.py | 0 .../{bert => transformers}/MachineInfo.py | 0 .../tools/{bert => transformers}/OnnxModel.py | 0 .../tools/{bert => transformers}/README.md | 6 +- .../{bert => transformers}/ShapeOptimizer.py | 0 .../tools/{bert => transformers}/benchmark.py | 42 +- .../{bert => transformers}/benchmark_gpt2.py | 2 +- .../{bert => transformers}/bert_perf_test.py | 0 .../{bert => transformers}/bert_test_data.py | 0 .../compare_bert_results.py | 0 ...nce_Bert_with_OnnxRuntime_on_AzureML.ipynb | 0 ...ference_GPT2_with_OnnxRuntime_on_CPU.ipynb | 4 +- .../PyTorch_Bert-Squad_OnnxRuntime_CPU.ipynb | 20 +- .../PyTorch_Bert-Squad_OnnxRuntime_GPU.ipynb | 30 +- ...low_Keras_Bert-Squad_OnnxRuntime_CPU.ipynb | 14 +- .../notebooks/images/optimized_bert_gpu.png | Bin .../optimizer.py} | 0 .../requirements_cpu.txt | 0 .../requirements_gpu.txt | 0 .../TFBertForSequenceClassification_1.onnx | Bin .../BertForQuestionAnswering.onnx | Bin .../BertForQuestionAnswering_0.onnx | Bin .../BertForQuestionAnswering_1.onnx | Bin .../test_data_set_0/input_0.pb | Bin .../test_data_set_0/input_1.pb | Bin .../test_data_set_0/input_2.pb | Bin .../test_data_set_0/output_0.pb | 0 .../test_data_set_0/output_1.pb | 0 .../BertForQuestionAnswering.onnx | Bin 0 -> 78606 bytes .../BertForQuestionAnswering_ort_cpu.onnx | 3778 +++++++++++++++++ .../TFBertForQuestionAnswering.onnx | Bin .../generate_tiny_keras2onnx_bert_models.py | 2 +- .../test_data_set_0/input_0.pb | Bin .../test_data_set_0/input_1.pb | Bin .../test_data_set_0/input_2.pb | Bin .../test_data_set_0/output_0.pb | 0 .../test_data_set_0/output_1.pb | 0 .../GPT2Model.onnx | Bin .../generate_tiny_gpt2_model.py | 0 .../test_data_set_0/input_0.pb | Bin .../test_data_set_0/output_0.pb | 0 .../test_optimizer.py} | 27 +- 54 files changed, 3844 insertions(+), 87 deletions(-) delete mode 100644 onnxruntime/python/tools/bert/test_data/bert_mrpc_tensorflow2.1_opset10/test_data_set_0/input_0.pb delete mode 100644 onnxruntime/python/tools/bert/test_data/bert_mrpc_tensorflow2.1_opset10/test_data_set_0/output_0.pb delete mode 100644 onnxruntime/python/tools/bert/test_data/bert_squad_pytorch1.4_opset10_fp32/test_data_set_0/input_0.pb delete mode 100644 onnxruntime/python/tools/bert/test_data/bert_squad_pytorch1.4_opset10_fp32/test_data_set_0/output_0.pb delete mode 100644 onnxruntime/python/tools/bert/test_data/bert_squad_pytorch1.4_opset10_fp32/test_data_set_0/output_1.pb delete mode 100644 onnxruntime/python/tools/bert/test_data/bert_squad_pytorch1.4_opset11/test_data_set_0/input_1.pb delete mode 100644 onnxruntime/python/tools/bert/test_data/bert_squad_pytorch1.4_opset11/test_data_set_0/input_2.pb delete mode 100644 onnxruntime/python/tools/bert/test_data/bert_squad_tensorflow2.1_keras2onnx_opset11/test_data_set_0/input_1.pb delete mode 100644 onnxruntime/python/tools/bert/test_data/bert_squad_tensorflow2.1_keras2onnx_opset11/test_data_set_0/input_2.pb rename onnxruntime/python/tools/{bert => transformers}/BertOnnxModel.py (100%) rename onnxruntime/python/tools/{bert => transformers}/BertOnnxModelKeras.py (100%) rename onnxruntime/python/tools/{bert => transformers}/BertOnnxModelTF.py (100%) rename onnxruntime/python/tools/{bert => transformers}/Gpt2OnnxModel.py (100%) rename onnxruntime/python/tools/{bert => transformers}/MachineInfo.py (100%) rename onnxruntime/python/tools/{bert => transformers}/OnnxModel.py (100%) rename onnxruntime/python/tools/{bert => transformers}/README.md (96%) rename onnxruntime/python/tools/{bert => transformers}/ShapeOptimizer.py (100%) rename onnxruntime/python/tools/{bert => transformers}/benchmark.py (94%) rename onnxruntime/python/tools/{bert => transformers}/benchmark_gpt2.py (99%) rename onnxruntime/python/tools/{bert => transformers}/bert_perf_test.py (100%) rename onnxruntime/python/tools/{bert => transformers}/bert_test_data.py (100%) rename onnxruntime/python/tools/{bert => transformers}/compare_bert_results.py (100%) rename onnxruntime/python/tools/{bert => transformers}/notebooks/Inference_Bert_with_OnnxRuntime_on_AzureML.ipynb (100%) rename onnxruntime/python/tools/{bert => transformers}/notebooks/Inference_GPT2_with_OnnxRuntime_on_CPU.ipynb (99%) rename onnxruntime/python/tools/{bert => transformers}/notebooks/PyTorch_Bert-Squad_OnnxRuntime_CPU.ipynb (97%) rename onnxruntime/python/tools/{bert => transformers}/notebooks/PyTorch_Bert-Squad_OnnxRuntime_GPU.ipynb (97%) rename onnxruntime/python/tools/{bert => transformers}/notebooks/Tensorflow_Keras_Bert-Squad_OnnxRuntime_CPU.ipynb (97%) rename onnxruntime/python/tools/{bert => transformers}/notebooks/images/optimized_bert_gpu.png (100%) rename onnxruntime/python/tools/{bert/bert_model_optimization.py => transformers/optimizer.py} (100%) rename onnxruntime/python/tools/{bert => transformers}/requirements_cpu.txt (100%) rename onnxruntime/python/tools/{bert => transformers}/requirements_gpu.txt (100%) rename onnxruntime/python/tools/{bert => transformers}/test_data/bert_mrpc_tensorflow2.1_opset10/TFBertForSequenceClassification_1.onnx (100%) rename onnxruntime/python/tools/{bert => transformers}/test_data/bert_squad_pytorch1.4_opset10_fp32/BertForQuestionAnswering.onnx (100%) rename onnxruntime/python/tools/{bert => transformers}/test_data/bert_squad_pytorch1.4_opset11/BertForQuestionAnswering_0.onnx (100%) rename onnxruntime/python/tools/{bert => transformers}/test_data/bert_squad_pytorch1.4_opset11/BertForQuestionAnswering_1.onnx (100%) rename onnxruntime/python/tools/{bert => transformers}/test_data/bert_squad_pytorch1.4_opset11/test_data_set_0/input_0.pb (100%) rename onnxruntime/python/tools/{bert/test_data/bert_squad_pytorch1.4_opset10_fp32 => transformers/test_data/bert_squad_pytorch1.4_opset11}/test_data_set_0/input_1.pb (100%) rename onnxruntime/python/tools/{bert/test_data/bert_squad_pytorch1.4_opset10_fp32 => transformers/test_data/bert_squad_pytorch1.4_opset11}/test_data_set_0/input_2.pb (100%) rename onnxruntime/python/tools/{bert => transformers}/test_data/bert_squad_pytorch1.4_opset11/test_data_set_0/output_0.pb (100%) rename onnxruntime/python/tools/{bert => transformers}/test_data/bert_squad_pytorch1.4_opset11/test_data_set_0/output_1.pb (100%) create mode 100644 onnxruntime/python/tools/transformers/test_data/bert_squad_pytorch1.4_opset11_fp16/BertForQuestionAnswering.onnx create mode 100644 onnxruntime/python/tools/transformers/test_data/bert_squad_pytorch1.4_opset11_fp16/BertForQuestionAnswering_ort_cpu.onnx rename onnxruntime/python/tools/{bert => transformers}/test_data/bert_squad_tensorflow2.1_keras2onnx_opset11/TFBertForQuestionAnswering.onnx (100%) rename onnxruntime/python/tools/{bert => transformers}/test_data/bert_squad_tensorflow2.1_keras2onnx_opset11/generate_tiny_keras2onnx_bert_models.py (99%) rename onnxruntime/python/tools/{bert => transformers}/test_data/bert_squad_tensorflow2.1_keras2onnx_opset11/test_data_set_0/input_0.pb (100%) rename onnxruntime/python/tools/{bert/test_data/bert_mrpc_tensorflow2.1_opset10 => transformers/test_data/bert_squad_tensorflow2.1_keras2onnx_opset11}/test_data_set_0/input_1.pb (100%) rename onnxruntime/python/tools/{bert/test_data/bert_mrpc_tensorflow2.1_opset10 => transformers/test_data/bert_squad_tensorflow2.1_keras2onnx_opset11}/test_data_set_0/input_2.pb (100%) rename onnxruntime/python/tools/{bert => transformers}/test_data/bert_squad_tensorflow2.1_keras2onnx_opset11/test_data_set_0/output_0.pb (100%) rename onnxruntime/python/tools/{bert => transformers}/test_data/bert_squad_tensorflow2.1_keras2onnx_opset11/test_data_set_0/output_1.pb (100%) rename onnxruntime/python/tools/{bert => transformers}/test_data/gpt2_pytorch1.4_opset11_no_past/GPT2Model.onnx (100%) rename onnxruntime/python/tools/{bert => transformers}/test_data/gpt2_pytorch1.4_opset11_no_past/generate_tiny_gpt2_model.py (100%) rename onnxruntime/python/tools/{bert => transformers}/test_data/gpt2_pytorch1.4_opset11_no_past/test_data_set_0/input_0.pb (100%) rename onnxruntime/python/tools/{bert => transformers}/test_data/gpt2_pytorch1.4_opset11_no_past/test_data_set_0/output_0.pb (100%) rename onnxruntime/python/tools/{bert/test_bert_optimization.py => transformers/test_optimizer.py} (86%) diff --git a/onnxruntime/python/tools/bert/test_data/bert_mrpc_tensorflow2.1_opset10/test_data_set_0/input_0.pb b/onnxruntime/python/tools/bert/test_data/bert_mrpc_tensorflow2.1_opset10/test_data_set_0/input_0.pb deleted file mode 100644 index 77361435f281cb4d9e325a7e3342754734d14112..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 47 scmd;JMTKrS2r001}!$^ZZW diff --git a/onnxruntime/python/tools/bert/test_data/bert_squad_tensorflow2.1_keras2onnx_opset11/test_data_set_0/input_2.pb b/onnxruntime/python/tools/bert/test_data/bert_squad_tensorflow2.1_keras2onnx_opset11/test_data_set_0/input_2.pb deleted file mode 100644 index 506cd795ba27cf589c851782bb11f7b2886733fa..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 52 hcmd;J&rZ#YFR3g@jn7Oe_L5=10{{RL2F3sY diff --git a/onnxruntime/python/tools/bert/BertOnnxModel.py b/onnxruntime/python/tools/transformers/BertOnnxModel.py similarity index 100% rename from onnxruntime/python/tools/bert/BertOnnxModel.py rename to onnxruntime/python/tools/transformers/BertOnnxModel.py diff --git a/onnxruntime/python/tools/bert/BertOnnxModelKeras.py b/onnxruntime/python/tools/transformers/BertOnnxModelKeras.py similarity index 100% rename from onnxruntime/python/tools/bert/BertOnnxModelKeras.py rename to onnxruntime/python/tools/transformers/BertOnnxModelKeras.py diff --git a/onnxruntime/python/tools/bert/BertOnnxModelTF.py b/onnxruntime/python/tools/transformers/BertOnnxModelTF.py similarity index 100% rename from onnxruntime/python/tools/bert/BertOnnxModelTF.py rename to onnxruntime/python/tools/transformers/BertOnnxModelTF.py diff --git a/onnxruntime/python/tools/bert/Gpt2OnnxModel.py b/onnxruntime/python/tools/transformers/Gpt2OnnxModel.py similarity index 100% rename from onnxruntime/python/tools/bert/Gpt2OnnxModel.py rename to onnxruntime/python/tools/transformers/Gpt2OnnxModel.py diff --git a/onnxruntime/python/tools/bert/MachineInfo.py b/onnxruntime/python/tools/transformers/MachineInfo.py similarity index 100% rename from onnxruntime/python/tools/bert/MachineInfo.py rename to onnxruntime/python/tools/transformers/MachineInfo.py diff --git a/onnxruntime/python/tools/bert/OnnxModel.py b/onnxruntime/python/tools/transformers/OnnxModel.py similarity index 100% rename from onnxruntime/python/tools/bert/OnnxModel.py rename to onnxruntime/python/tools/transformers/OnnxModel.py diff --git a/onnxruntime/python/tools/bert/README.md b/onnxruntime/python/tools/transformers/README.md similarity index 96% rename from onnxruntime/python/tools/bert/README.md rename to onnxruntime/python/tools/transformers/README.md index 5399fa08fc..cf6a318c38 100644 --- a/onnxruntime/python/tools/bert/README.md +++ b/onnxruntime/python/tools/transformers/README.md @@ -43,14 +43,14 @@ For tf2onnx, please refer to this notebook: https://github.com/onnx/tensorflow-o ## Model Optimization -Example of using the script bert_model_optimization.py to convert a BERT-large model to run in V100 GPU: +Example of using the script optimizer.py to convert a BERT-large model to run in V100 GPU: ```console -python bert_model_optimization.py --input original_model.onnx --output optimized_model_gpu.onnx --num_heads 16 --hidden_size 1024 --input_int32 --float16 +python optimizer.py --input original_model.onnx --output optimized_model_gpu.onnx --num_heads 16 --hidden_size 1024 --input_int32 --float16 ``` ### Options -See below for description of some options of bert_model_optimization.py: +See below for description of some options of optimizer.py: - **input**: input model path - **output**: output model path diff --git a/onnxruntime/python/tools/bert/ShapeOptimizer.py b/onnxruntime/python/tools/transformers/ShapeOptimizer.py similarity index 100% rename from onnxruntime/python/tools/bert/ShapeOptimizer.py rename to onnxruntime/python/tools/transformers/ShapeOptimizer.py diff --git a/onnxruntime/python/tools/bert/benchmark.py b/onnxruntime/python/tools/transformers/benchmark.py similarity index 94% rename from onnxruntime/python/tools/bert/benchmark.py rename to onnxruntime/python/tools/transformers/benchmark.py index 92dd059587..500b21751a 100644 --- a/onnxruntime/python/tools/bert/benchmark.py +++ b/onnxruntime/python/tools/transformers/benchmark.py @@ -160,22 +160,22 @@ optimize_model_statistics = {} def optimize_onnx_model(onnx_model_filename, model_type, num_attention_heads, hidden_size, fp16): optimized_model_filename = onnx_model_filename.replace(".onnx", "_fp16.onnx" if fp16 else "_fp32.onnx") if not os.path.exists(optimized_model_filename): - import bert_model_optimization as bert_opt + from optimizer import optimize_model # Use onnxruntime to optimize model, which will be saved to *_ort_cpu.onnx - opt_model = bert_opt.optimize_model(onnx_model_filename, - model_type, - num_heads=num_attention_heads, - hidden_size=hidden_size, - opt_level=99, - only_onnxruntime=True) + opt_model = optimize_model(onnx_model_filename, + model_type, + num_heads=num_attention_heads, + hidden_size=hidden_size, + opt_level=99, + only_onnxruntime=True) optimize_model_statistics[onnx_model_filename] = opt_model.get_fused_operator_statistics() # Use script to optimize model. - opt_model = bert_opt.optimize_model(onnx_model_filename, - model_type, - num_heads=num_attention_heads, - hidden_size=hidden_size, - opt_level=0) + opt_model = optimize_model(onnx_model_filename, + model_type, + num_heads=num_attention_heads, + hidden_size=hidden_size, + opt_level=0) optimize_model_statistics[optimized_model_filename] = opt_model.get_fused_operator_statistics() if fp16: @@ -210,14 +210,14 @@ def export_onnx_model(model_name, cache_dir, input_names, fp16, optimize_onnx, v dynamic_axes, output_names = build_dynamic_axes(example_inputs, example_outputs_flatten) torch.onnx.export(model=model, - args=tuple(example_inputs.values()), - f=onnx_model_filename, - input_names=list(example_inputs.keys()), - output_names=output_names, - example_outputs=example_outputs, - dynamic_axes=dynamic_axes, - do_constant_folding=True, - opset_version=MODELS[model_name][1]) + args=tuple(example_inputs.values()), + f=onnx_model_filename, + input_names=list(example_inputs.keys()), + output_names=output_names, + example_outputs=example_outputs, + dynamic_axes=dynamic_axes, + do_constant_folding=True, + opset_version=MODELS[model_name][1]) else: logger.info(f"Skip export since model existed: {onnx_model_filename}") @@ -481,7 +481,7 @@ def parse_arguments(): "--optimize_onnx", required=False, action="store_true", - help="Use bert_model_optimization.py to optimize onnx model") + help="Use optimizer.py to optimize onnx model") parser.add_argument("-v", "--validate_onnx", required=False, action="store_true", help="Validate ONNX model") diff --git a/onnxruntime/python/tools/bert/benchmark_gpt2.py b/onnxruntime/python/tools/transformers/benchmark_gpt2.py similarity index 99% rename from onnxruntime/python/tools/bert/benchmark_gpt2.py rename to onnxruntime/python/tools/transformers/benchmark_gpt2.py index 700c56eb3e..43eb2fe048 100644 --- a/onnxruntime/python/tools/bert/benchmark_gpt2.py +++ b/onnxruntime/python/tools/transformers/benchmark_gpt2.py @@ -232,7 +232,7 @@ def main(): remove_past_outputs(export_model_path, onnx_model_path) if args.enable_optimization: - from bert_model_optimization import optimize_model + from optimizer import optimize_model m = optimize_model(onnx_model_path, model_type='gpt2', num_heads=12, diff --git a/onnxruntime/python/tools/bert/bert_perf_test.py b/onnxruntime/python/tools/transformers/bert_perf_test.py similarity index 100% rename from onnxruntime/python/tools/bert/bert_perf_test.py rename to onnxruntime/python/tools/transformers/bert_perf_test.py diff --git a/onnxruntime/python/tools/bert/bert_test_data.py b/onnxruntime/python/tools/transformers/bert_test_data.py similarity index 100% rename from onnxruntime/python/tools/bert/bert_test_data.py rename to onnxruntime/python/tools/transformers/bert_test_data.py diff --git a/onnxruntime/python/tools/bert/compare_bert_results.py b/onnxruntime/python/tools/transformers/compare_bert_results.py similarity index 100% rename from onnxruntime/python/tools/bert/compare_bert_results.py rename to onnxruntime/python/tools/transformers/compare_bert_results.py diff --git a/onnxruntime/python/tools/bert/notebooks/Inference_Bert_with_OnnxRuntime_on_AzureML.ipynb b/onnxruntime/python/tools/transformers/notebooks/Inference_Bert_with_OnnxRuntime_on_AzureML.ipynb similarity index 100% rename from onnxruntime/python/tools/bert/notebooks/Inference_Bert_with_OnnxRuntime_on_AzureML.ipynb rename to onnxruntime/python/tools/transformers/notebooks/Inference_Bert_with_OnnxRuntime_on_AzureML.ipynb diff --git a/onnxruntime/python/tools/bert/notebooks/Inference_GPT2_with_OnnxRuntime_on_CPU.ipynb b/onnxruntime/python/tools/transformers/notebooks/Inference_GPT2_with_OnnxRuntime_on_CPU.ipynb similarity index 99% rename from onnxruntime/python/tools/bert/notebooks/Inference_GPT2_with_OnnxRuntime_on_CPU.ipynb rename to onnxruntime/python/tools/transformers/notebooks/Inference_GPT2_with_OnnxRuntime_on_CPU.ipynb index e7e244ce67..2a97343fd7 100644 --- a/onnxruntime/python/tools/bert/notebooks/Inference_GPT2_with_OnnxRuntime_on_CPU.ipynb +++ b/onnxruntime/python/tools/transformers/notebooks/Inference_GPT2_with_OnnxRuntime_on_CPU.ipynb @@ -356,7 +356,7 @@ "metadata": {}, "outputs": [], "source": [ - "bert_opt_script = os.path.join(bert_tools_dir, 'bert_model_optimization.py')" + "bert_opt_script = os.path.join(bert_tools_dir, 'optimizer.py')" ] }, { @@ -365,7 +365,7 @@ "metadata": {}, "outputs": [], "source": [ - "# Local directory corresponding to https://github.com/microsoft/onnxruntime/tree/master/onnxruntime/python/tools/bert/\n", + "# Local directory corresponding to https://github.com/microsoft/onnxruntime/tree/master/onnxruntime/python/tools/transformers/\n", "%run $bert_opt_script --model_type gpt2 --input $onnx_model_path --output $optimized_model --opt_level 0" ] }, diff --git a/onnxruntime/python/tools/bert/notebooks/PyTorch_Bert-Squad_OnnxRuntime_CPU.ipynb b/onnxruntime/python/tools/transformers/notebooks/PyTorch_Bert-Squad_OnnxRuntime_CPU.ipynb similarity index 97% rename from onnxruntime/python/tools/bert/notebooks/PyTorch_Bert-Squad_OnnxRuntime_CPU.ipynb rename to onnxruntime/python/tools/transformers/notebooks/PyTorch_Bert-Squad_OnnxRuntime_CPU.ipynb index d7f1bd6fdf..82e5bc6e60 100644 --- a/onnxruntime/python/tools/bert/notebooks/PyTorch_Bert-Squad_OnnxRuntime_CPU.ipynb +++ b/onnxruntime/python/tools/transformers/notebooks/PyTorch_Bert-Squad_OnnxRuntime_CPU.ipynb @@ -457,10 +457,10 @@ "source": [ "## 5. Offline Optimization Script and Test Tools\n", "\n", - "It is recommended to download the [OnnxRuntime Python Tools for BERT](https://github.com/microsoft/onnxruntime/tree/master/onnxruntime/python/tools/bert), and try them on the exported ONNX models. It could help verify whether the model is fully optimized, and get performance test results.\n", + "It is recommended to download the [OnnxRuntime Python Tools for BERT](https://github.com/microsoft/onnxruntime/tree/master/onnxruntime/python/tools/transformers), and try them on the exported ONNX models. It could help verify whether the model is fully optimized, and get performance test results.\n", "\n", "### Download OnnxRuntime Python Tools for Bert\n", - "You may copy the whole [directory](https://github.com/microsoft/onnxruntime/tree/master/onnxruntime/python/tools/bert) to a sub-directory named bert_scripts for this notebook. The list of script files might need update if import error happens when you run some script." + "You may copy the whole [directory](https://github.com/microsoft/onnxruntime/tree/master/onnxruntime/python/tools/transformers) to a sub-directory named bert_scripts for this notebook. The list of script files might need update if import error happens when you run some script." ] }, { @@ -479,7 +479,7 @@ "100% [..............................................................................] 21565 / 21565Downloaded BertOnnxModelKeras.py\n", "100% [..............................................................................] 26114 / 26114Downloaded BertOnnxModelTF.py\n", "100% [..............................................................................] 22773 / 22773Downloaded OnnxModel.py\n", - "100% [................................................................................] 7917 / 7917Downloaded bert_model_optimization.py\n", + "100% [................................................................................] 7917 / 7917Downloaded optimizer.py\n", "100% [................................................................................] 5478 / 5478Downloaded MachineInfo.py\n" ] } @@ -488,8 +488,8 @@ "import os\n", "import wget\n", "\n", - "url_prfix = \"https://raw.githubusercontent.com/microsoft/onnxruntime/master/onnxruntime/python/tools/bert/\"\n", - "script_files = ['bert_perf_test.py', 'bert_test_data.py', 'compare_bert_results.py', 'BertOnnxModel.py', 'BertOnnxModelKeras.py', 'BertOnnxModelTF.py', 'Gpt2OnnxModel.py', 'OnnxModel.py', 'bert_model_optimization.py', 'MachineInfo.py']\n", + "url_prfix = \"https://raw.githubusercontent.com/microsoft/onnxruntime/master/onnxruntime/python/tools/transformers/\"\n", + "script_files = ['bert_perf_test.py', 'bert_test_data.py', 'compare_bert_results.py', 'BertOnnxModel.py', 'BertOnnxModelKeras.py', 'BertOnnxModelTF.py', 'Gpt2OnnxModel.py', 'OnnxModel.py', 'optimizer.py', 'MachineInfo.py']\n", "\n", "script_dir = './bert_scripts'\n", "if not os.path.exists(script_dir):\n", @@ -515,7 +515,7 @@ "* The exported model uses dynamic axis and this makes it harder for shape inference of the graph. That blocks some optimization to be applied.\n", "* Some optimization is better to be done offline. Like change input tensor type from int64 to int32 to avoid extra Cast nodes, or convert model to float16 to achieve better performance in V100 or T4 GPU.\n", "\n", - "We have python script **bert_model_optimization.py**, which is more flexible in graph pattern matching and model conversion (like float32 to float16). You can also use it to verify whether a Bert model is fully optimized.\n", + "We have python script **optimizer.py**, which is more flexible in graph pattern matching and model conversion (like float32 to float16). You can also use it to verify whether a Bert model is fully optimized.\n", "\n", "In this example, we can see that it introduces optimization that is not provided by onnxruntime: SkipLayerNormalization and bias fusion, which is not fused in OnnxRuntime due to shape inference as mentioned.\n", "\n", @@ -531,8 +531,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "bert_model_optimization.py: Save optimized model by onnxruntime to ./onnx\\bert-base-cased-squad_ort_cpu.onnx\n", - "bert_model_optimization.py: Use OnnxRuntime to optimize and save the optimized model to ./onnx\\bert-base-cased-squad_ort_cpu.onnx\n", + "optimizer.py: Save optimized model by onnxruntime to ./onnx\\bert-base-cased-squad_ort_cpu.onnx\n", + "optimizer.py: Use OnnxRuntime to optimize and save the optimized model to ./onnx\\bert-base-cased-squad_ort_cpu.onnx\n", " BertOnnxModel.py: Fused LayerNormalization count: 0\n", " BertOnnxModel.py: Fused Reshape count:0\n", " BertOnnxModel.py: Fused SkipLayerNormalization count: 24\n", @@ -542,13 +542,13 @@ " BertOnnxModel.py: opset verion: 11\n", " OnnxModel.py: Output model to ./onnx/bert-base-cased-squad_opt_cpu.onnx\n", " BertOnnxModel.py: EmbedLayer=1, Attention=12, Gelu=12, LayerNormalization=24, Succesful=True\n", - "bert_model_optimization.py: The output model is fully optimized.\n" + "optimizer.py: The output model is fully optimized.\n" ] } ], "source": [ "optimized_model_path = './onnx/bert-base-cased-squad_opt_cpu.onnx'\n", - "%run ./bert_scripts/bert_model_optimization.py --input $export_model_path --output $optimized_model_path" + "%run ./bert_scripts/optimizer.py --input $export_model_path --output $optimized_model_path" ] }, { diff --git a/onnxruntime/python/tools/bert/notebooks/PyTorch_Bert-Squad_OnnxRuntime_GPU.ipynb b/onnxruntime/python/tools/transformers/notebooks/PyTorch_Bert-Squad_OnnxRuntime_GPU.ipynb similarity index 97% rename from onnxruntime/python/tools/bert/notebooks/PyTorch_Bert-Squad_OnnxRuntime_GPU.ipynb rename to onnxruntime/python/tools/transformers/notebooks/PyTorch_Bert-Squad_OnnxRuntime_GPU.ipynb index 4fa129d4e3..6ef81a2dbb 100644 --- a/onnxruntime/python/tools/bert/notebooks/PyTorch_Bert-Squad_OnnxRuntime_GPU.ipynb +++ b/onnxruntime/python/tools/transformers/notebooks/PyTorch_Bert-Squad_OnnxRuntime_GPU.ipynb @@ -576,10 +576,10 @@ "source": [ "## 5. Offline Optimization and Test Tools\n", "\n", - "It is recommended to download the [OnnxRuntime Python Tools for BERT](https://github.com/microsoft/onnxruntime/tree/master/onnxruntime/python/tools/bert), and try them on the exported ONNX models. It could help verify whether the model is fully optimized, and get performance test results.\n", + "It is recommended to download the [OnnxRuntime Python Tools for BERT](https://github.com/microsoft/onnxruntime/tree/master/onnxruntime/python/tools/transformers), and try them on the exported ONNX models. It could help verify whether the model is fully optimized, and get performance test results.\n", "\n", "### Download OnnxRuntime Python Tools for Bert\n", - "You may copy the whole [directory](https://github.com/microsoft/onnxruntime/tree/master/onnxruntime/python/tools/bert) to a sub-directory named bert_scripts for this notebook. The list of script files might need update if import error happens when you run some script." + "You may copy the whole [directory](https://github.com/microsoft/onnxruntime/tree/master/onnxruntime/python/tools/transformers) to a sub-directory named bert_scripts for this notebook. The list of script files might need update if import error happens when you run some script." ] }, { @@ -598,7 +598,7 @@ "100% [..............................................................................] 21565 / 21565Downloaded BertOnnxModelKeras.py\n", "100% [..............................................................................] 26114 / 26114Downloaded BertOnnxModelTF.py\n", "100% [..............................................................................] 22773 / 22773Downloaded OnnxModel.py\n", - "100% [................................................................................] 7795 / 7795Downloaded bert_model_optimization.py\n", + "100% [................................................................................] 7795 / 7795Downloaded optimizer.py\n", "100% [................................................................................] 5885 / 5885Downloaded MachineInfo.py\n" ] } @@ -607,8 +607,8 @@ "import os\n", "import wget\n", "\n", - "url_prfix = \"https://raw.githubusercontent.com/microsoft/onnxruntime/master/onnxruntime/python/tools/bert/\"\n", - "script_files = ['bert_perf_test.py', 'bert_test_data.py', 'compare_bert_results.py', 'BertOnnxModel.py', 'BertOnnxModelKeras.py', 'BertOnnxModelTF.py', 'Gpt2OnnxModel.py', 'OnnxModel.py', 'bert_model_optimization.py', 'MachineInfo.py']\n", + "url_prfix = \"https://raw.githubusercontent.com/microsoft/onnxruntime/master/onnxruntime/python/tools/transformers/\"\n", + "script_files = ['bert_perf_test.py', 'bert_test_data.py', 'compare_bert_results.py', 'BertOnnxModel.py', 'BertOnnxModelKeras.py', 'BertOnnxModelTF.py', 'Gpt2OnnxModel.py', 'OnnxModel.py', 'optimizer.py', 'MachineInfo.py']\n", "\n", "script_dir = './bert_scripts'\n", "if not os.path.exists(script_dir):\n", @@ -634,7 +634,7 @@ "* The exported model uses dynamic axis. That impacts shape inference. Without enough shape information, some optimization cannot be applied due to the constraint on the input shape.\n", "* Some optimization are not supported by OnnxRuntime, but it is feasible in offline script. Like changing input tensor type from int64 to int32 to avoid extra Cast nodes, or converting model to float16 to achieve better performance in V100 or T4 GPU.\n", "\n", - "We have python script **bert_model_optimization.py**, which is flexible in graph pattern matching and model conversions to tackle these problems.\n", + "We have python script **optimizer.py**, which is flexible in graph pattern matching and model conversions to tackle these problems.\n", "\n", "In below example, we can see that the tool provide an extra optimization - SkipLayerNormalization and bias (Add) are not fused in OnnxRuntime due to shape inference.\n", "\n", @@ -660,8 +660,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "bert_model_optimization.py: Save optimized model by onnxruntime to ./onnx\\bert-base-cased-squad_opset11_ort_gpu.onnx\n", - "bert_model_optimization.py: Use OnnxRuntime to optimize and save the optimized model to ./onnx\\bert-base-cased-squad_opset11_ort_gpu.onnx\n", + "optimizer.py: Save optimized model by onnxruntime to ./onnx\\bert-base-cased-squad_opset11_ort_gpu.onnx\n", + "optimizer.py: Use OnnxRuntime to optimize and save the optimized model to ./onnx\\bert-base-cased-squad_opset11_ort_gpu.onnx\n", " BertOnnxModel.py: Fused LayerNormalization count: 0\n", " BertOnnxModel.py: Fused Reshape count:0\n", " BertOnnxModel.py: Fused SkipLayerNormalization count: 24\n", @@ -671,13 +671,13 @@ " BertOnnxModel.py: opset verion: 11\n", " OnnxModel.py: Output model to ./onnx/bert-base-cased-squad_opt_gpu_fp32.onnx\n", " BertOnnxModel.py: EmbedLayer=1, Attention=12, Gelu=12, LayerNormalization=24, Succesful=True\n", - "bert_model_optimization.py: The output model is fully optimized.\n" + "optimizer.py: The output model is fully optimized.\n" ] } ], "source": [ "optimized_fp32_model_path = './onnx/bert-base-cased-squad_opt_{}_fp32.onnx'.format('gpu' if use_gpu else 'cpu')\n", - "%run ./bert_scripts/bert_model_optimization.py --input $export_model_path --output $optimized_fp32_model_path --input_int32" + "%run ./bert_scripts/optimizer.py --input $export_model_path --output $optimized_fp32_model_path --input_int32" ] }, { @@ -1117,7 +1117,7 @@ "source": [ "## 6. Model Optimization with Float16\n", "\n", - "The bert_model_optimization.py script have an option **--float16** to convert model to use float16 to store weights. After the conversion, it could be faster to run in GPU with tensor cores like V100 or T4.\n", + "The optimizer.py script have an option **--float16** to convert model to use float16 to store weights. After the conversion, it could be faster to run in GPU with tensor cores like V100 or T4.\n", "\n", "Let's run tools to measure the performance on V100. The results show significant performance improvement: latency is about 3.4 ms for float32 model, and 1.8 ms for float16 model." ] @@ -1131,8 +1131,8 @@ "name": "stdout", "output_type": "stream", "text": [ - "bert_model_optimization.py: Save optimized model by onnxruntime to ./onnx\\bert-base-cased-squad_opset11_ort_gpu.onnx\n", - "bert_model_optimization.py: Use OnnxRuntime to optimize and save the optimized model to ./onnx\\bert-base-cased-squad_opset11_ort_gpu.onnx\n", + "optimizer.py: Save optimized model by onnxruntime to ./onnx\\bert-base-cased-squad_opset11_ort_gpu.onnx\n", + "optimizer.py: Use OnnxRuntime to optimize and save the optimized model to ./onnx\\bert-base-cased-squad_opset11_ort_gpu.onnx\n", " BertOnnxModel.py: Fused LayerNormalization count: 0\n", " BertOnnxModel.py: Fused Reshape count:0\n", " BertOnnxModel.py: Fused SkipLayerNormalization count: 24\n", @@ -1142,13 +1142,13 @@ " BertOnnxModel.py: opset verion: 11\n", " OnnxModel.py: Output model to ./onnx/bert-base-cased-squad_opt_gpu_fp16.onnx\n", " BertOnnxModel.py: EmbedLayer=1, Attention=12, Gelu=12, LayerNormalization=24, Succesful=True\n", - "bert_model_optimization.py: The output model is fully optimized.\n" + "optimizer.py: The output model is fully optimized.\n" ] } ], "source": [ "optimized_fp16_model_path = './onnx/bert-base-cased-squad_opt_{}_fp16.onnx'.format('gpu' if use_gpu else 'cpu')\n", - "%run ./bert_scripts/bert_model_optimization.py --input $export_model_path --output $optimized_fp16_model_path --float16 --input_int32" + "%run ./bert_scripts/optimizer.py --input $export_model_path --output $optimized_fp16_model_path --float16 --input_int32" ] }, { diff --git a/onnxruntime/python/tools/bert/notebooks/Tensorflow_Keras_Bert-Squad_OnnxRuntime_CPU.ipynb b/onnxruntime/python/tools/transformers/notebooks/Tensorflow_Keras_Bert-Squad_OnnxRuntime_CPU.ipynb similarity index 97% rename from onnxruntime/python/tools/bert/notebooks/Tensorflow_Keras_Bert-Squad_OnnxRuntime_CPU.ipynb rename to onnxruntime/python/tools/transformers/notebooks/Tensorflow_Keras_Bert-Squad_OnnxRuntime_CPU.ipynb index 3d084324f1..76b9d45b90 100644 --- a/onnxruntime/python/tools/bert/notebooks/Tensorflow_Keras_Bert-Squad_OnnxRuntime_CPU.ipynb +++ b/onnxruntime/python/tools/transformers/notebooks/Tensorflow_Keras_Bert-Squad_OnnxRuntime_CPU.ipynb @@ -96,7 +96,7 @@ "100% [..............................................................................] 21565 / 21565Downloaded BertOnnxModelKeras.py\n", "100% [..............................................................................] 26114 / 26114Downloaded BertOnnxModelTF.py\n", "100% [..............................................................................] 22773 / 22773Downloaded OnnxModel.py\n", - "100% [................................................................................] 7795 / 7795Downloaded bert_model_optimization.py\n" + "100% [................................................................................] 7795 / 7795Downloaded optimizer.py\n" ] } ], @@ -113,8 +113,8 @@ " os.makedirs(directory)\n", "\n", "# Download scripts for BERT optimization.\n", - "url_prfix = \"https://raw.githubusercontent.com/microsoft/onnxruntime/master/onnxruntime/python/tools/bert/\"\n", - "script_files = ['bert_perf_test.py', 'bert_test_data.py', 'compare_bert_results.py', 'BertOnnxModel.py', 'BertOnnxModelKeras.py', 'BertOnnxModelTF.py', 'Gpt2OnnxModel.py', 'OnnxModel.py', 'bert_model_optimization.py']\n", + "url_prfix = \"https://raw.githubusercontent.com/microsoft/onnxruntime/master/onnxruntime/python/tools/transformers/\"\n", + "script_files = ['bert_perf_test.py', 'bert_test_data.py', 'compare_bert_results.py', 'BertOnnxModel.py', 'BertOnnxModelKeras.py', 'BertOnnxModelTF.py', 'Gpt2OnnxModel.py', 'OnnxModel.py', 'optimizer.py']\n", "\n", "for filename in script_files:\n", " target_file = os.path.join(script_dir, filename)\n", @@ -378,7 +378,7 @@ "source": [ "## 5. Model Optimization\n", "\n", - "[ONNX Runtime BERT Model Optimization Tools](https://github.com/microsoft/onnxruntime/tree/master/onnxruntime/python/tools/bert) is a set of tools for optimizing and testing BERT models. Let's try some of them on the exported models." + "[ONNX Runtime BERT Model Optimization Tools](https://github.com/microsoft/onnxruntime/tree/master/onnxruntime/python/tools/transformers) is a set of tools for optimizing and testing BERT models. Let's try some of them on the exported models." ] }, { @@ -387,7 +387,7 @@ "source": [ "### BERT Optimization Script\n", "\n", - "The script **bert_model_optimization.py** can help optimize BERT model exported by PyTorch, tf2onnx or keras2onnx. Since our model is exported by keras2onnx, we shall use **--model_type bert_keras** parameter.\n", + "The script **optimizer.py** can help optimize BERT model exported by PyTorch, tf2onnx or keras2onnx. Since our model is exported by keras2onnx, we shall use **--model_type bert_keras** parameter.\n", "\n", "It will also tell whether the model is fully optimized or not. If not, that means you might need change the script to fuse some new pattern of subgraph." ] @@ -420,14 +420,14 @@ " BertOnnxModel.py: opset verion: 11\n", " OnnxModel.py: Output model to ./output\\keras_bert_large_opt_cpu.onnx\n", " BertOnnxModel.py: EmbedLayer=1, Attention=24, Gelu=24, LayerNormalization=48, Succesful=True\n", - "bert_model_optimization.py: The output model is fully optimized.\n" + "optimizer.py: The output model is fully optimized.\n" ] } ], "source": [ "optimized_model_path = os.path.join(output_dir, 'keras_bert_large_opt_cpu.onnx')\n", "\n", - "%run bert_scripts/bert_model_optimization.py --input $output_model_path --output $optimized_model_path --model_type bert_keras --num_heads 16 --hidden_size 1024" + "%run bert_scripts/optimizer.py --input $output_model_path --output $optimized_model_path --model_type bert_keras --num_heads 16 --hidden_size 1024" ] }, { diff --git a/onnxruntime/python/tools/bert/notebooks/images/optimized_bert_gpu.png b/onnxruntime/python/tools/transformers/notebooks/images/optimized_bert_gpu.png similarity index 100% rename from onnxruntime/python/tools/bert/notebooks/images/optimized_bert_gpu.png rename to onnxruntime/python/tools/transformers/notebooks/images/optimized_bert_gpu.png diff --git a/onnxruntime/python/tools/bert/bert_model_optimization.py b/onnxruntime/python/tools/transformers/optimizer.py similarity index 100% rename from onnxruntime/python/tools/bert/bert_model_optimization.py rename to onnxruntime/python/tools/transformers/optimizer.py diff --git a/onnxruntime/python/tools/bert/requirements_cpu.txt b/onnxruntime/python/tools/transformers/requirements_cpu.txt similarity index 100% rename from onnxruntime/python/tools/bert/requirements_cpu.txt rename to onnxruntime/python/tools/transformers/requirements_cpu.txt diff --git a/onnxruntime/python/tools/bert/requirements_gpu.txt b/onnxruntime/python/tools/transformers/requirements_gpu.txt similarity index 100% rename from onnxruntime/python/tools/bert/requirements_gpu.txt rename to onnxruntime/python/tools/transformers/requirements_gpu.txt diff --git a/onnxruntime/python/tools/bert/test_data/bert_mrpc_tensorflow2.1_opset10/TFBertForSequenceClassification_1.onnx b/onnxruntime/python/tools/transformers/test_data/bert_mrpc_tensorflow2.1_opset10/TFBertForSequenceClassification_1.onnx similarity index 100% rename from onnxruntime/python/tools/bert/test_data/bert_mrpc_tensorflow2.1_opset10/TFBertForSequenceClassification_1.onnx rename to onnxruntime/python/tools/transformers/test_data/bert_mrpc_tensorflow2.1_opset10/TFBertForSequenceClassification_1.onnx diff --git a/onnxruntime/python/tools/bert/test_data/bert_squad_pytorch1.4_opset10_fp32/BertForQuestionAnswering.onnx b/onnxruntime/python/tools/transformers/test_data/bert_squad_pytorch1.4_opset10_fp32/BertForQuestionAnswering.onnx similarity index 100% rename from onnxruntime/python/tools/bert/test_data/bert_squad_pytorch1.4_opset10_fp32/BertForQuestionAnswering.onnx rename to onnxruntime/python/tools/transformers/test_data/bert_squad_pytorch1.4_opset10_fp32/BertForQuestionAnswering.onnx diff --git a/onnxruntime/python/tools/bert/test_data/bert_squad_pytorch1.4_opset11/BertForQuestionAnswering_0.onnx b/onnxruntime/python/tools/transformers/test_data/bert_squad_pytorch1.4_opset11/BertForQuestionAnswering_0.onnx similarity index 100% rename from onnxruntime/python/tools/bert/test_data/bert_squad_pytorch1.4_opset11/BertForQuestionAnswering_0.onnx rename to onnxruntime/python/tools/transformers/test_data/bert_squad_pytorch1.4_opset11/BertForQuestionAnswering_0.onnx diff --git a/onnxruntime/python/tools/bert/test_data/bert_squad_pytorch1.4_opset11/BertForQuestionAnswering_1.onnx b/onnxruntime/python/tools/transformers/test_data/bert_squad_pytorch1.4_opset11/BertForQuestionAnswering_1.onnx similarity index 100% rename from onnxruntime/python/tools/bert/test_data/bert_squad_pytorch1.4_opset11/BertForQuestionAnswering_1.onnx rename to onnxruntime/python/tools/transformers/test_data/bert_squad_pytorch1.4_opset11/BertForQuestionAnswering_1.onnx diff --git a/onnxruntime/python/tools/bert/test_data/bert_squad_pytorch1.4_opset11/test_data_set_0/input_0.pb b/onnxruntime/python/tools/transformers/test_data/bert_squad_pytorch1.4_opset11/test_data_set_0/input_0.pb similarity index 100% rename from onnxruntime/python/tools/bert/test_data/bert_squad_pytorch1.4_opset11/test_data_set_0/input_0.pb rename to onnxruntime/python/tools/transformers/test_data/bert_squad_pytorch1.4_opset11/test_data_set_0/input_0.pb diff --git a/onnxruntime/python/tools/bert/test_data/bert_squad_pytorch1.4_opset10_fp32/test_data_set_0/input_1.pb b/onnxruntime/python/tools/transformers/test_data/bert_squad_pytorch1.4_opset11/test_data_set_0/input_1.pb similarity index 100% rename from onnxruntime/python/tools/bert/test_data/bert_squad_pytorch1.4_opset10_fp32/test_data_set_0/input_1.pb rename to onnxruntime/python/tools/transformers/test_data/bert_squad_pytorch1.4_opset11/test_data_set_0/input_1.pb diff --git a/onnxruntime/python/tools/bert/test_data/bert_squad_pytorch1.4_opset10_fp32/test_data_set_0/input_2.pb b/onnxruntime/python/tools/transformers/test_data/bert_squad_pytorch1.4_opset11/test_data_set_0/input_2.pb similarity index 100% rename from onnxruntime/python/tools/bert/test_data/bert_squad_pytorch1.4_opset10_fp32/test_data_set_0/input_2.pb rename to onnxruntime/python/tools/transformers/test_data/bert_squad_pytorch1.4_opset11/test_data_set_0/input_2.pb diff --git a/onnxruntime/python/tools/bert/test_data/bert_squad_pytorch1.4_opset11/test_data_set_0/output_0.pb b/onnxruntime/python/tools/transformers/test_data/bert_squad_pytorch1.4_opset11/test_data_set_0/output_0.pb similarity index 100% rename from onnxruntime/python/tools/bert/test_data/bert_squad_pytorch1.4_opset11/test_data_set_0/output_0.pb rename to onnxruntime/python/tools/transformers/test_data/bert_squad_pytorch1.4_opset11/test_data_set_0/output_0.pb diff --git a/onnxruntime/python/tools/bert/test_data/bert_squad_pytorch1.4_opset11/test_data_set_0/output_1.pb b/onnxruntime/python/tools/transformers/test_data/bert_squad_pytorch1.4_opset11/test_data_set_0/output_1.pb similarity index 100% rename from onnxruntime/python/tools/bert/test_data/bert_squad_pytorch1.4_opset11/test_data_set_0/output_1.pb rename to onnxruntime/python/tools/transformers/test_data/bert_squad_pytorch1.4_opset11/test_data_set_0/output_1.pb diff --git a/onnxruntime/python/tools/transformers/test_data/bert_squad_pytorch1.4_opset11_fp16/BertForQuestionAnswering.onnx b/onnxruntime/python/tools/transformers/test_data/bert_squad_pytorch1.4_opset11_fp16/BertForQuestionAnswering.onnx new file mode 100644 index 0000000000000000000000000000000000000000..17496ef04b77f6ae242af9d5315592a557f13830 GIT binary patch literal 78606 zcmb4McbwGJ_NOdu0ci_t-zmvVl9^q0b~;!PQ4zZcHWU|F5$UihGL8lSyVancd(1krv? z`sEEv#&!>N?^9?g#DAlVS5Kw~%9Npu7iD~wf>Y|3G;;k4do(U6E(~?=QE0{IeoNm| z7R+DLSUT zT>h+P?t{f3^$Hkj)7FcKYMV8?3A6fDAh*cWwmcp;g2EWN>=X4E(4_5}!FkJX{Y)_E34cXRHm+6qgQwv?W z87*9}WOn211@nzF!=hk${7SzTo?Gb7&8Bg|oQC<+8dofAFwUs|l7?CH8s;}nOIdYY zAd?|FP@NAie$*7S^gMayOyX8`L3|p3r*7~KeCmV;ZtOR=%Tjnj!_1{K8qRL0pU)Mg z{r+#|j|>2qv%3~AE=*FVx~AK$F}EDA9;>*f8#~R7o#ysfdY-pnSxPQSa8GR^ZJcuY?D{2Xo5FOiRLQC^ovX0F%jfcI{tF|_ z+4YS$e4T~@?m}-Wf=qD?hq_4+KxAC zK=*ciK=7*;i{Q zgMR5n2lv$$7Px$MsVh3F4{E13L>7av2g~O}^*)cK&jk%j$jr##Lu^|g%=8y7uAjdI zMz#iSNMXUkhQ;&7b{pGcZ1=G}aZo2MgpTF&X(v0Mc9zecQ?%0$4&^@d93ROdGlpfi zkg6l7)#0zz3s3Q*up*>3H-YFCo_Thu|fU1_=m_~FZi)d z{a(@wb>xce>xbGI_5x0+exg38_5#-1i-iurJgY*MfXlGnxosQ3;smhBfja$qt2*3S zIy{Cta@#iGMjdD@0hnaidU z2%;|p0r%pBgi~&ipe{6oV0tn);ZrD+ySCQSXVQYJ8t2t7&)75gB|=gBCyGJOxNE`c z)1y382X65hps>6Z2h#u{l+>Dr8wi35;IrK>$NV56Wd3p?eO8wA_>_#a?H1lCqFvtj;{hU^*A4V*xMeDE7=z=Qqm z0*gvRKKP9ff?dE|V1FQu7c@5j{0bX$1JYORz`&p(0VJ(M^#J-6$Opgifx8Rp!E@Yo zT7%0`q(DB%4HcoO0n{h;9E=aJ+X1p?AYZyb*mVHGiHDtoa{j^-y4K!?K+p{coY2j) zFuy?InE!8nS*Xpez;-kjfbD25Dr$2BT35^kP#fGDptjlb8)3d~m^r(?u|d~V5W}NU zU_0l*@X`rsXzfPG43a1gxDrRGLHGb09x!~*Qx{*A$}0~{S6DtIz@*q03QR|9OKCud z2CPn6loU=&Qge(TkSQD*sv8YeLFL$dfXcD?R5a@Lz;u!Jwi8jHPN9T3pf`_?dEBax z0XoM{LbYkE4YUe= zhR&SvF%O_9kVNq`c|jlZqIU!8R(*^V2ywL=a5b?JG*YkIz(~`rZ0uQHXjNoF<1iPG z!n*bQO3fwPp8&EGLq~(~deI=Le^Y81ItMS$NG zMc5_))zjoZt2P}Jb)Z{rpGqH<`qXLEWYI^rPoF+^P@_Y)+&~LR!{JQrMVkUjpn_31 zz<$+mvS;dCG=R_uPH5&fRXh;GLhC1$kL(p{cIh%XuyXffDc^}*ntrk1^p;s zyuxIGOpq}cj z5lR3v3ibusSRP6MwSyE$3EWyrfYyhk*A0|FW8nb%yK3P<9K;d)yMqQ~U)ABy(qX(> zn`_}g0Cesk8VisCLkkZ=6L=8ecU;QP;9VpT9>DfSGGHi>1zu9N?I&#unmw?#Z)o8@ zXaf*HDSwuQ!#zHP{?2RR$WM@%d1?)PL8V*ZFsKbH0^nX5Sz$u+0FGCCEe9`*P2&Yf z1F2nL>K@nzQ0Nx?2FUCJ_xAw0Cu1MbE%=QO0LUBBE%+yv1>463Y)7SAaEs4;om3o5 z1B6ggYZ@Ma?P^7`-7aTx_z2LV0ELoor+8=0R6 z$$|%PyOC+Z01N3+#&{(K=oVywmy}bJERBa&Xgice8v`VsS;>NF818Yt_o(x|2bRB; z4g$Ibx4LvoJ3ksjK@Wg&VkJeGlMBojm-{>lvIZy!zwtp=_zuYfeGVbE2O)NP^UDM1 zTD1csgLY$kxrur}k!dLC!ST=o5WDjQe)Jqb-G+)3C(Z^f`DMXFj{@h>Tma6axu~ek z11MfG7eH-rYk=B1qgx)t_G!B1$%E}PbPE!oJ~V2?_G!8WzwrSye41`~^1*k7Zb1Ub z3*oy*f$wN-L<3@Y4`6tOZg~K>8ybN^w;%y#h=!^lbZkCA=-7NJ8uj4Hw@SC*R+nzM zYV!CP0Q8KH0Xm0b6SZlqjXs9h)B{WCof|pU8gUp%LlQvb86N{sZCLBfqyd;6QelJn zpurR$^P!8wLVM@x(Z>K(8!F297@&5PL~(UV1AgNJ&5{Px0Oiof0aq(@3w|3IX=ilH z}-D8iZAk2VFA zKr>SV>{ksZd#27s0|Xic(A|L@=+4v`9l!*%I<&r@ts@r=1aJfhpeq79(A8l<0d$st zKQ9_Uw&2M_2GGU>==gy40pi1`g;yLHgHSsIx&)lZO&`kP>+FQvyhjGI=Qh zWKoDRr2SKUK@u5L0;nCPAnG-gfV3?X+6$CGwLUMvexrKlSU4oad{H}sf4!ZjjE<~B zKi)itesbE@i-&$d9(5S(5rc(;eiQ(NZ%nr!J<1q+0a!R>ffquwVA~2T94BTkfOTUF z$2P!|H(mhoS__AJeAwA05BP!?iJ2D<-01s)O1HpaP#abRz`ZiE!i44p9Iy780Xl^= zm>Nvo3rq3}-Gbj~y5+?cc^u1uZozL910Ziix8R>BhCKsoIJg6vE&6Lo-sU_@8KQ+pjVymy|65< zbP&)jxYeaw+WFBK3VH#I6Dt822ftzD!{t7&f~)}w!f$-g6~06AK%c{z#ETF+O}D&& zu2nlQGH4IBmj~73p_DWfgx~m}E9a@ubF$8>I{L_9e1P2!&^tr7 zpa%FDrcy7icccVxANwf1rNOnoJVs3IFII{qBbv} zc*R@*wZW|cYU_+{c@f*E>6SMSw$IQlNC5f}+j|k)r|A~_#s|>wX}aaj2j3OC1qm=O z>$psg5h!#E5@32XR0W}9^8rG~=2OwA7gxSjx&^nobgM28 z-SPrJ&-fUibL=Egt6**PF~p``SVHfd?_jMFhk-OEQI|Bl0IECpF+5BIl9&`4Oz|;4 zx;QMfcdj0N3_!J^qKuCLYDY;FSC=&4H#`8;t@;=#5aMbt;A(|#!EXa2?Tl`D0emZT z3x4a;txR93{+aDhuu@O@3k?Dm-_am^8G7QyMR=8N!L2Uca(97lc>~Y~X}YB-Qcp2> z0lzDX&~X8(>(Q+aiaO9OFRb|Is!?~py@>cV^$F_J#|~*CEi7?iCe7Y*R?Adp8L#%EnX|InU3IPqgN z(_1hTuybD?G63TsWPulQfDMLV0HZ!IdLNwqH^c-1C?5!*luyM3fXUz<9|HK*hnN~n z766CyWl;iO*C_!cL}i#o9!db_g#udK=j%EpfP|m}DS?>>ke3obdX&ja2_Os9 zMkoPnTVXGt<~)=DY6mHh68N-~04*FzuMa4JYT-VB{av(hNQn7jkrDhG(5j>jnKFfC%51Zb5pKG4=wqaKx}6Lt2#(TAZ$ZuY`$I~zfW?(9Y-)Xu9E8MBE59k*BMlk^LMsy4QiMe6Vz*@O7{9<)*CDsQB z-=I3GYl9Fa^K(A}! zXnsf*e1O~YnIDn>;6R2lK(`k>NqTP?kt(hE|yGVW!9V-lxv@J~)D+ zbkGD82TksnnXx<-z z8lW6brMTXmrd#k^mu~5_w+iQS=#~#Kyh69&w=Uhvn_m_@^eJ#2%?02*nv06se1PH= za{<%_w+5)KGrHwNY@eoEzC74IL$@FS>O-SOY@eoE@EadM!>8$%FCTnY=oTcvys$46 z_>R_=(tr*P7+#@UzO3UiIYyw+El2?Aap~HpAara#Kj{!P|ViUD#tc^Z~*whD0=w0(Mg2Mu6u7JpuB@G{d>dt)(57U4oBn?7?DLxj& z(F|aB=jzeN08|?)%J>+dc9cYMbx8w$;{(l-2GjuM(8mE+D|8Ef8yIP4bjt@Pa}>G- zzjf(Wrms~0%=RaM?4-ZYAYkzw4MII#>cd5Nm7c(@F5U8Ufo}QXcz=!}zj}(n5BObC z1da>0x^2=yQ3tx^hZX-^HR|rS9}&M=AGqyWeRSxSA7}wl1Yi{EHWV(}Jr6&?e${ZY zXX+d@00}V*aCbkj1KpWAqXU?L3V8il2ZOS8Bs5!7KQJ9}ejoVFGADOo8fw4rswB+~Y$4zn#%7=z5gYQUZV1 zDFGye5+Eh;=b;2?9a`My?>Z&$BP9TN)M0@8uosXL`14Z&NRKjkDFI{wnWZQJ%pKSZ zFg^M6Py(nOrXcDylz>Yj&{{67BuWNm(4(%SXAJD#$j$8|e zwf>$U4?1wEUyssZ+u}JNKO%f%x&`S`#@Gw6SrNm63~AMron!4_)!o>_u?_IFiyuII zmW3lo_X9}JZ{fgC{7B6FupDn7snRWQ7!b!DH$Q-TWypgG&7bcyj6$~{4W>q4z|$~k zx&^<}bjz=tKnJ=7zflZ;yb;}kf1()njOJ`QRtHyN{ebWds-w~^5JJf;(*W3RWE!~x zEMk2>uK4%?+UuGInjg^tKwe|wSx`Rz0E_t9kAU6}fL>Pznjew{Kj8L!=7%J}4`AI$ z2Iv-Kffurpf;iC+nBCCu@O%&mfD|C{jN!q25BE6V`_=j056j|;u>jqITV1-PoraC+ zmLI@4nIQqZ!f%2vun=iv4Nwq%cVxANwf1rPlSoJVs3I1lC$(4aOz@rt=1YJ)>9dW+Y(7Bf*nAo^iYwnL-SWe`vFXowWW7P-$wRmN z%3HKpVnFBENke-E6s!$C2EXw^SNIO(0n&H@a}59@S9~l0pt@5Zg9Mlq8Y}>>)ha#) zzwtp=TpWBX0HE4Xk>X>J8!7@o5&+1p_!#`g2bv{~bb-Lf1Awa)x&^-tjI=Yl6#($9 z&@K3_OSdw8rTS-gmz|F2bwy1a5WdiMI=MD*(s)a}}wl7y^Lb z6-D5T~RfqPhOCalDP_~X7FdD$)K_HJhjL*7&eiSf1$6~=u z(9ZLa0rXe=77oZD7=>KH^*{$Qm;f*t+~Y&T&jYBK47wgAwUi*x zbxHsUQ5ng?m=dIQXmMYl>y#jXlmO(Rn0Y8c04YHrKP7*7_lP zG!|Uy*VBR4i|2d-i13Z+7NkcRV=o|;f$1J(h;5TA&h!BQ>xLGNFZX}|N@ag41Mcx* zXPZ3e3rNfYupDo|PgJ@E4ujgTA^`4{ArB|Ce5YX)bP8!OHTnXchDp;c_?@O(0p$cb z&@K24VhH4o=ob7F#js~IXVbAdxDp!xgl|wCjcenHm;k_cjrjp=H!_Xf0T!`-0HJ*V zpuMhXp!ubAWDOr#P(J?vi}*Q!fIa|#URMX2ACiRt;P!mxha?~XVBJUt=oVyw7qYpE zC$Iy6*$oX3&j*13N@<2xobLgR0{{$a=6g7Tq8JO%tpKcb>wPLiJ3kuJtpI>=GD9Ma z!{gZ3YNT+ThjzwRJ|ff{5+Y zbSsz#+h^z&BtU&&)Ir4dX}Sf!@c}e^nr;R2!FPpjK?2MR`XZ>ncd)j!25{&gV0eXY z1+$LJfo4e~RUpLGLBQ1t-Gbi+M%o$O3c@nHLbu?zF5SxXmFk~a z{uER`po0bhi|^=9>gm!TF2bwy1a5WdmahwRD~MmrL66cXQcp1i0lzDX&~X8(>#dV? zP}G5L1(o-6HEPt|?;xJt*VG5TLDz$179|LF zof1GoP==Hsn1>Rib!c&4u*>I@#dAJEMEJ&Z3(}*Eu@|7BAch4Q(yF+BOSV0OuKl z_u|mP5u^tJr02JAltf|{gynbxNtJGa!(hf(5dinfkcSgmzSA%YI)yZl8kgjQcp4^6 zx8Qf0ZUvPS=s>sNH;U!?R0jML%Yr?lIh&5v!Ijt`Abf-BsB{a2P%_Ij0Ja;MM(zNM zSU-r+J_yiW*EG=lhzqas_ zw;&6=;7W?0Kuo~whK7gdgMxUDJP1fUvyz2f2V--8$RIP{7r1I+U0ku2iI6+o>k=;Q z3~5NXFnJmejUaj9=mJZA_&g1-Ysm9D@~A7cXJksAT-_uU%)&^#C~~76k#evwvF%iH zoR@0h;5uFbQz9@-mBU9Kb>-YuN}bjMbJP)Rm{Nx;UA1s{1+VGjl|#v+AU*;>o+4qI zvcYL`{13Dpy54yx3@hP!-z?PW2NmdL6EVyLS{<3#IV4B@&uHY$Q41Y0zx4K{ndFCW zqQIFS&{ozO8%nm$WJ8a@yDPZ2WGmgICzE#QAl4&W2patJ9Y=?kC$ zgq4vT^TB*V%ThVwTv`i2AFpZ4)~Eu&kN-$Faw^|E+d_n$k>qf&1``s4RxmwjQvmhY zrWpW>QtE=pSdvQr`x-uDj)s|}7eZ3n{iIP++S4GN_yeF?lkFRUN4uW|lVlVWfqKxdG!y0|5@r0ceG4Q`tyBi#F^e z12Rm&aUM)(aGXY?3Ft^%zmn2uxVt8^$Z#S(B#o>h#xWeHcylP${=g8w!dc$uLq=8_n&4_oZ zOoa4K7pVwoGvFdvhEz5NY(%Rm13FWhv~zwZy+ckP)dD3!eFngjv^$Uz5Xr+zQc`4K z&BIGb?~v1^wJBx->u=acq*sloNvd}cN!9{6!K;Ryq}qX00Qia4N`ZuQ*&qC5UH~!z zxg#6V<7OI7z)-ZBfPolk>fjG#1ELlV4WTB3t!vVjaQ+BIjE|Y6q=;hy?J)8h(&B)x zF)#EAI2lZ?q#=Py0EJ|IP8v&;wZh@tu94%`3}Nj6vzn4XOdRne4L4#A-ThD1Y6xrPek zA<=~G2;HKu2~7}+K}3xAhf`8$aL_M?hFVSXLYo6AxcUh!b^*mOG*PG|io4JjUBDbN zCWjIu5t5J`N$)6hgLGS36&VVEbdW|)l{C*DSB5*f;FJ&+ z2!~mh!YD|a!iqn%X`Kc$yKw10%>|GiP|GEV$R@j)AUlM;Crm2)$6NEOXEUGgpuL5{CzZ zp~oFNDC@`_UCLX`T2+}t0-y?+iiQ)zeb?%x#~oe3W=OpV1o0AcgUu5Sf`u;^YHTW- zGjxC;1foF-G$UX&hFBSzlZa$a2N4L8b(ShyQwInFN2*WJRF9MfABssC?y;PUgCn^@WL0?1CK!&aYv{sNt6=Qybq56(M3p$3rI`;O@vhM z5bw-pGOqszlW`Fy11Ye``Irn?AQ8!iqW=#j<04GPMVO2$ACrM%5KZ}+3}iv1@-rD1 zVKS~1lYuE7dxw}uJ|+WI#cm)>#-(L4q{U$`5&xOB!r=n4qK>iIR@%WI;1e8}0VzK0 zo9J4pR#RpVp7;Y$6eCR?{DJJIxZng7>ND7fCzcL}sR{t^8K0R;Nny*?<@qoV`UB)b zuILqh*aF8ZS`NnLHTDO1k&1{ALCsr;=noi<7q6XdsuABIJQHK|8u*TeJA#Li+*7qu zhiaTOApozYnRCEwhC4!VUy=iP-*pq$%KoJNUG-bF64SAVp-e%m?G~8ktY7p+W#(l1-{02IiyHl(n1!0d@^F zqQGgWuhPP%(zIldhXik!d z*dj28EbfTmY5{4uHAn-()MjSni6o`|S4^~%e;)3b34@E!5^mt?B8#M%NVtemF%&Ld zyTaK>Mg*y2G+aytv%bO|Ab2KNeT~a0-GbHJxvaX zUlM3r2b&dLg zV5m;@OY&W}Is%L^RQe`7_0HgC}xFbd$)i>!*xg*Bz&2UGI$D598 zp7lm$rW;kS&hbIzTS>%TV5bA^C3BJ(>!B{>B6oy)U5*v(3U`EHzNvQ7D$|@=sRP=f z7$ zivj-C>^p*8U}dwJ48|Dla_M>>Uh0Or+rI2hadlI4F!y@-P{wDoKR+2ZHvhy_2$1oGu~auhvTQ z#R|@a^I(MF40nVZg*#?K0V8+BSipES z`sfOBNBA$fl3vri=s>DR7Ctb_uR--RLjy<=UD@UXn+|HdvWL?ShDsQUo6r#w>{rix z&`{HwfO}{L?(9gu*_t{SD#7^}(HAlU4q|MmgylfE-n>HtMgqo+jF**=J3=nxN``E* z!Vgh*3X2SgfJ3=U5jhz_UKZ$^YVSh0>KwJf* zV&GqV)Ri-O#ND#aG3BVsU{uHtOG~yaGSPpLJ7V}>yecm>rK(1{7hYPx|3Kp@`;Hg` zSdTmEbcBjBbGajg08XJ*>Hw8AqT`S35a&XW<}SyBLx1%jNrX*ylU2TFkrAce&@7;Ixmr(43H(sHETK$ zCzw3y%J@JU9hiWjI@K?MY+yMFFan~G;f@$&82_PN(}C&`VKP7f9msG;45O-V(w%Zg z3`Csaju>26k2_|%QT6H^9|WpFicahWb~?~rG-sSK6tQA2XJ8%Rqffr+4T~FV8nQ^L ztF|lT5keDp)TTMN;?}%kemc?SEA(_pTSxZjR^Bkz)=YD%0fQT7*DTn@n(5I;HxQZ( zeZ&aN&{RNXwAcXnifUHb*q{RnNue}y3=1PA>tX|fLeiSDPCr2)WJ66IPzX6ihM|+O zz(iL^L535F4m z^AUE)f@EauHKfG}%=QrKf#0pf7ANU|T^RcVyyXUEB!{3d#BsJi5L$s@jPr9Q;yZ+D zVwhtCSq**kz~>N9X{vPU>n}KOVxVKSnT@7th>1*cAfADTq?D~k2x_d*$4qErq>mWe z7_P`Z47VQPzeJMsn&vGFQa!S&4UvkCs;3zmK#J(fHXqn`Q0tX{;KmTfSlCoS49o{z zLeZ2x2xSqFY)u`820}wHLNR7$FbEMtB}4}U73VcnLS8UFv5^$gN63X-$qEPA8o+qP zh9(M?B;sWB0P>JAQJj)f;fmp_sw&{u#w$LPi1-QvolXCF_@f61M20`Y4P@yi3#AZ+ zIQ0+2sTjZ*uU+A7h}JQLF_FhqNDLJI2(gJZhGOUd5QhO1lA#z6EGhiaLx@4wLKzc( z%R0-Hqb`F}AwQyySR`&+D0`0><<0eYXlD8k1+_KhSvSoad8A{=yTx*b^$$%q5VS z0LSN&ECYqd|G%@$Mz0WoFjXO9P$b+jd>B__P?~Q_A)ds5#|C(^Qwr%pN`j3?+CC2@ zQ7I%OAxeq;N3ds&4`tFl>elhxCwBR}k))ltZ(& z1Hq5Uqppk*q&I;M7^+i!69Ps-g;SUX*n;YtXtKAu+HtW|+141ks!|hLsHwa#1rRB_+ba0zwVzV_5=5(wef)LFJ1FoeM@VAT;Vg z|1}H=jbH>L9;1ly2R(=sAw1qZYzBu5$c0=9UM8~~e9WBW4plYUgoH*ViBiSTe1id@ zqL=K&cwsNb010cLA{Ykh%H}i}3R-phPI`xM8VnYV+2r9gil!`3ieaFQGV{>f_C-82?@LsL(Io%psLsngwtSb=L}#( zD+T93y})TSt19_II(P5~`0NX2K#C9BDZ1QFt0`;yAHzI%xy8wzIHVv2HrW0rZ3&;6 zf+8U~G582hN{TpEo{vG17AI)iOBhJ%Yx{okzAHv(HuMK*RBRJ+910^hXIh;217Q~! zojJch5Lv=FF?zFstcFE`hY_WuRzOUY0TgFWjNhy_a}Hj~Ows;3Bqp=batD+LYXu}kt0{|yWNYd$G!Vd#ftN8GgF%RxAUTtT@t5-&DxokK zcG*Y@StR5_uBkzR7y+DMGB#0irVQgRV^YnKjZ<!n_W5g<~98!R92g03M|kSD2#St2)3SY+qHnN#H;Z3v%?MWq0A z6&49M=A#DxORQSN8k0cvADJEP^ND^ph^& zc6Amy0?V5|W=7gR4=GVuBqSjfCH9XW$W>S*gn1@y58a`#NN+ygkztV-s1}Mscn8LG zCcR0lDy53B4==Ef42#6*%-%R0K*t_VWZ>B&Zd?*n>}&dY)J;R8Bgs4nB1kRMTp=m>*EXM7U_LgORt3Xf7Y93(I- zbcRJ@$Xf%W?vzDh)aVS0#7NM(EHcxPs%vNaA%>eKg-4Tt8)7n=bI%w!S}_@%(=i_O zphfl5QsrAxgSla~@Y7m2KAl8lNRyM-PkcZqdQDn5cXlIJR9H~IeD;#!zm>o4q3%7n z;|hB%!GDkKJ{Io#6go);?xDSwG}bR}g!Sh|OB)(iHl$KFEE%gL{+mc6OaK)_dp69U z*@^7X!Ho+R&$zn!n%Rxj4a*lUSlq}B9o%Oeq_3-;*tbU-xMrTYCiX4JeO9L<=hl() z=*W3>_0OhX?ZO zAP4#7%6WC#*{jpeUTEjscJV@fxjKB1U#<=x>!}i&Ox1a4njNUmL;fT2f>bZ z03T!R&`0oT)((9HAwG5FI<_->0kT6`;KNKE)4Dp^f#2294&Z}99n0eC*v=5XQb!Kj zIj3FVMcO+cFrw}yQ3YzG4hV}aCfu=IMCct&ZE=L z9-VgfKs)D_MQ1#KfP@`&;CPVZ)37_PBd62OUY&OKLObV{#iwHjI^zMXs&p)iPsa{? zI_>P!X=fj_bEYh1eR~Wp9G5%2P9$R&x21bu_=Eq$C(2#Q2HwP*x=)nsw(-JsEo1q2 z*u(Z=Jj=F->$^p1KfRJ3yXm0x48O-I^B!t_xEsHjIvP%te3w{J^_+0AbEnkPvfT1u z^HFXMH9UGI{YvP6w*T7aG2bV5+JdGXEkj0K)Lw6%l|0V5EjEm{GA|?taHk08IUbGw zubOf0iS2N4mKU7a1yeSHpkPz%Uzbndddahjzueqf=8_M^yxX@%zTf})6$Dv%Q8lbU-wXyG=KkGaP54{1hZDIMUB#Lr;fH6Kl$5IJgp zAYK`HvFhEB-TGYY=C;MAzQZ=lR>u&2!;U-R?IW)5GpFB^VyV6fEthhJEqa*@C$rj=@;`qlIK=BLW5$(%y z^Wov=+s>+ZjUHL{Tczcc{(0yn%y??Ny(0Xd(5~2fmL(N)Vph}O*m14bJAY{XOX?%-iqz4))}CgY!%uJe zs;Yv%tZfo~LE<6qclt`(VD|Vef3km@yu~xk2Ze82#>Y-`9%^YXKYv7f_;>4XJFas! zwXCvpTMEF$Oeq_yQ@gU@smgJzTdFsmU^a??

C;VRpxws1N1hwXv5z>M74O}I7uRqRo5SNIZXZLE@cAoh6T6|O*B z*HoE^+gw8~b6jkl-8|Uxk!huUg*>+PimKJN#ll4isbp{HpxnRo67z|nt47=!x0QcO z_oK>__mn-(Prm0b$1jfWtjj_SW1}lg^V+}ae^h>yol^WhGxOe3`9<-<&?AvgsJW#V znTJ^>avxQEZ0|06nTPp!?A+E{?7ii^CMOQdSGQgt`O2}Jx|cbJKdq)b+RkiddosVa{$l@+bmEr1#ediIwHz36uk--b zmmM!|V{5t1MU&drZmM#`LJf}Q$a&QxsUOQOv)|DC4gb^TS@b-<+F2NKN9ER`;XC3p zZS(o>l9SloJHMeHVI*cp@($@t)9TQ;)+N!?#gD|#`9|R%$yrR83WR4lRtjf3N1Lw` zA2Q8j_R-B1Uo$-u=h$X64{{1D?YJq?mp!NSaBF2GDo~C_>VWOi9d9w^(lJJ+KS4r*Q4YL1WC((SR=L-mm< zi8=9tAANbRPeG%XQ2a=dJd^(t2?MJv?y?D)OggMs#mWPrA4G zF+E<~F5PKslW5Dr?bnx{l`P!0kNR<&P@bsVS#%Zki}^10rs%b7H|{THVavnCCsUV2 z&Q9#M_#BP#FBqoOWZqF#oLDVRj?8DKjl7ke;CxbAL!XoQvV2kOa%mOwAw5D|LtV~L zL!xqks^`BG2lC5ehq(RXIi-aoWZ5Mx=XO|z%R&Bps*sz?@3I`SK2d&y^=|2O?lPv# z(oeS1+hZ?K9JN2zJnBZunLOP32;(g4UR~5S4HF|lbtxH*25ecSr(hHKclaWDI~y`NMRea!lK z=)usO*e%wo_U`gEVomEiu`TkwmFwtG^G|Izv5od#RX;b4vi~JrBb+Ap=(i$zJuO*B zHQiTSY+e@oRk&AtpY9{xCJ%R98@Zxrnf!U#JCrB~qH~2`s_tZA%z}N*wwvk4o2P}B%KeIekQzffORr4c%f3pT9NV9KB|Na@@ok$*X}W>u zsCOAFn>CcVl?6cWRV4dz$H%c=ibYC?ZmmYQ!dUqU7L zTjXP*VZt$aa$6GUWAyx4htJ zZ!L}AATDFR5qC$=Z~dJ8HnFiXlzfOf#+hud&{g5z*q*Tm%d-onEG$96riQEe`q=Rm zE&TiRMe!%8U+mS)Wy88ve#S4^S|B8CVV1X@N|!j^w_O?gchkM>Bpj}&IL z-a;>NP{MerNBExBp|-Nkhnl})>V!|_D7~-bed;9s_||UmmGVcX_3eL7k}H@9zy-zIFfd__;Rmu&dRGM<@7{|Sz`OL{K6G_jO= zv2rrKtYw8*nVci_G5u*@XZy}R-}HU?Tht|%+o3@)uDqEY z!M{U~uQ;8%w9mWp=gbGufqYHX%4Rxtv1ydElv{24KKvMWjGZUlYO9kkFIp$x82N!Y zBECsYpdN9oP8Ly@hfgZ`ML2tFCG)erkF?z~A=+ed*rTDt@>=OFlMug={S$>!JTk@%JJEbQA7#zbqk>$MH7j}4t}HN9OKmsVUWtrv?p58h3;j0t$)ZB?YBxt;!CZA*{$1m#R^(u!g)Nqvsb7&^!$$QRa>OF&fW4o&i)V7 z#H!oI+m_o-l~%P*W;TWXDQ)0B5#~uh(%sCVzVY`;#%>}+Yc{cT}!;tuX->UU`v-O&EG<9c>R#aHE(j!&iA*~wKa?C;RU{DkDE_94zY zsVaGlxHW!JTx`9cJ?!|6x{A3V^aHz@Jt7zKBD-0v<7#5{@k^ws(IEdt$!p0q@^Q8= z+Un)2ny;jXwZA17@&DzP^EQ5D`|FjvN;!(>`v;XsVjp-*!s*pv~iC|IYoP3G$R} zFGXI7-P?3#c%L+2=ba_4c&U^%D|u8`Di|odSlpl65c|fqKn_IqM$71-OjXNA5u15m z$xn%G;_=)_zMt@#$gmfe-X6Qw+%GX$YLvg12lT(+ak{1NJ#E(A@zWv?me+D??z+bG zU~~fi7`I~N4%>8g<=P?qBlfNF9ugh@N*?NrMdr8n9MxRh!|_`5Kj!_^uK3H$ZOrwu zJGQrcF#ni!v$V{1#*X>)E@lEVEYefDG45qH@^42i;$G&sw)aYcqBZo7N+)x-_}s`D z%#%?^xWMs=d=qn@I6K<6Wr{H0_M`Rn)-T1W@qbC-mNKb*OYN}7ZBv`pw*D5M5Z*!e zHm_{`Cb3_5f5aqlLHJ>*k$-ku?e_DmpM>9Ko8vc9`x18f%E(<6Cs{v_JjpC)4{rQa z7{|?(2bV-zis&8k!m0o>#`aXv$U%c_4^utGhwaytKNOzA&1NsMT~IxRyTP>3bWPQt z+~Mkh+@qEPOIdSMbP>D4@)Wm;UF|Hb{zwYZHsJ#22wMxaH+Fx?3tM(br`!9P&Y@@D zIl27S%DckfZs;u>qK`(GI;S(wS?=XmMZ=MS%usQS@Iv%LTYvjEbW8a{=1#}u@@(o9 z;p4=aHcQCQy}D(Ia67j$PPd-KFKoN1yxw%Ec`;WLIyF3bN5AM($)9Vg_@680+J;Em zHHDo4hnJta?Acv!$o`w$N^FZ^cbqr1HdV3jttj zNL=P6J}8)nzDszDzIfE_>~&0g`~$YNe7R$mFqwJVNmbrxx0&|XHV*oY86_7B_r$NH zP6>}Py=6MS*-||qv@brKzM^%i#9249zu7j#`^#mCscnAx^#op}DB@$7;r?}I_XPAI#n^-RgzyreW)6_NiZ z*sG>-Tg|7oM$11G-V?5(V)EJH7aadGeQkENZw^m0Js$m997whB?T!np{uKVSZ;4&Z zIYVPD*NURxGWBGR@LMd&uub|ZG^<>shea0&TUj&xSmk5XCDHS!n{0C{|Fj1AQg0%p}CQ3V%zvX=(8zTbO&!QK5V{B{)t~2xkJ1pb~C@4+FM=|do)^M zqZ}2|MRu{`F1epj#Kx(K;vZGB#rdU^<2za(6P^-h*s^9Nukt?T#G49E5pLKX;O)|d z!nWeC>3*iS_x6Fzp5*i5kMhp& zTydxD5>`2T+U{;C7ms5XbE15erG@$*$3}}1zX|urTN3TI>E$fF(ekpq)BdD=ZPk^g zPh-w_8-0&?m2J8y9{-BnWSiB#g>JAP4@|^u8=7!f#w6^vDPje8YuPmCS>ZPJ5!-L_ zH2&^TlsYE7=M<6)<68$YREU3@`7J(z@0aX8WVZCWR35unSVlczKM;Ft)BR3Y_=xRk zG0J{vexZ6!^%}?WrjPHuz3RdyyA+o%Gwo`zQ(LVE`PmicIR0hZZ@=BXujSLK8SxGL zZLzub|L(BGjwOb(EBU4JByLagQK=Oe(aYlR<|phUsc)oU$+c#Cv?M-REVK`g^lNt| z!@_43rSu=pN0<|(AD9=ZPpfO0BJR+-60T_bXxf|X<($WUO>eXAm0PGI_We~S*$UX7 zsh1*7aa{HM_FZf>yMuNlACCRhJT~!se9X?N)RDxoNPX-j={|airJ9-~`>X?-eI}=P zX6%c|&DK}q-P=zsJ42$Azr;V0&WgVQD=S&El2>)5f@Yqf+WFh;b^MX0?#$a(hc#)j zTYliTw7JX55 zo%FGLZ1>0hY40ta9{H>KR@N4Kls!`Y8MjWlC44EpwT;_yKq?cKiIX|2cph_dXh7Rx zXW4_NQSU{@lozzjqTXS<+mqtkj-RVaL*JKNMLp>#vHg?3hW(m*#d1}Aj#cE>w5^O3 za5F4*;zlu6vcGwa+%LQ+GJrjg9+BKuTkE{mcE7U+o1jjPej8uZ_ObbT`L=rlj;6%% z>@U&>%&F1U@_70xhl9V8>YW&8u8mHiE|reT*78zkH+#Fq;yl2P+EzC55M9kY+%m>g zBKDRG#rtEMD~sA@MPJyIi1sL*!v4K^k=g4kr%ti|({#7ot?Fu9sO|EKhn)|vxxV6Q zGeccwJw3i8K7=}fxhs4%zmvU$aWYX$d*rgv_0kmTbLK4PvvC&_k)E#l&i0t-m-jiA zJ8y`8*YrX1ekoktyJDDVX;~M0nJTG zvfaaurv7c8!&sS%EOxUpwl2^tql<#oRrs-k{ zy}5=;7RLA38rc$Rk1(2N;^vkcEWe5`F@^DmA2=s6M_9J~kTBiW*YQSZ(ZF}Cl9*jE z&3e@6_`%4B=G~6bk+T`G{6o42-6JvA7I6%=x?^9OM%#al9FWFHw?w{|HnuK}-sl_{ zlj0kNe@exX?exTEr+lG2PP{ZR)4H=_DAkzg&RsA5m%YwXV(%5df|H{W5qwz;Q$Y1CGrG^ z^J%*BsrCP=ST2jv15)|0&xG+U728e_Phd_Jddj|%Cz|Hh1emZe(7BV|CIoE1H!W|U zz4dp=MIUFfN1KEj*$c&u?ETV7)H9WjQ2V#MXFn<)cHAJ&v%Vg=+s98Z+bwSE-s zL7!y(d%N|X%ETkq$)yME$Ao*h)#6XWNMU>S3Z75x%7SyMYS{+?w=Aa~PLw4#we*+I z3f0};hZ(s20&6!(v>%NA!ai!BRCZ8)TD-z?6D$+lu>FqK=S#OG-itk4xkRcrH}8xH z`=nj+zcx=D>9@|0UtxAiACK(W@))%(@f!P@eHQ;0>x^2MOOi(|p4NWFf5#se7}NRX zXEBQ`7lo%f{~LYN#L^F$);k}zCTum#Rl=8Ccd19xMZc7|f!P!}%W-_{<`HE)Ev>d) z#PwA2`=Hx^6Kyu=9H42&PT*x z%fYrK_M!?KtSAgDY1;np_D`v=Ol>w(^gYYH!qiyQd1~~1X%EvoS?;8{q2cS=M{hh; zxS{eDZf^XRo!2BwBTvMF;X(56;$rD7`sdhE+ZmEIa<6z)4pjA*S$ZA2UGT+5mEHl2 zrk(TJkA^2Z{w4iN?PLN?7Uy(!Y*Cx-X8V~HUqtuQmpcO02b^5&B)X@(-#MVNxaz** zOKPu4oXqT}3dOIusg;l3e|2aS{kS;VzAf^8_flzI;$rc%p{FFPZI77-Ro!o1VXb1u zY^xAX0&=nj#*0(UtN5#=@%)?}S+kN)b)|w2tFEzkuiRlP6`pmT&2(=*7tUe8M-p11su{kME}_3iBwVsgdscu(g2$nWgu z;Hxo5a9q~76i&3DHaSo_5$@uS5}wd>>G zOS{88n_mdGIR9aJEp~6LI=RzcEjStcW{Js5>^$&7~ zrPt_>*~j?{61~LpMPI@i8)N^Ny+iT}7g`$=&GN~@<@{&K72Gf4p|;19>*;z+rLa<% z8o8RkZp35EI%lLgVe6i_W9!fMkEKA>ozfl2W9Db*)1>X;NtVYpZsDA<@e;?r;qb|x z!RIsY4tprPSxWMK&7W`GE|ip?FYl|^J7j4%96r)K)qDk0#=R7tC&j7Dr1RSTqW$p$ zk^k84XrIM3*k)D?qo&wSim?16;n~}#hF%}JMf|<|7(JqO5Is45sr~n)ReUV^op_by zV)krlj(wker7_pjnLn6R9X?alUYKKY$=PJ;kYpAWN&ngpl)f6@h0{H z>fF%t^Z{!@c`z!PR@<&{+A6b$GIeRUyCfZA76{*Ly~K9F%8PrYGc2>Dg8Pg3r1O~R zb6crokw4z{JpZ-bQu$ze!g)NipLwR@>ev^X8%wX}j#>YTpJAQLoZfzdeSq{#;w0N< znR7nntfOYNtgJRMP4SbXk8a=B+`@d3_|AH%^ki~(#cNDBDY9O2hx##c!FR z_Vc+1D#nz)%P{gT>6snfm{UbZ)3ebJ;s?Z#?PTVWBF?m_(mQNitDRZQZ02v|S*fyY z%666+7`>pyp4`@a4Ra#%sk|z7gzwS(Fg@A2XVe3d-}WZ>^e^%i$;+i3&Xdabi&LF% zS*Nx!*^9A$O;_%cV{OkSKCn4M-!|RtunRNH&sTgye@0DmtZ({GIzWx#2U#w1j4rw< z{tt7vxSjpoQPxt-wZ%RQkCtz;{Mmdfb90rGE-SC+&ZqnATuTk%7mDHSviMN>rO}lw zf0#aI_f}ZTriGtj{v#xEe zd_=s={7~$qNPT=+WuPsEOIkD3Qr-;x(I|KiTK*SD+>u&|bkzBt<6&vOr8oO@ z(f52QEi|t(?O6JShZ=F@em+Uyq-08emeun+3HAH_1 zt7rB0W6j-$b4;_niM?OGllqxoFH8}R=N56_gjR5Ga&kOi+QQ9D_N)AZjarA(drK(x zWZ|ajE8}OhePQ{+@!N>;OwdvtmHFdsO)VGfEJ{2-Vp;fR^CU|*8@2VE$h7!c({^yq zZ(A=m&t{sczKRZuzi+P);%3%yI<>p@J9T4hvI))Wanqa>*>ngsEQ8M5KNEje8XP{#71?jzR#-hg_L^lAQxUIazKFb9-m*<( zELFwQNLDL!1 zCi`UPrNV9E2eE^($+BoZlUXZF7UmVZmGKnqUEbQiQv3;NXg#S$*#NJh0ZaY2h+3|PmX<$yHC` zWa5TY99Mjp5*%Fe@4-vf1?=C8f^!!akPBnyicgwf9JycGo|q!`mkyga<__~8%oxjQ zP9Ha~{V`@}{Ep=DI|}99)(xc|{w?+m{_ic>1yk3fx?83Bw&|N4;k~B4p`Y8HAMvR* zQC+vwA#Y%>Vc#e`TDiwEs$zw``o521Z^mM#lba_vKPp*gYT(_Lw!Y&iA2V!Q9euYn zBw1{txpi$4Tjy9D`&xQ4d;pB#hs5vA@9>w}9%goPk4K)19b=Luh0-UyDL#~)zN1X| znRP|73KrCK<;8M~WohA^J05c`qf53w8*a9p=h(=u;j1MV^PDxp7nDwR{4A}6NpS}I zs<42mXudG^j{IHxy5w3@v14Lnqx6jR4eQL%pNYp*22K!Lgg%?sMNUmFrahs{ zDjeb%b~!V$eT?O0@rjxnhpy)Bju88~qsjI{Bo^<>%(o0-e-j?JObuCDW{dA6mq;sG zl1vK>QqRfrxUI2Yw)bS(VVhu8K{n^h|^oP7g8XDRVGg&3)Sn4C^+U9A_ zy7G^T-=a<``H=mxXwa~difbIB!mI6a=>tPA*d7)A=9TfI%pR#&{=wcNUMt=f??;c2 zUKWm49J8!S9ube(uadqf`!caC{;N>qe1#gqj;UU14?3@uzHJiib@sQZ3z%l^1M@n& zNN=RpFvXfbk9WM7{+S(_^ZjS#i>Z<$? zdpNox;u~@?cOmmvvNu;+@q2Rk&IDa+ITX2zrdwIZ3G5^C+SWM$k#DVz#cqfn5RT*4 z$(NT5vpqyz#BYkdEHA3MUg80HYEC-@RaYuFlbtFj6O-c6&~$0Ma3b|b$&e(y^G0z% zw1=&&dRW37+LU~l-6DT2|6uvW?zfFCVHAqf=+ zBX3%1ThXTKX(Gs(I_Rjn}ty9Ci)br)zqB|ga*XtGQ*v-xa;NhiS?#`I!1Ff z%v$z0ehC-n+L(bBIGrHhEfts^5DSNQ57`_S%foCdxi5>%O=q!o#~|uS`_<7`+FoQY zVpmdhjmxqi{43pU&8wCZ~DybTtD^;%bw^{=H2WVKHl;; zFpdM^`h+JmF7|ljQ*M6Md$g3et?*l6ucrnf>%r0Z_GjvXq^uUy2R8`&1W+cHH;Fmsu^W6w*YqBlo}GBwg3Imw@AT^U+y zeVLyhsfoQJ9kQKjEp@h2`>YdMUoiJERVNCmGpY_-_fxL;qp{!EpM}37pV%hvEJ*B0 zyyp0lYZb2MxMcQFrY`MvAKcPI{F52N9G0JF$FiqDaW)Bmh9}bZ%DqQC9Cyc8JH8M1 zx_5Q!!_te9znhN>uZj(emBi2FE@Xdlp3gr*pDsP-xF;fR;^bc~=Oj*xO4Z$C--c|= z{OYFaHHii7zY7!W_et-?ev{`&yP5AyH#$FZ+Jt%XWsdps|6(t2*#*!W<-@P5vZ#P{PT%W-DH)}G-Z&U>g;%)a5lWLW-f^DJ&t z^2PZ3wpC`k?N#PSx_f0Z_IspAzMuP`swin=pN%b|AK+H;XNr65cL??38^te)Jatw4 zBgszL zOLa|z6&?gcU%}d{N66QQeUT-x&E_%m3hD^Ahj}l2Z$CrE?k=jaX0ki^wuITLJd zN{Ep~O$o=N?KjbK#l^99F|_I4gwuZR=9i*#tDQ+vZX9utvPO=CPKap>rbd7V68H6e ztL%03kLF18J=9C)Wb%QwQzBnktYXCUK4T5{x4b6*Q1+vB0X@xnh$*f3gsYZLpnh#S zo!>5=A}qV}F6jyBTY(aPlm?m#o5nC-#R{6c$FGo&r~a1aRJ_N(=bRqh#}8(&4ZmZX zZ;?#joA-=7Lz>C_!q!<`%$Ze71?{Y)MiYm9r8lW3_|a{H*s0V9md0lHsPV#!@}oPp z#Lwlu^8c0g*5OfI&;K}HNRSXBn`Gts-h20Mf+Pg8Kyi1s;1*m0G+6NhK@x~(eXpD= zy9w^@#R?Qku>vh#pio?Y_rvG&rO3XQzMtQd2mat;Upcd9UT5aa%$cbx45Tjj_6Zqc zd*%kutn`u9ifZtOxdGDO@+wmc@}m66GYlJzm32=y_B3Wwk?3mrkoQ`p`hf9P%&8>q z7pICD)E?%Np@+3OJ=XIpJ`y|Unqtb86RCr62$rupl@Em&v_-bhWdpN9s!+Z8cYI0o zA$(Gp7Eu_e~(N^5o#(wdzQ zQAArgU76s?HFbAOye8(LV~De^GD+zxk0`n$43=hK>EJ1v0u3U+7vdEsJ-}N*o@w1{ z``cwVtOsjwv+JOG%vF@9X?_(OVo#7?K!|$(DKCzW0DNQmuC^H+q^kv1kXQ4jz*f^4 z*RYT{S29%{zL&Fr+6D$-ZX!(#iWoudGIpZ{)(dqN^}d#r-2?AFK4wNOV;(67&mnNyN80g8P;$$+pFR zK`*;|5LMipz!;6wItUb?g<@j(cp=G9kMAgt;jh8(h*jXWYk*?otGkbrH_p@H zTEYMU0_O8X3;30Fq8Y{;QLAJ@JzyqCO^Ie)XLJYkJ5neP#Y&(*AYoE9r6oRMYLt663s9d&& zSWo#u=>eyL8k%Pn%kw3?ZHOY|n90O(Xj5jQbl%vD>}9=$163&ouv7VZNIKp_H~Md%7Y`!ud>N8uceR-?@WpjkLi;;jpsO`&6pzt_$MB zHJ~2#hM{2tb;)(yHHzI`sbyY>I9zgJB|#X7rcIuzTrP1Hc=eajU8V%FFV>#eD-8Co z1?|W!-fB!FGeNltM!IjX8;o_ZgW?${AZwZrW=%K3q6riO!dd~eREDc3#y?k*e`cSn zN106x6m}=NP#djBfX3*y#;Rw<4w7K|@FX5&n`UnuO=P`QH0~96yopf?sa!Gvx{n~P z4dQe<#Tx=-Wg$C`cmT3wTqu-tQ5m%3HOvLoK`-S=2OaE-lyKFMuc14te&x>v!?^|6 z7A0Evn|gxnwBG?oK1p~bR{tm-^+WU^aV{8!&vwRxjnW}Hi-0kka*G-)P8Bx8ML;2L zMd}G;U=xqCzk$KZ}}7mSd-&)oecFcxWHBv+4xX5%O?hb>a}W5^S_h z&`prXtEF2@=o|2J{({kM-W^>=9^~=Jr{VsnD9mT7lMlpCzOfHe)zxn-PnqFvW2^>^ zSUNi9T0}2*>4>)^#dZvP;?5CANVV(_u37U5Low5qrI3}$xu~7k zPwb8CZEQ_FjQ(_C|3v9a7T{aK2>dpY;W~_!Cxbbgb~8JRTo>Jp-;TZG&d?X_yZCN! z1NNqVsJ5gOibiTrs)c=Tpb8)rdTX19@Nm2AzOG-5{q);dxu6jngV%FiSeZm#6KLyM ztTOq))i!hOida`AbP_hwS;C+Q8C+RpmN1*Vk?%!1V4ql}4|7V@!zYUzA|TjYP>M*y zyyDLV<+SgxF?egNy(d&U3fhSH;!@mM(rWvkMt0xroB3-<-U&%Cp#7#rGL?NFUoR;+F}whvL4jD$CPRx)-Xi0Ba=D>uL!^Vw{gsUf+IePW!CPeF>S z?S}3UsnNsPrvfZp5!Q&Q#>Z@tPvONVqY6Jo2*H!lK10|f-P`w=9*_e zS5v6rtVAyqYUA|{wX(W_{Sri+;}+&V%=v*_j{Jbnq-G&;q|R`HC7BZVx?L2JF5J5qc@L-~kyPvyC($OFTCBAFe6%EkkAFqx_@fc77Rk zB07mGl^5oE%r63F{VkJFblas>aQQo~GI|reLxjkMQjqv7z9zOBT@t^J{cdi{INdxM zg(P|V&^`3`@atI>gu|Ylu9D{4%zf;hrXG0}b-JcdR%eV@37k@6)oFao!jbq~p|vqy zED>FSU#QQq7vVNj8MShK0kOoB32sLHu1m}nl`Z)xCAxb#B#shlQ(axBxjFI~aT7NVDeu078Qph`PpN)N5L`{OL8Nt;T(2?{PZ$6T4dSg9AK#C>%2jfm= zn7bjFNe}UCaE+iws}B%jjD3|}gu?BH&ysO&moYy!B4)pCF4Eh)n;k+XVVKis8p@_+ zZ@>xaGFz2La36)3W~eF6zp$g^U&uCRmReS{oo*97hiYuKDV@;n?hb`IMCpT2I1?mxkxQrO|%T3OwCbiI;!y(`0aQ) zK?%!Pc2RGh;b%I_QjhS%VolN}8EpOJ4796wQY=kxCP(o*NhEWD(1&C)4`Dl;3wWRH zHCGBCwoX)AzOB+3jx$xoH-7TS<)5jl>KR*Y#Z6_e5KC9}S(r{@%fgaeH1)SQfH2}0 ziRr-wa0zZ1a|?Y7 zK!Iki`6=2TTq*3xb)yb2(|Ln8*nLLL-NuWz)S~3y2vE3Ln#)f0W_(f{m{{|pc$66% z=WHrBa~D!8$apYY*zBoj>c~LC1Dp_h+7F>ksAkeqZXH-GV#*S^2=p}_BYK6^)^tQ2 z&O@S(J7pZSDkh>k63+fi&(x0NI=V(uXAvQ)xu-QT*1py?1U)7!H{X{U7R+Rhdp5f3 zhzB%U=WAL3VdA`uW1oZ;6Knnxk181_eT#cQ847i;WfoB<91DyKnd|%>_jM>i%};OD ztwv`G%bi9|6C&Ak6TGEQV?VLSW1L*1r>1j&S_pFtn<;H{j>gZ4GqJ5$7PJfM!A)kT z2n$su8Lgm>12N^$sh(oGkvzX@65h#qL+X*8NTeYXjrq=W&FN2S@ctQFr*5Um9imU> zqQxTeFREEgOR%0AX#5E_$=})+5IWNV?KgZ4XoB3Hzw4<+tl%CIN5p*uzz+)1!U^4M z_KsIaSBBz`0PL~Jg8Ycv~fiBC*jq6Msv?`LK^YhcIPdqd}Rvu#OyA-x%N;{PCa z=t{;G>5s3BL+>M>SfvjW(A2}Hs#prFR@2aR#4NEK-rKnodnuGD0fBLzLavznTUSCF z>k-w)Wk$_$H-goKI<8%ktdGYqZgOrS)8JYz9BswbMY>@@ zVp}3Q8n%D$IPQ4~PU5@tE0L3NdyIX_SLQVOH>REH;Ge~pave0wb$_B=@qN-gQz4iQ zUP2AfME4D-GG;K<;;+i_u3=&VrSLr+*-lqv1^ubQ#)fMAVYHvOziE;cAd8}#>rYw} zoO@)Kryf2CSHQ5Sdt7h!h8W}=C^hBEczLKmiTEUf_swzZHYn($|$?CfpXJHerq^6(lAU_v5YP`dC zB%ctC)ROx-{B>bEHclMQ{zQC3NBC|N$FgHxC&Oz-d`Idy9$E8d>Mw{uwg=_ zoP+W?PyzgeA6N4SH-$$Sg9UN2*pZ#(tRBL%<28#sF;uj;#JHa<&RRgliZb?sNDybU zkystCg0`qyf`qAth!2fq*E*eiC&Me<8L1*@;I>6vmC9m&fjA687pSqoi4-ZWb#DxJ z@E);=dyacLzb&Vy>oq@KXe_0K9Rs_$8{{5!d&?7jgqrpl&`qf;QxTx4($W}Ikj5H%eZ04WJV=@}+@4BlULL_R&_0)3h8(_+#@C5837pl5&W2A%bu~;nW5Gs;1Xk)#|4nR}5*=~<; z(&gYo#AmKeqBU$b{UZ^t{N(Zw3b^bUrq_V;w)*a8)=lIM>IK%;`3O7y=^6WBUYmN9 zJ#a7OP0(6wi*>hqf(T`6e2v(AXM4r2z$E1v6LT83=-1|+=jS>&Q+F&L%Y%lCN2q&Y zaZDO<2ORsH7jlFSlvA!kF+ z@*-lR(m(^WP@m!EKB;y{to2c7u{U!AtSW44+JZ)@HS!PWMyB7TFuDqrNlya($QB-t zalEp?-aKrDkYJh%hLR!V5qfrEvNI;Q4uveuVNAvrTyNJP(4Adjv>5xK1qB>d&iEw1 zhqRo!WLk}l!e&z%p`~%UJI4N2+Z0#a#~r%7JJdGHZ7OSO_(^CnvDQbS#VUxI0L(7% zGl1~sj^0Wxj>q_!`zRsWXM8Q70Y5z-5fO>E4`{CSDt)JH6;5G zI|~Q#EvN!hXIe+Rau1u*NKlncTvxnt3)aGI-TkI-F22(jMi~prfa$8>pD97 ziz~PrBEj|-Q^fVcF?J8PoS((kh`J}i0UzAJRQ;N;7=~exQQ#q zGlhAE8bS~{g&pjwu6e5W+P*=?SL!Gqz>gAJoGpx%&FR)3h0An41Dk4Rmv>FhndHhd zo0*y7AhIR(GrkOL0{NycN?E?9uq*l*UhHWK?aRxCHj=62NU9n8w_F+x7qU&Gt-HW* zekQSAm59t+L{7u^qh-Mk@frvG80ajhXpE%?Scmf+)u)}UG~ba%Ws)^a z>_Lp?GaUoL@7z2Q);~js5Ve4r`WXs_T-;EqqY_R&Anoo`c(}U@K36Sg{V0O> zVL*(k7?;2k$KUx2JUxZZb|+zT;xYBv&DObc3APvi)E>ee7E@dc$ylZcI|)1GU3{;+ zi`-;-G}4_sR?rR(@+{>ylI6U!of<=lxN>SOcU8kwPz%qf)`G!JcOfYQ-8AtB7E_3CA6;a~5HV#xyCB5{{mBsGO#_!AydzDMtXiU93OalFI_a<}sL z;6Ie@hZmtUp_kwtISyT^eYQMF%l}BH@GSGv*xnkf4z^-+ zxS9dl#~q)6u42^@%unb*dWUH%b%89( zeZcpI-(pMM->s6QDa_BG#+;ie14H9MJeHo^aguIxUO&-p|$*!H3^U9Yzl`Ki)-o4YNX>7pu#Id`$;`KPnk&e z&@1ltnRi{q=tIn9YKixbYoI~sv8H$KiiFR!7fTmLF&**B?DkJR|M-vMQ3IIK)|M5I zF!}U(^XAxx%076z_7HQ@feLGlQL0{ChHoR6xmx5)f$^4bHKlT_)yN!iFsGw6l1AA_ zJfs)vE{R9z1$Zs}5v3RSMI7(datCOc>rTI-X;oD5*v2|pk=exqQ#+TR_QcW64 zYfa%yYy1zmmF=~sGhRDp7f4n*aZk8AhKIKM054ZcH#Yna_b}zMh$zmSNU^Qhl)WFl z3;)Tu6#Rn4nyZMbnZINmmkMvD?8*h|mXalhhIgdLse!VO7WSdfzPgnYnGv=vj)qpZ z^9HSPmvN`kc|-!)5S({9un`o`HpWic`U@UUNB4etCiPr)6XW>G(h&3aFzPNx9N;lA z3Tuy#C(GzIQZ1BRyq~EzovS|a45s6wdXjIXx9VGi7bzuU z87=}l_^DO;(63!Re7?uVvteSMZ!F7$+DKQ4R;EI!BE|nCRg9X$4N^|Kuc7V1FI+9> zeX>86W)~_)B0K33XbthEo%E!^J=73nfAd*NAjC(q$pFu(l22=4##8r08$t%dRWdH9`Zv zx_uxe;1|MAJ4v{!zW=gB#~@E#vB2Czjh*9W)0fN<%a|79E*a&(zg@P+O9?Bi9tyNR!q2xMMV=93oq)QHCQi z^Q3r8E!8)6G1h^bP^q5%%*^~^_unGU&nUEj7fu&dn{Fvg^E}pd@f@Z~^3BOQjKy@_ z7)@=1Xd;eXPu`NY+FRhQpdX+*P$+%V_?uem*%29;myxyE@P@K*2C4<~H?i8V61|k? zBbK@143(T)g)FMHli;UG56M057xG={U6r2Tg{j!}kk@m)m0I30g>9K&v7wqX9N=xm zUDIvl7kYuIGP8ph5Q4xD@n*`^Dwxd6blAp3q3_xinAy zn}jToQ%q6lZ%BOM&&JuLfqA2)M3Uc|7oxtR-6Fj79LAclX+$^Lpmu#YKoqRpY^;OZ z>HBCYHw+?$=Hz3(7IGP@gLP-lxYqIA-9t$?Qrgwu^a|GsIS4=^K0W+Dbe30-vTtFv z43fR6EsZT}Vk-pYQyqsFaVLDVZyC7-v|0*@FomF@Z)O6NTE zQ#F^;3+W-Sw(G(qXC@E;hTS95$RGHMQaz@ktdOX(M~Va;nA>7wcyRU}vAKO1ztick zY4zK!TTrXi4=wrG!v3imEjdH!Rd^A~#P8U?{lag@0G3GYI`eBW44{`&+?`D zXeC_=7p<(KHlpbhcEC49>dqv%SA;j@rYf=Ny_mXU7`870mg7n7465xajwmLCs0P}E4JZQsE;9v zL2FlHTxDvcuK`<3_ErWnq0$+0Mt(V$mq^uLW=fbkNr$Kj#*-ARuyVTkc%nM?mwlx; zP~NR3WIwV>AJ@`H$ah^yoCn{FT@%h2UeM#!BI}2u0SvXhv9@4DUSmE)UN??VyQ6dy zPuO-r9noxMs`Iocy6QU(u7TV;kd>cA*0AN!*GzrkKg3#&qZH#@m{$^%d$#!qVp(+KNjM^V4KGZ~y&LPb!$DAco;^r?>;cg2>|O$&d* z9(R6IZMmxj`3mw#$!uq9Tdsht%085P;v?`W%H;gco_y^?r-m?M1>{q}$Hsaq8E4V^ zxcT}g=sTvet%-8QJBCl!zozOc2SL@C=4wxc^2Rk%)3EfaotSvv6Oe8y&{b6Hv1hqQ z*!OrfdEIdVE8zPQ5FPIxt?f&MTeom?Tt8=*6Av=0;ET{wkcdI%W5O%b3VlB;TU_o5 z<&PUA*G2T22$MCDk<29PFZ_JxC3hTIn@T4TaS^T)88>cf1Gm%P)HifpM{7%EyiifvvJR ziCj%Og_E`|qR|0qY;m!=Nu~t+Ft!MO?*3lr?CC3(x9)Zd*evG>!yfntyO#Y!TO>TA z`ib?)@4-Deg&N^Nxpq({R>kKK=XkH7fATlECZ6B4YQ$Ewi8IA!-l6y?CHno7fe%*c zLl0ZkDxD)n*pj6P-W+{WTiUe7`HD%UIzflP{Z$j~y`?+APR?TMX`a(|Pcoj$G*Ze! zInYcnRgpZ~0g#@EEEc+=(B-lYsCEI?8y_+0Mo}6HZxryn$);`eqUeq2UZ>4mNEnoD z&Un*mQ$@AFF&sHjuf7|4=G+aOo@?wRAiKv>8qmWTPS!Jhiw)!N z;Z@i*_FB4pU2VugRYbvY*?nyoIj#N{EvkUiG!4OE*vUpc|sj!_D&Bc^+cGehY>0 zLG&fEo^b|O9qc##Kz*OS&(lD3LFve3Wvb_zsdS-}TtsHm^~BB2bNCo(6L`u7QCXxU z=fP!6>!hZ6v8c0PC%N2S$v9f5MZR`RAX;n&so`g`5ou+&8;^oy;Xe2VzE957jB!K! z0M(sdA+KUzlk;Lq!4B&M@+G107SB_ufF2Hx+1K(HO-r4{v3ercv|0R>oI|4gLShuQ zPEg1lgw2SM1w=V_Gtk6~K?4gy)iyWB_h?9p>PA6WdnJBiFjR z3ItnrC0+l0`zri@N3Dz*G<3+2l(EAn#ro}w+dAY2S4H=XMrERpRR zVssxs8@VcJuHmP#(~ucGW%pr&ogIi5_GsI2BEjB2uWNRRf?zCJ zAHrTh&DN5J**ZC=A?ibB+Ji}ooM&$f>6~Fiys@^^OP$8Tp@mScrV+l`AhU_sJlqZ= zx}xaYXc=^9P6GZTQ39&wyaY*526mov*zTC7X(k(Z%~WhFf){2WjhO}3>sf81Uc!B< zc45PXdqgojqi~S!UGzG4QvNG7F%HFCb+ig2^~;drLY93>+=%=xFIF2DlUDVuixLs? zg64F@;PTl9%-R87>q%l6?IusEwyU+P<5kvqkPGG_qO+mqO5uUXAarQqeU?rxU+Vq0 z|9^vD|M{j<$Bh~~wts5c_@V##`-fH)GLf%mm(F;ee>LVeUPDCW2U|B4Y_PYUJ=V}R zA_b?PS1do|J-+#K4VgM_;*kH0ysP5OjLza(XlG099ZmL_ysf@>3{c4%93w&`a)Qp_mG zu`ZNe!j;XX88_Ao9E6)dcfnoc1w%3f&Z%$tJLyy9)aYF1PcsQ$aTnTo%{K8{^fxAr znF}ovctnG~0MF4W=8X)8jsjgZZ;=$S8nl#2HuZyAf&V`Z3zumeVUM}9#v}Igo^^z=Jgr+&SN>hj#>Uyt3h2pxX^+<~BBwDE7 z=ML9eA;ho`+A3`&vdxJMZ5t21i;jbBq6uGVo?^Nb^#Hi4j>3b*N7w_Vo_Q>EiH*`# zc21DfLtB|O=0Eg3`R;``;XE)mJIvggS&f|Hb6wjoOne4qGT$+B*hbI-%7#{nbKq?A z1ZJS&jF6(Mt2+-~!HD^~`A2(s&?mg5HW9qeNif%D?pCfYT(%uva#B1Bhnd$i({%Si zJNIy+g;-7Z1Cz>Rpp!u>C_8$&WUTmt8Dt)#i4$t&7et%_=7K)4nWje2Pe7+xhlHyy za`g;iUJjKNj#`5uOd0_{Fpo3!fg<=~+c12C*aSaip3e}f{~no7Wn1_pW(T7n=Rq=(h13#<Yi)zQ6c2_?Q1y9n{ylD=dzUT^yfd^iPhu+LabhKCd`zx130WSy z+O*%kRftE9S=Wg#;SuKQrm{9xh{;UB=Zi_uRdZc)gnLL9~+Y!yU{R16f(|PNC7TRH{PMH&unk>)^akwo? zw?|C~Y0ZC|k_}q{m)%GW_xD7$nPZqqSX0o*J{`FVeu_TG95P?i`#22kjQs+ppr_4w z%nRHJWY3|zQ_{w;1x#lZzos~ny#)^v&!QgYCz?RgMGDfuJHQV77jp%3FGEj|n%CEG z0{n^1Vt!$MS%irVa?|mHk{(JoO*H>$NR(#VPD33)HuBh<&-{sHfX%x3j@eQPWJqj0 z(-esYd!zf~!_|_MYs_GCcU>hh)wW;#{3FA3b7My9Jj&a!pP(?Qd%lO+z*L7<3nk&d zuxz#M8)L3+{u3D}Rz_<;wA=?D$ING%!_9?;+6UN3sXB6yX<~j0iTn(GGA2pU1#{FW zg;Cd6;H=@qJ8@#(H|8?tUr~$O9x@6IlBvS>%p~SI>H@3P>XW--Vdzb!7PADeBt(S_ z)8>PEP;l%phVJs-?pXfU?tC&I{NwT9U(6F2q&DvVzxgeyx5AC9>F^~guGN1eblCJ$ zUAo(Q(E9%^=%;Q=$YaYH%PIAb%ks)X*CXrE9p1wx{yW$qL&r`U`hLK`hjnJDg^TC- zqKMu|8%HbUKrG$nJ)pm7?jH+3Y3S%-v7gPEYaKFpMg6MN7xi3mXT{r4eP)+Xy4`!Q zgs-gbJ(xO_U}D;*Ym45zEb-el&5BZsZmd31qr33mzuK#C;qDN6GO7gbZH7XisoG}8CXJI`zPJ_ma5bnh1BsgIWC z+LPtIl1-`Ob9gx*7k#6cy={fnBHil!SL1Kmef9r0D&#+S(g%&hi}FBiuuQX598%*~ zCE%8Qp{1WGltuNZTUu&c-Xt}%G)iihL^pfCuK#noK2!ex1M{OsuhH#fE43|lc`Q42 zE@ckHp?^Elrgg}#E=`Rc$}KcsNvZW_&5QT|90~~fnaJBM9?N-4p=Gwpq{WuamH-^` z1M5flcxAFP#oK_pLe*F5n#ZmVupu7<`sWb6@?LgPM)0benZaSHE8Yd-&=+8SI5Zi* z=+d!WxFc%!)1&MlIRJ;gfb+wlvn#h+aaTn+mHjPry*?!nhrTvOsCC0+Wrz2bH?(-R zD^%&}UE&M!cK4+zW4%||E%IL9dGB~7U2LdK_DV{mGDcY;&$10vhA0)iIljN-x5_gi z0EYrkX=xo&jJGPf?r!4@CZ81_jn%CSxMg4CkkztWJ&0=Z8sK z^9j5cLg`NESWg#C9!t-7Z;k$gv|lqvjP#0MF0D~~(i|m8q^_h@>y?jrTZ4OwLteqR@qMo}RZQ-Dd0r%~&P-4vKz=74ea#*X^7`;xU>6hB z?8GxyJ2CLSeNC@gS-Puq>R{<(NwmC68e^fGzh9Z(p;{kS=0}@?%n`mla=+?TZr8FoDvh)n@eKeI^_ED@ISmT{IO%cGc*`@HVMx<~5%UH4ht@T9AC1F`AD z%KT_kM(j1YG}(-~El+mz6o)_o?npI18h-u7@;)I`?byF!ZORO&jCX0R&BZ+u0{C|U zK|hoEdrPW?wal`7XR%s-Nm>xdzw-m@M|guY8j4lAlTB+K%fF-?mIF8o9|QVlQoWE7 zdD9|K<;~22tuG?aS_3#_Ux4``($x@xxT!<++p=bP_8?K*e*@=-M>n8rniU>X-kzw{ z)~1!Nu>tN#HNP?@vUS6a@-W{!ab(dEWvF+rvQC2JS4w&DHD93op|nssDIL5c1dZ(Q zsZU9i$;w>spWc2_s5f8g<7@3b@7;$4@b7AV<&HE|jPPxC^=5VSsp3fb}AV(SiKC59{)yMpwD!{EVoR_C`Q52OImD18}I;zvfV=DGU3n zXaRrTyR-VWqUC|^NC81V6FI{&SiKwdvFI&*EtOTl1Kg4P!1@t>7T$~+0xl8XP$T0! zOmd)0>&JlpIn-&z?64(~k1G$#uab9Q)%uVCO8*6z9}e{)1|#9xH3Sh>16pXBivKro zemHa#KWCi{rRKLPtmX+aOmqg|P_3_WsE?NrZ|FOFSBk-6H+d)5ny(;S_vU+>D#hwu z=$KMpnQtE)bJ`Ulk5$$i`zt4k1m&P|b=})y;Qe0MAqU`4t*>yX2~dke=pSsG$zHxV zJUY;|)(_NY3N^-(ZE3GMI5X5N-iD+_fjIO5%#Ru!bWaoO*`~W;@i60FdMY*mhvNS= zhoq=wN_epkEcY5?NBjEJ2w;Z-f_^4)iY3f4-eRzzmIfA+Wn>@@`GNH#d?RBurk%9| zy_ng;l=18j#G#J?{d4GQ<^*?1Pp0#E?nv95$X}fSl>Q4aKOCZ0uXBGJ-!2hZZHXOO ztn&LmhyK2R^TVMO*Gkr+t(`L(X-^bL(`*4a6#rEYH4!7#PU24$jO|>ED2meFJ4@*+ zrb_L655@Jq1Q{qLy*s=M*-G9v#UqqtV*fca6Q+iJ7?OmSv(qWGQZ zTVBnV{O5Rg!oTJaoZr*Cr|3D=qWBdvXm$0P0XP&8^fQs0ScY3hSQrap!7O)@$^_z& zA6P%a4|6T}W#966{;+rA3*890Q1A4 zY;h0L+gX>LNe)s=Jm!=C4V)hiCE#bX+eg-jX@M8VY|wWJWQP*I%At5LNJ;ZTu1QKE z-(89lQ+%YMuXs@!qul3K6i$_5<+5TIZ&@X+$mbd9TL=aThtz37SD^atJK|6FFmC`3 zC47ZL_j9zm9^`4kWztBGsRUoZE&G};_q*k~<+NpwWxx9OVD%>ehd!*!j~eMb1C;IF zZRDD~YV^-M2*jb<|C&SL+73k(R(FXj>+ONZtvL}FfI|U6KjTocB}M(|rQV7t%dMoR zNdbI0Kd^p;?+Z+h(L@T@Gj;?G$F1N3Y{y?S3v+fRQszOdMuz~f5qk9 zsgA3&3gf+za#?VM%w2s#tjxDkE*71XYb$?ACSOo7#U~cGQ|B}_@uI$U@|oh_eUGF8 zz5pDm{S^)^ao-}=()*RN#?7D=86U_F`GNXOp$aW))JS&^^^!fuQq|Hs5Qjd1`B9^9 z)XvAdl}AJ~S0$tyDh1+D;=krlEY+%bnvxnDu2`|(iZfyZa3~%<_sj0H+ zq>w0B1Nm}^U*XWtVhp*AJt^%Zosvv^7l=cCpgvQm4wljCeW;P8o64h-YW_8V9r^&~ zM~#x>DtHh29#?P3I>0Z+$$>aj=U;P3XD(YjqWF&*C!{RbA4Ohs01gEN{Y>O$mNBZh zb=9asdDWMz5Qsy5VEqWM`t?*~TunktZG$(Ovz-Ar^f91+4!v19G`uy`-##QaD}vDW z%MQSyFTnh8=#|=XbZ+%i;yCejai!uM@xOud!yygp&OGRjagArZTxsR7F#w0^e3e5} z|I9&dCh8W^(cQv`BU7gtd!O&hrPklBdLc{ zL2L`wi^(8>9jfyc4kak3s7?0bqI5h%*+mBh^5y(MeWp-tEHl*$c4IYNG}%(lGAIy- zK7jdAqh&;q*CwY|&$nY>0>qF3?n8C|HHU^%rHY4mmsPi_?JKjqS4{yp6cF??k^5QN zsscB$)KopWmq~vG^5y)%`Vl^j`wqLpwB%QsdO8+X$1(zN=wm?t9Qr2P5z{uiz3p+i zy}C7}GF<_b{tGZa99nHYhI^Uq{8^p6E@|E9Krw|c;QVmtj=eE>t!v4SL+Zit?&e4U z4%PiChhE5|lpp1GLWZ|7e^kf;EqqOsQqi`TtS*Ebwqw-AeNN==K z+B-+tCLi>T0GGU#0{L=vzrvw@-Z=VCzNuon81K#ENV<3Rpb9Bt$4^e}pE6`pC>jKp|0YxyR53JY z(7@EeBl=HDnKm@2QfTm)f#3C?G;~7$(L=}ft`QXUzn>U0aMGxM_mP@G!IOp#A2W39 vzyCFZkf5@YQU^{<4UGzlEFTnF?XyLdEbert.encoder.layer.11.attention.output.LayerNorm.weight_resize CastDef_1283 CastDef_1283"Cast* +to20cast node to cast from float16 to float32 on cpu +e +1578 CastDef_1276 CastDef_1276"Cast* +to20cast node to cast from float16 to float32 on cpu +e +1575 CastDef_1271 CastDef_1271"Cast* +to20cast node to cast from float16 to float32 on cpu + +8bert.encoder.layer.11.attention.output.dense.bias_resize CastDef_1262 CastDef_1262"Cast* +to20cast node to cast from float16 to float32 on cpu +l + 1770_resize CastDef_1259 CastDef_1259"Cast* +to20cast node to cast from float16 to float32 on cpu + +6bert.encoder.layer.11.attention.self.value.bias_resize CastDef_1253 CastDef_1253"Cast* +to20cast node to cast from float16 to float32 on cpu +l + 1762_resize CastDef_1250 CastDef_1250"Cast* +to20cast node to cast from float16 to float32 on cpu + +2bert.encoder.layer.10.output.LayerNorm.bias_resize CastDef_1227 CastDef_1227"Cast* +to20cast node to cast from float16 to float32 on cpu + +4bert.encoder.layer.10.output.LayerNorm.weight_resize CastDef_1224 CastDef_1224"Cast* +to20cast node to cast from float16 to float32 on cpu +e +1490 CastDef_1217 CastDef_1217"Cast* +to20cast node to cast from float16 to float32 on cpu +e +1487 CastDef_1212 CastDef_1212"Cast* +to20cast node to cast from float16 to float32 on cpu + +.bert.encoder.layer.10.output.dense.bias_resize CastDef_1203 CastDef_1203"Cast* +to20cast node to cast from float16 to float32 on cpu +l + 1759_resize CastDef_1200 CastDef_1200"Cast* +to20cast node to cast from float16 to float32 on cpu +e +1479 CastDef_1197 CastDef_1197"Cast* +to20cast node to cast from float16 to float32 on cpu +e +1476 CastDef_1192 CastDef_1192"Cast* +to20cast node to cast from float16 to float32 on cpu +e +1473 CastDef_1187 CastDef_1187"Cast* +to20cast node to cast from float16 to float32 on cpu + +4bert.encoder.layer.10.intermediate.dense.bias_resize CastDef_1184 CastDef_1184"Cast* +to20cast node to cast from float16 to float32 on cpu +l + 1758_resize CastDef_1181 CastDef_1181"Cast* +to20cast node to cast from float16 to float32 on cpu + +bert.encoder.layer.10.attention.output.LayerNorm.weight_resize CastDef_1175 CastDef_1175"Cast* +to20cast node to cast from float16 to float32 on cpu +e +1464 CastDef_1168 CastDef_1168"Cast* +to20cast node to cast from float16 to float32 on cpu +e +1461 CastDef_1163 CastDef_1163"Cast* +to20cast node to cast from float16 to float32 on cpu + +8bert.encoder.layer.10.attention.output.dense.bias_resize CastDef_1154 CastDef_1154"Cast* +to20cast node to cast from float16 to float32 on cpu +l + 1757_resize CastDef_1151 CastDef_1151"Cast* +to20cast node to cast from float16 to float32 on cpu + +6bert.encoder.layer.10.attention.self.value.bias_resize CastDef_1145 CastDef_1145"Cast* +to20cast node to cast from float16 to float32 on cpu +l + 1749_resize CastDef_1142 CastDef_1142"Cast* +to20cast node to cast from float16 to float32 on cpu + +1bert.encoder.layer.9.output.LayerNorm.bias_resize CastDef_1119 CastDef_1119"Cast* +to20cast node to cast from float16 to float32 on cpu + +3bert.encoder.layer.9.output.LayerNorm.weight_resize CastDef_1116 CastDef_1116"Cast* +to20cast node to cast from float16 to float32 on cpu +e +1376 CastDef_1109 CastDef_1109"Cast* +to20cast node to cast from float16 to float32 on cpu +e +1373 CastDef_1104 CastDef_1104"Cast* +to20cast node to cast from float16 to float32 on cpu + +-bert.encoder.layer.9.output.dense.bias_resize CastDef_1095 CastDef_1095"Cast* +to20cast node to cast from float16 to float32 on cpu +l + 1746_resize CastDef_1092 CastDef_1092"Cast* +to20cast node to cast from float16 to float32 on cpu +e +1365 CastDef_1089 CastDef_1089"Cast* +to20cast node to cast from float16 to float32 on cpu +e +1362 CastDef_1084 CastDef_1084"Cast* +to20cast node to cast from float16 to float32 on cpu +e +1359 CastDef_1079 CastDef_1079"Cast* +to20cast node to cast from float16 to float32 on cpu + +3bert.encoder.layer.9.intermediate.dense.bias_resize CastDef_1076 CastDef_1076"Cast* +to20cast node to cast from float16 to float32 on cpu +l + 1745_resize CastDef_1073 CastDef_1073"Cast* +to20cast node to cast from float16 to float32 on cpu + +;bert.encoder.layer.9.attention.output.LayerNorm.bias_resize CastDef_1070 CastDef_1070"Cast* +to20cast node to cast from float16 to float32 on cpu + +=bert.encoder.layer.9.attention.output.LayerNorm.weight_resize CastDef_1067 CastDef_1067"Cast* +to20cast node to cast from float16 to float32 on cpu +e +1350 CastDef_1060 CastDef_1060"Cast* +to20cast node to cast from float16 to float32 on cpu +e +1347 CastDef_1055 CastDef_1055"Cast* +to20cast node to cast from float16 to float32 on cpu + +7bert.encoder.layer.9.attention.output.dense.bias_resize CastDef_1046 CastDef_1046"Cast* +to20cast node to cast from float16 to float32 on cpu +l + 1744_resize CastDef_1043 CastDef_1043"Cast* +to20cast node to cast from float16 to float32 on cpu + +5bert.encoder.layer.9.attention.self.value.bias_resize CastDef_1037 CastDef_1037"Cast* +to20cast node to cast from float16 to float32 on cpu +l + 1736_resize CastDef_1034 CastDef_1034"Cast* +to20cast node to cast from float16 to float32 on cpu + +1bert.encoder.layer.8.output.LayerNorm.bias_resize CastDef_1011 CastDef_1011"Cast* +to20cast node to cast from float16 to float32 on cpu + +3bert.encoder.layer.8.output.LayerNorm.weight_resize CastDef_1008 CastDef_1008"Cast* +to20cast node to cast from float16 to float32 on cpu +e +1262 CastDef_1001 CastDef_1001"Cast* +to20cast node to cast from float16 to float32 on cpu +c +1259 CastDef_996 CastDef_996"Cast* +to20cast node to cast from float16 to float32 on cpu + +-bert.encoder.layer.8.output.dense.bias_resize CastDef_987 CastDef_987"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1733_resize CastDef_984 CastDef_984"Cast* +to20cast node to cast from float16 to float32 on cpu +c +1251 CastDef_981 CastDef_981"Cast* +to20cast node to cast from float16 to float32 on cpu +c +1248 CastDef_976 CastDef_976"Cast* +to20cast node to cast from float16 to float32 on cpu +c +1245 CastDef_971 CastDef_971"Cast* +to20cast node to cast from float16 to float32 on cpu + +3bert.encoder.layer.8.intermediate.dense.bias_resize CastDef_968 CastDef_968"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1732_resize CastDef_965 CastDef_965"Cast* +to20cast node to cast from float16 to float32 on cpu + +;bert.encoder.layer.8.attention.output.LayerNorm.bias_resize CastDef_962 CastDef_962"Cast* +to20cast node to cast from float16 to float32 on cpu + +=bert.encoder.layer.8.attention.output.LayerNorm.weight_resize CastDef_959 CastDef_959"Cast* +to20cast node to cast from float16 to float32 on cpu +c +1236 CastDef_952 CastDef_952"Cast* +to20cast node to cast from float16 to float32 on cpu +c +1233 CastDef_947 CastDef_947"Cast* +to20cast node to cast from float16 to float32 on cpu + +7bert.encoder.layer.8.attention.output.dense.bias_resize CastDef_938 CastDef_938"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1731_resize CastDef_935 CastDef_935"Cast* +to20cast node to cast from float16 to float32 on cpu + +5bert.encoder.layer.8.attention.self.value.bias_resize CastDef_929 CastDef_929"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1723_resize CastDef_926 CastDef_926"Cast* +to20cast node to cast from float16 to float32 on cpu + +1bert.encoder.layer.7.output.LayerNorm.bias_resize CastDef_903 CastDef_903"Cast* +to20cast node to cast from float16 to float32 on cpu + +3bert.encoder.layer.7.output.LayerNorm.weight_resize CastDef_900 CastDef_900"Cast* +to20cast node to cast from float16 to float32 on cpu +c +1148 CastDef_893 CastDef_893"Cast* +to20cast node to cast from float16 to float32 on cpu +c +1145 CastDef_888 CastDef_888"Cast* +to20cast node to cast from float16 to float32 on cpu + +-bert.encoder.layer.7.output.dense.bias_resize CastDef_879 CastDef_879"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1720_resize CastDef_876 CastDef_876"Cast* +to20cast node to cast from float16 to float32 on cpu +c +1137 CastDef_873 CastDef_873"Cast* +to20cast node to cast from float16 to float32 on cpu +c +1134 CastDef_868 CastDef_868"Cast* +to20cast node to cast from float16 to float32 on cpu +c +1131 CastDef_863 CastDef_863"Cast* +to20cast node to cast from float16 to float32 on cpu + +3bert.encoder.layer.7.intermediate.dense.bias_resize CastDef_860 CastDef_860"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1719_resize CastDef_857 CastDef_857"Cast* +to20cast node to cast from float16 to float32 on cpu + +;bert.encoder.layer.7.attention.output.LayerNorm.bias_resize CastDef_854 CastDef_854"Cast* +to20cast node to cast from float16 to float32 on cpu + +=bert.encoder.layer.7.attention.output.LayerNorm.weight_resize CastDef_851 CastDef_851"Cast* +to20cast node to cast from float16 to float32 on cpu +c +1122 CastDef_844 CastDef_844"Cast* +to20cast node to cast from float16 to float32 on cpu +c +1119 CastDef_839 CastDef_839"Cast* +to20cast node to cast from float16 to float32 on cpu + +7bert.encoder.layer.7.attention.output.dense.bias_resize CastDef_830 CastDef_830"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1718_resize CastDef_827 CastDef_827"Cast* +to20cast node to cast from float16 to float32 on cpu + +5bert.encoder.layer.7.attention.self.value.bias_resize CastDef_821 CastDef_821"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1710_resize CastDef_818 CastDef_818"Cast* +to20cast node to cast from float16 to float32 on cpu + +1bert.encoder.layer.6.output.LayerNorm.bias_resize CastDef_795 CastDef_795"Cast* +to20cast node to cast from float16 to float32 on cpu + +3bert.encoder.layer.6.output.LayerNorm.weight_resize CastDef_792 CastDef_792"Cast* +to20cast node to cast from float16 to float32 on cpu +c +1034 CastDef_785 CastDef_785"Cast* +to20cast node to cast from float16 to float32 on cpu +c +1031 CastDef_780 CastDef_780"Cast* +to20cast node to cast from float16 to float32 on cpu + +-bert.encoder.layer.6.output.dense.bias_resize CastDef_771 CastDef_771"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1707_resize CastDef_768 CastDef_768"Cast* +to20cast node to cast from float16 to float32 on cpu +c +1023 CastDef_765 CastDef_765"Cast* +to20cast node to cast from float16 to float32 on cpu +c +1020 CastDef_760 CastDef_760"Cast* +to20cast node to cast from float16 to float32 on cpu +c +1017 CastDef_755 CastDef_755"Cast* +to20cast node to cast from float16 to float32 on cpu + +3bert.encoder.layer.6.intermediate.dense.bias_resize CastDef_752 CastDef_752"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1706_resize CastDef_749 CastDef_749"Cast* +to20cast node to cast from float16 to float32 on cpu + +;bert.encoder.layer.6.attention.output.LayerNorm.bias_resize CastDef_746 CastDef_746"Cast* +to20cast node to cast from float16 to float32 on cpu + +=bert.encoder.layer.6.attention.output.LayerNorm.weight_resize CastDef_743 CastDef_743"Cast* +to20cast node to cast from float16 to float32 on cpu +c +1008 CastDef_736 CastDef_736"Cast* +to20cast node to cast from float16 to float32 on cpu +c +1005 CastDef_731 CastDef_731"Cast* +to20cast node to cast from float16 to float32 on cpu + +7bert.encoder.layer.6.attention.output.dense.bias_resize CastDef_722 CastDef_722"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1705_resize CastDef_719 CastDef_719"Cast* +to20cast node to cast from float16 to float32 on cpu + +5bert.encoder.layer.6.attention.self.value.bias_resize CastDef_713 CastDef_713"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1697_resize CastDef_710 CastDef_710"Cast* +to20cast node to cast from float16 to float32 on cpu + +1bert.encoder.layer.5.output.LayerNorm.bias_resize CastDef_687 CastDef_687"Cast* +to20cast node to cast from float16 to float32 on cpu + +3bert.encoder.layer.5.output.LayerNorm.weight_resize CastDef_684 CastDef_684"Cast* +to20cast node to cast from float16 to float32 on cpu +b +920 CastDef_677 CastDef_677"Cast* +to20cast node to cast from float16 to float32 on cpu +b +917 CastDef_672 CastDef_672"Cast* +to20cast node to cast from float16 to float32 on cpu + +-bert.encoder.layer.5.output.dense.bias_resize CastDef_663 CastDef_663"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1694_resize CastDef_660 CastDef_660"Cast* +to20cast node to cast from float16 to float32 on cpu +b +909 CastDef_657 CastDef_657"Cast* +to20cast node to cast from float16 to float32 on cpu +b +906 CastDef_652 CastDef_652"Cast* +to20cast node to cast from float16 to float32 on cpu +b +903 CastDef_647 CastDef_647"Cast* +to20cast node to cast from float16 to float32 on cpu + +3bert.encoder.layer.5.intermediate.dense.bias_resize CastDef_644 CastDef_644"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1693_resize CastDef_641 CastDef_641"Cast* +to20cast node to cast from float16 to float32 on cpu + +;bert.encoder.layer.5.attention.output.LayerNorm.bias_resize CastDef_638 CastDef_638"Cast* +to20cast node to cast from float16 to float32 on cpu + +=bert.encoder.layer.5.attention.output.LayerNorm.weight_resize CastDef_635 CastDef_635"Cast* +to20cast node to cast from float16 to float32 on cpu +b +894 CastDef_628 CastDef_628"Cast* +to20cast node to cast from float16 to float32 on cpu +b +891 CastDef_623 CastDef_623"Cast* +to20cast node to cast from float16 to float32 on cpu + +7bert.encoder.layer.5.attention.output.dense.bias_resize CastDef_614 CastDef_614"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1692_resize CastDef_611 CastDef_611"Cast* +to20cast node to cast from float16 to float32 on cpu + +5bert.encoder.layer.5.attention.self.value.bias_resize CastDef_605 CastDef_605"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1684_resize CastDef_602 CastDef_602"Cast* +to20cast node to cast from float16 to float32 on cpu + +1bert.encoder.layer.4.output.LayerNorm.bias_resize CastDef_579 CastDef_579"Cast* +to20cast node to cast from float16 to float32 on cpu + +3bert.encoder.layer.4.output.LayerNorm.weight_resize CastDef_576 CastDef_576"Cast* +to20cast node to cast from float16 to float32 on cpu +b +806 CastDef_569 CastDef_569"Cast* +to20cast node to cast from float16 to float32 on cpu +b +803 CastDef_564 CastDef_564"Cast* +to20cast node to cast from float16 to float32 on cpu + +-bert.encoder.layer.4.output.dense.bias_resize CastDef_555 CastDef_555"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1681_resize CastDef_552 CastDef_552"Cast* +to20cast node to cast from float16 to float32 on cpu +b +795 CastDef_549 CastDef_549"Cast* +to20cast node to cast from float16 to float32 on cpu +b +792 CastDef_544 CastDef_544"Cast* +to20cast node to cast from float16 to float32 on cpu +b +789 CastDef_539 CastDef_539"Cast* +to20cast node to cast from float16 to float32 on cpu + +3bert.encoder.layer.4.intermediate.dense.bias_resize CastDef_536 CastDef_536"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1680_resize CastDef_533 CastDef_533"Cast* +to20cast node to cast from float16 to float32 on cpu + +;bert.encoder.layer.4.attention.output.LayerNorm.bias_resize CastDef_530 CastDef_530"Cast* +to20cast node to cast from float16 to float32 on cpu + +=bert.encoder.layer.4.attention.output.LayerNorm.weight_resize CastDef_527 CastDef_527"Cast* +to20cast node to cast from float16 to float32 on cpu +b +780 CastDef_520 CastDef_520"Cast* +to20cast node to cast from float16 to float32 on cpu +b +777 CastDef_515 CastDef_515"Cast* +to20cast node to cast from float16 to float32 on cpu + +7bert.encoder.layer.4.attention.output.dense.bias_resize CastDef_506 CastDef_506"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1679_resize CastDef_503 CastDef_503"Cast* +to20cast node to cast from float16 to float32 on cpu + +5bert.encoder.layer.4.attention.self.value.bias_resize CastDef_497 CastDef_497"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1671_resize CastDef_494 CastDef_494"Cast* +to20cast node to cast from float16 to float32 on cpu + +1bert.encoder.layer.3.output.LayerNorm.bias_resize CastDef_471 CastDef_471"Cast* +to20cast node to cast from float16 to float32 on cpu + +3bert.encoder.layer.3.output.LayerNorm.weight_resize CastDef_468 CastDef_468"Cast* +to20cast node to cast from float16 to float32 on cpu +b +692 CastDef_461 CastDef_461"Cast* +to20cast node to cast from float16 to float32 on cpu +b +689 CastDef_456 CastDef_456"Cast* +to20cast node to cast from float16 to float32 on cpu + +-bert.encoder.layer.3.output.dense.bias_resize CastDef_447 CastDef_447"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1668_resize CastDef_444 CastDef_444"Cast* +to20cast node to cast from float16 to float32 on cpu +b +681 CastDef_441 CastDef_441"Cast* +to20cast node to cast from float16 to float32 on cpu +b +678 CastDef_436 CastDef_436"Cast* +to20cast node to cast from float16 to float32 on cpu +b +675 CastDef_431 CastDef_431"Cast* +to20cast node to cast from float16 to float32 on cpu + +3bert.encoder.layer.3.intermediate.dense.bias_resize CastDef_428 CastDef_428"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1667_resize CastDef_425 CastDef_425"Cast* +to20cast node to cast from float16 to float32 on cpu + +;bert.encoder.layer.3.attention.output.LayerNorm.bias_resize CastDef_422 CastDef_422"Cast* +to20cast node to cast from float16 to float32 on cpu + +=bert.encoder.layer.3.attention.output.LayerNorm.weight_resize CastDef_419 CastDef_419"Cast* +to20cast node to cast from float16 to float32 on cpu +b +666 CastDef_412 CastDef_412"Cast* +to20cast node to cast from float16 to float32 on cpu +b +663 CastDef_407 CastDef_407"Cast* +to20cast node to cast from float16 to float32 on cpu + +7bert.encoder.layer.3.attention.output.dense.bias_resize CastDef_398 CastDef_398"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1666_resize CastDef_395 CastDef_395"Cast* +to20cast node to cast from float16 to float32 on cpu + +5bert.encoder.layer.3.attention.self.value.bias_resize CastDef_389 CastDef_389"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1658_resize CastDef_386 CastDef_386"Cast* +to20cast node to cast from float16 to float32 on cpu + +1bert.encoder.layer.2.output.LayerNorm.bias_resize CastDef_363 CastDef_363"Cast* +to20cast node to cast from float16 to float32 on cpu + +3bert.encoder.layer.2.output.LayerNorm.weight_resize CastDef_360 CastDef_360"Cast* +to20cast node to cast from float16 to float32 on cpu +b +578 CastDef_353 CastDef_353"Cast* +to20cast node to cast from float16 to float32 on cpu +b +575 CastDef_348 CastDef_348"Cast* +to20cast node to cast from float16 to float32 on cpu + +-bert.encoder.layer.2.output.dense.bias_resize CastDef_339 CastDef_339"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1655_resize CastDef_336 CastDef_336"Cast* +to20cast node to cast from float16 to float32 on cpu +b +567 CastDef_333 CastDef_333"Cast* +to20cast node to cast from float16 to float32 on cpu +b +564 CastDef_328 CastDef_328"Cast* +to20cast node to cast from float16 to float32 on cpu +b +561 CastDef_323 CastDef_323"Cast* +to20cast node to cast from float16 to float32 on cpu + +3bert.encoder.layer.2.intermediate.dense.bias_resize CastDef_320 CastDef_320"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1654_resize CastDef_317 CastDef_317"Cast* +to20cast node to cast from float16 to float32 on cpu + +;bert.encoder.layer.2.attention.output.LayerNorm.bias_resize CastDef_314 CastDef_314"Cast* +to20cast node to cast from float16 to float32 on cpu + +=bert.encoder.layer.2.attention.output.LayerNorm.weight_resize CastDef_311 CastDef_311"Cast* +to20cast node to cast from float16 to float32 on cpu +b +552 CastDef_304 CastDef_304"Cast* +to20cast node to cast from float16 to float32 on cpu +b +549 CastDef_299 CastDef_299"Cast* +to20cast node to cast from float16 to float32 on cpu + +7bert.encoder.layer.2.attention.output.dense.bias_resize CastDef_290 CastDef_290"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1653_resize CastDef_287 CastDef_287"Cast* +to20cast node to cast from float16 to float32 on cpu + +5bert.encoder.layer.2.attention.self.value.bias_resize CastDef_281 CastDef_281"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1645_resize CastDef_278 CastDef_278"Cast* +to20cast node to cast from float16 to float32 on cpu + +1bert.encoder.layer.1.output.LayerNorm.bias_resize CastDef_255 CastDef_255"Cast* +to20cast node to cast from float16 to float32 on cpu + +3bert.encoder.layer.1.output.LayerNorm.weight_resize CastDef_252 CastDef_252"Cast* +to20cast node to cast from float16 to float32 on cpu +b +464 CastDef_245 CastDef_245"Cast* +to20cast node to cast from float16 to float32 on cpu +b +461 CastDef_240 CastDef_240"Cast* +to20cast node to cast from float16 to float32 on cpu + +-bert.encoder.layer.1.output.dense.bias_resize CastDef_231 CastDef_231"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1642_resize CastDef_228 CastDef_228"Cast* +to20cast node to cast from float16 to float32 on cpu +b +453 CastDef_225 CastDef_225"Cast* +to20cast node to cast from float16 to float32 on cpu +b +450 CastDef_220 CastDef_220"Cast* +to20cast node to cast from float16 to float32 on cpu +b +447 CastDef_215 CastDef_215"Cast* +to20cast node to cast from float16 to float32 on cpu + +3bert.encoder.layer.1.intermediate.dense.bias_resize CastDef_212 CastDef_212"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1641_resize CastDef_209 CastDef_209"Cast* +to20cast node to cast from float16 to float32 on cpu + +;bert.encoder.layer.1.attention.output.LayerNorm.bias_resize CastDef_206 CastDef_206"Cast* +to20cast node to cast from float16 to float32 on cpu + +=bert.encoder.layer.1.attention.output.LayerNorm.weight_resize CastDef_203 CastDef_203"Cast* +to20cast node to cast from float16 to float32 on cpu +b +438 CastDef_196 CastDef_196"Cast* +to20cast node to cast from float16 to float32 on cpu +b +435 CastDef_191 CastDef_191"Cast* +to20cast node to cast from float16 to float32 on cpu + +7bert.encoder.layer.1.attention.output.dense.bias_resize CastDef_182 CastDef_182"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1640_resize CastDef_179 CastDef_179"Cast* +to20cast node to cast from float16 to float32 on cpu + +5bert.encoder.layer.1.attention.self.value.bias_resize CastDef_173 CastDef_173"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1632_resize CastDef_170 CastDef_170"Cast* +to20cast node to cast from float16 to float32 on cpu + +1bert.encoder.layer.0.output.LayerNorm.bias_resize CastDef_147 CastDef_147"Cast* +to20cast node to cast from float16 to float32 on cpu + +3bert.encoder.layer.0.output.LayerNorm.weight_resize CastDef_144 CastDef_144"Cast* +to20cast node to cast from float16 to float32 on cpu +b +350 CastDef_137 CastDef_137"Cast* +to20cast node to cast from float16 to float32 on cpu +b +347 CastDef_132 CastDef_132"Cast* +to20cast node to cast from float16 to float32 on cpu + +-bert.encoder.layer.0.output.dense.bias_resize CastDef_123 CastDef_123"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1629_resize CastDef_120 CastDef_120"Cast* +to20cast node to cast from float16 to float32 on cpu +b +339 CastDef_117 CastDef_117"Cast* +to20cast node to cast from float16 to float32 on cpu +b +336 CastDef_112 CastDef_112"Cast* +to20cast node to cast from float16 to float32 on cpu +b +333 CastDef_107 CastDef_107"Cast* +to20cast node to cast from float16 to float32 on cpu + +3bert.encoder.layer.0.intermediate.dense.bias_resize CastDef_104 CastDef_104"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1628_resize CastDef_101 CastDef_101"Cast* +to20cast node to cast from float16 to float32 on cpu + +;bert.encoder.layer.0.attention.output.LayerNorm.bias_resize +CastDef_98 +CastDef_98"Cast* +to20cast node to cast from float16 to float32 on cpu + +=bert.encoder.layer.0.attention.output.LayerNorm.weight_resize +CastDef_95 +CastDef_95"Cast* +to20cast node to cast from float16 to float32 on cpu +` +324 +CastDef_88 +CastDef_88"Cast* +to20cast node to cast from float16 to float32 on cpu +` +321 +CastDef_83 +CastDef_83"Cast* +to20cast node to cast from float16 to float32 on cpu + +7bert.encoder.layer.0.attention.output.dense.bias_resize +CastDef_74 +CastDef_74"Cast* +to20cast node to cast from float16 to float32 on cpu +h + 1627_resize +CastDef_71 +CastDef_71"Cast* +to20cast node to cast from float16 to float32 on cpu + +5bert.encoder.layer.0.attention.self.value.bias_resize +CastDef_65 +CastDef_65"Cast* +to20cast node to cast from float16 to float32 on cpu +h + 1619_resize +CastDef_62 +CastDef_62"Cast* +to20cast node to cast from float16 to float32 on cpu + +%bert.embeddings.LayerNorm.bias_resize +CastDef_32 +CastDef_32"Cast* +to20cast node to cast from float16 to float32 on cpu + +'bert.embeddings.LayerNorm.weight_resize +CastDef_29 +CastDef_29"Cast* +to20cast node to cast from float16 to float32 on cpu +` +236 +CastDef_22 +CastDef_22"Cast* +to20cast node to cast from float16 to float32 on cpu +` +233 +CastDef_17 +CastDef_17"Cast* +to20cast node to cast from float16 to float32 on cpu + +1bert.embeddings.position_embeddings.weight_resize CastDef_0 CastDef_0"Cast* +to20cast node to cast from float16 to float32 on cpu + + input_ids214"Shape +& +214 +215216"Gather* +axis +$ +216223" Unsqueeze* +axes@ + + input_ids211"Shape +& +211 +212213"Gather* +axis +$ +213222" Unsqueeze* +axes@ +& +222 +223224"Concat* +axis + +216217"Cast* +to + +218 +217 +219220"Range +$ +220221" Unsqueeze* +axes@ + +221 +224225"Expand +2 + CastDef_0 +225 CastDef_1"Gather* +axis + +-bert.embeddings.word_embeddings.weight_resize CastDef_2 CastDef_2"Cast* +to20cast node to cast from float16 to float32 on cpu +8 + CastDef_2 + input_ids CastDef_3"Gather* +axis +( + CastDef_3 + CastDef_1 CastDef_6"Add + +3bert.embeddings.token_type_embeddings.weight_resize CastDef_7 CastDef_7"Cast* +to20cast node to cast from float16 to float32 on cpu +: + CastDef_7 + segment_ids CastDef_8"Gather* +axis +) + CastDef_6 + CastDef_8 +CastDef_11"Add +M + +CastDef_11 +CastDef_13" +ReduceMean* +axes@* +keepdims ++ + +CastDef_11 + +CastDef_13 +CastDef_15"Sub ++ + +CastDef_15 + +CastDef_17 +CastDef_18"Pow +M + +CastDef_18 +CastDef_20" +ReduceMean* +axes@* +keepdims ++ + +CastDef_20 + +CastDef_22 +CastDef_23"Add + + +CastDef_23 +CastDef_25"Sqrt ++ + +CastDef_15 + +CastDef_25 +CastDef_27"Div ++ + +CastDef_27 + +CastDef_29 +CastDef_30"Mul ++ + +CastDef_30 + +CastDef_32 +CastDef_33"Add +. + +CastDef_33 + +CastDef_62 +CastDef_63"MatMul ++ + +CastDef_63 + +CastDef_65 +CastDef_66"Add +` + +CastDef_66250 +CastDef_66"Cast* +to +20cast node to cast from float16 to float32 on cpu + +250 +292293"Reshape +* +293294" Transpose* +perm@@@@ +` +294 +CastDef_68 +CastDef_68"Cast* +to20cast node to cast from float16 to float32 on cpu +` +297 +CastDef_49 +CastDef_49"Cast* +to20cast node to cast from float16 to float32 on cpu + +3bert.encoder.layer.0.attention.self.key.bias_resize +CastDef_38 +CastDef_38"Cast* +to20cast node to cast from float16 to float32 on cpu +h + 1618_resize +CastDef_35 +CastDef_35"Cast* +to20cast node to cast from float16 to float32 on cpu +. + +CastDef_33 + +CastDef_35 +CastDef_36"MatMul ++ + +CastDef_36 + +CastDef_38 +CastDef_39"Add +` + +CastDef_39247 +CastDef_39"Cast* +to +20cast node to cast from float16 to float32 on cpu + +247 +278279"Reshape +* +279295" Transpose* +perm@@@@ +` +295 +CastDef_46 +CastDef_46"Cast* +to20cast node to cast from float16 to float32 on cpu + +5bert.encoder.layer.0.attention.self.query.bias_resize +CastDef_43 +CastDef_43"Cast* +to20cast node to cast from float16 to float32 on cpu +h + 1617_resize +CastDef_40 +CastDef_40"Cast* +to20cast node to cast from float16 to float32 on cpu +. + +CastDef_33 + +CastDef_40 +CastDef_41"MatMul ++ + +CastDef_41 + +CastDef_43 +CastDef_44"Add +` + +CastDef_44244 +CastDef_44"Cast* +to +20cast node to cast from float16 to float32 on cpu + +244 +263264"Reshape +* +264265" Transpose* +perm@@@@ +` +265 +CastDef_45 +CastDef_45"Cast* +to20cast node to cast from float16 to float32 on cpu +. + +CastDef_45 + +CastDef_46 +CastDef_47"MatMul ++ + +CastDef_47 + +CastDef_49 +CastDef_50"Div +` +209 +CastDef_55 +CastDef_55"Cast* +to20cast node to cast from float16 to float32 on cpu ++ + +input_mask204" Unsqueeze* +axes@ +$ +204205" Unsqueeze* +axes@ + +205206"Cast* +to + +` +206 +CastDef_52 +CastDef_52"Cast* +to20cast node to cast from float16 to float32 on cpu +` +207 +CastDef_51 +CastDef_51"Cast* +to20cast node to cast from float16 to float32 on cpu ++ + +CastDef_51 + +CastDef_52 +CastDef_53"Sub ++ + +CastDef_53 + +CastDef_55 +CastDef_56"Mul ++ + +CastDef_50 + +CastDef_56 +CastDef_59"Add +0 + +CastDef_59 +CastDef_61"Softmax* +axis +. + +CastDef_61 + +CastDef_68 +CastDef_69"MatMul +` + +CastDef_69301 +CastDef_69"Cast* +to +20cast node to cast from float16 to float32 on cpu +* +301302" Transpose* +perm@@@@ + +302 +313314"Reshape +` +314 +CastDef_70 +CastDef_70"Cast* +to20cast node to cast from float16 to float32 on cpu +. + +CastDef_70 + +CastDef_71 +CastDef_72"MatMul ++ + +CastDef_72 + +CastDef_74 +CastDef_75"Add ++ + +CastDef_75 + +CastDef_33 +CastDef_77"Add +M + +CastDef_77 +CastDef_79" +ReduceMean* +axes@* +keepdims ++ + +CastDef_77 + +CastDef_79 +CastDef_81"Sub ++ + +CastDef_81 + +CastDef_83 +CastDef_84"Pow +M + +CastDef_84 +CastDef_86" +ReduceMean* +axes@* +keepdims ++ + +CastDef_86 + +CastDef_88 +CastDef_89"Add + + +CastDef_89 +CastDef_91"Sqrt ++ + +CastDef_81 + +CastDef_91 +CastDef_93"Div ++ + +CastDef_93 + +CastDef_95 +CastDef_96"Mul ++ + +CastDef_96 + +CastDef_98 +CastDef_99"Add +0 + +CastDef_99 + CastDef_101 CastDef_102"MatMul +. + CastDef_102 + CastDef_104 CastDef_105"Add +. + CastDef_105 + CastDef_107 CastDef_108"Div +! + CastDef_108 CastDef_110"Erf +. + CastDef_110 + CastDef_112 CastDef_113"Add +. + CastDef_105 + CastDef_113 CastDef_115"Mul +. + CastDef_115 + CastDef_117 CastDef_118"Mul +1 + CastDef_118 + CastDef_120 CastDef_121"MatMul +. + CastDef_121 + CastDef_123 CastDef_124"Add +- + CastDef_124 + +CastDef_99 CastDef_126"Add +O + CastDef_126 CastDef_128" +ReduceMean* +axes@* +keepdims +. + CastDef_126 + CastDef_128 CastDef_130"Sub +. + CastDef_130 + CastDef_132 CastDef_133"Pow +O + CastDef_133 CastDef_135" +ReduceMean* +axes@* +keepdims +. + CastDef_135 + CastDef_137 CastDef_138"Add +" + CastDef_138 CastDef_140"Sqrt +. + CastDef_130 + CastDef_140 CastDef_142"Div +. + CastDef_142 + CastDef_144 CastDef_145"Mul +. + CastDef_145 + CastDef_147 CastDef_148"Add +1 + CastDef_148 + CastDef_170 CastDef_171"MatMul +. + CastDef_171 + CastDef_173 CastDef_174"Add +b + CastDef_174364 CastDef_174"Cast* +to +20cast node to cast from float16 to float32 on cpu + +364 +406407"Reshape +* +407408" Transpose* +perm@@@@ +b +408 CastDef_176 CastDef_176"Cast* +to20cast node to cast from float16 to float32 on cpu +b +411 CastDef_164 CastDef_164"Cast* +to20cast node to cast from float16 to float32 on cpu + +3bert.encoder.layer.1.attention.self.key.bias_resize CastDef_153 CastDef_153"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1631_resize CastDef_150 CastDef_150"Cast* +to20cast node to cast from float16 to float32 on cpu +1 + CastDef_148 + CastDef_150 CastDef_151"MatMul +. + CastDef_151 + CastDef_153 CastDef_154"Add +b + CastDef_154361 CastDef_154"Cast* +to +20cast node to cast from float16 to float32 on cpu + +361 +392393"Reshape +* +393409" Transpose* +perm@@@@ +b +409 CastDef_161 CastDef_161"Cast* +to20cast node to cast from float16 to float32 on cpu + +5bert.encoder.layer.1.attention.self.query.bias_resize CastDef_158 CastDef_158"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1630_resize CastDef_155 CastDef_155"Cast* +to20cast node to cast from float16 to float32 on cpu +1 + CastDef_148 + CastDef_155 CastDef_156"MatMul +. + CastDef_156 + CastDef_158 CastDef_159"Add +b + CastDef_159358 CastDef_159"Cast* +to +20cast node to cast from float16 to float32 on cpu + +358 +377378"Reshape +* +378379" Transpose* +perm@@@@ +b +379 CastDef_160 CastDef_160"Cast* +to20cast node to cast from float16 to float32 on cpu +1 + CastDef_160 + CastDef_161 CastDef_162"MatMul +. + CastDef_162 + CastDef_164 CastDef_165"Div +- + CastDef_165 + +CastDef_56 CastDef_167"Add +2 + CastDef_167 CastDef_169"Softmax* +axis +1 + CastDef_169 + CastDef_176 CastDef_177"MatMul +b + CastDef_177415 CastDef_177"Cast* +to +20cast node to cast from float16 to float32 on cpu +* +415416" Transpose* +perm@@@@ + +416 +427428"Reshape +b +428 CastDef_178 CastDef_178"Cast* +to20cast node to cast from float16 to float32 on cpu +1 + CastDef_178 + CastDef_179 CastDef_180"MatMul +. + CastDef_180 + CastDef_182 CastDef_183"Add +. + CastDef_183 + CastDef_148 CastDef_185"Add +O + CastDef_185 CastDef_187" +ReduceMean* +axes@* +keepdims +. + CastDef_185 + CastDef_187 CastDef_189"Sub +. + CastDef_189 + CastDef_191 CastDef_192"Pow +O + CastDef_192 CastDef_194" +ReduceMean* +axes@* +keepdims +. + CastDef_194 + CastDef_196 CastDef_197"Add +" + CastDef_197 CastDef_199"Sqrt +. + CastDef_189 + CastDef_199 CastDef_201"Div +. + CastDef_201 + CastDef_203 CastDef_204"Mul +. + CastDef_204 + CastDef_206 CastDef_207"Add +1 + CastDef_207 + CastDef_209 CastDef_210"MatMul +. + CastDef_210 + CastDef_212 CastDef_213"Add +. + CastDef_213 + CastDef_215 CastDef_216"Div +! + CastDef_216 CastDef_218"Erf +. + CastDef_218 + CastDef_220 CastDef_221"Add +. + CastDef_213 + CastDef_221 CastDef_223"Mul +. + CastDef_223 + CastDef_225 CastDef_226"Mul +1 + CastDef_226 + CastDef_228 CastDef_229"MatMul +. + CastDef_229 + CastDef_231 CastDef_232"Add +. + CastDef_232 + CastDef_207 CastDef_234"Add +O + CastDef_234 CastDef_236" +ReduceMean* +axes@* +keepdims +. + CastDef_234 + CastDef_236 CastDef_238"Sub +. + CastDef_238 + CastDef_240 CastDef_241"Pow +O + CastDef_241 CastDef_243" +ReduceMean* +axes@* +keepdims +. + CastDef_243 + CastDef_245 CastDef_246"Add +" + CastDef_246 CastDef_248"Sqrt +. + CastDef_238 + CastDef_248 CastDef_250"Div +. + CastDef_250 + CastDef_252 CastDef_253"Mul +. + CastDef_253 + CastDef_255 CastDef_256"Add +1 + CastDef_256 + CastDef_278 CastDef_279"MatMul +. + CastDef_279 + CastDef_281 CastDef_282"Add +b + CastDef_282478 CastDef_282"Cast* +to +20cast node to cast from float16 to float32 on cpu + +478 +520521"Reshape +* +521522" Transpose* +perm@@@@ +b +522 CastDef_284 CastDef_284"Cast* +to20cast node to cast from float16 to float32 on cpu +b +525 CastDef_272 CastDef_272"Cast* +to20cast node to cast from float16 to float32 on cpu + +3bert.encoder.layer.2.attention.self.key.bias_resize CastDef_261 CastDef_261"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1644_resize CastDef_258 CastDef_258"Cast* +to20cast node to cast from float16 to float32 on cpu +1 + CastDef_256 + CastDef_258 CastDef_259"MatMul +. + CastDef_259 + CastDef_261 CastDef_262"Add +b + CastDef_262475 CastDef_262"Cast* +to +20cast node to cast from float16 to float32 on cpu + +475 +506507"Reshape +* +507523" Transpose* +perm@@@@ +b +523 CastDef_269 CastDef_269"Cast* +to20cast node to cast from float16 to float32 on cpu + +5bert.encoder.layer.2.attention.self.query.bias_resize CastDef_266 CastDef_266"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1643_resize CastDef_263 CastDef_263"Cast* +to20cast node to cast from float16 to float32 on cpu +1 + CastDef_256 + CastDef_263 CastDef_264"MatMul +. + CastDef_264 + CastDef_266 CastDef_267"Add +b + CastDef_267472 CastDef_267"Cast* +to +20cast node to cast from float16 to float32 on cpu + +472 +491492"Reshape +* +492493" Transpose* +perm@@@@ +b +493 CastDef_268 CastDef_268"Cast* +to20cast node to cast from float16 to float32 on cpu +1 + CastDef_268 + CastDef_269 CastDef_270"MatMul +. + CastDef_270 + CastDef_272 CastDef_273"Div +- + CastDef_273 + +CastDef_56 CastDef_275"Add +2 + CastDef_275 CastDef_277"Softmax* +axis +1 + CastDef_277 + CastDef_284 CastDef_285"MatMul +b + CastDef_285529 CastDef_285"Cast* +to +20cast node to cast from float16 to float32 on cpu +* +529530" Transpose* +perm@@@@ + +530 +541542"Reshape +b +542 CastDef_286 CastDef_286"Cast* +to20cast node to cast from float16 to float32 on cpu +1 + CastDef_286 + CastDef_287 CastDef_288"MatMul +. + CastDef_288 + CastDef_290 CastDef_291"Add +. + CastDef_291 + CastDef_256 CastDef_293"Add +O + CastDef_293 CastDef_295" +ReduceMean* +axes@* +keepdims +. + CastDef_293 + CastDef_295 CastDef_297"Sub +. + CastDef_297 + CastDef_299 CastDef_300"Pow +O + CastDef_300 CastDef_302" +ReduceMean* +axes@* +keepdims +. + CastDef_302 + CastDef_304 CastDef_305"Add +" + CastDef_305 CastDef_307"Sqrt +. + CastDef_297 + CastDef_307 CastDef_309"Div +. + CastDef_309 + CastDef_311 CastDef_312"Mul +. + CastDef_312 + CastDef_314 CastDef_315"Add +1 + CastDef_315 + CastDef_317 CastDef_318"MatMul +. + CastDef_318 + CastDef_320 CastDef_321"Add +. + CastDef_321 + CastDef_323 CastDef_324"Div +! + CastDef_324 CastDef_326"Erf +. + CastDef_326 + CastDef_328 CastDef_329"Add +. + CastDef_321 + CastDef_329 CastDef_331"Mul +. + CastDef_331 + CastDef_333 CastDef_334"Mul +1 + CastDef_334 + CastDef_336 CastDef_337"MatMul +. + CastDef_337 + CastDef_339 CastDef_340"Add +. + CastDef_340 + CastDef_315 CastDef_342"Add +O + CastDef_342 CastDef_344" +ReduceMean* +axes@* +keepdims +. + CastDef_342 + CastDef_344 CastDef_346"Sub +. + CastDef_346 + CastDef_348 CastDef_349"Pow +O + CastDef_349 CastDef_351" +ReduceMean* +axes@* +keepdims +. + CastDef_351 + CastDef_353 CastDef_354"Add +" + CastDef_354 CastDef_356"Sqrt +. + CastDef_346 + CastDef_356 CastDef_358"Div +. + CastDef_358 + CastDef_360 CastDef_361"Mul +. + CastDef_361 + CastDef_363 CastDef_364"Add +1 + CastDef_364 + CastDef_386 CastDef_387"MatMul +. + CastDef_387 + CastDef_389 CastDef_390"Add +b + CastDef_390592 CastDef_390"Cast* +to +20cast node to cast from float16 to float32 on cpu + +592 +634635"Reshape +* +635636" Transpose* +perm@@@@ +b +636 CastDef_392 CastDef_392"Cast* +to20cast node to cast from float16 to float32 on cpu +b +639 CastDef_380 CastDef_380"Cast* +to20cast node to cast from float16 to float32 on cpu + +3bert.encoder.layer.3.attention.self.key.bias_resize CastDef_369 CastDef_369"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1657_resize CastDef_366 CastDef_366"Cast* +to20cast node to cast from float16 to float32 on cpu +1 + CastDef_364 + CastDef_366 CastDef_367"MatMul +. + CastDef_367 + CastDef_369 CastDef_370"Add +b + CastDef_370589 CastDef_370"Cast* +to +20cast node to cast from float16 to float32 on cpu + +589 +620621"Reshape +* +621637" Transpose* +perm@@@@ +b +637 CastDef_377 CastDef_377"Cast* +to20cast node to cast from float16 to float32 on cpu + +5bert.encoder.layer.3.attention.self.query.bias_resize CastDef_374 CastDef_374"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1656_resize CastDef_371 CastDef_371"Cast* +to20cast node to cast from float16 to float32 on cpu +1 + CastDef_364 + CastDef_371 CastDef_372"MatMul +. + CastDef_372 + CastDef_374 CastDef_375"Add +b + CastDef_375586 CastDef_375"Cast* +to +20cast node to cast from float16 to float32 on cpu + +586 +605606"Reshape +* +606607" Transpose* +perm@@@@ +b +607 CastDef_376 CastDef_376"Cast* +to20cast node to cast from float16 to float32 on cpu +1 + CastDef_376 + CastDef_377 CastDef_378"MatMul +. + CastDef_378 + CastDef_380 CastDef_381"Div +- + CastDef_381 + +CastDef_56 CastDef_383"Add +2 + CastDef_383 CastDef_385"Softmax* +axis +1 + CastDef_385 + CastDef_392 CastDef_393"MatMul +b + CastDef_393643 CastDef_393"Cast* +to +20cast node to cast from float16 to float32 on cpu +* +643644" Transpose* +perm@@@@ + +644 +655656"Reshape +b +656 CastDef_394 CastDef_394"Cast* +to20cast node to cast from float16 to float32 on cpu +1 + CastDef_394 + CastDef_395 CastDef_396"MatMul +. + CastDef_396 + CastDef_398 CastDef_399"Add +. + CastDef_399 + CastDef_364 CastDef_401"Add +O + CastDef_401 CastDef_403" +ReduceMean* +axes@* +keepdims +. + CastDef_401 + CastDef_403 CastDef_405"Sub +. + CastDef_405 + CastDef_407 CastDef_408"Pow +O + CastDef_408 CastDef_410" +ReduceMean* +axes@* +keepdims +. + CastDef_410 + CastDef_412 CastDef_413"Add +" + CastDef_413 CastDef_415"Sqrt +. + CastDef_405 + CastDef_415 CastDef_417"Div +. + CastDef_417 + CastDef_419 CastDef_420"Mul +. + CastDef_420 + CastDef_422 CastDef_423"Add +1 + CastDef_423 + CastDef_425 CastDef_426"MatMul +. + CastDef_426 + CastDef_428 CastDef_429"Add +. + CastDef_429 + CastDef_431 CastDef_432"Div +! + CastDef_432 CastDef_434"Erf +. + CastDef_434 + CastDef_436 CastDef_437"Add +. + CastDef_429 + CastDef_437 CastDef_439"Mul +. + CastDef_439 + CastDef_441 CastDef_442"Mul +1 + CastDef_442 + CastDef_444 CastDef_445"MatMul +. + CastDef_445 + CastDef_447 CastDef_448"Add +. + CastDef_448 + CastDef_423 CastDef_450"Add +O + CastDef_450 CastDef_452" +ReduceMean* +axes@* +keepdims +. + CastDef_450 + CastDef_452 CastDef_454"Sub +. + CastDef_454 + CastDef_456 CastDef_457"Pow +O + CastDef_457 CastDef_459" +ReduceMean* +axes@* +keepdims +. + CastDef_459 + CastDef_461 CastDef_462"Add +" + CastDef_462 CastDef_464"Sqrt +. + CastDef_454 + CastDef_464 CastDef_466"Div +. + CastDef_466 + CastDef_468 CastDef_469"Mul +. + CastDef_469 + CastDef_471 CastDef_472"Add +1 + CastDef_472 + CastDef_494 CastDef_495"MatMul +. + CastDef_495 + CastDef_497 CastDef_498"Add +b + CastDef_498706 CastDef_498"Cast* +to +20cast node to cast from float16 to float32 on cpu + +706 +748749"Reshape +* +749750" Transpose* +perm@@@@ +b +750 CastDef_500 CastDef_500"Cast* +to20cast node to cast from float16 to float32 on cpu +b +753 CastDef_488 CastDef_488"Cast* +to20cast node to cast from float16 to float32 on cpu + +3bert.encoder.layer.4.attention.self.key.bias_resize CastDef_477 CastDef_477"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1670_resize CastDef_474 CastDef_474"Cast* +to20cast node to cast from float16 to float32 on cpu +1 + CastDef_472 + CastDef_474 CastDef_475"MatMul +. + CastDef_475 + CastDef_477 CastDef_478"Add +b + CastDef_478703 CastDef_478"Cast* +to +20cast node to cast from float16 to float32 on cpu + +703 +734735"Reshape +* +735751" Transpose* +perm@@@@ +b +751 CastDef_485 CastDef_485"Cast* +to20cast node to cast from float16 to float32 on cpu + +5bert.encoder.layer.4.attention.self.query.bias_resize CastDef_482 CastDef_482"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1669_resize CastDef_479 CastDef_479"Cast* +to20cast node to cast from float16 to float32 on cpu +1 + CastDef_472 + CastDef_479 CastDef_480"MatMul +. + CastDef_480 + CastDef_482 CastDef_483"Add +b + CastDef_483700 CastDef_483"Cast* +to +20cast node to cast from float16 to float32 on cpu + +700 +719720"Reshape +* +720721" Transpose* +perm@@@@ +b +721 CastDef_484 CastDef_484"Cast* +to20cast node to cast from float16 to float32 on cpu +1 + CastDef_484 + CastDef_485 CastDef_486"MatMul +. + CastDef_486 + CastDef_488 CastDef_489"Div +- + CastDef_489 + +CastDef_56 CastDef_491"Add +2 + CastDef_491 CastDef_493"Softmax* +axis +1 + CastDef_493 + CastDef_500 CastDef_501"MatMul +b + CastDef_501757 CastDef_501"Cast* +to +20cast node to cast from float16 to float32 on cpu +* +757758" Transpose* +perm@@@@ + +758 +769770"Reshape +b +770 CastDef_502 CastDef_502"Cast* +to20cast node to cast from float16 to float32 on cpu +1 + CastDef_502 + CastDef_503 CastDef_504"MatMul +. + CastDef_504 + CastDef_506 CastDef_507"Add +. + CastDef_507 + CastDef_472 CastDef_509"Add +O + CastDef_509 CastDef_511" +ReduceMean* +axes@* +keepdims +. + CastDef_509 + CastDef_511 CastDef_513"Sub +. + CastDef_513 + CastDef_515 CastDef_516"Pow +O + CastDef_516 CastDef_518" +ReduceMean* +axes@* +keepdims +. + CastDef_518 + CastDef_520 CastDef_521"Add +" + CastDef_521 CastDef_523"Sqrt +. + CastDef_513 + CastDef_523 CastDef_525"Div +. + CastDef_525 + CastDef_527 CastDef_528"Mul +. + CastDef_528 + CastDef_530 CastDef_531"Add +1 + CastDef_531 + CastDef_533 CastDef_534"MatMul +. + CastDef_534 + CastDef_536 CastDef_537"Add +. + CastDef_537 + CastDef_539 CastDef_540"Div +! + CastDef_540 CastDef_542"Erf +. + CastDef_542 + CastDef_544 CastDef_545"Add +. + CastDef_537 + CastDef_545 CastDef_547"Mul +. + CastDef_547 + CastDef_549 CastDef_550"Mul +1 + CastDef_550 + CastDef_552 CastDef_553"MatMul +. + CastDef_553 + CastDef_555 CastDef_556"Add +. + CastDef_556 + CastDef_531 CastDef_558"Add +O + CastDef_558 CastDef_560" +ReduceMean* +axes@* +keepdims +. + CastDef_558 + CastDef_560 CastDef_562"Sub +. + CastDef_562 + CastDef_564 CastDef_565"Pow +O + CastDef_565 CastDef_567" +ReduceMean* +axes@* +keepdims +. + CastDef_567 + CastDef_569 CastDef_570"Add +" + CastDef_570 CastDef_572"Sqrt +. + CastDef_562 + CastDef_572 CastDef_574"Div +. + CastDef_574 + CastDef_576 CastDef_577"Mul +. + CastDef_577 + CastDef_579 CastDef_580"Add +1 + CastDef_580 + CastDef_602 CastDef_603"MatMul +. + CastDef_603 + CastDef_605 CastDef_606"Add +b + CastDef_606820 CastDef_606"Cast* +to +20cast node to cast from float16 to float32 on cpu + +820 +862863"Reshape +* +863864" Transpose* +perm@@@@ +b +864 CastDef_608 CastDef_608"Cast* +to20cast node to cast from float16 to float32 on cpu +b +867 CastDef_596 CastDef_596"Cast* +to20cast node to cast from float16 to float32 on cpu + +3bert.encoder.layer.5.attention.self.key.bias_resize CastDef_585 CastDef_585"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1683_resize CastDef_582 CastDef_582"Cast* +to20cast node to cast from float16 to float32 on cpu +1 + CastDef_580 + CastDef_582 CastDef_583"MatMul +. + CastDef_583 + CastDef_585 CastDef_586"Add +b + CastDef_586817 CastDef_586"Cast* +to +20cast node to cast from float16 to float32 on cpu + +817 +848849"Reshape +* +849865" Transpose* +perm@@@@ +b +865 CastDef_593 CastDef_593"Cast* +to20cast node to cast from float16 to float32 on cpu + +5bert.encoder.layer.5.attention.self.query.bias_resize CastDef_590 CastDef_590"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1682_resize CastDef_587 CastDef_587"Cast* +to20cast node to cast from float16 to float32 on cpu +1 + CastDef_580 + CastDef_587 CastDef_588"MatMul +. + CastDef_588 + CastDef_590 CastDef_591"Add +b + CastDef_591814 CastDef_591"Cast* +to +20cast node to cast from float16 to float32 on cpu + +814 +833834"Reshape +* +834835" Transpose* +perm@@@@ +b +835 CastDef_592 CastDef_592"Cast* +to20cast node to cast from float16 to float32 on cpu +1 + CastDef_592 + CastDef_593 CastDef_594"MatMul +. + CastDef_594 + CastDef_596 CastDef_597"Div +- + CastDef_597 + +CastDef_56 CastDef_599"Add +2 + CastDef_599 CastDef_601"Softmax* +axis +1 + CastDef_601 + CastDef_608 CastDef_609"MatMul +b + CastDef_609871 CastDef_609"Cast* +to +20cast node to cast from float16 to float32 on cpu +* +871872" Transpose* +perm@@@@ + +872 +883884"Reshape +b +884 CastDef_610 CastDef_610"Cast* +to20cast node to cast from float16 to float32 on cpu +1 + CastDef_610 + CastDef_611 CastDef_612"MatMul +. + CastDef_612 + CastDef_614 CastDef_615"Add +. + CastDef_615 + CastDef_580 CastDef_617"Add +O + CastDef_617 CastDef_619" +ReduceMean* +axes@* +keepdims +. + CastDef_617 + CastDef_619 CastDef_621"Sub +. + CastDef_621 + CastDef_623 CastDef_624"Pow +O + CastDef_624 CastDef_626" +ReduceMean* +axes@* +keepdims +. + CastDef_626 + CastDef_628 CastDef_629"Add +" + CastDef_629 CastDef_631"Sqrt +. + CastDef_621 + CastDef_631 CastDef_633"Div +. + CastDef_633 + CastDef_635 CastDef_636"Mul +. + CastDef_636 + CastDef_638 CastDef_639"Add +1 + CastDef_639 + CastDef_641 CastDef_642"MatMul +. + CastDef_642 + CastDef_644 CastDef_645"Add +. + CastDef_645 + CastDef_647 CastDef_648"Div +! + CastDef_648 CastDef_650"Erf +. + CastDef_650 + CastDef_652 CastDef_653"Add +. + CastDef_645 + CastDef_653 CastDef_655"Mul +. + CastDef_655 + CastDef_657 CastDef_658"Mul +1 + CastDef_658 + CastDef_660 CastDef_661"MatMul +. + CastDef_661 + CastDef_663 CastDef_664"Add +. + CastDef_664 + CastDef_639 CastDef_666"Add +O + CastDef_666 CastDef_668" +ReduceMean* +axes@* +keepdims +. + CastDef_666 + CastDef_668 CastDef_670"Sub +. + CastDef_670 + CastDef_672 CastDef_673"Pow +O + CastDef_673 CastDef_675" +ReduceMean* +axes@* +keepdims +. + CastDef_675 + CastDef_677 CastDef_678"Add +" + CastDef_678 CastDef_680"Sqrt +. + CastDef_670 + CastDef_680 CastDef_682"Div +. + CastDef_682 + CastDef_684 CastDef_685"Mul +. + CastDef_685 + CastDef_687 CastDef_688"Add +1 + CastDef_688 + CastDef_710 CastDef_711"MatMul +. + CastDef_711 + CastDef_713 CastDef_714"Add +b + CastDef_714934 CastDef_714"Cast* +to +20cast node to cast from float16 to float32 on cpu + +934 +976977"Reshape +* +977978" Transpose* +perm@@@@ +b +978 CastDef_716 CastDef_716"Cast* +to20cast node to cast from float16 to float32 on cpu +b +981 CastDef_704 CastDef_704"Cast* +to20cast node to cast from float16 to float32 on cpu + +3bert.encoder.layer.6.attention.self.key.bias_resize CastDef_693 CastDef_693"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1696_resize CastDef_690 CastDef_690"Cast* +to20cast node to cast from float16 to float32 on cpu +1 + CastDef_688 + CastDef_690 CastDef_691"MatMul +. + CastDef_691 + CastDef_693 CastDef_694"Add +b + CastDef_694931 CastDef_694"Cast* +to +20cast node to cast from float16 to float32 on cpu + +931 +962963"Reshape +* +963979" Transpose* +perm@@@@ +b +979 CastDef_701 CastDef_701"Cast* +to20cast node to cast from float16 to float32 on cpu + +5bert.encoder.layer.6.attention.self.query.bias_resize CastDef_698 CastDef_698"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1695_resize CastDef_695 CastDef_695"Cast* +to20cast node to cast from float16 to float32 on cpu +1 + CastDef_688 + CastDef_695 CastDef_696"MatMul +. + CastDef_696 + CastDef_698 CastDef_699"Add +b + CastDef_699928 CastDef_699"Cast* +to +20cast node to cast from float16 to float32 on cpu + +928 +947948"Reshape +* +948949" Transpose* +perm@@@@ +b +949 CastDef_700 CastDef_700"Cast* +to20cast node to cast from float16 to float32 on cpu +1 + CastDef_700 + CastDef_701 CastDef_702"MatMul +. + CastDef_702 + CastDef_704 CastDef_705"Div +- + CastDef_705 + +CastDef_56 CastDef_707"Add +2 + CastDef_707 CastDef_709"Softmax* +axis +1 + CastDef_709 + CastDef_716 CastDef_717"MatMul +b + CastDef_717985 CastDef_717"Cast* +to +20cast node to cast from float16 to float32 on cpu +* +985986" Transpose* +perm@@@@ + +986 +997998"Reshape +b +998 CastDef_718 CastDef_718"Cast* +to20cast node to cast from float16 to float32 on cpu +1 + CastDef_718 + CastDef_719 CastDef_720"MatMul +. + CastDef_720 + CastDef_722 CastDef_723"Add +. + CastDef_723 + CastDef_688 CastDef_725"Add +O + CastDef_725 CastDef_727" +ReduceMean* +axes@* +keepdims +. + CastDef_725 + CastDef_727 CastDef_729"Sub +. + CastDef_729 + CastDef_731 CastDef_732"Pow +O + CastDef_732 CastDef_734" +ReduceMean* +axes@* +keepdims +. + CastDef_734 + CastDef_736 CastDef_737"Add +" + CastDef_737 CastDef_739"Sqrt +. + CastDef_729 + CastDef_739 CastDef_741"Div +. + CastDef_741 + CastDef_743 CastDef_744"Mul +. + CastDef_744 + CastDef_746 CastDef_747"Add +1 + CastDef_747 + CastDef_749 CastDef_750"MatMul +. + CastDef_750 + CastDef_752 CastDef_753"Add +. + CastDef_753 + CastDef_755 CastDef_756"Div +! + CastDef_756 CastDef_758"Erf +. + CastDef_758 + CastDef_760 CastDef_761"Add +. + CastDef_753 + CastDef_761 CastDef_763"Mul +. + CastDef_763 + CastDef_765 CastDef_766"Mul +1 + CastDef_766 + CastDef_768 CastDef_769"MatMul +. + CastDef_769 + CastDef_771 CastDef_772"Add +. + CastDef_772 + CastDef_747 CastDef_774"Add +O + CastDef_774 CastDef_776" +ReduceMean* +axes@* +keepdims +. + CastDef_774 + CastDef_776 CastDef_778"Sub +. + CastDef_778 + CastDef_780 CastDef_781"Pow +O + CastDef_781 CastDef_783" +ReduceMean* +axes@* +keepdims +. + CastDef_783 + CastDef_785 CastDef_786"Add +" + CastDef_786 CastDef_788"Sqrt +. + CastDef_778 + CastDef_788 CastDef_790"Div +. + CastDef_790 + CastDef_792 CastDef_793"Mul +. + CastDef_793 + CastDef_795 CastDef_796"Add +1 + CastDef_796 + CastDef_818 CastDef_819"MatMul +. + CastDef_819 + CastDef_821 CastDef_822"Add +c + CastDef_8221048 CastDef_822"Cast* +to +20cast node to cast from float16 to float32 on cpu + +1048 +10901091"Reshape +, +10911092" Transpose* +perm@@@@ +c +1092 CastDef_824 CastDef_824"Cast* +to20cast node to cast from float16 to float32 on cpu +c +1095 CastDef_812 CastDef_812"Cast* +to20cast node to cast from float16 to float32 on cpu + +3bert.encoder.layer.7.attention.self.key.bias_resize CastDef_801 CastDef_801"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1709_resize CastDef_798 CastDef_798"Cast* +to20cast node to cast from float16 to float32 on cpu +1 + CastDef_796 + CastDef_798 CastDef_799"MatMul +. + CastDef_799 + CastDef_801 CastDef_802"Add +c + CastDef_8021045 CastDef_802"Cast* +to +20cast node to cast from float16 to float32 on cpu + +1045 +10761077"Reshape +, +10771093" Transpose* +perm@@@@ +c +1093 CastDef_809 CastDef_809"Cast* +to20cast node to cast from float16 to float32 on cpu + +5bert.encoder.layer.7.attention.self.query.bias_resize CastDef_806 CastDef_806"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1708_resize CastDef_803 CastDef_803"Cast* +to20cast node to cast from float16 to float32 on cpu +1 + CastDef_796 + CastDef_803 CastDef_804"MatMul +. + CastDef_804 + CastDef_806 CastDef_807"Add +c + CastDef_8071042 CastDef_807"Cast* +to +20cast node to cast from float16 to float32 on cpu + +1042 +10611062"Reshape +, +10621063" Transpose* +perm@@@@ +c +1063 CastDef_808 CastDef_808"Cast* +to20cast node to cast from float16 to float32 on cpu +1 + CastDef_808 + CastDef_809 CastDef_810"MatMul +. + CastDef_810 + CastDef_812 CastDef_813"Div +- + CastDef_813 + +CastDef_56 CastDef_815"Add +2 + CastDef_815 CastDef_817"Softmax* +axis +1 + CastDef_817 + CastDef_824 CastDef_825"MatMul +c + CastDef_8251099 CastDef_825"Cast* +to +20cast node to cast from float16 to float32 on cpu +, +10991100" Transpose* +perm@@@@ + +1100 +11111112"Reshape +c +1112 CastDef_826 CastDef_826"Cast* +to20cast node to cast from float16 to float32 on cpu +1 + CastDef_826 + CastDef_827 CastDef_828"MatMul +. + CastDef_828 + CastDef_830 CastDef_831"Add +. + CastDef_831 + CastDef_796 CastDef_833"Add +O + CastDef_833 CastDef_835" +ReduceMean* +axes@* +keepdims +. + CastDef_833 + CastDef_835 CastDef_837"Sub +. + CastDef_837 + CastDef_839 CastDef_840"Pow +O + CastDef_840 CastDef_842" +ReduceMean* +axes@* +keepdims +. + CastDef_842 + CastDef_844 CastDef_845"Add +" + CastDef_845 CastDef_847"Sqrt +. + CastDef_837 + CastDef_847 CastDef_849"Div +. + CastDef_849 + CastDef_851 CastDef_852"Mul +. + CastDef_852 + CastDef_854 CastDef_855"Add +1 + CastDef_855 + CastDef_857 CastDef_858"MatMul +. + CastDef_858 + CastDef_860 CastDef_861"Add +. + CastDef_861 + CastDef_863 CastDef_864"Div +! + CastDef_864 CastDef_866"Erf +. + CastDef_866 + CastDef_868 CastDef_869"Add +. + CastDef_861 + CastDef_869 CastDef_871"Mul +. + CastDef_871 + CastDef_873 CastDef_874"Mul +1 + CastDef_874 + CastDef_876 CastDef_877"MatMul +. + CastDef_877 + CastDef_879 CastDef_880"Add +. + CastDef_880 + CastDef_855 CastDef_882"Add +O + CastDef_882 CastDef_884" +ReduceMean* +axes@* +keepdims +. + CastDef_882 + CastDef_884 CastDef_886"Sub +. + CastDef_886 + CastDef_888 CastDef_889"Pow +O + CastDef_889 CastDef_891" +ReduceMean* +axes@* +keepdims +. + CastDef_891 + CastDef_893 CastDef_894"Add +" + CastDef_894 CastDef_896"Sqrt +. + CastDef_886 + CastDef_896 CastDef_898"Div +. + CastDef_898 + CastDef_900 CastDef_901"Mul +. + CastDef_901 + CastDef_903 CastDef_904"Add +1 + CastDef_904 + CastDef_926 CastDef_927"MatMul +. + CastDef_927 + CastDef_929 CastDef_930"Add +c + CastDef_9301162 CastDef_930"Cast* +to +20cast node to cast from float16 to float32 on cpu + +1162 +12041205"Reshape +, +12051206" Transpose* +perm@@@@ +c +1206 CastDef_932 CastDef_932"Cast* +to20cast node to cast from float16 to float32 on cpu +c +1209 CastDef_920 CastDef_920"Cast* +to20cast node to cast from float16 to float32 on cpu + +3bert.encoder.layer.8.attention.self.key.bias_resize CastDef_909 CastDef_909"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1722_resize CastDef_906 CastDef_906"Cast* +to20cast node to cast from float16 to float32 on cpu +1 + CastDef_904 + CastDef_906 CastDef_907"MatMul +. + CastDef_907 + CastDef_909 CastDef_910"Add +c + CastDef_9101159 CastDef_910"Cast* +to +20cast node to cast from float16 to float32 on cpu + +1159 +11901191"Reshape +, +11911207" Transpose* +perm@@@@ +c +1207 CastDef_917 CastDef_917"Cast* +to20cast node to cast from float16 to float32 on cpu + +5bert.encoder.layer.8.attention.self.query.bias_resize CastDef_914 CastDef_914"Cast* +to20cast node to cast from float16 to float32 on cpu +j + 1721_resize CastDef_911 CastDef_911"Cast* +to20cast node to cast from float16 to float32 on cpu +1 + CastDef_904 + CastDef_911 CastDef_912"MatMul +. + CastDef_912 + CastDef_914 CastDef_915"Add +c + CastDef_9151156 CastDef_915"Cast* +to +20cast node to cast from float16 to float32 on cpu + +1156 +11751176"Reshape +, +11761177" Transpose* +perm@@@@ +c +1177 CastDef_916 CastDef_916"Cast* +to20cast node to cast from float16 to float32 on cpu +1 + CastDef_916 + CastDef_917 CastDef_918"MatMul +. + CastDef_918 + CastDef_920 CastDef_921"Div +- + CastDef_921 + +CastDef_56 CastDef_923"Add +2 + CastDef_923 CastDef_925"Softmax* +axis +1 + CastDef_925 + CastDef_932 CastDef_933"MatMul +c + CastDef_9331213 CastDef_933"Cast* +to +20cast node to cast from float16 to float32 on cpu +, +12131214" Transpose* +perm@@@@ + +1214 +12251226"Reshape +c +1226 CastDef_934 CastDef_934"Cast* +to20cast node to cast from float16 to float32 on cpu +1 + CastDef_934 + CastDef_935 CastDef_936"MatMul +. + CastDef_936 + CastDef_938 CastDef_939"Add +. + CastDef_939 + CastDef_904 CastDef_941"Add +O + CastDef_941 CastDef_943" +ReduceMean* +axes@* +keepdims +. + CastDef_941 + CastDef_943 CastDef_945"Sub +. + CastDef_945 + CastDef_947 CastDef_948"Pow +O + CastDef_948 CastDef_950" +ReduceMean* +axes@* +keepdims +. + CastDef_950 + CastDef_952 CastDef_953"Add +" + CastDef_953 CastDef_955"Sqrt +. + CastDef_945 + CastDef_955 CastDef_957"Div +. + CastDef_957 + CastDef_959 CastDef_960"Mul +. + CastDef_960 + CastDef_962 CastDef_963"Add +1 + CastDef_963 + CastDef_965 CastDef_966"MatMul +. + CastDef_966 + CastDef_968 CastDef_969"Add +. + CastDef_969 + CastDef_971 CastDef_972"Div +! + CastDef_972 CastDef_974"Erf +. + CastDef_974 + CastDef_976 CastDef_977"Add +. + CastDef_969 + CastDef_977 CastDef_979"Mul +. + CastDef_979 + CastDef_981 CastDef_982"Mul +1 + CastDef_982 + CastDef_984 CastDef_985"MatMul +. + CastDef_985 + CastDef_987 CastDef_988"Add +. + CastDef_988 + CastDef_963 CastDef_990"Add +O + CastDef_990 CastDef_992" +ReduceMean* +axes@* +keepdims +. + CastDef_990 + CastDef_992 CastDef_994"Sub +. + CastDef_994 + CastDef_996 CastDef_997"Pow +O + CastDef_997 CastDef_999" +ReduceMean* +axes@* +keepdims +0 + CastDef_999 + CastDef_1001 CastDef_1002"Add +$ + CastDef_1002 CastDef_1004"Sqrt +0 + CastDef_994 + CastDef_1004 CastDef_1006"Div +1 + CastDef_1006 + CastDef_1008 CastDef_1009"Mul +1 + CastDef_1009 + CastDef_1011 CastDef_1012"Add +4 + CastDef_1012 + CastDef_1034 CastDef_1035"MatMul +1 + CastDef_1035 + CastDef_1037 CastDef_1038"Add +e + CastDef_10381276 CastDef_1038"Cast* +to +20cast node to cast from float16 to float32 on cpu + +1276 +13181319"Reshape +, +13191320" Transpose* +perm@@@@ +e +1320 CastDef_1040 CastDef_1040"Cast* +to20cast node to cast from float16 to float32 on cpu +e +1323 CastDef_1028 CastDef_1028"Cast* +to20cast node to cast from float16 to float32 on cpu + +3bert.encoder.layer.9.attention.self.key.bias_resize CastDef_1017 CastDef_1017"Cast* +to20cast node to cast from float16 to float32 on cpu +l + 1735_resize CastDef_1014 CastDef_1014"Cast* +to20cast node to cast from float16 to float32 on cpu +4 + CastDef_1012 + CastDef_1014 CastDef_1015"MatMul +1 + CastDef_1015 + CastDef_1017 CastDef_1018"Add +e + CastDef_10181273 CastDef_1018"Cast* +to +20cast node to cast from float16 to float32 on cpu + +1273 +13041305"Reshape +, +13051321" Transpose* +perm@@@@ +e +1321 CastDef_1025 CastDef_1025"Cast* +to20cast node to cast from float16 to float32 on cpu + +5bert.encoder.layer.9.attention.self.query.bias_resize CastDef_1022 CastDef_1022"Cast* +to20cast node to cast from float16 to float32 on cpu +l + 1734_resize CastDef_1019 CastDef_1019"Cast* +to20cast node to cast from float16 to float32 on cpu +4 + CastDef_1012 + CastDef_1019 CastDef_1020"MatMul +1 + CastDef_1020 + CastDef_1022 CastDef_1023"Add +e + CastDef_10231270 CastDef_1023"Cast* +to +20cast node to cast from float16 to float32 on cpu + +1270 +12891290"Reshape +, +12901291" Transpose* +perm@@@@ +e +1291 CastDef_1024 CastDef_1024"Cast* +to20cast node to cast from float16 to float32 on cpu +4 + CastDef_1024 + CastDef_1025 CastDef_1026"MatMul +1 + CastDef_1026 + CastDef_1028 CastDef_1029"Div +/ + CastDef_1029 + +CastDef_56 CastDef_1031"Add +4 + CastDef_1031 CastDef_1033"Softmax* +axis +4 + CastDef_1033 + CastDef_1040 CastDef_1041"MatMul +e + CastDef_10411327 CastDef_1041"Cast* +to +20cast node to cast from float16 to float32 on cpu +, +13271328" Transpose* +perm@@@@ + +1328 +13391340"Reshape +e +1340 CastDef_1042 CastDef_1042"Cast* +to20cast node to cast from float16 to float32 on cpu +4 + CastDef_1042 + CastDef_1043 CastDef_1044"MatMul +1 + CastDef_1044 + CastDef_1046 CastDef_1047"Add +1 + CastDef_1047 + CastDef_1012 CastDef_1049"Add +Q + CastDef_1049 CastDef_1051" +ReduceMean* +axes@* +keepdims +1 + CastDef_1049 + CastDef_1051 CastDef_1053"Sub +1 + CastDef_1053 + CastDef_1055 CastDef_1056"Pow +Q + CastDef_1056 CastDef_1058" +ReduceMean* +axes@* +keepdims +1 + CastDef_1058 + CastDef_1060 CastDef_1061"Add +$ + CastDef_1061 CastDef_1063"Sqrt +1 + CastDef_1053 + CastDef_1063 CastDef_1065"Div +1 + CastDef_1065 + CastDef_1067 CastDef_1068"Mul +1 + CastDef_1068 + CastDef_1070 CastDef_1071"Add +4 + CastDef_1071 + CastDef_1073 CastDef_1074"MatMul +1 + CastDef_1074 + CastDef_1076 CastDef_1077"Add +1 + CastDef_1077 + CastDef_1079 CastDef_1080"Div +# + CastDef_1080 CastDef_1082"Erf +1 + CastDef_1082 + CastDef_1084 CastDef_1085"Add +1 + CastDef_1077 + CastDef_1085 CastDef_1087"Mul +1 + CastDef_1087 + CastDef_1089 CastDef_1090"Mul +4 + CastDef_1090 + CastDef_1092 CastDef_1093"MatMul +1 + CastDef_1093 + CastDef_1095 CastDef_1096"Add +1 + CastDef_1096 + CastDef_1071 CastDef_1098"Add +Q + CastDef_1098 CastDef_1100" +ReduceMean* +axes@* +keepdims +1 + CastDef_1098 + CastDef_1100 CastDef_1102"Sub +1 + CastDef_1102 + CastDef_1104 CastDef_1105"Pow +Q + CastDef_1105 CastDef_1107" +ReduceMean* +axes@* +keepdims +1 + CastDef_1107 + CastDef_1109 CastDef_1110"Add +$ + CastDef_1110 CastDef_1112"Sqrt +1 + CastDef_1102 + CastDef_1112 CastDef_1114"Div +1 + CastDef_1114 + CastDef_1116 CastDef_1117"Mul +1 + CastDef_1117 + CastDef_1119 CastDef_1120"Add +4 + CastDef_1120 + CastDef_1142 CastDef_1143"MatMul +1 + CastDef_1143 + CastDef_1145 CastDef_1146"Add +e + CastDef_11461390 CastDef_1146"Cast* +to +20cast node to cast from float16 to float32 on cpu + +1390 +14321433"Reshape +, +14331434" Transpose* +perm@@@@ +e +1434 CastDef_1148 CastDef_1148"Cast* +to20cast node to cast from float16 to float32 on cpu +e +1437 CastDef_1136 CastDef_1136"Cast* +to20cast node to cast from float16 to float32 on cpu + +4bert.encoder.layer.10.attention.self.key.bias_resize CastDef_1125 CastDef_1125"Cast* +to20cast node to cast from float16 to float32 on cpu +l + 1748_resize CastDef_1122 CastDef_1122"Cast* +to20cast node to cast from float16 to float32 on cpu +4 + CastDef_1120 + CastDef_1122 CastDef_1123"MatMul +1 + CastDef_1123 + CastDef_1125 CastDef_1126"Add +e + CastDef_11261387 CastDef_1126"Cast* +to +20cast node to cast from float16 to float32 on cpu + +1387 +14181419"Reshape +, +14191435" Transpose* +perm@@@@ +e +1435 CastDef_1133 CastDef_1133"Cast* +to20cast node to cast from float16 to float32 on cpu + +6bert.encoder.layer.10.attention.self.query.bias_resize CastDef_1130 CastDef_1130"Cast* +to20cast node to cast from float16 to float32 on cpu +l + 1747_resize CastDef_1127 CastDef_1127"Cast* +to20cast node to cast from float16 to float32 on cpu +4 + CastDef_1120 + CastDef_1127 CastDef_1128"MatMul +1 + CastDef_1128 + CastDef_1130 CastDef_1131"Add +e + CastDef_11311384 CastDef_1131"Cast* +to +20cast node to cast from float16 to float32 on cpu + +1384 +14031404"Reshape +, +14041405" Transpose* +perm@@@@ +e +1405 CastDef_1132 CastDef_1132"Cast* +to20cast node to cast from float16 to float32 on cpu +4 + CastDef_1132 + CastDef_1133 CastDef_1134"MatMul +1 + CastDef_1134 + CastDef_1136 CastDef_1137"Div +/ + CastDef_1137 + +CastDef_56 CastDef_1139"Add +4 + CastDef_1139 CastDef_1141"Softmax* +axis +4 + CastDef_1141 + CastDef_1148 CastDef_1149"MatMul +e + CastDef_11491441 CastDef_1149"Cast* +to +20cast node to cast from float16 to float32 on cpu +, +14411442" Transpose* +perm@@@@ + +1442 +14531454"Reshape +e +1454 CastDef_1150 CastDef_1150"Cast* +to20cast node to cast from float16 to float32 on cpu +4 + CastDef_1150 + CastDef_1151 CastDef_1152"MatMul +1 + CastDef_1152 + CastDef_1154 CastDef_1155"Add +1 + CastDef_1155 + CastDef_1120 CastDef_1157"Add +Q + CastDef_1157 CastDef_1159" +ReduceMean* +axes@* +keepdims +1 + CastDef_1157 + CastDef_1159 CastDef_1161"Sub +1 + CastDef_1161 + CastDef_1163 CastDef_1164"Pow +Q + CastDef_1164 CastDef_1166" +ReduceMean* +axes@* +keepdims +1 + CastDef_1166 + CastDef_1168 CastDef_1169"Add +$ + CastDef_1169 CastDef_1171"Sqrt +1 + CastDef_1161 + CastDef_1171 CastDef_1173"Div +1 + CastDef_1173 + CastDef_1175 CastDef_1176"Mul +1 + CastDef_1176 + CastDef_1178 CastDef_1179"Add +4 + CastDef_1179 + CastDef_1181 CastDef_1182"MatMul +1 + CastDef_1182 + CastDef_1184 CastDef_1185"Add +1 + CastDef_1185 + CastDef_1187 CastDef_1188"Div +# + CastDef_1188 CastDef_1190"Erf +1 + CastDef_1190 + CastDef_1192 CastDef_1193"Add +1 + CastDef_1185 + CastDef_1193 CastDef_1195"Mul +1 + CastDef_1195 + CastDef_1197 CastDef_1198"Mul +4 + CastDef_1198 + CastDef_1200 CastDef_1201"MatMul +1 + CastDef_1201 + CastDef_1203 CastDef_1204"Add +1 + CastDef_1204 + CastDef_1179 CastDef_1206"Add +Q + CastDef_1206 CastDef_1208" +ReduceMean* +axes@* +keepdims +1 + CastDef_1206 + CastDef_1208 CastDef_1210"Sub +1 + CastDef_1210 + CastDef_1212 CastDef_1213"Pow +Q + CastDef_1213 CastDef_1215" +ReduceMean* +axes@* +keepdims +1 + CastDef_1215 + CastDef_1217 CastDef_1218"Add +$ + CastDef_1218 CastDef_1220"Sqrt +1 + CastDef_1210 + CastDef_1220 CastDef_1222"Div +1 + CastDef_1222 + CastDef_1224 CastDef_1225"Mul +1 + CastDef_1225 + CastDef_1227 CastDef_1228"Add +4 + CastDef_1228 + CastDef_1250 CastDef_1251"MatMul +1 + CastDef_1251 + CastDef_1253 CastDef_1254"Add +e + CastDef_12541504 CastDef_1254"Cast* +to +20cast node to cast from float16 to float32 on cpu + +1504 +15461547"Reshape +, +15471548" Transpose* +perm@@@@ +e +1548 CastDef_1256 CastDef_1256"Cast* +to20cast node to cast from float16 to float32 on cpu +e +1551 CastDef_1244 CastDef_1244"Cast* +to20cast node to cast from float16 to float32 on cpu + +4bert.encoder.layer.11.attention.self.key.bias_resize CastDef_1233 CastDef_1233"Cast* +to20cast node to cast from float16 to float32 on cpu +l + 1761_resize CastDef_1230 CastDef_1230"Cast* +to20cast node to cast from float16 to float32 on cpu +4 + CastDef_1228 + CastDef_1230 CastDef_1231"MatMul +1 + CastDef_1231 + CastDef_1233 CastDef_1234"Add +e + CastDef_12341501 CastDef_1234"Cast* +to +20cast node to cast from float16 to float32 on cpu + +1501 +15321533"Reshape +, +15331549" Transpose* +perm@@@@ +e +1549 CastDef_1241 CastDef_1241"Cast* +to20cast node to cast from float16 to float32 on cpu + +6bert.encoder.layer.11.attention.self.query.bias_resize CastDef_1238 CastDef_1238"Cast* +to20cast node to cast from float16 to float32 on cpu +l + 1760_resize CastDef_1235 CastDef_1235"Cast* +to20cast node to cast from float16 to float32 on cpu +4 + CastDef_1228 + CastDef_1235 CastDef_1236"MatMul +1 + CastDef_1236 + CastDef_1238 CastDef_1239"Add +e + CastDef_12391498 CastDef_1239"Cast* +to +20cast node to cast from float16 to float32 on cpu + +1498 +15171518"Reshape +, +15181519" Transpose* +perm@@@@ +e +1519 CastDef_1240 CastDef_1240"Cast* +to20cast node to cast from float16 to float32 on cpu +4 + CastDef_1240 + CastDef_1241 CastDef_1242"MatMul +1 + CastDef_1242 + CastDef_1244 CastDef_1245"Div +/ + CastDef_1245 + +CastDef_56 CastDef_1247"Add +4 + CastDef_1247 CastDef_1249"Softmax* +axis +4 + CastDef_1249 + CastDef_1256 CastDef_1257"MatMul +e + CastDef_12571555 CastDef_1257"Cast* +to +20cast node to cast from float16 to float32 on cpu +, +15551556" Transpose* +perm@@@@ + +1556 +15671568"Reshape +e +1568 CastDef_1258 CastDef_1258"Cast* +to20cast node to cast from float16 to float32 on cpu +4 + CastDef_1258 + CastDef_1259 CastDef_1260"MatMul +1 + CastDef_1260 + CastDef_1262 CastDef_1263"Add +1 + CastDef_1263 + CastDef_1228 CastDef_1265"Add +Q + CastDef_1265 CastDef_1267" +ReduceMean* +axes@* +keepdims +1 + CastDef_1265 + CastDef_1267 CastDef_1269"Sub +1 + CastDef_1269 + CastDef_1271 CastDef_1272"Pow +Q + CastDef_1272 CastDef_1274" +ReduceMean* +axes@* +keepdims +1 + CastDef_1274 + CastDef_1276 CastDef_1277"Add +$ + CastDef_1277 CastDef_1279"Sqrt +1 + CastDef_1269 + CastDef_1279 CastDef_1281"Div +1 + CastDef_1281 + CastDef_1283 CastDef_1284"Mul +1 + CastDef_1284 + CastDef_1286 CastDef_1287"Add +4 + CastDef_1287 + CastDef_1289 CastDef_1290"MatMul +1 + CastDef_1290 + CastDef_1292 CastDef_1293"Add +1 + CastDef_1293 + CastDef_1295 CastDef_1296"Div +# + CastDef_1296 CastDef_1298"Erf +1 + CastDef_1298 + CastDef_1300 CastDef_1301"Add +1 + CastDef_1293 + CastDef_1301 CastDef_1303"Mul +1 + CastDef_1303 + CastDef_1305 CastDef_1306"Mul +4 + CastDef_1306 + CastDef_1308 CastDef_1309"MatMul +1 + CastDef_1309 + CastDef_1311 CastDef_1312"Add +1 + CastDef_1312 + CastDef_1287 CastDef_1314"Add +Q + CastDef_1314 CastDef_1316" +ReduceMean* +axes@* +keepdims +1 + CastDef_1314 + CastDef_1316 CastDef_1318"Sub +1 + CastDef_1318 + CastDef_1320 CastDef_1321"Pow +Q + CastDef_1321 CastDef_1323" +ReduceMean* +axes@* +keepdims +1 + CastDef_1323 + CastDef_1325 CastDef_1326"Add +$ + CastDef_1326 CastDef_1328"Sqrt +1 + CastDef_1318 + CastDef_1328 CastDef_1330"Div +1 + CastDef_1330 + CastDef_1332 CastDef_1333"Mul +1 + CastDef_1333 + CastDef_1335 CastDef_1336"Add +4 + CastDef_1336 + CastDef_1338 CastDef_1339"MatMul +1 + CastDef_1339 + CastDef_1341 CastDef_1342"Add +Y + CastDef_1342 CastDef_1344 CastDef_1345"Split* +axis* +split@@ += + CastDef_1344 CastDef_1349"Squeeze* +axes@ +f + CastDef_1349start CastDef_1349"Cast* +to +20cast node to cast from float16 to float32 on cpu += + CastDef_1345 CastDef_1347"Squeeze* +axes@ +d + CastDef_1347end CastDef_1347"Cast* +to +20cast node to cast from float16 to float32 on cputorch-jit-export*,B1532J *,B1432J *,B1418J *,B1318J *,B1289J *$B1225J*,B1175J *$B1567J*,B1190J *$B1111J*,B1076J *+B976J *#B883J*+B833J *#B769J*+B748J *+B734J *#B655J*#B541J*+B506J *+B406J *+B848J *+B377J *+B392J *#B313J*+B263J *  +B1604J*  +B1593J8*  +B1587J=*  +B1575J@*  +B1095J@*  +B780J*  +B1551J@*  +B1233J@*+B947J *+B605J *  +B753J@*  +B678J<*  +B1490J*  +B1487J@*$B1453J*,B1517J *  +B1479J8*  +B1473J=*#B427J*  +B1464J*+B620J *+B491J *+B634J *  +B1461J@*  +B681J8*  +B1137J8*  +B689J@*  +B1437J@*+B292J *+B719J *  +B1601J@*+B278J *+B962J *+B862J *,B1090J *#B997J*,B1403J *,B1304J *  +B903J=*  +B795J8*  +B1365J8*  +B1362J<*  +B1359J=*  +B1350J*  +B1347J@*  +B1590J<*  +B909J8*  +B1323J@*,B1546J *  +B692J*  +B1578J*  +B1376J*  +B1373J@*  +B1262J*  +B1259J@*  +B1248J<*  +B1245J=*  +B1236J* +Bqa_outputs.biasJ* +B 1617_resizeJJ'&D|@)'))%7%z%@ר,4$$y¥X,"x"*$K%ʠ&'m(אַ&9 /Wa!g,%#(Ŭ *CP+-(,1*",* +B 1618_resizeJV'H*g啞1ۧ5+!R!0&C.,')S+% 6(j cu%8*I(!='ՠ'#§r #+}k !* +B 1619_resizeJ%ɣMƟ;r($ $yL'͙h !$U_q&';W &ާŤ!O+(D+$՛H'Ԝzy"Q&L% " 9!* +B 1627_resizeJ& n,m'4&(c"'$*!Y(D$D'젶"sk# },2%F&kb%[S%R*%`&)<) 5d!ץDF,۝Q#|,{'*  +B 1628_resizeJB)3a =! Ѯ*!'A'ڦm_ )AG*$$&(觵Y(*&1[+X#i" z'y@^&rWàש Y!>f&% $U++#w((&% m )V(dƦT$lW!"J*'5(O'K"ť" F)> " *Uh}+x$(NF.()('>."$)D)0*S+e+P-$['~g%m-, +3d&nS)%((K BsGݦtg($8g+zM,:!| m(%(N&c,)%+)N$+t$&Z((%ڬ~p!>.Y$X%b"'q@ΠaVk^|*  +B 1629_resizeJ' P|(,'H%(("cÝ%%D&%"p}L +$f$!)v)*(pA$[M"6+t(( !Un(_)H,%Nq]{(%$]($8$a w*ͦN +1x*"9Q$ +*\"#~#G*[(" #%$*$ͧ/$,瞴(,-,~ +n$%c$NXV$95N,(*)) 'ݝp,ä)&# :$L% 뢓hO% *)SI )#Ւj)l5)' #㠱k#j.ԧ%)t'/'jF y}%#/!w%+v^w$8l,w$"'" ]%'q&!'%_u t`)$Aͬʡ %%e*$(#)* +B 1630_resizeJsP_{$b-S\(!˦]$"."u!!Y$U(z(I"$(ӥ)" '-G"+~w(릸Q)$* &%)* +B 1631_resizeJ\-a<%W$'-([s&)&E%+&`晐)Uס}g%s+$D&ۜ(2੣%$?<z ^( '&9+Y* +B 1632_resizeJ$S)B2,"BR(B)")L!e#%&*ܥT))%"%J'"D(m%'¡uu$U%wyk '&'n $Y"$&զ&* +B 1640_resizeJ)%DG*Zթ(ڣ .z%X /,*&*)m&2]($H$+$]?M('"H"+'穌 %\(>!)!.*  +B 1641_resizeJ!$u&&ʢD-K% + ѫl%d&ם*ҩN*Y %]|(\q'ўu#)P)ěl,, .-D&x&E{U(ب*ߥm%!x- & !LuʟP,m/WV&H)).!%$-c&v$ Q)ӥ)#&9*%'y"(M'ӥr*$%e+})c+ਅ)Uy'%H',$;W)+$f(R)) +)2*.aY\9̬&af`%ҩ*x&ܥ/ܜ2^ +G-"Zޥ$&U**.s&*%"((PU<C#&'/f%+!ͪw{ (t%8ͥkB:$-$͟CB7͠HVȢʧKǨ 1*  +B 1642_resizeJ  *&o8ã%(ݢ&!򫘦<* ̦()Xv! t +G"#F0*j H*y&b)z'%,n/+̣!$((}3#w&In%(H(5(ӣ"(9#,(c(++ߩ(iVn&#Ξ\⨡0&\H%!zͥSot0P#ߡ)$ǫ'`?#㢻(x)דB*l%p8(;&&$'jV*i)W&R-V*~ p j*- *""q)y"*),-ݨ%&R,&$çŖG' N%iU%u,b("{ (ţT& '%p(+]i$CޫK&"7*˖s*z%?pa ۜr*.EU .l&ȧђ$ e'=!* +B 1643_resizeJv!A(&ۜ&$3)( "6Ury)&b,8-+k (ޭJ%M9B+ݪ-y!a)*ۥc";)!&I!Z-ͤ!**>p'* +B 1644_resizeJ4%%X/)|()x+gU(,$'& &(/`$C/ <"- Ǫ%^\*#)& \7%Ϋ$&.Q7+_d(*k*(|q%#!G* +B 1645_resizeJ=K5s'_*!Eܥ֥f$M+ !1_ $+T'X*,@"v1 (餖"&g%q)!%$YP$&l#9C(( ]r(* +B 1653_resizeJ' BϜۤatġ)%Ȩx$`N#}e'L04lj%M$1'![uY$.&$+W' )$JinUc&*  +B 1654_resizeJC~8'T% ҩy*)f+((  #٨U(l' +-#+1&4gͧ#˩b̪$fZ%আ嬦)) (Ħ񩮡l1{,&)r*f"$*)&b)%"ɩ''` ,%!{&b )&g%e'<%(3&+";'&V*_,L)&_$,p($( iΨXw$'ϧ,1&)!-k-+xވT'[ '$#פiN+&''ǝcl'&)zuS*ɦ!'$9v!'U +' 1%,(>($-0(*ۍ*:%5,m)%#é$'?&)$Ȣ,B-n)-)%+@Ī;\$aȨ%s"-$S6# 6 ,K䢁#ʪEI%ߨLЫy*  +B 1655_resizeJ($'2(#+#"'"𤖟3'{,g(_ C$&2+(Y 'a i6)&'G-)&)-*Z%*ϨwU*#P(A ,F$֢;g$)'+f$+&ؤj)*"fk#z*c"2͜ q )P%00,v&,)$Aq!}8+<)(ED%f++`$B!0V$Z#,'"",)/%-(;  +gŔV)q!5,$C' h&],}!($[#Gs$=(h)Y(,("[}V$(L,1(&7+x,~ڬ-"ɤ(%%k! +,A& +#9('4):$'k(#(W"'!%%ä0J!E %"*..Wn<- )h$夏P$zp* +B 1656_resizeJ8)K( **!*mʪQ*[(c'b,,4:#+/*%=/`T+% b%.$ +&t)$:(!~"( +Nkwۥb&+Рq #* +B 1657_resizeJ:ע!+:M(%%l&+#3 :'Ϩ;﨓u, ب%J,TBZd#$t*})|"'*,_1)k)$!l﨓K)1M?=$զ* +B 1658_resizeJ%&!87)Ϥ|! "0S%,$  + %+ɤ"JT!0 *%֡ʖ:%٥ "&v'u,&%#P*~#* +B 1666_resizeJxϨD==(E6n.8( ,%8&xi1% ')T)?$$͖'+~m#΢'*l#ݣ%?###U'*'*  +B 1667_resizeJN0)+ݪx$ L2( R#'ף)'Sש^"8~֧sa!Ǩ\7#n^!,$)'g),"(Y"4 ( QL(q"W\, )% ,#&.(f*1%ȫ~(L+<%&W*7)&b+Ǩ,)ngJ^$()ƬFSl$%+ -p6 dx+()襼V$z<):%\ 1K-V(C,9ןs'&w( $ & "\,E٨(,%*|:E~*lj9r%u&I#8 )%)6/$-$*5'U+$(8",b)@&'K%]+8-,*?%, Y0jA($ +*\h/%:'mXFC-& !y#(;?詊c'E!{fDk*  +B 1668_resizeJj'&,M(נ]%,*()#&u)(7O)[!F}")M-H(٢)58Ʃ82$@* }"ߨC!K"ɝ&'ޤ* d$'T] (0*~)E**Bo# S:0器š,"̧-'-c(jm !q(Y*%)RQ6#;')5T#tA]y*۟%a"z\h|(,&'ި8!%FS"*,D*)+64Q(!㗚Qu\!^(*ȡm$[P%7Ϣ&g*e&d$V&?)i\%%UZ'#f"X)Mk'ǩ +{( !Рy(s%I+T,)+$ )R$O%#9 &^,&* +B 1669_resizeJ(oY&#èI"g, &"ޘĤ'",<((ht}*#I#l(G=' ?&*,,2$gz- (Aǜ(#Z()3,%6,* +B 1670_resizeJwM%S{i,V(%s*ݦo度/( A,#K)a*H-#wr$p-,$Fäv,N+;rS6.'ќE*v&)L) +$,'S* +B 1671_resizeJ_$p((J&('Q*;(((E+ڪϦ&A(* 5B'(r()V)( R'j(>({%U#/7&Ѥf^],?ͦ\,#\()* +B 1679_resizeJ&,홋+'c!%$S+$;"W+:!')+*:"(,%a*Q%$pZz )#_(쥬#̟&*&穪橈(i&*  +B 1680_resizeJcB%oP!?awn%;"y'$,.!, -$B *$+*s8'殮,*8 +O!ͨV *(!(;"F,6*m(< +#4)Ф* +B 1684_resizeJ#n&b+ à#-2+)j) +"/y'-ݧѧ AH, e3"B$6(2% %*)2+rѡϦ(ԤZ&(*§( +ߤ )b)1* +B 1692_resizeJX"u +,w%&P+)-1(#U+z)c)˥o(Tԩ| +J##d)+y٩~ w"E;!R(3ir\(S,4a " *  +B 1693_resizeJcz-*ʥ!_)&44,I$D( !Ә,*ѥ%&%;-+ HEr%4[+;)x(;"ɦ.'+)+&Ţ(n")"\"iԬsy(%(©m*("{Π:.#!6 #,@$,_,2$?)j+/'R7!z('v %ɥ"{'fw.,$);-u'9,`홦'2'$R(*!'%$(5g$h8Ҥ%4&%/){)z #'%L,E%s**P!Ԭ;) )($%é%( U#[,}!h'>&#)&+N '<@y=S(o+('6')U)*) (J1)y6X*T(*%0"ۢ%%)+=)k-~ϥ*]&$Wɩq},*  +B 1694_resizeJK)\SbD>$% &-!)"'7&/J)V"R *ie"+#yע#%&-($!+t*%C$ $8 l(,g*} +:*.()s**( "n$ +&+X&z*!L)'+$'hʟ)W)z$%.1"o%"=($i('Ц* )"!);%Uϡa4B*n% +$\#)C(W#`+)o#e(?)vӤ%++v|RQw'UF,H'()$"Jb)S!z(ae(Q%#S$'檺'7 Ҡs||)nqO"V(l(Ĩ/(.P#z#ɤn.ʨ&E#,$#H! - +$I#$1)&U+*h*** +B 1695_resizeJ(()@)C)J%3+Ƣ{K*T),Q%G+*"NF-&(n--oH'֧k( ,,&m[+n** +B 1696_resizeJ&!^J),OA%)" +# ,ݦ٪A(H,${(9g*ƪ&{!'&(%*&)q%*{K'aȦ$f?**O|$I/%* +B 1697_resizeJ"!(W,(7#+T2%j!*-("-.)9*)&-}6s!;%y$*(֡ yH%(h(9q+%X,'Oȣl-,t$#=* +B 1705_resizeJ{'2%!$!C"# (# "̩#o%_#(*-=$GCw$ +;(^<$k֤,bǤ)v&n" !Id()2#1(Ku*  +B 1706_resizeJ$:)\$"t3/DƩO59a)ۮħN,,(1ɥ`+G(ܫf$l${'X')#*ȭ =, ũw),WF*n%j$ءv+7)!$S#G~)Z#!ѧz$w.'-#9򨙠,g(R) ؞d$J"J )&Lq+Vǟd9s\hc|$#i b(4*` }Ý%-&z* z٩@$"רlB*E* +3(v!+աf/])0!햅)`$&y +h)'U* TH))X@Ҥb]*((um#$Y| *̧8'o'+#(u,' -5(&&Y񤱠-,'a:(,W,^&(;*""EӝrLΦ4lc&7-"ۨ*  +B 1707_resizeJ5:&?&?L%#ޢW&mb#7p:$&8*"(-$#j'|b*.(,y/6 PP+w(lH)1e#ѩ t "!M\'^A%<ے:(*$(L. #s$j"~(晭"O$*  +B 1719_resizeJ(d"$&y$Ԛ$m%J$NҢY<8%H"#Uϓ:mYs'U#%% eW(+%$%'*d,203$g(P*Vf*5,I,L),y&x/<*ۣR[&9(!,& +&W.>%By+@('%v" &tg!৲t'ߪ>=8K(2&%!Z,ť21 *6%=3UŪ &إ("%})~"/(&e)%'${*Z,=4&B(ន*lĐ G(("v*'():)-+%x##& +%.*%$-L,(":[*W% ɣ-,$<(#v) $p(yHj%J'4 %+e+&7iV"*"Z.(F E*  +B 1720_resizeJ8-vB)%pjT"ݨk'[!wd'9*P&)6)Tӥթo"$'')^ +!*ݡ5#"N* i!)'8* z(*զǩ,]I$O(x'!*(&͟@m$1)C#%]j%&I!ȟt(?S C"Z!y<*%W+*쭍U%*L)Q)O"+& ;',f@(*dh)l('\Y&!(/'."Z-d(:$$'+ey%&)ޤ"%)"+|+$(" ҩ*K)BQ#VO"%(&l *qԦ %դ2%>$"32#]))P%ۦ%&)Z,})%ɩ2(^#Tѩ*(k("[(,$"Q)LLR!/+%!* +B 1721_resizeJç.*;,QH)`&'%&{ !,#*0&=).:,Q&,6+)C%%!ި%= "]-#_PU),{ ,E4&2&w* +B 1722_resizeJ-$oDE)(4 +P, %&X#C)E$\+&Y6P#ȩ'bV"*}oPj)"FJ&nd#ͧ-@},Σ8*ͨ* +B 1723_resizeJ/?%)#,)ϟ',¢ 5,#}'C&]+*$Ф2`$($}l%ة}$$ Wuvl##w*A:%QW7&y"r'(* +B 1731_resizeJ+]()EH)a'!+?&+1 qN&&˩&%ǧ+h*F?'-+S$[,nZ :*Q&.Β/#"*  +B 1732_resizeJƩ8%Z$ o(k%!*$*"E&s%P]%P)H%9&(&)&ßZ]§O*(a +*}&#7 $+34f5)X қB#^_+s"Fќ5«<%b%fZ +aD ,*  +B 1733_resizeJ*As!d(($Mɫ 'p%$W#7*d]'q&C^Ǧ"'P(&A){$og&*&͕.2x(!#z&;>pu+!E(!(M$Ja$=ά &0 ($f ǫب %Ω, $,$ +P~0p)'().$&5$8#G(!)@"#Qժ >$/$ #$r)(C{XpBiͨ(¨%&[2| *)X(娔l"#|*S/.Y"(-"&.($_d&:=O,&6&"!")$$W%)+)%5$%BmǢn$.( (*Kܤ`#iѡ*&4С&ޣfڝ=()$L$Ψ* +B 1734_resizeJ+ $q AN!()m%;ڦ+V$4$ g)/,y&#h'/$(*.%'%3''.~, ;(,(#֦")y>#,* +B 1735_resizeJZ,樓L+#+]*A+*)*s^(G'pF5+e((*] 8)",+'GZbV';'[#$'W)+$k$)H'{D%Ο))++@** +B 1736_resizeJ"-&,Ԩ %FZ,"(K$!ǫ$4+(,&S !*X#ߨ`$%#y& "7"\(#n,)<%(,P,$LުH"騔* +B 1744_resizeJ 'ר9&&$HH( ]%],ͧmP ++'$+'% d*$!p ]o0ߪ̪ͥ~'iu& 6&Pç%*  +B 1745_resizeJ$ .(*## 5,['Leܘ$P f$] ;)$)g$P()@([&bG)+R*&R,WQ#),&*,#8*3P_&_iͤ¥~19E$&&!&&rH_ c"?,4*'p;(p+(''&#+H*ˢЫh0$(%)ة*&Q`= k+1A%mK+9+i$([&0P#:'%1 +#vc0"p P%&G'-ʧ*,!+6us--,}b(H)%+S2%D Ę (t(j)E#+lg}&('((z%#%n$)&)m ֨ '.*`gPIx('sW&la$U); #d&3-) e)P((%S%*  +B 1746_resizeJݨ󙅣XR&%=Yɩi(.,$!Yܥu*l"V(͚M! +(/+&w3%G('$')>p))(m&*# ^d&:)($'). N!"Ρ A#T+(Ѫ/&@B$+o(Ŝx,ߦƟ1О?%* &$e("a.(&v&,ϣ(* *i%a%^%#} (o#@Ť%,%Dq'*i,N*&K Oĥ)J(u!}#\"}*`)æ&j%N&^&Hy%NhB<)(&,1((=,*Ѩev0טlw'#R,o$ (#-% +>&#m)6$'". h,#a*+)(Z!$xv,Ԭ* +B 1747_resizeJ2 ]))aɭZ);bإ(u$$i hJ+)% *Ҩ) (!r(WN롰#$+H%Z/>"*n+ &+$k,P%(* +B 1748_resizeJ`,lH ,*-.!,џ#'&!"ޥ3(,D)(c*&*ݩ*%3ˬYשp*V***s&+J%l,"n=(&* +B 1749_resizeJYJ"# #+$(7x'$m(7 Ҫ+,J%!IR%6]Q%nfީ"LD%'$.-,t!))謔P*o$* +B 1757_resizeJr,h@*t*&5(@ڠ$h)B$ѝ_Pū})&od-!#&Ʀ$$N%p()G%^+Z*; *4X($<#,)ڪ,$Ǫ*  +B 1758_resizeJ$pn6"T*w t)Ъ bӢ#Nץ3(_$y"*h#1(y!W$_-%$(G⠏+)!%(!$$&& !bz*'')!'+$W,/^F!,&}%,W!$*v!,ާt6 $sޡ>0a)%Ӡ(!T_7b9-Wxtb4U(/ $np>&$Rb!& *n#j<+T&2u$* (H4$''S'8!C)t,}%|;)6*HzS)F(8,;+') *&/(誻G%I 8 *'ߜ%P"j);)G,&#ݘ&/ )&z(¢X+Z3mL, @:`I+,(W#%$,*  +B 1759_resizeJɜ$Ҥp*c# %_42ݩ*J( ,KfT#E$)!Ǭ +%;(#:"l-f!^.c$$i&* $`!)_,.!+%,Q$F4(&9$* % i#')[+ n! + ")!$!+,,,,P=ĞVPQ! #n(*'~&$ yj^()S&&i"?"b!0+ 񬯧$).b'(e'(#%%!)+(9x,19!5c'۫m'ٞ$7# K%N)%@^'$d##,,\#5[&*",!$-]%r&(:"i+#"4'(,)(Ø$C$Ey&U*$&'8%*#!F(3"$8$V,աVe("_+%""(* +B 1760_resizeJ?!Z$;"h 2+mM%I*^'3,-) *PY"Ws$$&y( <, (#ªO,B!c©Ff#^#(* +B 1761_resizeJ|Y %Vө (Y +:̪Q()W'-&B)"N$i#* +B 1762_resizeJ)(R$Q!HQT;,,+s,#⪷ ΩoQ\K~)J7/)j>(Zc,(|&t,+Q)q')!)E*0$𞒩a-* +B 1770_resizeJ:%$*pr(eG,-('h$*$Uϝ P#wө=a:(1*+\!3),`,]$m*˨4)(ūZV -%,%*  +B 1771_resizeJ(4'EF%0'8mDӣ IyI)+NŜ #N*<')aL%j!+Uo%c*"k)㢑,%,))Y)#(,).zΠG( ]RL(X%h%vuR&'$(.. '~*ĥ#,Cm)9ǨFҩ'DQ,4+ 5(~h­V%$\& *4l\&ԕ,! 7$ -ͧ+^$#!(2"j,sצݢ/(6)&"%,%+K!M@#6v&%6')R%,+N%H+f! e%##(S(k(&(⤷&-:((ѝ' /.s+5%$!R\$~!)^=(i)h#M $#C}Ee%k&*  +B 1772_resizeJ&h]-+,t)Q ȨrZꨞ(~<*]%t,A }w$`!','e!%u+*#à-&4++ ){4% Fl%'5sn)W(<+?$f%36 ) "nR,W!ڨ4 +,*_)$2%(v)p iYz"'^O&r(^ u)k#!EM$# %b*(<'|0+)ģ> "uw+((*<%m +)(!+D.#( 'l3(E ѧ (A('̨]!(}m `$((!(&;*g͞+.&+((#&k$(&'"(&EC$ a(B)ͦ'"+]$̙2J'r,U"&`!(( *5 +B 1773_resizeJ  +s#=.%6Z&(T&*= +B%bert.embeddings.LayerNorm.bias_resizeJ Ĩ\(2(M*? +B'bert.embeddings.LayerNorm.weight_resizeJi;F;u;4;:I;59X9* +B1bert.embeddings.position_embeddings.weight_resizeJ&J&ئ4.ޜ /! (($Ds#} $7T'#A %$3ܟ%֤͞ -#Ҟ$cO&&5_T $]#-@#"} ()(L &4'6)$ A%7$%!"##e5%&  $ڥ+zuw$"%"B,L![S&!|b:( I("&x./y)"(c$O!:,tTTڨ"@ +#e*] +B3bert.embeddings.token_type_embeddings.weight_resizeJ ,ߦ& JnLi%* d +B-bert.embeddings.word_embeddings.weight_resizeJ }R"Ѩ"3-3+R$N%%-,ئ >%.*'Ȩ ֩#d+,$yQ%7M#L-+%!-,$Y!C ",(,&% Qtt,-(!࢛?#,y, "##v-,#kT"i ,]+^ H$%S#-I,ܨ6L$$ -S,d$$-(-f#w ,.,#q̩#p,,@^` % ,A,w˦ݛ" ,2,* d"V-_-!̝0-t,!%-z-7d$ jl!W-+,""̪%_-,I!vS%"$,5,!Q$./+"u!J,,D֣x$#-N->㦽!L,-  zv&|,2, "i&G,y, &-7,*px/!+%-v+﨡k% $,<-n ݤqI--XV!-J,"Ҧ$$ ,x,G !-q,aէi9ڨ!"--ʞ[H75-6,0ޝɊѩ!-,zMg&G,t,#uI ' ,b-m/3Ҩ-[.+A Ш"o#,"- ̜ $h!-o+[ f%C%-,&$YGO!-,.&Ħ|"n-,n%. !-i-${u$--&UX Y,*l!:!\-*1fĨ!-{- +!}%ka"-+-<,$2V!3-,H#qTs,,"ڨ]""3g%+/+Ky X$N@$.,\K$q-(-ĢyI-s,%/ ps".+Ş4"͜!h-0+i%: -9-,!%kh$[,x,*V$*V+ -š)"S;$-R,%訾O'$ -.,;t!#-\-M #; #<,,ʍ"٩^.,_ qt$!,*m$A!n ȩ#:-+j#&4;--!Ed!"h-,I:'nw$ ,-"A$&&I!O-$, '&=- ,( %;m-t+; |g¨!-+]A#X"-,s$DA{",-$#R$רv#-,%ȘҪ,R,( 2j!b#,*&") !%- -YXt\Ϩ$~,,׈$>%Ǫ +r-7 O"-,$"d.4,D"_%g`&,c-V t%Ǫ{-B,̦$ V,,0!!$%,-0-"a#1 *[%o,,F!F@$j0",C- |]$,,'\&r--#;k# +)N,s,##멛,3,% f< .f,*S +B;bert.encoder.layer.0.attention.output.LayerNorm.bias_resizeJio275O9*U +B=bert.encoder.layer.0.attention.output.LayerNorm.weight_resizeJ;;;;;;*<(<*O +B7bert.encoder.layer.0.attention.output.dense.bias_resizeJ%ئZ"KI.*K +B3bert.encoder.layer.0.attention.self.key.bias_resizeJc={XS*M +B5bert.encoder.layer.0.attention.self.query.bias_resizeJ7?Et1V*M +B5bert.encoder.layer.0.attention.self.value.bias_resizeJ!T (&_虴"*{  +B3bert.encoder.layer.0.intermediate.dense.bias_resizeJ@ ¯y {j{*]ê8-L*I +B1bert.encoder.layer.0.output.LayerNorm.bias_resizeJ'g6d1٬*K +B3bert.encoder.layer.0.output.LayerNorm.weight_resizeJ;;4;H;6;:E;A:M:*E +B-bert.encoder.layer.0.output.dense.bias_resizeJ#(I,$.)-,*S +B;bert.encoder.layer.1.attention.output.LayerNorm.bias_resizeJ١1#i34*U +B=bert.encoder.layer.1.attention.output.LayerNorm.weight_resizeJ;;;;;;;;*O +B7bert.encoder.layer.1.attention.output.dense.bias_resizeJ4ui>)=9-n*K +B3bert.encoder.layer.1.attention.self.key.bias_resizeJ*핯 t*M +B5bert.encoder.layer.1.attention.self.query.bias_resizeJu%ܡS˩R(̤b*M +B5bert.encoder.layer.1.attention.self.value.bias_resizeJ֒% !ti*{  +B3bert.encoder.layer.1.intermediate.dense.bias_resizeJ@]X +Vyn᤺îڭp@um~ad *I +B1bert.encoder.layer.1.output.LayerNorm.bias_resizeJ%IK (.*K +B3bert.encoder.layer.1.output.LayerNorm.weight_resizeJ;;;;k;;:n:*E +B-bert.encoder.layer.1.output.dense.bias_resizeJl%$K(XK.d,*T +Bbert.encoder.layer.10.attention.output.LayerNorm.weight_resizeJR;O; ;];::::*P +B8bert.encoder.layer.10.attention.output.dense.bias_resizeJ(,5'{]) *L +B4bert.encoder.layer.10.attention.self.key.bias_resizeJ|! *N +B6bert.encoder.layer.10.attention.self.query.bias_resizeJ-'H[+*N +B6bert.encoder.layer.10.attention.self.value.bias_resizeJ&%Y")S*|  +B4bert.encoder.layer.10.intermediate.dense.bias_resizeJ@,:٬ILf楧a%9ͭJKD+*J +B2bert.encoder.layer.10.output.LayerNorm.bias_resizeJb%7M*L +B4bert.encoder.layer.10.output.LayerNorm.weight_resizeJI;V;;O;[;7;:m;*F +B.bert.encoder.layer.10.output.dense.bias_resizeJ +-«^0]}',*T +Bbert.encoder.layer.11.attention.output.LayerNorm.weight_resizeJq;;;o;:;:O;e;*P +B8bert.encoder.layer.11.attention.output.dense.bias_resizeJM"&,"&̗*L +B4bert.encoder.layer.11.attention.self.key.bias_resizeJk Jr/*N +B6bert.encoder.layer.11.attention.self.query.bias_resizeJW+ڟ@/f`2 ,*N +B6bert.encoder.layer.11.attention.self.value.bias_resizeJ_d\7O *|  +B4bert.encoder.layer.11.intermediate.dense.bias_resizeJ@C5( A$ŮݦpEN<#+Ӧѫ*J +B2bert.encoder.layer.11.output.LayerNorm.bias_resizeJq%UZ&}M*L +B4bert.encoder.layer.11.output.LayerNorm.weight_resizeJ999999:9*F +B.bert.encoder.layer.11.output.dense.bias_resizeJ.(E,uXd *S +B;bert.encoder.layer.2.attention.output.LayerNorm.bias_resizeJ6..| 3W5*U +B=bert.encoder.layer.2.attention.output.LayerNorm.weight_resizeJ;t;;~;x;;:;*O +B7bert.encoder.layer.2.attention.output.dense.bias_resizeJl .V(D3͙ݬɫ*K +B3bert.encoder.layer.2.attention.self.key.bias_resizeJ<hzҘ}!֜*M +B5bert.encoder.layer.2.attention.self.query.bias_resizeJ&&#~"*M +B5bert.encoder.layer.2.attention.self.value.bias_resizeJ +D *{  +B3bert.encoder.layer.2.intermediate.dense.bias_resizeJ@fjˬeì GQOh&u^[Jڭ"*I +B1bert.encoder.layer.2.output.LayerNorm.bias_resizeJeZ*Ұ*Ѳ*K +B3bert.encoder.layer.2.output.LayerNorm.weight_resizeJn;;;a;;T;:*:*E +B-bert.encoder.layer.2.output.dense.bias_resizeJF|ўA- '^-*S +B;bert.encoder.layer.3.attention.output.LayerNorm.bias_resizeJ+$𯅦ڭ0⯎4*U +B=bert.encoder.layer.3.attention.output.LayerNorm.weight_resizeJ;c;;[;;\;;;*O +B7bert.encoder.layer.3.attention.output.dense.bias_resizeJ~%E)d&)j/,]*K +B3bert.encoder.layer.3.attention.self.key.bias_resizeJQbƓ*M +B5bert.encoder.layer.3.attention.self.query.bias_resizeJ[&c"&&1 +w%*M +B5bert.encoder.layer.3.attention.self.value.bias_resizeJ%؜ tI +r*{  +B3bert.encoder.layer.3.intermediate.dense.bias_resizeJ@[&R UJ 孙Ddͬ=cӡn'_ϯǬ*I +B1bert.encoder.layer.3.output.LayerNorm.bias_resizeJC^*(Y/%*K +B3bert.encoder.layer.3.output.LayerNorm.weight_resizeJm;;N;o;|;F;::*E +B-bert.encoder.layer.3.output.dense.bias_resizeJǢz<{!,.*S +B;bert.encoder.layer.4.attention.output.LayerNorm.bias_resizeJ'.l[1*U +B=bert.encoder.layer.4.attention.output.LayerNorm.weight_resizeJi;;p;;#;>;+;j;*O +B7bert.encoder.layer.4.attention.output.dense.bias_resizeJA',MO*,,Ŭ*K +B3bert.encoder.layer.4.attention.self.key.bias_resizeJqj٠*M +B5bert.encoder.layer.4.attention.self.query.bias_resizeJ*?4M7";.j]*M +B5bert.encoder.layer.4.attention.self.value.bias_resizeJi;6l"N&z*{  +B3bert.encoder.layer.4.intermediate.dense.bias_resizeJ@CL$R"N~\tM񩵮5 KhM"+xp@*I +B1bert.encoder.layer.4.output.LayerNorm.bias_resizeJ," 5;*K +B3bert.encoder.layer.4.output.LayerNorm.weight_resizeJ;;;;;;F;X;*E +B-bert.encoder.layer.4.output.dense.bias_resizeJ䣑}7x+?x,*S +B;bert.encoder.layer.5.attention.output.LayerNorm.bias_resizeJ!X¯)G,b02*U +B=bert.encoder.layer.5.attention.output.LayerNorm.weight_resizeJC;g;h;,;";$;:;*O +B7bert.encoder.layer.5.attention.output.dense.bias_resizeJf*$)--*K +B3bert.encoder.layer.5.attention.self.key.bias_resizeJBE *M +B5bert.encoder.layer.5.attention.self.query.bias_resizeJ"Z9}(b(((*M +B5bert.encoder.layer.5.attention.self.value.bias_resizeJ5%֕N1G%S*{  +B3bert.encoder.layer.5.intermediate.dense.bias_resizeJ@4by sVi(\ܪmw.Syjbȫ =О*I +B1bert.encoder.layer.5.output.LayerNorm.bias_resizeJW(ӧ+)5*K +B3bert.encoder.layer.5.output.LayerNorm.weight_resizeJ;;;;;;c;c;*E +B-bert.encoder.layer.5.output.dense.bias_resizeJ`(*Ԧ,*S +B;bert.encoder.layer.6.attention.output.LayerNorm.bias_resizeJO S/W%//*U +B=bert.encoder.layer.6.attention.output.LayerNorm.weight_resizeJ];i;i;Y;t;);::*O +B7bert.encoder.layer.6.attention.output.dense.bias_resizeJ\+l&iX.h*!/*K +B3bert.encoder.layer.6.attention.self.key.bias_resizeJ{[[Cf*M +B5bert.encoder.layer.6.attention.self.query.bias_resizeJ'_%(r*M +B5bert.encoder.layer.6.attention.self.value.bias_resizeJF!/$vY*{  +B3bert.encoder.layer.6.intermediate.dense.bias_resizeJ@#_p(ϩIܯѫ6+ +)7M7C#հ`*I +B1bert.encoder.layer.6.output.LayerNorm.bias_resizeJ&*Ŭ'I(4*K +B3bert.encoder.layer.6.output.LayerNorm.weight_resizeJ;;;W;;;;Z;*E +B-bert.encoder.layer.6.output.dense.bias_resizeJ%&E "U#,*S +B;bert.encoder.layer.7.attention.output.LayerNorm.bias_resizeJ.)Izt.$߰.*U +B=bert.encoder.layer.7.attention.output.LayerNorm.weight_resizeJ\;;?;;u;=;:;*O +B7bert.encoder.layer.7.attention.output.dense.bias_resizeJQg,My)Q(.(*K +B3bert.encoder.layer.7.attention.self.key.bias_resizeJmI {DD *M +B5bert.encoder.layer.7.attention.self.query.bias_resizeJ."1G,/5P'*M +B5bert.encoder.layer.7.attention.self.value.bias_resizeJ+Ƨ+*{  +B3bert.encoder.layer.7.intermediate.dense.bias_resizeJ@I;X< a骳%(*ͯխ&ժŪȭ2+ѧ7*I +B1bert.encoder.layer.7.output.LayerNorm.bias_resizeJԩ(Ѫ(&x*K +B3bert.encoder.layer.7.output.LayerNorm.weight_resizeJO;l;N;A;U;W;;6;*E +B-bert.encoder.layer.7.output.dense.bias_resizeJ]/ Ǯ0@'u,*S +B;bert.encoder.layer.8.attention.output.LayerNorm.bias_resizeJ-h1Ϫ䯭-*U +B=bert.encoder.layer.8.attention.output.LayerNorm.weight_resizeJE;m;U;9;[;;: ;*O +B7bert.encoder.layer.8.attention.output.dense.bias_resizeJC)/2&)!-*K +B3bert.encoder.layer.8.attention.self.key.bias_resizeJeJ _d&^*M +B5bert.encoder.layer.8.attention.self.query.bias_resizeJ%0ҩo *M +B5bert.encoder.layer.8.attention.self.value.bias_resizeJǢp,'*{  +B3bert.encoder.layer.8.intermediate.dense.bias_resizeJ@wp5I[[kc( +-̦լ쩻ɭ W L]*I +B1bert.encoder.layer.8.output.LayerNorm.bias_resizeJ5)%* +*K +B3bert.encoder.layer.8.output.LayerNorm.weight_resizeJK;};p;B;;u;;b;*E +B-bert.encoder.layer.8.output.dense.bias_resizeJ&0$q ,(*S +B;bert.encoder.layer.9.attention.output.LayerNorm.bias_resizeJg)d0+*U +B=bert.encoder.layer.9.attention.output.LayerNorm.weight_resizeJ^;L;B;C;2;:::*O +B7bert.encoder.layer.9.attention.output.dense.bias_resizeJyx$,H+X0$*K +B3bert.encoder.layer.9.attention.self.key.bias_resizeJLN*M +B5bert.encoder.layer.9.attention.self.query.bias_resizeJ-%,:l*M +B5bert.encoder.layer.9.attention.self.value.bias_resizeJBHo"