From 7f46f347db55a121e82f50609d4ce23b6301ed10 Mon Sep 17 00:00:00 2001 From: Tianlei Wu Date: Fri, 17 Apr 2020 16:23:53 -0700 Subject: [PATCH] Add GPT2 Attention Fusion in optimization script (#3488) * Add Attention fusion for GPT2 * Support distilgpt2 in benchmark_gpt2.py * Add options to disable Attention/SkipLayerNormalization/EmbedLayerNormalization/BiasGelu fusions * Add logging at the begining of each fusion * Update notebooks: Add Gpt2OnnxModel.py to list of script files. * Add test for gpt2 model optimization * Add optional parameters (--input_ids --segment_ids --input_mask) for graph inputs * Fuse BiasGelu * Handle model that does not have segment_ids input. * Allow fuse embed layer without mask --- .../python/tools/bert/BertOnnxModel.py | 220 +++++++--- .../python/tools/bert/Gpt2OnnxModel.py | 180 +++++++- .../python/tools/bert/benchmark_gpt2.py | 44 +- .../tools/bert/bert_model_optimization.py | 57 ++- .../python/tools/bert/bert_perf_test.py | 13 +- .../python/tools/bert/bert_test_data.py | 40 +- .../python/tools/bert/compare_bert_results.py | 13 +- .../PyTorch_Bert-Squad_OnnxRuntime_CPU.ipynb | 38 +- .../PyTorch_Bert-Squad_OnnxRuntime_GPU.ipynb | 40 +- ...low_Keras_Bert-Squad_OnnxRuntime_CPU.ipynb | 48 +-- .../tools/bert/test_bert_optimization.py | 30 +- .../GPT2Model.onnx | Bin 0 -> 157561 bytes .../generate_tiny_gpt2_model.py | 394 ++++++++++++++++++ .../test_data_set_0/input_0.pb | Bin 0 -> 35 bytes .../test_data_set_0/output_0.pb | 1 + 15 files changed, 967 insertions(+), 151 deletions(-) create mode 100644 onnxruntime/python/tools/bert/test_data/gpt2_pytorch1.4_opset11_no_past/GPT2Model.onnx create mode 100644 onnxruntime/python/tools/bert/test_data/gpt2_pytorch1.4_opset11_no_past/generate_tiny_gpt2_model.py create mode 100644 onnxruntime/python/tools/bert/test_data/gpt2_pytorch1.4_opset11_no_past/test_data_set_0/input_0.pb create mode 100644 onnxruntime/python/tools/bert/test_data/gpt2_pytorch1.4_opset11_no_past/test_data_set_0/output_0.pb diff --git a/onnxruntime/python/tools/bert/BertOnnxModel.py b/onnxruntime/python/tools/bert/BertOnnxModel.py index a23cb0da8e..3917730a7e 100644 --- a/onnxruntime/python/tools/bert/BertOnnxModel.py +++ b/onnxruntime/python/tools/bert/BertOnnxModel.py @@ -15,6 +15,19 @@ from OnnxModel import OnnxModel logger = logging.getLogger(__name__) +class BertOptimizationOptions: + + def __init__(self, model_type): + self.enable_attention = True + self.enable_skip_layer_norm = True + self.enable_embed_layer_norm = True + self.enable_bias_skip_layer_norm = True + self.enable_bias_gelu = True + + if model_type == 'gpt2': + self.enable_skip_layer_norm = False + + class BertOnnxModel(OnnxModel): def __init__(self, model, num_heads, hidden_size, sequence_length, input_int32, float16, gpu_only): @@ -38,15 +51,31 @@ class BertOnnxModel(OnnxModel): self.bert_inputs = [] + def cast_input_to_int32(self, input_name): + cast_output = input_name + '_int32' + + # Avoid consequent Cast nodes. + inputs = [input_name] + output_name_to_node = self.output_name_to_node() + if input_name in output_name_to_node: + parent_node = output_name_to_node[input_name] + if parent_node and parent_node.op_type == 'Cast': + inputs = [parent_node.input[0]] + + cast_node = onnx.helper.make_node('Cast', inputs=inputs, outputs=[cast_output]) + cast_node.attribute.extend([onnx.helper.make_attribute("to", int(TensorProto.INT32))]) + self.add_node(cast_node) + + return cast_output, cast_node + def cast_graph_input_to_int32(self, input_name): graph_input = self.find_graph_input(input_name) if graph_input is not None and graph_input.type.tensor_type.elem_type != TensorProto.INT32: - cast_output = input_name + '_int32' - cast_node = onnx.helper.make_node('Cast', inputs=[input_name], outputs=[cast_output]) - cast_node.attribute.extend([onnx.helper.make_attribute("to", int(TensorProto.INT32))]) - self.add_node(cast_node) + cast_output, cast_node = self.cast_input_to_int32(input_name) + logger.debug("Casted graph input {input_name} to int32") return True, cast_output + logger.debug(f"Did not cast graph input {input_name} to int32: found {graph_input is not None}") return False, input_name def undo_cast_input_to_int32(self, input_name): @@ -69,7 +98,12 @@ class BertOnnxModel(OnnxModel): return self.mask_indice[input] # Add cast to convert int64 to int32 - casted, input_name = self.cast_graph_input_to_int32(input) + if self.find_graph_input(input): + casted, input_name = self.cast_graph_input_to_int32(input) + else: + input_name, cast_node = self.cast_input_to_int32(input) + casted = True + if casted: self.mask_casted[input] = input_name @@ -125,19 +159,12 @@ class BertOnnxModel(OnnxModel): vals=qkv_weight.flatten().tolist()) self.add_initializer(weight) - weight_input = onnx.helper.make_tensor_value_info(weight.name, TensorProto.FLOAT, - [self.hidden_size, 3 * self.hidden_size]) - self.add_input(weight_input) - bias = onnx.helper.make_tensor(name=attention_node_name + '_qkv_bias', data_type=TensorProto.FLOAT, dims=[3 * self.hidden_size], vals=qkv_bias.flatten().tolist()) self.add_initializer(bias) - bias_input = onnx.helper.make_tensor_value_info(bias.name, TensorProto.FLOAT, [3 * self.hidden_size]) - self.add_input(bias_input) - attention_node = onnx.helper.make_node( 'Attention', inputs=[input, attention_node_name + '_qkv_weight', attention_node_name + '_qkv_bias', mask_index], @@ -350,6 +377,7 @@ class BertOnnxModel(OnnxModel): """ def fuse_gelu_with_tanh(self): + logger.debug(f"start FastGelu fusion...") input_name_to_nodes = self.input_name_to_nodes() output_name_to_node = self.output_name_to_node() @@ -435,19 +463,23 @@ class BertOnnxModel(OnnxModel): nodes_to_add.append(gelu_node) if len(nodes_to_add) > 0: - logger.info("Fused FastGelu count: {len(nodes_to_add)}") + logger.info(f"Fused FastGelu count: {len(nodes_to_add)}") self.remove_nodes(nodes_to_remove) self.add_nodes(nodes_to_add) - def fuse_add_bias_gelu(self): + def fuse_bias_gelu(self, is_fastgelu): + gelu_op_type = 'FastGelu' if is_fastgelu else 'Gelu' + bias_gelu_op_type = 'FastGelu' if is_fastgelu else 'BiasGelu' + logger.debug(f"start Bias and {gelu_op_type} fusion...") + input_name_to_nodes = self.input_name_to_nodes() output_name_to_node = self.output_name_to_node() nodes_to_remove = [] nodes_to_add = [] # Don't need to fuse Gelu+Add here because ORT native code can handle it - for node in self.get_nodes_by_op_type('FastGelu'): + for node in self.get_nodes_by_op_type(gelu_op_type): if len(node.input) != 1: continue @@ -476,20 +508,21 @@ class BertOnnxModel(OnnxModel): continue nodes_to_remove.extend(subgraph_nodes) - gelu_node = onnx.helper.make_node('FastGelu', + gelu_node = onnx.helper.make_node(bias_gelu_op_type, inputs=[matmul.output[0], add.input[bias_index]], outputs=node.output, - name=self.create_node_name('FastGelu', "FastGelu_AddBias_")) + name=self.create_node_name(bias_gelu_op_type, gelu_op_type + "_AddBias_")) gelu_node.domain = "com.microsoft" nodes_to_add.append(gelu_node) if len(nodes_to_add) > 0: - logger.info(f"Fused FastGelu with Bias count:{len(nodes_to_add)}") + logger.info(f"Fused {bias_gelu_op_type} with Bias count:{len(nodes_to_add)}") self.remove_nodes(nodes_to_remove) self.add_nodes(nodes_to_add) def fuse_add_bias_skip_layer_norm(self): + logger.debug(f"start Bias and SkipLayerNormalization fusion...") input_name_to_nodes = self.input_name_to_nodes() output_name_to_node = self.output_name_to_node() nodes_to_remove = [] @@ -552,6 +585,7 @@ class BertOnnxModel(OnnxModel): self.add_nodes(nodes_to_add) def fuse_reshape(self): + logger.debug(f"start Reshape fusion...") nodes = self.nodes() input_name_to_nodes = self.input_name_to_nodes() output_name_to_node = self.output_name_to_node() @@ -705,21 +739,12 @@ class BertOnnxModel(OnnxModel): SkipLayerNormalization """ - def fuse_embed_layer(self): + def fuse_embed_layer_without_mask(self): + logger.debug(f"start EmbedLayerNormalization (no mask) fusion...") nodes = self.nodes() input_name_to_nodes = self.input_name_to_nodes() output_name_to_node = self.output_name_to_node() - if len(self.mask_indice) == 0: - logger.info("skip embed layer fusion since mask input is not found") - return - if len(self.mask_indice) > 1: - logger.info("skip embed layer fusion since there are multiple mask inputs found") - return - mask_input_name = next(iter(self.mask_indice)) - mask_output_name = self.mask_indice[mask_input_name] - mask_node = output_name_to_node[mask_output_name] - nodes_to_remove = [] # Find the first normalize node could be embedding layer. @@ -731,6 +756,14 @@ class BertOnnxModel(OnnxModel): if self.find_first_child_by_type(node, 'Attention', input_name_to_nodes, recursive=False) is not None: normalize_node = node break + # In case user disables attention fusion, check whether subgraph looks like Attention. + if node.output[0] not in input_name_to_nodes: + continue + children = input_name_to_nodes[node.output[0]] + children_types = sorted([child.op_type for child in children]) + if children_types == ['MatMul', 'MatMul', 'MatMul', 'SkipLayerNormalization']: + normalize_node = node + break if normalize_node is None: if len(self.get_nodes_by_op_type("EmbedLayerNormalization")) == 0: @@ -780,7 +813,9 @@ class BertOnnxModel(OnnxModel): segment_ids = segment_embedding_gather.input[1] if position_embedding_expand: - subgraph_nodes = self.get_parent_subgraph_nodes(position_embedding_expand, [], output_name_to_node) + input_parent = self.get_parent(position_embedding_shape, 0, output_name_to_node) + subgraph_nodes = self.get_parent_subgraph_nodes(position_embedding_expand, + [input_parent] if input_parent else [], output_name_to_node) nodes_to_remove.extend(subgraph_nodes) nodes_to_remove.extend(word_embedding_path) @@ -788,22 +823,37 @@ class BertOnnxModel(OnnxModel): nodes_to_remove.extend(segment_embedding_path) nodes_to_remove.extend([normalize_node]) - nodes_to_remove.extend([mask_node]) # store inputs for further processing - self.bert_inputs = [input_ids, segment_ids, mask_input_name] + if self.find_graph_input(input_ids): + self.bert_inputs = [input_ids, segment_ids] if self.find_graph_input(segment_ids) else [input_ids] - if not self.input_int32: - # When mask has been casted to int32, use that casted one as input of embed layer norm. - if mask_input_name in self.mask_casted: - mask_input_name = self.mask_casted[mask_input_name] - - # Cast input_ids and segment_ids to int32. - casted, input_ids = self.cast_graph_input_to_int32(input_ids) - - casted, segment_ids = self.cast_graph_input_to_int32(segment_ids) + # Cast input_ids and segment_ids to int32. + if self.find_graph_input(input_ids): + if not self.input_int32: + casted, input_ids = self.cast_graph_input_to_int32(input_ids) else: - self.undo_cast_input_to_int32(mask_input_name) + input_ids, input_ids_cast_node = self.cast_input_to_int32(input_ids) + + if self.find_graph_input(segment_ids): + if not self.input_int32: + casted, segment_ids = self.cast_graph_input_to_int32(segment_ids) + else: + segment_ids, segment_ids_cast_node = self.cast_input_to_int32(segment_ids) + + segment_id_path = self.match_parent_path( + segment_ids_cast_node, ['ConstantOfShape', 'Concat', 'Unsqueeze', 'Gather', 'Shape', 'Cast'], + [0, 0, 1, 0, 0, 0]) + if segment_id_path and input_ids_cast_node and input_ids_cast_node.input[0] == segment_id_path[-1].input[0]: + logger.debug("Simplify semgent id path...") + self.add_node( + onnx.helper.make_node('Shape', inputs=[input_ids_cast_node.input[0]], outputs=["input_shape"])) + self.add_node( + onnx.helper.make_node('ConstantOfShape', + inputs=["input_shape"], + outputs=["zeros_for_input_shape"], + value=onnx.helper.make_tensor("value", onnx.TensorProto.INT32, [1], [1]))) + segment_ids = "zeros_for_input_shape" embed_node = onnx.helper.make_node( 'EmbedLayerNormalization', @@ -814,10 +864,9 @@ class BertOnnxModel(OnnxModel): position_embedding_weight_node.input[0], segment_embedding_gather.input[0], normalize_node.input[2], - normalize_node.input[3], # gamma and beta - mask_input_name + normalize_node.input[3] # gamma and beta ], - outputs=["embed_output", mask_output_name], + outputs=["embed_output", "dummy_mask_index"], name="EmbedLayer") embed_node.domain = "com.microsoft" @@ -826,13 +875,50 @@ class BertOnnxModel(OnnxModel): self.remove_nodes(nodes_to_remove) self.add_node(embed_node) - self.update_graph() - logger.info("Fused EmbedLayerNormalization count: 1") + self.prune_graph() + + return embed_node + + def fuse_embed_layer(self): + embed_node = self.fuse_embed_layer_without_mask() + if embed_node is None: + logger.info("Fused EmbedLayerNormalization count: 0") + return + + if len(self.mask_indice) > 1: + logger.info("There are multiple mask inputs found!") + + if len(self.mask_indice) != 1: + logger.info("Fused EmbedLayerNormalization (no mask) count: 1") + return + + mask_input_name = next(iter(self.mask_indice)) + mask_output_name = self.mask_indice[mask_input_name] + mask_node = output_name_to_node[mask_output_name] + + nodes_to_remove = [] + nodes_to_remove.extend([mask_node]) + + # store inputs for further processing + self.bert_inputs.append(mask_input_name) + + if not self.input_int32: + # When mask has been casted to int32, use that casted one as input of embed layer norm. + if mask_input_name in self.mask_casted: + mask_input_name = self.mask_casted[mask_input_name] + else: + self.undo_cast_input_to_int32(mask_input_name) + + embed_node.input[7] = mask_input_name + embed_node.output[1] = mask_output_name + logger.info("Added mask to EmbedLayerNormalization") # Change graph input data type int32 if needed. if self.input_int32: self.change_input_to_int32() + logger.info("Fused EmbedLayerNormalization count: 1") + def get_bert_inputs(self, include_mask=True): return self.bert_inputs if include_mask else self.bert_inputs[:2] @@ -856,14 +942,13 @@ class BertOnnxModel(OnnxModel): graph = self.graph() batch_size = self.get_batch_size_from_graph_input() - input_batch_size = batch_size if isinstance(batch_size, int) else 1 new_graph_inputs = [] bert_inputs = self.get_bert_inputs() for input in graph.input: if input.name in bert_inputs: - int32_input = onnx.helper.make_tensor_value_info(input.name, TensorProto.INT32, - [input_batch_size, self.sequence_length]) + input_shape = [batch_size if isinstance(batch_size, int) else 1, self.sequence_length] + int32_input = onnx.helper.make_tensor_value_info(input.name, TensorProto.INT32, input_shape) new_graph_inputs.append(int32_input) else: new_graph_inputs.append(input) @@ -924,6 +1009,8 @@ class BertOnnxModel(OnnxModel): | | +----------------------+ """ + logger.debug(f"start LayerNormalization fusion...") + input_name_to_nodes = self.input_name_to_nodes() output_name_to_node = self.output_name_to_node() @@ -1013,6 +1100,8 @@ class BertOnnxModel(OnnxModel): """ Fuse Add + LayerNormalization into one node: SkipLayerNormalization """ + logger.debug(f"start SkipLayerNormaliation fusion...") + input_name_to_nodes = self.input_name_to_nodes() output_name_to_node = self.output_name_to_node() @@ -1043,9 +1132,9 @@ class BertOnnxModel(OnnxModel): return def postprocess(self): - return + self.prune_graph() - def optimize(self): + def optimize(self, options: BertOptimizationOptions = None): self.fuse_layer_norm() self.fuse_gelu() @@ -1054,20 +1143,28 @@ class BertOnnxModel(OnnxModel): self.fuse_reshape() - self.fuse_skip_layer_norm() + if (options is None) or options.enable_skip_layer_norm: + self.fuse_skip_layer_norm() - self.fuse_attention() + if (options is None) or options.enable_attention: + self.fuse_attention() - self.fuse_embed_layer() - - # Fuse Gelu and Add Bias before it. - self.fuse_add_bias_gelu() - - # Fuse SkipLayerNormalization and Add Bias before it. - self.fuse_add_bias_skip_layer_norm() + if (options is None) or options.enable_embed_layer_norm: + self.fuse_embed_layer() + # Post-processing like removing extra reshape nodes. self.postprocess() + # Bias fusion is done after postprocess to avoid extra Reshape between bias and Gelu/FastGelu/SkipLayerNormalization + if (options is None) or options.enable_bias_gelu: + # Fuse Gelu and Add Bias before it. + self.fuse_bias_gelu(is_fastgelu=True) + self.fuse_bias_gelu(is_fastgelu=False) + + if (options is None) or options.enable_bias_skip_layer_norm: + # Fuse SkipLayerNormalization and Add Bias before it. + self.fuse_add_bias_skip_layer_norm() + if self.float16: self.convert_model_float32_to_float16() @@ -1090,6 +1187,7 @@ class BertOnnxModel(OnnxModel): for op in ops: nodes = self.get_nodes_by_op_type(op) op_count[op] = len(nodes) + logger.info(f"Optimized operators:{op_count}") return op_count def is_fully_optimized(self): diff --git a/onnxruntime/python/tools/bert/Gpt2OnnxModel.py b/onnxruntime/python/tools/bert/Gpt2OnnxModel.py index 949ee27357..6a0633c61b 100644 --- a/onnxruntime/python/tools/bert/Gpt2OnnxModel.py +++ b/onnxruntime/python/tools/bert/Gpt2OnnxModel.py @@ -2,11 +2,189 @@ # Copyright (c) Microsoft Corporation. All rights reserved. # Licensed under the MIT License. #-------------------------------------------------------------------------- - +import logging +import onnx +import sys +import argparse +import numpy as np +from collections import deque +from onnx import ModelProto, TensorProto, numpy_helper from BertOnnxModel import BertOnnxModel +logger = logging.getLogger(__name__) + class Gpt2OnnxModel(BertOnnxModel): def __init(self, model, num_heads, hidden_size, sequence_length, input_int32, float16, gpu_only): super().__init__(model, num_heads, hidden_size, sequence_length, input_int32, float16, gpu_only) + + def fuse_attention(self): + """ + Fuse Attention subgraph into one Attention node. + """ + logger.debug(f"start attention fusion...") + + input_name_to_nodes = self.input_name_to_nodes() + output_name_to_node = self.output_name_to_node() + + nodes_to_remove = [] + attention_count = 0 + + for normalize_node in self.get_nodes_by_op_type("LayerNormalization"): + return_indice = [] + qkv_nodes = self.match_parent_path( + normalize_node, + ['Add', 'Reshape', 'Gemm', 'Reshape', 'Reshape', 'Transpose', 'MatMul'], + [0, None, 0, 0, 0, 0, 0], + output_name_to_node=output_name_to_node, + return_indice=return_indice + ) # yapf: disable + if qkv_nodes is None: + continue + (add_qkv, reshape_qkv, gemm_qkv, reshape_1, reshape_2, transpose_qkv, matmul_qkv) = qkv_nodes + + another_input = add_qkv.input[1 - return_indice[0]] + + v_nodes = self.match_parent_path( + matmul_qkv, + ['Transpose', 'Reshape', 'Split', 'Reshape', 'Gemm', 'Reshape'], + [1, 0, 0, 0, 0, 0]) # yapf: disable + if v_nodes is None: + logger.debug("fuse_attention: failed to match v path") + continue + (transpose_v, reshape_v, split_v, reshape_after_gemm, gemm, reshape_before_gemm) = v_nodes + + layernorm_before_attention = self.get_parent(reshape_before_gemm, 0, output_name_to_node) + if layernorm_before_attention is None or layernorm_before_attention.op_type != 'LayerNormalization': + logger.debug(f"failed to get layernorm before gemm. Got {layernorm_before_attention.op_type}") + continue + + if not another_input in layernorm_before_attention.input: + logger.debug("Add and LayerNormalization shall have one same input") + continue + + qk_nodes = self.match_parent_path(matmul_qkv, ['Softmax', 'Sub', 'Mul', 'Div', 'MatMul'], [0, 0, 0, 0, 0]) + if qk_nodes is None: + logger.debug("fuse_attention: failed to match qk path") + continue + (softmax_qk, sub_qk, mul_qk, div_qk, matmul_qk) = qk_nodes + + q_nodes = self.match_parent_path(matmul_qk, ['Transpose', 'Reshape', 'Split'], [0, 0, 0]) + if q_nodes is None: + logger.debug("fuse_attention: failed to match q path") + continue + (transpose_q, reshape_q, split_q) = q_nodes + if split_v != split_q: + logger.debug("fuse_attention: skip since split_v != split_q") + continue + + k_nodes = self.match_parent_path(matmul_qk, ['Transpose', 'Reshape', 'Split'], [1, 0, 0]) + if k_nodes is None: + logger.debug("fuse_attention: failed to match k path") + continue + (transpose_k, reshape_k, split_k) = k_nodes + if split_v != split_k: + logger.debug("fuse_attention: skip since split_v != split_k") + continue + + mask_nodes = self.match_parent_path( + sub_qk, + ['Mul', 'Sub', 'Slice', 'Slice', 'Unsqueeze', 'Sub', 'Squeeze', 'Slice', 'Shape', 'Div'], + [1, 0, 1, 0, 1, 0, 0, 0, 0, 0]) # yapf: disable + if mask_nodes is None: + logger.debug("fuse_attention: failed to match mask path") + continue + (mul_mask, sub_mask, slice_mask, slice_mask_0, unsqueeze_mask, sub_mask, squeeze_mask, slice_mask_1, + shape_mask, div_mask) = mask_nodes + + if div_qk != div_mask: + logger.debug("fuse_attention: skip since div_qk != div_mask") + continue + + self.create_attention_node(gemm, gemm_qkv, layernorm_before_attention.output[0], reshape_qkv.output[0], + attention_count == 0) + nodes_to_remove.extend([reshape_qkv, transpose_qkv, matmul_qkv]) + nodes_to_remove.extend(qk_nodes) + nodes_to_remove.extend(q_nodes) + nodes_to_remove.extend(k_nodes) + nodes_to_remove.extend(v_nodes) + nodes_to_remove.extend(mask_nodes) + attention_count += 1 + + self.remove_nodes(nodes_to_remove) + self.prune_graph() + logger.info(f"Fused Attention count:{attention_count}") + + def create_attention_node(self, gemm, gemm_qkv, input, output, add_graph_input): + attention_node_name = self.create_node_name('Attention') + attention_node = onnx.helper.make_node('Attention', + inputs=[input, gemm.input[1], gemm.input[2]], + outputs=[attention_node_name + "_output"], + name=attention_node_name) + attention_node.domain = "com.microsoft" + attention_node.attribute.extend( + [onnx.helper.make_attribute("num_heads", self.num_heads), + onnx.helper.make_attribute("unidirectional", 1)]) + + matmul_node = onnx.helper.make_node('MatMul', + inputs=[attention_node_name + "_output", gemm_qkv.input[1]], + outputs=[attention_node_name + "_matmul_output"], + name=attention_node_name + "_matmul") + + add_node = onnx.helper.make_node('Add', + inputs=[attention_node_name + "_matmul_output", gemm_qkv.input[2]], + outputs=[output], + name=attention_node_name + "_add") + + self.add_node(attention_node) + self.add_node(matmul_node) + self.add_node(add_node) + + def postprocess(self): + """ + Remove extra reshape nodes. + """ + logger.debug(f"start postprocessing...") + + input_name_to_nodes = self.input_name_to_nodes() + output_name_to_node = self.output_name_to_node() + + reshape_count = 0 + for gemm_node in self.get_nodes_by_op_type("Gemm"): + reshape_after_gemm = self.find_first_child_by_type(gemm_node, + 'Reshape', + input_name_to_nodes, + recursive=False) + + return_indice = [] + nodes = self.match_parent_path(gemm_node, ['Reshape', 'FastGelu'], [0, 0], output_name_to_node) + if nodes is None: + nodes = self.match_parent_path(gemm_node, ['Reshape', 'LayerNormalization'], [0, 0], + output_name_to_node) + if nodes is None: + continue + (reshape_before_gemm, root_node) = nodes + + matmul_node_name = self.create_node_name('MatMul', 'FullyConnect_MatMul') + matmul_node = onnx.helper.make_node('MatMul', + inputs=[matmul_node_name + "_input", gemm_node.input[1]], + outputs=[matmul_node_name + "_output"], + name=matmul_node_name) + + add_node_name = self.create_node_name('Add', 'FullyConnect_Add') + add_node = onnx.helper.make_node('Add', + inputs=[matmul_node_name + "_output", gemm_node.input[2]], + outputs=[add_node_name + "_output"], + name=add_node_name) + + root_node.output[0] = matmul_node_name + "_input" + self.replace_input_of_all_nodes(reshape_after_gemm.output[0], add_node_name + "_output") + + self.add_node(matmul_node) + self.add_node(add_node) + + reshape_count += 2 + + self.prune_graph() + logger.info(f"Remove Reshape count:{reshape_count}") diff --git a/onnxruntime/python/tools/bert/benchmark_gpt2.py b/onnxruntime/python/tools/bert/benchmark_gpt2.py index fe3a46791e..6be7310ee0 100644 --- a/onnxruntime/python/tools/bert/benchmark_gpt2.py +++ b/onnxruntime/python/tools/bert/benchmark_gpt2.py @@ -11,10 +11,16 @@ import psutil import argparse import logging import torch -from transformers import GPT2Model, GPT2Tokenizer +from transformers import GPT2Model, GPT2LMHeadModel, GPT2Tokenizer logger = logging.getLogger('') +# Map alias to a tuple of Model Class and pretrained model name +MODEL_CLASSES = { + "gpt2": (GPT2Model, GPT2Tokenizer, "gpt2"), + "distilgpt2": (GPT2LMHeadModel, GPT2Tokenizer, "distilgpt2") +} + def dump_environment(): if "OMP_NUM_THREADS" in os.environ: @@ -44,9 +50,7 @@ def pytorch_inference(model, input_ids, past=None, total_runs=100): with torch.no_grad(): for _ in range(total_runs): start = time.time() - outputs = model( - input_ids=input_ids, - past=past) #attention_mask=inputs['attention_mask'], token_type_ids=inputs['token_type_ids'], + outputs = model(input_ids=input_ids, past=past) latency.append(time.time() - start) logger.info("PyTorch Inference time = {} ms".format(format(sum(latency) * 1000 / len(latency), '.2f'))) @@ -88,6 +92,12 @@ def inference(model, ort_session, input_ids, past=None, total_runs=100, verify_o def parse_arguments(): parser = argparse.ArgumentParser() + parser.add_argument('--model_type', + required=True, + type=str, + choices=list(MODEL_CLASSES.keys()), + help="Model type selected in the list: " + ", ".join(MODEL_CLASSES.keys())) + parser.add_argument('--cache_dir', required=True, type=str, help="cache directory") parser.add_argument('--output_dir', required=True, type=str, help="output onnx model directory") @@ -132,6 +142,24 @@ def setup_logger(verbose=True): logger.setLevel(logging_level) +def remove_past_outputs(export_model_path): + from onnx import ModelProto + from OnnxModel import OnnxModel + + model = ModelProto() + with open(export_model_path, "rb") as f: + model.ParseFromString(f.read()) + bert_model = OnnxModel(model) + + # remove past state outputs and only keep the first output. + keep_output_names = [bert_model.model.graph.output[0].name] + logger.info(f"Prune graph to keep the first output and drop past state outputs:{keep_output_names}") + bert_model.prune_graph(keep_output_names) + onnx_model_path = os.path.join(output_dir, 'gpt2_past{}_out1.onnx'.format(int(enable_past_input))) + bert_model.save_model_to_file(onnx_model_path) + return onnx_model_path + + def main(): args = parse_arguments() setup_logger(args.verbose) @@ -147,7 +175,8 @@ def main(): if not os.path.exists(output_dir): os.makedirs(output_dir) - model_class, tokenizer_class, model_name_or_path = (GPT2Model, GPT2Tokenizer, 'gpt2') + (model_class, tokenizer_class, model_name_or_path) = MODEL_CLASSES[args.model_type] + tokenizer = tokenizer_class.from_pretrained(model_name_or_path, cache_dir=cache_dir) model = model_class.from_pretrained(model_name_or_path, cache_dir=cache_dir) model.eval().cpu() @@ -197,10 +226,11 @@ def main(): setup_environment(args.use_openmp) import onnxruntime - onnx_model_path = export_model_path + onnx_model_path = export_model_path if enable_past_input else remove_past_outputs(export_model_path) + if args.enable_optimization: from bert_model_optimization import optimize_model - m = optimize_model(export_model_path, + m = optimize_model(onnx_model_path, model_type='gpt2', gpu_only=False, num_heads=12, diff --git a/onnxruntime/python/tools/bert/bert_model_optimization.py b/onnxruntime/python/tools/bert/bert_model_optimization.py index feee35b721..ec1689d531 100644 --- a/onnxruntime/python/tools/bert/bert_model_optimization.py +++ b/onnxruntime/python/tools/bert/bert_model_optimization.py @@ -31,7 +31,7 @@ import argparse import numpy as np from collections import deque from onnx import ModelProto, TensorProto, numpy_helper -from BertOnnxModel import BertOnnxModel +from BertOnnxModel import BertOnnxModel, BertOptimizationOptions from BertOnnxModelTF import BertOnnxModelTF from BertOnnxModelKeras import BertOnnxModelKeras from Gpt2OnnxModel import Gpt2OnnxModel @@ -137,6 +137,33 @@ def parse_arguments(): help="whether the target device is gpu or not") parser.set_defaults(gpu_only=False) + parser.add_argument('--disable_attention', required=False, action='store_true', help="disable Attention fusion") + parser.set_defaults(disable_attention=False) + + parser.add_argument('--disable_skip_layer_norm', + required=False, + action='store_true', + help="disable SkipLayerNormalization fusion") + parser.set_defaults(disable_skip_layer_norm=False) + + parser.add_argument('--disable_embed_layer_norm', + required=False, + action='store_true', + help="disable EmbedLayerNormalization fusion") + parser.set_defaults(disable_embed_layer_norm=False) + + parser.add_argument('--disable_bias_skip_layer_norm', + required=False, + action='store_true', + help="disable Add Bias and SkipLayerNormalization fusion") + parser.set_defaults(disable_bias_skip_layer_norm=False) + + parser.add_argument('--disable_bias_gelu', + required=False, + action='store_true', + help="disable Add Bias and Gelu/FastGelu fusion") + parser.set_defaults(disable_bias_gelu=False) + parser.add_argument('--verbose', required=False, action='store_true') parser.set_defaults(verbose=False) @@ -152,6 +179,21 @@ def parse_arguments(): return args +def get_optimization_options(args): + optimization_options = BertOptimizationOptions(args.model_type) + if args.disable_attention: + optimization_options.enable_attention = False + if args.disable_skip_layer_norm: + optimization_options.enable_skip_layer_norm = False + if args.disable_embed_layer_norm: + optimization_options.enable_embed_layer_norm = False + if args.disable_bias_skip_layer_norm: + optimization_options.enable_bias_skip_layer_norm = False + if args.disable_bias_gelu: + optimization_options.enable_bias_gelu = False + return optimization_options + + def optimize_model(input, model_type, gpu_only, @@ -160,7 +202,8 @@ def optimize_model(input, sequence_length, input_int32, float16, - opt_level=99): + opt_level=99, + optimization_options=None): (optimizer_class, producer, run_onnxruntime) = MODEL_CLASSES[model_type] input_model_path = input @@ -177,8 +220,11 @@ def optimize_model(input, f"Model producer not matched: Expect {producer}, Got {model.producer_name} {model.producer_version}. Please specify correct --model_type parameter." ) + if optimization_options is None: + optimization_options = BertOptimizationOptions(model_type) + bert_model = optimizer_class(model, num_heads, hidden_size, sequence_length, input_int32, float16, gpu_only) - bert_model.optimize() + bert_model.optimize(optimization_options) return bert_model @@ -195,8 +241,11 @@ def main(): setup_logger(args.verbose) + optimization_options = get_optimization_options(args) + bert_model = optimize_model(args.input, args.model_type, args.gpu_only, args.num_heads, args.hidden_size, - args.sequence_length, args.input_int32, args.float16, args.opt_level) + args.sequence_length, args.input_int32, args.float16, args.opt_level, + optimization_options) bert_model.save_model_to_file(args.output) diff --git a/onnxruntime/python/tools/bert/bert_perf_test.py b/onnxruntime/python/tools/bert/bert_perf_test.py index 2495440463..62de94635d 100644 --- a/onnxruntime/python/tools/bert/bert_perf_test.py +++ b/onnxruntime/python/tools/bert/bert_perf_test.py @@ -240,9 +240,10 @@ def run_perf_tests(perf_results, model_path, batch_size, sequence_length, use_gp def run_performance(perf_results, model_path, batch_size, sequence_length, use_gpu, test_cases, test_times, seed, - verbose, inclusive, test_all, no_warmup, opt_level): - # Try deduce input names from model. - input_ids, segment_ids, input_mask = get_bert_inputs(model_path) + verbose, inclusive, test_all, no_warmup, opt_level, input_ids_name, segment_ids_name, + input_mask_name): + + input_ids, segment_ids, input_mask = get_bert_inputs(model_path, input_ids_name, segment_ids_name, input_mask_name) # Do not generate random mask for performance test. print(f"Generating {test_cases} samples for batch_size={batch_size} sequence_length={sequence_length}") @@ -347,6 +348,10 @@ def parse_arguments(): parser.add_argument('--no_warmup', required=False, action='store_true', help="do not use one sample for warm-up.") parser.set_defaults(no_warmup=False) + parser.add_argument('--input_ids', required=False, type=str, default=None, help="input name for input ids") + parser.add_argument('--segment_ids', required=False, type=str, default=None, help="input name for segment ids") + parser.add_argument('--input_mask', required=False, type=str, default=None, help="input name for attention mask") + args = parser.parse_args() return args @@ -367,7 +372,7 @@ def main(): for batch_size in batch_size_set: run_performance(perf_results, args.model, batch_size, args.sequence_length, args.use_gpu, args.samples, args.test_times, args.seed, args.verbose, args.inclusive, args.all, args.no_warmup, - args.opt_level) + args.opt_level, args.input_ids, args.segment_ids, args.input_mask) # Sort the results so that the first one has smallest latency. sorted_results = sorted(perf_results.items(), reverse=False, key=lambda x: x[1]) diff --git a/onnxruntime/python/tools/bert/bert_test_data.py b/onnxruntime/python/tools/bert/bert_test_data.py index 5ab0f5e7da..b2c609056a 100644 --- a/onnxruntime/python/tools/bert/bert_test_data.py +++ b/onnxruntime/python/tools/bert/bert_test_data.py @@ -107,15 +107,22 @@ def fake_test_data(batch_size, sequence_length, test_cases, dictionary_size, ver """ Generate fake input data for test. """ + assert input_ids is not None + np.random.seed(random_seed) random.seed(random_seed) all_inputs = [] for test_case in range(test_cases): input_1 = fake_input_ids_data(input_ids, batch_size, sequence_length, dictionary_size) - input_2 = fake_segment_ids_data(segment_ids, batch_size, sequence_length) - input_3 = fake_input_mask_data(input_mask, batch_size, sequence_length, random_mask_length) - inputs = {input_ids.name: input_1, segment_ids.name: input_2, input_mask.name: input_3} + inputs = {input_ids.name: input_1} + + if segment_ids: + inputs[segment_ids.name] = fake_segment_ids_data(segment_ids, batch_size, sequence_length) + + if input_mask: + inputs[input_mask.name] = fake_input_mask_data(input_mask, batch_size, sequence_length, random_mask_length) + if verbose and len(all_inputs) == 0: print("Example inputs", inputs) all_inputs.append(inputs) @@ -144,7 +151,7 @@ def get_graph_input_from_embed_node(onnx_model, embed_node, input_index): return graph_input -def get_bert_inputs(onnx_file): +def get_bert_inputs(onnx_file, input_ids_name=None, segment_ids_name=None, input_mask_name=None): """ Get graph inputs for bert model. First, we will deduce from EmbedLayerNormalization node. If not found, we will guess based on naming. @@ -154,8 +161,31 @@ def get_bert_inputs(onnx_file): model.ParseFromString(f.read()) onnx_model = OnnxModel(model) - graph_inputs = onnx_model.get_graph_inputs_excluding_initializers() + + if input_ids_name is not None: + input_ids = onnx_model.find_graph_input(input_ids_name) + if input_ids is None: + raise ValueError(f"Graph does not have input named {input_ids_name}") + + segment_ids = None + if segment_ids_name: + segment_ids = onnx_model.find_graph_input(segment_ids_name) + if segment_ids is None: + raise ValueError(f"Graph does not have input named {segment_ids_name}") + + input_mask = None + if input_mask_name: + input_mask = onnx_model.find_graph_input(input_mask_name) + if input_mask is None: + raise ValueError(f"Graph does not have input named {input_mask_name}") + + expected_inputs = 1 + (1 if segment_ids else 0) + (1 if input_mask else 0) + if len(graph_inputs) != expected_inputs: + raise ValueError(f"Expect the graph to have {expected_inputs} inputs. Got {len(graph_inputs)}") + + return input_ids, segment_ids, input_mask + if len(graph_inputs) != 3: raise ValueError("Expect the graph to have 3 inputs. Got {}".format(len(graph_inputs))) diff --git a/onnxruntime/python/tools/bert/compare_bert_results.py b/onnxruntime/python/tools/bert/compare_bert_results.py index 248a25d246..13b603a6a2 100644 --- a/onnxruntime/python/tools/bert/compare_bert_results.py +++ b/onnxruntime/python/tools/bert/compare_bert_results.py @@ -78,9 +78,11 @@ def compare(baseline_results, treatment_results, verbose, rtol=1e-3, atol=1e-4): def run_test(baseline_model, optimized_model, output_dir, batch_size, sequence_length, use_gpu, test_cases, seed, - use_openmp, verbose, rtol, atol): + use_openmp, verbose, rtol, atol, input_ids_name, segment_ids_name, input_mask_name): + # Try deduce input names from optimized model. - input_ids, segment_ids, input_mask = get_bert_inputs(optimized_model) + input_ids, segment_ids, input_mask = get_bert_inputs(optimized_model, input_ids_name, segment_ids_name, + input_mask_name) # Use random mask length for accuracy test. It might introduce slight inflation in latency reported in this script. all_inputs = generate_test_data(batch_size, @@ -161,6 +163,10 @@ def parse_arguments(): parser.add_argument('--verbose', required=False, action='store_true', help="print verbose information") parser.set_defaults(verbose=False) + parser.add_argument('--input_ids', required=False, type=str, default=None, help="input name for input ids") + parser.add_argument('--segment_ids', required=False, type=str, default=None, help="input name for segment ids") + parser.add_argument('--input_mask', required=False, type=str, default=None, help="input name for attention mask") + args = parser.parse_args() return args @@ -174,7 +180,8 @@ def main(): path.mkdir(parents=True, exist_ok=True) run_test(args.baseline_model, args.optimized_model, args.output_dir, args.batch_size, args.sequence_length, - args.use_gpu, args.samples, args.seed, args.openmp, args.verbose, args.rtol, args.atol) + args.use_gpu, args.samples, args.seed, args.openmp, args.verbose, args.rtol, args.atol, args.input_ids, + args.segment_ids, args.input_mask) if __name__ == "__main__": diff --git a/onnxruntime/python/tools/bert/notebooks/PyTorch_Bert-Squad_OnnxRuntime_CPU.ipynb b/onnxruntime/python/tools/bert/notebooks/PyTorch_Bert-Squad_OnnxRuntime_CPU.ipynb index aaff0445f9..d7f1bd6fdf 100644 --- a/onnxruntime/python/tools/bert/notebooks/PyTorch_Bert-Squad_OnnxRuntime_CPU.ipynb +++ b/onnxruntime/python/tools/bert/notebooks/PyTorch_Bert-Squad_OnnxRuntime_CPU.ipynb @@ -484,25 +484,25 @@ ] } ], - "source": [ - "import os\n", - "import wget\n", - "\n", - "url_prfix = \"https://raw.githubusercontent.com/microsoft/onnxruntime/master/onnxruntime/python/tools/bert/\"\n", - "script_files = ['bert_perf_test.py', 'bert_test_data.py', 'compare_bert_results.py', 'BertOnnxModel.py', 'BertOnnxModelKeras.py', 'BertOnnxModelTF.py', 'OnnxModel.py', 'bert_model_optimization.py', 'MachineInfo.py']\n", - "\n", - "script_dir = './bert_scripts'\n", - "if not os.path.exists(script_dir):\n", - " os.makedirs(script_dir)\n", - "\n", - "for filename in script_files:\n", - " target_file = os.path.join(script_dir, filename)\n", - " if enable_overwrite and os.path.exists(target_file):\n", - " os.remove(target_file)\n", - " if not os.path.exists(target_file):\n", - " wget.download(url_prfix + filename, target_file)\n", - " print(\"Downloaded\", filename)" - ] + "source": [ + "import os\n", + "import wget\n", + "\n", + "url_prfix = \"https://raw.githubusercontent.com/microsoft/onnxruntime/master/onnxruntime/python/tools/bert/\"\n", + "script_files = ['bert_perf_test.py', 'bert_test_data.py', 'compare_bert_results.py', 'BertOnnxModel.py', 'BertOnnxModelKeras.py', 'BertOnnxModelTF.py', 'Gpt2OnnxModel.py', 'OnnxModel.py', 'bert_model_optimization.py', 'MachineInfo.py']\n", + "\n", + "script_dir = './bert_scripts'\n", + "if not os.path.exists(script_dir):\n", + " os.makedirs(script_dir)\n", + "\n", + "for filename in script_files:\n", + " target_file = os.path.join(script_dir, filename)\n", + " if enable_overwrite and os.path.exists(target_file):\n", + " os.remove(target_file)\n", + " if not os.path.exists(target_file):\n", + " wget.download(url_prfix + filename, target_file)\n", + " print(\"Downloaded\", filename)" + ] }, { "cell_type": "markdown", diff --git a/onnxruntime/python/tools/bert/notebooks/PyTorch_Bert-Squad_OnnxRuntime_GPU.ipynb b/onnxruntime/python/tools/bert/notebooks/PyTorch_Bert-Squad_OnnxRuntime_GPU.ipynb index d9ac643786..a0a0e32e1f 100644 --- a/onnxruntime/python/tools/bert/notebooks/PyTorch_Bert-Squad_OnnxRuntime_GPU.ipynb +++ b/onnxruntime/python/tools/bert/notebooks/PyTorch_Bert-Squad_OnnxRuntime_GPU.ipynb @@ -21,7 +21,7 @@ "source": [ "In this tutorial, you'll be introduced to how to load a Bert model from PyTorch, convert it to ONNX, and inference it for high performance using ONNX Runtime and NVIDIA GPU. In the following sections, we are going to use the Bert model trained with Stanford Question Answering Dataset (SQuAD) dataset as an example. Bert SQuAD model is used in question answering scenarios, where the answer to every question is a segment of text, or span, from the corresponding reading passage, or the question might be unanswerable.\n", "\n", - "This notebook is for CPU inference. For GPU inference, please look at another notebook [Inference PyTorch Bert Model with ONNX Runtime on CPU](PyTorch_Bert-Squad_OnnxRuntime_CPU.ipynb)." + "This notebook is for GPU inference. For CPU inference, please look at another notebook [Inference PyTorch Bert Model with ONNX Runtime on CPU](PyTorch_Bert-Squad_OnnxRuntime_CPU.ipynb)." ] }, { @@ -603,25 +603,25 @@ ] } ], - "source": [ - "import os\n", - "import wget\n", - "\n", - "url_prfix = \"https://raw.githubusercontent.com/microsoft/onnxruntime/master/onnxruntime/python/tools/bert/\"\n", - "script_files = ['bert_perf_test.py', 'bert_test_data.py', 'compare_bert_results.py', 'BertOnnxModel.py', 'BertOnnxModelKeras.py', 'BertOnnxModelTF.py', 'OnnxModel.py', 'bert_model_optimization.py', 'MachineInfo.py']\n", - "\n", - "script_dir = './bert_scripts'\n", - "if not os.path.exists(script_dir):\n", - " os.makedirs(script_dir)\n", - "\n", - "for filename in script_files:\n", - " target_file = os.path.join(script_dir, filename)\n", - " if enable_overwrite and os.path.exists(target_file):\n", - " os.remove(target_file)\n", - " if not os.path.exists(target_file):\n", - " wget.download(url_prfix + filename, target_file)\n", - " print(\"Downloaded\", filename)" - ] + "source": [ + "import os\n", + "import wget\n", + "\n", + "url_prfix = \"https://raw.githubusercontent.com/microsoft/onnxruntime/master/onnxruntime/python/tools/bert/\"\n", + "script_files = ['bert_perf_test.py', 'bert_test_data.py', 'compare_bert_results.py', 'BertOnnxModel.py', 'BertOnnxModelKeras.py', 'BertOnnxModelTF.py', 'Gpt2OnnxModel.py', 'OnnxModel.py', 'bert_model_optimization.py', 'MachineInfo.py']\n", + "\n", + "script_dir = './bert_scripts'\n", + "if not os.path.exists(script_dir):\n", + " os.makedirs(script_dir)\n", + "\n", + "for filename in script_files:\n", + " target_file = os.path.join(script_dir, filename)\n", + " if enable_overwrite and os.path.exists(target_file):\n", + " os.remove(target_file)\n", + " if not os.path.exists(target_file):\n", + " wget.download(url_prfix + filename, target_file)\n", + " print(\"Downloaded\", filename)" + ] }, { "cell_type": "markdown", diff --git a/onnxruntime/python/tools/bert/notebooks/Tensorflow_Keras_Bert-Squad_OnnxRuntime_CPU.ipynb b/onnxruntime/python/tools/bert/notebooks/Tensorflow_Keras_Bert-Squad_OnnxRuntime_CPU.ipynb index d1b0773cf7..3d084324f1 100644 --- a/onnxruntime/python/tools/bert/notebooks/Tensorflow_Keras_Bert-Squad_OnnxRuntime_CPU.ipynb +++ b/onnxruntime/python/tools/bert/notebooks/Tensorflow_Keras_Bert-Squad_OnnxRuntime_CPU.ipynb @@ -100,30 +100,30 @@ ] } ], - "source": [ - "import os\n", - "import wget\n", - "\n", - "cache_dir = \"./squad\"\n", - "output_dir = \"./output\"\n", - "script_dir = './bert_scripts'\n", - "\n", - "for directory in [cache_dir, output_dir, script_dir]:\n", - " if not os.path.exists(directory):\n", - " os.makedirs(directory)\n", - "\n", - "# Download scripts for BERT optimization.\n", - "url_prfix = \"https://raw.githubusercontent.com/microsoft/onnxruntime/master/onnxruntime/python/tools/bert/\"\n", - "script_files = ['bert_perf_test.py', 'bert_test_data.py', 'compare_bert_results.py', 'BertOnnxModel.py', 'BertOnnxModelKeras.py', 'BertOnnxModelTF.py', 'OnnxModel.py', 'bert_model_optimization.py']\n", - "\n", - "for filename in script_files:\n", - " target_file = os.path.join(script_dir, filename)\n", - " if enable_overwrite and os.path.exists(target_file):\n", - " os.remove(target_file)\n", - " if not os.path.exists(target_file):\n", - " wget.download(url_prfix + filename, target_file)\n", - " print(\"Downloaded\", filename)" - ] + "source": [ + "import os\n", + "import wget\n", + "\n", + "cache_dir = \"./squad\"\n", + "output_dir = \"./output\"\n", + "script_dir = './bert_scripts'\n", + "\n", + "for directory in [cache_dir, output_dir, script_dir]:\n", + " if not os.path.exists(directory):\n", + " os.makedirs(directory)\n", + "\n", + "# Download scripts for BERT optimization.\n", + "url_prfix = \"https://raw.githubusercontent.com/microsoft/onnxruntime/master/onnxruntime/python/tools/bert/\"\n", + "script_files = ['bert_perf_test.py', 'bert_test_data.py', 'compare_bert_results.py', 'BertOnnxModel.py', 'BertOnnxModelKeras.py', 'BertOnnxModelTF.py', 'Gpt2OnnxModel.py', 'OnnxModel.py', 'bert_model_optimization.py']\n", + "\n", + "for filename in script_files:\n", + " target_file = os.path.join(script_dir, filename)\n", + " if enable_overwrite and os.path.exists(target_file):\n", + " os.remove(target_file)\n", + " if not os.path.exists(target_file):\n", + " wget.download(url_prfix + filename, target_file)\n", + " print(\"Downloaded\", filename)" + ] }, { "cell_type": "markdown", diff --git a/onnxruntime/python/tools/bert/test_bert_optimization.py b/onnxruntime/python/tools/bert/test_bert_optimization.py index 6fa0422c32..528acf85e5 100644 --- a/onnxruntime/python/tools/bert/test_bert_optimization.py +++ b/onnxruntime/python/tools/bert/test_bert_optimization.py @@ -29,7 +29,9 @@ BERT_TEST_MODELS = { "bert_keras_0": 'test_data\\bert_mrpc_tensorflow2.1_opset10\\TFBertForSequenceClassification_1.onnx', "bert_keras_squad": - 'test_data\\bert_squad_tensorflow2.1_keras2onnx_opset11\\TFBertForQuestionAnswering.onnx' + 'test_data\\bert_squad_tensorflow2.1_keras2onnx_opset11\\TFBertForQuestionAnswering.onnx', + "gpt2": + 'test_data\\gpt2_pytorch1.4_opset11_no_past\\GPT2Model.onnx' } @@ -202,8 +204,8 @@ class TestBertOptimization(unittest.TestCase): 'Attention': 12, 'LayerNormalization': 0, 'SkipLayerNormalization': 24, - 'BiasGelu': 0, - 'Gelu': 12, + 'BiasGelu': 12, + 'Gelu': 0, 'FastGelu': 0 } self.verify_node_count(bert_model, expected_node_count) @@ -222,6 +224,28 @@ class TestBertOptimization(unittest.TestCase): self.assertTrue(bert_model.is_fully_optimized()) + def test_gpt2(self): + input = BERT_TEST_MODELS['gpt2'] + bert_model = optimize_model(input, + 'gpt2', + gpu_only=False, + num_heads=2, + hidden_size=4, + sequence_length=2, + input_int32=False, + float16=False) + + expected_node_count = { + 'EmbedLayerNormalization': 0, + 'Attention': 12, + 'Gelu': 0, + 'FastGelu': 12, + 'BiasGelu': 0, + 'LayerNormalization': 25, + 'SkipLayerNormalization': 0 + } + self.verify_node_count(bert_model, expected_node_count) + if __name__ == '__main__': unittest.main() diff --git a/onnxruntime/python/tools/bert/test_data/gpt2_pytorch1.4_opset11_no_past/GPT2Model.onnx b/onnxruntime/python/tools/bert/test_data/gpt2_pytorch1.4_opset11_no_past/GPT2Model.onnx new file mode 100644 index 0000000000000000000000000000000000000000..c1b13ef70b3e3f04f129b829fc415e91a5570a80 GIT binary patch literal 157561 zcmb5X>(VUAb)E-chM2(z9z>Y5sVyEy7AX!XXm@uVX<0Lp9HK*EMmR|M_l_aBNx%XI z1Iz{iA{=2|3NOI7;lI8I--zMoeX}xiWp!tD_xz~&c2{QBs;pcq*Wq0&KYagNAAIo- z?|%5>?|k>wd#7(-{_Ow#f4=eZ`!C=4-sfNZw|Nil-@4tNe zMI-2PR;SCW5C8nLci(;c<2UDzzMlU2_?s{9fA+KQefe8o`Nyw*?dst}e)y$%_}G2; zSN9Je^zcy+pUT6>+YkTOYC5Uu`s$ni+vi{Y=qGRA{`~Em>7#Fye_I zU*5kA=TY5Oy3Ma&X->ZL*}FIMM_*5WwXgTbVQ%R)zgDl)t8e_}+b=!E%SRtB@rJQ>TW>2W^jq1jmIx11njIlubGHXEnQ?dkaMMU^{f zbu5=xFL@$PL!F7&FTeiNcW=M_)3@LIyYIex^W(Q)e(&dR)zJ?bUVUR*QK#}%dD~zA z=@-{MmW!S;)*e${2QR-Zs-&xWPc{BnG|sxLTvmO)Wzhg9NdEr! zzVr6Y`3%YW>*Y_?LesaNU%mX*w#?6$k3QJ`ZddvLprQ2k>*dw2{N>xf`N?9tR zK7Vr_KH_`je}4QM;$t`?zrioQDi-m?YJskr{B?Ns^}qVbU%xqzAHB!_U;c%Di_H8R zdG0>^Ctv+f@59TV>fJQRs;{%^>#Tg8|MiDIeRJmP@c$Rlx0=4WrEhtFK2s3CUQ5CM z^?(1-Tqw}&uAb91t&hv=v_Af+ZXf~9uU{jte)QvaZ!V{g-q#<6bZ+VX>DAZ&r|u-Pm^EanTrJfjnd(- zzxUaf8&9bDWg~1T3zy$?W5^5lGUkPg7(c&`liwHPrjrjqdv%%$3tbR&x8yh+vpu+Q z%%yOQ9~|ereQ*@velYWXe$Bj}U$5oC!|OW_J|T<~s}869#xT?wkykgaJFI#>#a(XS zI4|^#RnJJd{>WS7%&O-T49f7jTy%jO&nJB(G3c8?A4z|SRry^v$I7b1)L~T*s!w!k z#QZSd5!#3hx~LyByb0}L=?ZN=tvYDMpI=v@J*-t||2+-PuU~#g3-z;i?>_(bci#Aa ztrY)f``hhuKBIg5`P(1-;LZ5@qxbd4$G`USOP_uJi|>B+n_v0LfBobizxvgWfA!`4 zzkd6!{979x>r;eX;VcZ+nT{RBLATL#BOs7X4UgQRv=B9UweUT#d&G#G`-`$O$*lEv5 zl>W$j_Pt0ZUBotnR@_CrGnAcn(#I<7JLpReVW*wu4*Rmt(NGu3?19Zvo5i%I&B71z zjVUhF`Sk&FYvMK3-1InRhC$+HSeqVwX81Fa%LdOg%nVv?gVleF+LJ6%?&TXeD%8@^U^4$A2mH=)gtE(FU3b=dTftwIBA@_ zkKS|A=ovtRKaWdcweN2kd>IW!KLA&No#)^>NdC>@zEsFBg5~E;b%Tt0w0aO(QJ}v+C4i@oHvTaF${zC zv>x-3tfw>9UF>npM-#RX&oCcpI%MLU=W0i-9X|TQZI!EwB>fyLsddLivu1H-t1Eo- z1`WK!MeluTC7x};_PzHc?9pv8uMHo;RdtaZ^AW}>G2+ZznGv#d&se(O%m(1;b&sEZN6jUG773tb zdG&$zrPJ$;q4ZQa7tZVJSHBXS3w@3eKiJo)fkIR==FA4p1eD%%Zpoks{DFZT1&zPpaQu*qHOQOSA+xCQ$haHW} z%=j0q!^VUhHg^8dH3xNUV_STX&^&Bh=iS(6N}3lj!aqM6NXTb-Emo6^MU7&`^J7MF836Rlu z!RV_Ka6yvaoB%D56C#!QQlAMQ1U{A%mcE@VS~@3$?FFMSGR~#%*mM_tAMC-VOW%;x zJfDyBgeZsqeez(~A`B*<2t_*cRI{tr%UP3jW(&-dqz`M=y3mN}6=_6|@9pEJ@yLLP zH0nE@&n853d!GEKmhV|>^Ngju(KHmtnpsuf_N*^)!5d^|9yad{&ZY~Vp~^C?vnkoM z@0;Oewr!)Fx9^ElAUn-2td;I4pZ@lL6n~1Ai(cX);c&r|o@l{qCQp)YYe8PL&@N+s z$dVo+9+csB!EpQLU;XfJ-~HgTpS?NB!r%T;9NXyGUz4|*mJ)+nXcuhTEXXk3vmg`K zU)U!u7Zzd=lQPcBKW}OA;&7Q*my0Id_V8&e8!!g(r7_;x{XYsgzb`5$?bQi+oA-aL zfSVR^c{B!Gd31E4VgH9u(LOpZCp}uW9qsjKHlXt8jt{LqT3h(`*2RgYwTvIAu7Z1l z9Vbw~@m{dw{OLE|3-&?$f%(Tp+&gFqU_WN|umNwD)uNJZc)>QTa&784p34Ma73K+; zhPbn_2>N8k53kw6FVk<_yptL~p?6(IF{_M2y6K{QS{C9aA0~aYG66C@E@RZYF!1|= zebmAkHKpUcu`m|DSy)e%bK!zX7$x9*s6?^$)TI^db6<(_yNPS3R=dNf@5x!MhfzyG z<{kai8D7g87l!@K=-9uY@kQMxz0`CL+7^a>iHvDkD4ck~P|VAsG3hTXlKQ1vmW+!+ zGdt<4O`4Gj2Ww`@o=smLjXP^&NlcF#7ro%y={cAkKj98k8%7#_E9rS~Pwf1Z55WR4vxVz5T_H zfB3(*@tc?E-uCUF>o7X@=ZAK06O4P*zpYu2SK~r>HO|6%%)pc0h5&2RrO&`xnTuut zV^SV&_l7_6?BP`C&fX%nl5u&Faam_Ca4@ zJ(~F+Ft79epio~|TTG8GDjU(e{AiSr;n2K}o0A*I&15gpcS3q#cI!)Z8fPIryk?6Y zWQNmC7x-{cbTw%#!~&r?Yb@O0F2;2c(uDpx8VllymTKO}8bX10gtWf!Nz;r$oW%^$ zvc8yEMYoLPK}K>GGiSUaMM(chEykBu3NrrS`(F@({M+xeEB)oSntRQf9x$YncqZ8Q zgm`9+il{~_yhw9XsAmL#f+m1cAm3Knn0 zMOO8IRUPIH-`_TgLS+oRa$QZWcw&`)vKg3+q0#i+h@RTOJfHCj%E0s}F;uu8J-F1#_sW>tA$Q1GQYLNXk(%8 zCF5S^LUo;@edE2%#U*pMagn*Q{)a)IPX;qoi8OsF30fdg`cg&a%7`8a+g6#2t$W)_ zl*}FUp|Glhes{*+?FkSo#)CR4>N+Uux~X=AH~X^VBDxiZ9SFl#*)ayy?k8$-e);Vx zS8wRWK_XwP#e)L1R*TPo9v*~!=vAbp4w&7|9P}NQo;JTr<3A+O*PS-8#OOJ&GjPBN z9k9LY-1UtL-b$TKVlXO*cviVSCs0|l$J8?hJ*Q^%vqwbtqof8MkeITA>$AsQT zZw;|I6$!Q$wRqs~x|&-#3zrmWE%S)B8U+DmwRl3W+|*)z)SYoh;`9KC62m|zdNc&J znDIq}cx2kFhcjLlPm{#3tmtWaP)LaJ3Q=GE0eAA)FZQj6tK5ggCcvq~*K-Gt)} zAbgX^#eBBNMJ?t>sl{85R)rw?o4z3yw2705R#bOTR5v*gh}LFdB{|8z>I58%XxZ1a zSErzx;wtIN$@J)|n>KEf;UI#$7txB~(q50o*Son*2(8|Gw8B01WD|}|nTMMFnd=VrU-gkRE+&n&GVhO6Pi&=vLns*TU4*sAr_`MxGCLvd?wWNMmfeUg z8{`84-NP;?B+z(j+@S=SyIp`oK$9*`7LgW8Zw*m(7{$tO3K!rI+qzdTw$G8+aM1x3 zH7_ip85cNs1RNaVdVK1~{%BkAbeF+9V(?a;0_ENClwy4&R&L}eoC+PD+OT^C zz^1tyO2mw?e_mVSs>bR)=Bfdmg!bxGBW48uZ*x`q_~MrU2hcpe@0Zx~m9WO6%-wOQ zehGj9k37EkCWIFVoCQXq&n$b+!cOlu-gDMrfO^kagP2OO$> z;D+vo$k_#FM4z)1m>v~+9tk}s&caDjIBW2o#%D<|zq{r|;~>Uw!^i~p54$LgJz~8^ zW@PSr%qY%M7@4P-vm|(Ay^dI~g|qk^H#w`o_q1FUE*=RNSKg90B`d@pSJR0-8+l6( zltWw!s2BD)F7gD7n3EaxLiD?!UZgE3CORq-I?5unT*L79>|+t~qwW-okob38osC7f zm(KguNo>8pI(I0K5mT|>xLL{~QfqS7GLrE(%I+NJ;=Up%3C(47$}}7?4XfP8u)n=J z>jV_zP*L>2p3fME<1@zL_>6Hl$~a_rU9|pX9FEmEB;*|@K*r%1jYFd0oe9ta5onkB zQXe)x!S@J_C(n$YwMg8ywj)vVF5|e1z7JMo(kRc|yrTb`LMJ_@TLcjd81K*>MI#$&d1HS6jQ)AX0 zsvnFYGT3`2Ou3?0W1-))R}n7h(G?r4$a{EngAHyU9hbbmglxV^Hs6HJS0CN?w{4HE zN>|O8U@cD=Hgy%;5$qFK@_yr;V4tvY`i*yjeG(X%;dOy~p-Q9`_E4O@3U+SjF0Q4l z&=c^bGNfihv4GooiuW^68BTP)4L|3sPusE>81<&^n}H@ z$p?k(!vrYUIT7p(6L63=!BlBPP3d_H*C|kMaiRyx>oBzdX2R`VDXlIt`X{nSgU}@8 ze%M83=tNMo>Cy)YYAFDB)>Cv5uMslM8rOiSe^cWky3B%C54{xN=b~*9*{|A`mW2YS zlZ{3@gNftkWq~m=J(%4pZWK(K_Ucqkd}7>vX^M0VW+XOiLa&|1*}R{W*~;)-l-UYZ8laf>g!v;g z$ReGvNGmgR&3k<)smT*oV_b*lqbvk%+3u)mCRSS3AYOs)8kAdt#_E9rnvo%Z0_qz4 zsanj93;afy68K(mfnUcdC+yGU0tZm=gbSRN3TbVuR_KGCX66;Hw;@U!Ck~fFuZx&a zst#SO^=R5LB+Kl=x;& &dOhLe-uyuIl37>tH2LIgMJ$invZ#sg6?;A}(=Ce&h~8)(GDl z;H&$gRN|CVlM>s~5aN`K4-K+eCq-O6O&~IIoRS}PNBORbQw}7Oq^cpF1d*>W>!dKN zY(@bYY%_Y$p~NY#;CmC{SF4LeKT4dEA4{Ba2j81Wk==+xy%f?!LR#U~iSX*?jRdO@ zg*4&T`Vm@58Fy}z>NnAckcqqn3$YLSlG2YR*jgT4f`!^28tiiW=)KBDY*y{nsj6(q zlzWdZDjQ&i{MHhshvs#BZ#RycNk{G$A>9ODJB^Ewo~5$M44-E4x`JvG)^^rdTr48n z;@r^PsIrlFe%lt8tP${eBGWa>ngdKkk}#r2Ri7pd;v!}!8_^Xr0>9H(J%U>Xu~Nm% zkJREK_}=LNz8CHVQ&i(pzQQmP5D(9YXO;4rT3Z{;Kv&j!%KRR!N(PwXNIc6>o|2(# zDM~!G%p2(o-FZ`D@3e?Stm-PB@xB!CEb&B;vvOxEo>--yY-TEBFq&}86Yt+P zV|O~hD1!&dfRJqzNh)EK1qcCJQQ&*iL4l_L-=nW+(Y_(LvW#7j62;iv9wLz>Y_cV3 z237clqd00`IKcPltA}WX6T00VvWZA=BmDTS_hu~i1m7F*mlXKkd}Om%_#VAq7=kIC zWwR$bnNkT=Np<15xue_I?CC8xdjOfskGfN}(~6Qn@XbvkZ!7qs>@fw*ChgTJXr~9s zCEKH`xUb12esrAQ_R%|;JIg|!Wuec(LLb0J^epuB=xU)4I612OZ*DB~y=08YTg=X2h}B#%tidu zM;(LgkRzl6{bicfLra{%Y?U2wd66AIp#>go;CorF-q4GKM7~ywXCk+o+yxTxI5LEF zD#9ArI4F#+5Y@EFdY-Yx<5q=>`YwN z(i-?jQH%LecP={;A1U>R7@n~KdizdLiy2=uh(`osmXH)?B&)^zs5>f}Behrvk)jrF zc&H=tQPYl&nC9J z>85}{vaphzM7ysO(8|dus`L$HU(;Tls%|`3PUEZ=&x#GtGGOVZi}+Aw zsFI@}JV+GL3a`%C)WvrpXh~m03-&x?5(ilWtfVVzl8Dw$DD`f}NY3irhzq4FX5@IG zv3i8$IdCLZME_JRCiyPT$MC)OcDu64B~C&bbjmYAS{|Ia4f+BIIhX~$g5IXh9R@Vg z?V?Ypv|+l~cOi)x;=Ka4bZ1W%KaJHR7!`uIlnt%`*i2Q?cEG6Q^(-!a+_;@5KhsoJ z;}#6bZEdG{d^K>#rYh(BxPd#%z@{siHMbeIz*G*qERsOfXNn@_0CWKy0-AJjQm&aqdA2jgY7s8T%bV;`;Cq@+ zA&O^W!$pq)Yf@Q63t9-`83b{N^MOpc6X)ANY$mCMe893q&T3L`+f6F&6I#^*qB`5&FE;=FOltGvd*rGGATyY5x@ajF#_V7?NSVl0vWFM!;krk5 zi*>+NwA2M*w8-|A%W_m^NaiXz8WxP(xZZ(<9C4Mf*bDLKrpqB$@o05voG~Nvy!RBi z3Yv5=C{_+zdTWRZiUq%Oo2xeX9#cwl^$Czz@IDPsag!EUxsj)k^qtuG9=-=xEvEvC z1&x!}7W>uGSiSASjw?&=c0yRuUY%;h3|!CcgwYH4EA?c;ZRwFrMq zdp*ATB|yGB^7y)2lW0;M1Pj^9{q-N;AI@3`(DfVdw9n-*K)nkiFDICa;kgJC230)G zFy_Qr_NQd0EH`vFM9zQ!_4g}UYFNb4qC8u3mXIJDXYna@rvb(S#p$kjF98^TX~W0> zcMiKKjJ;sJMrJ&yz9G(17@4O?lT(#XdNTypYuT@GoAmapRB}?hrI_tPxVY|D@}>m% z9yisZfyRzHW#I!(oGjl7*(h?+X%1_G?&#W({RBw zEOH-oP-k`42`I)PD4}Koj`p&v2{_{A=AA_01H7Eo$wkKDBIA&5x=6-X;}9f9oB$bz zOEeAv|8yom3q(O`=1YCp_~e-ZCzL!hx}$f5S7&Z3y6{OiDbN3>rW@QvyM*f~S z4>EjjK^>@Ex*Mg!_ZF4&r7g7Nyis69C87a=+!iQ!5hyr4gjfWB=$Hq7)E(^*8+;E0 z5lgTG-=igcbp1_SwN9vweL`{=mj{b+p-*?dbV{0kc6&ZBG5 z->K$IXe~ce^mjQd`nwnGj}-k)+COh3FfzmIBKA;Bb*V*v1uF`IUH0OFy;uc1h=3e`SL`Gsh3_*@_=(-*_+B8!NHXxCnL?ds<{HE(&Kt6|acJnB_y0 zHo*3faTJh^n@wWuXA-QgdIY z#>8JA&wXW#m-lG@EiTo>2UgT)Nj-Y4d9Of)f;3nkFYLq^dNeb;hP+oWL-!tCWng38 zug~PXuFvGWt_?W)4l}IEd1ZKA5O+|e7c4oi0FHt*;0AYN&MSDM4l}S0X{qK-gbr-d z!D`o*EqY|0tf8Jf6VK_k^Q_*1nQ8yEPEFeUK4;sU>pQ?8}F6Bwr_ zT;Qx!k=Dd&rvv|jCQjv@*0z)~P^j+2A#-?uF3x)r7-|jWohBR$H*F?3n7860<8qDV zodlUG_7ZZ;5PRWnDGc-ZKiG-Ag;*|6j4SHu`o~_v6U&z|iicV6u zjh*&X%Z$)hqxjtk^a)i0*r~)RR|Q#Df~*m~2h`~S5{Xl;O?qrgLx@u{zGx7aDB_x? zX$$897vV?U(XlUzQ%)q3lsIJ~UuNm5Fsp1v08+_jWS8Q{5~sX^@2!Milj@3!k;{zm zV~JDl;CpK>6a`$W#3}hv&#vMMrJ}lQR`8VdBeZ%`+__DvK7@{uS!E(`u22wCT}in; zx~gpW72>0#CeS{5r?Rc82D{36*Cv6s(>QA^t12(8GQ;Vni}(<0J8LX1C6R4$Zs=}Q*+50yR@pqP zJ`~lLqq1;-iEx}!<_>-6h(TNwg#$PQ(@YN;$*YXyj1mS}RmIGY)Z!%g-uVE&7w!d9 zRO3=a&G0=yS7X zjIsnR5Uv8>TMw#21^6C)p+#`zRdD4*%Vv19;cO3)ND?+#k`zEy5>|TQgR)NnzDHj@ zL@QinpZpN~D1KqdW0iecRrbl%_XO`M@V)iOX0Px)dTSg_sciN{CsWEFg6}cUTx7Ga zve^>>n>`@uBb$A#iM;cv5qwP|Z!7qs?C~Qso3hr|P}&LXR$uU?M;CDqiYp5~EcErj zLf^|=S?IJ^rz-Aeun}FEn;u=nJs7L}Xo=BhvUk^KvUk^KvUgWSd>LLB@x1&1Yt7yT zvjxpX=E{g(WA-iz(}B!YX{WV?41?3^3XmjoW%?)(yN1$Epqu)#Lz$^GR*xt<@P`!D z?k8%o2)>u)>J7a(NaP!8aVhKs66_hdD-Y4y7Ip&Wbub57U+U8M4+S7QlDjJGL>D;& z&4miiZZ$SQjrHeF?txXo#5ERn0%KO^PV8;XRT$(|7-VxlPzrPZztI5tT-ef4=LA3Y zfX=DGDY&fO8e(%Qf~u-&@nL3hg75vV2GBYtF8luJwNi^Mu9Z^4A20v*#lQMqTO$yN z*#lJmp?k&GvO zXbSc9B)mBgvV?IsoSplN?-Q2K`O4tRI+8ripg$hAbI_k>6sew2G86~N%(F13b**`J zK+$eXq<^XAIBDG6okFS|*tIH@{=sbqxjHV#XutOMZs}u(CuMrZU-KX)J59%Hlh>dr zX1q5C)4`;h@7sLi`2gsFmxFmr)8UX0SSo%AVPa5r2Q!JMz|Gw$m?MJmm96m)a{V&% zd$o%}(B;)`c6-n;_Uup(R}Y^2;GitwO1Y^l?P#bn2hA|boF==sl{r@|kBKZAyljJ! zEL`3LG*nAj*u0GfbLPXQa%%mkt(;m%q;f|_*jP5#V=eQBn9I6a0$?!f?_x0n;)8Z; zZ{~VTX59}q^Ng-|+h+D&Mr>FeyfjGbO`DONDGKqYPwjhc*6xf;<&Y65>BX>Z-Y$T@ zJDn`DNY{-L=m5;|iAX2g*H5xyWP(ZI)h|Im!V7cLZTKgJGdzZOWFpVSss|%c*~q*K zKv`oqWk7)I-H9XqUaEDLCbGnoK=|c#XZ~xFd)u`i_nP#MmKmh@ikgbPoqN?lA_Z8? zTU1N1z{q`#+1yfa#eD)PbFSt|6bPlfxu|AbG&i}gp`hSzg$<0ncNis)vtVvQ99L&S zhnwn-+gRG2)|K}-9!YOzIY+!&e|vp6E!aCQpudLqu)){WJmNhpZgsE#lg%wf za8X7|;XOPYw7{$m906wUn2mset5d^9VEm3dVNbxd6{Rm2cFzZ@_cY-3`T=^{xw!H131-sVL5O43FWRgur&?z*sfJE8eCz z+Iz&%0Bh`V;C9gOA8L5AcDRyX|Ne=86p+rMWRdNxUuh%U^Hm?3=Gzxh0StW+*^K<^ zdqsf+p%@o1@BYLBX1Iga5u2jr&VL83BalPuij}G>xwG!+U$j6^RM6T?HlT=c0dvRq zLtS-Xi=71=Cc~+?Nuq#@47;gLVSb#OPl$qk>-c(UG9KXlvYVc4zw!i6hIks<{IbP< z1(w)Oh8K$}Ef;Rad-1`|L}ITUg^U?iB9SwsQ}k6hNk?DBWBlrS?QVKdh0~qL8dleK zH&sGNfuywM1U)%$dw(jUF#nIqY4h2;cb~^|#Z0Y4JP$T=d*Uc&rC&6SX;;9xq25y<9f^?lz6R>UdjJU&lN!j+Khx^r{#{fKU>lu-N zf{GSuG?*DDIK6U5OF&$3i2#2j+oz{c_85%leFCbsS93f-CAv5#<)TkE<>D5}h>TT~F>)!)_xFmSV2si`DpKifwq4BEO$bdyQU z(L}Eqma~L(-Qt*_M-$Jd1!p1KV!J_=c76W zyi^1Shx9an*^;PoF}6^#Z#p{Uc14)DYkI$sV-13|+FD*R#F_Pw;;8^$+AuKOU)Olc zz;VGOBK@`QObky>4=qXLZReFBP~0x)SN zzzh4~MB&u>NFBIhnkHi1VX?|I%sONX>u)cMnlCI^YOs?|wxmxnV;E##M}nlm z{qfquSfEIcSOUGk%hd+dDU`+%P_@5A?P#I7zE!M_1&**T(SOiO0{f1Ufg8bIK#(Oe za24&<2)RlY$kAsa1FT|`*PxOGg6*kUX%5pnHTIpru)R)c_WV6ltt|T+PN7(q(6RlQ z1QD-yG^w#(7Rc8+la5CE86+FioFKMCzu;X{obcaj5F5xR4CEO6T``94VR9FaWsW61 zr!$*weI1TlEg__h}mkB7k zBFNWFfMmDMTCOv&m&S3oORbM|#l56IHv#Vb*n9Qp{8}=B^PGM48aat;Ig&Z6M__BC zr00(B2VD^??SZ7{#AbJBrPKj1g)h!w8HplKdpJ)C1a4nGG3|a0F$rfXSGSYkxg~P> zrQ57RQeF;_l;K_*{pIlGE&2<9_Zb<_`W0bqFdR&7S6rv1@3(1P1z>o)o%}t?Z`PC2 zOqv@BS!aa~@bAGKl5DN6iPWIbfwHZ0D6w|N30N0#E7n-Tp9obcv2-SB*{$7YGl_xg zRx@2^9QQ{`g%RUO7H~q4TU4vih|oHw>7Zy=Mpt?yxfyVSNPjyEP%h}4mt7B>fSm0*K)G3PS#&qq*RAA? zV#{xAuLLJ1>uI4R7kJ$R$+;0w_l}wtln98E-V$&ny6tB;+s|+gg}FeT-Y1~$9XM}- z`%`Bj*R{GAuW=46x>+#&)>oqY;RI~2UPsx_q+n})&Gz~0N+M*+YP|U$knnH zQuu?L?oWS0(xw@_g?|GJ?h2nIa8g&V_fV({EbxKYp;DKgWJZX!x&$;XqWhH>KDl=!v_?A%uC0-}608GIPOQCqJINOQMSng(A}#V!Ya&I*Qa zz)Wz=2lHr02hH`}V%08y&vhOp4A8(en?THU1!Hb;0nEkXLOMFX`d)kXu&~dM>FD(L zC}{>ZuPP|js-vF|^T!iN(}K$;9IZ_cqRoOIBy@pka$K?RwY4CHf4)zvg)D2KGHNT# zqmItm$9g9Rw$J%EVt~(S)-25J6gcA)*mQJ!M=6adyonK6oK(E&=<*dQ80B3X+Z&wb za2uz22EU#*5Y;PTC*hD~(KAR#-}>y-6~gc7+u}O%x3!>mO#SQwryS)037y59chQOk`VhYYiN0)zyPVUO4vI!^x5gi5dkG)n?FYhea|G%8G zo#+-c%%6Mcv-~l`F;cU=y<7aa2EO66Cq{o5e{Zqnxbhi?z^_(EDueVP`0i!b)ETsU z+p8Zg`!m01niBLkCRJ|IGv0Hf%D7RMe`kiHg4TiOip>!KQ*~ zt5kGKQS7L|eO#z0Ih~RKI{UZJrBMo|*3Ve!mxrx%?a~}vtgEebdirm19Y+8|ym4c~ zW@izgErDBZFueNZECOiKD^BqT?&0Zl2%S@XUwtU|5kTTC%Qnrm_VNq%c(c55@AN#5 zsh1Y#1%tfpb<9eTpk5?ux4rtT^r>G$>M+kTD=A9A2p)$T6-7Sb4!Y^&{L{iE?kFYd zg5@1V6yTYnjf!=-?MCEw*O02v*LPy7zn_3==0=(3c?DKKk@=v>?e6cwo@K+fFJyR& zDkOj5gI+b5h6c9nOhc^*lW2&>c}v3;FOuM`r~9=q&;tz}UW_!5`CVyfIq?uraK(wZ zqxyWi--~r}@7>nki_M9gki{F4Jg%$f-7bO)hJE40S|i4sXzQL6eIqA!%fy%yIsaJ0 zTclyhZ=9&KBx#B1Yx^dKdcz$cdk#5~40{F!BGC4_!-;knT!{U*z3yUiY>)rz^L5Vuq)X*}~Ho}fqt|Ljq5#f!$U zUu?l&$o>vgba-(`MbHs&5d{~>g@0Bm?rFvsJV^c4SKJ81=WgrDjidR{A^wZufY7ED zuKe-5FWP}*sDrtYY;iSMA~%Bcx$OioHySl?F)HAKA-{2>Fg=AEN%m(@jP!;>0QDSl zBX#W5YsXhWeGYpWXTbTuyJy|uMu+%$mbuZ|1XbPibX>e;U&LK>lN;F=H!jh_b{w5E zg(E-8jg$*WdSA4yegU`M?2B?8m3{Ko*`|M_2k1Nr_{TdLHu5(&I1?~U?70ou=`Z6j zH7ED#MKE)>FQeip`ZB+#QEX;jgdNW;RcozUZEFnOyNel^OMiLpVLO+@Tc@|4-D({}r_ zx6MaoZkxS4K{H`flQVzyrB$Ch&j%xJCv$pSTExkcVHA%g z%4M zhX$Li^l(4Kc{WVQQo5}UMG~}T?Vw~{8V}{r;LuwV;@b0PGE$r#CV?HVlF)HcMiQ!0 z2n~4_%m5Ep%1*7LA+_i2{obL|Z0ii~TkrP{j@|WV)2=~J|Lql`I%gYOfz``A?*_Ju z2cpr{K>#+gBP+vvC?h+6k~Yytr`h&tj%u>!;}Fjt-!Qd)Mll=|`er69IUIk6Hhp{y z<1!yL(}P>>Pyafx+Zh1Q72|A|Ne1NnVblZ3HVJ*sfqry(}`Y zhod+qmUP_9X7@leyY1CSnOaOZ5m!T{|s1$CcyxFrZ%`baFiWLl#PVy>A+8v+1Wzv!94Lq zg2XWA+lmAs^@~L56~GO%Ac%hEAiNJj`x>4~6W0N@b2fhyKW6x-mOYa2PJ>*(+-TE`!-k((t8KBcF3&kjW5xK1xms^uewbC}K}-gjcbU~77iHC@6u%ETzsDLI+qz%l! zMul?rY~I=Mq*%F4^Rt$a^BxvTEvZ5&`Y@l{wzx`6F|kC=i}VAD6Wm(^_I;ISclH1m ziJ^ywj9R6t#N2YSSpaV&1g#QxOG85ZFf9pJS|w3;y^pm@JPdUYkRRn92|=r@sZ^HJ zhdze57pB%nTIH1%)kM|r?g388OQ8FKT5oz;GqGaqS*vutm*SBCx2X?;e<5g5s+N1x z0{8qGm`^OZ6LjI)~4S1iLCYIX032GxQ|-HVZ1qp0#>bSJ3Lm##OPG zx^W&P(RFTYowI%Gy+qf^v3FY+t)3%p{eu3o_8CB%pQ%rn+;5~@M6G^w&9{;0eU<3S z;OI+q+GF!TR=S06mAjZbFaTkR(5(B=fV!xwJ|$ z=^zg%$(B(Rb=NSS2HXqDe|mk9H?; z)a#zyN9vJU$YTxnXo)&Kk1#Fi+hlWytdgqJkM=8!hT&VMKjVl>Aks83ejD3-BhG}R z)00?Ib$Wi5PCwp8k#gAUCeFzbsX9GBOQ+vDwK@&wzvhpOAFAl|{H%{mOh3y$Qb)4v zv-l;#UuWgc4q!jSXg31q!&dRDRFl5=-AJek^|^H|jR&#&s~y0xdwTe-LhSy07pU$4h6Qkntk$5z=Nv?ZqPq69FKy!jY zt&>$B=6~Dt^w0uO;6H1CB)Qz?|2-K&47GKKb;4?oj4K&nyPh55D4X*!yqH!6ql zIf~W&*um|5B88?xXZX&(*&?RTIED^H1$5gg!Gv%40!+?fFXz&Xq`SAh`eANCN;&z} zSLO?DvVq_X_)!_~MMU97y2(@jsxpd3F<73XRHK-pN>33}W?$#VkIIcNSQ#WPPNie) z%pbgb{lC1;90~Bf?8w!>uHIo0B^E#G7;jK=@7uM-<2og@|EQM9=^BBrpDy&w641{C%1_n ztuA`{DlMFIWb3Y6Yxam6x#b2C;7tzB0XK5!touo>P1JkexKU|T8OX*r_RZbKjbu6G zE87_y`P%XouA8G?kf3K_xi{QME>*9WxN$Q6foGW;JyYI-H~=71=y7tJ}?~fCQsflo@CKJi-2?^A3#+?*A zB@;GGC1cMuR~(7)(r1fhQnF9PmQId;PmXy{vVE<(C~c=;OZi#E2HTgUn`SjPw(s55 zjN;s1&3k;1&PDCFj%(lR*p@YCt?K1%;&M)_iLZXY3*L~pnr#DfFlw;+$o%Q7X1QXw z`3P1pc{t5{JY@yz7}Uq!(0e(t*SoDNciRf)2_oB8Fo&{gK7tiYYELsCwt_hzHO>bj z<$$}5OJ@c>F#9c$=r^F7H~*FS)V1iQouNz;&oQ6cyq*Z9$DNa;pKdSD5ZZBnhN6%+ zwsbRtgZ=Bdvgnud4_(-A;DU3L^3yl(0qH+^-qI_2p<#eF#NtlriK0Yobf)cDClV_& zhR2&AZ;#3qhd3{F!z4Kyv=Uc~*;oMEc|ZewB1Nqo;~>4K2Ay=)XPvgai!qho@K82h z^H9qE_SeRhQoNNq^0PZsz?M!fflg~vh1s-Gg|||nu=F(RfQG(Lg1&E0BT~d)P73E| z_t$omv85NoaeCNN4g}R?+7ZgxpFudPV$24-mxaPj4~?1_EEIBm?nq!u37eVIT($BG z?u@94GW0$X0T zPN-IQpM#QV9v(hLn@ zpKKXV4q8txLpNa`7a74ORHezrHwS<2di8Ov788Q`^V+g)>C3Bd4rsN~E-Fu)94PI^%p~PG{!{ znrWxALz_pb-VRc>VE-afIDFI%qPOGH^>&CoX6yJI8QLyc9I!pTgI?$j(kJNUr-dR6 z%H`@VR#u>U$$a8e(P+G|nHA&Z=Wet%F^C&sXt1Rv25}<}%~7N|20`A^GcgF(vk2S; z<}U{x7fTCenxr;0=7xlViPTzoemnUytP*n zYaE(mOf{4v?6XcfcO06Gtr~~sXOT-xy@A4#&-DPgibLy|&>k%@4$UJ>iy2Gdb;EKm z?3@K#%JHEdP-VUyhrX~=);M&28?P!w)su~^%q%$+El5da&vEE$Z0St2eN%36r4(!_ zKbJW49$U)!poLQO;fyWiXHy!a9|doMRym`>)>&xP^G&K21L$UOG5lGEeU(Aa&}$q*&YT?0gWW!NV(9(A^aF!CaR9&Y7o|q<`AmiJTYdKj8H4OMfTwb4aMD#jg-2GC;a4 z>WI2kroIxppJZ$)!)V%#+ULm5P38 zXidFUbX6$xtuN8jECg4@(4I{-Kb5bqBQ`6;+?D8PTX$O*pXzK!2v5+Lwa)XUAM37Al2j6FW3{?=xP1G|%`TTknN8{(F zQr&>NsHmYobEC7crCDmg-Guag!Rh;$(|2|123%0R zX|z;#(gmk$TB<7=HY*&Tb$asn4kpw7656|0G?7TjtCMNrko0Y1?l}%K3q67dmLH!m z-~7xtoP^mb$)w#qpd{-V4B)G#VWEnB(&-0POP2psf>noR*45vNapcRVd2^w=htgW- z7|pyRC`U^>LLyg!j^i@-WOJpQY1Jx3Q%hf|as_|eh1(j zO_Sh{gI^+$&kcY8`_g`?2>r{bFDMCur(5 z9Y2RQQHaMOdqxM8h~U@ZmKy-VDA>|LxmewNvBeelIK{FRc;Nebq6>k?4j^lkUVd6g zawT`FFSPjl?q!ZqWq_*F%l8zqr3Tal<>*>C!S7RdKgDTTm(UfM^ zOHjX1UxH1)JYmx>w(04i1<=^^_=;|-Upq3QUEkJ9UYa!gRt#Oq0NeHSFd0z{^@=6$ z$q1sTR!lapK4shMWXvkKI-Ff>`=x=BSXpU5Jr*craFrg@i3PgYEKg-bVDZ1FC;TzVx89taqo zVnIg!d~`K{zV7xCnhn}~Wn+MwygfLhegP3scnZ+zZJw&wQVTZ&=Q*`9h$$r=7QQn3 z8YjPilaG;Gk_zucZZo3N>lRZ!@5q1v8jfso>|0idJCWboNb|*c_MC57w?kt0w>h#U zqGG*q+P6eRv!x~CiXUx=at5Xria_$&FjVH^O|j4tQIX!bIWiIE#^yM?J;%hkNUbCM zZr^&_XDc^0CV!`O<;KZ;X!Cz@0yGD0>U9m;PdR@`a)>X{7H+vg%;HuO;{i9?=3i{{ zFVv_{+^FQlf@I?xd(d{{CcUuBA!(I(%La|m@D)1GAvYQru>dk?dZGLDdYK!If8gXx zbE7;4j3An+z*o5uSI^BIE@Mm0i#GEY%>1e{klMK8MdRrgZ2N{6@&9yi-dEUCPbf#) z6>h{a6!*u*+HV;lu0YQ-XW!{fy*HDLUl~CafNnXQE4DO|Q5`kpAhu-0t)?a8iXRQO z=fFx&dvMf@xQ`wpqiPEzGG;Q&V{DiG9WTyDYW;xbTRHL&op7ZmFV=&MBjCtN&AVL_ z97tLAM=};0j&!R1qE!2$CoB2!fFlh};6O^vB!>RRk$8_PN4k8%vO^GzWLx#PB&`%n zj~#-5l*Qv-hB9y_WXq9gaJ^@lBaI+fgdkvllpJFOUWkJt14s7C{#0zK);4|IGvd9; zcx-u+lO)}!#e0+A*W<~6EoE0V3#G?|(tJul%m(|k1_Pg%h_-LOCQp1V1eQFCA`n4X>w z?IN6lSmy)RNZ4<=TqGnB~+>~dV{ zmTPl@+kQ1hA#ZHyW(EiQ*E6Oaca`%GUQcuAc?0`i))57S zw|NW1Vt}^9_63-UZA#4@5o>|t6X~gGAmWrpPBCrqIcEy$#fe}S;%X@ze(+~POKImPdML8Ox37&WrFbiKJYRRHfGy?lKMiHeXf0{rN{=*E zCft!sZi;Y0-_zQ{QBts_{Op_Bjxx5Cj6xP{4_mq#fzFY;?uUMn>z0ysScIc2l-0oW zbyvw!5nf!jU+Lj~Ar+hq$lIz& zAR&O@Nq=leXh85S36W7uf^A%m0~86dhWkil=B73jNeFsj%{ejyJXOG5M*!B9wP%;B zz2lJr?$(xm2p0TX4qyO=|I5# z8^(3v**nD1$K5ftKEkbD?Hw+wvDz|T9khlv%?I{z5i@Pp5x3l+(F_l>j+i@5w@-KD zguu;PPiAEj0cq`9^6o(U;~gIeOVp&adxx?tdwYkUW#gOE0UIH%Y|CCRiwOtMIg8M* zVjd+vUaT~wU*5FRge^t5xR>7%)PJ?nNUE&*px2J$<(%?YBnnxS4Ld)P-_9_4uJhYD z#jn~}q(UCB^F~7TcTBCnV|^mGonrt^f~*2g@{1&N?5j8nRjy90w_|-OuYEG#2Sa@3_uU%)vhDrjZ7k~w8o)1(eGRDa|ryd%<6>p{!ragy7slq5Qo+y zUi2Uo7p^$xjawo|w8WwHvv2C8U&o;*QnMc|s@35uZ!%ebLiaxA}Inqx*n}{I&upp|RjRx@xwJKfI9uNYw=s#ZyIY$gFs zhoO#q3~?S(>m#jlw+AqG471@E&Z4OnQ2uoxTVNr`HSV&jh5&fRETx{auNiSIrOO zrYHUB>rbaj`II?qG(W7IBK?8faJw(+Sj?Yn@E;8$qJ-M#il{S`aZ-V0VPDcv5vQ`3 z;00S+hO*2PB0ouRmCM<%r8J3b(+(A|rL^;lDpQ`n5o?L6jL2boD^Ig<6A(X1zKp<0 zgh#>FJ}qEN8OlOU0C@gp>)D>R$z5Pa`dK$s(k@|3CtOPJ>(4YrNA8P?E!6-TOzsP= zWn;lDWO1F|3xA|;x=AjgIuVca2eCMn`+_DxrLLG7`~=*J)7*;FD9Jp@@@%Mw`$d)d zAYF_0iG;AJ)O4*xLY(`Sge!?IR|P|Q{^N;r6HuD|rjkeqt_o6y8;RbyD)v%0&VwcT zGx_>d1C2bfAA6_O)7qb1`TFYVtCv8?*C(a8k?3+H&^YGnlPG*7(Un$DkDVcUT*d|w zvgYiJbxqZ(vF}UE$s*1ANiHLp*NltP+ZTVs}nWfXuH?gJM z&2Ds(ii1v1t}xirtw zrF5JJ?*I?i6-ccg@B(EN3zS&vll9Kucy~mbJ|P~5>={+ga*e9#mKy-VD4t@{jgNK! zVg2dXiY{`GyP_+JEAPJY6VNv#xsrl>AhhJ7QhWD`Nv^YePXSv>qe*_9`50T;-rjKm zu7yprpE0E#_v*GB@aenpk9v~@vigA82s}H7A z-Ve2RuV^BXFI}gzqA3%BRsAAiIj?;2KW9Kb%>SHB=VvF1gH1oAWDlWlvR;8&GQ zfHqTDZF@4CqZ~Ndw&$RPCv$$8QQIYR_+)LHmAf(OlA?3N-3+xQ-r7csb+1zB#Qbs+$1ifm;jdW83~a+?vAiMSf2BL})+g(k=Tct<`7X-+kI&l|Y- zx+C}M0yRWn#%Dj<+*Af)$Y{2-MC|wElM?LJL$e}~e8!fN^?aKjTOumbd&a92c4dzL zJm9bWuFM(82d%HTkyUkD%0xv8A4EoB5op6M2yccFT*x({ofpi}NO_y2p!G*iv&N z=j0Vez%dk*(^F&Zw~P>1pk*cJ>4_N2K9cb(Bd7w)Cp8$!x zu=Nsy_BW2Srzs;U&XmK{T2ZQf)sw*%9qf;SCeXu@P5JbRBLyNV`=iPy(4<9s&wz*y zc}w*W>9yl6kVS{R1ZALCi}s!mO8t0$6oNp{GDmua+GMBfQ}sK{IF)+)<5suSJgFEx z$B9I|H>ut`p0pWHnVJUgO)hwkCo8s;z16~1^(=TyD9tx3eQ7JrXJM)MH%@9;r9Hg2H%vE$nJ7qMkc-k!*|iA&m= ziI+s!RA6`NVtTWw~ljOZ8{}svB$$w=bs)~0E+U~at zpqoQJwRsKDtJyh~XWi@N8N!FuUtdwk8(X@W!NLC3%Lok5HSNA$O{!?eFSY(Hxzl`f zucOph3yp};j@Z5c6R|jUA0*anDtJ5UfP6PR$$#tFjN-AR0j5R1)KWN!%k}N?H@jI5 z?~s^XgHAZh0wrziCSXe$M3m&A@UPihN#<+yR^ou@4^^?H45D>OYf)9wz?G$)v8CLU z(B-C^0Da${Mx;o-QSDb)U%sjBC}T@Woo2yyu%-0y7quf4{jy?9m7Sf1qbw9|dT5qy zs9Q=R>>UX#DS^uBzEvwH-8oAMc?{S}!Ys9_NYnuQfo;Fi!~H@kILXtuRgp+S1;LYL z-I5SLR7*l+lth9*mJ+l%IVx0fC&BtI388}fQge>X0MAr#=fOw2!G7;J^QgWgh>}$T zj1S_gnagose)7u(wz8X}>>Zq4=WgKZpGGrS*8|h^e~XTXv2sCq2>MRr&3(N$_}(&V0bm ziG+~bPVROiQ6A52=dlvQoGxV&g1+9}TE{={YSxh`-c^$5 zg$`lTY;>Jt%{ubf4;HIkHni6lYiial%EpQ@2yQnqh#O&Ou%#siaU%}RX;L``vEX8P zA_l>FHi2zmo*en3DdF3s*J`ao^5eU`%=-ezGZ}Cblj^S!`C=8Kl|qHka^Om9|-K~84Td7r$IRXq-)`GDRJmC z)odzn?UlqDhvx7p50&PF$gShhB+S=1^w}}>GcolhY`)-Ic??^s6QX*w#5gpMFfCSG z6kgw^ZVTZn4t=&ce#RW%;?U&1hdA{1Ha5pO5^A%NNqSELdKgt-lkKg zA^iz=2h}YH5Sp|HcwWeZy~tAmJ=&fwg613TNs@jr^G!H6-6Ug$){MZ>99#gV z^qv;6rJU>Jr->sjDrx7Zk<6c-CUMWtzDY^Dge@hDKVVBQNA8P&Exp)%L2C^r_l3pv zid(2)OZf8TFSP-;&9Sv1y;ht81c_v@~ ze0e5cpE(wj@dRHb-Jwk{8}ju@w1@a2(YfUYF<+mw{{xB6VN&kV8loy%&=W^|eiTp{ zbsPufzCjRCU7#e^Vpp?n#V^$BC8^F+e1(@dRYv>5MdA32EnR`!bfr3P=)RU@ z&AD_`R;ZI%_3eu>_ZMOAll%aDQ?RA{?5FPznO|`d-B*(J32=0a z<*cjCCD$!wD6e~m<1Z>IV4Z$dj#kE&^0UarozUp?EBKoG)W(!c+dp_@RnU&(EH6tqZw6byn`|0QNJ`Lqp<5Cgx2g)sVOa zBG`o54qegZ9Uw7={R9^k$tyyO57-x4c2UvZPmD>ft9(z z7eiMvz;-=7L`JmhDIoC>8Ns$+YYAoZ>QlD;DA`at2o3VH!ZSqIrXO%O9wa|VP~MtS9VCa$@8U*dVY_36>g-PJXNu!=1BwRFQ5Wy zF23qX6ftG?HBSDbg7qPCt3ndTRe>_1GEHd3Z&4be;m9V({?3kUr1?da?0MgS_NIcw zZB~dxaD-5C^{uSsAQp67P+hDo5qo}Q7(M_!AYu?mKKt1yO7RF073oc7iA;pJQ3nQ9 zBK91!-<1c9<9_S?t~@j*|9{%F2t=Ee35Zzwl#=%|! zjsr3`N)|AJpB|9j6jZp)jTu`iUgTgN_lr3172argaj+3KfTwSGQN@7@Z+`siFYkZx z_QyZ?^edl!?b9o4slO8E7$ zop8mGPIcu_AkVlN$F2#E_Q@PcPQN~7@sw~&Gon=cj6S2uAx9dTz*#@;w*_bD6F8FZ z?3;~8m?n2P(zOBxZ1|vn)9VgL8p^<-JG)+YIMN6Lo@I`-1T!LF01=Q?cply1$Xd75 zGwm4t08F6lk96Jhq|NvNGd}Pn&AL1pu%+y(W_@jFoNQCzQ>vCw;8Rjvxaq59glu1_ zF0|TgY{*$N&=h_Ca}_(8el`3P1pQ^wM~X$5-;DX^{?R6p|G9g|jZh~&SrL|4T- zZZ`WZh{)r352x6;SF_*Jt8ZR=hOl1xGZclqv89_C9PD4cjKT1Og6-(So;_8xX` z^3=tjNKtFWICz82DJHkx&{Uk;yBPm#iC@u96Qt5$T=B07fxu9~TRC7P)Gx3bDqu_L z?xDPA=QMCot9UErsx#q^WDbVDbNESq8jg~JE#+sCf&p8wrHr9ehaR?+wtB7|p#w&2 zu3JjCDja2@aMMGxY>O7eyULkS{@rLrwQebWEmvwKcls3_K<>Jsv140RB+5)LF59p4 zaKBo~oiPt8Ol1-Z2%hFmg{h$d!M7yDwUEV7`@D=#_R%IwHJV{Rc`ngIH6VT?XY+L$zgjzB{6)tXDX~LGmAnujkF_a9T8zANE*45hK3>6PuJRV^4 z8ykx=xx?CN**S7jdQ*K@pW@B6dOTDdpwG@52{pH!=0)|9P~%j_E(v>y!s#j^!Lju* zr zG`1B7B6t>qw}U1<+sYr(HS4HD+Adi{Ur+C#7kYzqHhSr4p=KQqBU7MP;fT-gUfS8l zShJ39WMk#U`g1qBG%<)9VQ8?WB?fUL4ow}nEe65pgascy6(X(H2W z?W{svEWL>|?;9M?=_s01$38gKB-n#ETKd!J3`S_AJ(4@xm&8o4QQQgLWf zZnp9_pJ>KyUXR4lV_=syj~p_uXE1=To(AFkt=XM~%Hc29IMN9wm0Nb2T|GaOp)3oX ziw%r9#i1buv7Y15{OosW%dFDokus}NSjQ#@zNIC`p+Wrmwai%gqn5BU3G+#-V{AlsNP?wsa)go>Xq?dDJ*GKbJW49$QL0 zD}Q9AZDnj}3UlF}FVc_l6WR8(0LI{`U+B&rU@!mZ`1&X(!RBywX^^f;mr1BvrD}r& zpEveNmxQZ5z{V9lH2-)Q0tq-$df3uTLe(m%?vY85rUNGi683UB?uDs!+_d(sul9f@ z%-(5T_kbj5?TZ*)>u!@0I2^JCPF0U=NqR@LB|YL_s9Gf@KRQ~aMVHb3oSxF`0jC29 zjrV|!D9?R3oQ~8x_TTOv;Ea&bUVl>o_Qy>!Z85WXgR=^hI_#R*B8W*kRs5G z`&4EDE7|>9eUW7At{nFix&i}-t?Z_-59YG;x1WGsmi`u;M9zyhkYXhV6X{I;(SC&y zze4o{p72nW&D5d}o1!vxE9zLx#WrUcO}kP1V?KSJ1WEj?GN{u~F+jW@s$xs?P?mW@ zv$%IjyHO?Wj4dUCblcgfM`gWd4!+A`y*S zs9;O^2yWu-Med9A>d0slV|~`{3(hpD61`$-EYX|H{W}ufo{iCN3yw4i;`At8i-TD7 zX@aZbRMWK*332XQ60Rh=T@|BU6&z_2NI)j{CHh7}*i>)>1)u8JxGMHi_e!Eam9IYn zUauc~Kla|IIy%@tI$xij`o4CGj5S|h1xXr-Zb!lx^Yv-cI~&X5$Z7Rs3y`NzUl~mj z-D}^$_c2thQ@xM_LFB4+wAUga8Xp7^?xG@yex^4itN^w&OYIvDSxXQVtx+;#@Uy zw7<%H;%DjfTc=j1;Q*4nN#KVnIz2!8HG=GGleKh~NS&2d{6@*=Pi;Q~Jv?Mos8#i* zr%-1Sf`iC0;!YDCw6_vT|DNY0X z6XF(#@g$D@JQ8@gAWV+^1RTu~gcg3LuFzuc^%8arHG8~NPwW94j-yH3Mf*9g?`TT+Hc~vNwqAg$$8>vX58x@jBJ-G3f4msyr|itZ zblUsD;U9i(jPY|qSDnrZ>XiC5MVlV4Q81{dCv19#5#6@wr)bjyLAlNUdorpR0&CEc z(HBEkGQf5{r)~Jnf*2w${1_Qk6cJp^WSFy`vh8)~#a+(+g z*EXX$7sG#hM+&w*&TNP;*qO<;{nXg@c>6kXz&p4@_|xb;f^_?|QQ@MgXg0pHrnxP0 zPt}UPmSX{9i+93=j{*i0uge$im3V>t`Lc4FVX0R|6^*F!mD$%g`H4DqF>;G0awl?|VV&kkO!*vRK%VC{D1&H10jsO( z&W>!PIX&F3CfiY0rNfae5fygBCvAy{W=l)N6+hY#om>oW5=cI~`Y32}n;#n@>REYE z-E!k0fZ=LaMhC0&*5R+pjc8+cTi0D#C-G6S!1B+qfyw>^4!|h8vMTpPZnPs{!Vyq+ zWm0pGxY0KMWH9$6o8PWrvI_$@+Wo?@6fK7UDLS%L_$XQ*pb;9r!gX`p%XBkk+X)$KrgX)(>dDl_L+)30M2$YF;#s zU@{<)6D+jK@d9K$nE-ghkxsSe_=w1nV2f_+$=)9gP2lVZ2Ie@3%#i{S$&gFmJl$+P z!bC8SdCT<>Iqsq5Es#Y=90>^~q(!6>+vxVv{x};!z_ZMe;;h+q06FO)?2kB=ZmRGV zTbg=mr{U0t$dht5Wi?(r$!#}k@ZOZj>GEX2ma?l#(CYL`%CXead=#evIuPoXa#N3T zRxk%31#`H9 z=D=1kCmi^1?IJU^=*xWI8VUO?x7;9F!77Gxdp^Y79CP5SOt_i6_Y{*p;~#NMS__e% zb#rGZpARYpr(XVcZLYeC+;sWWGt`ud-^-4M*05yG#n)bTguPwukC27TZ(=tI8F~+N?Cusf*ql=j%`cw zkhf5snN2ua0uSDl+G_5!Vtcu*6$NbRY@o?pQ^Dz1bO6~Ws?T3%sZ~Xy24GIG{Ynq_ ztEu3Oc~D_0lTcE@Y2H+rQZ4@;36W6}3F+#Og^gRaA#gA^l`_l9Z5FCtNGX>_Ruc87 zBi1$g@h$6^|ix zb2_pz%m;>Xwpvw>CwNe2cHSrE;}Az5cSp}&V-fG?daoJIE@MaCf6s^aRo8o`YJZ)N zHtUF6TA)O=!+acO9Wi$v-rEC(mcpa%mEJLw4B)ANl(RnQ zwc~jCT=Uy$(%4w4@?Y1^9f{%;0ea|zh5U9E0w znspH0*jC)~Y(#GdrEoeDB@O9M*w(BAi+Gr}gWlj>+%@awP_vGXWD4|fMYCNtv=@zv zlyA!u7Ev}~XpnU+htwIs!w|5wW=Ace8i&?N1C_USnq7!Pb7%n+(b_{1 zxpf?xIzu%M&G7@i*_K(Q7SSVRR;94&+<+b}F%G?W>S*mgXqjp1&{0bqdNFE$!5rV> z&=e&Kap?Ru`c51Ou-VAe`bjpjIEkN29Qqnt$^ij+XTz0J;?VqD;?R3+>5>cKF(c^1 zikM6K@w14a1YXe}2(<(Oys&lFSvlSV7{){TE6GWa9nRDo>hA%WgsN3iFRdjZewdbo z{T{HmY&XXY_^pCgNeJRT2|=r@d0W~8bVNX%g}t2K?*YrhJ_e87gxU9F@Am)>Iq)1y ztK^UY|1D*Q69lRjN^ZU*yON_uA=;fN*-fr1|j_U*RLBGmQ{6Kk~-<6X~Oc7W)+z{0c$(sYM-| zP+iVfV)v6=x0GQ_j=3qHK2N6ugVr z(!~YuXsrnrG~WQrOGe~q>)D=WHwN|kvy9FN920&}(k^04>8=4?YHCqQyX|S4+!gGI zpG^w6D+ybwicA$-s#5&@*{+@tD*Sse&$g=ArHuu*ki~U+FWF6t-4|5j&%~gi`h!@a z+kHXp{VLHbrbd2(rODiH#c7mei5Ph{=%Ie0Y8}CtBXOEY2(AhWD>V}Rz*TW2(e0|B z9KT73n+g%2uAhL(ar>~T;06kIvFhp9kFuA#aULwA*$(=x_ja+$;@@puVwfBWpgDHF zK0W=nmq5tZr>+ykSBY*%0wwUHU94=ZzC?GXI(6w=L-e?e4I;?oIQUMz`e1^}P1Kd> z&Q+(qz5vnqAc$}m6+!efy(wV@u%%gQ-%#pZu=fAb>X# zs!C0vsg{KJX<8Dl_5cU)sbjCD5(D_EFxB4!5(y!APr;!~f~9&ByuZ5#JX~2Ijr{=X zrS1WS1}wJU7e~++!vq$R0djn3AW|3i0Eg8VmD?-6;z8M1{XM{F|AjbvbVgDJsw=z@ zdF%l;HS8&%Hhd6Uc$)fqfSpvd=ZjG;K8h`QpR#A?|3H{<`{j4Nz(0HU?(=Vd=Z*jO z)3@LIyYIex^W(Q)e(&dRwMW1<@qWt(>uZh(N!mR%s-D^h9MY!Hd^;(4t$KR8_W?V) z)?Lkj;Q;TWeSoL<>S?t^Ut_F)z$?|AP~OmeWfM?{K29ips4}4i*lC+@4R-y2=P20q zs`~z@1Yj6F+qikkzXu6`zvw3W?a2s&s4bzqBA0q(?4`hJ+nyepf>RVky%NfMGJ-Iw zWt7FOD*9f@xF-oz?!Di7uMMik``y+>5>TPub1kN$r~j7NqvU&)1hvqQ8>DXyejrE! zoJqGOfiwkby!)zMoTBS_vyx*d<7fEJzS-u}Q)TOrFZe9()oN9$=XJzQkleB^RgXgK49{CFVPT?!Fud(tJO>Z!5 zD*SaEz1>ii=_>(*FMD$JF=(dP7UzmRZ8C9m5Y*%DEqH`Lo| z#6pu`-x6`fk4~~*T@Y^-P(Hi+hF*1`mWYb>9@_dq>mi6C&V_jw?yvK<=iMuAY|Q>n z>&lHjxPcPwnsY-3piOgb;J)lJ)OS2VVPEE!8^lcRp<5rw+-SR>s_O2CB5Zu=pd_op zjV_$9V$?TVH?X2ZzH&AK70-9Q?r@{M3lucp^}552#z62abECz}D7WKE6~4-ixO;AL z1pt;cr zI8-k8`v$Z(HOy{vV*z$mfp6@Bdy6R$3iyq^J1O?g{Fb*ZtAnrs6H( zmOhNq?W?8?zUW|obWZ(h^uel}`oxg}5*3adbN&`x3^D2HLmn&?)QMJ)y!(@ul+qnXg$nXkGqlg6x-I6?l#%h{A_2h6x+*o-{@nf>T0$XykZ5{ z)vO|5w^wtRk6;H=EUDq{r|jVMDLZ&&f%Rwaio0zG^90R@?O@7&>thBxmkHDhGCZKD>ksPG<%^F#D~2SSkuu@TFAh_RV&`RVA=P?zVxw?()?s@73+aY6QP_ zq`yQV|37bzAlt#g0QO|bQO~2z>@qMJ)`Qm~xFcrEHgT4lqLyZ@hCgDh^xiR1dt@m^ zYNeM`yxRl`&A^lVx2~^+mEXx!@E6OH@HPH!;b$qM36A|Gqtqa1npK zq??|2EZ7P#r63q9wgWI_APWe2L-`v8BLhq+4pyWiB3L)wgmqg0rEgT^>y8LUOwiZg zB$RWfu2>cm7$}$5jUn6WB2j31^$9Gm9_|ZZxptcet<6?0Lq|`)tXD`V z<4)DA%vN4{vXD9PF&68juvkY}hEIZJoE2qihc)xTBIyhGxGp?A|2eX}dQp&4HF;sd~RJV;owm@w9#YTbGj8MItO-2lpv-8Rzz zm`Y5zm){ZSe~s=@hoCBiUONh!Gjw%%kX4`8jwQNyb@dMQqCl6)@!ZG4b;KtQ z7OQ+VJj**{fh9PbQqnj|qbvPR6eJNmvNAig<1dGaoXcwO!Nwn@M zW1dQ-^Yhf!Fr*qPt&Y2uIV-Ur6RUf1_s_2}H5B z5{}l-zR8(=seCEb*J?Pryp6w;D)!rhL20bw!C-``hNG`Arra)@8wRj9w?x!j!_hmC zDShyjY_y^iD|GICAT2WD>P3eDyHlNQ9M~McQ7eR_2H`cY%4O=I*O#Xtd!4Dpb|8ck z353`S2^hT%3GvgkBwT5hs^87S{dtr>UrE5fnf{Ruga6)_zx9=W{OZ@@EEubwoNSbO z%OrRoICM0`c}T6}#&vfeXwv07t+U&kSF>iRs5^gqzgL))zk|W%dNtV$*DW=`$q?KU zs%YE`N2{%Sw8D!DY+d1%_5pu>72#)u;K`Z}P&8VYdKI|}xJEA)RoklA`g=DTU&tgN z-28*@e?c_*Z@=^HHWvL)+nd1-_uv$mOo3>Mygju8sDLBHwOS*!+zzMXwTdy&tJNAO z@HoAF!#jZT=hcfV5{?n57+sx$)JZ$o0hH9Q_U;w+KNXcL4nc{#6EkoidOz7f^xB@ff=g2s%5DqX?bvISoqNokMxgaPI zSB?ElxV9fkLvON=a&SrZc@7S$+Xa}Zy6b6z<8x*EvBP7+<7Ckb0!csnCTT1QW~v%- z1!l_aquH+h5Q;~8FweFM+F4*zz%3E2ZIzSy1qiZwO2vNV%! ze}cIB#ae+HWm$TXr`pa7sybJJs`_tTftpFE+lmUn1(|+2a9HeR`pFIpdWiH`z>X?z zcS#5iiwT2&BSBJHHJs}#?B#Bpht&E3d$;KJD%sp0d#~G5a9ZC~sry_d-=WPE*3^Bf z!9jeH=@fT%gP6KcEw}@jPT6Ml*kR=-jQ*ZDvP^eX8hW{J6hv|sbzr7ifqIFHnW8CR zM;*5+h<@%yE?`$=`VG}Siu@azJ@u4}Tn9+=IM5U_Egg0R>?T#hDHeND7T$jODayhJ zDqq64Nw3v9XDKh&?IpQ>@j{2 z*PMU@Y@ZpALecE`S@Y(T_L5FZvAZn)ybJKpIxly20ec)yb|{>1C=|7tLU;!vcq5@Y zY^T=Aq@v3`60UXuMe=zce;zy(frJAmQ=Wwy#iznvo(0SG!!iR`yTHR`20Zrv-`-Wn zRk3_=Lec;cFhL9yQIxK`XF#yA6I<*KuoXcNFaQ+;0|P-tu@O}6p26K+=4+YYFsjtRvFUvu80$J1migl@~oS#gTl4 z)`TPW>^fVi6ddAVw0@dxswmM^oZf^ydR9~7U$KOWLc0>pS4f*@ycuj$M`ah+^RF~P z1?iQ-xMe)Pg`qCSz^7^ou98Fd#bYvShG~G+5Svs<{bt;LnoU94iK4`BMgS=7gzDl& z+KE5yk&yw5oe)->>~oANsH}}DNZN^_v~R}LSI`N;8U>zz6m`PGx!xL5?&IrAmNr+KsD`~F$fl8jf@>wY+f86 z!pdK?Mn(%*WQ~lfS!`4a-Hh6*Si52`MXV*YD=((6;%u0Gf<>$)ja_*$y=ki@SRFc$08)Q$Zm^;q)Fe5 zCN22^ZP2ssrv@&Z1x+hX(543@D|jS~(OPUlg{#j?ILlN-WXc{Q zzJGE5n|;Z%V6qh0YXsLt*wpvD?BTmt}>CoiB?12{el;-X8m=m{>qH`p1Y9wTxy}eH5*O8JD6|AH^Z4 znsxuDbujx1%ir};gty{4n0?bm(1f@SX5Tpy#8p`bGp?7K&aIoBsZ3he86>}yTBVAGFipX)qcO=FSSOKw%1-m?q zqfrqU6JmP;L2OY^q_I66ouZ8uR!)4P*c3Y_u6*4rno^jIz=(vjH>Hwj%J`pzro^QY zW5rQ!z%cGaMX$t;i(8uPrqDwn5nc?Cda+nZVWKI!EZ!9GsDR0oQ7W<#t1Z!#QGE!{ z6{DeylA~ETpNk=Q2N0Wr^Y?L^BHi-?6M|iO~Z>NBHab>-iTknF_VU zx+-oti)X0%B9evdUtA&u;0Q_atYH;ycP|#L(709?XDUS(R8ry@!=j`Ml9YI97gQ}p zRF`6;iHlrzDpAE9Vkt3y?ShJ9SabqOG*`B2D;jox1rK)2LVIa7@c0)_chPYY!9&Qz zXRLuDY>3;x>;RmgZxv2=Mz~mD^WxAISIF!ThU#R=2EXRXQuURnxQ3E5NQ{B-zqp1h zPq7$%sKTS5YnFZa_tHh+A#R+rCorwBn&PkE8nU#|qG^T&dQ`xa-4Y8(T*KndATi|% zky@4*Ce|z1gY{yjAx7-QfhWdeSgwzBV*N`Hgbh)#$v?oOm>`JoES4W9*80N@WoDvn z5*MZk);IB2n4(#Vh}eP(g@6}}d}v&z35lj`nhNyYO5tcKX6O31@YTgXGT3q=7B#>1#Oqp(1`)-L=xqp%$u zQ7Td8swdqURipM9r9cZs^D5(BEa-wb%KytLccxTT*!y$-QL`Y)Ub~jkg%-r((V~C? zU^16n{97CyDyQ5T&!WVF#DzHX+*M{2fwEEaj3Vk!bn=;_D=`no&8T(A3boi5JH<|k zx>sefV|0$%2P@OPSW7@Q<}j!XR>tI5G+2dg2zFb#4WVH-TCxqHVLO@`N5M7(BYpfY z+YoF=Sm@#(JM@f%uUMt9y%VvPbYaN&9<|mItzXQQX9G!8t7uPb@mv_B@n%M;)%ux) zg093BHDg*;RVa(S6k5L+WEQuISs}5DY^D^qqZt!q(RMUrT@`Qs@!@&K6ZsGOyTUVA z1LChj4JhBxjdn(+s@#qi7$yt8smY!uPI_!> zUBY`L@JjyMcC?%1OuV!&s@u_wj8zazT&}wbyX*?pu}q%A?PzvcP#s&5D-%t<*-5QJ z-XqpkQ5tbY?j~%mi!nuS;hU0Ki?*ZxOf+SD zs)c=JKC!TME>6PYzB(gPEey6X9?F6lPzbiUv5oekck~w=5NOPV8e%;a&(d2UzR{ND z$$BP~XItzA@jh=BbWw3`-6Z+@(k`gsfI#NW8WX|{j6=1^v-B3CDSzuik!R`6A}Go( zh&WJepg|Q66{S^yOUukI`ihFu#VyNPTg%VO2R zhM!nV>NPUvORcp;1P~6=vu6pG#Il{m7F7H{#&}tbDuvVitghHw7F{R`2nvLng$N+5 z&;>@)BD0oGcWgNO6(pLviBSSpTC7vBtY%!1MT>p6Vp8*iKeW&AgFM9p8c|kO-%jtF38Oh;Yay2ac&V3^A+=SK`M?y z2dYz>^l7Szrf%Xv8aFW?p7E3xK216G3K>Z3vd|TQ>Q0epDv)IsC7QYw7sODBhp+~O z#>60on-IiMcA9Bhn6MRciK{0uh~Xv#F_fY!7dhgbE(S5&R8I3V-phj7Oq|mtr}?E5 zs_I-ODp`zgF!EASK9!wB@8V8yv7U+LRbZv)1ec-o;VT7&bGioCUUkaX;Mp@W(8770CDQzd1S7_= zD|S$Noiqcj~nAaJpsQ9Z;s^YoKnhKI37-Oe)butOXx{6wo zr4l>0RjAG)ubWYVid`%?Tf_)Di#&UGF{MMKuUa0}9Wc z&2fcMTY>4b7!1JlC zFRGqBdxZEn0dpDoXTjtv9OV)wUtyYOm&Kdx#ERkwnKuha6?ygr(unVgkq#(5dm#WN zc=ip%7Q9R?hL7w$PUKu>I)n0Mq|be2%S{l z#lm3N0O3DIifwhaJ=tOO@F8P+GoUsQ|0+CxrpL~oIhvaqgbDZkO9=l8N#->z^w~D5 z%1|n2Y9O}dXk>2WtE(5lQqfC@r5Mf>4LGf(B$;X&OVzf}mxfub3!Z3du3Npt`x4oH ziifr_)n}wZ1){nsM0G0=Ral3Lq6(i)`)64_Bqpq}e@r7D5_STur6fDA8cQ)*g#*S# zdMid&_Fb$3Sv?f8GT9ZcRXE(Kc&*~W4B^jzc(2lLCTZ*+la+mhM5TXPOG(bJXe`BK z71mOJm$j{F33kSdjakJQ_EHGT#(Y$>yi_ zViGq>V=)KzPuGlPXAjNE2k+J3qx_9afPuV`(UV)r}yukbYV8(Bgz17CJ4>=5><*7BQ)EXE1byK@lb`(g%g>W#i|h6Q_PnT{{EXs z!lYqaiGLlCLNQm#5oT>dvt-D>>V^r;Hvay>1G6w6J1Vr_V5gx&#ySm{IC@m*SVv1^ zU2~%V!KL?TVO$UkakNlg^K4;U5KSq(=GDTuAk3pu+gqh}QJX%RuDUt5FfOHRwx|U+ zmlno)>Q`MgUUk!W)m`INCRd>hg*|g)lIg2IjmcK1gQBubwi4=PHOl7BWHVPQ>&|3T zZ`Ym4rrxeQlTAZeCY!p>+?j0ZI&){TX}HQ{({Po^rr|1+OJeX|i5_>Y)G+fmUyqRq3I`h^zCcT+#>N@k*_};vkZ0b7m)@X0u8tu(n z~%txbaJ{o27(I}e_lTBS`J{rfQ50gz@XFi(V8zVZ_pfe9= zP4?#DtjXRO%drNF^>EfSCOw>)Y-&35aAvZp>5LH&YnXo-=dcFZ7_+dZvMx+EHJ!1T zjK;D|HZ`3w(qD~bnQUs~j4}CY(u0d8dt>Ch8p~=LlZ-!Clg?b3Z0b65)hL^*M%i37 z%I3;sQ`ecqCn>-H4_79ey3SlRzBgAUo4U?iHQ5`Zm(`>ljeZFui`7_`$)-Ne+?Z_Y zI&;%xZythQqAr`8M%mn$Z0b65(>NvtzeK%VjeZGZQPrd~!7oumNJfvU@v6o#snIX- zaA&fq>&%_Wrmi!0jj{=TiMnhW{SptsFHzT-M!$p+gKBJ7<9ieQ67_K=_$BH(6Z{el zS2d1FjeZGZ;MAlu!7owQnc$bG`I8=kU!txvjeZH^z|^EOjeZFuw$#|J#`h-pCF(j8 z{1SET2!4r%s~X3oM!$qHHfqwD;FqZDOz=z8btd>F>N?ZtmoT0&O*+%)moV-cP3>y* zOBla_CY=d>iMn@ zGR57G^b4>YTwnS9IF*e@#}3A&n0lP`UwiL-r4|BwFkz>9?&c{4JpdJ>s=<1h@K zcmg`tXv)*O(IkN7GDroLu%$S<8JA%CGdF!yk?j6tL%Y|qrc?A!phQ1^<~TSGV(Jm9 z`7f=qnzVcBM;q-s3Rbhf@NG&@rVcY&(9RF~hy+**DiBLcWmmSo^_QS2iSr zdvqqB<2#V)vse2Sw*E(2tYaTzb3sKW&8kDGDLZl83NuQM8R3z+(R{}jqxn{4T2RYV zKVbNn5c!CUuVv#F*`i^BABpKQ2TwNPvG%?tXj^GGA5cN)eIb#hs=RCU*n1Ej5~=s$ z#a?LYdJIy+^>OBmSs)wv0$)Zs%8t)Ykq!1Vr72g|@pU5`qG!)4r0lIDuqxIU4{|2l ztfL;-{PaTDJ5`r%9X<=6W|_gBEtzZ&D_ZE5npwSEBBD@|md97s@nE&fA-9!{Hi z3cqw14Q0w$=X%Ai!IUy*@Q~d`G+Mtv-ZGK!4_a0uQ>I0cvX{%#aW!U!+3HxHyS&(<^6b|lu6?Jc% z;l^&UfxsE2_;EuyQnvnC{$h+3IETK*8qb&G-pVuJY+GAM7_te?KIqXB&S&J&4{Abp z%Shfja4i?@TZ&c>d5TziEzFEQ%{4smmT!G{E;QI{Md`z<{B*nfaPH9;XtZS)j5@M} zUom{0oQH0F(-%o_EMqA&TbK#HOWokzo)7RbZzHdN^)b3nJ}hhR-WiAVZHSBfj=VP0HTZHNxsu8T08hXG~PCodUr{t)rWeKoxOIEUUQcLtzMm^ zJuQ=WcwRTWwcgHe;ECxZ&dZ7T2A;z1bAzePTEtG3y3r@SdU^YtpGmVX1Yy@>aioLu zHnMM2Ik>TAKPMo8R8Xl6D{%_uPC(rWyZDPvZrDGmn@qr|!fI+pT!>Q%cMhXDp2;nL z;vqczR1HUuIs+S1x&kiSjQtI!!J`tp`KM_PFyeG?n7DESbQv}n9ER-#lG`3ewp)yn z-f6G_C6e5_HbTCwX6=vSZDVp-D7%_kGDfboYKViSAc@E zd!Sl9J-m{Z#Y5xo7#|u-ClC3JA$n$1|3Kaz0jD&l zx;v`iRPFvc{FuiBY42AX_yLc1p}tNh8d+-&u4z1uJ}SEo>OCHfRhoprn41UqOD36I zm`z#eUy-ApdxpY?q*|C(yDg3wR+G-@P?oeCK_+LU|xC<_%-KJ3A$5w-3_xi^4%^e3~tm9A59E^pJ zv4iQ^(eBh}-bHTPgEY8S-<%sWUxuH5Mq;9F5ZZHRp?$Ala_;>m5pz_uBpu^QoO(YW zdCP+wG@afTTc)-Wv8|wv+7MU4so3265K-|8f90_ezTGwy3_ok=fp`XTH; zran*SwH9$oq9>_#)BnGoQmhY^IHmX^{huEqbL^;CP`y0(iO*_Yk~G|QkXDc0MBI{G zsBd#SD&M<}oc_?4-#Diftu1RzpPXIE`F5K@1t&rK_q`pT7Okk_nuL&(hH)gfgcGD) zpNY~4>V>TrZN?YjU|TSUQmk1#&D;e4pIe<42ldYb>Wa69M(zk%R^<*}*v zS-9HcGjwX-8?I~^L`nu!BB7RZac!?OTyWQ&x|9mS2j_cX)A^?GsA5U--7kj7%1?*t z9j`*Or-!k8z7H4OIvxZ5@}+Y}cZIt-^Lz{bwB<0)_sM*@%Svy>@GM-4sTuO%f;H9g z^q+F)vnBFsbXW}oht(t}T$fRYK~u<$S37Y+_f<5(8fgp9N@QHM_GIv}OZ2qma(p;G zk9^dm8+b2P-eBSDeSyWxB$g~P(!N*isJFf*NI`&ezK(H z5i&qN9ZGHgGpGbwgN`BD4u0Dln zcGRc&VVoW3LJf50^8-%i0LJ}-ENF+9rVPYAw<|!ity|$qG=au6iI&X_!0$I_V#Q0d zFk^FPsQE3QcYgB|gJb(Z(v5U%zs(KGJqm&^-f@`J)R`XfIfP3yOz43puH?joLAd7l zdYsX224_w;!d)9{TAOz!<3H`>C;jX~HoP~X=Pon@+bvJw-0-95*zN_?zZ^ldS# z{r%-$x7~>7VHLRle||}!AbjA-Rl#1_=eXqjR`m4?3uz^_p`KB+w%WP43+OIvOIFMp zLSsVYuy#Nl(qMgCI-}DJE@{vrGIfk8ndtBsj#`(+MK!iVu$wN)dUhE1)-fmBJJ!J& zjV}2{9-K)$em#U~Tja#G_7rlZ{cJQfIgH!x21B>2Tloh4U-0|eZ6@i?5#-msL>y^r zNrrFu?wc1~j`SRV9@@)p;Lt=%YLu^w!E4Lm+11D3mH9R>Ty=}&caNgBN3Wyh^s?mh zwp#SUnW|iOtVlv9Kaj_LsX|x0t4yq(e#hH`>yf2HDv|n*OR3k4G@7(xB`$2efDAr= zn?A2Qmvl0-B(YaK$@^IoiRthjwBr&(y!LdK!m%vSt_EBEKWIr8SHc{nVRfYht@Olx zH0)s?Xf)0U&R(fTbnL!xfh$JS9?w(gqXRF|C;0;$?kJ-#D#*dBxd#zJt8n?C28Al1 zRpwwXdhqE)m@+X6>n^Lucidh9e$}*vDNo<~rll-|4eet2O|N6{{6IspH^Y*8?|&m3 z(6A(3nrel9{T9KEA3XFkTg%6JyP!pU8IhN>KS8VdS?n z;w$-({*~;1cwYf38rN9L_K(ifm2Yd2&K=gbfs4ad`iP(YC7fA8Jx3-iI1rm`HaS zRHxrx#A5vJ6BzNh1^qR(j{>v`oLn{=tA+zrKx^FKa@6>7FVfF81M`<=;EZdlv0a~d z61>Bh#(q5jr^gSbk3*cGV_rPJJk^@?>Rtk>f9c8Hi5X4$HJOC1^IO5LN9(cSocUyJ-(K+S4Z^cc%eV&vN|PT6 z0ob1O;n#MW0*hY%=F<(zad9T=A$P(?=u;|&c-8d78OHC>_FWR*w`@$yWbZ-S4^H&* zBEriWSD{>R89M%gnIdeWq9*BBSH=<+#axBX&EG@Lv3!}8S1%BeuAq|IP*(x1tiE$# z{V*O3Y=7}b?b_3T=Jko=$ZN8*JDoD-J>x5r zs;{o|$3i`Tt5S=)SFJ`Iy=K$i{Alw1gaN(b8Ao1DIIjRLiK3+X{eMJI3R)~8=&B&B zS2b7V&egR=JyGXpG#=Ld%{kBwXr8nh3_97tnh{Sa@BJRnJ(rVHwV!eo6LL^LWv=L2 zDO>B;TMe>QcCKpHo=J1#9&mRmErRSFZDDrcdMdLORt8 z18bWG#HIQP`tnnC%3E8K9beng)`@zgR$@A7+|G=QoOTU+h5Nv=L%#5+_cQtWI(NXT zbah&NS{$vM%)`u$bHUjykWLDlMEl;%!dfqTm{)%h+ z1fjvAvht&MN77GT-Eojt2G~9B&Oho|TiQDfc%_U8jwjXEa_1`cWC&k+aA#V>q8aMlY((c9zQ@&*htpN^ za=<8*3sSpu#{p-$E#k}6!`XM~2 zzWzEMPMu1ZWMa1B{td8}9X)3ByLrpiMk<$l>$fAmPnB z&eb%6R4{LX_9t_&?U;>p&qgcKW%^DeH`dXP-Ud{*7RbP3YzJ&Ybxc}WflJx^4ST)? z;(k91f8q!AwxP_+5Tko4T9UqJWkk@^AQmr;x(Qw8Hh@Vz5+rrh zhPX;ZFn!`d-p8&peX-^h-{<=v{?N2vaP`-2E~nO8Y#{G}(eI;i@T?#lw(A&n)t|^W zJ?KskIaa3kX1C=Ib|?jXO)HbQpfsoq6>vaoc{;Glb)erWQQP4%-t5OajLR#9vu(>@ z!kV&BVS+t<=&+x2J(7qAydBZxo-VhWM5&My~k-tTr_ZpOuf}UmrUM^H$7; zUS*@O*@wxT{8JEf1@d7>ms-^2v@gCau@!P-b?~;g3z7Ss<1SR*4Iu{iv65FbmzQk< zmV4H7gT`)xO{d&QPOcFqe7wS+{n!T6y6aJ#7mZg;OT>^%fo?U}=>Ks_4X50+ z;#B6;f%L}bXf!*W!%dr&Mm`3X_iH=Pp8SkEOzNM~%{zDMB019XIpIog!J3xwa>1KY zwpKh&{TQ==*PCceS5-72SF9r7Q(iyP&1E>y^K$Eg;nqxR66;x`&N}Sq0W(aR-Kb{U- zyog_W_J!ze7FJU;;zFFd?-K`0%k7m%RWFCTd%IBE^FQHiXM1S4p({kh+L1w)M`e+z zju`kZ5SFJNgy2m*<)_TH@Tad^)4-|qq`lK%14<;Z)ZfR2kZ75Q%!Lc{ttI9_xM{_y z){%2awIwUag=uTZRqHGfNtC6fvMXD^aXX7k@w0#`k+op{sEZ;_iRCro)QyOFWMaG3 zq-B@oWC?CAYF!IXWqb~z>GyYW!#4-fW^R39;Zz$M6X#8JEUQtgF99I;J_RvL@8Bgr zUiKjGzC5z7C2cV_7uP)bhK9!OB2Kye6{o&_H%I?fM!dWKZhX~e7YsRVAq!c*4b42C z(6~mU_-xW3_viKc98g2@tdu3TYrTNpTGkEE)mVb3{KN5mh#tLoVgQD>UyQT6HHVSQ z+H(=hkK_90v2<&|C3LJGhFxC%hL%11;nfPCxM#=g$fZPI8nkB_=e)lJoi5AgYW~=Q zLAQI*S}8BMl_`P5#_ld)&N;k%tTK$cRu&SP8*;adPviDmBV{edrs3$do#aOj-R71b zn#`RalnDb5uH#lz2qh=-7vbYwmAUPkUqSzvQasMrr)`ruK-N?__U`pjZlCoU=&%Rq z^H>KCwBE!S)k_5-TS7%k()X;yseO5rPwsLRRRu?S%WV}tUJTA8K8i-$E4b6wu#M=&BZjR@3nfvI*5H|ZJ zH*CupZuYaf#Po70yp=r~s@Cg^J$BY0&4<>4`x{S!8C0MxW@m7*S;p|aVhlE!eUQ8R z_%(MixeCT#JHo9^%jD`;{RTB0Oh~C&)iJ$G1`c-!f!hNsa4_`*G+ty52Y=3yTQ_M) zGK2H5Hq=7fh>Cn^T|@YFur~iY{xfedC^vWdt3lX0d?nYgj4m|$?MuddJOoP??uJc{ z>7eJ508KyWqtoX-=zn}UB;>8Y3ac*2vo>uKajHPK8f^6cIHiVD?pkqbagEP>Fa8SL z-%yQ2Up&s;^uH``5i}7@N@l}tS5rEC#9IDXWC`-xWIfgo-65_Ql&uwyQ@J&Ifvnp_ zSlZ?j)`=QG_T?^sg)=Xs<;SDA{D37MD!+wH{qhR(-XA2vVZK=SygH(2On}9ZYtAnGPm3 znwWG_; z0kqRnVB*sp-_FiYxQeEwidxr*Q`aJ*Nu6mi#2_+}jDN>g<}A-pY+VaZ z8Ky;}!~G=`A3VZa3!Lb#2e0}2C8u)TUysG(6+>uh+p_am%*me9x3gw4$joiT-3rs@%Q89egzuI$oKB zC0u6nlkPQxTey#!uQA7-=C{eU{P*1S&g0RgdJ}H*m?`)xClwv4or0T*!?9coT^e(Z z;%m2gT&u~Ma6YgqXRl*M>THeX2XBqREu-4tm})C=@5@Wj?V$zwt?N!Uv|R_^cIlCs zjjZVV(baKBl?*(YmyFfQ_vHt5@}&j~+u%C=RLJdMiwPI?$%f8XFtf@YS--{0(EdyS zd3f?RmupYu^^ZG{b6@Lmn{NJuppE8aPv9JAbo@Nm?pH_ZzikmE51(Ti%p($+hCXRauTMi?7O3HthzRfysEV+!Nk$c)dLrm-*oL7MIa1Vm9?M zTMq%%=40nWacD8RKRz!x6a5_e0B>sT`=HYf$fLcujYF()!PFc0xaJGY`E(YsdZ<0_q23FV%cU24EVNDo;K};{6orX?w#=ms97x% z^G+UuiQSE$+_DqUq(imb4c8vy*nao;sKtJ~`Q!eO9OexxOe#RDW@dQH^a|&G&|dCv zV;J`NS`{rL(%|c*QQSyf0`FzDMVyl8NvhrdQHiRp~8Yn|}na zUfm(Du)RCJn0A3Hqt}xo+Yey3UgrF(iO0CpyB~7GJgjW3c$|6=Lr9Y@E$FpFWuVK8 zIdJ&LeX7&rEjK?u90Gr@fd~52aCUb+epZf*l<9FFpX;=tk!QN{A0DQW1AEGH8&Aw8 z`zH*b@d2Zu;nv%5)~q7FO6vndvK&eD^#vTJ8p*!z@<7wfC!n9um3~~lgF3o*q#1`U z;1F3R9KV+Yy7OgpMcpv6WYbJ)eAa+GsWzItKk*3-4{rkhaoKos+!t8ZDS%vBGFv`k zXBx>5Uxv4L?%-D5;=~n;R8Z+CQ{vQ%v1Raw&JRAz?X`UQl{(^js<4`x5f|cAh`k>6 zS#M4j=pa!L>9TfE_cONjbAJygchTb|($-R+tanzjF_K z>Ge~LeM=uB?VS>*CUs5c#*bXf)jK(lGdp4`B8i7qoSNvpoosVCN@mVJM*?QD+zN&y z%F-HJzjU=OoOfEr#c#8Ma}W8V)-~eP@rYTZ_Vn4r|MgsA5vjl_v2`st7304epB|`& zlLk$o)?59Sc2%z9*Mfzno6zzzmcx(7MsT#}40JB*h~)JJuu0(YLhdxI`|>GgvLz3y3~m8I zUwt7%z6RT-R3L+o+j0{()Z>~j8I8wcdt%hb(U9@{0j55S#qm~kNzj2>(Dh&x&foTx z@6+0t+??tU?=N=8247d9n|p0qZ^}m4w6hr`k4&Ns@BHEy+79Dm_J0;L0adglea}jq ziu!z&|M9algxze3AC{#^>ZlEI6`bn6at8d6-GJ)>r(s9j54>H)ksiJG1YUL5rRk@V zXv_ysVm@Iz4880R&yQ50U)othzDX7g-mwev^H;-|Mjq6n3#;F*1eVHej7Nvvfq6L- zvF6fiuy?ngSH1?V#P)HW%ME(QF#C~O?!h~0v14n zr}nraDwiv9d<2(uK1n`hAwiq?p?t(1GjvED2zkqm$lKiu$j{j0z=f104Rm$MjQ*MU zIdi2v+-f#)$adsN)4wxwc=FeO_kxp_y%N8GjkeU_bz`lIgww2F|vwF%90TM zR{YJ(mGnjUWRlR&oOW~@CC1zn@9jp~d6Xivf*NB|oIP*7 z_7{+#7g#SS20mXOL_6KGqT#Ow(rriHap!K8!8!>BG$>&Mf8Ty4xzlYY-(v1iw8^r= znUM|1?)Ia(cCyQ0b<6;x%a@~N`KfSaK^AUyxeoWcrD6qrGmBeJrK3!VQypj^%XTosjM$r813h09aJ;aZnh_V`)Tvp34O6R7`KmKqm*<_~c%w|1 z7cw4l&VB`_H`CE__Y%arEBS;K^N}7~hGzc0oaM-xP&=|6UjOh}#3|`xq`gz(6rD@C z!0&^(Zo2!qsK&iTB=OXWQ+)Ua(lK%iNq)1N9H3i8oKlw7*!qg$-te|-70$+C4p-uN za8c_TacWM)3{u){8c7eILtYMNN7on%imhwGsTbwG@MZd>;iW1)a?LHi;8&-g{Ji-4 zIIvp+F70y&m!|-pZjgZPi8pZTrTYArIjb>mx*L{1Zh%mwvKaUC{3}j%n)DS9eVPv= zw}*pErB0-4(+#lMJRe`?Poi!6%p$E$z4BQ%G#cFPQs}8sn=o+U5w3myb#y%R1kZJj zm1Qq}2i?l_!4+OQWW>;Z#K5O4j{XP~bQjZ@i1%Puu{^z9ZV!f-I+JGc-*HstU3ss> zrexLKbI?cs3^>PD^L08M4Eqy?Vy?40fWsGAgQLNuO|Thl)qOhIEiVgBpVRP42{){! zvrK;I=^XHS-IcrJpTqT9`5m%q4##|NBY8EF$?_EmvFtnnwy)ZVGqTR(^5I>nmH$wF z@pvEdnZ-o9cYX?y;qxGId}nxbG#sY4=r69PRJ0_0&q|yc-pmm7)?Eja`^+Sbjl@`C zK^?Uru7XqfD-Xkj{s-_^^%$t-eH`}pkU>(CK5<{^h`XI4ap9VUSpQoj>RgSH?fp0# zZ%ql{y<={}rLiSA?<0|*duIiv`Sj#+di6sRoGo9wG{cL({Ik!Zi(7vTcDB>||7$^c|Zh zUzXhjoD)saK6Ext&zsK~+Q(ss>|i`IZ52+5TM2iQyJGEer*VC9fQVBPJxR6uf1Fao zDKD)!^?lMlaI0}JWiF^ydMJX=aHXNmcz*^BcZS1RxIK86Q4(!(fn0opjW&3^tQ`WsOVUo*2!sw zUmIlL$ng}6t{T(56`SMRFAP8}-{OaI4RE!69vohk4CmI&gsV1=-FtPys@=uj1xdHN%lF zOlZdkQP^qbA%4c>IRsYdoqWYenlmG8}Jh1n*Sk~(TbTm`4x>g96o!KJYE z*_%Mj=Hi5woB847I9{1}iO;s^O(I{nqCPfLfjnJB9xs~($v;lYEXGG+sas3X=;Z~l zT)CT+Z?^+Nm+ygvTery`&Dsw63sXtZ)3)&L)qH#s-iCxdYD`TpB$6Y6$cH>VfGb}u zh486vSU<2K9)H{%Zbx>)mB%jgI~$$lbiR~^wU!|`F!d+*cxOl6rbk;+JF+!+yRC%2 zDM7UImn+EC?L;yvPK6r-3IC}?1FoJ|2-exT2}Xz8(h?Rx^m69%uI486J{LnXB3Hrb zd&XSt&E=rw?|#^(>rohZD2+t5@`Y|Kw{Sw9bb)R~JxR6uf1FaoDQ~SfRdVJ`zGKy+ zG&`s){l4@7Hn)7iS^h4I&Q>SL<|(J}=^sU#X0v4mTQ4YED;}qoPj3dB zmYdR+DcRsvs|I=3#sp-Wc8sAqNJW8+ThMOxA}t5$MBbCTf8 z`7d~OvI#k7rAuA2*@+gm+LJy;+B+pqZO*&K?bwL2u)NUFk;*_^moC>VClYFw< zOu9$pkZ;E`MVwNW*4TQ7r&qa0@!z;b4GrP^s#`^^Ys9HnY=7#+*(kENZ5+{yj4Wzh z%X(`4qK9~p|Ay;B{pj5dc{uoG5}M4*MebpL+OKvfwclEvPyV(H7r)uS%fF7}y38Mn zU#3??|IImk-;xzXobvuFPR+4f1Z`RkCq0J^gh20i^7k=&$;GT|Tw)Ctczrj4)NWg! zyt-k=jhs`G+qWu7p0PFumQGuY18&TrOApV->j^11(ApEG`&EbIkB-2=z##6%&Y2jM z?Tnf63u&{x-Qbvx54K$Mh%dL)fRFeviY=IH^J`qXLhr@y{NX3>;aEDwKKoDbS1t{N z1&MXwaaBM0g!~NhCQ%22V(rPzkW+ZheG?w3>p)8PX^Zc_b)e-hr1RMB8tjgr$G>^6 zM*^Ggg_g0d_%oyosr&36Y?!wdZ~E<{qg-oK`QsMo-uEQeaKUrT81#eRkaZ3BT|FgB zx8EhMja0NGea{7Pzg?qobN&SWkhLClecwRDse(FcLtF)?R;KR6EVf-d(R?U|&0*va z?w#nsV13$p*lHZWf8*cH^QJzRYLIb3*P(jr$zC5`m4N(Vzu-*fENButing9m8Wv>6 zLv^!*ILF3?j@fyguQp{D-?qdAn6RZO7q!NRA5!T8@PVi36*f9_heQ&)-=}hI+MdFo z=^X9;uraNfoQgkN_|wA+>yWyR6X~(9wYjrh8llaVS~z$@IyUao6UA<#vdcmK8Sj^U%$tnAW2@pqzZblKQxZK%wfldZQo|`9tvGe(@DOO$uq;&7If0h3 zrsVUbCw%}xhfi0XiaDF8}pkYH(V!X(Z`)b#KE@)ngTI3m#&i&ZtKx#aV&{>C}?W>Y5CC_ss z_8H-v;3d3$OjlSodZD;tkqRmuWlEgdw{{^eUHAj~&}ZC%Rn{U-6;@L-;zFFtxo1en zJkg^eCbPKAcVWcPx+A%3*^IBTee`{RSy9*PwfVY-5Aq3hH}WkXPUG+R zwG#KIeEy14`Mq?h|D;pUX|FTHxNo85$JBt4zkR6pnjLt(oE#^%$$=X4r;(t+W~B4f zoBZyN+weqaBfiI-`DkcfmgKgS&duw&%~7!Wg+9$Byk&qYiiL+aE(IZTB7 z>tVR%Lm7OO6-51Zp2bJ+FCpUvgQ+x?ofa%XqK=>BE=}?xUww?}_{Q#JY-Nsbb1MV- zo}3R6o)lj-_v7yx8&Q)yV|wNH=-sU|jiBj~wm3D?7Pv|SsL3N!8e?}ESEiPtQ>U+j z$2*gu_uK1S&*l112cDq0>yw19)#&?qz=a)t<9==R zI~DiVFEnM5Ux0=$dO4;DT!I2?Tcqw&r)7L4hZc0@yJ(ry6CZ9t6UGJBsR#0H_T!D| z9r&-cX3KUi*^H%vdPCrsCGx%V7Vf!hpNxUFSE2iPN8f&d-(*&^Px04u^hnn8lk&^$ zZ8)bBrg$sFkkD0A<WAtcpw?1Xbh?6^}(6fz$iN%Gw7&W8~wvN7!?>1hASHDi;xYu4>n!FO-{4#*VP3*|+*q@0G z3-utO40Cl(+fY9%0~)s|m`>SZNM^-n!KOB8n6dabZX7-k#y(4fNquW#!$zIqj>9)t z`Xv-kL{>*${}N8szX;nZRi!I>aNL8QX7g@30}02bh{z7=;ZO(|h=U gk$pW(rJppjsA*c=Sl7Z", input.type.tensor_type.shape) + + reshapes = {} + for initializer in initializers: + tensor = numpy_helper.to_array(initializer) + if initializer.data_type == TensorProto.FLOAT: + dtype = np.float32 + elif initializer.data_type == TensorProto.INT32: + dtype = np.int32 + elif initializer.data_type == TensorProto.INT64: + dtype = np.int64 + else: + print("data type not supported by this tool:", dtype) + + if len(tensor.shape) == 1 and tensor.shape[0] == 1: + if tensor == old_parameters["num_heads"]: + print("initializer type={}".format(initializer.data_type), initializer.name, + old_parameters["num_heads"], "=>[", new_parameters["num_heads"], "]") + initializer.CopyFrom( + numpy_helper.from_array(np.asarray([new_parameters["num_heads"]], dtype=dtype), + initializer.name)) + elif tensor == old_parameters["seq_len"]: + print("initializer type={}".format(initializer.data_type), initializer.name, + old_parameters["seq_len"], "=>[", new_parameters["seq_len"], "]") + initializer.CopyFrom( + numpy_helper.from_array(np.asarray([new_parameters["seq_len"]], dtype=dtype), initializer.name)) + elif tensor == old_parameters["size_per_head"]: + print("initializer type={}".format(initializer.data_type), initializer.name, + old_parameters["size_per_head"], "=>[", new_parameters["size_per_head"], "]") + initializer.CopyFrom( + numpy_helper.from_array(np.asarray([new_parameters["size_per_head"]], dtype=dtype), + initializer.name)) + elif tensor == old_parameters["hidden_size"]: + print("initializer type={}".format(initializer.data_type), initializer.name, + old_parameters["hidden_size"], "=>[", new_parameters["hidden_size"], "]") + initializer.CopyFrom( + numpy_helper.from_array(np.asarray([new_parameters["hidden_size"]], dtype=dtype), + initializer.name)) + elif tensor == 4 * old_parameters["hidden_size"]: + print("initializer type={}".format(initializer.data_type), initializer.name, + 4 * old_parameters["hidden_size"], "=>[", 4 * new_parameters["hidden_size"], "]") + initializer.CopyFrom( + numpy_helper.from_array(np.asarray([4 * new_parameters["hidden_size"]], dtype=dtype), + initializer.name)) + elif tensor == 3 * old_parameters["hidden_size"]: + print("initializer type={}".format(initializer.data_type), initializer.name, + 3 * old_parameters["hidden_size"], "=>[", 3 * new_parameters["hidden_size"], "]") + initializer.CopyFrom( + numpy_helper.from_array(np.asarray([3 * new_parameters["hidden_size"]], dtype=dtype), + initializer.name)) + elif len(tensor.shape) == 0: + if tensor == old_parameters["num_heads"]: + print("initializer type={}".format(initializer.data_type), initializer.name, + old_parameters["num_heads"], "=>", new_parameters["num_heads"]) + initializer.CopyFrom( + numpy_helper.from_array(np.asarray(new_parameters["num_heads"], dtype=dtype), initializer.name)) + elif tensor == old_parameters["seq_len"]: + print("initializer type={}".format(initializer.data_type), initializer.name, + old_parameters["seq_len"], "=>", new_parameters["seq_len"]) + initializer.CopyFrom( + numpy_helper.from_array(np.asarray(new_parameters["seq_len"], dtype=dtype), initializer.name)) + elif tensor == old_parameters["size_per_head"]: + print("initializer type={}".format(initializer.data_type), initializer.name, + old_parameters["size_per_head"], "=>", new_parameters["size_per_head"]) + initializer.CopyFrom( + numpy_helper.from_array(np.asarray(new_parameters["size_per_head"], dtype=dtype), + initializer.name)) + elif tensor == old_parameters["hidden_size"]: + print("initializer type={}".format(initializer.data_type), initializer.name, + old_parameters["hidden_size"], "=>", new_parameters["hidden_size"]) + initializer.CopyFrom( + numpy_helper.from_array(np.asarray(new_parameters["hidden_size"], dtype=dtype), + initializer.name)) + elif tensor == 4 * old_parameters["hidden_size"]: + print("initializer type={}".format(initializer.data_type), initializer.name, + 4 * old_parameters["hidden_size"], "=>", 4 * new_parameters["hidden_size"]) + initializer.CopyFrom( + numpy_helper.from_array(np.asarray(4 * new_parameters["hidden_size"], dtype=dtype), + initializer.name)) + elif tensor == 3 * old_parameters["hidden_size"]: + print("initializer type={}".format(initializer.data_type), initializer.name, + 3 * old_parameters["hidden_size"], "=>", 3 * new_parameters["hidden_size"]) + initializer.CopyFrom( + numpy_helper.from_array(np.asarray(3 * new_parameters["hidden_size"], dtype=dtype), + initializer.name)) + elif tensor == 1.0 / np.sqrt(old_parameters["size_per_head"]): + print("initializer type={}".format(initializer.data_type), initializer.name, + 1.0 / np.sqrt(old_parameters["size_per_head"]), "=>", + 1.0 / np.sqrt(new_parameters["size_per_head"])) + initializer.CopyFrom( + numpy_helper.from_array(np.asarray(1.0 / np.sqrt(new_parameters["size_per_head"]), dtype=dtype), + initializer.name)) + elif tensor == np.sqrt(old_parameters["size_per_head"]): + print("initializer type={}".format(initializer.data_type), initializer.name, + np.sqrt(old_parameters["size_per_head"]), "=>", np.sqrt(new_parameters["size_per_head"])) + initializer.CopyFrom( + numpy_helper.from_array(np.asarray(np.sqrt(new_parameters["size_per_head"]), dtype=dtype), + initializer.name)) + + new_shape = [] + shape_changed = False + for dim in tensor.shape: + if (dim == old_parameters["hidden_size"]): + new_shape.append(new_parameters["hidden_size"]) + shape_changed = True + elif (dim == 4 * old_parameters["hidden_size"]): + new_shape.append(4 * new_parameters["hidden_size"]) + shape_changed = True + elif (dim == 3 * old_parameters["hidden_size"]): + new_shape.append(3 * new_parameters["hidden_size"]) + shape_changed = True + elif (dim in old_parameters["word_dict_size"]): + new_shape.append(new_parameters["word_dict_size"]) + shape_changed = True + elif (dim == old_parameters["max_word_position"]): + new_shape.append(new_parameters["max_word_position"]) + shape_changed = True + else: + new_shape.append(dim) + if shape_changed: + reshapes[initializer.name] = new_shape + print("initializer", initializer.name, tensor.shape, "=>", new_shape) + + for initializer_name in reshapes: + self.replace_input_of_all_nodes(initializer_name, initializer_name + '_resize') + tensor = self.resize_weight(initializer_name, reshapes[initializer_name]) + self.model.graph.initializer.extend([tensor]) + + # Add node name, replace split node attribute. + nodes_to_add = [] + nodes_to_remove = [] + for i, node in enumerate(graph.node): + if node.op_type == "Split": + nodes_to_add.append( + onnx.helper.make_node('Split', + node.input, + node.output, + name="Split_{}".format(i), + axis=2, + split=[ + new_parameters["hidden_size"], new_parameters["hidden_size"], + new_parameters["hidden_size"] + ])) + nodes_to_remove.append(node) + print("update split", + [new_parameters["hidden_size"], new_parameters["hidden_size"], new_parameters["hidden_size"]]) + if node.op_type == "Constant": + for att in node.attribute: + if att.name == 'value': + if numpy_helper.to_array(att.t) == old_parameters["num_heads"]: + nodes_to_add.append( + onnx.helper.make_node('Constant', + inputs=node.input, + outputs=node.output, + value=onnx.helper.make_tensor(name=att.t.name, + data_type=TensorProto.INT64, + dims=[], + vals=[new_parameters["num_heads"] + ]))) + print("constant", att.t.name, old_parameters["num_heads"], "=>", + new_parameters["num_heads"]) + if numpy_helper.to_array(att.t) == np.sqrt(old_parameters["size_per_head"]): + nodes_to_add.append( + onnx.helper.make_node('Constant', + inputs=node.input, + outputs=node.output, + value=onnx.helper.make_tensor( + name=att.t.name, + data_type=TensorProto.FLOAT, + dims=[], + vals=[np.sqrt(new_parameters["size_per_head"])]))) + print("constant", att.t.name, np.sqrt(old_parameters["size_per_head"]), "=>", + np.sqrt(new_parameters["size_per_head"])) + else: + node.name = node.op_type + "_" + str(i) + for node in nodes_to_remove: + graph.node.remove(node) + graph.node.extend(nodes_to_add) + + def remove_past_outputs(self): + keep_output_names = [self.model.graph.output[0].name] # remove past state outputs which is not needed. + print(f"Prune graph to keep the first output and drop past state outputs:{keep_output_names}") + self.prune_graph(keep_output_names) + + +def generate_test_data(onnx_file, + output_path, + batch_size, + sequence_length, + use_cpu=True, + input_tensor_only=False, + dictionary_size=DICT_SIZE, + test_cases=1, + output_optimized_model=False): + + input_data_type = np.int64 + for test_case in range(test_cases): + input_1 = np.random.randint(dictionary_size, size=(batch_size, sequence_length), dtype=input_data_type) + tensor_1 = numpy_helper.from_array(input_1, 'input_ids') + + path = os.path.join(output_path, 'test_data_set_' + str(test_case)) + try: + os.mkdir(path) + except OSError: + print("Creation of the directory %s failed" % path) + else: + print("Successfully created the directory %s " % path) + + if input_tensor_only: + return + + sess_options = onnxruntime.SessionOptions() + sess_options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_DISABLE_ALL + sess = onnxruntime.InferenceSession(onnx_file, sess_options, providers=['CPUExecutionProvider']) + + input1_name = sess.get_inputs()[0].name + output_names = [output.name for output in sess.get_outputs()] + inputs = {input1_name: input_1} + result = sess.run(output_names, inputs) + + with open(os.path.join(path, 'input_{}.pb'.format(0)), 'wb') as f: + f.write(tensor_1.SerializeToString()) + + for i, output_name in enumerate(output_names): + if i == 0: + tensor_result = numpy_helper.from_array( + np.asarray(result[i]).reshape((batch_size, sequence_length, new_parameters["hidden_size"])), + output_names[i]) + with open(os.path.join(path, 'output_{}.pb'.format(i)), 'wb') as f: + f.write(tensor_result.SerializeToString()) + else: + tensor_result = numpy_helper.from_array( + np.asarray(result[i]).reshape( + (2, batch_size, new_parameters["num_heads"], sequence_length, new_parameters["size_per_head"])), + output_names[i]) + with open(os.path.join(path, 'output_{}.pb'.format(i)), 'wb') as f: + f.write(tensor_result.SerializeToString()) + + start_time = timeit.default_timer() + + sess_options.graph_optimization_level = onnxruntime.GraphOptimizationLevel.ORT_ENABLE_EXTENDED + + if output_optimized_model: + path_prefix = onnx_file[:-5] #remove .onnx suffix + if use_cpu: + sess_options.optimized_model_filepath = path_prefix + "_optimized_cpu.onnx" + else: + sess_options.optimized_model_filepath = path_prefix + "_optimized_gpu.onnx" + + session = onnxruntime.InferenceSession(onnx_file, sess_options) + if use_cpu: + session.set_providers(['CPUExecutionProvider']) # use cpu + else: + if 'CUDAExecutionProvider' not in session.get_providers(): + print("Warning: GPU not found") + continue + outputs = session.run(None, inputs) + evalTime = timeit.default_timer() - start_time + if not np.allclose(outputs[0], result[0], rtol=1e-04, atol=1e-05): + print("Error: not same result after optimization. use_cpu={}, no_opt_output={}, opt_output={}".format( + use_cpu, result[0].tolist(), outputs[0].tolist())) + print("** Evaluation done in total {} secs".format(evalTime)) + + +def main(): + parser = argparse.ArgumentParser() + parser.add_argument('--input', required=True, type=str) + parser.add_argument('--output', required=True, type=str) + parser.add_argument('--float16', required=False, action='store_true') + parser.set_defaults(float16=False) + parser.add_argument('--no_past_outputs', required=False, action='store_true') + parser.set_defaults(no_past_outputs=False) + parser.add_argument('--output_optimized_model', required=False, action='store_true') + parser.set_defaults(output_optimized_model=False) + args = parser.parse_args() + + model = ModelProto() + with open(args.input, "rb") as f: + model.ParseFromString(f.read()) + + bert_model = TinyBertOnnxModel(model) + + if args.float16: + bert_model.convert_model_float32_to_float16() + + if args.no_past_outputs: + bert_model.remove_past_outputs() + + bert_model.update_graph() + bert_model.remove_unused_constant() + + print("opset verion", bert_model.model.opset_import[0].version) + + with open(args.output, "wb") as out: + out.write(bert_model.model.SerializeToString()) + + p = Path(args.output) + data_path = p.parent + + batch_size = 1 + sequence_length = SEQ_LEN + + generate_test_data(args.output, + data_path, + batch_size, + sequence_length, + use_cpu=not args.float16, + output_optimized_model=args.output_optimized_model) + + +if __name__ == "__main__": + main() diff --git a/onnxruntime/python/tools/bert/test_data/gpt2_pytorch1.4_opset11_no_past/test_data_set_0/input_0.pb b/onnxruntime/python/tools/bert/test_data/gpt2_pytorch1.4_opset11_no_past/test_data_set_0/input_0.pb new file mode 100644 index 0000000000000000000000000000000000000000..42d734d9aede4c5635b60664f8692f5505c01c30 GIT binary patch literal 35 gcmd;JÛÑ>&ÏIÀ¢?½;g>v,²>©3CÀYŽ²? \ No newline at end of file