From cb5c4292b8938244cc59843f24eb2f57e73f4940 Mon Sep 17 00:00:00 2001 From: Tianlei Wu Date: Wed, 8 Jul 2020 15:59:57 -0700 Subject: [PATCH] GPT-2 Attention Fusion without input mask (#4456) * Allow input mask to be optional * Add test for model without input mask and past state. --- .../core/optimizer/attention_fusion.cc | 2 +- .../core/optimizer/attention_fusion_helper.h | 80 +++++++++++++----- .../test/optimizer/graph_transform_test.cc | 36 ++++++++ .../transform/fusion/gpt2_one_layer.onnx | Bin 0 -> 8815 bytes .../transform/fusion/gpt2_past_one_layer.onnx | Bin 0 -> 10143 bytes 5 files changed, 98 insertions(+), 20 deletions(-) create mode 100644 onnxruntime/test/testdata/transform/fusion/gpt2_one_layer.onnx create mode 100644 onnxruntime/test/testdata/transform/fusion/gpt2_past_one_layer.onnx diff --git a/onnxruntime/core/optimizer/attention_fusion.cc b/onnxruntime/core/optimizer/attention_fusion.cc index fb8d17b43d..fa97ffbf29 100644 --- a/onnxruntime/core/optimizer/attention_fusion.cc +++ b/onnxruntime/core/optimizer/attention_fusion.cc @@ -416,7 +416,7 @@ bool AttentionFusion::FuseSubGraph(Node& layer_norm, const Node& add_after_layer // Find mask nodes: Unsqueeze -> Unsqueeze -> (Cast) -> Sub -> Mul -> Add -> Softmax --> [MatMul] // The "Cast" node in parentheses is optional. AttentionFusionHelper::AttentionMaskNodes mask_nodes; - if (!AttentionFusionHelper::MatchInputMaskSubgraph(graph, qkv_matmul, mask_nodes, logger)) { + if (!AttentionFusionHelper::MatchInputMaskSubgraph(graph, qkv_matmul, mask_nodes, logger, false)) { DEBUG_LOG("Failed in match input mask subgraph"); return false; } diff --git a/onnxruntime/core/optimizer/attention_fusion_helper.h b/onnxruntime/core/optimizer/attention_fusion_helper.h index b588efcc10..75b6cc5d16 100644 --- a/onnxruntime/core/optimizer/attention_fusion_helper.h +++ b/onnxruntime/core/optimizer/attention_fusion_helper.h @@ -434,16 +434,22 @@ bool MatchUnidirMaskSubgraph(const Graph& graph, const Node& add_node, MatchUnid struct AttentionMaskNodes { const Node* softmax; + bool has_input_mask; // When it is false, the following nodes will be NULL. + const Node* add; const Node* mul; const Node* sub; - const Node* cast; + const Node* cast; // optional, could be NULL. const Node* unsqueeze_2; const Node* unsqueeze_1; }; void SetMaskNodesToRemove(const Graph& graph, AttentionMaskNodes& mask_nodes, std::vector& nodes_to_remove) { nodes_to_remove.push_back(mask_nodes.softmax->Index()); + if (!mask_nodes.has_input_mask) { + return; + } + nodes_to_remove.push_back(mask_nodes.add->Index()); // When the last Attention node is fused. Original mask processing nodes can be removed safely. @@ -463,25 +469,54 @@ void SetMaskNodesToRemove(const Graph& graph, AttentionMaskNodes& mask_nodes, st | (optional) v [Attention_mask] --> Unsqueeze (axes=1) --> Unsqueeze (axes=2) --> Cast ---->Sub(1,*) --> Mul(*, -10000.0) --> Add( ,*)--->SoftMax -->[MatMul] + +When is_input_mask_optional is true, this function also matches the following subgraph: + {UnidirMask Subgraph [Where]} --> Softmax --> [MatMul] +In this case, we only match two nodes: "Softmax" and "Where". Note that "Where" is the last node in unidirectional subgraph. */ -bool MatchInputMaskSubgraph(const Graph& graph, const Node& qkv_matmul, AttentionMaskNodes& result, const logging::Logger& logger) { +bool MatchInputMaskSubgraph(const Graph& graph, const Node& qkv_matmul, AttentionMaskNodes& result, const logging::Logger& logger, bool is_input_mask_optional) { DEBUG_LOG("Start MatchInputMaskSubgraph"); - std::vector mask_path{ - {0, 0, "Softmax", {1, 11, 13}, kOnnxDomain}, - {0, 0, "Add", {7, 13}, kOnnxDomain}, - {0, 1, "Mul", {7, 13}, kOnnxDomain}, - {0, 0, "Sub", {7, 13}, kOnnxDomain}}; + + std::vector softmax_path{ + {0, 0, "Softmax", {1, 11, 13}, kOnnxDomain}}; std::vector edges; - if (!graph_utils::FindPath(qkv_matmul, true, mask_path, edges, logger)) { - DEBUG_LOG("Failed to find path for mask"); + if (!graph_utils::FindPath(qkv_matmul, true, softmax_path, edges, logger)) { + DEBUG_LOG("Failed to find Softmax node"); return false; } const Node& softmax = edges[0]->GetNode(); - const Node& mask_add = edges[1]->GetNode(); - const Node& mask_mul = edges[2]->GetNode(); - const Node& mask_sub = edges[3]->GetNode(); + if (!optimizer_utils::CheckOutputEdges(graph, softmax, 1)) { + DEBUG_LOG("Output edge count not expected for Softmax"); + return false; + } + + result.softmax = &softmax; + result.has_input_mask = false; + + // GPT-2 might not have input mask. In that case the subgraph is like: + // {UnidirMask Subgraph} --> Softmax --> [MatMul] + if (is_input_mask_optional) { + const Node* parent = graph_utils::GetInputNode(softmax, 0); + if (parent != nullptr && parent->OpType() == "Where") { // UnidirMask Subgraph ends withs Where node + return true; + } + } + + std::vector mask_path{ + {0, 0, "Add", {7, 13}, kOnnxDomain}, + {0, 1, "Mul", {7, 13}, kOnnxDomain}, + {0, 0, "Sub", {7, 13}, kOnnxDomain}}; + + if (!graph_utils::FindPath(softmax, true, mask_path, edges, logger)) { + DEBUG_LOG("Failed to find path for mask"); + return false; + } + + const Node& mask_add = edges[0]->GetNode(); + const Node& mask_mul = edges[1]->GetNode(); + const Node& mask_sub = edges[2]->GetNode(); // Match optional mask cast node Node* p_mask_cast = nullptr; @@ -547,7 +582,7 @@ bool MatchInputMaskSubgraph(const Graph& graph, const Node& qkv_matmul, Attentio return false; } - result.softmax = &softmax; + result.has_input_mask = true; result.add = &mask_add; result.mul = &mask_mul; result.sub = &mask_sub; @@ -999,13 +1034,13 @@ bool FuseGptAttention(Node& layer_norm, Graph& graph, int64_t hidden_size, std:: // Find input mask. Unsqueeze -> Unsqueeze -> (Cast) -> Sub -> Mul -> Add -> Softmax AttentionMaskNodes mask_nodes; - if (!MatchInputMaskSubgraph(graph, qkv_matmul, mask_nodes, logger)) { + if (!MatchInputMaskSubgraph(graph, qkv_matmul, mask_nodes, logger, true)) { DEBUG_LOG("MatchInputMaskSubgraph returns false"); return false; } MatchUnidirMaskResult unidir_mask_result; - if (!MatchUnidirMaskSubgraph(graph, *(mask_nodes.add), unidir_mask_result, logger)) { + if (!MatchUnidirMaskSubgraph(graph, *(mask_nodes.has_input_mask ? mask_nodes.add : mask_nodes.softmax), unidir_mask_result, logger)) { DEBUG_LOG("MatchUnidirMaskSubgraph returns NULL"); return false; } @@ -1076,16 +1111,23 @@ bool FuseGptAttention(Node& layer_norm, Graph& graph, int64_t hidden_size, std:: } // Now everything is ready, we will start fusing subgraph. - NodeArg* mask_input = graph.GetNode(mask_nodes.unsqueeze_1->Index())->MutableInputDefs()[0]; - NodeArg* mask_int32 = GetOrCreateMaskInt32(graph, mask_input, mask_int32_map, layer_norm.GetExecutionProviderType()); - NodeArg* qkv_weights = graph.GetNode(gemm0_result.gemm->Index())->MutableInputDefs()[1]; NodeArg* qkv_bias = graph.GetNode(gemm0_result.gemm->Index())->MutableInputDefs()[2]; // Create Attention Node. - std::vector input_defs{layer_norm.MutableOutputDefs()[0], qkv_weights, qkv_bias, mask_int32}; + std::vector input_defs{layer_norm.MutableOutputDefs()[0], qkv_weights, qkv_bias}; std::vector output_defs{graph.GetNode(reshape.Index())->MutableOutputDefs()[0]}; + if (mask_nodes.has_input_mask){ + NodeArg* mask_input = graph.GetNode(mask_nodes.unsqueeze_1->Index())->MutableInputDefs()[0]; + NodeArg* mask_int32 = GetOrCreateMaskInt32(graph, mask_input, mask_int32_map, layer_norm.GetExecutionProviderType()); + input_defs.push_back(mask_int32); + } else { + // Add a missing optional input for mask. + std::string empty_name; + input_defs.push_back(&graph.GetOrCreateNodeArg(empty_name, nullptr)); + } + if (has_past) { input_defs.push_back(past_result.past); output_defs.push_back(past_result.present); diff --git a/onnxruntime/test/optimizer/graph_transform_test.cc b/onnxruntime/test/optimizer/graph_transform_test.cc index 6ab3de3888..fd1f6e83b5 100644 --- a/onnxruntime/test/optimizer/graph_transform_test.cc +++ b/onnxruntime/test/optimizer/graph_transform_test.cc @@ -1736,6 +1736,42 @@ TEST_F(GraphTransformationTests, AttentionFusionGPTWithPastAndMaskTest) { EXPECT_EQ(op_to_count["Attention"], 1); } +// Test GPT-2 Attention Fusion without input mask +TEST_F(GraphTransformationTests, AttentionFusionGPTWithPastNoMaskTest) { + auto model_uri = MODEL_FOLDER "fusion/gpt2_past_one_layer.onnx"; + std::shared_ptr p_model; + ASSERT_STATUS_OK(Model::Load(model_uri, p_model, nullptr, *logger_)); + Graph& graph = p_model->MainGraph(); + + onnxruntime::GraphTransformerManager graph_transformation_mgr{5}; + graph_transformation_mgr.Register(onnxruntime::make_unique(), TransformerLevel::Level2); + auto ret = graph_transformation_mgr.ApplyTransformers(graph, TransformerLevel::Level2, *logger_); + ASSERT_TRUE(ret.IsOK()); + + std::map op_to_count = CountOpsInGraph(graph); + EXPECT_EQ(op_to_count["Transpose"], 0); + EXPECT_EQ(op_to_count["Softmax"], 0); + EXPECT_EQ(op_to_count["Attention"], 1); +} + +// Test GPT-2 Attention Fusion without input mask and past state +TEST_F(GraphTransformationTests, AttentionFusionGPTTest) { + auto model_uri = MODEL_FOLDER "fusion/gpt2_one_layer.onnx"; + std::shared_ptr p_model; + ASSERT_STATUS_OK(Model::Load(model_uri, p_model, nullptr, *logger_)); + Graph& graph = p_model->MainGraph(); + + onnxruntime::GraphTransformerManager graph_transformation_mgr{5}; + graph_transformation_mgr.Register(onnxruntime::make_unique(), TransformerLevel::Level2); + auto ret = graph_transformation_mgr.ApplyTransformers(graph, TransformerLevel::Level2, *logger_); + ASSERT_TRUE(ret.IsOK()); + + std::map op_to_count = CountOpsInGraph(graph); + EXPECT_EQ(op_to_count["Transpose"], 0); + EXPECT_EQ(op_to_count["Softmax"], 0); + EXPECT_EQ(op_to_count["Attention"], 1); +} + TEST_F(GraphTransformationTests, GeluFusionTest) { auto model_uri = MODEL_FOLDER "fusion/gelu.onnx"; std::shared_ptr p_model; diff --git a/onnxruntime/test/testdata/transform/fusion/gpt2_one_layer.onnx b/onnxruntime/test/testdata/transform/fusion/gpt2_one_layer.onnx new file mode 100644 index 0000000000000000000000000000000000000000..850ae1f392f112f47c02a18293687d60096aeea3 GIT binary patch literal 8815 zcmcIqdw5e-wkPQ$>4BC~DUa~56Qn|blJiJ%LbFewT5jdxP((j?AJG9bPXMuZWaxq8<+$vHV`8}9t!ZNKm2 zWbd{2dj8g0d$-o?0!!PXXluBk(U!rMxF+qdVv3lYP)l2T)E}B1Dac?wZkwg9G0+zD zJMCHO3sb`2Fg`p;Alj@ofoNke>}T!S@i#|bCNr=w6dBTIlgZ3XL{bq*y}YgO4K0y* z?ZM#Upr5nn=wIRsf{}7qV1WghyugXD%**34!Dg*$ZD|NZ{k%Op{+7z2SY4osB&x3{ znCP`xW&~R12K|CPOMOu#5xuyj(_0cSgfe zCbW?kt>YDK*%h;A`&p+w13sCpCY%wOL5(HOlBO0v+fAQ=`gnaP5J_lRkioGoTj7+z zqF{JhYq&Yk6j~gJhFW1cYcJ?I>=?jUf^CsdQ)^4%Gp}Fm*<>;~1|~w~C+)`sF~iyB zv`2!oi}WpuBJK5a!-2NOND(uPLAJ=-YqQ2nKmcct(LD|0MZnPbHye|LKRETDtqT`BxMY5BWgcV2zC zG1z%jlm|u$jId(wAkoF6SaBSx*c562xFY`XjTw)VZk+Vk zm^uy3tlg4$PX)7TU$h19H0sBthrd&WF#0gA8u&1w8kYwDR{<@nT10guXJa&&azYY5 z8z96vo?}K}CGx5dIpUDa|wKWC``}FB3 z!;sEnGV6oUo*g*wzWpUdtQ5v{taN5*%k~pm-i_*muHj%;4I7-#sZ!U$2qtm z+i`AJk^}Ednh`m;5q@p}jJ~IzXVY;om1iIaJFdgUUIw;K6eqe1McZ2lXi) z!90Rgi(nYXNwA_8!8}+yT?7LmPzxy$%%?{v#C&`O1dAc3@M2m_>#9M<&{Q*mlV_nE z@|?Oc!~oBtJSvb?a0*6n!Z=RyN!5gO;6!_KfHa_nC4kWEy7qd%z}Yk4lR<6>L7wM~ zY6yH9h8A_7cAV2HT82*ryRQAHKgZtQ7PNMJ;Y=zI}UvX-vg;GneWN<>?TqS3zY z^bJiMVwhmCwD3?`bk)Fzy*1nzHy%}H)fUcV#JdeO3kxm}?X$7qwuM`7>(*Kt3$8kC z$Q&S!mrWF0x94XoxUeaTrbYpvT!X9IP)OBw8Et~?GOxAEZf~+(hC#hu=DO*mwaZ*d ziZX)J+iSb*asI!y%fPlELx9ATK(9v+LqHFV;{)PLz@nUJmpxFF)1U{)fHt+A^Q5l{ zfZhau9R?m__7GTrfxtrW3oMu9pFVp48PG!MHe?S^`s@Md1!8#cYMcwm2nw1O;vP7c z^%mkuvGp$BBO1^vu#j{GEXx9(Hw0`<1P)bH(d)tLt)Umjagt4Qt9eRNfsZ+9} z2;Ehh>=f4uZOEYyIq+d3F^S;A-c$qhf=6%xIucrZ9RQ*s>#qa!!X#tEez>`*4Z7ty z4c)?gNXlGlsKe9-t=J)b1DE zcB}S<8yY`c)?7c6!%4&0a#C(XYja6+s3F`Mfy$2^C+q{8x%Ai-jg<$+fx?CUawTI| zOka6`3~1dRW9*9QE04|}dXWTnz4|>2Viy|&L4d3+K;_r4>rHbJgR%;k39xh5qsi}0 ze-uOPV9yQ`W97ppOK>O4gE!52Oc&dwryQ>id8mAX3sa5@cX#3LE(}~3s;FYet9N!# z`CuF;#UysTmub|25JhBD^t)W6j#@zlh9$TznHIchPSJ=RLA8kLNJ9{M4ZYra#EzS1 zh9~jSOW`vht7qL4+|UWA2Tj)L{CNO08ij&Z30w%udf*OBD}1aIP6&xF%m^%cnAbd* zwLDOM;}=D6D!`kfX>f$&y#?BFu^@gfEzsIvaAI3iDC*}BC-|oK%Vy}89r;XF1V_s= z@fXg81^Mdz34Y<+wbnj~AAQVIYc*+O1Ffb6rC4j#u8$na zQEzXfytCHYug4C&P-`{!9CaBa(D&s*t_+PfFW-c+@}AnvKIOyrCAssx%EW_Anc7-? zM*(E&DZjwhTJw9-a=p^>hJD9bGr=x+QBYfySg(I*O6u$eASqns8tkeL!3CB<`S=hA za_ZKf$C#e9B1MJzCD<@fOB_Epa$ssC7HOA{<_PN2xd52{|mva zMZR137l=2TttR+|pL~pbB_5<@cGtOO2bN_*3iu=Fx>A#eXcL^$#7E zf)D&d{(j!OQr7O@lCO`AlU_V^XX#DV>&U4?G3n!-8)!8*RH?`SkWhKIP!ydTI8>a^vgACOiZ+eaD~ z9wf6y)yT?;g7~<{E1jR7Sulj8Jc=WNu2sG z>t{>;x=GTU5hJA5$<0S-{r4@GXP#-3r{>iv0}uU^ zG|dadn%_TH+Wz23ntgAXwDQLPpl@9()2mHOeHd0a;Yb^mI(G~oDt^7uWI=zsfie5((* z=)b+afpjf3DSv2MO{abTy>H**1j`l77mgI-q$eM3D zN#WQU`c3XyO3zl2Ik&Cz4Rc*ba`z6QZ9iT^w?DTzR$u&2LdWMyrGFnIUAMMazHqOZ z-nr@!y;Rpm^701l_T2LX?aX_JzFxGAT7JJiHf5b8pPKRuX~~^;O9PG!R@z6cmfkP- zD4CBhSDwf^NY@^(Q{KG)C|x!Ed3pNIL3DafRQfo3t#o4dZf%f?^~M+J=~$L*T+<`Oe{Z(ttIImf^JjiYnvzqLA z;}C6}b(ETpz2l2MJAzcLVCcoCu8;ZjNPUr?I6g{)fswy)x~Z{%tF?jsYs3Kiwkqw=z% zljM)5FOlkJUnZ^lH_DGSJsFe9&2r6|2gn;+A0d-BJWVD&!jtp6E|80Rw#gYsKcm9p zeWf#mSu|8%K_4vJPwu<<19?yIEUmokF~QM^H43vqhv|z6zmaWYYL)*wzr1w9=U(6X zYazyNxRuJ&_tIm_>g4Z>*OczuxSjU9&QGc?ZH#r@zhT#r(GSzAkw?ib?>Z(mRQ*s| zxaN@j=RRiXg|FU<9hou0H{l>dj|Z|P8W2{ zA3Hh!Z{*=VSHvoN{-xXTk=Z&>6|c+u+#M(A=p#jxK58L{a&n~8rK8DxMK$!ARzj?= zzDaLdX;vP-xta`K>67+;`4av36G|7)>{Je4bCXo?-Z*-9XgZy?{3JcU?I=0><6E@- zoX590W>dC*T`B!~;Y{C-y-(9I8_eY1ytTys_6Vh{FiZLMgEuKlp3Rn4uYG|$Gkr7N z`p{2wThkm8{c6Y!sX?b&z(q46tl>Egao%r!ldXe2qZ=9Dc{pZ*M@!T zCm4vfhv7dL!Vw_IW0*jwq_w4GA>91rn=1!0`P%nrI1p+Hwaf*o;mi4I*XNF#(r4A5YJHnk;HbZ&LipfD|q1Bia=eEHH%u%o?%B&~p` zjff*WT%Mrfh)+gFM{yV2=()%UA+ z)ECt|H#uC)8BAHcJ<*wt#22PKMXV%w-Sb+a30NukO7u78EC$E%Ap?aR$nhc;<^}i$ zGEWi5^4I?|Eg*PHrz~6;;W=Lsd@|Mg1U>{^38=S# z>~+tLwqFWo^1c%N4OE2VFv$k2O63JxrH+9WEG}aOTqb&%xv`XPTVC|JvhU2Vm=S4o zmqi`Pwpg;}%0#T@%2@o;mUJW;OU0MPa6R}S$L93Q2Mg=hZ}BCR8IEOT>#-sGW7B<( zg{63G&h>d4vx0ts3?E=yj+L_dl{5P7ao~)^_^eX4)C6i;+aqi*QAZV|X^KZvS!KZf zyy&f(8C@Dn&IW}=TjR^3>3BzbL=!Z|!KWdgP^Xp-m2le^3^-KO6rX zfM3qLs52E?ILh2&RI0P-(quHzk{ZQ;-@>tYHuyybodRq4Eua&|;US>s2&{<$!6s6m zM+hv?ib1O*2@ol4(3v|wXr6D44jhl#=TZB-lsn%(Ish4-FXo+ZA00&4UDO{z9U=%L z5e~|R2*D;qm~0COVcqcs$xs`>of9~G2>8I=1>9Fa5p}o=7#d8t3lL*CA(Vr=7_fo6 zu3yxJm|oP(tIF@yw^27*47pg1z$0~_)lRwi zoGI1!pQ7h1zwu{7CWWC;B99NY&eli}{3P?rKs1CI ze1H$~0z|`U@hc)h-d6;l7)oFaCIEa{Oax6R114Y`A1J?wnh>$15HalNm&^z~X$6^_I@-Y>I>+yTd1DZ5YFeyizD+A8cnaeX@{_ zQw#WTPrwPNMh}N*tRUJc7&3)Ho9eL(#SMmZI<#kq)n8E)Arvtrm`hC40uEsuCq=Jo zDi)%lpsyHT{AG+Y-7%yL3r?t8fB+tXgj-_0$A^G{3!qAkMM5E8iT;MQ5{%=d1lsIa zlx~YIiG(C@l+3$X)`5pR^Gjd!Q%HG6=^Fw77Ak!KcUMGPJ7aoG;Grzzc__<_T1U#q z9n2vCIF*UCi1dHqPLBzwGal7~IW`*;q+EP9V!}_>w2p`c!BI=-1R)S6i&%*_5sP3A zA(W)NNF_mKXYyDm8lasda4-SuN&)K1oXnPZ35bU|BruC!DQINDvv3mF6EIQ=5Gf5^ zOOR6w=^Bs2z$kEe$y+)hEVJjE5l1p%shMQ$OigKcqp$=lxu z0Bpb#0Y*Ruh06d^AeroS;a!Xku$jxk3%by9!OH^IOWHnR8ZH!EO}>~=J`6TX`rKuK z4RHT|w*@Ew1gsAP0dCP5&t(Iq>!O2U$twuxlTbo1*~S7J5DG&MT??yU0diZpnR`KJ zqHQY9u>hgcV8IMJIt%))h#3K5waJ1_)X|AC$U(7@$AW#>fuIW1_|P*R-bDq-!AutR zY#>+7<>5ZZI>d-9>nxTA7y_6Hs0+xq0@f0Os7r#5ge7~XLSk9Gndzx_5|TjJ7evn6 z-{=ysEN26K6jDTuKxt+blC`fDGNNG&649D)p)=)U!I~La%>=9Wi4Z3O@{N_9AcibB zVJ%tXj|c=P1FO@P-T>a3+gcNFb6C{e3k+jy>GA;q*EY9)X<%8XDSnAt0}6ztDp(f; zf@(1beAaRun28kV1W=<4I0tH(>9N+%2+K-7xA7M@G;;@^e%^czXDwIo$>q%*Z8dH2 z=43|-$`&lLVI$a0$~ib|W;IkVHnRi?%_&(NoU=)5Ix+GOohhsBIs3e3vH*3J;i%9Y z%sMKvfOUs}g@pixg^4lf2o}ujItv&!ppllxf_;MtF(xA#pl^mHAb6A*fTIkIanJ!M ziDqJC(&29ZOuo@3ZCjT`1!~u2#a(oD$vP3eLinT*0 z-rL`FYThPu%*M`z>@j)(o=lsu4}4BCoH2l6Ec8jJhY58suek_+-A15d@qT9bbnc0Y;Jm3Ian9a5Na&`w)p8fgI#E!Se9kChwe&W!JymHml0?z6 z#;3W`zzNTt1WtyzojOtg!j;Aj#*Y5Soo)yGhkun0Ul#zftIn7!{E71?qA}*}Xsq*v zXnN20rYSUUrLmhgXihZWRoBqxGTFQ5=a0AtfT{vTM~^E#*Ol(L^_2r!@T0lf1Ft)^ z%R64A6Si#Awr|)-j~0jN)z1#4sq3TaGa;fi*j>u|9B|CxF4oWJaWriFM%(eHRoeVn zuh6H)Y}Lf4W6G6#&Z6PSKIQ!loZ{G3uO)x1RU-Ef+A(tRbLs+SD{&ketc+_g^A0KjS=by8LJpcEoa%5s9y?kj{+r76*S$Mo&dvU`e^^WKNqD(n> zt@8PfO`4{jqAc~@rrda77in3tn=BYTUDXb)R2okH8xMaSBnpSGlKFV3^4vd1Xct!x z)~+hrO>OnoyV&xlKyA21{awR1DZl=#PMh|^E2L`8_v)PTW!m`b8?^-=eLzn;Qch~w zziKa?Hd6UR<0WLv3&Yhf4xdRj&*)Yg-fK|)uD!V3mg~NipkK;xJ^klI_mjI1h7XNf zNws_L>+b7E`nQ~xJD7iRX64`3EmR`&CM%0Zj8HmeOj8<8;`>qK)TjPu64|)9glw){ zSG#xUig3YG^I9RssoxDNRa!1tpnSJ>p0Z-irn-Ws_G+~FTiunY-m)1!GT^FhwDX=* zH04znefgG^>ijPg>a2=JZP1>7lh(!2?zVT1)^^@Ll9t|7r>wf*Kj`b{sC3*58|X-H ztJ-i>W%fJXSqXrymd95{lgF8U7M$>w@)E-`5$I> zFFfyYq6VL$&u^bc`&WNUDw7Ga=If25>dfi%>+-df9+^rOU3PPLn0PKJ-#LUPema{z z`o!k$rs{taIBrqAs@?iN5ZdZ=SL$1I zIj@-i9&JqxS3cX|B1fushUZpQ5qV;YuGs!48Qou2T}AiM?^?>jp0b_v&$CyPzeKcT|1UA1$C1vK6?h2CEGGP&iVchw!SBedb9 zZ-f2^F@Iv@>2iEF>36Dke53YX->j^i^r;+PcMkaB`Ik_2&Q5w@#XR+g>NU07HatoP zoEsrizu(Z^bL;x&_l>=iP93?QT>O@Qa`V(5Ypd4mQUB7+LM zwa59y)raWVeWNJ7+eP-2l_?+BjwQE@noiH|AjJK`YxKfZPVLT%rjgTDg_WJ3KSSUD zfYN32H)?y&zEJVJJ%L^qpF?M_JWRjYvY#CJ>2=z9G!)+4?bROrszLeflKJ7Mc0NqU zuXmE0D%KL;8zZ#3suJzDw_m7T^;oI0dhJ&7$ehje!8?AYTUr;9^yiz&(xx$V%CrUK zgYiF-yS`7--CaJa7X6tnFM5YO;XPfu=aO*v&1qA*-@dq*yya>l-o#a87~847yZus9 z@$ED86{e0{Gf7srobqXS?t6sXytI^lxt!4s_dkOSjPQ!T;s!eF<)nJsvt7hJqnzIV z=s{ZFHcIQ9x|`Je&7~ar>27+QeUM(TxK#PiGd<+p%5FO3p>NeE@1LlgS$dTGrsos- z%><6V{fSD(3%{eiRNeb`Gjsc0G|}=i9M9>@2)D~Klo{Y*nxg6EmWckFo-&Us6 zjkRAi+4!-UC&1LW9Ued9bUS;I<4A>Pfc~Sj>^h`sI-AMekUbI;gyo#g^1+@#*5wnd z+mv9tfNOA8_HQ_iIW@1-R@2_xa%D5zKu%|d