From 24d6b0f5c48a724f2d985dd3f96cf8c026e95934 Mon Sep 17 00:00:00 2001 From: Sreekanth Yalachigere <17345104+sreekanth-yalachigere@users.noreply.github.com> Date: Mon, 10 Jun 2019 20:18:56 -0700 Subject: [PATCH] MKL-DNN Subgraphs (#1116) * subgraph with memcpy fix * Linux compile errors fix * Linux compile errors fix * subgraph with memcpy fix * Linux compile errors fix * Linux compile errors fix * memcpy (PR1020) fix implemented * check graph viewer GetNode for nullptr at other plances * documents * Review changes (UseSubgraph simplified) * static_cast removed * static_cast removed 2 * fall back to CPU implementation in GetCapability() * check shape for null. fall back to CPU implementation in GetCapability() * backend data errors fixed * PR review changes * disable Opset10 tests * removed tests from main.cc of test runner. added a check at GetCapability() * backend data and Model-Zoo related fixes * subgraph with memcpy fix * Linux compile errors fix * Linux compile errors fix * subgraph with memcpy fix * Linux compile errors fix * memcpy (PR1020) fix implemented * documents * Review changes (UseSubgraph simplified) * static_cast removed * fall back to CPU implementation in GetCapability() * check shape for null. fall back to CPU implementation in GetCapability() * backend data errors fixed * PR review changes * disable Opset10 tests * removed tests from main.cc of test runner. added a check at GetCapability() * backend data and Model-Zoo related fixes * patch to run tests and models separatly --- .../MKL-DNN-ExecutionProvider.md | 35 + docs/execution_providers/MKL-DNN-Subgraphs.md | 65 ++ .../images/mkl-dnn_node.png | Bin 0 -> 51197 bytes .../images/mkl-dnn_subgraph.png | Bin 0 -> 43124 bytes .../mkldnn/mkldnn_execution_provider.cc | 339 ++++++++++ .../mkldnn/mkldnn_execution_provider.h | 92 ++- .../mkldnn/subgraph/mkldnn_activations.h | 152 +++++ .../mkldnn/subgraph/mkldnn_batchnorm.h | 362 ++++++++++ .../providers/mkldnn/subgraph/mkldnn_conv.h | 637 ++++++++++++++++++ .../mkldnn/subgraph/mkldnn_func_kernel.cc | 257 +++++++ .../mkldnn/subgraph/mkldnn_func_kernel.h | 234 +++++++ .../mkldnn/subgraph/mkldnn_kernel.cc | 57 ++ .../providers/mkldnn/subgraph/mkldnn_kernel.h | 121 ++++ .../providers/mkldnn/subgraph/mkldnn_lrn.h | 183 +++++ .../providers/mkldnn/subgraph/mkldnn_pool.h | 427 ++++++++++++ .../providers/mkldnn/subgraph/mkldnn_sum.h | 172 +++++ .../core/providers/mkldnn/subgraph/subgraph.h | 73 ++ tools/ci_build/build.py | 35 +- 18 files changed, 3239 insertions(+), 2 deletions(-) create mode 100644 docs/execution_providers/MKL-DNN-ExecutionProvider.md create mode 100644 docs/execution_providers/MKL-DNN-Subgraphs.md create mode 100644 docs/execution_providers/images/mkl-dnn_node.png create mode 100644 docs/execution_providers/images/mkl-dnn_subgraph.png create mode 100644 onnxruntime/core/providers/mkldnn/subgraph/mkldnn_activations.h create mode 100644 onnxruntime/core/providers/mkldnn/subgraph/mkldnn_batchnorm.h create mode 100644 onnxruntime/core/providers/mkldnn/subgraph/mkldnn_conv.h create mode 100644 onnxruntime/core/providers/mkldnn/subgraph/mkldnn_func_kernel.cc create mode 100644 onnxruntime/core/providers/mkldnn/subgraph/mkldnn_func_kernel.h create mode 100644 onnxruntime/core/providers/mkldnn/subgraph/mkldnn_kernel.cc create mode 100644 onnxruntime/core/providers/mkldnn/subgraph/mkldnn_kernel.h create mode 100644 onnxruntime/core/providers/mkldnn/subgraph/mkldnn_lrn.h create mode 100644 onnxruntime/core/providers/mkldnn/subgraph/mkldnn_pool.h create mode 100644 onnxruntime/core/providers/mkldnn/subgraph/mkldnn_sum.h create mode 100644 onnxruntime/core/providers/mkldnn/subgraph/subgraph.h diff --git a/docs/execution_providers/MKL-DNN-ExecutionProvider.md b/docs/execution_providers/MKL-DNN-ExecutionProvider.md new file mode 100644 index 0000000000..716ef27ed5 --- /dev/null +++ b/docs/execution_providers/MKL-DNN-ExecutionProvider.md @@ -0,0 +1,35 @@ +# MKL-DNN Execution Provider + +Intel® Math Kernel Library for Deep Neural Networks (Intel® MKL-DNN) is an open-source performance library for deep-learning applications. The library accelerates deep-learning applications and frameworks on Intel® architecture and Intel® Processor Graphics Architecture. Intel MKL-DNN contains vectorized and threaded building blocks that you can use to implement deep neural networks (DNN) with C and C++ interfaces. For more visit MKL-DNN documentation at (https://intel.github.io/mkl-dnn/) + +Intel and Microsoft have developed MKL-DNN Execution Provider (EP) for ONNX Runtime to accelerate performance of ONNX Runtime using Intel® Math Kernel Library for Deep Neural Networks (Intel® MKL-DNN) optimized primitives + +## MKL-DNN/MKLML +To build ONNX Runtime with MKL-DNN support, build it with `./build.sh --use_mkldnn` + +To build ONNX Runtime using MKL-DNN built with dependency on MKL small libraries, build it with `./build.sh --use_mkldnn --use_mklml` + +## Supported OS +* Ubuntu 16.04 +* Windows 10 +* Mac OS X + +## Supported backend +* CPU +* More to be added soon! + +## Using the nGraph execution provider +### C/C++ +The MKLDNNExecutionProvider execution provider needs to be registered with ONNX Runtime to enable in the inference session. +``` +InferenceSession session_object{so}; +session_object.RegisterExecutionProvider(std::make_unique<::onnxruntime:: MKLDNNExecutionProvider >()); +status = session_object.Load(model_file_name); +``` +The C API details are [here](https://github.com/Microsoft/onnxruntime/blob/master/docs/C_API.md#c-api). + +## Python +When using the python wheel from the ONNX Runtime built with MKL-DNN execution provider, it will be automatically prioritized over the CPU execution provider. Python APIs details are [here](https://github.com/Microsoft/onnxruntime/blob/master/docs/python/api_summary.rst#api-summary). + +## Using onnxruntime_perf_test and onnx_test_runner +You can test the performance of your ONNX Model with the MKL-DNN execution provider. Use the flag -e mkldnn in [onnxruntime_perf_test](https://github.com/Microsoft/onnxruntime/tree/master/onnxruntime/test/perftest#onnxruntime-performance-test) and [onnx_test_runner](https://github.com/Microsoft/onnxruntime/tree/master/onnxruntime/test/onnx/README.txt).. diff --git a/docs/execution_providers/MKL-DNN-Subgraphs.md b/docs/execution_providers/MKL-DNN-Subgraphs.md new file mode 100644 index 0000000000..5a82a1726f --- /dev/null +++ b/docs/execution_providers/MKL-DNN-Subgraphs.md @@ -0,0 +1,65 @@ +# Subgraph Optimization + +MKL-DNN uses blocked layout (example: nhwc with channels blocked by 16 – nChw16c) to take advantage of vector operations using AVX512. To get best performance, we avoid reorders (example. Nchw16c to nchw) and propagate blocked layout to next primitive. + +Subgraph optimization achieves this in the following steps. +1. Parses ONNX Runtime graph and creates an Internal Representation of subgraph.. +2. Subgraph Operator (MklDnnFunKernel) iterates through MKL-DNN nodes and creates a vector MKL-DNN Kernels +3. Compute Function of MklDnnFunKernel iterates and binds data to MKL-DNN primitives in the vector and submits vector for execution. + + +## Subgraph (IR) Internal Representation +MklDnnExecutionProvicer::GetCapability() parses ONNX model graph and creates IR (Internal Representation) of subgraphs of MKL-DNN operators. +Each subgraph contains a vector MklDnnNodes, inputs, outputs and attributes for all its MklDnnNodes. There can be attributes of same name. So, we prefix attribute names with Node name and its index. +Unique id for subgraph is set as an attribute. + +MklDnnNode has an index to its inputs and outputs and pointer to its parent nodes. MklDnnNode directly reads blocked memory from its parent to avoid data reordering. + +

+ + +## Subgraph Classes +Primitive like MklDnnConv, MklDnnPool, etc are derived from MklDnnKernel base class. + +The following UML diagram captures Subgraph classes. + +

+ + +## Subgraph Execution + +MklDnnExecutionProvicer::Compute() function creates MklDnnFuncKernel and call it’s Compute Function. + + +MklDnnFuncKernel::Compute function creates SubgraphPrimitve pool and add the object to a map. + +SubgraphPrimitve constructor calls the following member functions +``` +SubgraphPrimitve::CreatePrimitives() + for (auto& mklnode : mklnodes) { + if (mklnode.name == "Conv") { + kernel.reset(new MklDnnConv()); + kernels.push_back(kernel); + } else if (mklnode.name == "BatchNormalization-Relu") { + kernel.reset(new MklDnnBatchNorm()); + context_.kernels.push_back(kernel); + } else if (mklnode.name == "MaxPool") { + kernel.reset(new MklDnnPool()); + context_.kernels.push_back(kernel); + } + . + . + . +``` +In CreatePrimitives method, we iterate MklDnnNodes and creates MklDnnKernel objects and add MKL-DNN primitive to a vector. It also reads attributes. This is done only once, at first iteration. + +``` +SubgraphPrimitve::Compute() + for (auto& kernel : kernels) { + kernel->Bind(input_tensors, output_tensors); + } + stream->submit(net); +``` + +In SubgraphPrimitve::Compute() method, we iterate thru MklDnn Kernels and bind input data. Then we submit the vector of Primitives to MKL-DNN stream. + diff --git a/docs/execution_providers/images/mkl-dnn_node.png b/docs/execution_providers/images/mkl-dnn_node.png new file mode 100644 index 0000000000000000000000000000000000000000..d2863f8938143c3c4d4a01155e5a9b472d5cc979 GIT binary patch literal 51197 zcmeFZXH=8j`!0xrfE2+>6-1OS9V1;t=?IEQ2azVydkaKBf(l4)QX(%+1gX+X1f-YH zI|QT?dJhROJNW+2{AZnW=FEqgS@U7CmJ11p5Bu5szOVZ#&qvLtiqsU06huTs)XGYa zpA!)g2N4mS=OHHr??kU{2!IdgT%IdFBFgV(UIw3#Sjnl&5fK%IQ{qi8fX^>FD(Sfp z5ncLC_;W6n_YxBkQA?EaW4RZe#;cRmh1aydES?6E*Y$q3ll(b(q^PfFZU>Kx(`010 z`};% zE0QAXtLE9;83Ei}*UsL{c|}3-=8&X5v)(a60e;ap!Z|yyA5$gzpdvw%x|* z$5gZ5ExBNZTJy#+acW28S=SBUB?21V(VHB`ACq){vhhsm(5gaB`HqxVZSDPmXcbr8dJy z*5`jeO_9JZ-;t3={h1AyWijS9+L)@Htlp@@Wb6FUpFf#lJ*f|46l?e0tS7VYP8J_r zCK|`?6~NZAG!X0PMHeTc5U1JZd398hL*23u=fUdetj~=^4k9r=MGbaLs<6eU)0!EV ztDH}F&+%pz+Ub98*dl`B+g4192{=VPD&q4%o1zR5*0Jcn|LAYeoMzvxzz4-OCHya;Z}ss zHX^scg&>RHh24v(KHX#=lu1-~`-IG+Z{^qhnUa7fr26iUny$b2_2!RphYLy?qBbb% zJlDE{zQG9h6_&c+EHk#j4sn<=wE4g%uR#q-O|$B;|EG}^`wR28aIsKObhLV}IH#4{ z&o+nIH^Qmisy?cQrk26i?byF%=m?$sxLto2Nf&xKRR5t0joG#bd~0#}iTN8I>5u!0 z8WzGmXBQdS3>GeQDJXq7KO#}Y>Il7X7d9M!LtYAN#9IGS9y6F5PosNp`c8z!7dFJS zf-s&xf-UW9{ARBr>yLGD^HGbAIIMLKu62WI%{R{E=bOm&r#yrUe6DbhwH+Mp+vf#Y zP@3e(TuI02cdROHHzf_q?bjT3w_zu!QzwMqQBL)8RuZ08<@{|7+iVsuIQ0j$E^A|@ zJK-gsYo)_@pS)mMpKzkL!}&&o1+e42VHvKB_)8PZF<&uYgE-xkbl)96*+HBdnoYuv zF($t6hRf^>&{q%V<4ukf6$PbS!S3^`?G(w2ok{zlU;3TJ*}LOj`QZULeA?)K<%c7g zs&uC;a)LOp*ixs7vn}DchY@J)E6n{`N~wZ;gWk%py<8J*fwLa+{HgWWLD#bowU-HT zSGGJ!Q^z^a?uJ#JiXGbK@EhSgd{FxTV0X!T!ojPidU4A4z^EM58^OPq-BrIi)7bc8 zUQ<(-Mm3Hvr&WcW@ayz3Yrn-AH!A)=R7C$rC0R;+3U+!ds6Aee_QS~FHyi)G!amPy z*HTG0>Tvl!m>=a0hIR!t+n;b9CMQ=G`fy9>;S=Ee(k+}7`^TTOG(8#dTj4NQk0~#! zgxs~Ya?|K&m?F#CyTLJQ+jxQMUKY%+k(%Sy>-Q7F_v92Dd6w4#heX zHV?#kP6^_(Z;S3w`bux6#N&S%ftm+R!eygo=V@mz`G6WZ-A%nfMcp^mg zK)vp)O^Ry&d%)oS?rURWotYw_tV7-oe5;Y0NfvjR08M~(;nVC6*w7w}2$&z}K%l(G zuP0iiIvG{DI`wC39LUWwIn>O6ej-)wwPRWCz4z8*+^!spx^kzlBcaWFVx}>8vIHfA zk#JvK;;)>$a(a@bo&+rz2gQE!({foo@qB%8=Ug3X# zyy}O(xa$W_f3;DyJJ$8GS}`-GdbdyY7%|V(0o`h%E8SKn?2&FMH)}M6a`e^1l{Z^| znoN08wv*h1^sU0=j(@T@GJ)Bc^k z>59jq#pj}6_nIY&{Zmho32o7w!8uynnUTKg>jH>lu&EjiV{K^@+epn#`K8D6*Vva$ zs$6qm_u?Oi zd8wT=#R3zgpDBL|wDATC>@~r8!_$-F#uwMiOdD@AbgMW_G}6efo-S3U2lI>1i7;&* znieSA4Cn7jBluJc%+spBbNnztPEpef-s;p&vkGUa*S-JEP1&4QM1k;?PkTl8;AM>Pm~9Dpc{}7Rn(;=o09r*a_^g6zeFfe8@OTV#*{ch0dEFKJ zpcUNI;f2!&KDRW}7kSnr=fu3X$XS%h5ZQ{C_)tDWB1+Tk8`nZ6nVWyh@nNzK1U}vx z?4h*wAh|0$Qnr`hYg6ewuQFWi;UJ5SXbrzWdEO?{^N;Diwczy;Oh4ZdyT#>Vkz)#`wPM z6ePgRo2ISY1pkLlrgrX}wMD{`+py81k42&-`p_`hA*O!P18tPzK)7$t3(Dhl4))is z4atwG$VWWZh@u`3aK zOM6j;|Ao`a&oxIvN5fvE!cM#(ZYu+u-wC(qLo4_HWt5#1DY;(3uJ(5z%K&)eqbGi= zrSZjNrq{RXvmd6UCTGe}dwPOfPw|*?Li^$DJm+H!3+5XeH<92l3VKk4BU*z|I}vzL zMBnQ?ossTq3S7smfZgv3nuC2p?lPz%&JK0htW(*K$Cl%-kj|O)+xLcL{zBB_pHME) z|2~ojRJpG)E^M%iGb;#r#Iy*jstdmvBv&U|#X7tnz zGt^aTKRJdxZRahhSnkiR-d`K94rcWm@da4Ko-hrRjoa^RAWx1kekmH#dofJH<{w~P zdHSX8>d6m!3w8o1`5kk03fwuLCOZ<)%Er~strPLZO@^!k0lU_FpV#xeNPO=kq`q*6l>UFX}#nT$YJIHjXK`d*K7^xz19nNVWU>|ZTT z%WX&hAPQPpY0#{a?s+B)Ap&)qG=DY*a-yr}qEs8ka`A+@1Jo4f)1!h@M&q+v&uuie z?^_ZbDPd6K>6E~;3ZOtDz{f$AY6}41CW_e%fg>_HBBeH8=DOV1*^?$O((?HxWx+t4 z!%e<;>Y4^a4iVFa_kyu)geEK_4R^h+sv3$~Ck%2H!jw1oTA|=Bo9352V1-jNy{wfT zeKkZVs^^deppudakKP*72=t{Fu~bWt3_AXJpiHRH8T@@^8%a!mcjOW_UGX!EtT(Ac z#SDDY-dn_~+tqID=S7$0-dT_RQLA8a9kZLettme1m8RX5GQLOeb9VD<_s0b1@l}6J zQ+JJv(YrF+2k}V!-5RZ?zAM@FJt1w|Wvl7oGAqFe9K&(~W@%|9XEiPg0czZmwMADK@V-H&Za}|gakhD!wqZF52^ID`Jobo8c&T}d!Niq{?Q$ZD?wi8eK|n> z_9E>?dWhZFwZAJH`5a6{#D1t2hbgsurhM9jEhY)ao$G!uol9rq24RG`$A&brU0Tzp zxr=+OfmkzR1ehgzL$5Sxla}a;S_@=euiE{G519BuPEMW?R_3vDf0*S8+ZxTYi=lO= zCqA(L>-HJN4rm#OAO3KKscrfQQ{b_)(3M-)+gaeZ$BWh?4afl#Tpz&F9mtk0y+bMT z7|X6N^CrRfQ0mr8vA9{*)1|;3aAum((?PdZP_UB{4XtYTHT->DI_s*{;*N zQvc>Czfsj;QYS5aG{;bV#0z_#et^so0(?j}%uD1YV-c#R<3-rX;!4u`Sq&J_&YXIa zbwQSos{i>vcU9vm0Qv~7T=1VsCrO#Ox2Gofc7Tgjv>BOrR+e%?7DnncE z;b~u}*lPhgYmih&%Sh3xY10XFUqnJZ_ zf;&!P;96IrNDCM_c+Pl@*2IT$Gu97Zxh$r5z$?e5K#e=y+!ytHil%`b%AZbqZob1K zPjN^Oj|rz>guoyuO-1)bGD)lH917jT10oIRnTf0LF&?R~X9?@cu7|^?Yh_ zus)gVQ$l`UF_iupW@cws@UrV1CiWjTgo6?W?Ab-8{vdeG4H4_U3_)!s3@0r|Sf!}@ z7zNuczTF9h?dYHJVKv`?Dt}_KBVX>IHJ^NF$w41Uui|IsoLIjF z%6q7zkXY3;;U6?`osxB^b;*GXb-g9>Q|xt&1zJyot?~@4&8ZD+r{xLWyZg_x2|z3e z#$WO$s`bGJJc6Sq!Kh%$f0N|_I!B9>u-hsTC%f5objx6<->&C8LzXU!PY(bDuGTT} z9RyXk)5xVu>_nPzjcRSqDHeCI;UQ4F8wEQaGH}>CEeRK~>~sWo${Dx6wj=ijl`1`n zT|{i41s3*(O%7_~cj16Xxl^MbMCAGB_eVdjHLt~##i4v7%+urtGX!vq=CmVB7x(mk zZd&`CU>A>N^vB4R$iF#fhNWtGqjfz1YMTUiY>~iF>=2IXK?j|DIKTB=fdjg3W{`)( zO==YZm0nDCnP>>2^sVaO=dlN{*KN74snoi^8RfA)F$z|pdxNzxSg(AZ&Py?+5;Q)% z6i%V;dE6Y+Jj!p%wO{?j*PD%7tFP1-=T1QMCU9A7n}#81B>1LtD>G*ZT;_0^xcqe! z#c;YJjfsHWB&SBI>v)-+i=_MNCLPN|quU4*pS98z=%SYlIQh;$49fQrdSI?cbBt@m zS+iw?7;E5%eN=XC!9l25QCN)uN~;v{`w@w5|z4k z5Lhz>$Q1#t718lqVHsg=fe+(C=vZO+BHTif1K!V(`5FJMp1H%H(D4$haI|dSmwiuK z!0cuH;9tX8`UelIPVyPoJW$JYS?WnsyH85vQffC*Aync#-!2$G1*Rt9QMdyKljtEf zQ#Jm`&42HQUP-eCQ(uJL;n6Zc=T^mX{~(&dW)vPJ;_6J%FBkXTePcQH)7q537K%%u zhQ7^1?9!Jt?o(Ow-?giOSav3;-G9(3b-df32-uAv4;3x0H9||g&~g8x6l}Gq1?r8J zWio_&e7`f?RL9uP#q>U?l4mt3c87rHW0t|x^Y%x$n|UC^DNLws+j4yCh&%pJ1dJL# zp7FQMQ;yd9cq~*drc$~jc2Ww|M2uFs>h{S^XFrRkadttsJcQ&D*lwyVG8EuR(~Et+6X!y_%W08qRD zw_V<_x8<_iYb4VZYY5icZ-))F9Q#GwC5s7SpS+Qh&dK}3ya4m=95FsZv2eXYZJDj> zE|Z|n0Y}Ty@oiMAmN(zBMOZH{eF4JxWADz5cxx#mD(S4Ph!@_AdGS%MJwCScU92sk z)a>)V-k;3!!fTWR)4oDwn#-c)&D$UtBSLYuzI5$c+(r45H?9lSGZaRH6NXtaq^?(} zU!NzgUZ#3e$O#*6((LTuukO?kG3RfS(a=}EDo-;j3#W{ePy1P9B03$C88+kBpQXOz z)u)S;%%g;}GM-nl?|JR-&De(Sj}*A^Pz&{^hP_ZF1l=Y?63zC{9H*Skc9!Bmk>~N8 zT@e}`8}K#yR-$W*9E(M2u8i>GYXdrcCOWI~#odtw)K1ZNz*Rp*+1=rm3%sVdyvhq(agTL%%p5-rfAfN-U?5>|0k9 zdEfi=;aElDQ)Vdbt!pYwIN#YYNEMi;ano?&ThrDQsnG2Z*Mg@joZ?@aMbT}Vu(`b>BIDH-J9w8f|K+fi0g#4SJ4-QivpOu z6w4?B^-Smphu?*~m?5RFr2W=gVM%YA^X5T`6t&K}`Ne>Dgc)Q|p7>&1o~c^FwegR^ zlFf2t)Mob6at_fy!WAlNQH62=;i9#id4Z`_cj7qhL>H{TUd3M~k02B6yZAXT_*=fU znRH(G^e3j+o=wh&uTqJ!bi-|vt%L&X%j7rJzP9%~4$$HyTW4mX;q3~37szz}^6x^H zecAQvW~^(Ui|?Dq4bxMoOn10f_I}*+yMn*ENq_!{Hwz3u%Zg)*APL+3UcW7^kgA(< zOXD7`xsM6#xb(DUP-DnKI*HF^rwgL9XjQCs&!jc|XX698MCYpshVmwQ*%zEiJ`GT+ zN_w@azvDA$>z~`S{!D{e%gEyrF-YAj_S+9W_(LgHCLokc1At?nR0f#XAwZ;Hv&&@r z^;Hfbp!+I;d7}x5xN6Ns@{puI$PcCe>A|nFt^JfojB8-CSNJPgw&uoVX>khjOt#*u z%A3uOO5V*B)JFUlq%-g*v2HF!@&v%w4PD{R3s$L(x}ERMK>*+*YW$-0)_l@#Wt}GF}U*1VE%@_FPj}fzHrTE`?RPzHuYz zJPQeuP*sNoJvATDpbG^06mAVNZ}z}a@4cbPrXx%Z)_ed(e6vEO+5-~ZH2=PLr)R`5 zfrB#^pKpz1H~ns}%ZceOPX%g7tL&WAa2fJ=-vG(C+z~{!v*TxkFi?D`yCm*i=69lT zyEdb`sL#DBlvQ?W@nfn9H{rJo`!O%CXR5FAxZnkVGL43&RN;Ua3mmWUDgjiqj%O*) zbQO?YCb#E1CP7J`_;{6bLIfo>d$hlX@#!J|LufdF#+Z`NyttP`bHCX3GL=AKC4A)_vxu!wAX<; z1Lio;cm8QEwi(LHdD;QMw@n_Uq{n(Ep%H>sX^4GP1*c-TvLH<0$cGONh3$Ya#G6ny zS?w{bG@d3OjF%{`cU*{6WZBxtZYs_Yz|v)B)AlLo&mv=ab>JQvZLtDy?|vi6HGHb#=#pC)ASbxh#L<*i^!#H0h%#DZ0Li&0Wj!n6 zG`1uj+OH7KRampIzH;t2cTvnNbRJz*x7kQ@9A-#0TX^v$Ba=1zo6?`Xv&v(u2k_m`ef=13>7pIV~p(-NNsn4*V%0E^NBifYddCGvzs^$KEq2t4NXbI7-UM zSl0W8EcDen&|bD6nrsPpAib;xb^;3e420(c%Jzg*Wa2^4S~!ce1mBt=_<&c}Us4~h z^R1@&=MwV+&Dy$g`|7R@l&V1O9I>eV>z!a(d|Gp1H*~}!py+|gVaYLv`oshH!*Egk zx*qv98>g{~UMg$cZokH{x{K2IkMb#fcDt5vRu7=o?OXeR#tC)ovLSHVr~3}43l2Z! z#Wne}nNXZRe<6H5foyQb>nzHp{BdiCtf~7FYHLQJN^P$irJpW6p!!I>&xpR;^NkLS z*qd+1s(T32qmmQQnRVy_TNyrB(evCQRPadn*VG1YO$}4OiQ1U7Fsx&C&rgU2VXIu` z4hGyUFIOJfzb=BVV0idnV}VlBv;Sf?Naff-p86$x4_myvjh8+9r;*KBUpGakv7AyK z{H!b|-Hn4x6sQBA!Ph#5bQoD_K3XNsJ<)ocI@eL)Je!7_ zyAIjl&L(4pZiad%!1PzI;^I00cd^Yv$sApy6JPAP+pE~|98poJWp-05=xns=wxyuZ zAZE1x?_`qC@G-(LdBA$o2y-N;7M^?8%ESPjTNBo#9L-S)%EY)VRjr{Xk@Pqi%5*$4 zt=TlbsQdz_&1$&L7gtOc6myy8NC0wvyZN_FkwYb+wzKk9ehwl~%Ln##*oCN$*DC`# z7C9PpMy`>P2NN!hhK)Qi4q-ovFz>;dXKHEcT2fJF`XdTv6dW3+>jN7NC4gY%RzBsO zYf$T5YETFD@+^hYudCcL7-lOdgXSCOfI+C6tjd~khA?3cHnv04=IxHL@SPhf28?_y z+qRrJBXy(28Eo-jJpYYE6zC-SzCOmr3BZ3HMyc{&bSb?&H^;kkG-qJcpL7HFyDo)) z&Yu`_L^&xatF09wtL zMiCqm=l#u1;{xN_219jG+ojTu13wwjp9H?tR&-I47U zSf-646ko@t31cZZvs-l6-i-7;utF+d#T^2buEHk2vRQy~J8_u6lA}dx!OdTR-h;oJ z2g23Xd{fDtWnbYK2u*i(vJw;Ku0xKC=n=QWJ}wM0wTWw9OxPGNFK|#RF?mVDZBiz8 zM0$yNX;>pM4~QT)0i_H3EbHCXzw6g)_B4gPYj?$ew=-G1zX0L3DYI-^{Zhb%+ci&@ zRzoi)V9j|34#iCe(m;=^K)Z+fs%m8g(Y87=N5#i@N%g&RuEgY5Pd_MD*hT_#=2&#ba607it%o>Vj+mB_hsjDMg8KNC#P4KJKx z<_VLOAxAXzrZAu{ixHM7>Hw!Wcw2I<@C5imASdxUb&DFzmG9f7qlbwQIZECb^KeyyDNW0s^8~w+;4A zxJ^4m=H*DZCc_>*7bOry+2SjqL;$XV~SWR2&%ua-mhQMW72IHd%$>p{>o_?pLjCAURI1j-AokqN6}thg_GyO zt?b@BXPo@U{*ct^ic8O|xFHd;ZNix{18z$0{7$}V{X6=fJX?8M)SSsVTHHd9Oybj; zGG8+#$3M8ys?}Xz>3E4-bk%*L!bu!uv)kE+YpD_+ijr3nNB2il^gXJJclHDQmYWV3 zXdcam!=KkNiS=lAMSL@Cl%X^oVSrJk&vDS>V{_{-FizV>vw4?Tq4v+6Zye|=PV#mK;JJQthdPG&3}o)AL(`)e&b$&F+=d!tMDMAEgV9dfvHt!|<7+^Lz+f;n&Ew zfSJHq`*UP1ip2!jIy)uC{ zNP78A?)mEH3@q)tUgFEM9(?$rOy-J5PhY)BSUB((vZ3m~T5LZlA!~PdoN(bXiA5f2 z+aLdjLKgQ1Y5koy&dNY2fH!};#ILVe^xO-x^fV>2sIS-6`a53X)Umw@rN(yxzE>Q? z!!!5V1EvFx%xYCcfBo<{zhCJ*^M`C+p)zVCBTTZ3F_&f)f{P3pYMWg%yfLR((_& zOb31v3cj~AY^!1@HQ=*>Xqx+^D-12R+plRJ7;|HwweRs$(P%M9eDRITks?}@%@J_W z6(KLC;kTP~XXBH?<{p^^ea#qn_n!0)_2u35$#I(CjCylwlw2Sw7ETvdbjNpPJOjIKi%xt?Ek$I6_I?YdLv8OxXsvWkrZl37m= zTl~Bjbqr(4aMusM^>(y(N#vEs9v4515OpfKDC2-t8-2!+hahFLF1EP%o!t0Mjc`Uh zsf6`U?4uDYdzEwN4bESOJ4TVkZ4@fV7F=cCT)KF*AQ1nG{0^kHa8D0KtebkAj+blv z!epgcwn_u2*fIAI)9SY7AAiN5LQAfRE^2X zOqj3CIk{!012nlJuZ*mHI!9z6Hf*&qbS;6Cf*dmUUk81@z_|6&r1cbRLOWzg8h#YX zdc344L-Xugr+C~qtCv|#x|P5A>|QFkLksN#*cXR|5IzI z6FGlv`y)|t8vNBW_n0(MW=e|LRJk16t)OHTbGi*4aT>sA+=*qpwi0=5FJeEgR4O&C zN82v{3x>kU-cW9FI)f~@MM3cQPF~51>`LNPnC$Vg#!2L5k~wRi-aE410@nDbf@+VA zZk!TX4)gAzkZX*hV~>`}k|lo^HC@q?X(nex>SGpxe<)$MZ>yUCZPO(kfc0+!ycY?? z!tJhSnPnSp=?s4%_y5E;suBGNmGS4sQrmdOnQ;6;l=MPS_Bju+KMeL*$8rf4(`Vn$ zv~1-{M~@~$lt14E)brW0<2I{lF}I0}FTXp+8@Bhb$Ox1=B@dK;b`c7Lenx2MUz*U1 zw~{Qj>qcc8aPR%dVa?9YReSO{gM;wH!_pto)GWMQICCng3Sb{DIVTERs3g4gl=A$t zD0Tk^%Tps(0!vftAcC4z@yusMy^@OFq}{5T_McI&RHJMwxA$qqFH=!4?L3&9*^=6= zm3i=V1-EpQ#o7t*T=CNHZaIis0o8#(+06_##}}xn<~X=XKkb-zjm`P+VIbpcVJH2K zF#zCe!qWenx2dl)Ra%mug%y+EUyf}LYQCU@!1R5a=kfNXy;ZnoSeu>^|8ud?XIJJL z<7b5?@TrEys}gq&$8Usw0T};4dg)gek6$Q1(`{ijA0D2@+3_XuS;Ii%hFB! zGH2DATQ#Q>I-AksLNRKY2JI+EbGcr!CQmN8m|yWOdV;Jhu^_)i$)^Z)Cld=zgUYAP z#_SF6)r3Eqf4vw1U0ir=@z&ZVPx$)f$S=E_N8ZyR6$8Nr-6@j8z>&)1pxH{B381fR zXJ#X2a}Yw|a+n?*yGi@jTNwL1d-44+!Tf{_0~4~gNz>E7y){52q)2)7{m?CHkOu16 zI^tv*f%n<{mTy?uM?k9u@p}f1R2kt96D*WPyn4zVOio$o7sPyzToLZ75FEj)RS5tl zstX~D&Q94YvENSvXxVi6&xi-hcT{m3S?LaxsiOl7oKsy%5EQJrJ>@{jA@}8viLp|f z<3Oy}yShEv68j<5#LW``>vMNvv7qKw0y?Ul;6^yengU30JuAuHnP9|1v;|i=Z`2-C zX~-TA0K0bo;!UrolNH1h4adQk7THoq~q`u56 z=MF*&QKx&z6P;7R9YYJ1UkWKw1kDSuM-!n8B2bnLZn_m>ugme=xKW_yY<~O>1>iAv zb}Nq{*A-l~?nt&E@Zh!fCtL!&S_`8lL&`?CO<)JrXzal+sSSg~7K<;WSm)Idp;JYf zDA>ylK09}2j(!Dc3mCUUdyJh2@JJ)cw!^0VXH!qPMOG->+S`+MF3dMqc_FIeNyCox zN-VK%BbALkn@9lgp+{UB0(FP~ysV2X1BSDs*$-~a(%u6EC60BNge8NZ>-nLLft3^? zE5*ekC6#J?$Rh21=&dlB@pNwkjb?Rd8mVh>nC}q(t{%+e; z+Ioe#WtgBB#c-d?$xQNExs0v18qwMBxF+$m9y7qU9b0UPh2dLZR2w&%#m#3Lf==+w zMZc>?Hg-_5Xe6rWO$+~`N1vRL9pGwU2w}@4zN7**z0W`IcD#6;Q8bw8i{{(h^v+3Y z3ZV8?fPPUx8w z+b@1MW2Ui1@>9feTNgtgCOZ@vT9Yk?Rpdo4zJD}gUt`>@xaFi;HS6ZWtZTeBRV%qO z7p-w*-MDCH@{OjlecG&&d&`Yk?i54- zWnNNEyK)B31a){MzA1Urd*^VC3N~!yVdRjy7X5knZP4OKS)dk^N0fzhbqcCUsxR{ z`P>^r1}5yC_p?Fss^Tw~o-q>qN1x|M-T8Nynk;nqw~DSfAf_@1-keGvu+yluv|Fwq zKJLU$wRoP=m$gg4PH|}5vCTJ^a~5CJn0AumUo+F}&Ou1MhtzWB?-Q&oM-ORMuIm_H zo>JzVayJ|n4|bIK*x$G@xi+Cz;ON}yI}zRH^d(n~p%*myFhv5u{3yd^axx&ZtVX^^9imbk(H3rO$?etMi&iK2;Ke9ggW<_32XR{zb zBsaE29*?pp6O`BhuAP;@|Ee4QL2 zmx|gG@-w8dQ1k^0VmhSy*SEjF!py@|0R}>SdM5_!%f@zN$@~ z6F}N=B1|*<(nuwdi5uZwp35^$#_WWU#uXF20iV zLFukdJ$)ZAq39Qze?+D!2|-7<@3zDq8NRW-yc)^`;jf8eF+K=RVpDXKOAkL`#L3FiHYVIzwf2aOd^dVyfGoGgT9YLvX-!s+wEi6R%Q9uGT8hNr z;yEw>iq6rpK?`rp(q+rU*SfNhuC%Ewtf4h3{=TNpy$4x*6*H99ckD7a>$uLfONtM~ z&8F5^9LI=Z8BG$}TDrxB3}m_?{Pw!X#n8uwmIGRS8dGE9{4!Izsa57~U7scQgM5p< zkMOvCdp@Gf7d^qn zl^6Uo7CuE$r}X3C-bc9LM&?BY#xECwtQm>>q;&ss`(oN}HNOE)=U40i^c-8owvWX5 zNBMt*LqmCs4NGGe!r6>sB>hdtOzQ}vNMB7(b|DMTp2{P}EypawKWTRA>iKDDLQH?| z5Wd_M0uNVd`!|;;%E~Bn7O1&>`?}))Fqas2*~5>|btks@x62w(zqbTA=O1}-B3xeb zq$)R0K3ZcNvXV%ZY);YWm#Hu!XdjI6!0T8w9YDE)U%JTRXTmBqmk>;XxD>_^dVitH z%kPfI?H@Gll3m1glrw3liM5~NBtcRo(M0i5ISx~2%19#!j3@vXx$9r7-y0+EX4M15X@J5_@Umnl7_Oo-Zq{SEv z^`%>c;TS|L4^QiD(iOtC>lK0137VeB5`*1Y#7$ibkYnKog0>1F`G%w_(D)+SrjcDw zs}K-Q2zPX)NJ4<6%XN#$$2%R^M)hT--Bw<(whI2<_{f>+NU%MQ*MJa#@0jOP)^qD; z$p;$jLyaNcf|@_Zepm}6^#a3w1qgqfiVvL|lJwksC2K_G^=K+0pP+V3f1Hln_5WF5 zP@eBSNe{6%baUWqbW;|D9vV4`v!k*xAL!Lt62juv5pwu8)j1jjZ11`cG<~*HSQg=rCO?J; zji8Sw(b(b0ydBecy}D$A0Z zfxs*R@*9_2w5>Zx{ei&Cjrlxm(W*0Kio2R_c$gaw#k@+|!o-}O0XX1rex5Ef4KkGl zC-`U$nZ!{bW+3jbjsu>w(Xe?KVf!j!wA!NpCNO+5->}GqoI`_|qJfjb%xBJeJG@wp z=Y@NQSHt^@XCcxIsg=GuhfAW+pC2oZR&pC-T!6sIhl%**>B(OEQkh_-($cQ-*jV?e zW@M5!wGBaQJw?b&9#9zrW)wJom$v@Nj%4AEv!Z{?zxz_moJLoFi1_06B>cNQ@Qfc& zG#?#oux5RSCl?c_)m{;X4a6Q}@hR{xtJT7G*ZWG6@diKNJaapuFh#VY=1g*snHC?G zS4T@4&vGE5fERdv4QUV}bggJa@HhP7CJ=jAS(#}^1b1CPVsd7(f!^Jw(ea{+ZfaVL z)%oo>-*6%PpdeM8X|YpJli!txOy?h;c!sBavyN2U@?H~I>vq)W*7n#N{=}r-*KWh>KKWU*XP0&B&6-Z@Mhx{e!Dhw$fk9p_XeNtbhepO z!R`8|!mXXbEF#|DsA^5yrURsk-hH(8YfPYAUOb4(y!GI3Mr-h-H+pE~u_34le;Mw2ZA!%IQ04_ni18rpv<-B{=^t0>uxL9<+S$UcLkJH&Y zgA>u#+@s`ON1nj9X_`8`#=$TKG7{~clr_Gy2nbHF{2N$}9x<{MQvIRu3eK~Gx_J}z+pHQ6YrP#ZZyP*L@ z3D9=2-}kKtziSV6Fa$8~Pul-#l<6h5f@HF3@S#^pb;7YT$)m4=uOEXbp8ENUc$ZV z8#(h1J!=bL`@ZXbnLf^?;`@vH@Y=t+G&k!WNO?5Lv}Bx{V|>n3GLx%*89ChH!Ko#< zD%3@*Mc}-w^+lmCRQ^p%XJC8&4NzasdGKFy91Z;$Jq~@IHG$GDf5X-pLfl+Z)D`Jg zc+K5A_T~*c2TvjE586`{lO(o3Ssg{bO!c>-!D?VYaiP-cqD1jz@J9;F(V)nsQn)z# zib0-WQ^ijzS_GTik_BIM*Uu` zxGkIv3)z#bh$>K*b|m=Ol;y!RAjB7=cyG>Tvu{Vxb8}iyhmr9cD=3`_svBV78f?^N z7;1<1fqeBs;Gkix<1W#baZxIlcm7J0OdMO!E36?oRKjC@HSjaYHpMbu>#INo+ksOJ z<0pi-H?V(BpU)d)o2E!rrH?Fbrx|)5tgm7832jKG>6?Ic0AstD$<$(lJC^tNidr2C z1hIa_TksFa#r?q0VgMPncTdO2N^h#8JTLPH|DGY}&U3ETSWQB_wD;>eplK3Rtw5r} ze&E|PufPUzmxYQme(Jg+@h)(H@6`%O8JPpd#U6;Y`9MVm{@)P#M}#tte&q{z@!J07uK0U;F;Y z@FaJyBh~HY;2voi>ck!7Vbnn_^aaeKJs1)OmwDgs3jf>z>5+91n^+&ULJ(qmZt*65 zl`sT;Q7oVs9KoZXQ3d;cLVSv2%CmndwUKaO-8wR@18(?KRu!K=zb_60V;0hA|0909 z)FvM7A)_Hvj3D9N)$bc`xNRqE@sT9oNy#rYg~8yPv@+gh~3_=h#}BthYk((S3R6fsIjm1$bd5kmDp{5lE{gG z3YP#qSunm0=WQ$Ee1?)W~l0z`O@^*YY0VT=Vz(+38^UX}_f_GuMX+A%6@#w!uL-hw^1`|MY{89ipk0T)BH$Uw{it zhbHGKf5(RCkKkxCf0mI$-D%Gu!RMgvh+hTypR4DY8B?(KZy)K1ozuwi*Al-2lYJte z+$@NDl%ALt)`o;X^lPf;ev=%3g+w$9fa&)+JX8!d>s6Qr!{b849J@b*yd0o{5C9UudI79;cF=_#(oHAv96#71 zSa&OQ<$Zx?(61_oB#7iKo+VOh1{OeCElweK9%)33)j++xLPxY}$wA`p)b?@pg^KoZa)C*|Vu^zG z`dr$oKz&?u_*%cHCoul;@}_<|x=fWwJ}~8$Xmd~hb9ig9F{SXuia);+6?`Vh$u`E- z)hW`>X%r^?u8WMHqw^2Tj9)91USDXVHuXIH#GAi)E@F?Cqud5_%aLC6O$1_*RP=_@ zCd;|Ry-O|DoTnEH+gw&?Htq)BUug6mec9P(H^LgAuenbZBsyUTUt;*a^Tr^kVW4R=L&brmsoDhCyeYj2k2*E zOUu;_EgkxH72hJZp{=45x(U9__D)`j5{xJQ1Up2{&{iCs{_uxuj zQk+9d^N(C6@?8SN6q(N?px$QG!MD!`FXJ$nOYDzo55T!vWZO~UY;wV4pf>t zbXk|kz+Qg!x-)3K+ca-YKnAeGT=>L)4OD9?1FlfE5|W4iW=xelOFw0<0W+jCp@o_t zGs9|bioJ~?WN5~VEkbNYi!)F0AgFHy1SzP#Ufiqip!CWVCP{kaqu5i~l?oM8>DnQE zPyBwq>1_b=!Js-|Zh&o!uf~90UIt=6u{6wdN{Cjj13Hcd(H>s4S&csC?-cjIA37l{ z_nS5of!S>=-T-*l8gdC6t55gLlRT3 zVo|%eUL}ENl8`9{z9_2yn15o;zx=@pnKd7tD$lmZ@*cb9`*Lxhg-0J2DU5Mwq%vm- zCjwQ-lHfxEUL&mS=4!U`RTz9&kK-j5n$Su*|4WzBCa6mI{V3R0-Lh9gzpWGeL!g9?;NCQ6uU@OIZ7$-%CPcf-@}(m z%wlwo5X@@?Q|t%^bki#f-{Cy2>l5Og+NMhwFh?0?qo1FdOfxdSZc@oPeA(l_T(OC( zhQIcwtyk2l?%+n=oIChHqJ-CA`QJEu@2Doz_uJcof&yZtOA%>OLX%K~h|&cF6;O(R zfG9;e#3TqPs1SM)DG{kE(xnNBfbU%%ToZs(z&U?-}?|T23S!>pe zj0T?PzOQTV{n=e#88BZR)dk>jE#CCR2l!JP9GEtzgKY4x>Avy{U!3)!XzE^vz>uKA z#U1wo@r{uSTh@Kbz5jgubgqsbjX~b`9%tep?=V;Nv%Tr1Dv)_bUn%_8J4`b10zd0p zT<=go8`=%I1l=p!EI{lI%*6h?*p3Xs7I2tzmsb^8Y8{Z`mKl9N1_gN_t?AqT05(z) zZ(61MxF(?hn12g8V;dV!T2bS;w&q4_hUiNZ9HM-y*MN{x1rl0P)%zPb><42vxt~G2 ztdM^t@ND37xf4N}K@}SjDe*xhq0?uVnKYmuEz!t-STJ%r7Fp&5Obtb6{1iLb9~vBa z4m~BtC-zev3!-FSO;E$ZF?soTOMI|*6o)(@K>qfee>MGEoOWM|e$*5o9tRnb*3jZ5@$X%&`% zVe+o}J$JSP!yV3)p41J-K!4>GMIS1RFWP50eB%R(TZ9?Le75P4OnvrvSmK-P`H90NY~wM zT0Y3iEbM#gyC7>dL_7gxD(k%g>%GlBj3qRWSmW7|H*^F7@&p>Ax*@Uvl-N!MPXqMe zEWCZ3uQT#V8^$|(-@JPRe5Eay6z^Vh%*zE}0{pZbR-Iu1zEcP_H~-c$n!w(bD?|*{ z98v$2%sef;=l547up7cvU1zf-s~IYZtP|DthSv|ox@CslkLfd8hY~V)p?#7W7!>mZ z_ICfon;>>K?AF1z`c|Snr*rI(cmN=EAwJXDlcoKC0FW5YYRw)f(Z6VA(320rlu(Da zG|UT0;03qJCIGGas=$*9Y_~C5&cWyme39$P2VnX7L1`Xx$bJRjF@R~A-`=8o%YB%h zg=fY7yHY=5^_3C?CF*_yt>#$$4AOf#_74#yFif{Tuv?Te9csjK$%>ig19nMU`2u*k zI}~Hrz;L1eMv7nGUIr`^JGqs8#G&Qnfs3)sDG;E*xxbLJ4Jh(*vfwb5N?f8qzRg(X z6EeWW;N>t-Z2u>u;_f}XKa%n!J9yN8H_0}DhG!iSlh+X z2ULn%R>>_l3q*p+$xvnKxuI-K8&#?8d(2SV1PF942UO;nmk4=dzJs~T+%6SXy?E6D zM2az+U^>1$L*HBu6!8tGR=%WyJXn7DKNAyi(-!=L9r?EW{f@bza@amY&X@m&mnI+%koob=yy9Eu#&q} zW#d1)liBA4LZE+ALwtq9l?=k z8Jx}q-`bq z%qfp_KH>EhIBiR3?F)ZEa5;Ejq;9`RPkurG--#_Lefru?>kZ-Tp28?p+0W& zX*dtap2WA8+X#lK&Ff!#MOlwc_4B^z(+d(lOF<>Y{+xKrLw zv4??Y4qmEJS`xVQ_s1&=vc>JkU;gn%VY(CBC-S5z{Yk6@<%@n1hRiXX57 z7qC55i|_*?3$*b@5qB0YI5K@IZejyBZ%!1dJx74k4c#BoAs0+s4rC?V)W6`iJRw{F zN%rmWAw1In9zwJF24a|eaVw2O7rV-pzG0RS{x)Xx1ZFvu5MKR8id&C6wQenWUTWkl z1tA6=0Atbr5yF%n&pzMK^>@MnHk=~B=Bj_yNXB)zVo?`KcVYOAz^mV(Ru?&!<+_gT zGSKy(z4adT*-ml^!vnOPe&(=2(CUvB%@Nd}*6 zgVY%{YleeD8joTtbQp*DuLv3mKIB&rEEFoJP=2hES9&smKi5We?w*QQG><}?G8du+ z(^RXtr2&)d7jm)*Xz3Hnuc)KH(yQ@?d*Pu!5nsr?T;}#$b&Z4!9dOO(k7eq}DwkZa z&o;QZKG%4~dp}?axd2QJ_(|eX0yi={LMSnbbb)lC`X-Sv^=T3$Kb%TA>hSEu*umzX z3WU$$cJ68Wd(rID`K?J!bsN>p&vJBX?l9xigy}KWg*&2-M;VV#^Se0H=?tHW``z3n ztHbwpHX;i;NsZyaX!r=kqq~xCu#-19E5oKZ8N-zb_x~j|-F&r}m@?nwhF7 zQve<^f2qqGx)((20;o~AWX10wB{gJFC%5oJ>n*41I>q0kWL&?PdIteL&|_bb2wJr>uJ@&&I4+A7sT zy}J=lOoqcyW4DqPlfX@;$&du-3>ek8rZ`cPZM-FNIt!6|F;|#$B?(AZ-_#1CR@DGm z6SD&}z(?58-+LRpdl`^XZ3L`z>Y*p_LieqBa0cD^-^yPP4u^vy=fUC6?O@ji#I61G zL-&K=miB$I>s?}aPmsMR`N^m(J1LF_JQBfQLcP#7$Am$Ji$5SS1_(}PtN2(!XkZlSFkqau8ECm z)VuPokdMWC;4-%Mvk_^lP8kO&v+WMYxyUpf8^f>tkwW2Z3*=Ad#cbHT`E+{fOzk`fRBjuY zZP{tVG~Af$U5@ei!}>b>80L-%aTx#m83SQKql?j=pmXu>UGSR4NjiXwzbnX5{tDy@ z>;j+WqwU?!AO%G1tYi)yl2s}?WTGu`%c<*36J=VP2#`Xkf&s8`uK#-5*&M(|?Zbc1 zfob@-XXjoSYytaNe1*F7RJV<->@mTOj@IaQd>&1`|Kt7jU%tWVwC_3xn|#k3}^ zC(XR@Yp+5zSx>tK^T(|`TO#ng^d@_Q8}G*aNjdjr_V4gae#`2nj@wrZ`$39q=ZXFy z%^(3iSq6nl4yHD97}u=8P6*%6g%hoo*USS_@F@1tr0P#12ia$Z3Ln&v&rJ(d2%I0L z3)u-B`A$!l15KUQ!v6PF^T=FIjwJSwwho8;_77;+b_=c1n=2ni4PCwJ7w-=3r@Q4` z;*l8~z^H9hKWEeZD!~~@&9jbp2nX0h2 z!T8Ud(%l1@6naeh{rnpF;T|oaQ*zlHw)=xnOv3*MgM#1Nvzs;Ck!soqvwhdWwM=Oc zd}XdMaoqm4cG0!CPL|U8ivII5s?K=PSz|%v(rsdNDg6-qTu=JXXrZ*HEwbE#YH1lQ zML&CH^f{Flk`sUsJgjE&<)q}*wCPo>V~2dE9Ie-EBY#5K^wE2lTaT18KNuzHdcK3g zl;l%t)vD9LESQf`K=I2}Lkv$EZ$0&$PoelSxP3}YGg1JUYmvnVb{z`c-LgT^3*4B`pUrM5Z>;l`AXOTl@jXlII{^Tz25&B@N%jx`Db_|3 z9{)j(9-{K_WC8o_6gs62Jc+~mnMWH`I`es@9yJvVxV|Cyz*Y1s;ju9oHXhT7${qDn zvu1ff1eBWnli`N=GMIc>w~ZYLRMk&We+pM?*Os$VzrC5ClU1{;)BT?fya%EqP5<@k) z{qOd9z{*l49;3M`oP|x_>HAN76bn&`7qUjs$gwG_*s2saUi0iO_eW+#Xa{GZ1;jpq0;*CeMeR+ z5v!lO+&S%(S3jHTpdwixg`T~7H+R;llfM#S@>jurq@m0@SN|tBW3}mDbhO9Af5bKS zu}rP}v%u-OstN~8q5P2jAuzuJSQKezIwAN^unv+PK!L}7D5LX+M%7KSJJ47JEx*>m z5cU;9PsSFcJBIKFfC(}G7vndb7&-SN&R}-ip9j^2RnJ4N>9)m9utQzgvqCLhhXy}_ zxb;W6i*yXo+=H8t6;D1oZQI}VKiC1hM*})8_uUGJ_B@md-n}Fo*!%VL%bV`ew*IgL z{C(;Gw9?CcA8(8CdXYTZBz9%`~e&i7`1#rtt$+v8viI%dp~^DBl>qk_JrKWE7t%wuXnz`sGedC;-foL zA`}QH<~`T{gP3z2%3b2C@QY18bAErq;+5P-{ikRy#yG$b>%C>VO|#9RkKeqqDS9S? zx9oaq=FKbkn#StI~@z#Al*-XMLd8x%NMUO?3DRqR!#+Tvo~`@^@5J88UUIJnSt$Y~ zMc{K9n+6&Rm3}1F#KEVIwZ>LmLn?dy{oNK-MJ#Hlj6i=GkJ~BUY@F?ly&Bc37>#}> z9;+(OUEXtMMr=A8aThHX!k0|BG$ZHVwinm&g3anLX7eX*6U9$zVVkj|n*Byg-tlnF zLI^HnOCkCotbVU&6*8{NWZ|(Oi(QadYyq$bhAd5rTiW9(O0_aQS`p)52i{NNVqj9} zZI-1b&U^0W;G+@B2_6&4=NX|Ca1+0Mq6(`vS5rQk6pf&kfzNYS_C>$q%k2mLtv{cn z47)d-$DfGtSxco14uNPtuKedae3U~3EJC!q1rr-U=9tO=nZ)6}qP-sVPi}D>_O&!I zTFn8x04?8c#Ao3Fny@qPhECg^gcFwh9Y{8D2ZzXC>w9l9c)#>`pMQ}k;gixxVq9|9 zRen)+WxFP-05AP96uI2U<#sDkY&NIkySpyl-6|_B_@>$31Yw&ES5SJrUw-JzI%B+O zjLX5+p6%u@IYvcrf;9K9`f`)%wBuL~-#O~A`^UNXZ4gpqP-K}$o62MB7GQEu3 z>4qY$fRLBkBF_Wl*Q;9=+VWj(On@T>Ehu6gYQ2|6fd^_wt}TXjv1%ev0_WoAu@2a7RZ_w5OoUr}C<)IDr?k<4+Lf3OEvS zBsr+Bw&kjMVMNTuB2+aYOC)ZWTvlvdoYchgRJ=Va*5yf0Zat(K@3LQxyJ@jE;3GqV z*5+WDz6yYJ^Oq6y5kDWGGh98}gO$EH+LCd<9AjPDZ^KM@TZ+@S$I-$tC>(4m;6m1u ze|f^Q!Ck+BcEeJ4;jws<{LJB|$XBzM{NJiL&`>g@;~etNW?;{^J~sZw^IrO~1xqme;0U(B zES}RvBNkJ9R$;^%?x#+K*yeIS^C#j1_^3#C#FEndVUs+4^_^*OMeiIJ8^G|ZdMlbv zDDy0z_vO}V84udVt~1kS?n$<|+>~rs6SPi>Df3J28QI`&+HG8w^G|7>zWHC?{nxg| z^umqjE+E5oYj0cUALUOmd|9rq(jq$nJ_>0XR!&4#6} z@$hW(Il267`8dCkmeJd5Nmhr4V8}OP{5(s)TFCLb`MVxjs-0~4gYJTbw_0p&lVP)y zGwGj`#iHY4(?jc#*vjtaMN){d^BcyA*!JWHqacR@g1$cZ<)>ydGGHTe{-7!DLm0ED zDLOMTtAAIh=mgRywnC=)oV}QAYb5$*8$8LaWt1ZvfI1w}dM6ko3qnlRD^9x&O-9ZO zg^G9Ii<#>%4EFvh_6=ESoTjon5LwYAZJsPn=pQ@OAfRGW87_t!H4C*sc2Tw_+T>*M2b zYE&r%;ldDm!A;Zd_lYl8-fnC+`HV$NqU3L|*ocmK!nV@N@IS%~U`{Ep6vCV`*1O0$ z!H{fYLGp+fZMt$=^}GCC;YR;ga_o4n8ISx+=cZg*1xMW?vVcJu7={f7D5xeUbd+T0 zrzQC$K)p-%iO^OQ1)9D=CrWgZYo_59HpH0v{5F&DRK*Y7j6m;!ll*u(_PSalo~?To zPi(NT5tNzyfXHXPt{yXX*SbZ~AsA=7(<%K$u4f9??3^Yd4~d~?ki);z!<`hq&rU|# z2uL4l0Ia$#d#?AV9_MPb9A&q@`ht_KlGB+MtA`pcD z8%H&etH1it*~6m9GD2F?O8VAmr*2Kv3lx#52fwf3ixwW>FJH5Z^PU&1THw_i*cz zYQG^STG-*r#;f;vabzN=^=l#dWVj&v+o@d>}K!Nqk%$f5R(3 zV14;z9fi_)@@3FKAH}hl6_p)$#V)GxoM~spA?=fz-^@R5=*7)bsFj^592O6w9`J=M0x>%aZR=??8`c-xPVF8Lq9XUG=-s5hylf8@4o> zN6eF&YDL>n9_mc*DSx0k`To*sM{Atj>(M%6GdKPK<`F&Z!FiPSK#3dQ;``6GtIR0L z-3VZOJEdTt$Rg|>@q0ZygfdxS8Ot(lI{@r5Th$f^OSC`yyYH-!X^rd-`;(~=%twkB zJ9GxW@%P6kMgxr8p}CK|!XAeX*C#$l2gb4@9>YEOPU;-JnimRVuWyNbH#McUS82^A ztc9g4z2y+azm&^8Nal+U3lx|YOK!W*{i-Rf_hRy@7mXv+9s&v^Ff ztuv{&ZA)!;DRL+B+kh{A<>5uCjx`!TeCwK3Q2_ZMWm!-=ZBZ*YEcQ#5vIO-xEarhB zAMFv=hEs>k!Thq6rl}FH^4s?62X!v%zj2l5KSh)~YF7rurGB?1|32$}SDVZK8v1>8 z%xWwQK9s{gM6UQfgSIpiAaV+4N{sF)n7x<$USD~<2px8+VzB4{qJHz3NE$x>9qSuZh4=coiKq}2yCI*Vm*YX}5)9frIdYd`*i4)&<;JR-0sZb!)9EJq0?%PH zqy(=PZV?INLfFi5b3+ANU zNY?6-0F+k$i<@iF6L#*Z_BAa>YvNy_c?XNWDZWAS=0wgfDY|~Vit*&pANqO?BM8gw zuNp^RsNusk2a}7U45?|fu2Y(>Ebg6DU!`It zsfTOJis+qEqOyvmz8d0anBl?+6Z_$1Ewgco^MMH*pPSXKJ$IM$dIRUZ z_-BpvfWU5QTyJrWtY5#bG=;(NO5XlEaU0W|WT2iNmuGy5*%Vg!+9^Z%qOvRNxnj{lg6Ny!B!J z>BvLP{q61lwdrV)sXn#F&o5tfZ(6_BCSjUJ27BtWtzZNFClrCMEW7bmRE&Jd_mkVq{5~&n_bwvt9N=>?{h5QDux*i= zG~fgr0q5ihu>Frf)*YX@{y(l&(?WX)RrlGx0$(szLyGNM0N6ZxLJkB9%o;zu1C@e6 zZdS1K{Rrl*C4Uvlx#p>UWSs^Bf#lu z0Zh6(TXYxq+c^GZF_E-F?03n+YVJ@^07zHUPg9NlfGoP{x-s>eO-Rui80hJr&T(!1 z1PVK09xR>3vGN(?MDz-)>fQ-M~(im_P$eX0Ck1j3Jn&h;w)sH|W; zUii;CJl@%vC;O4@0Qi=LR$fB+faBUGzZHN6WWWub2=XI>pmhpj34uLFJGa6UzG`E^ z%3b4b2W>t1-))eZTWx#~Y|Aa=%}#hGaXR?}aQvgZSco*=-n%;NNu+JtkevOgPPIWt zFoT7>U@~O})7ao7U{q{9@S+BvV^ZpZJXg zpP9p3tV4jx`?>l(&rO_JIZ%DHJ)@!50EF1>61z^}tWjS1_(RXDo*Dk!mobC>gCPz9 z{v^<@u19pm;Ky=oLHR51uJJSLr-MpC5m$|YQyB1dwE=juPa0mFWG0MNw4lRjwv;|tpg0> zp8&N4M#B0y8rf{?t;40NHE~5s!;Q@;JEx#${`3KZWcd#I8agGiKuzSk@^}){U3}2| z#bsgMZcR4}%b!P)n{!8D0R6UP-`~}*){2k_vTuF+j@T8fYt_KO;6Fk&fnIr?7F9p= zRWo<^FJbo}A_9qQ2drq2O=I`yC7%*j3(zeE87>go;}lcE=IvExV_@~0NQA(=+r><) zz;~KKIkq)z7%-z&FAqd1uJvTncH{cm#UhEItH`FY5S6y2P2f?-DY?+yOOQVl0m1iK zzLINzw|9dWJSvBL<9EJHMT14>Fvxj|Oiv1luFOEW0|;#6t~e*0!RCt~DXUi8;55HG5*MK>A9vGN?)@BZ?r0ux{cC5ts{7VrIYeL!EiN znTr8-xrT$bV5_0C>bYebD|F+y$(JSnHLO+F#ZduEm5=F?GbJipsgJE~q2LJAHn&H( zRk>Dv_63z0=w|7pb&IVl7^6GQw->;A{4E9ggLzE;yRDXfUIHk4Hj-+b6TmX%1tHsk znb{i>v4)E5LiZcgwpPAFRX-3n*~S-}Ts)9=xZ| zze0`1Bsk)rq8Vsa4w3zD=L~gU+XUdmiGNH^)7e@HYHM6`!V{g2Gysi}oNEIfSM=z~ z@aP5ba%~Ps^J)h)7sz#sZ)f1sVXM7ZP_zIR@mZMN`$Z^$cgafLA7Mmb`TGiWDkchh zhoEQr3UFUR9Cmo-*C(|37w3U)RiJh+@a`?~?ojgr{uis0>>ZMs;eUBv9m{|_!6Ed& zvK!|TT;4cBAq~K{b3Tv&mhm^gxp&)MCiB-yQ&e!ffETN&?Zt z=cbSDg5k~qi0k{g3{yasx7HnG0=6X6Bv35y!t_yF`H6(Kqir^lCIS*HDvMXJ-ZA;M zPSSa@T)zQ)9^xXYto7U3&;%~-3!E?q{=#a|u$M#?4#Cr=*QXYid{3*G!)1-;PUhV7d$fNA{q zzIv~{ve2b`9rlA6K74XiBAn-6zyoCPGs)@l)Fc|>M}>OTBMpc37`l4_@mMKKLLLMc z`oKpcm1vb?kU9*4Lgyz_C>=5@r46(ClN$U?SpBs(z&KTDP_MFFxzxx7_##N#=^8P5 zuztf!rvPu}V)Aw5os!yzemS}+Icl3fBI|C&Lxjam2hBCD|KDfesc#L0&~dn{)2R(^ z(1QugwU$}ih~hW@W-`9)XEKJT>y6A>W3Hfqzi%P!>B2F}Szq9D-Qv#|XWJ{vXVY66VBLCA`SHH+t7i?a>{ox+f%DIeji5U2`{*6L^FQYwW#~W8 zKMW)5Cp~~FzC<6yf&8j1OE@TQ|8?aVbx7&!Q<3Rx;W*{;Mu4WUga z3wJEu(>o-UFdk5*MHFVppF2_H9PA?e=i=({Tlv7;OLpN5~mB$YOT zLos#K0(u21mVCL-?G0^)OAK`EGg^@rD&djO>X3gK>W(sV z-a-p-j0*IR2PD{Rhy-%OnTq&hRyKZY2uIl}JUm_FEMCv7J;tAziA~cQEzle67b@Tg zl1|X7wg~?gE_d=_lCMy%s|)I;uWzNhbC0Mkk6lsleYARZpn4eh&#O z9+880+29z~#7PWqkBx!9&&-V-n5dnl`_;qM-CCR(>Sgw}*>@)JNQZO%q5Iv}2AyAC z@?R;z?X*K6BMRZaFMLqiU-z?I4r8ELjT@+)@hW*&2plYb;#<`Z=sX!X+o}#8hfMx| z3KFw)N=W(A+c9b1jWN6+Q>6IduSks>Eo4^SamDDLXr>dQ$Ha_fMl>#@G0GUpzx*n6 z-%+hy-@SNKH0d9Xd-Npev4I{l@Ri3z{-ed#@C;9E6bmuGWa{+(rS2t^3q&$PJ4nvD;D0v^Ie_IF$LTYLmsAi>6;RT0d>Fta-%!JxeH;L3CiTLcERljD zz0W}RRb)WF3YHuqfB)MvO>;NTEA>zm`=izp$|W=uqLm0P@{ z9FX$MSvCFc+W51|5!9m1JBgdukgq4`w{G=$d~mZGDqEi5qMTu(rhcPsL$qN^CV*iL zmMJ!|)*&XVUfy~*a({<5V~1V_4TY_r{tcc8HF%`^q_XoF7A|6Sb;eubm>T4W*N6WhWPG1ExuqObYEm9(ZbhtB1*nRA`-h&*V1Z;!`Jb{7@ zAE!2?Z-t2s+$(c-hY~Ab@$6s;P2J3#-I@iipl}Sug9h=#1>k>0xcNf}0xw|ar4Bt1 znTM{gU^t%o2{FYsiLn$2kV~DA5$MI&f3&Jmo_Jz;z&zrTa>a z&Al?WYN$tSKZxSmI?Nf1!T@U;#$W6wK#G@cqB%Y0lyMnaVaPKX(Qwa6ec%e4qIs$A zZ7(l?z8`P!BUI+b2eHjC8?yF|yG6{c)p!PNki3&z&*0t3RWbtxk)X|k!tj8cXxOF(LN z%^xs_UBR@wn{v%R2Hs<@-1#9gC3#>C*otpH9!b&I*lIDvj1hX6`Nby9Mi!|@$fN-e!2S9oMc!DS=xOKURKPbi~g6oF7S@Q!f0`PaY9FbI>w~c*}g5O{W81TFNls@nwaDP%^QA~Q=RLjV_nE0mEj~; znN(;+0r|h%Q>lPP2EKF~fM0xnc`C{I%AXu_Lm84zMh#95vYE%XKY|g1=+n=)E^>SY*9*xXV8TVH>pp>*a5USvIPiJ>q>W%@IZmVjp>}*uwC2jMe<45 zT4eQ&u}g&2F?K3~Cn&yseFKdY*M^(UOfzFc1-k8kXsqhxKG5~}yUfbFLKV}AWoF8y zbJHbhO!b&zLda-CML3to8}`*umz!=Mc=LvkW6c5~Qg>PC1Gr_gm;?kz$T_caKDe#C z?5}Du?Y??m;5?HH`SMeo6(3iSF3Y8w4@D0~@>#m0=hWXLq9={JwqifA;m{>JW0BR_ z!mM{lf746WW5+hX$*HK~@D3<(TU*ES0^4V;VfB#H8_s z{b);<{5^bn)b4}v=pFr|Z4FPxBdh2>K}JNFUblL1JU^E8zL9ma@S5VTo|Qu=DS6$F1Un)9cDrXD|nPO*r4V+F^ zxSt!~%(JjG07@L@;?G96KY6}Ur9YKzn9m2D1ACk_e#ZAsD<`kSd1Nr=fAkbg<$Io{ zoc)>Kg312+pbLSv!}Qa=KJ42|k117z9?>BGT`cj3n8V*2@5233OQY#&oL=Pf6yYU* z4DLwx_0h~@b7m35SO;K5s_ChsK~K8o;}2|#xyq3$)R&Zz^hA5$$3Pg-sV3ybsz}Gd zn+!DGX}#IBU&U@;I8crfa9LU7;v`qI9MU|5o?V(yjd)>va;4EbIgGUo@yX3S16S7>Z+DMTSt^tkV<1eTuj7-7dIFXEoZ+Gebu*VQS zUfM)EI;q3!{q{hj(gT4@c?N;N5psIV*87>0-ec)Fj+&MPwm1pzcx4$l9W|nQ(l?RJ zNRV##PKn8x3!vH>^0#azsIYdwT*mGxHobnS{qt9lbS6~Z13MTQAQA$*!)Xj|f}k<< zT5t%gz4V|?c%$*Kk>ESs6Nj;C8b)Y#? z!?S`vHp+NTaN73z8Duk)nFG%;NNFcqeioXq=3Rf-wb&9*UuonMcTAPu8S+2opRj#+ zONf^x$^wQf+L>%c4)X~ZP0lERHJ*c8^-?3J>44LBb3Sd1dkG^a*8TjuKg14xZMM}= zF6D2W{=L61_zAI$uPkv`38>`;RJ+#yeRwS)ozsN!rS85`eM&K&C7i_>vQBg=Vt4&)L}T|vkw_9n|Jh!`x5R0{QnP|{7duX z%gp}S)6s$=j`zfy+}mdhH(95+4oZ6m6JyJVS6*y#xMf74R+;lcf754%ah-BMljFCy zJdtcU{0{FCoQ~Vuo*}kM7g4&f~mS2nqyP0|MVKmvF?v%mq82=>Zw=ZYE4r&byX>8OH=d9=BsI$T6opN z=IckMJ?{2;B8_dT`OZRv(esqpvQ)Y~dh0QG5sojh0^2mHzU?PBx4gJ>E*PspH7)p) zeo<}+mGUSE(WVlljB5J1q|FlKo2G7VYLcyg%;aewo=R z%SJb&uq&$XDrj>OBe?7mUmU#xW+~bG-wu$>+Cx=Z`YH;Dc4Q_!y9OTNe2(8-@y z-bHNyijk^6?#!d9@+zL3%%q9sRNfC^v`%Z0IDjSe{N+fkHp3KJpl3ii$7kn)e)$hg ztD-+kn~d|vv+z@Q@FmE#IgZIk7gW7F;QcfSYxJe;t|G8;_Yo(zCEC*NkYvt9Z~myC zoehM|81uR+`Yu?&ANAx`tN#IwJj45~FR#bxkkG&rnzG#70-_-mU^iI_0*cQKZ>i>@ zK`d_p0Djh4uB!RWz4GzzT`T#(ptEaAaSn>Dy8|;2uf*8{K^&9 zB13)d;Q>ke2OtNccl8pfW!qznZ31>{RGlMwD*c)S+)m z%1V9rV$-WracaY>L_+yQ?UZ6@p*W>z56qj(w6lHi86QlLS!X~*aOl*3w>hv0c`#)P9m zoRU0b-DJQO$<7lg2PE#AmfP8KgV1ocJP(BR#U_cST-A$e`EF!I?~tlN;Cv2HG0H}{ z8IeS|*5YMuOj=7ebUWDZf_=_cAZmXFyH47wvGD6ZTMt3M)9yA@jANbH568#O4R@$a z3mSe%?BQC&t^~{QTlO?7`#jCX2reqh%JlP~eph@ak!mR*TN?*S4xaVyV$JlmQSrh*I8(vW0hpjp#0MoX! z@bLr9;IacpK><(QR55cQh@?TzVtxB$qd9xRnRNhpp9$vAK(`H(F%q+Q6@nh-JptiQU$of;xFD z&h_pAj3k)3X$jjZL%O}urDh38n|^uko(Ei&U`*Sx@#f?DMjMyc+$lm&)i_5{a+urh zDte%V`wUe@ZsYAzH|rID#Ch!6^2X4%!SKAXebD|iZ-Bn=`Vm@p0<*#6IH%(LVtu^K zM9EU|eYA7}@u-2~?89OUpi3AF;9iTy$8|#7t zXWTz~pWu{hwniVpVV^2hd`xsP+jPoDbw90*zky8K7;hCW$Z43Wy!xsXrA zOz?R|i$9LMV|IaJ)86N0-msVo>#(Rgto*)fRdhZ=AmUV6NBoM#-z?p=-#2*`93)1z z-Sc0yT}*Og{R*}!wg6ssIoez1E5;!Z{3Co;9jj`tBe2C zaFamh^eYkGDZAa^^Z|xUP;4r~vy)Rz+jI1>fs)o3 zrxJ?ZjJP-4A>+{%y!$S88S8x)A2=?DXj)3Xc1jcmO^R&v!d({Dh~JFZ#+y{VT)cw2 zFrk^F)6SaJJ!7DMN=MAeRzYvONvcimVO~)LFjAPpxZq69ES!>b;@e9I?1BeBaeOBy zT-QnAGAlLw&nxNHizl8i+VFD9bX__!(RT6@2+p(5Jk%I75bjQ>21%}a8i+oM*_;`l zp;KYQR)h7R9lyjOKG(JH*8?}qZ2HBR5y}P|`gePhEt;gUqdz4~lfJbEl?rn;xJ&5` zxTi~u!e)}0R|TF`KsiS0SFq-wCQ`PA*eg z>3rJA1d}O?h+s^YoKj-#E<4uGgA)W!0S<$~lhpG$cTQAxcNRG+L>!9%3+|>yQ} zwBaQF31`Na+UKAT*Nx-2r8K!UAYD18_nh^8i_4{afiNO?liq^A{d=qkcEVsW<>hDT zIyNTeKNY4yX97g6<m*Xo$v&r{F~T3cZD$Qm8`6urhG$iqR(1ou&b%COtSzdG z576M87JPj(LT#g#W!o))(|B0w;z*{11m}Y}`qkmChs3_2H;l6{+oJ4hk>fMLb}z@f zW)L(3%knf2B(DOg-#^&J!%s9^dR#y3}wb zBN2AIEXijC^VA#IQuZIsj1ZV*n`3K)3_mlOzW(-DWBKJ2e{7?u#=HEg znU6DEbx~Gpkvs3g{cGa>5a^uZ!dzm#hPu>xwUG0SX&@riZ94+iC7LX?@BXMF>M>t0Xn7{ju{W>VR1mBRC&VjSy~PU&a%KbN#+=VvsKwjT z?rFxMl~2Cm&GF`l#yk~f9KYc=T9C>g^x7-CcShi)g!N?hcce5-MO!p0Gq&cPFfQYc zbl=oYzP$M%@($a{L=OKB{IWkaaYy79sh`a?|6znagK zsSozcPejWW3|Z!G^(oLgm0Qe}{vQEv#?I5P?%R z{Zq@IO>SZ)u8OsUGHd1mb|mP6B_v8b3L(jGlc&1#C9kIP6Y)hJd)L`1S+d^S`-*+; zk5KNnEo-@i?AgqRb#72IY@g%)_@X+t4gRR#yhyMd)_oQw- zGkLoXj#6AoGk7*}ect6u-?Q#gfKA8Qiu=;P& z1+&P|(w8se4%dXOR4qEz2zPJ%t$YuEy^N?y0{)uU$m6r=#swE&zU1pWxnSnO@Co>m zOmaXZTIT;!%yUk92s~s53G%1F@?#17tRsSn7Gu0A z1}9g8dAGlLFANXBJ97+i3t)WVE_Wj#paRRj5=TQ{pfj2l*a2s4L{2C-@FziKN3KBD zgQ|t7PWLg8HMpiD8U7WruNL$b)&Q$(739&Y0f*Q^e~}#oL0X~|xdurlfIMRIUI2)i z?l%py$n83Uid5hrodigA%N-?W|DbZK8sslO1~v5t;EtT_(-{QCn19N_D9Hk*$h!LH zzThIUx4}JF?nc?0qB(%YhBqWx4ygh7k9SoV|Nqn8c}6waw&~h}A|hg?_o{+)=|#Z; zB7&gwrW8SXhme4Pg3>~hVrWVgkS;|b5m0)O-U$$DLhm7zxgY$#Z}yMbGqd)ZKYOq3 zA6)O7mB`9-m+QLD^ElP#>7_)dln3!h zW>9&+rRbY^jU`B>rZxWHp9VB(a_ay_2$aMz-bSO4UlIQcK#S|1@<91O=0H&fs7=4z z-nrJd`ix%8O2e%n zvFSAZH(1U&QlF@t3fwlB>2VQw^;Ls%mZ%3QQ7BsUkdqDz5$^WUD~(1R$n&0 zgpKfHU8vZrw}xy+K?{U==JBWUuxUHJW@K?_ztd)~PSBSe5q|Msaq5_nBJ_C2&@UF3405PnlHNDH(m15KcG` z?Mjf;F_z*ETiueC<7GBorH`kj>qK6*#y`O=gJ#e6a+-V%*_|hA9wwyiWsjq25?^Sk zaN-BkBl5H_s$chvxotMryz=Raw-+jY-(je`aZ`}Oxrqc_if6rKH@?xW-4YNyptIC# zmS&(N#qUvZ`f=QmcN3)F6LBagG9jyNLYVF3KR2qXkFZ>G0AD{k@x|VOD^MIBA^Goq z1Y53!BC-hp(B>eFBA@~!ujH)DiSOONiB99Cvh|6!IAgps8b47EYm(8a4|D!ea|VJI zc)R9xsxyG`j@CY@sC3R~%EM_xs2Cksi@vx*-(@G!&0er0Hv;e*hz_ffA0JN31(U^0 zjRKY2o|$Nr7d2_$RK!!@4Oa+f_fpqhsokfm2pEo7fF?J!Y z>w3l~H)G)Mmd~gCtYUNfXLW)vfvY2$L9`ey^}D zOvah&L3*S;l@knQHE>yo9%h^u7yc;#sO9o6a$S5R+)BN%RamvDe^Ka?sb9T}lH%8k z>Oakor?Wc-2N}^Qt~|VbnK1gJoK@b6z$0mvGU(f-Jbq{QT)`mvBHQKHGYnC>Hv;7^ zX2UU1Y0lMc<(WDnBYx)@6KMc{#BCS4DY{gMa%_n=dov2kQca7$1- z^jfodW*O!&4cdR4h_#I#Pc}A$J*M#dA_J};jBcl=tP`=O%w*oSC^7v?U3LEDlJmM= zzuf}YUij{CE>AKtM$*^|{c4Mtl!m|{T|FQ%-9g|AV9{r2e-u4+hrNy|qiXJ16+S2T z)&BiwkiO|lE>Qgfd`QV%YNl#EvcRTf4Qs=V_`hN)DVOXc<_J*yvS+L=Z%0Kb=|Z}w z^a-n->=@vPftIcEItb5&HZ*{Fa{5O~$~)*J{`=pq{^Moulu!LbAIzay^Pj$~8#HX^ zzzTE&gvxU+NACjE%#(ElfMPu9Hvw!SjDI1EQ;)QP4d>rK2y-2>sr+vx$46+XyZ@zI z=l{R#?tcvv{;Q7rFPX#t54?;p0Ukvvt)s`#ITNfFM93p34nX5M(8kjLeGt>7=pc1{ z4%6AQgy(>zcU8|C+I#MPO#6!-g_g;G|M~xKE|dTGWojdU`7#1dN1BqhGV23PoMV#YPF3_W5 zxC;BAKubAqOdZ-vB1Kh(pyb-rLY6M=kI)n&bsn_i9bYE7#io?MM!qI2lyq#>Fv|H? zweNxZ(+Ff@Qvh=Hj-7tM7@M%K-qeN=DXfLSgfxl(#yWI=Giy*QarCmv%6$hG zC2fj1x>{*4y4hX^gKT+bQyB=G+v0#lk2(;#R0TI{M!`pqfT;n(w24$z@7RakU-IzL z^H6-k01fYULL?GwZkHFq6AC~-+F`q)>{~( zf)702B_9~4mLM2icT1e02k=R(b*Fj=^+!DJ`uXP>jEiCy8pfEs+=ZM?a_^`W_6jjh zt-!Z1jTMDxHYq1tx^azEj*k+>`xr62N2^2)GA0(vxVi&=^C++!$lo@Px6;kh_f zVs6?&67AXK21IPSB#k^A5{&LpAp(v z{6wvyoFjh+tDhb_uq$qOP^yI(|AbfjlD(xdJ*sWRUjF0z>~zt-HpjQYl9wg$J~ezy z{$TFIRC!27YQJEA8U4d%RzlmYyrY}X-FEodv#8_#wNO5+g4qh7v$z7c%DLqp>N@4h z=<9@U69s<{*DBzulY9DVyZC6Ea!LQ6JiPofddAFEui9d@Cqt!>O_$#b!#FAFz@NowN2UtWwsgR+SRo_? zAeAukuViSX+?FnY!ZHQ*RDq1}fiGZ*jfD6M3p(E}J-7!`AE(~Jn4t?bVPE)OI%IX}rM84Zxm-of&{1zMm zM=B$i(A0=8cw|Y7L%T%2Ah2nZ7K;&?HTNoO0?2X(tdaSa#x;p@Ff{2bi}PfY6qYqo+wuvvJzT( zITxv4^x|H7snb0;@MFoR9PG_1kU9Vxz74{jB>^Bd8a}{7QXhgO<$&FP31hql2Oc&w zzcMG`Us?*%_`_YWuo1hBDdy;_FNn1;;^*>31{n3u8ZTx+6he<-11EBXbwYh1tQO|G zCum)bg6JMXRZFx2nWuGtVg#*IngzIJ$V+}MvT*x}2RU9_K)p?>nx>lrH6{pUpmO`R zd;5DfX{rsAGg|DxO|5sG?N~BMwJ0D@kA^8$tAyBTja7Et4~XAw@QL)< zN|27>aZKHg+*NT%WS(WJGy3jIk&^vT#w~EqjIm1R6@t^KpnB^gb`~-QK(_48d@mzbOQQzW&dEcqh-l$WFY4us6`A}o_P z|AQpQ`N~@esHhM- z9sQU|9zG!WQdz+*%;jYF)HYx);=1oA6bhEf_uZ=;0j_kPWH`iAgQI6!B;-3hPA-=q z&@UHAQN`ni_eRH=9;sV1(HS6fla!*cDzV8x1qN_*PE}8k!5Lw@50k4vfAK(<%Lv}j ztH+>PJ;%jCl@`Hgs z{0~(O_!+Y;icMO9?!XJQOLEp}fn_c4v4IE}b?mQ-pCcvMT_aleK{ry8NH}P63e>TG z%;U?T1MLbx=33Mm%Ls0{kS;R@(zo-!kDQ%0$cNph*GmrQ_(l3zHn~gFQKA4g785t( zx?T9OpF(kPQyHhE4EtZ!>`AL|_?~~quq;t=zX;BLSVHBhz-&fh(keqbNkzuZ0Hj@` z`N?o0j~t)DT|N1HY!z$2Nwci6i9T^noY7P}D*>+JooB!@bRfIG(S0xyH*$V`iruS@ zR5_(;uqGLlBzP^&mTkU;1!Rb?&hjY@O*-QS-V<2oUmE2FxtM5*s5O;>oR3f6(Y%CV z5J!UWq1#GvsbsfH|FHgaZRGY~lunt+R`ta!=5(op{9uBCczg>=JQL@_-OUuDlHS?J z&heKIas&oYz1wj?C$G}K@oN7IC2wcYvGUIGWu!52y9qQ{eJ6jH;l{|jEb799K#sN0 zhsR*DzwhI;nUb4jC^a2yt*;b#?hAU&XKU~!X}a=zxcLxWejKGv<)R#ncj1Ntp#=N0 zld@*_1r){@#y2%pu+KtXzUz zGq2LNQ-uS+7CUI$XG}5bED>!-E^9TTU2Hpl++L=rHs0%yE$FAU#@+CRk94;DxllKM zoBAXnt=*sH1fEll>h~X2bGdJ+2QL{reHeNq@CyUW{gf_f_=7-GHj6eV$gzO@dRJm{jZAMGhb^ z+k!Z%a$O-p{XFthD^epchWuMHC$vx^c7# zN94-Bm@>)laC{_og`qJK3kI1te1-i(gsdZjnl=XELwnN$?a#<7ul)ixdNO~EA#JyF zZ@~+%p{Eg#1>h^BpKze^nGlrjp`8s0ly1NCTJ7iWh?T&Y+;*;$F9bM>5rF{>5+R=j zxZ(~`7$Qoas4CyXE$G~P%LjXnqw5NYQ#$E?JO6Z7Bj>wG%J+N~MPgx?`r8_&I{d`i z8yUjnBt>pd_G6zuH97b`sxo=RTR?*Wqv2cVYt~h|Y<1GO(yOpM#Y=wItDkLoQ!%I; zsv$3mb^fr{Rj4)$3lnp?B@>(5=?>(M-&6J%Ub^KlP8p1}e7xIX55HA?MeO6GNZJE| zrO{kIn^56cR@;EFB=eJUV`T^$u@T93E2|+^uL$wNYj4crSU66=lzv|9$cilB|FYgB z{Zm$M_4mMm9`d9$4ESUA#j+jx2rRQ|Yl?TjkY1cQ@#SP5(wGLeKWP?L?_y}VN<7Y# z>z)`eUMH`f;QYAQOTNJgMI$FqmYW_1i($HXto^yRKsw6B#YiAFQDB4S65UL#Qxj=H z9RUkm)U^#g#?$|{U`O&(>%+=Fji>x~s*b^qx%4{VPo}0mK%kRk_94gzCLWM)QppvLWW_}h#yXk(3on-YN3dA3e4$b;JB#M{-f_Qb%y{6 ziI@c@9GG^Wl5@i|Ey*ah2qd_2Zq%(*&{GiWs1IwzB+?<(0HcmBr)&ZU%|RUpz#p$K zC)l8P@hw#H*{$WHAjYdBuHztvWkbor4&#+nFCgJsc>fY{C*^S6H{WR;Ra&8UQZ8e} zk@x*+7HiV1@4@WKL+c89GKrU|`I-gv$+p&95uYXwpvj@HfFAY?%N#e7z~^NSEX%4% zlSmN^E?j2sf-YCcNizw7 z&*uxN>>F!|0XVM(4{S}}2kq&BvPIov*kOMkhX z7_cLIhqHmBdt~odV07{0t+yz@5%*y`AA>33+2>vMvlZ|&_LYCG^z&w3zl^_GXD3zQ zVyJOo8{%@z?Y%7Dn>{XS;J`FSD6y^0`5rDG8X*=|tM4#RCpH$<_zY6%objIjH^Q#MvX}}Bl3VSs6+x3g9Y%?_g^bj_4~o8zXgH6aNz+AKf67V-aT}%*!~2_k z`qTE1fy1PvMsEfr8J%~@DjZd8cu`b(RdZoa+ZLlM-&R&x^WgfmT=*t8R!w_qBTb-h zmdfA&SV5Z?~T1- z74!3x16J=CxgvE&tlqFkZs4bw`;A)O;~oO;T8Wwt#acI(t_*5OV6D%| zBKcX@0$)bmiYaYSbisT%PwKyd^k^nMY9eL2qE(8`(0_;zK)KCw>TMet^*KRW^hQm9 zyO*v*Xb@>Fo1BGsn=_<=VkLDF?kOph($p(Cd+|zM%R#(DcHrq@TuoOXs-ZAU@s1+< zLIX%czd93a(`p(Tln=7L0=yW*VpzTGr+eZn*wwX|<2uaFpB-(B-G!2BgDPwD&NpmE z2MT+&x@#=xjz1^CEdk#hgdOGi46#yg2|P$cvBfS%3Miph7!LbSH8>d|OX?MU){ePs zy+1_39Ch);%*5a%Uoeca*3=kyWII+R_2?GM33~tAHUnGp-3=1E)~R$1I`H=7{f71F zvz6e6HVPE_BT(nSSju8$yyn1nY<3xHOhX;Ff?nEr{X$bebwO}6L?B3qpHRuWumJ^%;;S|>}P4}I3)W%czp4~aIx0Oaz_s3Gc1#Th*4l`f{G zH)Pgjzt)jT7sS}w9155V#HzZ*J~*SVOW?gfts_D9iP}xp=Cj?J91NF$Cz{Hovhx~O z#1IrOI9&Gi5O;$Ji4Z`B)goh(*lkczE;7cD=|Y8EC*AZu0Em5uA{)lu0HF9!W>CsO zSa8tT^*%^2-?*|?b<@8N>J_@pcX9I@eR$keXg4g534@voM~PX>t3Pd6oYnNtOwJuQ zW#ng7(DpkPcr5tzXT(Hzh$AGDPmTcIhuvHPs9)Vdhg>`C_G>X=b)hC{-5KB^Kew!1 z)WP+;VEwfH`GFy9%MNPxF!Y_Mh8uiKnv#?PItUws-o<>BH9FGfjH;@ zl{dnWMwj<&ob*jC+RLtsHpeuSyk6rN{~YejZwUSc5KpVOCHet~y5E?e6a`R#948;P z{T?r%#Y}7fSbpyAdMDF{_elzei`av&yL&3%yJKbMKk+Y)e3W&dsS`;YmWoxzuHqUH zh;d0S$!SmSn;Ual?&-r__q|s5wbo)?mj_E4mQl>A?qe^jdoH(A*mkUqwgUsZZ#9;t zfeEY5WaEB(1&l@336@^XSNQ7y&@^Itz{41~XEjS(_Ivd=^IoG1x&RN$3w{^#R};f| zJ>8y0(3mOnY>W6G5g^D1>~0Xt()b7fzCoRSndecXImWflkOz{V^zXpM?F&{9z`EVH zDVnO10&|oZNabRFl0zY7-?<$~X!2lV<^j?v;kId9E(-&~7^Y&XZ>@lg5!d8XL?Y6q zuqS~Bzf>Iyuso4cu5%jgrK`IDAij49%2?n0{>58e?oiP#7EtLs1x_UE;UNN=dr|_p5NMNKhPoQz*zdtrm zWaAflQfvll;r;l8Jho zj0mTL#Lhl#t0a`WellvCrNLvd`|*&I?p8TbR7howaQ*t83f%PJH8N#+!#IQbST6m# zvhmG71oGn}&N9)daQXo|N{#CFz8tXG+o3UBKZouU$u9pur_DGWfB$CkGnZLhlJ41L zE_TtH-r_N`0?vrBKAx6Y!8W^Kv?(DWRaFD=@V4hUH-QTVJmj$jUf1 zm}X`LSft>>ZH=PI$$=C7Ef3FaqAaQ_ zz;}PYjSKBN{2iAR+OK`3z?mz zzI&H%n$>vxDC^`6Rf#@NOHWz&aDWhp&0EMl&~PKFA+z&RilORxw<2=vXRE;Sh0PYU@xSfu_R+Y;aT&VOULoX?F-i=V|y=9 z#0-!1ze;>P`#Xddm8xDSm`zVX*gGiWmZCPu8N>J<;(U%H_1yd#vM;xGQEET%>M3|r zgF7Lv5I9csS1s6tH~4_5)eO0FeV@eeE}5cPZXQg_vL->=Ej(>3<~J#fo*2mm`zit( zA$h~eo}GB&Mi5pQ6Y_bocBMJ(EBLajtQ$ZJ+ZL~Fy5z%0jsN*7e!VZeT}e(JD1u$i zw~spYP~acSc|DT3Rt|IKpDug2uv7M)b3wzuha&X!Iu4wp+Pu!&Uo%&Rx!_hY=dRL%16PZZ9L%u1 zx8yyinQoFNYroUJmWb7*md$!X=&g=Ubh97KjoEQnI5D?_zcfo&cKcSp&HlycxG3ua zH~VIZ`p-pKa$#N#+wm@rqw8=BrhPj@yDHgK%R+2}p-y!x06JPoUd#v98}E_%$H3%MW55~TAf zK5{jseELvJX_X{`S86+&S+(@0~e!=JH1-vy!z)zFB#mPg{vL(9@v0!gvJ$08oK6RgC}u zl2!nK_y+|!VZ_##p#}g51AtVOP5h?TVWCmZvn}Xvn>=HDPP!W=pIJRkNE;Ce5r z#TO1#zvbPb%y|`cdhb`5ahf9eCOgUI)7K_ks{Jr^hQlt=UqshMe;LMZ8C(l!jojh8 zj@>>!TOew|(B1^U%QsE3neHyhMjkIE>W`r&WqbRdEj8Ozq=61l&gE=5u(4&>*p%QP z5#d$+=U>No2?YO}G;K4GPls+b{8VwkiOC$Aa4|wE4fEdq@WWDDjdJ?uEG# z=6ey|<9`nRe)#X;pNIcG^S?9re_!aoUfr!c@r8~GXXtQ!vgZN+k#wVxZ~;@)n&w$3 zzekH%Y`#^Q?HuZxlT_<{BEXGS0AS18#|T$2|M{p)k_xS_Vy#feBSeF^XXqnI702m! zJbd>iupLgVIsa zR=`Ew{rS?Fg~+9)T(u+95owR?KeB6L;6!xGW<)qFW(GHer2i@e$)z}`PrWdbkk#kg zYyP3A7hRnmUzf$>u~`i+F;v?_&@gXh=`fYU6UqFs7{fe2KO-J2r|YbUm?SIrY2ES1&1`5?c4WALAg8 z>o^ejjca*?6SHSSiflMV-`_q7vgr+WTP&xy-sf@Jo#{Q3-N7$KlWrA6eW0vfF;A$A z?!9+au)>$z8| zgS>We_U4@a!+ZV9AAzIY+6dQ&>;rZ*zG?_?GQ{atc$1w=uFDXLZkuVsdpnr~{Zk77 zU=Je#%&pP3+B^#>CY|Btq}I@Wtz|A!Vb1aQyUB^ygTc>QKZ#(*uS}2}!9|vx;t%=rXAsiEGf01%5-R}# zU6R6K?x%{u$#q#hN*zkEvyF$MzjB8E;}vAXZq84p{D8a@i!P>^3#Y!bc4e+3&)6+Y z91(wEO6ES^=@__s1;*Lo5N5|DO*X%BiC#~1$UxlQD&>%PL3~*D(Pyj8_qW{Ab{W;9 z%wy(iam=`AirOG;L85;ycX{F!rpIK4hSc%fHs|<{3l7>jacb_bdA`$1PFMU=KOgUl zzCMDd0&-dNG_RNakObGxo0tw&+6`XU{^T6Lrm^a2ImEbA%SBwAhrxZ_8;#-X*McgB zly^J=b@)yv*{NVEx9OErm3z7a6YB(d&f`We?0a`#3Dni*!AcPGHl1wbGD8SghC21I zx>*+|SR&*HcF4fv#2H!?f}353t>4|sO*NK`0c6)7*&i#gq}~p`R5pz;$~!4jntEB^>{FfJ@n}z+6AM`$@)aNat~1{%`M^f_K2rYvR{j9BiES$ZT}VatG;Wdr0B}M{B#a#xeW8$e5M4zI zXsshuLK%L-4PgcVkef*S=bjd!X#UFkJ&Gm_gz5e66~ski)YUnW6_%O*+RX2@T4<_+ic%@%j78k_7f%E0u_!CFQvVxB=p8YyWs_%DdFXzCkqNd}9f81~1cr}u>{Fo7S2N(dK>5jLHgt&4( zVaKCL&$APqe!ZhW$KRjPM%iSrBzHwfk3HP!7q8>K?)EX5a}~cuyzj#p zP&RSZG`>bW8w9wdJQDKVyFU8x1?Pja$I2jhpol~+&aU(1S|e?FPVDp^UsbBg-klYkUqKw)I>Yz% z%5vggWJn3CMsY)e@?wuJTIlER#xFd|-PB?{hHREbH}QV@a=SZ{)31i2;RDXDFYzRy z_Qr*Mcnu2z&tY51@L2Kl;jWXEE{b!>d2Sc_UR5C!rK;Ml6sQdP}dS7v`ii-W4N3&`dCAp)8ISa{zwr7 ztW51Xcz5Nowb@67?gW(T;H=?t#N)np#I|FdiII z8(5!5<<#RV&p(U@2imy)Edu7#phB^X+9v3@w2MFgncpHdmAnubUo-2FS9)i#j!w~{ z{mBQ;u}v=QnD;KOErr|79=j*0`v}heSf5SpwUdW9-(wm*^2gNnO!h}D(0dcWotX~- zEgv9pm*=c|clEU7{K+bepMK-;zwT6)Ap;hyWOh00nAtTW7j`*6h>?FCFcp(ziA;i) z(P!1l?lQlaAA3LJmJS#c^pf3~S+Z8hPHX#^(3joGme+C5jO4a`a*7xY-W}AiPdTv@ z!(Xl!F+lXYPXV$E6J;RPlX>zBE~~z*DqGdJo}}clMVDs|E6>o!SM`VMLSv?U--$iB zcj3KNFMop@(~4^v5DteAYFbVYh=hZofsuH~_kA{_jz|0|R;ycjRNQ-zw{JO# z^UMtZcu^N|=O1>T$aTs2@jKOUbVKLl0-^cXUY_ITVzn%wGA@V?64uUqBsHs*?UhB< z570MjXn0?=>StvL$g!uea*H!hIA|982+GrKFj;xtI9cMYPwFfbJ5hZ%cJV}6 zHRw@;?v)A1z<`+%x5Hx!0MS)KaeJ9HVj#V1zZ*Oj7#Li43WMLj@7da%I9Wyczbc!`6#jYRzw~ z3Lj0K8p!Vfqj>_s4Wa3gf?>t0GrAOW?5l_`{UV-k3!ae?Sqa>2jpqrwZhg>?OJ^2o z^mvG&vu&d2oB2M3vqwHi_!*K+nc@T_vJyw0x^~S`0f=uA>Lr70VVP8!5BUl^X3qao znefF)I<+jN_r{W=Z;2*KyuHYwOkRL5mFzbMIWo*39*Y*tUf< z7t)FNOUk|0L=C`6^U-8`7VnK#z3k$~#_1yneR;ToIoqy%fYx;~ipPXjlZwkj zU^>RQy06W;aOHob(A~Hj;p_2zJEcKE$dI`g9SgkoM0)v$RIj(>bf=_{zSWYP*{sVL zPt@)WUzbU2nO2>yr8WHe3M#QBvTOdFLrutU5L-{%YT;DeG+EXAA*Vvx@N16Xk?=_O z8|qGR{$fM*29}IK+jJSKK9ftFo8_~oLx=V+c_!bqd5o+j`Ld5XcQ58dVMxK1L zWznTe!aQYVi1C=pBRKTTM4!6hqD(-{`in34Jb2K37S6bMOKT>8-{tGFY;Pc;UCXZ( zt~q~kVJy;9hXk8mZYY5?^=U#`l^BZxnKQx*RrBu|J`L%%$2Hsdu5Fw%rzqL>T+{|P zm>}DeGku-2$*hI5dgk;el1kjN-&E|K8YEBmOw^!q%W+G#Yrh&Nov(cFdCc)=TM$~V zjr6I|)yMjO_z!{LY%Ohz4TLn*dr~1=2w%tBuCThe@ z`Vx@or15?XwipV`Fp>YKC%5%2pi^B6UX#4N#7>|5ez_-7Mve5pWJ>=*xom@Zv7vF^ z7nc(;Xm3iUrjfCz?u)Un7xGiJfwH)6?>wu6N&*1*jjRB2zn}YPkw4V2n)}oKqw}(t z^n-Q{=bG)7xjdX{XlVi__zh-S#k`G>9IaE%6mx@&<6v2;--CY(Q+Xnz&Ubgl;~o4U zbIS)@;AWpr$yo!`(i^5|Go1@Ka5|EsVk>yUxt(g#7`W47`yyDRTRd1AdBLLlJzs{- zA!T1W|0T&Kw3eL8WyvpA>Iv#OwGMhj%>?o_?9P`Hx46rYLInW)*`fO~g=mRS`xC+b zlB2`aM9h5A&r5oiwWYiHIb=||0H1^up9l)w6D!G=BP|+1WeWNx3Yiom(=VN%QT0^Y zNv0WbIt{bM3l_vuW;U*-yacH8XIWa=!nDUMz37ROuk=0zI~VMbjbzJAvXIL}>#6`e zHycYbeGXh70>8#)cB8e@V!=1Q{>(S>_tZbvu-S@t|HcuRFBIliQ=P)Q*(c<2qlZrM zU1K#FaXmFi(a0Iq5Bq?hPo(@VYpaIoxS(%ALyS5h(# z;^90lQ_~6f=uHi@dy>d260x!4ATpv3>F+j|=%$t)GspM!l|o$&k~L+-yCJxE3Xd~S zZJQ2s{p4+1lEi}qX0{^FYmhI+!Yd37lhlsTaurXUwz&I`6z2elTuNPbU`sA5 z>4F74mh!B49`3TYM;Ag_*T5}eFF8F?RLfoVHnaPr#f{`*dR$h}X05#G;R34QDO3a$ z$T)bHtT z{r8LYievXbwb>4*-o2sxb>W~|Is~-vEmM}jP+B5)8hG*b_wk!O;_^p<(BWwOc1Y5i z$`mdpZ4VtP;-Asu0pmW>!aUbPsV7_;sLZpyI4{dlJ~6{k;#vFTR|$8P3SwkMPMBEunG@5H?Aj><$<7`1-v2M8>`smV!$ zFr`D9B)vn5T?X0883F?f)?M=uqmlYMS!D}IaLAaaIYvY@x1xM~!v?z{YP$GZ_NIZi`L$giYn}6Oung$L4Dy zqgtlDpqBuGgW0~ro6#c|7IvZ+%;qp&6kji+eq;6)@0n$4C|xm=hx*wv3)h3x&TX67 z>XIP|pB0>dUef1No6E&z4frytfo}fo4pMiu&C&)*kGOuN&eqI}7%f|Vh@91Ka0K9- zuU%)MZ$%G_`;oK@mtn2Q=rVN-qYKV`OEFzuw|$$Ln~=r_#elA-*ttzpsY6%M!4)av=W&SfbkE5$6MHo{$p2`C^qL zqutBJjsfH;*^|kE9`m2b;!nF@-&lQG^?JM-$Vc3lGR;lxsIz6=aKtbi(TAeYy8d?0 z`WUjT_l*5XdTEEtMTyc>6{$Oyk$JD!<+-9p^WLN?N zsd&kDGBp5>lhcu!Cwm|&B^{FnZ^=`ke-Q20#3IHgj;R=j%X8VlsEwPzAG$!4MH4LG zQULFcHIOKX?g-qH3m9y)O}ngMZ3w`D7;3LjB$%m^*4RV?;P0<$Jw#90Nc7cB`v&wEvaKTR+-g#HQv@irp?vvO zCh_6OVl`{wupFo7t-_NGifl^yoKcadZo-`?8zVpt6H_q(ZsUZA)^lRkqWDT0JC&;& zErVz|W3(QiU^sb}EAha|GxY*o%a2rg zjaS0R`|L95Z}cteX=O$1P6ItETxd#sftHL>d@|nE`2*vtAyoFx3TJiT0~({os{vnT zUT7u#ig_o{WfRzida9kNKZ`+Jm#jCk&5gdC7%r+y-)3M3(DFn?8; zsYj>p-`l3FI40;-P`KMVP{B0!wKLClHpa*oP^JeyT81S5z_^hwWME0IL+m@M%d!ti zDX3^Aq)DvIw#WFEW{@k!3eZs=JXm2Kb~c3HeIHnNm+FtGkeev*GIY5j3n=iAsP&z3 zVB2y{)go%2cQ8E=wQ!g$b23ZSW$JMxS+~u!rDJuB>q=2o zvhG9M<2PKSL@E>lhwlb`3G{CgbW_eA@5OmUP~|K)JE0Yab$SjgCgrc17>G`y(hb`sYgR4Zr8>n%=&8(j`R zT4z}e^e7|1#S$z$`a4_TGQ8jHQ3fPfe{lD~wk6)aYv(zmLV68B+EvuYq^ibtdhFsA zdeh5QqKmoky4)d?aO4|Drm>6AT|BHRH_9DWsL*PtQK8Mc;iE$~8! zD2Q+SP0$L%W81R0gfTe9u9&y0#?a@rJ$A_T&<)2_xXy{;O9vtJK99A^i|nuoKdNQ4ri(jhy)Qbxi3opyvJWY92t=@S=r$W+1uw`4A6)l*#|*sVg) zyK>TMxstHF#s%limy8QYjugfsuBOTrob13{W!uFLELJC@t%JBEs4?|sN2~>MluoZ@ zHj0U&I6gzIiiiSUcv+9a7@%z{_WehwuFqS(OGFbDc#Lg9OG;7(Bf(jt?ChyEtTNVQSk$<^H zW;P~i`Vt5#TqkJEN^7UjL4_{bMSPLdkNMU7_Nwq{8b=rsThBtVnl9=FVpwSle+#UM z=8w#1K*kn*jt2Jg3I~t1&{T%oyN0pm@u?xc34RJPPd=-vnZJH`QUnU_TJ@TB6p21%n|h}e`}ea25H&v`vhsK*_AZ9- zRuHs-1rkd)a@63{B79-hLpK~=>YLwe4a9sUbj-OFORFoW03PB)QBl5lGw{92o6th| zy$9v-b_Vk!osOZBcfvHZlCD801kTpE!L`i3$N}V!FCMq?GnUFa79P`LPOD3PtaXL} z=lO9-|3{91JZ50Jy`*B_i?tCEv|&(cw!IzMTDD~n{Yfj9&2J4F{;Ff3g;O+oWW@==4yuS&50}xVX>CJgIOIN8#ZmmG zTe*qH7@dd#|K>!mkC_+RK+3jC1v=iMeCD!_0f{cq45s9fHoV3C9nK{JZxKn|UFgxw zlu%Z&GUyt?{!Y51DIv9^Gt|z@yi@+}C*F8m00C(9-a`48Mh{CHdA#v`;>!(ZIVwSG zY^FIs$k^WjWz|+#`EZ94G}DhG@hHC+whE4;D|b0-zLP(&fIfzaq2KTfn)2f+X7(P< z>W{Bzjuzgj%jU1kpcdmYm6Qoi1=oU19u6_mG!zJ*NjeAnpUqWCwjH;1I$D&^xV$Q= z0FLKZ#cA6+lZd4adsh!6sDnLJzGgHE3o(flhlhFlgkHmXaa`0Ht$w5T@9G1GNCzIa z@IU$50WQ(I&NV*WSg4);I{Y1_@zQ}s3>71l@$21ifn9hlBJn-jqBBw zw(J_+#dMe()Y0!NqN#k9N6_^T17==Aczxx8x|8!bHVW> zcKVqEOFL5Jh#{3mh&0dk5J`H{*1nD3DRf+c-$LV`ZmG;JJxB0W~ z8UtheR2!$#bC)0F=qsiOZp0Jy6HMr*qSM#-!C`o*7HBAk&KFvA>91qs%0`g3ws`0W zy%|u_239{Af~>(jVehAC>fB1-*F#O@f76fuy(g1>qeWTwkY%o}e}i%Sc%V&qtR8id zvZ>u7;x0N zzTK%5_%2>nYESXYvn~klVq1G6l6e86Ma&&v9@_r)pUcRuQyqnVg6rETOTd2Bo9CPj ze$5$KCV%V=-$)uz=mjMfytaBIHL~Z@j)W3~9 zKfDy}#<#+z{bSp@thiBrIFRvOW;u49! zSjRU~%y&`Dy)7z-C(Vb0@NojRXD{Q-%vU)LeM&b<-CQ0hQtH~dNv=+Ov+}jej(qVJ z7K0Td1T6>(?&D590SPgmC$EjuyzlTATo?w|_Pv0G4db5I&YV_Ru$bQn@u72l-eEgQ zahOd(A=t&p8J4Lvt*d@sdh4_+@7ovBVR+OtM^(?yVILTwICb^ANA++|+YspRlI#s= z;YN{jmzL^mffWwY0dvyu1f^&!x5SeOUcSA<4 z1krK{16{P~06gBY0W13*sW9d+buuStAQ%z>aV}LUzlD|E`U`SnA4YeUIgSlpNS1Z=>p$>&*XGP-T{v!)la%C zRMpHv_a;x|RCid1e6VBC;9=|yDc{}drP`#FjtrP0zVB8w*st!<+Xu7&Pb zwY^!UZZ!wqvk_(wgHm_*1liI;E3iHvs%L^S@9a-f&S)fRIJ^1A2YhoHP!Tk;QCoL? zymMMw*c8YQZA?nk$4B$mA>JMk;taphg$zH>^Y7FsZq66iX4eZMg6h4h(|=aviS@5q zBc$6Z6+Nr%%b%$@wPm;8ev-<6 zF!ruv7E(=BLM{x~naaPkCGY+ipwPE4lhxil8~NnrNEH~^3;E5+2-=|xe;Dl+E~rlI z0roP^+5&=;!IBe%!{4;`f0rGy9045-DouSXL?4sZqI7QQ@(5p3|}Dq|uo& zkJ38ausH5)h{+r_;yHCQa1SqR7x7mUUf#?*C6^iJ{7}Vw&i-;Y>eXfO8rs9gTbi@1 z`rpq}@|@i$heWLwrWW2ieuVC#zTK&a32Ad~bVd!^hVmTzx^%;*u7fo=xGGWNFD&|` zR|Hc91TvB_KPKJs`JkiXL^+(?jk57fZyBw*?N;N`I0Pq}v;5s(=8}DlLd+Hx?$0!d zh=qrAISPGb0dj|cFm=OQsNnScKCUol>YTV~q)V0n(RwHxJAB^?aW5x=$$KL{ubg*0 z&ML@h<>>yYT*8;ImV;{Oh4d}O;XP$cW{-Ttj-)%E6Ry{Pd?#2f*|Pq&8F+cMzU5qr zWjgW1Mmu|x=QSEiumqS>czn69YkWC}=R`Cm1>v+CIuI`0<9gf=w><=%u1gq-?PG?0 zZ?CX{rUW6IpRMQ-j(pxFWSvTzPJLJ%#KC8+X+yngT$H>w4=S7An0IB%16LIAQ!J}Z z0j+`9v6K$wwhs|me&KQ7WWTYb9~Yy$*@w@W!61csj4sTJGU4eFmuS0Vs_Fub%ldvG zS2Pg@(;a02dgMk5;)%Zr4>7(>y=?rzDBf!yi&8NL(|^&6T+O^*6wK6f!DVM$JM-d8 z)}hc5mD6-VB+>7U6gEB$@=ks}eCVoRU<%QEPhKj`tNf|IUMlIQ&qao)+MahqS~^ z^nIo-OU|?xGi={`)d7P(#e>>J>z40xqOZruuOeB*egN0+-~wwFqP{Xim#e_!zCLC> z(^WwQRF6kv9^wi{R>~^7%K{uU1P0Eo-V0%hZc;8M{u8dM;|gN}Uh;ZM+IuPWOz*J{ zAW&m;kTjW#SgLO_(`BMJwlkm1{FCbTtDvzbN!2stU=M!}K~D7(?IC*7-Q?+0*?5qi zGUKML9#rN%8th@Q+x@s#j|?ytbnfd zJAwICz62=vRx@ClU8(>2Fd+N5xFM#~R-WR}jOr0D;WKSLU{8B#ByF#s{}p(&~$aFNB+; zN7}NJyeBN`zvlW25Y0&X)LR+=a(da{X62*bO|&T!%}Wq=-B)>Ik%M3?P2C~5fvzZ@ zR@LrNx-}Ejso@AlJO*s_Ri8nt4$bF~$uC%YZrktvHA^f}pwXGrE@Nx#NNfiOILdC; z~lM!-ynFgUDyqH`E*O#;sH>#?` z+E%VV{U&sT%czN@Y7Q`BH3cnOt2dAliJFX$y${n|eE6{Hkv=d9XS9P=oA(tCq$y#W z!CW;Lhv4*7pTD8{L|`lOU~6>1BmpeiCW|(haV|X^q#VCyDm`(6E`;b?o#&8g<*|w^ zUYK#C!t*#hcAlPVZh1Tx_P1CRP+HBqks zn+1y}1_-PW)-Pe_EFZW-TPNj={?PWJR?GJAT(2{P=in*G-`lu}}iHABM zRd*h(n~FV-Eb7vIu5)y$^mUw1Yg)L2l~s=*GqV-x+|p!D^`WyP_)cnBO(VoFO~gHq z>h!Zu#D1m+dEj;-TqKYE(+xNCNiXsIlD}*NP;LdtiCPFXQv#eYAS`#SvB?yVTZHgI zF_1Na#?{V#8cVsz+fSUDN`cj-oKs$GVv;Shm~z2iQ4BnCSVwc5-(WDPH_L%( zo;AM&pps=kMyrbjJg*2g|8K=6jQ&^N3vkFnVHwml?LIYd{6^(ZLh)R8=u?{JBn<%9 zR{l%|hRjbWb;QJ-sFG&T_tf#PNXL6NX;mEV%HC${lW5bL!Q6gm9bH`5@xo#UE$(!k ziE}?st|(!!={A#ty2?CAbY+!^;(wv-mpopo zr^ortyf?eeDnz_7scHqVi+_hH$;h@bMI1yQzpcU`d2IYk1Xo{ z(NOFEx2w;6`~PWVwzb;Z2{Q-H3r_X- z&{?|QRY^1H>&~2>J9hj`mTHW*laKC;{CS_*?Jkrg+V}ZTFq~)9UC3Uv#{+EJQMJU9 zLU4)xEHU71WDRWl7FBVzCY9nbFQyc_a8eyLXA{}c5e$1X8ZRVmBZ+0HPm3@vVfd$d zKxtYdhuwK|^uNsWsR2%S8dQKi1GHoniGgphZ7L0D&DZsGK?>F(CVkeGl*iX_io;*)UuB$iKVP&ByH9V6RPEGN0zv`QcPTyXcyN_z<^nFDD z8oy7L0Gufv$K|lxDWI6smPSp50S-JiTr~k}$dKFq<5ttLq=I2hf>-ABxHr455Fa+Z zx3ihUJ^uMmf1W4-`0%$IQ)-C3gc8=d@Xs`UMn~IfFWJ&@q<+~o_+p5hbgRtLN^b*a zYpV>x)rPpR;tJ}9un z=}yOjpxthn-Z%#w$68E<5B5YAbNQkhwd|ViM-q8k_n%z|kSR2e3mqQ^zs5R76yGFV z-G8EOFrHKD=S@M!QBMgM>K~dm2aY7KgC&B5L_SQA86Nbd5qT!%9_^BC~!bWqW^oD+wLr7>m2iP~&YC8FZkZTxr zX9VWUGC__bwKMV4=6lbvg~O?N9LV;fXY!Q1(q$KW2TfwrD2*rD2!imIsz2 zgU?mQ&iKEL%j6n58vG+{TM*l099onusd?9>)h1=|s{)ag+#W1oka$gwKl>BYa7NEV zd1K{HE%EwP+CM}P7_T$FUnc*4A@(|&sd=|r#p&yx2}cdTW#=Ntvv zdqPb!bDi<_>J=jGUke9YG9uz%%K}ZO<|)Sy7Xs)CZ+nC$w9G4H1UE?ZZ*4kIJihZX zEAQt;sKCW_o;%9!8}*a@zG9feqq{z~4z5Af_Q4YIDcGaklUL3c+#}4))|{4^X|Tp&G5}`hVCLknF&G$9d-&6?+!|@9r2g9 zPwYc%n5=VjSPMQZNTtR}?Fc>d;HVA#lAUkqdG;;HHlhAfm(L)Nu4Md9BTb;+2RXF1 zf~$2$_8a&0z+~KXkNi{Q0}BlESH}WDrv^R#x;&)@Y5&{Ds{!%=ANK96zRJKlDJ7pg z#A#!$?ii07S8rl)igTA)>D_7LgEY;PhpGVdSBXKjDf-#khy(8H1Dew_F*O4mZYq*Y zssqNhAA^hePZ$fEacRqYQbQX1CJnX?lS3D^a-ZV_HoEz}73ii5)Ars-Y0`Fwc?XXmeU@3<#n z(wvuJzaSWS<4bUbmUl@jyg*asyd*J1sXb=+0#7-z^39_L%72j1bqV<5QS;;)?+){3 zVfZzA>Fyo9j)=N5w>%-?MOOb^NsCH(ehq|&O9zgiVTOc5sy~5mG>oS^_3e3jJFCvd zk!`w8*L?&bVaHY}L{Vkb-FOH0rD{u8aw&`IU=c<)m7PaMNQ6o_QMK{z%trCD_{3DK zf9=#4J?Ag5bw=4uhnl`|hZ}Z;(6EL#>KgH_-%gHg4uG}cu%usAdVoXj>jIBtAaXRa zo7kzIw%fGZl_jeTI?zL96{J$MozR?rd9F;$mW!)YpChcVD}H>?p>M!}O+`xd)4WMp zOOjpwRm0()iWu({U~DQyq^U&9sz&Cz`l9DL4D(&H;hU}uBUqx?hmc4!%Tmabqs$ja zINvBNx5qi?r%XIg5cS(LA(oQs6Sr@1RL|^2+$m4nA>P&640fE#d6?m)5D8c!N#kq?pK`8#=W~!(vv(fV!Bul_mFEsnX%od|_Eq z{;0xYrpR)hSoob(2kvX3r8&vX`oNobhJCbWvD?e)7F9Xh#!-pFUakx#e!Lz{>&F?N z+0%ZbhYsm=dgy#PO8+-A+ywv|p6pb*)`05jkHD+*k`=hbDZ_LFZmrOLGmi{2RrVJl>;h?`1{M^QTa6xSxG(j z;XA@Cb=9ejdTukA1WZJqBFQOg04bC8R&#=mNY3VIUtjjO?4%XlXSoHm4E4RnU@ESN zW|#uqjaHVrdXY9a`WjDHTU@eyG8alo|xa;_hbJgk#az2m5DvD8?C<~kl*(#6TeFWE@q!oKfkdM zgMI9Qd$6`g$VF2?Wfx`+I*+KBp73-vtQ4(X>hic;of@rW`><%~4gM9>mEG>mA&7h9 z5ov_(t1f|9V{E^?rVJtAzCIccJ@ZP6X#p}={c4z0f83;{`Q|iYt=x!tx~T`ROY1vv zRbAE_r<0Cyh+2=lxxDv8ZFgF?S9`9%v~GB#6uM!eaG&^>(auROqU_laorDUd_mG+Y zSS?-3a5hsQTOou^potRMZjirG8wR*0n|7Tili|ryT9Lk@+S9Mq9YYQ#fkxvk1Z3<%2`o@U`tF43=A>o1t zU~j*%fYm9QUFJ)H^}fYt=U!2Y5*7C(kcGi5=co~XF03?FH{NJ9=$Xs@Obti$!kO(~ zH?|GRS=)5*%`%vzWQ_+iAIqsv+xqX&@YZV+>(hcKb4Od%pXY;*89Ryo9Vd26CXo!h zsMYiMe>l_jKgE@QV~fAKki5uw9TOAyB*b&u?~U`Ry1FbfP5q==1uCS{qP?fv<jYbCxP^Vt_29cZ>)E|fqq1}%{p!9#v>V@eE&|W}GX=uSYlZIpKQ9o9} z>}9nG;~r#I5S;(kcmeHZ=qCIWzFBo2{2=J)G^XFT)Tw^P05uz7WSlKb4i@^uTYrS!=cG!@3|lwb*^0vRmaI1YPLQL7E3yBdTY% zsmOeKtpK}MFZ0-rHv5CqD}$~jFn2RCH6ouZo@msazqgRiXC?_5+iZNCNyC-0iH?G~ zip69e)L-75!MHK_a@E@Gg=}0UlKVJ%=}*2rYee8>ho^n!C=K2|#cdtl|6)^B)m{pm zs!*$@bi~m6+3A*^ihdTso9%+2J_5No==eR28#)b{pB&1Qy_VQr?;#jsrX9PFxHU*bu=q^t+@+6zEU22Cs7`K&C@28H@oEH;9Oh?vP4!Fg9#89uS= z-Y4a#7s$HW21NKK<9)xug6&qpnNQs=F&y~V4gJ&vE`@x++~2V_@AL$325-`#^M!W? ztHnOuTm0w-mrc5GMO~8K@TgI}DZx3;BaIn-ML4_0oYQ!??k3H{BElc`U8r7oST9fh z(KkJ!KQgS%6W;u+?i7z}gpaqBc`ohMrepXE{!eC}t8}UUnO@U}Jw6w}c!=wxU|0qNsSn3jv^`{tSP8uaD!p;ffdnVP7q;K3qrK`R$qB)W z5g2Xfb^@_qqW6`*)(C>k{YG|MgS^K(T>K`V@>4W$zxop%*e!lrQvet*ZnnJH*vZgR zWtmB(>uCdPJ%8L0fQ%w)d^~H&1;EAe!?TdQZ7NN;f#k?pw@ey0!7#a|vEK?R1_m@Vgoa9r>ZO zVa(A=u3t}n<*BGj=fz#-fyNyec$Ub~RlBEuJ?mSzN(-KVPpD z8Q&-r4Z6Rg=v7TMPEI%$|HTU7P{FU8S)2D>C3C;W16t=y<2z9zEGM!IY`VgPn55dc zraRTmRQC6R5cqI;LL23*uZU=cf#;+c#Nxiy!RAj#Vf0}4R@M9LZ5#QwO#eD9CE(6<%PJa^VmSdo!99;U> z4^fG?Xvh<(xJ#kHIA0&!fUk|0;y99$N=@fWbiZ<2y;p-xfmFIR|E~Ae({**sBhy30 zvbp<;v`zYVe|#`E5h{+elXm=2{l4qk*b~}monlLL?V)l>lX!Nr>l5>xVU&_pjwm@p zZ0gn9LNs+7R5fmY!Wa{NyyDSOKXzzti|7*U&lUEavjv)h?%|v98C?y63MARL96b(S zueg=0jPU9Qg#f3RAnQ3*LJ4>H3d-L+Q<3mK?Ly?ATt$Ma@^qTgcz=U~R%iOwY(Ysy zN@j=7r-QLM{&HCn*%Xc!=EzTGi%pdjt!IRaAdNlgbNVIc5l@v8wQkx`?9HmNv|E}q zP6-Kb+dvdXVW}iBrH{)$_xE^pRJj#ZiJ35tX7Tf-rN{I0}WnQMuG`aOLy4> zb&s873nNFg)KP(e>jBDQ21McC7=NK}yt?sd)6+!hmK7*d0q3~XTp~k7!Y$+GD7`zH zKrmi7MVK9%&C;Rd)u&9Xo@+LK^J>_yrgOXC=~*ZOu1SrlU@RH-YIw}hCB z1DeFU*NXJWAzBerVjBFC8?L|gLwJl{pC6Ny82TK z*{}KyNj10iTv8s&S#zK&1;(3hF4xB!mAMwu-!ZzDJS=a>najCM_`Q=m3lt~F7K0pZ z^~aeSsxD*w2Hpd0uWGe(CS`nfwJ}U&7&ktqVUp`-nyB`MZ^X#BVw}D` zm<_xPZrwVWSM(SABX!5sWQxO?xgrq*U}H~|Dv>7pW4Y0{+#(p8XNq=Qtc zQUVAPq$X?>1S!%xs5I%)0@8v?FM`xSfY6aZ=skpdOZVQ-+0QxWJ=b^M>-zrr)*o3} z?Y?K`p0(EeX6Bw%vB_oGiN%LhE``xab;e0VEtyfz3wwHycSJVnvMWDUkI1d`y8`Wd zXCY9ieMf7GF=Bu^EJy@uob2sV9COeYRDJ>FkCAg7OXg*Qjx&ZlzJt4Gmj5~LWdY#G z|Jks|J(!US)^Wwml+gHbluoIpMMrI207n6q69{F`>yTBd9~t`C5HvnD@21Ar zet%ELX|m%%aKTRk1EbNXgGjqzL(SciCRKJF`yY7*V-1B^{PCq}y*%~QrHbS4}56eJ1@hJxfMIk#Wzm(1u#?}b_!fFy7LRN@8bjC+(2ihJ;YX}X^epl*Jqg(8y&KKxy% zD)e&@Zb5^JO?o$TeCpIC5^xDyiaxKN*a7Cd*Q`uM$XnX8SQ~-D14oQ9zS2Ma%~FR9 zpJMNRbcXfDq``;U_c5zfx;;JjcXUwMUIlDPyF;|R<-E^wx(T+(I{f$clt+%uZ$48y zq@1;Qdp4Di(~dPXk?+*yDO(*WWL_nE_@Yx!B#lj7+GJ$k&d7;XI@{M#$jxvh0r@2K zD;Ti6;URk}II@LO$z;8`tEwvPEjb~3DY}hW^0?cTS>7mYoWEbGQnm5N z+ks7|eYrgOi;Uv8f24?g*7K_~kJ=Q^Ka z+}VR9&6}O%e5967a$6oD8m5JdWGFK};))KwCEDMtYxI=kGLvS))PE-3(gZM)iL#%-97rs&S*+mamDBAE>gdruUV2}*A zzZwG4CJN_#3+KcOfO1wq3WoPa@6(#ix5VbF^+d(puZOW-!O!+{h@8Zqif2;o6*RE3 zPRwzU{&92g#3UV0^*)R4H{w!DA3uTjGKHI{`NUZqkr$G2_Mt{Q*hmkk@BoBKj*oYT znDASQ&07SREAxd#U7{@U3p<`v7B-GKiY4|Ems3bk;fmtvr#Z-z5+ruGD<&V^4=Z!S z!gBA`ivBO(LjbK9Y&4Xm1(HD-Vjx59+uX|@bCQ#xn;+`T=<|OxiDY!-|S9I^01#STf2n;M$p!S)4e1NF(r|RKoTkxgHPB$ zkQ`x7`W5k43kpx+pKXRgDE?z!B+%XSu8Nr(N1U)z-N2LD-nEaQv(Q>(CsA^qb@>l% zQk#;+oKqmsGk61eF8tj=01SiKZX~wD@K=}HHj#u);)J4<{K3V}_&aXbLFU}V#j8M6 zjQY6jE*Wf(eTLYt=(nd!&_6vpB<@Zeuh`0))^w>lOh>W!*u0pHBxUJr#f*v7h9^78LQ` z7{Z#d+S$^7lhApacQc^>xRkE-UNiWYWKMv$(S(K8v{(r^At2maH* z{C`Wy|3W1^?|Gj0H2zOA_?M<`{h`H^psR=n>OS72J&$HQt&RW%CrjS`GPC?`b(L@F z9x54n`#vp1l2|dZnG3l|JEMgrR2?acT>@L-3HLYpH6wee&K-AwF0|;odJ=GAOnu#IR4yu z53PL@4BRWS=K} zY^o-NCYHFrfC>A|6=_;kXDY&i6!RBOxO~Ktz0eq&pqKTBTv;>8GZ6I&<2;KnfRj(2 zoCD@YK`ZELkEC~y*V+|3O+H-0geFLFu7GCdT-xq4US{a+>+#_$&rc7ybW@h`G{tPt zWVucUb$qn5=C@k*Jj)A9V-t@i*XTD@oqMy|h&Xceg!YK@T7IS<+18tjHk;uFJEOS3`i z;U?~H@bMyOtH8t)DhL+Oh+b$Iaef+$XLgTe=YDP|A2jn$yIg7Y)d{M?3M9+7_1>$i zcH2Fu?h{lFCP>H$mE2ZJey_+}1y6JS| z7x3{@70L4`ku}vbNUF zxHTEluV&8XV-qCu;dM;mQSdY3$lMa!Mlw2c21D`5xXk$QC%UsSpIu) z9X5Ht^1KbxG$qY`k~z!q#SVjHy#&-#D=YB&$oWSGDh}@XVCHMQHA2zd_hJH^evU|> z^1Gf6#JIGc(EE2#Zu|MYy&H;RN%z{$H6Pb-!)SiI`tgEwwxVylQy{&%!9yyIJRco} zXm*RhHbLE$JPL(Oi5XI7F8c1>7`{5sCf@IXeDsHD{EGdrA_Ls8HN_hf`uniVctnKr zR=StA?u0faa3@;LU*fSf;Pz{&;FJ{83)PDiR#f}Q{m|FqOglKID5N$Yv55rrdBWwR z9bd_yRFu|dpQXkCqw>bMV3AAZrmD4k4q16M1}V07vhI?h$%|U%>idQ7gWPWd50tnD zcK(2-zAe!vp=R%G6SXA7c(VN&VIHsD=kHSr!a|Q4Ms)vBlb2 zeY<-4XD0P69veZqlk<&glkP^Dh<@yKVj<(|8}cFd*rlnU?M;O)-%Z2MQ5Vu9JOp4C za}zmgvfi0L*eLMQp4P?;Q)R%M`KkfC?p@R49ms+~4EM|6Qo0|vwIhZRofO4F?Dg#t zwS#QqhOnTNI7DvL!I8p48V7QH}HR(1DH0y1O>1g=E%-5b!+AR06R_ zCd8$JZ$Qk;5<}`fkf8Vq%OPZIEzVAj%e(o{QI+^G-@|f43*R`rWTN;}*)_b{+1P3( z^AP!!W(7T*zS7QRAewE7u^#GNI<^|egsujZ<{q0R8Fb(6rjjy0aHHVSw58!TSV@Vi z$t%0urICR<)aYmdUIw#|4m*cnhk2N&<7=G)2tV}>Ur1fO{(4_#A-c(x)3)|S)-~jy ze)JK@OuSX9K!}eCDJlwCuJyaZAZ5N}N0?3}GGs)bLQ>8&p1j_5d}z(nq~qK22Z-7M z$@@W^2VTMy>i+PY=Ri%J^W)ZCy@{t2EHo!e3Fa&yVb#e6#G=R;TW26{K=ZlA@RY|( z(2Ov+Nq~GrgRV6vMegZCvaL9}Rx_IW@jpHuBrifzZ$|nW&Y=0ePF+B{#NZ+E%1ck) z&MU(b{fib~^9~$_M#*SK_1hJ)6Iu^RT z6=^~a*ASaxiio(L1Kw(fj-fTj@}L8dhbOH4U_2XR9Ar6u2~oqPi0IF}{4}pqnd0h`}!yKn>y;= z?JND0)mQIdWxm|%d{`^aJ2%5R8kTr7=!(6Hhv+jB9 z&-B^rToHO^&b^Ia*a7*kZf_lU4`A?wEa0vITBu zQ&8h2@RV_`qSLQzKVm;U=XaoN;gG3ObUk(T)t1UpPS*exjwQO*FM5_f<2E_t?9(or zpp~F5ER&93_V;e<3XB0d4@B)lf1SWuccfzhWK*nKJOqX7JQA_yFGL9d)Uzv-8u}&)srDT`oAy)M+tGqdj`Ow>yk%OD}Iz*87b>HEruaW>&2h&js_7O1z5G_ zN{ixwC*R1POi5meZ9_&-77EOBV9=N(`#~{UScnQP&=$1ZP$5 z$N_V`1G%o*4zx>YYuwykUb48}>i@+xtRSQzo_xuIa;@CU<(ng&&vE8Q|4YPi?a$$x z`(oYk#l5rdEPIakxtENu1e|}iSyZ)A(or_&$gH)O-_lzg^R4PCd9S?X%xxwB)_+ckzz}lEOf>YK?mnZj!NiTpDf6tm(R)aryX!zstRX(?hLVdIDvXJF_FJh0v++T zkP5banEn1eJ>C2MM)5PfczPK3js@D9f=@N@ZM0ml2Y2U*Bb*Q|gQW2y^T3L`Z{(e0-<B| z-++RBBFK&|+y@O)lMDiI11Q%-`nyQsUv~emfCGRh{sIO7g!s>BVGx?22ISKNm;Z@8 z&fCya-Xs0v7gl*kr3`2}7?g9NlLk&s_6uSB0wBXPV2}bM)vqFu|NX8bVx2GXecAKX zC!I7AP?ODaXs%mgX@gON#yppf@pNjr<~hduU%-XWzMNCCxNQha9fMu7ea$Caf4j!9 zAaMdXU^ zK?#7IfW!1Q^ftg=VE^lBO-nT%o`R)JHTWjxsf5N)S1qkx>0?|1*^6z;+YAg0!{#y5 z9$_umiJuL*k zTV~(~#L)p<@D{{?{;d>aAxWfgvQ!lLCkQK%>a6O9x))xbM38x0hj$SGZbrWf8AXqkTtxk7FQI z?GjFHQ<0M@B@*K=bz&4WNt)`tN(WHkVFYzN<^C5eDeI=t18&vn@f1Jipf|?s!>Ng6 zM{^Y{>|H8#!|(kF%2pL!VKW1$)NdR+KcU!%Gwf43-_U$<$9q={+pGG#xrirAESi3i z`?1e+S2ChnCN!d_B~@=rODIQreLUSmS%=j zzEwX{XdkCk(7PU_@B0J3GfdVwYInRoLf)@`v%)?3)|b`ApVeHg$2~uid>S23TD>PI z!tEVX?Oh_-cM)$Xe z+?twMC~c*e_8DD2!M{|-*A-b_y24t`%I#>bx$y~D-43jrZON8vHIj&jQg7QoJlja~Uz#X< zZZKaP95e>6dAhDW!qKvsyt!^Q90w99X1Q_yBDDhsEm%*r96`fvHuj;Mr*h#+4#?xm zy6UdG)HD-D{&x4-97){|{AGHHARC*f9T)x3Lb^FyyE4c^LVJ0J!sl8+3MMspr3B?x zqy|FAKAc?>PB8oGs{Rz~&C~j%&>#-98roY)7IIPCblNZi^JvBcP>;PhR3vjwh(gLh zvKLP)dgEKJlgzt;1c_s4&|GBJ6Drco_lb4Fkib>)lm7>!WH4*JO#*SMk$5r5&hotl?SGX z_n%lT8s?p@7t=S!EUUrCN6t7zVQXY4kIez@4wGdqcvd7Sq-ZT-(#b8YKDc<&)t)|n zkez?0McWvQZM+&6)KlKXHg9TOc8Ca+Ds#J`SN~`Yl0;WlBsTp`7B(Y zH96kPKev$u(hs+xwybC55;z^N%6;^TI#tS1__oP8SirrI33m+ zyGx$96uX-<*_DKyA}bCsDRDT7*!N~SGeI3;%sKh%UH-b7aFLr3^BP^ZUa=A}WKdBx zt9fmnf~rf_Mt&c@tGyzMk|0z%-jD{X*YH*q%3ba-naN8MYOA|AYVE5A-=>9vZkWED zozhZ=>x#t$GOEJ^_}_oNO{x>%*H)X+bn0=9(o382jFgfmWV+RalNs`ughexcdBQX3 z-f_ABG)%e%c1??2Q{I-+Q*-i@{%uqneXDrw_i`;8^c|d!Tl{IVSplY~`T20lOZQZ; zvijJ2A!4JWrIA-+8P}$f*PbT*Y+(W@{Dt>&N1YA|QpUNRYv?JfA!fDK(4C7%JHAUy zz5jrI!pgBdQ@~)OxOm)q-=A~A=FxR_?-nf2z}qElxYB9a$CqDuS_P`2J)BCBO_Wn%h$?M!?1poDsBYdn6C)eG3AGcuanQEnCVwloNN4a?7{?|qq1t4C#|VUEDh^(m9S>6U zop2&q)`llAF?W_CJ?w+OsU6RFFXU!C@jnY=nDzU4Ky!H9mP`z}G@>Cr=8+AQ1E>xhcNzh7xSdLX#u5i-N@@9cj^4YUh5R@30Roa(n zOLB4U1dy+|?UQ&;>WP8UJeC?SY4SP`{@`fCuP`Bp817b)zIT3llv}0sS^K^}X&9Nv z4*K4m8F95tDff)`Ojy%<-~RR;KdGH#c@u1PdgSzu+IWKONI(9M6O#E*?Z;IoO4$TY zRV$j!RdnxHrsnr;5qcqHM-uTh#upN`VZ50TE0MRLffHg7rD!4l8sV1yOhHs*_R_lf zLxx#Ll}nk7=yHeV3KjYxr?{heJI`;4+>o4jBZRzEi;tL=lknO4B|vi1)UC2=6JN06 zx3bQ0wURPwZKi{G9ck@D9}a6pQzV0Yr;oHA2%>z%NQ~piUKo`T8+lp}TDlmjH_YSq z^}EeBN~_A$%u#;hzaK8^kfLB41~v>Y;m7H>bC(~09eECFK#{f6+wVG9yf~$xw{9GNu(Y&mn#QjW=7?;?QJwEo_@!2X(|;H^JNi zBM7QFrmx$}3o=?>38ry^XOg*4-y2m(qQc`rjZtLvMI>%3)RbEY_19O@K;ie5&6=G= zzMur(g4S4Hv&$i+I*YebLi%L< zWq$Otw2RD+o#-3j;2lcO?3cW*_Yw|6cv6Zh6!*wbouw7^^E5&%OSg1CkXt*ed(M!d zvWqP3Nm$4mWs=KOsQm?Rg56ao{qMxvB!DoRN4z+;S$`G!OkRbT0(!<9eBko>ZHI2w zgwvxIhYBy})D}zqg+ya}t;+lkgX*4xo+!W56`DS%q25C5;l>T{fnCT4|=%Dcw%_Sy`$57 zEc=r*s^LN`c5~}d)z+hu$m?2@UUk4TvOpo1U16c{&TtH;5iu^{8&Xr1yR3_Uwm`Qcy;LUMjMur)ad58CK20DgrOsS-9D~>QPmmX;D!421) z!w~fXAGZ-j+|`q|q{o}xq|=fq?^yyInV(n9?5A=?vHvl6IA$1CiJnVe4Vc>(PdGhT zB9uq)u1O#GSPl<-J!LM=!S+5noIY~$tzp`18mzeL|@jU1b55QG0^@axh_6Lg1?4qMbdJC`0+jayPy}v?CR@UtN+Yx=>p6 zitrHA)=?nSMaP_=fWpM{fE|}W01!lgU%YnuAC2@jnZHp2Lc8Zr!~Zve_|xz&B^98s zL390T$StGoJm~k?2<=}G<3CRLCo%wFg&N#h@fY&E7xm;X8aTk@Alpj<6aWIPr1}Lz zY5$2J{)t8SC_uKXGfh99?hRFN{6eScrwDZbrr6ouL`-W2s+nHZ($b2R9|O1AsRDQh zSbnt)EO0Ua`2!1pdwwkdF!{Cm9}fZ~{#pRU{A=|Kz5H4L>V7T%Zw~%1`ugt)`Cs() z-%|O1!Sufq@_)hfzZ3HB&Hazkhc{|ICHXB|;0G?{EOGr=7&&PBbprrYX@@~_XQtbK z0cD2aDy>)tiq-~uTib@sRc&esXAeLZ9HO+nDI{WyKpbp(R{;-#?RB^+yTJLUpx3G+ z=Rl%ispP-p@mjUb7#O`aiQg(HcN0PKi^gAY_|H`v&jrfghe$5LQOY;Kv=T)Ze*fm4BS@l!2hD*chhWi=75i782d}_;4hTjf*Sej z3|6Qy){R+M-=i}g+(91Tdc6k9HYHj2O^whhtQ*$(zImLr-j5rx_{y^-yETU!>8-5Z z!00&D4Ii*lP$zq&bjXiY_Aj6RV_APZU%0Zpb1avn`^y+^&r3byosVp^d!c)Wlg@|& zcic%IgQBYMEvI@1SZ!EAp@F?6NSwZsdHs1h$f6ftPXr6>{ z?UkWidunV><6XK^cFwrkc{%H2eITa6#cvK|aMajDW?9bd;P*Li!S8l*W8DL4!w<_u z!sC$5gX9;X&fnGGhy8d3l!CSA;M$BGfnHLTRwAWASb?sn_%cR--ri$BzfGuCR9)BbYHGQ$FpQ5f?`B>{J}F(- zmC3^B2xwjl0Brv}dYi`3QPs`?ZoTJXZ<&jbN+eCf_pw?9k^aaJR*DG&F(Y9xF4*`~ zxGM2<=b*3E`Q0CG56{iVCEkfUJoPov{-mJaJEnS~AskGm&;D#rby;VmX$ zS)>zmIJEBVHt{@AE^^)VL7;+xXdNe!$1#Ute!loP_{F9xR-Di!rVD?bg^RRWIv*} zh2KfNh2i9twpXzW%iE+EfQ_mHKOB`Q?+z*Bayy()T!9qXRkgAl?giha%l9ySEKj6`oHAFuZucFH8BI@?N z@HL+JdO$ZnL``{H4c;lMk9<~|zh0UUg$x2UJX@${p%NdI_Fbs%y`z5KPIWfB$fnYf z-0y@~$dlS!v22|TGBp7&<@2KpxJqT-I!T*+dB+9TPr<32i}&JsztON)&86!234P}a z)F|0n|4T)-Hw6ixV7bByjx1_oX-=O~Qi(Z@+uUA4ceVsIwLTWVI7sT6b>IpuLuk0$ zRo3DPXxy;%Y}CQYMos<&2Uo)#XAr_hMVd2J4(;`>5yw6eo7~O|wIth8x*r(Kq1;}N z7N%-46s~<*o~PhNK9eQ*$1EB5pGh=Tpv#h~gU5#yT?(ehF!vghI4Q0_Z^SJ}{=#Cr%20ENkk=@t_5D?$#%_YZxdw{D517bUI=Dy8FqkLw605sqeSOTV%eDN_|;x z4GeCN9@qmcWV)|hOY*LF;yF^L7JuOERb(BgxXMTIPa4E>oa?tIV*Eo`&)*xg$Zn9} zhEf6Pv&JMAZyl*%Q+AQp8drEU$|X)Mg>hPW603mr~ewxWV1 zZf2sw5vRzj5w(YLU9Uo~8zt4Og)CMl*1OXPnsA1qO&ha5AB7B<;Uph_g=%Uw?t9f_ zv?zLg?BKoXvc>-W!DFRSkGh+<96(dEKqk{ig05~^iV49MY1KJ-&5{BesLe_9y$1Kd zEeC@=8cG`tERWfxXY?xG3_~=!dSrFD#}c+X_7C;99KWnY)2v|pwOS8)r?ave}5pV zR2@^HKkpw$G0o*02s2}_K`P276KS+E6M`G>wkl9})b7D@-a&uL`PFhi{KC%(`>Y z$vsOr?8;2bLJb05+MG66`T3#N4c(gW*u6*sMOUh_G-lU1#=2Xc-=QDJg+y_tob63G=)grb zQSM64b*R+bJGs+U%K^>`Xnp*SXGPPrq!bWbNCd5@w^aQGaoyT8*vc-lyB#q;6&JW# z0esw;Ci-38VEcKywzZ#`SG#*aFF*RB{i!K0xWhmfKJLJI*gev+{*L;lY@wZ0@ar~l zY5$S9O}3uHuS(K;x|FLqraQ9lA&l^^oVy;2&0O)HtQxkJuIf1Wne(Tz7JE?{ezizO zR*b;&M0#C{Kn%j-vWX6>xwa5LAweY?D8j)i09!5PzSA;u`>CAgFpcEL`fYAYiXEaj z<0XvGLhj<+yKm8ykLPyUm98S2=sBLYG&lSVKUvpwn~LBf3+Hc{R%@CBFcRwO{BSW- z>zeKME>qfV*;H zuSRql>!=ifpo2(?JwWb|OhSGDRI7kgza6c-j zEJL)zuWEZow0-5=$Nry6hw?nGGtZ8ruHGkUQf@EmRlzY}dS%zA*53`X3&}Z#lD}5f ztr4aIa?{s)J@4<9s#)MJYkdSvJyH%yJ+(Oz#T}>1(5d;DmNEP611DY(ZEUuNC+~m+3+G6B>R)4qFB=)$a`sEBay3?Xw$z zh51w;uyQqRuY)=i$s|B+oLCb0H6<<5-(i-2->?<<+rXWtt7QBIVQd+IpI@f>um8hC zm*XQMBAm?%I%`O6YyqQs0Ib~ z1YqH7=f7J1*T&V7n+$((!ZYeoIFyOm$COMznHWVEX*J3hK{>&}r2b;^68&{GQ_<*E zhYox++WEO`GvLk^UtUrS@Rr5Ny_FPEPs{nRY*4z#paT!G#x!T?cogb{>?3746`HhN z={+3V^T%+{17%rzW*y>!h{B=o!&|Z>bDlOwr1VzHv6Jf9_pKZ*`RN274al3%KkM0o z%JjZ~EkUuxMeL?nx(A1XA*j@fp;4QTi4PO-tth!ujck2%t z;eo47taoyLLeu4b_%n#4*!clb3Q}{U38xlwo2g-ooo5y0p{&keBiUzEMl=nIZOi!gpA;`RfEGDGI)d4B*P@ zf+2(q1`_jEcw%`@i0nPHe28@kp|lisIBVC=Vc#JEUtxG0;7B^Ww7n~v1gnitbp41$ z-h|Zq$nNJ~;?<8e{Zv*cs3-sy?hbfwB?ER>pCBMf&7j;>I#P>x3O-ze<4IWU+|U3C zu17{TZa{ieGI+%=7u)n?%vV>)*W^T-JN_f}&0djSU-GJ!>jfUZ4q0wpM2cl`ALj%m zDa?=Q1^IH2kRn zs6qR40iSxy>oqp;kxsf>ORvrbc*ql=(kXX;-rRP0*#0g1Itur2-rxs1{i}bOtlSvc zkziF>tzi|1X)g$LOGI7yz9IWCK8aPl!?ifLr1K>-xN_STbr`O|a&ls9&wRZnfU%m0 z>pHmBK_L()YBpAXDZ>c8$J8(B=|vd# zk{MfWxE|LrF)5EautjJYiGM^T?$mJh1{aW%Y-%z=`{CSr`wbe>@VChzno+NyW~<_Z z!=*sskavRn*VBDYvQGket8}sw@wJ*r-}#KXVH3^@_Tnsk?HJlM#OlZWDT!T?Zda)x z6UlttFiBLrf>B%pYcc3DiF>_7k;};&58~CG>5D!#HmUKaIgPd{u0~6;j7gbX1xPCI zgF1s@4F8a-vC)o~Gn5PMa=WtPX0;jYOqkI=a?Q~sLgdt zKZ{p`J=UXW(iy?0xXhrdw0#Y8o~i}PCbgK}Axs>kXfw!G7)4QvYVfws#n;5IE2Z;D zfA=aU_bL(FjEvBI@~*~xGQT_?2#2nQw1-^JGs<(b^d6jj)W4?RKBHkTD-_>1YM&)# z*chbL%sy{&3tbeMh4-yDSmM$XyBb|L!Y`}g+WagQ}i%ym?|h7Kgj(=46h=yDq<*ZxzJ5 ziV>?Ot=B?hDn{y`S|SDj<5;&{)DrQD)xD%@0%x*$@$3DPs`V0I2ie+Z6|3_p&f%3x zkF_r0Gp4&lt=4GS90EnBh_y`$t*`X76)f@Fi!E%HFm0~ovP<5$D_T*eLhc8hd9TF!F#ao5 z3h{NIpdVm~s7|X&FIT2i<-(+L#XGU6JCMY#uAFK38pl#`W@Qo1Uibc| zYcX$fgf2NjO!9-S$F+#9={K72kq)375)tJ$o**dXp5uANPrT1J4OpCMBKU`LWZVN| zZRI-=357`3hE>fM4a+}$&BjzsShH$+M zvr9;ddTzT$Pp@~0Kf+ZUF__pWN*a9xezfZ}oo|7WkX6o{!!%;m_xh@n_`J%8f+zNY zvL$ZLm6{$a)g3(Q9jN!xb`#E(pCPt8gMH&0O1Ai%Qqg{Qt~2OGg!yI+PO>2!T8;Bm z$en(?@kBvjRh_pr7YJw6;znCGOY$1+`ZuRNXp()F-IIso#I(> zrPTK!d9o_~psY7nYkwl7hP}x5E-6s)1tGLAmnjYoZn$r7yvVC~?9GlhJw__i*WUEu z%V#Mq0uYTQ*L&xl5Sjo_>|JiK&KpYv5=HhxKE4Z=cj^2dViY=IqLVi*JS=hzX&W@o z`$B{h-K}RY8?;d{ur+muj^+FI8Aq-a0KRtleUm;HB?<`T9g3Y;DpqCjNOz?B<7J*Ngnk?7!<$@;ly#=rc3W2SOcn)?b({BU zvqi}zL2IAoo7=Fo(5;HgoJ7;L89OA4lSHc;+dtsWoWO8&GEyalGllqu5#^)0Ck zou;1q2H4sRu*C6wUJbu8#SNzZ>L9`l$mGd4rYmajZ$UNCN7X;c?oJuRIS#jPs;)jH z|89O!13qAR5ajH{6eJD!Y|K;qj*rJ3FTGgNhRX^EJDPfY^S7yzR3&NR?^Ze#>ddVx zI?l=|Ctzc1#?EZEG)-WqmbC+?%XKLlM2m`X5}%1Td%2#@0i}u~oP?oZMHcS5y4WMg zJb{Grll;%INUqkDxE724I8cf{H5T_V7|Ies*LZGzR9(fQ#d9KMW6VL(Afdhn*{Zz7 z;WhiN0Pr?xc5CN=v!%c>LIM1g@%xYBmd2dg2EM8%vzX5EDkbw;NhiQ>3{H#HWVnUi*1WwJFcgrsFVU zDHWV*+}Ij+m92(8`$5C?B7DpB)>oR3dt7L*xxdAvS2kHXfAg#`y4Dni4`>uaMlur< zU$YIaiic&m90y4ewn715iruRet%x6`?^A_iX z;U&Jst^2{O@(Zz}(JYDQLq|wQF*>97PzQB5wVmgslRh8+*eQ3}C3W8QB&wT_9i}s5 ztXbKbp!Une?8fk%6u0Q9?wLTtc;mp6VS&pQW7-CgZ$XSX-?cA2?R8yo50tht;I@Ry z4q>K^gI4dI;G!dU=c;Nwt;aq`UY+|)7c@~W;DRnL34Na89#}BRW+)@kJMMJ9POHf! zqw3W-xeM~$Kxu^r5fgu?J96KO-MZz&f#~~8NoT~iv%rcB&0!EO?I^Fez-}aelJ^y% zqf(~p8DLsi79mmG0w0B55js*V3Ypww8oMFEGT9vXtu+t3X!Kc)`w`WIQX*JOLTR~y!)to zrIn@YV&&45Hs&S*(vg0GZTy6LMY?f@2QoF(6J67MuQ9Nt`K>+sQ4pcD=Du^|8Pp=t z2yuG@`?;Iqs7yUe#K;U_jjbc*Yeq z#Tp!N-oBAR$1o0m>%E-c7hvcN*8G(}f7I%iA{>929I%F|b#$V(Vytrg2(*?x`x6eFvTP*G`wJfyz6 zxatud1$i7_RmB>1^dZN6GHRrDy`xUA&s%ZLze7f(MEruo zZiBFSce_a4B@9L@g6=)CGz02f-yPm0p+9-2Ew1(tAFH^@3NbORvJQ= zhf{maCSmH3Fz0mEmqSP*5fyS%n!`ru^qf$mo7r@+*9-Qh^C*lycwh#K< zIL%B>M-4h`Y7;Bn`Ihj`4kb=ekUglCxLs#egn{$!Kx9Paw!L8as8|Q8y5qUo!j^js z6d4N@V0`*3S|de2tNOT0$|vUsb7Lpn3o^upYX=Y8F^;%jrB$bnV<#MS=n{z4EzSvw z-aDZsImEF;7?DQM&aJGmeWp`)U0E+$I^NN~v-h*@Qe64x`{ljQQNH2J4T`onwb|o*0yEq>e;+* zkS?K`xWWrNsb~&9E+NXxJUUGwZj4*Oo-`Bsb%TpfT&+)rQj&{=Biib>hHc(_DzIG< zZNB=#bGs^tx#VnI*X?Xnu_w~m+RgWDuNs5@OgyGJA$q3SKEBq-_<33=)|+ws`d*dP zgH25xXKGu9R~8m;wZsZ!EEzf2mJHA4*a=_veS=MKv4l|Dy?fi6r-U40g0I5aHir09 z6|;S4o}7Kp?Hmsxnh8H?nf{b-!Qt~x5mxbRv9M(!!$1_be$@)oa#qHE@J(TZ||+1(XGgO;@5$nqlAih_m!+Eoc`GF+2Op`%|XQCttWEbSHMs+_&xUxW0x)wAoZ`6ql!Q#tCn1xp2M0omF-Lk4kNZ ze_UI2*-Mwcj_SB^p1~Rg-gBsE z36$L-mgBeT+T8Sw<94#9b&UMVgdF7G0`a$#xSen^AYl;A_qOK9fmMB?kq12U7BThq z@NjxDlo!fHx4X}rSL2$hJLWNmOMO``xMZ5JM3dK_UeJ8%B)PO{S)|%tT4%*&HkEjs z-E(Mc&$NaW7Z3(MVHa7t!n~hGwVNj7*UX@v8BmZF5I5jn>|e3?Q^0S+$KPhm`KPK= z)hTol)ZHHixq;B3{3Y5@t>6B%i~M=)Ppv<;4HE;3;!#G&jXa?QR;lCAF>UogSAO*u z15Gd?A`m}e8ZuCaH-}}M#qES~F+0Tzjv2;3j&Bo#8V|rWO5)F-h&2mP>SDMiZh#fu zD%Zt{gUi(#OVy+U#dh^aD|m2&2Hp6|lNQ~W8IuU~6E3>F7ViDm+v+Q>>cws!jzTmB zUGxWxM%}fKXYvK>Za9qW{#1@fH6#|^P~TOHua9%iC@4)aZS+}e9%OfkbC7VzYE+wV zDWvZ-8U{WlApU+vQhL1innq)am%9Ko3(+q#9p9oKx5KJJ%|#`SILAx4^IzCjD0zG* zqLU+@{ z?VWi%TiM^oV`+`)q7!XZjnYaOqS7*?EgEfAMT8d7rq-%usI5AvR)j92T9hEMr<<*6 zC$>m*Lq$bvUt>3Ni=atKc;4dkAe)Sj;%GbvL>y{~^snTBacP^-QdP=aiirC!8 z_+KYX*2*o?g?z zCYibfkCt-J83eYIcXSVoi(Ys}*;YYt+xQRmV)h3CnAs@JBfKsn8gdMUbL^%q3lD&Y z{aHTdA2942ilJ{YnS+Bw*2%vB1P%F;h2 zKWd0c|1O>GN+T7vUlfE_6I^z&>6>7D`P)9v_) zN*mts#O@m21{y&(2t7xan?uevu@7Tg(SiO~?1n1I5CQO7O0r`6%X}?sx-F5iRvH%D zaeE}fX>G(6mJMIDkzacI8rbQ6By)srCW*LFZs!lBu;bQ?uMm9&ctLTl4lWzx(qF2j zH@NcDGs+Cu=Ud3|`L*i;kaoYV@id{%rvFVud0mjQb~Cw-x^M18pOmjI z_D;tGO$IoxJ6p5p8Oa(kQIN`wT%C4hOpjXbX+0sq4w@I-ENlJU*1aQte+a|38An1@ zx)lJCTdcT!MoDX$x|F_`ywvF$?TxdUsEi7Ww-NC{CQjY`Rxqo%qS{f*9Mx~4JAjq| zg1T*I0;x?>#^U`Rz-{pa1t`*RV9|0wX^_5S}w z=wCS1aUW3Rr%&d5`Xg~ydvbDeW`OshZ?2smr`fk*5O-RSTpT6j@_~%vqKux}ujg#E z_8wz=HthVk5#-a2S$Dj?GB}9ElFOn)zSPK2tV9M`MSpg7Tz2lg*byr7TeIzCGe4{0 zYiW?ptJeuRWopj%%UKr(*mrI!q#kbWK&nuBh0?dNrjK#_A)E*QQtqw4pbk|XBm|Fh zGf~+V5a~WMl)>Y{NCuL_nkz_!EF`6Rz_m@;q!1;viwjc@yqi*$-pnh1n8?*6DwmHh zy`C-iB$j8=2oR>zqmFzvu445xyCWv4;0j%-&qp7cC!c#_0&}Sn>(B~0dRZqb(~qgp zIjN9ccS~DFjiYJHs7d6uVKV0{Jn|#sUG+NLR7!<7DHCsf@0ZK{@oTeZ*RJ9eJAUGn zX8Be$2etl`r_PfyfoZu6`OQN>h$NX$T`a?baLAd*VJYNCEzSECGO@L z0giPk)*y7dw$-Ufr&+|ZI{6~dhT+0`g-}6t)&`0g`9iTcq(C%JN4Z?EmJkvl>TeTW zpQ*a#H+jgIyRZ{VJ10a>QHA|46;plJ%wK5Pu@W(rX^+bLNsvZ2Y%4RJ^XlNl9D1-> zn|nAZH}Hk`(~nCL_ldM173M%(ND%f=$f2l9y3kV#92W;tSwcKW##OO%E)EpAdM}W- z2UnO7_)-=_Nu(#PpSSXPGBwK#ND5G%Fe_LH?A0@i52}qbzEqh8?-$H)@z&I<(`r!l zw??FRgjknUU@3TTM+gJc_CgDAZGC$=X;&fF5y0Y+pm41c=az5WRgIpu4k?uUDw;?+{^Y96dZ?R;K=yZeO@Q=L%@(^!1o(L9 zvvr~cUT-O7d(&*kdFf$I&!|{SEjOX0;L{&Iz+nFTjsKSz&S%R$(0_- z>!UtGzR)P}_MF{I3XXq1gllz&SObjltJ`J4FQN;gPbi@d$)|PK286qKoRa>U^75je ziTFT9JSb8^mCpK-!e5IIq>{=8aBkO3(|SUb07ZrC{_9Qk4+c`|s1uvB+;`M%Rq`0b zGRYJiH2Rr~{nK}iez9m$m8+)c2FY38eyNI3VR|}_wA;IOx~zQ)nChsgApl_73NP@c z58CQ9Ut`$|^g=H<<5yWTOpl8gtI%#B*-eUBKnpCiL9Ho6U$tJ~RR@sGU5h7XO_8bm z@YYwQ6Zn1heYesAi`6(!w)|6*I1p@Ivm4{;??8HS-4YySDBl!$FZa;hImHkp#C(m% zR!Mrd2>29O`oyaHAx+)e@km;Hn z<#Sb<^g|1^N2R2KTpiP9xDhi->sAA2@@}6!Yz=4lu3folIB@1yFz+f34x3SKj0t5F zc3r_L_DnO^{>TrBXY^l!y%PAh^|)*Cqz^l1DJyRs#kEIE;rMdzPu3}2H0@y$k7q83yLWO z6}O>-)jt!+Hlvl$Kv*6m-?X>1uRs%pwu5$t!0kTX27&|O`PmNMy|CkCJ2IyMqg9_Rk|WgI#1@~Z-^WIB9O?Av|a3?T3NfkjTt zZ{ssB${i`_Mk~~ML3G4Pam^iq|N?46l4anMI8a`$QmI*ajV0abldZbsw?Y5;p-f+9` zvyi%heYoFn_Yc7Gx-JR=($O^x;xgySwsBIDgEe}Tve{oKb{9^ z>Gx77YM$Ckw3q3mvTfMhe4lCM+rS7YZm5gn%DdqH<5&j0{g@>1dUMd6c|&xmzTxA%x6oxkKUW<^CMIiG-sZ;_|pK$$o+I(0q#&OAt=%eo~SR=z6o2P{A+ZyhF=fgEh6-m}4w(ilRnkv~^ z+$5V`Eqmc`UmHegv7K8nou#a27<*bz<&sMj4@k4JEz|IA(0cv&cO{bzQspM``<^sX>YDfntgsp_9;z8djfEM?G1!E0RhgdNp3djw zJ3Wgl;9@&MlyPU-V5 zqwch#QFqYVai96`GxEc`66>vDN8*5;nz&UfZ}$le4;Br&glPl7hW}7iAEGF5U{OA? zyvO}&){O+d3^&aczTCFAR!H1$(-(2|u!`6+Q@;sE1w27%I?Z}8u>(~fD6l-^zJuFa z*cf%EF2nG8YfSjnema}}Mt-O!jh69xK~1j3Yy7?xy;J%tY*#GLl_k<08P7gFQ9zbwUD-;S;??P_5?GK* z5AFMrR}@nxIDCQX>KrQ6a-1CjN!hc?fOluE3D1iPm?Y!o$%ICj?#3BwuENqP3G(dr z@#ei-Uq@*?&)6tQ)a?*yU#(AF9tttx?KGH>JoZjXF?dn0=gYepk~t%W&}ez|Sr`0>4;y0j%v3b=c}iP1^U zSMI*n(LIK?+UkXq_F`e2O2O+x3Ce#=)j24OX`#>S$MLwI&WZs8^r@Y&cszP=zU^!r zh_M5AF^1-eN8qu{cr@YIUL)^YPo#@w46;nYp`)Lb_7J)yd(=L89B1xJV<4YZ?UlZq zige)<-|gER*^Rx8mtpp!KY@o4`Kn_G0(N)}oY+VS@nt>O11g!0kmL2lz|Spw;KC5A z%K8586`TeSAC89d3hOU)-!0#@^%y$j85BX^4Y${-<1~;beO`k05xX5-*8`Kx1?Zy6 z@eL{8wP3cecibs>U}M|2HJD3_c0+lM0!0~@0}w{phzm{G3hXl%NQ_x%>tWRIKF%*c zOf9`2cpRo>lRKkZR1Bsq*~|+MTl!nsx*_X7n8>FFgl=$d*aliQr?udyXXmA9c+%VY7=jPz-@ilKkjFD0 z>eNil=y6NFgsiBEkdJ9|ghIb2(w0%^ygaDn&L(oScmzCrhChO{-RPf^^X2hB)x8WY zf=g3u6Zk`9g2D@pN65zvuO1+h}1j(#6Q<95`NU!Zf^AJ!&pX z@U`XUB;(8Q2P6!gfk4`?F(z=4b_%e9b;nfBA37K=4vb(V|ET#GVLp6emdP2}K9wm{Jn&Nmj9u+N zCWW0%UFAbkw#%gUU6lUAg_Kb>kL%K1r4f97n^>f5<97s&*F~(+>Btbl1R^0!Gj_z@Rw^;&0TOYkmUJIDvp=DIJjv2?D2qTWBmw_sQ@JY8_*%k zOxV8>C;|O6>Z!8uo6rNsm0Ln zd4rfsj;rtyf} z2Q9-dtuOR@T8ajbqWoM)2yH^J!!XfoC;Wdc4Ijb6VO8=a>(MgWbP;M}W=X?$F z8aEGzv;D;fFnN|;xS^spfmnmzkH!=RdZym%;N-q&XJ75Ngf!>lMh_ZHEJ27LU5flhXt{lrnN_Q`E^F*} zxf_$Ixa?8d{rBp>zH%1ub*9WL9@*@OlIMZ`Ir__{m*`tp-SzlE0kGWOY>01x0npK{ zpaZrbHvrSTOp1D~4nUEh;h|`g=l5)Zj^g6~t$%_*M~-dTEC9%ok}j^?0Q?XC{#76w z%P`QVj{--W{eK2rQosjI2H>Y7QYyZYpHWUc;L!j7{hw@sukS=o`DtwYW$SH?n-_pO MYhhe?+WF3Z0V19CT>t<8 literal 0 HcmV?d00001 diff --git a/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.cc b/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.cc index aae5959264..307cdb7454 100644 --- a/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.cc +++ b/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.cc @@ -1,11 +1,17 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +#ifdef _MSC_VER +#pragma warning(disable : 4996) +#endif + #include "mkldnn_execution_provider.h" #include "core/framework/allocator.h" #include "core/framework/memcpy.h" #include "core/framework/kernel_registry.h" #include "mkldnn_fwd.h" +#include "core/framework/compute_capability.h" +#include "core/providers/mkldnn/subgraph/mkldnn_func_kernel.h" namespace onnxruntime { @@ -121,4 +127,337 @@ std::shared_ptr MKLDNNExecutionProvider::GetKernelRegistry() con static std::shared_ptr kernel_registry = onnxruntime::mkl_dnn::GetMklDnnKernelRegistry(); return kernel_registry; } + +bool MKLDNNExecutionProvider::UseSubgraph(const onnxruntime::GraphViewer& graph_viewer, + const std::vector& kernel_registries, + std::vector>& result) const { + // switch between mkldnn-vanilla and mkldnn-subgraph implementation using + // MKLDNN_SUBGRAPH environment variable + bool use_subgraph = true; + + bool FP16_graph = false; + if (graph_viewer.MaxNodeIndex() > 0) { + int index = 0; + auto node = graph_viewer.GetNode(index); + while (node == NULL) { + index++; + node = graph_viewer.GetNode(index); + } + if (node->InputDefs()[0]->Type() != nullptr) + FP16_graph = node->InputDefs()[0]->Type()->find("16") != std::string::npos; + } + + if (FP16_graph) { + // FP16 not supported yet. + use_subgraph = false; + result = IExecutionProvider::GetCapability(graph_viewer, kernel_registries); + } else { + const char* env = getenv("ORT_MKLDNN_SUBGRAPH"); + if (env != nullptr) { + if (atoi(env) == 0) { + use_subgraph = false; + result = IExecutionProvider::GetCapability(graph_viewer, kernel_registries); + } + } + } + return use_subgraph; +} + +void MKLDNNExecutionProvider::CreateOrUpdateMklDnnNode(const Node* node, + std::shared_ptr& subgraph_ptr, + mkl_dnn::Subgraph::SubgraphVariables& sub_var, + bool fused, + std::map& output_to_source_node_map, + NodeAttributes& subgraph_attributes) const { + const auto& node_inputs = node->InputDefs(); + sub_var.outputs.push_back(node->OutputDefs()[0]->Name()); + + if (!fused) { + mkl_dnn::MklDnnNode mkldnn_node; + mkldnn_node.name = node->OpType(); + mkldnn_node.num_inputs = static_cast(node->InputDefs().size()); + mkldnn_node.input_start_index = static_cast(sub_var.inputs.size()) - 1; + mkldnn_node.node_index = static_cast(subgraph_ptr->mkldnn_nodes.size()) + 1; + const auto& node_outputs = node->OutputDefs(); + mkldnn_node.output_name = node_outputs[0]->Name(); + if (node->OpType() == "Conv") { + mkldnn_node.weight_name = node->InputDefs()[1]->Name(); + } + for (size_t i = 0; i < node_inputs.size(); i++) { + auto iter = output_to_source_node_map.find(node_inputs[i]->Name()); + if (iter != output_to_source_node_map.end()) + mkldnn_node.parent_nodes.push_back(iter->second); + } + subgraph_ptr->mkldnn_nodes.push_back(mkldnn_node); + output_to_source_node_map.insert(std::make_pair(node_outputs[0]->Name(), subgraph_ptr->mkldnn_nodes.size() - 1)); + } else { + const auto& node_outputs = node->OutputDefs(); + output_to_source_node_map.erase(subgraph_ptr->mkldnn_nodes.back().output_name); + subgraph_ptr->mkldnn_nodes.back().output_name = node_outputs[0]->Name(); + output_to_source_node_map.insert(std::make_pair(node_outputs[0]->Name(), subgraph_ptr->mkldnn_nodes.size() - 1)); + } + + // Add inputs which are not in the outputs vector. + for (size_t i = 0; i < node_inputs.size(); i++) { + auto itr = std::find(sub_var.outputs.begin(), sub_var.outputs.end(), node_inputs[i]->Name()); + if (itr == sub_var.outputs.end()) { + sub_var.inputs.push_back(node_inputs[i]->Name()); + } else { + // Vector of node outputs, which is input to other node + // if node output is not input to any other node, then it's the end node + // which we will find later + sub_var.outputs_as_input_other_node.push_back(node_inputs[i]->Name()); + } + } + + NodeAttributes attributes = node->GetAttributes(); + if (attributes.size() > 0) { + size_t index = subgraph_ptr->mkldnn_nodes.size(); + + for (auto att_it = attributes.begin(); att_it != attributes.end(); ++att_it) { + std::string key = node->OpType() + "-" + std::to_string(index) + "-" + att_it->first; + std::pair att(key, att_it->second); + subgraph_attributes[key] = att_it->second; + } + } +} + +std::vector> MKLDNNExecutionProvider::GetCapability( + const onnxruntime::GraphViewer& graph_viewer, + const std::vector& kernel_registries) const { + ORT_UNUSED_PARAMETER(kernel_registries); + std::vector> result; + + // temporary switch to toggle between mkldnn-vanilla and mkldnn-subgraph implementation using + // ORT_MKLDNN_SUBGRAPH environment variable + if (UseSubgraph(graph_viewer, kernel_registries, result) == false) { + return result; + } + + LOGS_DEFAULT(INFO) << "Using MKL-DNN Subgraph"; + // use sub-graph implementation + mkl_dnn::Subgraph::SubgraphVariables sub_var; + std::shared_ptr subgraph_ptr; + + // We need graph name make PrimitivePool keys unique. + // There are several identical graphs in Model zoo and only differ in + // few attribute values. GetGraphName return graph-name + first-node-output name + std::string graph_name = GetGraphName(graph_viewer); + subgraph_ptr.reset(new mkl_dnn::Subgraph(graph_name)); + + // output name to node index map. Using it to find sub-graph end nodes + // if output of a node is not an input to any node in a sub-graph is end node + std::map output_to_source_node_map; + NodeAttributes subgraph_attributes; + int node_index = 0; + + while (node_index < graph_viewer.MaxNodeIndex()) { + auto node = graph_viewer.GetNode(node_index); + if (node == nullptr) { + node_index++; + continue; + } + + if (IsDimensionSupported(node) == false) { + node_index++; + continue; + } + + auto op_it = mkldnn_ops_.find(node->OpType()); + if (op_it != mkldnn_ops_.end()) { + sub_var.subgraph_node_indexes.push_back(node->Index()); + + // can we fuse (at mkldnn level) nodes? + bool fused = false; + if (sub_var.subgraph_node_indexes.size() > 1 && node->OpType() == "Relu") { + if (subgraph_ptr->mkldnn_nodes.back().name == "BatchNormalization" || subgraph_ptr->mkldnn_nodes.back().name == "Conv") { + subgraph_ptr->mkldnn_nodes.back().name += "-Relu"; + fused = true; + } + } + + // Create MklDnn node: + // Update inputs, outputs and parent nodes + // Collect attributes and modify the key to make it unique + CreateOrUpdateMklDnnNode(node, subgraph_ptr, sub_var, fused, output_to_source_node_map, subgraph_attributes); + + auto temp_index = node_index + 1; + if (temp_index < graph_viewer.MaxNodeIndex()) { + if (!sub_var.subgraph_node_indexes.empty()) { + // if next node is mkldnn node and if it's input is not output of current node + // if next node input is output of any of the nodes in sub-graph continue + // else + // break and create sub-graph + auto next_node = graph_viewer.GetNode(temp_index); + while (next_node == nullptr) { + temp_index++; + next_node = graph_viewer.GetNode(temp_index); + } + auto sub_it = mkldnn_ops_.find(next_node->OpType()); + if (sub_it != mkldnn_ops_.end()) { + const auto& next_node_inputs = next_node->InputDefs(); + bool input_from_subgraph = true; + size_t inputs_count = 1; + if (next_node->OpType() == "Sum") + inputs_count = next_node_inputs.size(); + for (size_t i = 0; i < inputs_count; i++) { + auto in = next_node_inputs[i]; + auto itr = std::find(sub_var.outputs.begin(), sub_var.outputs.end(), in->Name()); + if (itr == sub_var.outputs.end()) { + input_from_subgraph = false; + } + } + if (input_from_subgraph == false) { + CreateMetaDef(graph_viewer, subgraph_attributes, subgraph_ptr, sub_var, result); + subgraph_attributes.clear(); + subgraph_ptr.reset(new mkl_dnn::Subgraph(graph_name)); + output_to_source_node_map.clear(); + } + } + } + if (!sub_var.subgraph_node_indexes.empty()) { + if (node->GetOutputEdgesCount() > 1) { + // If current node has branches + // iterate and see if all nodes are mkldnn ops OR + // it ends in node with same number of input edges (mkldnn node or cpu node) + // create sub-graph + bool create_subgraph = false; + bool break_loop = false; + while (!break_loop) { + if (temp_index > graph_viewer.MaxNodeIndex()) + break_loop = true; + + auto next_node = graph_viewer.GetNode(temp_index); + while (next_node == nullptr) { + temp_index++; + next_node = graph_viewer.GetNode(temp_index); + } + if (next_node->GetInputEdgesCount() == node->GetOutputEdgesCount()) { + // if all nodes in the branch loop are mkldnn nodes + // then continue with adding nodes to sub-graph + break_loop = true; + } + // inner nodes. if inner nodes are not mkldnn nodes + // create subgraph (inception v2) + auto sub_it = mkldnn_ops_.find(next_node->OpType()); + if (sub_it == mkldnn_ops_.end()) { + // break and create a sub-graph + break_loop = true; + create_subgraph = true; + } + temp_index++; + } + if (create_subgraph) { + CreateMetaDef(graph_viewer, subgraph_attributes, subgraph_ptr, sub_var, result); + subgraph_ptr.reset(new mkl_dnn::Subgraph(graph_name)); + subgraph_attributes.clear(); + output_to_source_node_map.clear(); + } + } + } + } + } else { + if (!sub_var.subgraph_node_indexes.empty()) { + CreateMetaDef(graph_viewer, subgraph_attributes, subgraph_ptr, sub_var, result); + subgraph_ptr.reset(new mkl_dnn::Subgraph(graph_name)); + subgraph_attributes.clear(); + output_to_source_node_map.clear(); + } + } + node_index++; + } // graph_viewer node iterator ends + if (!sub_var.subgraph_node_indexes.empty()) { + CreateMetaDef(graph_viewer, subgraph_attributes, subgraph_ptr, sub_var, result); + subgraph_ptr.reset(new mkl_dnn::Subgraph(graph_name)); + subgraph_attributes.clear(); + output_to_source_node_map.clear(); + } + return result; +} + +void MKLDNNExecutionProvider::CreateMetaDef(const onnxruntime::GraphViewer& graph_viewer, + const NodeAttributes& subgraph_attributes, + std::shared_ptr& subgraph_ptr, + mkl_dnn::Subgraph::SubgraphVariables& sub_var, + std::vector>& result) const { + std::string graph_fused_nodes; + std::string node_list; + std::string subgraph_id = std::to_string(sub_var.subgraph_index); + sub_var.subgraph_index++; + + // This is a list of initializers that subgraph considers as constants. + // Example weights, reshape shape etc. + std::unordered_set input_initializers; + + // Create ng_required_initializers attribute of NGraphCustomOp + ONNX_NAMESPACE::AttributeProto initializers; + initializers.set_name("initializers"); + initializers.set_type(ONNX_NAMESPACE::AttributeProto_AttributeType::AttributeProto_AttributeType_TENSORS); + + for (const auto& init : sub_var.inputs) { + if (graph_viewer.GetAllInitializedTensors().count(init)) { + auto tensor = initializers.add_tensors(); + *tensor = *(graph_viewer.GetAllInitializedTensors().at(init)); + } + } + + auto meta_def = std::make_unique<::onnxruntime::IndexedSubGraph::MetaDef>(); + meta_def->attributes["initializers"] = initializers; + meta_def->name = "MkldnnCustomOp" + std::to_string(sub_var.subgraph_index); + meta_def->domain = kMSDomain; + meta_def->since_version = 1; + meta_def->status = ONNX_NAMESPACE::EXPERIMENTAL; + meta_def->inputs = sub_var.inputs; + meta_def->attributes.insert(subgraph_attributes.begin(), subgraph_attributes.end()); + + // Find the end nodes + for (auto& mklnode : subgraph_ptr->mkldnn_nodes) { + auto itr = std::find(sub_var.outputs_as_input_other_node.begin(), + sub_var.outputs_as_input_other_node.end(), mklnode.output_name); + if (itr == sub_var.outputs_as_input_other_node.end()) { + meta_def->outputs.push_back(mklnode.output_name); + mklnode.output_index = static_cast(meta_def->outputs.size()) - 1; + } + } + + ONNX_NAMESPACE::AttributeProto ap; + ap.set_s(subgraph_id); + ap.set_type(ONNX_NAMESPACE::AttributeProto_AttributeType::AttributeProto_AttributeType_STRING); + meta_def->attributes["subgraph_id"] = ap; + std::unique_ptr sub_graph = std::make_unique(); + sub_graph->nodes = sub_var.subgraph_node_indexes; + sub_graph->SetMetaDef(meta_def); + result.push_back(std::make_unique(std::move(sub_graph))); + mkl_subgraphs_.insert(std::make_pair(subgraph_id, subgraph_ptr)); + + // Reset subgraph and meta_Def + sub_var.Reset(); +} + +Status MKLDNNExecutionProvider::Compile(const std::vector& fused_nodes, + std::vector& node_compute_funcs) { + for (const auto* fused_node : fused_nodes) { + auto attributes = fused_node->GetAttributes(); + NodeComputeInfo compute_info; + + compute_info.create_state_func = [=](ComputeContext* context, FunctionState* state) { + auto* p = new onnxruntime::mkl_dnn::MkldnnFuncKernel(context, attributes, this); + *state = p; + return 0; + }; + + compute_info.release_state_func = [](FunctionState state) { + if (state) + delete static_cast*>(state); + }; + + compute_info.compute_func = [](FunctionState state, const OrtCustomOpApi* api, OrtKernelContext* context) { + onnxruntime::mkl_dnn::MkldnnFuncKernel* custom_op = reinterpret_cast*>(state); + return custom_op->Compute(api, context); + }; + + node_compute_funcs.push_back(compute_info); + } + return Status::OK(); +} } // namespace onnxruntime diff --git a/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.h b/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.h index 0e7ba75eae..7e844adce5 100644 --- a/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.h +++ b/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.h @@ -12,6 +12,7 @@ #include "core/graph/constants.h" #include "core/framework/allocatormgr.h" #include "core/framework/execution_provider.h" +#include "core/providers/mkldnn/subgraph/subgraph.h" namespace mkldnn { struct memory; @@ -46,7 +47,7 @@ class MKLDNNExecutionProvider : public IExecutionProvider { } void SetWeightsMemoryBuffer(const std::string& weight_key, - const std::shared_ptr& filter_dst_mem) { + const std::shared_ptr& filter_dst_mem) { weights_mem_map_.insert(std::make_pair(weight_key, filter_dst_mem)); } @@ -59,6 +60,13 @@ class MKLDNNExecutionProvider : public IExecutionProvider { reordered_buffers_.push_back(std::move(buffer)); } + std::vector> + GetCapability(const onnxruntime::GraphViewer& graph, + const std::vector& /*kernel_registries*/) const override; + + common::Status Compile(const std::vector& fused_nodes, + std::vector& node_compute_funcs) override; + private: // mkldnn weights(filer data) memory blocks from first iteration // saved by weights name @@ -66,6 +74,88 @@ class MKLDNNExecutionProvider : public IExecutionProvider { // Save reordered memory buffers in list so that memory is not freed. std::vector> reordered_buffers_; OrtMutex mutex_; + + // SUBGRAPH + private: + static int GetOnnxOpSet(const GraphViewer& graph_viewer) { + const auto& dm_to_ver = graph_viewer.DomainToVersionMap(); + return dm_to_ver.at(kOnnxDomain); + } + + std::string GetGraphName(const onnxruntime::GraphViewer& graph_viewer) const { + std::string graph_name; + + int opset = GetOnnxOpSet(graph_viewer); + + int index = 0; + if (graph_viewer.MaxNodeIndex() > 0) { + auto first_node = graph_viewer.GetNode(index); + while (first_node == nullptr) { + index++; + first_node = graph_viewer.GetNode(index); + } + auto first_node_outputs = first_node->OutputDefs(); + graph_name = graph_viewer.Name() + "_opset-" + std::to_string(opset) + "_" + first_node_outputs[0]->Name(); + } + return graph_name; + } + + bool UseSubgraph(const onnxruntime::GraphViewer& graph_viewer, + const std::vector& kernel_registries, + std::vector>& result) const; + + // Some dimensions are not supported by MKL-DNN + // example: Pool with NumDimensions <= 3 is not supported + // Fall back to CPU implementation + bool IsDimensionSupported(const Node* node) const { + bool supported = true; + if (node->OpType() == "BatchNormalization") { + auto node_inputs = node->InputDefs(); + if (node_inputs[0]->Shape() != nullptr && node_inputs[0]->Shape()->dim_size() == 3) { + supported = false; + } + } + if (node->OpType().find("Pool") != std::string::npos) { + auto node_inputs = node->InputDefs(); + if (node_inputs[0]->Shape() != nullptr && node_inputs[0]->Shape()->dim_size() <= 3) { + supported = false; + } + + if (node->Op()->SinceVersion() == 10) + supported = false; + + if (node->OutputDefs().size() > 1) + supported = false; + } + return supported; + } + + void CreateOrUpdateMklDnnNode(const Node* node, + std::shared_ptr& subgraph_ptr, + mkl_dnn::Subgraph::SubgraphVariables& sub_var, + bool fused, + std::map& output_to_source_node_map, + NodeAttributes& subgraph_attributes) const; + + // Create MklDnn node, update inputs, outputs and parent nodes + // collect attribtes + void CreateMetaDef(const onnxruntime::GraphViewer& graph_viewer, + const NodeAttributes& subgraph_attributes, + std::shared_ptr& subgraph_ptr, + mkl_dnn::Subgraph::SubgraphVariables& sub_var, + std::vector>& result) const; + + public: + const std::shared_ptr GetMklDnnSubgraph(const std::string& subgraph_id) { + return mkl_subgraphs_[subgraph_id]; + } + + private: + // supported MklDnn Operators + std::set mkldnn_ops_ = {"Conv", "BatchNormalization", "Relu", "Sum", + "AveragePool", "GlobalMaxPool", "GlobalAveragePool", "MaxPool", "LRN"}; + + mutable std::unordered_map> mkl_subgraphs_; }; } // namespace onnxruntime diff --git a/onnxruntime/core/providers/mkldnn/subgraph/mkldnn_activations.h b/onnxruntime/core/providers/mkldnn/subgraph/mkldnn_activations.h new file mode 100644 index 0000000000..406542fd15 --- /dev/null +++ b/onnxruntime/core/providers/mkldnn/subgraph/mkldnn_activations.h @@ -0,0 +1,152 @@ +// Copyright(C) 2019 Intel Corporation +// Licensed under the MIT License + +#pragma once +#include "core/util/math.h" +#include "core/util/math_cpuonly.h" +#include "core/framework/op_kernel.h" +#include "core/providers/mkldnn/mkldnn_fwd.h" +#include "core/providers/mkldnn/mkldnn_execution_provider.h" +#include "core/providers/mkldnn/subgraph/mkldnn_kernel.h" + +namespace onnxruntime { +namespace mkl_dnn { + +template +class MklDnnRelu : public MklDnnKernel { + public: + MklDnnRelu(const MklDnnNode& node, + MKLDNNExecutionProvider* provider, + const NodeAttributes& attributes, + const std::string attributes_prefix = "") : MklDnnKernel(node, provider) { + ORT_UNUSED_PARAMETER(attributes); + ORT_UNUSED_PARAMETER(attributes_prefix); + } + + Status CreatePrimitives(const OrtCustomOpApi* api, + OrtKernelContext* context, + mkldnn::engine& cpu_engine, + std::vector& net, + mkldnn::memory::format& source_format) { + Ort::CustomOpApi ort{*api}; + int input_index = mklnode_ptr_->input_start_index < 0 ? 0 : mklnode_ptr_->input_start_index; + + TensorShape x_shape; + if (mklnode_ptr_->parent_nodes.empty()) { + const OrtValue* input_tensor = ort.KernelContext_GetInput(context, input_index); + auto tensor_info = ort.GetTensorTypeAndShape(input_tensor); + auto tensor_shape = ort.GetTensorShape(tensor_info); + ort.ReleaseTensorTypeAndShapeInfo(tensor_info); + + auto xshape = tensor_shape.data(); + auto xdim = tensor_shape.size(); + + mkldnn::memory::dims dims(xdim); + + ort_source_format_ = GetSourceFormat(static_cast(xdim)); + source_format = ort_source_format_; + src_format_ = ort_source_format_; + + x_shape = TensorShape(xshape, xdim); + + mkldnn::memory::dims src_dims_mkl( + x_shape.GetDims().begin(), x_shape.GetDims().end()); + + src_md_.reset(new mkldnn::memory::desc( + {src_dims_mkl}, MklDnnType(), src_format_)); + src_mem_.reset( + new mkldnn::memory({*src_md_, cpu_engine}, nullptr)); + } else { + src_md_.reset( + new mkldnn::memory::desc(parents_[0].get()->primitive_dst_mem_.get()->get_primitive_desc().desc())); + src_mem_ = parents_[0].get()->primitive_dst_mem_; + x_shape = parents_[0].get()->primitive_dst_shape_; + ort_source_format_ = source_format; + src_format_ = parents_[0].get()->primitive_dst_format_; + } + + primitive_dst_shape_ = TensorShape(x_shape); + + mkldnn::memory::dims dst_dims_mkl(primitive_dst_shape_.GetDims().begin(), primitive_dst_shape_.GetDims().end()); + mkldnn::algorithm algo = mkldnn::algorithm::eltwise_relu; + fwd_desc_.reset(new mkldnn::eltwise_forward::desc( + mkldnn::prop_kind::forward_inference, algo, *src_md_, 0)); + + relu_fwd_pd_.reset(new mkldnn::eltwise_forward::primitive_desc( + *fwd_desc_, cpu_engine)); + + primitive_src_format_ = static_cast( + relu_fwd_pd_.get()->src_primitive_desc().desc().data.format); + primitive_dst_format_ = static_cast( + relu_fwd_pd_.get()->dst_primitive_desc().desc().data.format); + + if (mklnode_ptr_->output_index >= 0) { + // last node of sub-graph. need to allocate memory for output_tensor + if (primitive_dst_format_ != ort_source_format_) { + // reorder neded. Use primitive output as input to reorder and + // allocate buffer for reorder output, final output of this subgraph + primitive_dst_mem_.reset(new mkldnn::memory(relu_fwd_pd_.get()->dst_primitive_desc())); + } else { + // Last node but re-order not needed. Allocate buffer to output of this node + primitive_dst_mem_.reset(new mkldnn::memory(relu_fwd_pd_.get()->dst_primitive_desc(), nullptr)); + } + } else { + // Intermediate node. Use mkldnn kernel internal memory for output and + // use this as input to next node. + primitive_dst_mem_.reset(new mkldnn::memory(relu_fwd_pd_.get()->dst_primitive_desc())); + } + + relu_fwd_.reset( + new mkldnn::eltwise_forward(*relu_fwd_pd_, *src_mem_, *primitive_dst_mem_)); + + net.push_back(*relu_fwd_); + + if (mklnode_ptr_->output_index >= 0) { + // one of the end nodes. Allocate output buffer memory and + // reorder is necessary + mkldnn::memory::data_type t = MklDnnType(); + InitDstReorderOutput(cpu_engine, t, net); + } + + return Status::OK(); + } + + Status Bind(const OrtCustomOpApi* api, OrtKernelContext* context) override { + Ort::CustomOpApi ort{*api}; + + int input_index = mklnode_ptr_->input_start_index < 0 ? 0 : mklnode_ptr_->input_start_index; + + if (mklnode_ptr_->parent_nodes.empty()) { + // Sub-graph's first node. Read input from input buffer + const OrtValue* input_tensor = ort.KernelContext_GetInput(context, input_index); + const T* src_data = const_cast(ort.GetTensorData(input_tensor)); + src_mem_->set_data_handle(static_cast(const_cast(src_data))); + } + + if (mklnode_ptr_->output_index >= 0) { + auto& y_dims = primitive_dst_shape_.GetDims(); + // Allocate memory for output bufffer + OrtValue* output = ort.KernelContext_GetOutput(context, mklnode_ptr_->output_index, &y_dims[0], static_cast(primitive_dst_shape_.GetDims().size())); + T* dst_data = ort.GetTensorMutableData(output); + + if (primitive_dst_format_ != ort_source_format_) { + reorder_dst_mem_to_->set_data_handle(dst_data); + } else { + primitive_dst_mem_->set_data_handle(dst_data); + } + } + + return Status::OK(); + } + + private: + std::shared_ptr src_mem_; + + std::unique_ptr fwd_desc_; + std::unique_ptr relu_fwd_pd_; + std::unique_ptr relu_fwd_; + + std::unique_ptr src_md_; +}; +} // namespace mkl_dnn +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/mkldnn/subgraph/mkldnn_batchnorm.h b/onnxruntime/core/providers/mkldnn/subgraph/mkldnn_batchnorm.h new file mode 100644 index 0000000000..17728b927f --- /dev/null +++ b/onnxruntime/core/providers/mkldnn/subgraph/mkldnn_batchnorm.h @@ -0,0 +1,362 @@ +// Copyright(C) 2019 Intel Corporation +// Licensed under the MIT License + +#pragma once +#include "core/framework/op_kernel.h" +#include "core/providers/mkldnn/mkldnn_fwd.h" +#include "core/providers/mkldnn/mkldnn_execution_provider.h" +#include "core/providers/mkldnn/subgraph/mkldnn_kernel.h" +#include "core/providers/mkldnn/memcpy_s.h" +#include "core/util/math.h" + +namespace onnxruntime { +namespace mkl_dnn { + +class BatchNormHelper { + public: + static common::Status ValidateInputs(const TensorShape& xshape, + const TensorShape& scale_shape, + const TensorShape& b_shape, + const TensorShape& mean_shape, + const TensorShape& var_shape) { + // defined as per spec and used for validation + constexpr int kNumInputScaleDimensions = 1; + constexpr int kNumInputBiasDimensions = 1; + constexpr int kNumInputMeanDimensions = 1; + constexpr int kNumInputVarianceDimensions = 1; + + if (xshape.GetDims().empty()) { + return common::Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "Invalid input X: Empty dimensions"); + } + + int64_t num_channels = xshape.GetDims()[1]; + + if (scale_shape.NumDimensions() != kNumInputScaleDimensions) { + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Invalid input scale: NumDimensions() != ", kNumInputScaleDimensions); + } + if (scale_shape.GetDims()[0] != num_channels) { + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Invalid input scale: 0th dimension != ", num_channels); + } + + if (b_shape.NumDimensions() != kNumInputBiasDimensions) { + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Invalid input B: NumDimensions() != ", kNumInputBiasDimensions); + } + if (b_shape.GetDims()[0] != num_channels) { + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Invalid input B: 0th dimension != ", num_channels); + } + + if (mean_shape.NumDimensions() != kNumInputMeanDimensions) { + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Invalid input mean: NumDimensions() != ", kNumInputMeanDimensions); + } + if (mean_shape.GetDims()[0] != num_channels) { + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Invalid input mean: 0th dimension != ", num_channels); + } + + if (var_shape.NumDimensions() != kNumInputVarianceDimensions) { + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Invalid input var: NumDimensions() != ", kNumInputVarianceDimensions); + } + if (var_shape.GetDims()[0] != num_channels) { + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Invalid input var: 0th dimension != ", num_channels); + } + return common::Status::OK(); + } + + static void NormalizeDims(const TensorShape& x_shape, std::vector& new_dims) { + new_dims.clear(); + auto& orig_dims = x_shape.GetDims(); + if (orig_dims.size() == 4 /*supported size by CUDA*/ || + orig_dims.size() == 5 /*supported size by CUDA*/) { + new_dims = orig_dims; + return; + } + + auto rank = x_shape.NumDimensions(); + auto num_samples = rank > 0 ? orig_dims[0] : 1; // NCHW + auto num_channels = rank > 1 ? orig_dims[1] : 1; + auto width = rank > 3 ? orig_dims[3] : 1; + auto height = rank > 2 ? orig_dims[2] : 1; + new_dims = {num_samples, num_channels, height, width}; + } +}; + +template +class MklDnnBatchNorm : public MklDnnKernel { + public: + explicit MklDnnBatchNorm(const MklDnnNode& node, + MKLDNNExecutionProvider* provider, + const NodeAttributes& attributes, + const std::string attributes_prefix = "") : MklDnnKernel(node, provider) { + ReadAttributes(attributes, attributes_prefix); + } + void ReadAttributes(const NodeAttributes& attributes, + const std::string attributes_prefix = "") override { + auto attr = attributes.find(attributes_prefix + "epsilon"); + if (attr != attributes.end() && + attr->second.type() == ::ONNX_NAMESPACE::AttributeProto_AttributeType::AttributeProto_AttributeType_FLOAT) { + epsilon_ = attr->second.f(); + } + } + + Status CreatePrimitives(const OrtCustomOpApi* api, + OrtKernelContext* context, + mkldnn::engine& cpu_engine, + std::vector& net, + mkldnn::memory::format& source_format) override { + Ort::CustomOpApi ort{*api}; + int input_index = mklnode_ptr_->input_start_index < 0 ? 0 : mklnode_ptr_->input_start_index; + + TensorShape x_shape; + if (mklnode_ptr_->parent_nodes.empty()) { + const OrtValue* input_tensor = ort.KernelContext_GetInput(context, input_index); + auto tensor_info = ort.GetTensorTypeAndShape(input_tensor); + auto tensor_shape = ort.GetTensorShape(tensor_info); + ort.ReleaseTensorTypeAndShapeInfo(tensor_info); + auto xshape = tensor_shape.data(); + auto xdim = tensor_shape.size(); + mkldnn::memory::dims dims(xdim); + + ort_source_format_ = GetSourceFormat(static_cast(xdim)); + source_format = ort_source_format_; + src_format_ = ort_source_format_; + x_shape = TensorShape(xshape, xdim); + + mkldnn::memory::dims src_dims_mkl( + x_shape.GetDims().begin(), x_shape.GetDims().end()); + src_md_.reset(new mkldnn::memory::desc( + {src_dims_mkl}, MklDnnType(), src_format_)); + } else { + src_md_.reset(new mkldnn::memory::desc(parents_[0].get()->primitive_dst_mem_.get()->get_primitive_desc().desc())); + x_shape = parents_[0].get()->primitive_dst_shape_; + ort_source_format_ = source_format; + src_format_ = parents_[0].get()->primitive_dst_format_; + } + + int num_dimensions = static_cast(x_shape.NumDimensions()); + if (num_dimensions == 3) { + primitive_created_ = Status(common::ONNXRUNTIME, + common::NOT_IMPLEMENTED, "BatchNorm: Please call default CPU kernel."); + return primitive_created_; + } + + const OrtValue* scale_input_tensor = ort.KernelContext_GetInput(context, input_index + 1); + const OrtValue* b_input_tensor = ort.KernelContext_GetInput(context, input_index + 2); + const OrtValue* mean_input_tensor = ort.KernelContext_GetInput(context, input_index + 3); + const OrtValue* var_input_tensor = ort.KernelContext_GetInput(context, input_index + 4); + + auto scale_tensor_info = ort.GetTensorTypeAndShape(scale_input_tensor); + auto scale_tensor_shape = ort.GetTensorShape(scale_tensor_info); + ort.ReleaseTensorTypeAndShapeInfo(scale_tensor_info); + auto sshape = scale_tensor_shape.data(); + auto sdim = scale_tensor_shape.size(); + TensorShape scale_shape(sshape, sdim); + + auto b_tensor_info = ort.GetTensorTypeAndShape(b_input_tensor); + auto b_tensor_shape = ort.GetTensorShape(b_tensor_info); + ort.ReleaseTensorTypeAndShapeInfo(b_tensor_info); + auto bshape = b_tensor_shape.data(); + auto bdim = b_tensor_shape.size(); + TensorShape b_shape(bshape, bdim); + + auto mean_tensor_info = ort.GetTensorTypeAndShape(mean_input_tensor); + auto mean_tensor_shape = ort.GetTensorShape(mean_tensor_info); + ort.ReleaseTensorTypeAndShapeInfo(mean_tensor_info); + auto mshape = mean_tensor_shape.data(); + auto mdim = mean_tensor_shape.size(); + TensorShape mean_shape(mshape, mdim); + + auto var_tensor_info = ort.GetTensorTypeAndShape(var_input_tensor); + auto var_tensor_shape = ort.GetTensorShape(var_tensor_info); + ort.ReleaseTensorTypeAndShapeInfo(var_tensor_info); + auto vshape = var_tensor_shape.data(); + auto vdim = var_tensor_shape.size(); + TensorShape var_shape(vshape, vdim); + + primitive_dst_shape_ = TensorShape(x_shape); + + primitive_created_ = BatchNormHelper::ValidateInputs(x_shape, scale_shape, b_shape, mean_shape, var_shape); + if (!primitive_created_.IsOK()) + return primitive_created_; + + mkldnn::memory::dims src_dims_mkl( + x_shape.GetDims().begin(), x_shape.GetDims().end()); + mkldnn::memory::dims scale_dims_mkl( + scale_shape.GetDims().begin(), scale_shape.GetDims().end()); + mkldnn::memory::dims b_dims_mkl( + b_shape.GetDims().begin(), b_shape.GetDims().end()); + mkldnn::memory::dims mean_dims_mkl( + mean_shape.GetDims().begin(), mean_shape.GetDims().end()); + mkldnn::memory::dims var_dims_mkl( + var_shape.GetDims().begin(), var_shape.GetDims().end()); + + mkldnn::memory::dims dst_dims_mkl( + primitive_dst_shape_.GetDims().begin(), primitive_dst_shape_.GetDims().end()); + + scale_shift_md_.reset(new mkldnn::memory::desc( + {2, scale_dims_mkl[0]}, MklDnnType(), mkldnn::memory::format::nc)); + mean_md_.reset(new mkldnn::memory::desc( + {mean_dims_mkl}, MklDnnType(), mkldnn::memory::format::x)); + var_md_.reset(new mkldnn::memory::desc( + {var_dims_mkl}, MklDnnType(), mkldnn::memory::format::x)); + primitive_dst_md_.reset(new mkldnn::memory::desc( + {dst_dims_mkl}, MklDnnType(), mkldnn::memory::format::any)); + + // scale_shift_mem will allocate 2*C*sizeof(float) buffer + // + scale_shift_mem_.reset( + new mkldnn::memory({*scale_shift_md_, cpu_engine})); + + mean_mem_.reset( + new mkldnn::memory({*mean_md_, cpu_engine}, nullptr)); + var_mem_.reset( + new mkldnn::memory({*var_md_, cpu_engine}, nullptr)); + + batchnorm_fwd_.reset(new mkldnn::batch_normalization_forward::desc( + mkldnn::prop_kind::forward_inference, *src_md_, epsilon_, + mkldnn::batch_normalization_flag::use_scale_shift | + mkldnn::batch_normalization_flag::use_global_stats)); + + if (fuse_relu_) { + mkldnn::primitive_attr attr; + attr.set_int_output_round_mode(mkldnn::round_mode::round_nearest); + // Execute RELU as Fuse PostOps + const float ops_scale = 1.f; + const float ops_alpha = 0.f; // relu negative slope + const float ops_beta = 0.f; + mkldnn::post_ops ops; + ops.append_eltwise(ops_scale, mkldnn::algorithm::eltwise_relu, ops_alpha, ops_beta); + attr.set_post_ops(ops); + + batchnorm_fwd_pd_.reset(new mkldnn::batch_normalization_forward::primitive_desc( + *batchnorm_fwd_, attr, cpu_engine)); + } else { + batchnorm_fwd_pd_.reset( + new mkldnn::batch_normalization_forward::primitive_desc( + *batchnorm_fwd_, cpu_engine)); + } + + // out format of this kernel + primitive_dst_format_ = static_cast( + batchnorm_fwd_pd_.get()->dst_primitive_desc().desc().data.format); + primitive_src_format_ = static_cast( + batchnorm_fwd_pd_.get()->dst_primitive_desc().desc().data.format); + + if (mklnode_ptr_->parent_nodes.empty()) { + src_mem_.reset( + new mkldnn::memory(batchnorm_fwd_pd_.get()->src_primitive_desc(), nullptr)); + } else { + src_mem_ = parents_[0].get()->primitive_dst_mem_; + } + + if (mklnode_ptr_->output_index >= 0) { + // Use mkldnn's internal output buffer + if (primitive_dst_format_ != ort_source_format_) { + primitive_dst_mem_.reset(new mkldnn::memory(batchnorm_fwd_pd_->dst_primitive_desc())); + } else { + primitive_dst_mem_.reset(new mkldnn::memory(batchnorm_fwd_pd_->dst_primitive_desc(), nullptr)); + } + } else { + // last node of sub-graph. need to allocate memory for output_tensor + primitive_dst_mem_.reset(new mkldnn::memory(batchnorm_fwd_pd_->dst_primitive_desc())); + } + auto bn = mkldnn::batch_normalization_forward( + *batchnorm_fwd_pd_, + (const mkldnn::primitive::at)*src_mem_, + (const mkldnn::primitive::at)*mean_mem_, + (const mkldnn::primitive::at)*var_mem_, + (const mkldnn::memory)*scale_shift_mem_, + (const mkldnn::memory)*primitive_dst_mem_); + net.push_back(bn); + + // Allocate dst buffer if reorder is necessary + if (mklnode_ptr_->output_index >= 0) { + // one of the end nodes. Allocate output buffer memory and + // reorder is necessary + mkldnn::memory::data_type t = MklDnnType(); + InitDstReorderOutput(cpu_engine, t, net); + } + return Status::OK(); + } + + Status Bind(const OrtCustomOpApi* api, OrtKernelContext* context) override { + Ort::CustomOpApi ort{*api}; + int input_index = mklnode_ptr_->input_start_index < 0 ? 0 : mklnode_ptr_->input_start_index; + + if (!primitive_created_.IsOK()) { + // abort as MKLDNN cannot execute this. but + // ORT try to delete output_tensor buffer data. allocate memory so that it can delete + // fix for test_averagepool_1d_default node test + return primitive_created_; + } + + if (mklnode_ptr_->parent_nodes.empty()) { + const OrtValue* input_tensor = ort.KernelContext_GetInput(context, input_index); + const T* src_data = const_cast(ort.GetTensorData(input_tensor)); + src_mem_->set_data_handle(static_cast(const_cast(src_data))); + } + + const OrtValue* scale_input_tensor = ort.KernelContext_GetInput(context, input_index + 1); + const T* scale_data = reinterpret_cast(ort.GetTensorData(scale_input_tensor)); + const OrtValue* b_input_tensor = ort.KernelContext_GetInput(context, input_index + 2); + const T* b_data = reinterpret_cast(ort.GetTensorData(b_input_tensor)); + const OrtValue* mean_input_tensor = ort.KernelContext_GetInput(context, input_index + 3); + const T* mean_data = reinterpret_cast(ort.GetTensorData(mean_input_tensor)); + const OrtValue* var_input_tensor = ort.KernelContext_GetInput(context, input_index + 4); + const T* var_data = reinterpret_cast(ort.GetTensorData(var_input_tensor)); + + auto tensor_info = ort.GetTensorTypeAndShape(scale_input_tensor); + auto tensor_shape = ort.GetTensorShape(tensor_info); + ort.ReleaseTensorTypeAndShapeInfo(tensor_info); + auto sshape = tensor_shape.data(); + auto sdim = tensor_shape.size(); + + TensorShape scale_shape(sshape, sdim); + mkldnn::memory::dims scale_dims_mkl( + scale_shape.GetDims().begin(), scale_shape.GetDims().end()); + + mean_mem_->set_data_handle(static_cast(const_cast(mean_data))); + var_mem_->set_data_handle(static_cast(const_cast(var_data))); + + T* scale_shift_buf = static_cast(scale_shift_mem_->get_data_handle()); + + size_t src_bytes = sizeof(T) * scale_dims_mkl[0]; + size_t dst_bytes = sizeof(T) * scale_dims_mkl[0]; + + MEMCPY_S(scale_shift_buf, scale_data, src_bytes, dst_bytes); + MEMCPY_S(&scale_shift_buf[scale_dims_mkl[0]], b_data, src_bytes, dst_bytes); + + if (mklnode_ptr_->output_index >= 0) { + auto& y_dims = primitive_dst_shape_.GetDims(); + // Allocate memory for output bufffer + OrtValue* output = ort.KernelContext_GetOutput(context, mklnode_ptr_->output_index, &y_dims[0], static_cast(primitive_dst_shape_.GetDims().size())); + T* dst_data = ort.GetTensorMutableData(output); + + if (primitive_dst_format_ != ort_source_format_) { + reorder_dst_mem_to_->set_data_handle(dst_data); + } else { + primitive_dst_mem_->set_data_handle(dst_data); + } + } + return Status::OK(); + } + + private: + std::shared_ptr src_mem_; + std::unique_ptr scale_shift_mem_; + std::unique_ptr mean_mem_; + std::unique_ptr var_mem_; + std::unique_ptr dst_mem_; + + std::unique_ptr src_md_; + std::unique_ptr scale_shift_md_; + std::unique_ptr mean_md_; + std::unique_ptr var_md_; + std::unique_ptr dst_md_; + + std::unique_ptr batchnorm_fwd_; + std::unique_ptr batchnorm_fwd_pd_; + + protected: + float epsilon_ = 1e-5f; +}; +} // namespace mkl_dnn +} // namespace onnxruntime \ No newline at end of file diff --git a/onnxruntime/core/providers/mkldnn/subgraph/mkldnn_conv.h b/onnxruntime/core/providers/mkldnn/subgraph/mkldnn_conv.h new file mode 100644 index 0000000000..ceafa69e5f --- /dev/null +++ b/onnxruntime/core/providers/mkldnn/subgraph/mkldnn_conv.h @@ -0,0 +1,637 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "mkldnn_types.h" +#include "core/framework/op_kernel.h" +#include "core/providers/mkldnn/mkldnn_fwd.h" +#include "core/providers/cpu/nn/autopad_type.h" +#include "core/providers/mkldnn/mkldnn_execution_provider.h" +#include "core/providers/mkldnn/subgraph/mkldnn_kernel.h" +#include "core/util/math.h" + +namespace onnxruntime { +namespace mkl_dnn { + +// helper function +template +Status ComputePadAndOutputShape( + const int64_t in_dim, + const int64_t stride, + const int64_t kernel, + const int64_t dilation, + AutoPadType pad_type, + int64_t* pad_head, + int64_t* pad_tail, + int64_t* out_dim) { + const int64_t dkernel = dilation * (kernel - 1) + 1; + + if (pad_type == AutoPadType::NOTSET) { + *out_dim = static_cast(static_cast(in_dim + *pad_head + *pad_tail - dkernel) / stride + 1); + } else { + switch (pad_type) { + case AutoPadType::VALID: + *pad_head = 0; + *pad_tail = 0; + *out_dim = (in_dim - dkernel) / stride + 1; + break; + case AutoPadType::SAME_UPPER: + case AutoPadType::SAME_LOWER: { + ORT_ENFORCE(dilation == 1, "Dilation not supported for AutoPadType::SAME_UPPER or AutoPadType::SAME_LOWER."); + int64_t legacy_target_size = (in_dim + stride - 1) / stride; + int64_t pad_needed = (legacy_target_size - 1) * stride + kernel - in_dim; + *out_dim = (in_dim + pad_needed - dkernel) / stride + 1; + + // make sure padding is symmetric + if (ForceSymmetricAutoPadding) + pad_needed = math::roundUpPow2(pad_needed); + + if (pad_type == AutoPadType::SAME_LOWER) { + *pad_head = (pad_needed + 1) / 2; + } else { + *pad_head = pad_needed / 2; + } + *pad_tail = pad_needed - *pad_head; + } break; + default: + return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "pad type not supported."); + } + } + return Status::OK(); +} + +template +class MklDnnConv : public MklDnnKernel { + public: + MklDnnConv(const MklDnnNode& node, + MKLDNNExecutionProvider* provider, + const NodeAttributes& attributes, + const std::string attributes_prefix = "") : MklDnnKernel(node, provider) { + ReadAttributes(attributes, attributes_prefix); + } + + Status CreatePrimitives(const OrtCustomOpApi* api, + OrtKernelContext* context, + mkldnn::engine& cpu_engine, + std::vector& net, + mkldnn::memory::format& source_format) override { + Ort::CustomOpApi ort{*api}; + + int input_index = mklnode_ptr_->input_start_index < 0 ? 0 : mklnode_ptr_->input_start_index; + const OrtValue* winput_tensor = ort.KernelContext_GetInput(context, input_index + 1); + auto wtensor_info = ort.GetTensorTypeAndShape(winput_tensor); + auto wtensor_shape = ort.GetTensorShape(wtensor_info); + ort.ReleaseTensorTypeAndShapeInfo(wtensor_info); + auto wshape = wtensor_shape.data(); + auto wdim = wtensor_shape.size(); + + TensorShape w_shape(wshape, wdim); + const int group_mkl = static_cast(group_); + + TensorShape x_shape; + // std::unique_ptr x_shape; + if (mklnode_ptr_->parent_nodes.empty()) { + const OrtValue* input_tensor = ort.KernelContext_GetInput(context, input_index); + auto tensor_info = ort.GetTensorTypeAndShape(input_tensor); + auto tensor_shape = ort.GetTensorShape(tensor_info); + ort.ReleaseTensorTypeAndShapeInfo(tensor_info); + auto xshape = tensor_shape.data(); + auto xdim = tensor_shape.size(); + + x_shape = TensorShape(xshape, xdim); + + mkldnn::memory::dims src_dims_mkl(x_shape.GetDims().begin(), x_shape.GetDims().end()); + src_md_.reset(new mkldnn::memory::desc( + {src_dims_mkl}, MklDnnType(), mkldnn::memory::format::any)); + } else { + // get the output of previous node (mkldnn block propagation). + // TODO Sourcenode will set src of this node. + x_shape = parents_[0].get()->primitive_dst_shape_; + ort_source_format_ = source_format; + src_format_ = parents_[0].get()->primitive_dst_format_; + mkldnn::memory::dims src_dims_mkl(x_shape.GetDims().begin(), x_shape.GetDims().end()); + src_md_.reset(new mkldnn::memory::desc( + {src_dims_mkl}, MklDnnType(), mkldnn::memory::format::any)); + } + + primitive_created_ = ValidateInputShape(x_shape, w_shape); + if (!primitive_created_.IsOK()) + return primitive_created_; + + std::vector kernel_shape; + primitive_created_ = ComputeKernelShape(w_shape, kernel_shape); + if (!primitive_created_.IsOK()) + return primitive_created_; + + const size_t kernel_rank = kernel_shape.size(); + + if (kernel_rank + 2 != wdim) { + primitive_created_ = ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "kernel_shape num_dims is not compatible with W num_dims.", + " kernel_shape: ", TensorShape(kernel_shape).ToString().c_str(), + " W: ", w_shape.ToString().c_str()); + return primitive_created_; + } + + for (size_t i = 0; i < kernel_rank; ++i) { + if (kernel_shape[i] != w_shape[i + 2]) { + primitive_created_ = ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "kernel_shape is not compatible with W shape.", + " kernel_shape: ", TensorShape(kernel_shape).ToString().c_str(), + " W: ", w_shape.ToString().c_str()); + return primitive_created_; + } + } + + std::vector pads(pads_); + if (pads.empty()) { + pads.resize(kernel_rank * 2, 0); + } + std::vector dilations(dilations_); + if (dilations.empty()) { + dilations.resize(kernel_rank, 1); + } + std::vector strides(strides_); + if (strides.empty()) { + strides.resize(kernel_rank, 1); + } + + const int64_t N = x_shape[0]; + const int64_t M = w_shape[0]; + std::vector y_dims; + y_dims.insert(y_dims.begin(), {N, M}); + TensorShape input_shape = x_shape.Slice(2); + primitive_created_ = InferOutputShape(input_shape, kernel_shape, strides, dilations, &pads, &y_dims); + if (!primitive_created_.IsOK()) + return primitive_created_; + + TensorShape y_shape(y_dims); + primitive_dst_shape_ = TensorShape(y_dims); + TensorShape output_shape = y_shape.Slice(2); + mkldnn::memory::dims dst_dims_mkl(y_dims.begin(), y_dims.end()); + primitive_dst_md_.reset(new mkldnn::memory::desc( + {dst_dims_mkl}, MklDnnType(), mkldnn::memory::format::any)); + + mkldnn::memory::dims filter_dims_mkl; + if (group_mkl == 1) { + filter_dims_mkl.assign(w_shape.GetDims().begin(), w_shape.GetDims().end()); + } else { + filter_dims_mkl.assign({group_mkl, + static_cast(w_shape[0] / group_mkl)}); + filter_dims_mkl.insert(filter_dims_mkl.end(), w_shape.GetDims().begin() + 1, w_shape.GetDims().end()); + } + mkldnn::memory::dims strides_mkl(strides.begin(), strides.end()); + mkldnn::memory::dims dilations_mkl(dilations.begin(), dilations.end()); + // mkldnn dilations start from 0 so we need to subtract 1 from each dim. + for (size_t dim = 0; dim < kernel_rank; dim++) { + dilations_mkl[dim] -= 1; + } + + mkldnn::memory::dims padding_left_mkl(pads.begin(), pads.begin() + kernel_rank); + mkldnn::memory::dims padding_right_mkl(pads.begin() + kernel_rank, pads.end()); + mkldnn::memory::dims bias_dims_mkl; + if (mklnode_ptr_->num_inputs == 3) { + const OrtValue* binput_tensor = ort.KernelContext_GetInput(context, input_index + 2); + auto btensor_info = ort.GetTensorTypeAndShape(binput_tensor); + auto btensor_shape = ort.GetTensorShape(btensor_info); + ort.ReleaseTensorTypeAndShapeInfo(btensor_info); + auto bshape = btensor_shape.data(); + auto bdim = btensor_shape.size(); + TensorShape b_shape(bshape, bdim); + bias_dims_mkl.assign(b_shape.GetDims().begin(), b_shape.GetDims().end()); + } + + auto fmt = mkldnn::memory::format::any; + if (kernel_rank == 1) { + fmt = mkldnn::memory::format::ncw; + if (group_mkl == 1) { + filter_format_ = mkldnn::memory::format::oiw; + } else { + filter_format_ = mkldnn::memory::format::goiw; + } + } else if (kernel_rank == 2) { + fmt = mkldnn::memory::format::nchw; + if (group_mkl == 1) { + filter_format_ = mkldnn::memory::format::oihw; + } else { + filter_format_ = mkldnn::memory::format::goihw; + } + } else { + fmt = mkldnn::memory::format::ncdhw; + if (group_mkl == 1) { + filter_format_ = mkldnn::memory::format::oidhw; + } else { + filter_format_ = mkldnn::memory::format::goidhw; + } + } + if (src_format_ == mkldnn::memory::format::any) { + src_format_ = fmt; + ort_source_format_ = fmt; + source_format = fmt; + } + + // Set the memory descriptors to format::any to allow MKLDNN to decide what the optimal memory layout should be + // for the computation given the input + filter_md_.reset(new mkldnn::memory::desc( + {filter_dims_mkl}, MklDnnType(), mkldnn::memory::format::any)); + if (!bias_dims_mkl.empty()) + bias_md_.reset(new mkldnn::memory::desc( + {bias_dims_mkl}, MklDnnType(), mkldnn::memory::format::any)); + + if (!bias_dims_mkl.empty()) { + fwd_desc_.reset(new mkldnn::convolution_forward::desc( + mkldnn::prop_kind::forward, mkldnn::convolution_direct, *src_md_, + *filter_md_, *bias_md_, *primitive_dst_md_, + strides_mkl, dilations_mkl, padding_left_mkl, + padding_right_mkl, mkldnn::padding_kind::zero)); + } else { + fwd_desc_.reset(new mkldnn::convolution_forward::desc( + mkldnn::prop_kind::forward, mkldnn::convolution_direct, *src_md_, + *filter_md_, *primitive_dst_md_, strides_mkl, + dilations_mkl, padding_left_mkl, + padding_right_mkl, mkldnn::padding_kind::zero)); + } + + if (fuse_relu_) { + mkldnn::primitive_attr attr; + attr.set_int_output_round_mode(mkldnn::round_mode::round_nearest); + // Execute RELU as Fuse PostOps + const float ops_scale = 1.f; + const float ops_alpha = 0.f; // relu negative slope + const float ops_beta = 0.f; + mkldnn::post_ops ops; + ops.append_eltwise(ops_scale, mkldnn::algorithm::eltwise_relu, ops_alpha, ops_beta); + attr.set_post_ops(ops); + + conv_fwd_pd_.reset(new mkldnn::convolution_forward::primitive_desc( + *fwd_desc_, attr, cpu_engine)); + } else { + conv_fwd_pd_.reset(new mkldnn::convolution_forward::primitive_desc( + *fwd_desc_, cpu_engine)); + } + + primitive_src_format_ = static_cast( + conv_fwd_pd_.get()->src_primitive_desc().desc().data.format); + + mkldnn_filter_format_ = static_cast( + conv_fwd_pd_.get()->weights_primitive_desc().desc().data.format); + + primitive_dst_format_ = static_cast( + conv_fwd_pd_.get()->dst_primitive_desc().desc().data.format); + + src_size_ = conv_fwd_pd_.get()->src_primitive_desc().get_size(); + filter_size_ = conv_fwd_pd_.get()->weights_primitive_desc().get_size(); + dst_size_ = conv_fwd_pd_.get()->dst_primitive_desc().get_size(); + + filter_mem_.reset( + new mkldnn::memory(conv_fwd_pd_.get()->weights_primitive_desc(), nullptr)); + + if (primitive_src_format_ != src_format_) { + mkldnn::memory::dims src_dims_mkl(x_shape.GetDims().begin(), x_shape.GetDims().end()); + auto src_md = mkldnn::memory::desc(src_dims_mkl, MklDnnType(), src_format_); + auto pd = mkldnn::memory::primitive_desc(src_md, cpu_engine); + + if (mklnode_ptr_->parent_nodes.empty()) + src_mem_from_.reset(new mkldnn::memory(pd, nullptr)); + else + src_mem_from_ = parents_[0].get()->primitive_dst_mem_; + + src_mem_.reset(new mkldnn::memory(conv_fwd_pd_->src_primitive_desc(), nullptr)); + net.push_back(mkldnn::reorder(*src_mem_from_, *src_mem_)); + } else { + if (mklnode_ptr_->parent_nodes.empty()) { + src_mem_.reset(new mkldnn::memory(conv_fwd_pd_->src_primitive_desc(), nullptr)); + } else { + src_mem_ = parents_[0].get()->primitive_dst_mem_; + } + } + + if (mklnode_ptr_->output_index >= 0) { + // Use mkldnn's internal output buffer + if (primitive_dst_format_ != ort_source_format_) { + primitive_dst_mem_.reset(new mkldnn::memory(conv_fwd_pd_.get()->dst_primitive_desc())); + } else { + primitive_dst_mem_.reset(new mkldnn::memory(conv_fwd_pd_.get()->dst_primitive_desc(), nullptr)); + } + } else { + // last node of sub-graph. need to allocate memory for output_tensor + primitive_dst_mem_.reset(new mkldnn::memory(conv_fwd_pd_.get()->dst_primitive_desc())); + } + + if (!bias_dims_mkl.empty()) { + bias_mem_.reset(new mkldnn::memory(conv_fwd_pd_.get()->bias_primitive_desc(), nullptr)); + conv_fwd_.reset(new mkldnn::convolution_forward(*conv_fwd_pd_, *src_mem_, *filter_mem_, + *bias_mem_, *primitive_dst_mem_)); + } else { + conv_fwd_.reset(new mkldnn::convolution_forward(*conv_fwd_pd_, *src_mem_, + *filter_mem_, *primitive_dst_mem_)); + } + net.push_back(*conv_fwd_); + + if (mklnode_ptr_->output_index >= 0) { + // one of the end nodes. Allocate output buffer memory and + // reorder is necessary + mkldnn::memory::data_type t = MklDnnType(); + InitDstReorderOutput(cpu_engine, t, net); + } + primitive_created_ = Status::OK(); + return primitive_created_; + } + + virtual void ReorderWeights(const OrtCustomOpApi* api, OrtKernelContext* context, mkldnn::engine& cpu_engine) override { + Ort::CustomOpApi ort{*api}; + int input_index = mklnode_ptr_->input_start_index < 0 ? 0 : mklnode_ptr_->input_start_index; + + const OrtValue* input_tensor = ort.KernelContext_GetInput(context, input_index + 1); + auto tensor_info = ort.GetTensorTypeAndShape(input_tensor); + auto tensor_shape = ort.GetTensorShape(tensor_info); + ort.ReleaseTensorTypeAndShapeInfo(tensor_info); + auto xshape = tensor_shape.data(); + auto xdim = tensor_shape.size(); + + TensorShape W(xshape, xdim); + const T* filter_data = const_cast(ort.GetTensorData(input_tensor)); + + const int group_mkl = static_cast(group_); + + mkldnn::memory::dims filter_dims_mkl; + if (group_mkl == 1) { + filter_dims_mkl.assign(W.GetDims().begin(), W.GetDims().end()); + } else { + filter_dims_mkl.assign({group_mkl, + static_cast(W[0] / group_mkl)}); + filter_dims_mkl.insert(filter_dims_mkl.end(), W.GetDims().begin() + 1, W.GetDims().end()); + } + + { + // lock to make sure reordering is done only once + std::lock_guard lock(provider_->GetMutex()); + std::shared_ptr filter_dst_mem = provider_->GetWeightsMemoryBuffer(mklnode_ptr_->weight_name); + + if (filter_dst_mem == nullptr) { + auto pd = mkldnn::memory::primitive_desc( + mkldnn::memory::desc(filter_dims_mkl, MklDnnType(), filter_format_), cpu_engine); + mkldnn::memory src = mkldnn::memory(pd, (void*)filter_data); + IAllocatorUniquePtr filter_reorder_buffer = + IAllocator::MakeUniquePtr(alloc_, filter_size_); + filter_dst_mem.reset( + new mkldnn::memory(conv_fwd_pd_->weights_primitive_desc(), filter_reorder_buffer.get())); + + MemoryReorderParams params(src, *filter_dst_mem); + DoReorder(params); + provider_->SaveAllocatedMemory(std::move(filter_reorder_buffer)); + filter_data = static_cast(filter_dst_mem->get_data_handle()); + provider_->SetWeightsMemoryBuffer(mklnode_ptr_->weight_name, filter_dst_mem); + } + } + } + + Status Bind(const OrtCustomOpApi* api, OrtKernelContext* context) override { + Ort::CustomOpApi ort{*api}; + + int input_index = mklnode_ptr_->input_start_index < 0 ? 0 : mklnode_ptr_->input_start_index; + if (!primitive_created_.IsOK()) { + // abort as MKLDNN cannot execute this. but + // ORT try to delete output_tensor buffer data. allocate memory so that it can delete + // fix for test_averagepool_1d_default node test + //auto xshape = input_tensors[input_index].shape; + //auto xdim = input_tensors[input_index].ndim; + //AllocateOutputTensor(output_tensors, mklnode_ptr_->output_index, xshape, xdim, input_tensors[0].dtype); + return primitive_created_; + } + const OrtValue* winput_tensor = ort.KernelContext_GetInput(context, input_index + 1); + const T* filter_data = const_cast(ort.GetTensorData(winput_tensor)); + + const T* bias_data = nullptr; + if (mklnode_ptr_->num_inputs == 3) { + const OrtValue* binput_tensor = ort.KernelContext_GetInput(context, input_index + 2); + bias_data = const_cast(ort.GetTensorData(binput_tensor)); + } + std::shared_ptr filter_dst_mem = provider_->GetWeightsMemoryBuffer(mklnode_ptr_->weight_name); + if (filter_dst_mem == nullptr) { + ReorderWeights(api, context, GetEngine()); + filter_dst_mem = provider_->GetWeightsMemoryBuffer(mklnode_ptr_->weight_name); + } + filter_data = static_cast(filter_dst_mem->get_data_handle()); + + filter_mem_->set_data_handle(static_cast(const_cast(filter_data))); + if (bias_data != nullptr) { + bias_mem_->set_data_handle(static_cast(const_cast(bias_data))); + } + + if (primitive_src_format_ != src_format_) { + if (mklnode_ptr_->parent_nodes.empty()) { + const OrtValue* input_tensor = ort.KernelContext_GetInput(context, input_index); + const T* src_data = const_cast(ort.GetTensorData(input_tensor)); + src_mem_from_->set_data_handle(static_cast(const_cast(src_data))); + } else { + src_mem_from_ = parents_[0].get()->primitive_dst_mem_; + } + + auto src_size = conv_fwd_pd_.get()->src_primitive_desc().get_size(); + src_reorder_buffer_ = IAllocator::MakeUniquePtr(alloc_, src_size); + src_mem_->set_data_handle(src_reorder_buffer_.get()); + } else { + if (mklnode_ptr_->parent_nodes.empty()) { + const OrtValue* input_tensor = ort.KernelContext_GetInput(context, input_index); + const T* src_data = const_cast(ort.GetTensorData(input_tensor)); + src_mem_->set_data_handle(static_cast(const_cast(src_data))); + } else { + src_mem_ = parents_[0].get()->primitive_dst_mem_; + } + } + + if (mklnode_ptr_->output_index >= 0) { + auto& y_dims = primitive_dst_shape_.GetDims(); + // Allocate memory for output bufffer + OrtValue* output = ort.KernelContext_GetOutput(context, mklnode_ptr_->output_index, &y_dims[0], static_cast(primitive_dst_shape_.GetDims().size())); + T* dst_data = ort.GetTensorMutableData(output); + + if (primitive_dst_format_ != ort_source_format_) { + reorder_dst_mem_to_->set_data_handle(dst_data); + } else { + primitive_dst_mem_->set_data_handle(dst_data); + } + } + return Status::OK(); + } + + private: + void ReadAttributes(const NodeAttributes& attributes, + const std::string attributes_prefix = "") override { + std::string auto_pad; + auto attr = attributes.find(attributes_prefix + "auto_pad"); + if (attr != attributes.end() && + attr->second.type() == ::ONNX_NAMESPACE::AttributeProto_AttributeType::AttributeProto_AttributeType_STRING) { + auto_pad = attr->second.s(); + } + auto_pad_ = (auto_pad != "") ? StringToAutoPadType(auto_pad) : AutoPadType::NOTSET; + + kernel_shape_specified_ = false; + attr = attributes.find(attributes_prefix + "kernel_shape"); + if (attr != attributes.end()) { + ONNX_NAMESPACE::AttributeProto proto = attr->second; + Status status = GetIntsAttr(proto, kernel_shape_); + kernel_shape_specified_ = true; + } + + attr = attributes.find(attributes_prefix + "strides"); + if (attr != attributes.end()) { + ONNX_NAMESPACE::AttributeProto proto = attr->second; + Status status = GetIntsAttr(proto, strides_); + } + + bool attr_read = false; + attr = attributes.find(attributes_prefix + "pads"); + if (attr != attributes.end()) { + ONNX_NAMESPACE::AttributeProto proto = attr->second; + if (GetIntsAttr(proto, pads_) == Status::OK()) + attr_read = true; + } + if (!attr_read) { + pads_.resize(kernel_shape_.size() * 2, 0); + } + + attr_read = false; + attr = attributes.find(attributes_prefix + "dilations"); + if (attr != attributes.end()) { + ONNX_NAMESPACE::AttributeProto proto = attr->second; + if (GetIntsAttr(proto, dilations_) == Status::OK()) + attr_read = true; + } + if (!attr_read) { + dilations_.resize(kernel_shape_.size(), 1); + } + + attr_read = false; + attr = attributes.find(attributes_prefix + "group"); + if (attr != attributes.end()) { + ONNX_NAMESPACE::AttributeProto proto = attr->second; + if (GetIntAttr(proto, group_) == Status::OK()) + attr_read = true; + } + if (!attr_read) { + group_ = 1; + } + } + + private: + mkldnn::memory::format mkldnn_filter_format_; + mkldnn::memory::format filter_format_; + + std::shared_ptr src_mem_from_; + std::unique_ptr src_mem_to_; + + size_t src_size_; + size_t filter_size_; + size_t dst_size_; + + std::shared_ptr src_mem_; + std::unique_ptr filter_mem_; + std::unique_ptr bias_mem_; + + std::unique_ptr fwd_desc_; + + std::unique_ptr src_md_; + std::unique_ptr filter_md_; + std::unique_ptr bias_md_; + + std::unique_ptr conv_fwd_pd_; + std::unique_ptr conv_fwd_; + + private: + IAllocatorUniquePtr src_reorder_buffer_; + IAllocatorUniquePtr dst_reorder_buffer_; + + private: + Status ComputeKernelShape(const TensorShape& weight_shape, std::vector& kernel_shape) const { + if (kernel_shape_specified_) { + kernel_shape = kernel_shape_; + if (kernel_shape.size() + 2 != weight_shape.NumDimensions()) { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "kernel_shape num_dims is not compatible with W num_dims.", + " kernel_shape: ", TensorShape(kernel_shape).ToString().c_str(), + " W: ", weight_shape.ToString().c_str()); + } + for (size_t i = 0; i < kernel_shape.size(); ++i) { + if (kernel_shape[i] != weight_shape[i + 2]) { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "kernel_shape is not compatible with W shape.", + " kernel_shape: ", TensorShape(kernel_shape).ToString().c_str(), + " W: ", weight_shape.ToString().c_str()); + } + } + } else { + auto& weight_dims = weight_shape.GetDims(); + kernel_shape = std::vector(weight_dims.begin() + 2, weight_dims.end()); + } + + return Status::OK(); + } + + Status ValidateInputShape(const TensorShape& X, const TensorShape& W) const { + const int64_t C = X[1]; + const int64_t M = W[0]; + + if (X.NumDimensions() != W.NumDimensions()) { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "X num_dims does not match W num_dims.", + " X: ", X.ToString().c_str(), + " W: ", W.ToString().c_str()); + } + + if (C != W[1] * group_) { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Input channels C is not equal to kernel channels * group.", + " C: ", C, + " kernel channels: ", W[1], + " group: ", group_); + } + + if (M % group_ != 0) { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Output channels M is not divisible by group.", + " M: ", M, + " group: ", group_); + } + return Status::OK(); + } + + template + Status InferOutputShape(const TensorShape& input_shape, + const std::vector& kernel_shape, + const std::vector& strides, + const std::vector& dilations, + std::vector* pads, + std::vector* output_shape) const { + size_t rank = gsl::narrow_cast(input_shape.NumDimensions()); + for (size_t dim = 0; dim < rank; ++dim) { + if (dim >= strides.size() || dim >= kernel_shape.size() || + dim >= dilations.size() || dim >= pads->size() || + rank + dim >= pads->size()) { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Out of bound access to array"); + } + int64_t dim_size = 0; + ORT_RETURN_IF_ERROR(ComputePadAndOutputShape( + input_shape[dim], + strides[dim], + kernel_shape[dim], + dilations[dim], + auto_pad_, + &pads->at(dim), + &pads->at(input_shape.NumDimensions() + dim), + &dim_size)); + if (dim_size <= 0) { + return Status(common::ONNXRUNTIME, common::INVALID_ARGUMENT, "Invalid input shape: " + input_shape.ToString()); + } + output_shape->push_back(dim_size); + } + return Status::OK(); + } + + private: + std::vector kernel_shape_; // must use ComputeKernelShape(...), instead of kernel_shape_ + AutoPadType auto_pad_; + int64_t group_; + bool kernel_shape_specified_; + std::vector strides_; + std::vector pads_; + std::vector dilations_; + std::string activation_; + float alpha_; +}; +} // namespace mkl_dnn +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/mkldnn/subgraph/mkldnn_func_kernel.cc b/onnxruntime/core/providers/mkldnn/subgraph/mkldnn_func_kernel.cc new file mode 100644 index 0000000000..15a7510505 --- /dev/null +++ b/onnxruntime/core/providers/mkldnn/subgraph/mkldnn_func_kernel.cc @@ -0,0 +1,257 @@ +// Copyright(C) 2018 Intel Corporation +// Licensed under the MIT License +#ifdef _MSC_VER +#pragma warning(disable : 4505) //Unreferenced local function has been removed +#endif + +#include "mkldnn_func_kernel.h" +#include "core/common/exceptions.h" +#include "core/session/onnxruntime_cxx_api.h" +#include "core/providers/mkldnn/mkldnn_common.h" +#include "core/providers/mkldnn/subgraph/mkldnn_conv.h" +#include "core/providers/mkldnn/subgraph/mkldnn_batchnorm.h" +#include "core/providers/mkldnn/subgraph/mkldnn_activations.h" +#include "core/providers/mkldnn/subgraph/mkldnn_pool.h" +#include "core/providers/mkldnn/subgraph/mkldnn_sum.h" +#include "core/providers/mkldnn/subgraph/mkldnn_lrn.h" +#include "core/session/onnxruntime_cxx_api.h" + +namespace onnxruntime { +namespace mkl_dnn { + +namespace { +template +class SubgraphPrimitive : public PrimitiveBase { + public: + SubgraphPrimitive(const OrtCustomOpApi* api, + OrtKernelContext* context, + const SubgraphParams& params) + : cpu_engine_(GetEngine()) { + context_.stream.reset(new mkldnn::stream(mkldnn::stream::kind::eager)); + + if (context_.net.size() == 0) { + CreateKernels(params); + Initialize(api, context); + } + } + + void UpdateProvider(const SubgraphParams& params) { + if (context_.kernels.size() > 0 && context_.kernels[0]->GetProvider() != params.provider) + for (auto& kernel : context_.kernels) { + kernel->SetProvider(params.provider); + } + } + + Status Compute(const OrtCustomOpApi* api, OrtKernelContext* context) { + Status status; + + for (auto& kernel : context_.kernels) { + status = kernel->Bind(api, context); + if (!status.IsOK()) + break; + } + if (status.IsOK()) + context_.stream->submit(context_.net); + return status; + } + + ~SubgraphPrimitive() = default; + + private: + void CreateKernels(const SubgraphParams& params) { + for (const auto& mkldnn_node : params.subgraph->mkldnn_nodes) { + if (mkldnn_node.name == "Conv") { + std::ostringstream os; + os << "Conv-" << mkldnn_node.node_index << "-"; + std::shared_ptr> kernel; + kernel.reset(new MklDnnConv(mkldnn_node, params.provider, params.attributes, os.str())); + for (auto index : mkldnn_node.parent_nodes) { + kernel->parents_.push_back(context_.kernels[index]); + } + context_.kernels.push_back(kernel); + } else if (mkldnn_node.name == "Conv-Relu") { + std::ostringstream os; + os << "Conv-" << mkldnn_node.node_index << "-"; + std::shared_ptr> kernel; + kernel.reset(new MklDnnConv(mkldnn_node, params.provider, params.attributes, os.str())); + kernel->fuse_relu_ = true; + for (auto index : mkldnn_node.parent_nodes) { + kernel->parents_.push_back(context_.kernels[index]); + } + context_.kernels.push_back(kernel); + } else if (mkldnn_node.name == "Relu") { + std::ostringstream os; + os << "Relu-" << mkldnn_node.node_index << "-"; + std::shared_ptr> kernel; + kernel.reset(new MklDnnRelu(mkldnn_node, params.provider, params.attributes, os.str())); + for (auto index : mkldnn_node.parent_nodes) { + kernel->parents_.push_back(context_.kernels[index]); + } + context_.kernels.push_back(kernel); + } else if (mkldnn_node.name == "BatchNormalization") { + std::ostringstream os; + os << "BatchNormalization-" << mkldnn_node.node_index << "-"; + std::shared_ptr> kernel; + kernel.reset(new MklDnnBatchNorm(mkldnn_node, params.provider, params.attributes, os.str())); + for (auto index : mkldnn_node.parent_nodes) { + kernel->parents_.push_back(context_.kernels[index]); + } + context_.kernels.push_back(kernel); + } else if (mkldnn_node.name == "BatchNormalization-Relu") { + std::ostringstream os; + os << "BatchNormalization-" << mkldnn_node.node_index << "-"; + std::shared_ptr> kernel; + kernel.reset(new MklDnnBatchNorm(mkldnn_node, params.provider, params.attributes, os.str())); + kernel->fuse_relu_ = true; + for (auto index : mkldnn_node.parent_nodes) { + kernel->parents_.push_back(context_.kernels[index]); + } + context_.kernels.push_back(kernel); + } else if (mkldnn_node.name == "MaxPool") { + std::ostringstream os; + os << "MaxPool-" << mkldnn_node.node_index << "-"; + std::shared_ptr> kernel; + kernel.reset(new MklDnnPool(mkldnn_node, params.provider, params.attributes, os.str())); + for (auto index : mkldnn_node.parent_nodes) { + kernel->parents_.push_back(context_.kernels[index]); + } + context_.kernels.push_back(kernel); + } else if (mkldnn_node.name == "GlobalMaxPool") { + std::ostringstream os; + os << "GlobalMaxPool-" << mkldnn_node.node_index << "-"; + std::shared_ptr> kernel; + kernel.reset(new MklDnnPool(mkldnn_node, params.provider, params.attributes, os.str())); + for (auto index : mkldnn_node.parent_nodes) { + kernel->parents_.push_back(context_.kernels[index]); + } + context_.kernels.push_back(kernel); + } else if (mkldnn_node.name == "AveragePool") { + std::ostringstream os; + os << "AveragePool-" << mkldnn_node.node_index << "-"; + std::shared_ptr> kernel; + kernel.reset(new MklDnnPool(mkldnn_node, params.provider, params.attributes, os.str())); + for (auto index : mkldnn_node.parent_nodes) { + kernel->parents_.push_back(context_.kernels[index]); + } + context_.kernels.push_back(kernel); + } else if (mkldnn_node.name == "GlobalAveragePool") { + std::ostringstream os; + os << "GlobalAveragePool-" << mkldnn_node.node_index << "-"; + std::shared_ptr> kernel; + kernel.reset(new MklDnnPool(mkldnn_node, params.provider, params.attributes, os.str())); + for (auto index : mkldnn_node.parent_nodes) { + kernel->parents_.push_back(context_.kernels[index]); + } + context_.kernels.push_back(kernel); + } else if (mkldnn_node.name == "Sum") { + std::ostringstream os; + os << "Sum-" << mkldnn_node.node_index << "-"; + std::shared_ptr> kernel; + kernel.reset(new MklDnnSum(mkldnn_node, params.provider, params.attributes, os.str())); + for (auto index : mkldnn_node.parent_nodes) { + kernel->parents_.push_back(context_.kernels[index]); + } + context_.kernels.push_back(kernel); + } else if (mkldnn_node.name == "LRN") { + std::ostringstream os; + os << "LRN-" << mkldnn_node.node_index << "-"; + std::shared_ptr> kernel; + kernel.reset(new MklDnnLrn(mkldnn_node, params.provider, params.attributes, os.str())); + for (auto index : mkldnn_node.parent_nodes) { + kernel->parents_.push_back(context_.kernels[index]); + } + context_.kernels.push_back(kernel); + } + } + } + + private: + struct SubgraphContext { + std::unique_ptr stream; + std::vector net; + std::vector> kernels; + + SubgraphContext() : stream(nullptr) {} + }; + + Status Initialize(const OrtCustomOpApi* api, OrtKernelContext* context) { + // Propagate mkldnn block format + // dst format of current node to src format of next node + mkldnn::memory::format source_format = mkldnn::memory::format::any; // ONNXRuntime format + for (auto& kernel : context_.kernels) { + Status status = kernel->CreatePrimitives(api, context, cpu_engine_, context_.net, source_format); + if (status.IsOK()) + kernel->ReorderWeights(api, context, cpu_engine_); + else + return status; + } + return Status::OK(); + } + + SubgraphContext context_; + mkldnn::engine& cpu_engine_; +}; + +// Pool which allows for reuse of MKLDNN Conv primitives which are expensive to instantiate. +// To address thread safety, the primitives are stored in a map on thread local storage. +template +class SubgraphPrimitivePool : public PrimitivePool { + public: + static SubgraphPrimitive* Get(const OrtCustomOpApi* api, + OrtKernelContext* context, + const SubgraphParams& params) { + Ort::CustomOpApi ort{*api}; + std::string dims_str; + for (auto i = 0; i < params.subgraph->mkldnn_nodes[0].num_inputs; i++) { + const OrtValue* input_tensor = ort.KernelContext_GetInput(context, i); + auto tensor_info = ort.GetTensorTypeAndShape(input_tensor); + auto tensor_shape = ort.GetTensorShape(tensor_info); + ort.ReleaseTensorTypeAndShapeInfo(tensor_info); + auto shape = tensor_shape.data(); + auto dim = tensor_shape.size(); + + TensorShape x_shape(shape, dim); + mkldnn::memory::dims src_dims(x_shape.GetDims().begin(), x_shape.GetDims().end()); + AddDimsToKey(dims_str, src_dims); + } + + SubgraphPrimitive* primitive = dynamic_cast*>( + SubgraphPrimitivePool::GetInstance().GetPrimitive(params.subgraph_key + dims_str)); + + if (primitive == nullptr) { + auto subgraph_primitive = std::make_unique>(api, context, params); + primitive = subgraph_primitive.get(); + SubgraphPrimitivePool::GetInstance().SetPrimitive(params.subgraph_key + dims_str, std::move(subgraph_primitive)); + } + return primitive; + } + + private: + SubgraphPrimitivePool() = default; + ~SubgraphPrimitivePool() = default; + + static SubgraphPrimitivePool& GetInstance() { + static SubgraphPrimitivePool pool; + return pool; + } +}; +} // namespace + +template +Status MkldnnFuncKernel::Compute(const OrtCustomOpApi* api, OrtKernelContext* context) const { + Status status; + try { + SubgraphPrimitive* primitive = SubgraphPrimitivePool::Get(api, context, params_); + primitive->UpdateProvider(params_); + status = primitive->Compute(api, context); + } catch (const mkldnn::error& e) { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Status: ", e.status, + ", message: ", e.message.c_str()); + } + return status; +} + +template class MkldnnFuncKernel; + +} // namespace mkl_dnn +} // namespace onnxruntime \ No newline at end of file diff --git a/onnxruntime/core/providers/mkldnn/subgraph/mkldnn_func_kernel.h b/onnxruntime/core/providers/mkldnn/subgraph/mkldnn_func_kernel.h new file mode 100644 index 0000000000..59105a3b8c --- /dev/null +++ b/onnxruntime/core/providers/mkldnn/subgraph/mkldnn_func_kernel.h @@ -0,0 +1,234 @@ +// Copyright(C) 2018 Intel Corporation +// Licensed under the MIT License +#pragma once + +#include "core/graph/onnx_protobuf.h" +#include "core/providers/mkldnn/mkldnn_execution_provider.h" +#include "core/session/onnxruntime_c_api.h" +#include "core/framework/func_api.h" + +namespace onnxruntime { +namespace mkl_dnn { + +namespace { +struct SubgraphParams { + NodeAttributes attributes; + MKLDNNExecutionProvider* provider; + std::shared_ptr subgraph; + std::string subgraph_id; + std::string subgraph_key; + + SubgraphParams() {} +}; +} // namespace + +template +class MkldnnFuncKernel { + public: + explicit MkldnnFuncKernel(const ComputeContext* context, + const NodeAttributes& attributes, + MKLDNNExecutionProvider* provider) { + ORT_UNUSED_PARAMETER(context); + + params_.provider = provider; + params_.attributes = attributes; + + auto sub_it = attributes.find("subgraph_id"); + if (sub_it->second.type() == ONNX_NAMESPACE::AttributeProto_AttributeType::AttributeProto_AttributeType_STRING) { + params_.subgraph_id = sub_it->second.s(); + params_.subgraph = provider->GetMklDnnSubgraph(params_.subgraph_id); + + std::ostringstream key_os; + key_os << params_.subgraph->graph_name << "_" << params_.subgraph_id << "-"; + key_os << params_.subgraph->mkldnn_nodes.back().ToString() << "-"; + key_os << params_.subgraph->mkldnn_nodes.back().output_name; + + if (params_.subgraph->mkldnn_nodes[0].name == "Conv") { + std::ostringstream os; + os << "Conv-" << params_.subgraph->mkldnn_nodes[0].node_index << "-"; + key_os << GetConvAttributeKey(attributes, os.str()); + } + + if (params_.subgraph->mkldnn_nodes[0].name == "LRN") { + std::ostringstream os; + os << "LRN-" << params_.subgraph->mkldnn_nodes[0].node_index << "-"; + key_os << GetLrnAttributeKey(attributes, os.str()); + } + + if (params_.subgraph->mkldnn_nodes[0].name.find("Pool") != std::string::npos) { + std::ostringstream os; + os << params_.subgraph->mkldnn_nodes[0].name << "-" << params_.subgraph->mkldnn_nodes[0].node_index << "-"; + key_os << GetPoolAttributesKey(attributes, os.str()); + } + + params_.subgraph_key = key_os.str(); + } + } + + std::string GetPoolAttributesKey(const NodeAttributes& attributes, + const std::string attributes_prefix = "") { + std::string key; + + auto attr = attributes.find(attributes_prefix + "kernel_shape"); + if (attr != attributes.end()) { + key.append(1, '#'); + ONNX_NAMESPACE::AttributeProto proto = attr->second; + for (int i = 0; i < proto.ints_size(); i++) { + key.append(std::to_string(proto.ints(i))); + key.append(1, '_'); + } + key.append(1, '#'); + } + + attr = attributes.find(attributes_prefix + "auto_pad"); + if (attr != attributes.end()) { + key.append(attr->second.s()); + } + + attr = attributes.find(attributes_prefix + "pads"); + if (attr != attributes.end()) { + key.append(1, '#'); + ONNX_NAMESPACE::AttributeProto proto = attr->second; + for (int i = 0; i < proto.ints_size(); i++) { + key.append(std::to_string(proto.ints(i))); + key.append(1, '_'); + } + key.append(1, '#'); + } + + attr = attributes.find(attributes_prefix + "strides"); + if (attr != attributes.end()) { + key.append(1, '#'); + ONNX_NAMESPACE::AttributeProto proto = attr->second; + for (int i = 0; i < proto.ints_size(); i++) { + key.append(std::to_string(proto.ints(i))); + key.append(1, '_'); + } + key.append(1, '#'); + } + + attr = attributes.find(attributes_prefix + "count_include_pad"); + if (attr != attributes.end()) { + key.append(1, '#'); + ONNX_NAMESPACE::AttributeProto proto = attr->second; + key.append(std::to_string(proto.i())); + key.append(1, '_'); + key.append(1, '#'); + } + + attr = attributes.find(attributes_prefix + "ceil_mode"); + if (attr != attributes.end()) { + key.append(1, '#'); + ONNX_NAMESPACE::AttributeProto proto = attr->second; + key.append(std::to_string(proto.i())); + key.append(1, '_'); + key.append(1, '#'); + } + return key; + } + + std::string GetConvAttributeKey(const NodeAttributes& attributes, + const std::string attributes_prefix = "") { + std::string key; + + auto attr = attributes.find(attributes_prefix + "dilations"); + if (attr != attributes.end()) { + key.append(1, '#'); + ONNX_NAMESPACE::AttributeProto proto = attr->second; + for (int i = 0; i < proto.ints_size(); i++) { + key.append(std::to_string(proto.ints(i))); + key.append(1, '_'); + } + key.append(1, '#'); + } + + attr = attributes.find(attributes_prefix + "auto_pad"); + if (attr != attributes.end()) { + key.append(1, '#'); + ONNX_NAMESPACE::AttributeProto proto = attr->second; + key.append(proto.s()); + key.append(1, '#'); + } + + attr = attributes.find(attributes_prefix + "pads"); + if (attr != attributes.end()) { + key.append(1, '#'); + ONNX_NAMESPACE::AttributeProto proto = attr->second; + for (int i = 0; i < proto.ints_size(); i++) { + key.append(std::to_string(proto.ints(i))); + key.append(1, '_'); + } + key.append(1, '#'); + } + + attr = attributes.find(attributes_prefix + "strides"); + if (attr != attributes.end()) { + key.append(1, '#'); + ONNX_NAMESPACE::AttributeProto proto = attr->second; + for (int i = 0; i < proto.ints_size(); i++) { + key.append(std::to_string(proto.ints(i))); + key.append(1, '_'); + } + key.append(1, '#'); + } + + attr = attributes.find(attributes_prefix + "kernel_shape"); + if (attr != attributes.end()) { + key.append(1, '#'); + ONNX_NAMESPACE::AttributeProto proto = attr->second; + for (int i = 0; i < proto.ints_size(); i++) { + key.append(std::to_string(proto.ints(i))); + key.append(1, '_'); + } + key.append(1, '#'); + } + + attr = attributes.find(attributes_prefix + "group"); + if (attr != attributes.end()) { + key.append(1, '#'); + ONNX_NAMESPACE::AttributeProto proto = attr->second; + key.append(std::to_string(proto.i())); + key.append(1, '#'); + } + + return key; + } + + std::string GetLrnAttributeKey(const NodeAttributes& attributes, + const std::string attributes_prefix = "") { + std::string key; + + auto attr = attributes.find(attributes_prefix + "alpha"); + if (attr != attributes.end()) { + key.append(1, '#'); + ONNX_NAMESPACE::AttributeProto proto = attr->second; + key.append(std::to_string(proto.f())); + key.append(1, '#'); + } + + attr = attributes.find(attributes_prefix + "beta"); + if (attr != attributes.end()) { + key.append(1, '#'); + ONNX_NAMESPACE::AttributeProto proto = attr->second; + key.append(std::to_string(proto.f())); + key.append(1, '#'); + } + + attr = attributes.find(attributes_prefix + "bias"); + if (attr != attributes.end()) { + key.append(1, '#'); + ONNX_NAMESPACE::AttributeProto proto = attr->second; + key.append(std::to_string(proto.f())); + key.append(1, '#'); + } + + return key; + } + + Status Compute(const OrtCustomOpApi* api, OrtKernelContext* context) const; + + private: + SubgraphParams params_; +}; +} // namespace mkl_dnn +} // namespace onnxruntime \ No newline at end of file diff --git a/onnxruntime/core/providers/mkldnn/subgraph/mkldnn_kernel.cc b/onnxruntime/core/providers/mkldnn/subgraph/mkldnn_kernel.cc new file mode 100644 index 0000000000..19007e99ef --- /dev/null +++ b/onnxruntime/core/providers/mkldnn/subgraph/mkldnn_kernel.cc @@ -0,0 +1,57 @@ +// Copyright(C) 2019 Intel Corporation +// Licensed under the MIT License + +#include "mkldnn_kernel.h" + +namespace onnxruntime { +namespace mkl_dnn { + +void MklDnnKernel::InitDstReorderOutput(mkldnn::engine& cpu_engine, + mkldnn::memory::data_type& data_type, + std::vector& net) { + // Allocate dst buffer if reorder is necessary + if (primitive_dst_format_ != ort_source_format_) { + // reorder to ONNXRuntime format + mkldnn::memory::dims dst_dims_mkl( + primitive_dst_shape_.GetDims().begin(), primitive_dst_shape_.GetDims().end()); + mkldnn::memory::desc dst_des = mkldnn::memory::desc(dst_dims_mkl, + data_type, ort_source_format_); + reorder_dst_mem_to_.reset(new mkldnn::memory({dst_des, cpu_engine}, nullptr)); + net.push_back(mkldnn::reorder(*primitive_dst_mem_, *reorder_dst_mem_to_)); + } +} + +mkldnn::memory::format MklDnnKernel::GetSourceFormat(int dim_size) { + mkldnn::memory::format source_format = mkldnn::memory::format::any; + switch (dim_size) { + case 1: { + source_format = mkldnn::memory::format::x; + break; + } + case 2: { + source_format = mkldnn::memory::format::nc; + break; + } + case 3: { + source_format = mkldnn::memory::format::ntc; + break; + } + case 4: { + source_format = mkldnn::memory::format::nchw; + break; + } + case 5: { + source_format = mkldnn::memory::format::ncdhw; + break; + } + default: { + source_format = mkldnn::memory::format::any; + break; + } + } + + return source_format; +} + +} // namespace mkl_dnn +} // namespace onnxruntime \ No newline at end of file diff --git a/onnxruntime/core/providers/mkldnn/subgraph/mkldnn_kernel.h b/onnxruntime/core/providers/mkldnn/subgraph/mkldnn_kernel.h new file mode 100644 index 0000000000..0a4bc68eea --- /dev/null +++ b/onnxruntime/core/providers/mkldnn/subgraph/mkldnn_kernel.h @@ -0,0 +1,121 @@ +// Copyright(C) 2019 Intel Corporation +// Licensed under the MIT License + +#pragma once +#ifdef _WIN32 +#pragma warning(disable : 4244) +#endif + +#include "mkldnn.hpp" +#include "core/common/cpuid_info.h" +#include "core/session/onnxruntime_cxx_api.h" +#include "core/providers/mkldnn/subgraph/subgraph.h" +#include "core/providers/mkldnn/mkldnn_execution_provider.h" + +namespace onnxruntime { +namespace mkl_dnn { + +class MklDnnKernel { + public: + MklDnnKernel(const MklDnnNode& node, + MKLDNNExecutionProvider* provider) { + mklnode_ptr_ = std::make_shared(node); + provider_ = provider; + alloc_ = provider_->GetAllocator(0, OrtMemTypeDefault); + } + virtual ~MklDnnKernel(){}; + + virtual Status CreatePrimitives(const OrtCustomOpApi* api, + OrtKernelContext* context, + mkldnn::engine& cpu_engine, + std::vector& net, + mkldnn::memory::format& src_fmt) = 0; + + virtual void ReorderWeights(const OrtCustomOpApi* api, OrtKernelContext* context, mkldnn::engine& cpu_engine) { + ORT_UNUSED_PARAMETER(api); + ORT_UNUSED_PARAMETER(context); + ORT_UNUSED_PARAMETER(cpu_engine); + } + virtual void SetProvider(MKLDNNExecutionProvider* provider) { + provider_ = provider; + } + MKLDNNExecutionProvider* GetProvider() { + return provider_; + } + + virtual Status Bind(const OrtCustomOpApi* api, OrtKernelContext* context) = 0; + + protected: + virtual void ReadAttributes(const NodeAttributes& attributes, + const std::string attributes_prefix = "") { + ORT_UNUSED_PARAMETER(attributes); + ORT_UNUSED_PARAMETER(attributes_prefix); + } + + Status GetIntsAttr(ONNX_NAMESPACE::AttributeProto& proto, std::vector& values) { + ORT_RETURN_IF_NOT(proto.type() == ::ONNX_NAMESPACE::AttributeProto_AttributeType::AttributeProto_AttributeType_INTS); + values.reserve(proto.ints_size()); + for (int i = 0; i < proto.ints_size(); i++) { + values.push_back(proto.ints(i)); + } + return Status::OK(); + } + + Status GetIntAttr(ONNX_NAMESPACE::AttributeProto& proto, int64_t& value) { + ORT_RETURN_IF_NOT(proto.type() == ::ONNX_NAMESPACE::AttributeProto_AttributeType::AttributeProto_AttributeType_INT); + value = proto.i(); + return Status::OK(); + } + + Status GetFloatAttr(ONNX_NAMESPACE::AttributeProto& proto, float& value) { + ORT_RETURN_IF_NOT(proto.type() == ::ONNX_NAMESPACE::AttributeProto_AttributeType::AttributeProto_AttributeType_FLOAT); + value = proto.f(); + return Status::OK(); + } + Status GetStringAttr(ONNX_NAMESPACE::AttributeProto& proto, std::string& value) { + ORT_RETURN_IF_NOT(proto.type() == ::ONNX_NAMESPACE::AttributeProto_AttributeType::AttributeProto_AttributeType_STRING); + value = proto.s(); + return Status::OK(); + } + + void InitDstReorderOutput(mkldnn::engine& cpu_engine, + mkldnn::memory::data_type& data_type, + std::vector& net); + + //void AllocateMemoryAndReorderIfNeeded(const OrtCustomOpApi* api); + + //void AllocateOutputTensor(const OrtCustomOpApi* api, int index, const int64_t* shape, size_t dim); + + mkldnn::memory::format GetSourceFormat(int dim_size); + + public: + std::vector> parents_; + bool fuse_relu_ = false; + bool fuse_sum_ = false; + std::shared_ptr primitive_dst_mem_; + std::unique_ptr primitive_dst_md_; + TensorShape primitive_dst_shape_; + mkldnn::memory::format primitive_dst_format_ = mkldnn::memory::format::any; + + protected: + // ONNX Runtime format + mkldnn::memory::format ort_source_format_ = mkldnn::memory::format::any; + // input format. + // It can be ORT format (nchw) or blocked memory format from parent nce + mkldnn::memory::format src_format_ = mkldnn::memory::format::any; + // Pointer to MklNode of subgraph IR + std::shared_ptr mklnode_ptr_; + // input format expected by primitive object + mkldnn::memory::format primitive_src_format_ = mkldnn::memory::format::any; + + // memory used for reorders + std::unique_ptr reorder_dst_mem_to_; + + protected: + Status primitive_created_; + AllocatorPtr alloc_; + MKLDNNExecutionProvider* provider_; +}; + +} // namespace mkl_dnn +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/mkldnn/subgraph/mkldnn_lrn.h b/onnxruntime/core/providers/mkldnn/subgraph/mkldnn_lrn.h new file mode 100644 index 0000000000..c27914e79a --- /dev/null +++ b/onnxruntime/core/providers/mkldnn/subgraph/mkldnn_lrn.h @@ -0,0 +1,183 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/util/math.h" +#include "core/util/math_cpuonly.h" +#include "core/framework/op_kernel.h" +#include "core/providers/mkldnn/mkldnn_fwd.h" +#include "core/providers/mkldnn/mkldnn_execution_provider.h" +#include "core/providers/mkldnn/subgraph/mkldnn_kernel.h" + +namespace onnxruntime { +namespace mkl_dnn { + +template +class MklDnnLrn : public MklDnnKernel { + public: + MklDnnLrn(const MklDnnNode& node, + MKLDNNExecutionProvider* provider, + const NodeAttributes& attributes, + const std::string attributes_prefix = "") : MklDnnKernel(node, provider) { + ReadAttributes(attributes, attributes_prefix); + } + + Status CreatePrimitives(const OrtCustomOpApi* api, + OrtKernelContext* context, + mkldnn::engine& cpu_engine, + std::vector& net, + mkldnn::memory::format& source_format) { + Ort::CustomOpApi ort{*api}; + int input_index = mklnode_ptr_->input_start_index < 0 ? 0 : mklnode_ptr_->input_start_index; + + TensorShape x_shape; + if (mklnode_ptr_->parent_nodes.empty()) { + const OrtValue* input_tensor = ort.KernelContext_GetInput(context, input_index); + auto tensor_info = ort.GetTensorTypeAndShape(input_tensor); + auto tensor_shape = ort.GetTensorShape(tensor_info); + ort.ReleaseTensorTypeAndShapeInfo(tensor_info); + auto xshape = tensor_shape.data(); + auto xdim = tensor_shape.size(); + + ort_source_format_ = GetSourceFormat(static_cast(xdim)); + source_format = ort_source_format_; + x_shape = TensorShape(xshape, xdim); + + mkldnn::memory::dims src_dims_mkl( + x_shape.GetDims().begin(), x_shape.GetDims().end()); + + src_md_.reset(new mkldnn::memory::desc( + {src_dims_mkl}, MklDnnType(), source_format)); + src_mem_.reset( + new mkldnn::memory({*src_md_, cpu_engine}, nullptr)); + } else { + src_md_.reset( + new mkldnn::memory::desc(parents_[0].get()->primitive_dst_mem_.get()->get_primitive_desc().desc())); + src_mem_ = parents_[0].get()->primitive_dst_mem_; + x_shape = parents_[0].get()->primitive_dst_shape_; + ort_source_format_ = source_format; + } + + primitive_dst_shape_ = TensorShape(x_shape); + + mkldnn::algorithm algo = mkldnn::algorithm::lrn_across_channels; + fwd_desc_.reset(new mkldnn::lrn_forward::desc( + mkldnn::prop_kind::forward_scoring, algo, *src_md_, + size_, alpha_, beta_, bias_)); + + fwd_primitive_desc_.reset(new mkldnn::lrn_forward::primitive_desc( + *fwd_desc_, cpu_engine)); + + primitive_src_format_ = static_cast( + fwd_primitive_desc_.get()->src_primitive_desc().desc().data.format); + primitive_dst_format_ = static_cast( + fwd_primitive_desc_.get()->dst_primitive_desc().desc().data.format); + + if (mklnode_ptr_->output_index >= 0) { + // last node of sub-graph. need to allocate memory for output_tensor + if (primitive_dst_format_ != ort_source_format_) { + // reorder neded. Use primitive output as input to reorder and + // allocate buffer for reorder output, final output of this subgraph + primitive_dst_mem_.reset( + new mkldnn::memory(fwd_primitive_desc_.get()->dst_primitive_desc())); + } else { + // Last node but re-order not needed. Allocate buffer to output of this node + primitive_dst_mem_.reset( + new mkldnn::memory(fwd_primitive_desc_.get()->dst_primitive_desc(), nullptr)); + } + } else { + // Intermediate node. Use mkldnn kernel internal memory for output and + // use this as input to next node. + primitive_dst_mem_.reset( + new mkldnn::memory(fwd_primitive_desc_.get()->dst_primitive_desc())); + } + + lrn_fwd_.reset( + new mkldnn::lrn_forward(*fwd_primitive_desc_, *src_mem_, *primitive_dst_mem_)); + net.push_back(*lrn_fwd_); + + if (mklnode_ptr_->output_index >= 0) { + // one of the end nodes. Allocate output buffer memory and + // reorder is necessary + mkldnn::memory::data_type t = MklDnnType(); + InitDstReorderOutput(cpu_engine, t, net); + } + + return Status::OK(); + } + + Status Bind(const OrtCustomOpApi* api, OrtKernelContext* context) override { + Ort::CustomOpApi ort{*api}; + int input_index = mklnode_ptr_->input_start_index < 0 ? 0 : mklnode_ptr_->input_start_index; + + if (mklnode_ptr_->parent_nodes.empty()) { + // Sub-graph's first node. Read input from input buffer + const OrtValue* input_tensor = ort.KernelContext_GetInput(context, input_index); + const T* src_data = const_cast(ort.GetTensorData(input_tensor)); + src_mem_->set_data_handle(static_cast(const_cast(src_data))); + } + + if (mklnode_ptr_->output_index >= 0) { + auto& y_dims = primitive_dst_shape_.GetDims(); + // Allocate memory for output bufffer + OrtValue* output = ort.KernelContext_GetOutput(context, mklnode_ptr_->output_index, &y_dims[0], static_cast(primitive_dst_shape_.GetDims().size())); + T* dst_data = ort.GetTensorMutableData(output); + + if (primitive_dst_format_ != ort_source_format_) { + reorder_dst_mem_to_->set_data_handle(dst_data); + } else { + primitive_dst_mem_->set_data_handle(dst_data); + } + } + + return Status::OK(); + } + + private: + void ReadAttributes(const NodeAttributes& attributes, + const std::string attributes_prefix = "") override { + auto attr = attributes.find(attributes_prefix + "size"); + if (attr != attributes.end() && + attr->second.type() == ::ONNX_NAMESPACE::AttributeProto_AttributeType::AttributeProto_AttributeType_INT) { + size_ = attr->second.i(); + } + ORT_ENFORCE(size_ > 0); + ORT_ENFORCE(size_ % 2 == 1); + + attr = attributes.find(attributes_prefix + "alpha"); + if (attr != attributes.end() && + attr->second.type() == ::ONNX_NAMESPACE::AttributeProto_AttributeType::AttributeProto_AttributeType_FLOAT) { + alpha_ = attr->second.f(); + } + + attr = attributes.find(attributes_prefix + "beta"); + if (attr != attributes.end() && + attr->second.type() == ::ONNX_NAMESPACE::AttributeProto_AttributeType::AttributeProto_AttributeType_FLOAT) { + beta_ = attr->second.f(); + } + + bias_ = 1.0f; + attr = attributes.find(attributes_prefix + "bias"); + if (attr != attributes.end() && + attr->second.type() == ::ONNX_NAMESPACE::AttributeProto_AttributeType::AttributeProto_AttributeType_FLOAT) { + bias_ = attr->second.f(); + } + } + + private: + float alpha_ = 0; + float beta_ = 0; + float bias_ = 0; + int size_ = 0; + + private: + std::shared_ptr src_mem_; + + std::unique_ptr fwd_desc_; + std::unique_ptr fwd_primitive_desc_; + std::unique_ptr lrn_fwd_; + + std::unique_ptr src_md_; +}; +} // namespace mkl_dnn +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/mkldnn/subgraph/mkldnn_pool.h b/onnxruntime/core/providers/mkldnn/subgraph/mkldnn_pool.h new file mode 100644 index 0000000000..e4ac5099fc --- /dev/null +++ b/onnxruntime/core/providers/mkldnn/subgraph/mkldnn_pool.h @@ -0,0 +1,427 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once +#include "core/providers/mkldnn/mkldnn_fwd.h" +#include "core/providers/cpu/nn/autopad_type.h" +#include "core/providers/mkldnn/mkldnn_execution_provider.h" +#include "core/providers/mkldnn/subgraph/mkldnn_kernel.h" +#include "core/util/math.h" + +namespace onnxruntime { +namespace mkl_dnn { +template +class MklDnnPool : public MklDnnKernel { + public: + MklDnnPool(const MklDnnNode& node, + MKLDNNExecutionProvider* provider, + const NodeAttributes& attributes, + const std::string attributes_prefix = "") : MklDnnKernel(node, provider) { + op_name_ = node.name; + ReadAttributes(attributes, attributes_prefix); + } + + Status CreatePrimitives(const OrtCustomOpApi* api, + OrtKernelContext* context, + mkldnn::engine& cpu_engine, std::vector& net, + mkldnn::memory::format& source_format) override { + Ort::CustomOpApi ort{*api}; + int input_index = mklnode_ptr_->input_start_index < 0 ? 0 : mklnode_ptr_->input_start_index; + + if (mklnode_ptr_->parent_nodes.empty()) { + const OrtValue* input_tensor = ort.KernelContext_GetInput(context, input_index); + auto tensor_info = ort.GetTensorTypeAndShape(input_tensor); + auto tensor_shape = ort.GetTensorShape(tensor_info); + ort.ReleaseTensorTypeAndShapeInfo(tensor_info); + auto xshape = tensor_shape.data(); + auto xdim = tensor_shape.size(); + + mkldnn::memory::dims dims(xdim); + ort_source_format_ = GetSourceFormat(static_cast(xdim)); + source_format = ort_source_format_; + src_format_ = ort_source_format_; + x_shape_ = TensorShape(xshape, xdim); + + mkldnn::memory::dims src_dims_mkl(x_shape_.GetDims().begin(), x_shape_.GetDims().end()); + + // reorder for better performance + mkldnn::memory::format fmt = GetAVXFormat(src_dims_mkl); + src_md_.reset(new mkldnn::memory::desc( + {src_dims_mkl}, MklDnnType(), fmt)); + } else { + // get the output of previous node (mkldnn block propagation). + // TODO Sourcenode will set src of this node. + x_shape_ = parents_[0].get()->primitive_dst_shape_; + ort_source_format_ = source_format; + src_format_ = parents_[0].get()->primitive_dst_format_; + mkldnn::memory::dims src_dims_mkl(x_shape_.GetDims().begin(), x_shape_.GetDims().end()); + + if (src_format_ == ort_source_format_) { + // reorder for better performance + mkldnn::memory::format fmt = GetAVXFormat(src_dims_mkl); + src_md_.reset(new mkldnn::memory::desc( + {src_dims_mkl}, MklDnnType(), fmt)); + } else { + src_md_.reset(new mkldnn::memory::desc( + parents_[0].get()->primitive_dst_mem_.get()->get_primitive_desc().desc())); + } + } + + const auto& x_dims = x_shape_.GetDims(); + std::vector y_dims = SetOutputSize(x_shape_, x_shape_[1], &pads_); + primitive_dst_shape_ = TensorShape(y_dims); + + if (x_shape_.NumDimensions() <= 3) { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Please call default CPU kernel."); + } + + if (global_pooling_) { + kernel_shape_.assign(x_dims.begin() + 2, x_dims.end()); + pads_.assign(kernel_shape_.size() * 2, 0); + strides_.assign(kernel_shape_.size(), 1); + } + + size_t num_outputs = 1; //OpKernel::Node().OutputDefs().size(); TODO + if (num_outputs == 2) { + ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "can not call cpu default op"); + } + + mkldnn::memory::dims dst_dims_mkl(y_dims.begin(), y_dims.end()); + mkldnn::memory::dims kernel_mkl(kernel_shape_.begin(), kernel_shape_.end()); + mkldnn::memory::dims strides_mkl(strides_.begin(), strides_.end()); + mkldnn::memory::dims padding_left_mkl(pads_.begin(), pads_.begin() + (pads_.size() / 2)); + mkldnn::memory::dims padding_right_mkl(pads_.begin() + (pads_.size() / 2), pads_.end()); + + primitive_dst_md_.reset(new mkldnn::memory::desc( + {dst_dims_mkl}, MklDnnType(), mkldnn::memory::format::any)); + + mkldnn::algorithm algo = mkldnn::algorithm::pooling_max; + if (op_name_ == "AveragePool" || op_name_ == "GlobalAveragePool") { + algo = mkldnn::algorithm::pooling_avg_exclude_padding; + if (count_include_pad_) { + algo = mkldnn::algorithm::pooling_avg_include_padding; + } + } + fwd_desc_.reset(new mkldnn::pooling_forward::desc( + mkldnn::prop_kind::forward_inference, algo, + *src_md_, *primitive_dst_md_, + strides_mkl, kernel_mkl, + padding_left_mkl, padding_right_mkl, + mkldnn::padding_kind::zero)); + + fwd_primitive_desc_.reset(new mkldnn::pooling_forward::primitive_desc( + *fwd_desc_, cpu_engine)); + + if (mklnode_ptr_->parent_nodes.empty()) { + // Sub-graph's first node. Read input from input buffer + src_mem_.reset(new mkldnn::memory( + fwd_primitive_desc_.get()->src_primitive_desc(), nullptr)); + } else { + // Sub-graph's inner node. set input to parent's output + src_mem_ = parents_[0].get()->primitive_dst_mem_; + } + + primitive_src_format_ = static_cast( + fwd_primitive_desc_.get()->src_primitive_desc().desc().data.format); + + primitive_dst_format_ = static_cast( + fwd_primitive_desc_.get()->dst_primitive_desc().desc().data.format); + + src_size_ = fwd_primitive_desc_.get()->src_primitive_desc().get_size(); + dst_size_ = fwd_primitive_desc_.get()->dst_primitive_desc().get_size(); + + // reorder source memory for best performance (AVX512); + if (primitive_src_format_ != src_format_) { + mkldnn::memory::dims src_dims_mkl(x_shape_.GetDims().begin(), x_shape_.GetDims().end()); + auto src_md = mkldnn::memory::desc(src_dims_mkl, MklDnnType(), src_format_); + auto pd = mkldnn::memory::primitive_desc(src_md, cpu_engine); + + if (mklnode_ptr_->parent_nodes.empty()) + src_mem_from_.reset(new mkldnn::memory(pd, nullptr)); + else + src_mem_from_ = parents_[0].get()->primitive_dst_mem_; + + src_mem_.reset(new mkldnn::memory(fwd_primitive_desc_->src_primitive_desc(), nullptr)); + net.push_back(mkldnn::reorder(*src_mem_from_, *src_mem_)); + } else { + if (mklnode_ptr_->parent_nodes.empty()) { + src_mem_.reset(new mkldnn::memory(fwd_primitive_desc_->src_primitive_desc(), nullptr)); + } else { + src_mem_ = parents_[0].get()->primitive_dst_mem_; + } + } + + if (mklnode_ptr_->output_index >= 0) { + // last node of sub-graph. need to allocate memory for output_tensor + if (primitive_dst_format_ != ort_source_format_) { + // reorder neded. Use primitive output as input to reorder and + // allocate buffer for reorder output, final output of this subgraph + primitive_dst_mem_.reset( + new mkldnn::memory(fwd_primitive_desc_.get()->dst_primitive_desc())); + } else { + // Last node but re-order not needed. Allocate buffer to output of this node + primitive_dst_mem_.reset( + new mkldnn::memory(fwd_primitive_desc_.get()->dst_primitive_desc(), nullptr)); + } + } else { + // Intermediate node. Use mkldnn kernel internal memory for output and + // use this as input to next node. + primitive_dst_mem_.reset( + new mkldnn::memory(fwd_primitive_desc_.get()->dst_primitive_desc())); + } + pool_fwd_.reset( + new mkldnn::pooling_forward(*fwd_primitive_desc_, *src_mem_, *primitive_dst_mem_)); + + net.push_back(*pool_fwd_); + + if (mklnode_ptr_->output_index >= 0) { + // one of the end nodes. Allocate output buffer memory and + // reorder is necessary + mkldnn::memory::data_type t = MklDnnType(); + InitDstReorderOutput(cpu_engine, t, net); + } + return Status::OK(); + } + + Status Bind(const OrtCustomOpApi* api, OrtKernelContext* context) override { + Ort::CustomOpApi ort{*api}; + int input_index = mklnode_ptr_->input_start_index < 0 ? 0 : mklnode_ptr_->input_start_index; + + if (x_shape_.NumDimensions() <= 3) { + if (mklnode_ptr_->parent_nodes.empty()) { + // abort as MKLDNN cannot execute this. but + // ORT try to delete output_tensor buffer data. allocate memory so that it can delete + // fix for test_averagepool_1d_default node test + //auto xshape = input_tensors[input_index].shape; + //auto xdim = input_tensors[input_index].ndim; + //AllocateOutputTensor(output_tensors, mklnode_ptr_->output_index, xshape, xdim, input_tensors[0].dtype); + } + std::cout << "MKLDNN cannot compute shape with dim less than three." << std::endl; + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Please call default CPU kernel."); + } + + if (primitive_src_format_ != src_format_) { + if (mklnode_ptr_->parent_nodes.empty()) { + const OrtValue* input_tensor = ort.KernelContext_GetInput(context, input_index); + const T* src_data = const_cast(ort.GetTensorData(input_tensor)); + src_mem_from_->set_data_handle(static_cast(const_cast(src_data))); + } else { + src_mem_from_ = parents_[0].get()->primitive_dst_mem_; + } + + auto src_size = fwd_primitive_desc_.get()->src_primitive_desc().get_size(); + src_reorder_buffer_ = IAllocator::MakeUniquePtr(alloc_, src_size); + src_mem_->set_data_handle(src_reorder_buffer_.get()); + } else { + if (mklnode_ptr_->parent_nodes.empty()) { + const OrtValue* input_tensor = ort.KernelContext_GetInput(context, input_index); + const T* src_data = const_cast(ort.GetTensorData(input_tensor)); + src_mem_->set_data_handle(static_cast(const_cast(src_data))); + } else { + src_mem_ = parents_[0].get()->primitive_dst_mem_; + } + } + + if (mklnode_ptr_->output_index >= 0) { + // Last node of sub-graph. Allocate memory for output_buffer data + // Reorder if needed + auto& y_dims = primitive_dst_shape_.GetDims(); + // Allocate memory for output bufffer + OrtValue* output = ort.KernelContext_GetOutput(context, mklnode_ptr_->output_index, &y_dims[0], static_cast(primitive_dst_shape_.GetDims().size())); + T* dst_data = ort.GetTensorMutableData(output); + + if (primitive_dst_format_ != ort_source_format_) { + reorder_dst_mem_to_->set_data_handle(dst_data); + } else { + primitive_dst_mem_->set_data_handle(dst_data); + } + } + return Status::OK(); + } + + private: + void ReadAttributes(const NodeAttributes& attributes, + const std::string attributes_prefix = "") override { + global_pooling_ = (op_name_ == "GlobalAveragePool" || op_name_ == "GlobalMaxPool" || op_name_ == "GlobalLpPool"); + global_pooling_ = (op_name_ == "GlobalAveragePool" || op_name_ == "GlobalMaxPool" || op_name_ == "GlobalLpPool"); + + if (!global_pooling_) { + bool attr_read = false; + auto attr = attributes.find(attributes_prefix + "kernel_shape"); + if (attr != attributes.end()) { + ONNX_NAMESPACE::AttributeProto proto = attr->second; + GetIntsAttr(proto, kernel_shape_); + attr_read = true; + } + ORT_ENFORCE(attr_read, "No kernel shape is set."); + + std::string auto_padding; + attr = attributes.find(attributes_prefix + "auto_pad"); + if (attr != attributes.end() && + attr->second.type() == ::ONNX_NAMESPACE::AttributeProto_AttributeType::AttributeProto_AttributeType_STRING) { + auto_padding = attr->second.s(); + } + auto_pad_ = StringToAutoPadType(auto_padding); + + attr_read = false; + attr = attributes.find(attributes_prefix + "pads"); + if (attr != attributes.end()) { + ONNX_NAMESPACE::AttributeProto proto = attr->second; + if (GetIntsAttr(proto, pads_) == Status::OK()) + attr_read = true; + } + if (!attr_read) { + pads_.resize(kernel_shape_.size() * 2, 0); + } + + attr_read = false; + attr = attributes.find(attributes_prefix + "strides"); + if (attr != attributes.end()) { + ONNX_NAMESPACE::AttributeProto proto = attr->second; + if (GetIntsAttr(proto, strides_) == Status::OK()) + attr_read = true; + } + if (!attr_read || strides_.empty()) { + strides_.resize(kernel_shape_.size(), 1); + } + + attr = attributes.find(attributes_prefix + "count_include_pad"); + int64_t temp = 0; + if (attr != attributes.end()) { + ONNX_NAMESPACE::AttributeProto proto = attr->second; + GetIntAttr(proto, temp); + } + count_include_pad_ = (temp != 0); + + storage_order_ = 0; + for (size_t dim = 0; dim < kernel_shape_.size(); ++dim) { + ORT_ENFORCE(kernel_shape_[dim] > 0); + ORT_ENFORCE(pads_[dim] < kernel_shape_[dim] && pads_[dim + kernel_shape_.size()] < kernel_shape_[dim], + "Pad should be smaller than kernel."); + } + + ORT_ENFORCE(strides_.size() == kernel_shape_.size()); + } + } + + private: + size_t src_size_; + size_t dst_size_; + + std::shared_ptr src_mem_; + + std::unique_ptr fwd_desc_; + std::unique_ptr src_md_; + std::unique_ptr fwd_primitive_desc_; + std::unique_ptr pool_fwd_; + + std::shared_ptr src_mem_from_; + std::unique_ptr src_mem_to_; + + std::unique_ptr dst_mem_from_; + std::unique_ptr dst_mem_to_; + + private: + mkldnn::memory::format GetAVXFormat(const mkldnn::memory::dims& src_dims_mkl) { + bool is_2D = src_dims_mkl.size() == 4 ? true : false; + mkldnn::memory::format fmt = mkldnn::memory::format::any; + if (CPUIDInfo::GetCPUIDInfo().HasAVX512f()) { + fmt = is_2D ? mkldnn::memory::format::nChw16c : mkldnn::memory::format::nCdhw16c; + } else if (CPUIDInfo::GetCPUIDInfo().HasAVX2() && (src_dims_mkl[1] % 8 == 0)) { + fmt = is_2D ? mkldnn::memory::format::nChw8c : mkldnn::memory::format::ncdhw; + } else { + fmt = is_2D ? mkldnn::memory::format::nchw : mkldnn::memory::format::ncdhw; + } + return fmt; + } + + std::vector SetOutputSize(const TensorShape& input_shape, + int64_t output_channel, + std::vector* pads) const { + ORT_ENFORCE(input_shape.Size() > 0); + std::vector output_dims; + int64_t N = input_shape[0]; + InferOutputSize(input_shape.GetDims(), &output_dims, pads); + + output_dims.insert(output_dims.begin(), {N, output_channel}); + + return output_dims; + } + + inline void InferOutputSize(const std::vector& input_dims, + std::vector* output_dims, + std::vector* pads) const { + ORT_ENFORCE(input_dims.size() >= 2); + if (global_pooling_) { + output_dims->assign(input_dims.size() - 2, 1); + } else { + for (size_t dim = 0; dim < input_dims.size() - 2; ++dim) { + int64_t dim_size = 0; + ComputeSizeAndPad(static_cast(input_dims[dim + 2]), + strides_[dim], + kernel_shape_[dim], + &pads->at(dim), + &pads->at(input_dims.size() + dim - 2), + &dim_size); + output_dims->push_back(dim_size); + } + } + } + + inline void ComputeSizeAndPad(const int64_t in_size, + const int64_t stride, + const int64_t kernel, + int64_t* pad_head, + int64_t* pad_tail, + int64_t* out_size) const { + if (auto_pad_ != AutoPadType::NOTSET) { + switch (auto_pad_) { + case AutoPadType::VALID: + *pad_head = 0; + *pad_tail = 0; + *out_size = (in_size - kernel) / stride + 1; + break; + case AutoPadType::SAME_LOWER: { + int64_t legacy_target_size = (in_size + stride - 1) / stride; + int64_t pad_needed = (legacy_target_size - 1) * stride + kernel - in_size; + *pad_head = (pad_needed + 1) / 2; + *pad_tail = pad_needed - *pad_head; + *out_size = (in_size + pad_needed - kernel) / stride + 1; + break; + } + case AutoPadType::SAME_UPPER: { + int64_t legacy_target_size = (in_size + stride - 1) / stride; + int64_t pad_needed = (legacy_target_size - 1) * stride + kernel - in_size; + *pad_head = pad_needed / 2; + *pad_tail = pad_needed - *pad_head; + *out_size = (in_size + pad_needed - kernel) / stride + 1; + break; + } + default: { + ORT_THROW("Unsupported AutoPad Type."); + } + } + } else { + *out_size = static_cast( + static_cast(in_size + *pad_head + *pad_tail - kernel) / stride + 1); + } + } + + private: + IAllocatorUniquePtr src_reorder_buffer_; + IAllocatorUniquePtr dst_reorder_buffer_; + + private: + std::string op_name_; + bool global_pooling_{}; + bool count_include_pad_{}; + int64_t storage_order_{0}; // MaxPool_8 only. 0 is row major, and 1 is column major. Default is 0. + std::vector kernel_shape_; + std::vector pads_; + std::vector strides_; + AutoPadType auto_pad_; + + TensorShape x_shape_; +}; +} // namespace mkl_dnn +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/mkldnn/subgraph/mkldnn_sum.h b/onnxruntime/core/providers/mkldnn/subgraph/mkldnn_sum.h new file mode 100644 index 0000000000..a45af46acc --- /dev/null +++ b/onnxruntime/core/providers/mkldnn/subgraph/mkldnn_sum.h @@ -0,0 +1,172 @@ +// Copyright(C) 2019 Intel Corporation +// Licensed under the MIT License + +#pragma once +#include "core/framework/op_kernel.h" +#include "core/providers/mkldnn/mkldnn_fwd.h" +#include "core/providers/mkldnn/mkldnn_common.h" +#include "core/providers/mkldnn/subgraph/mkldnn_kernel.h" +#include "core/util/math.h" + +namespace onnxruntime { +namespace mkl_dnn { + +template +class MklDnnSum : public MklDnnKernel { + public: + explicit MklDnnSum(const MklDnnNode& node, + MKLDNNExecutionProvider* provider, + const NodeAttributes& attributes, + const std::string attributes_prefix = "") : MklDnnKernel(node, provider) { + ReadAttributes(attributes, attributes_prefix); + } + + Status CreatePrimitives(const OrtCustomOpApi* api, + OrtKernelContext* context, + mkldnn::engine& cpu_engine, + std::vector& net, + mkldnn::memory::format& source_format) override { + Ort::CustomOpApi ort{*api}; + int num_inputs = mklnode_ptr_->num_inputs; + int input_index = mklnode_ptr_->input_start_index < 0 ? 0 : mklnode_ptr_->input_start_index; + + std::vector coeff; + TensorShape x_shape; + if (mklnode_ptr_->parent_nodes.empty()) { + const OrtValue* input_tensor = ort.KernelContext_GetInput(context, input_index); + auto tensor_info = ort.GetTensorTypeAndShape(input_tensor); + auto tensor_shape = ort.GetTensorShape(tensor_info); + ort.ReleaseTensorTypeAndShapeInfo(tensor_info); + auto xshape = tensor_shape.data(); + auto xdim = tensor_shape.size(); + + ort_source_format_ = GetSourceFormat(static_cast(xdim)); + source_format = ort_source_format_; + src_format_ = ort_source_format_; + x_shape = TensorShape(xshape, xdim); + } else { + x_shape = parents_[0].get()->primitive_dst_shape_; + src_format_ = parents_[0].get()->primitive_dst_format_; + } + primitive_dst_shape_ = TensorShape(x_shape); + + mkldnn::memory::dims dst_dims_mkl( + primitive_dst_shape_.GetDims().begin(), primitive_dst_shape_.GetDims().end()); + + for (int i = 0; i < num_inputs; i++) { + TensorShape x_shape1; + + if (mklnode_ptr_->parent_nodes.empty()) { + const OrtValue* input_tensor = ort.KernelContext_GetInput(context, input_index); + auto tensor_info = ort.GetTensorTypeAndShape(input_tensor); + auto tensor_shape = ort.GetTensorShape(tensor_info); + ort.ReleaseTensorTypeAndShapeInfo(tensor_info); + auto xshape = tensor_shape.data(); + auto xdim = tensor_shape.size(); + mkldnn::memory::dims dims(xdim); + + ort_source_format_ = GetSourceFormat(static_cast(xdim)); + x_shape1 = TensorShape(xshape, xdim); + mkldnn::memory::dims src_dims_mkl( + x_shape1.GetDims().begin(), x_shape1.GetDims().end()); + + src_md_.reset(new mkldnn::memory::desc( + {src_dims_mkl}, MklDnnType(), src_format_)); + + auto mpd = mkldnn::memory::primitive_desc(*src_md_, cpu_engine); + auto src_memory = mkldnn::memory(mpd, nullptr); + srcs_pd_.push_back(mpd); + srcs_memory_.push_back(src_memory); + coeff.push_back(1.0); + } else { + src_md_.reset( + new mkldnn::memory::desc(parents_[i].get()->primitive_dst_mem_.get()->get_primitive_desc().desc())); + auto mpd = mkldnn::memory::primitive_desc(*src_md_, cpu_engine); + auto src_memory = *parents_[i].get()->primitive_dst_mem_; //mkldnn::memory(mpd); + srcs_pd_.push_back(mpd); + srcs_memory_.push_back(src_memory); + coeff.push_back(1.0); + ort_source_format_ = source_format; + } + } + + primitive_dst_md_.reset(new mkldnn::memory::desc( + {dst_dims_mkl}, MklDnnType(), mkldnn::memory::format::any)); + sum_pd_.reset(new mkldnn::sum::primitive_desc( + *primitive_dst_md_, coeff, srcs_pd_)); + primitive_dst_format_ = static_cast(sum_pd_->dst_primitive_desc().desc().data.format); + + if (mklnode_ptr_->output_index >= 0) { + // last node of sub-graph. need to allocate memory for output_tensor + if (primitive_dst_format_ != ort_source_format_) { + // reorder neded. Use primitive output as input to reorder and + // allocate buffer for reorder output, final output of this subgraph + primitive_dst_mem_.reset(new mkldnn::memory(sum_pd_->dst_primitive_desc())); + } else { + // Last node but re-order not needed. Allocate buffer to output of this node + primitive_dst_mem_.reset(new mkldnn::memory(sum_pd_->dst_primitive_desc(), nullptr)); + } + } else { + // Intermediate node. Use mkldnn kernel internal memory for output and + // use this as input to next node. + primitive_dst_mem_.reset(new mkldnn::memory(sum_pd_->dst_primitive_desc())); + } + primitive_dst_format_ = static_cast(sum_pd_->dst_primitive_desc().desc().data.format); + + std::vector inputs; + for (int i = 0; i < num_inputs; i++) { + inputs.push_back(srcs_memory_[i]); + } + auto c = mkldnn::sum(*sum_pd_, inputs, *primitive_dst_mem_); + net.push_back(c); + + if (mklnode_ptr_->output_index >= 0) { + // one of the end nodes. Allocate output buffer memory and + // reorder is necessary + mkldnn::memory::data_type t = MklDnnType(); + InitDstReorderOutput(cpu_engine, t, net); + } + return Status::OK(); + } + + Status Bind(const OrtCustomOpApi* api, OrtKernelContext* context) override { + Ort::CustomOpApi ort{*api}; + + int num_inputs = mklnode_ptr_->num_inputs; + int input_index = mklnode_ptr_->input_start_index < 0 ? 0 : mklnode_ptr_->input_start_index; + + if (mklnode_ptr_->parent_nodes.empty()) { + for (int i = 0; i < num_inputs; i++) { + const OrtValue* input_tensor = ort.KernelContext_GetInput(context, input_index + i); + const T* src_data = const_cast(ort.GetTensorData(input_tensor)); + srcs_memory_[i].set_data_handle(static_cast(const_cast(src_data))); + } + } + + if (mklnode_ptr_->output_index >= 0) { + // Last node. Allocate output buffer memory and reorder if needed + const auto& y_dims = primitive_dst_shape_.GetDims(); + // Allocate memory for output bufffer + OrtValue* output = ort.KernelContext_GetOutput(context, mklnode_ptr_->output_index, &y_dims[0], static_cast(primitive_dst_shape_.GetDims().size())); + T* dst_data = ort.GetTensorMutableData(output); + + if (primitive_dst_format_ != ort_source_format_) { + reorder_dst_mem_to_->set_data_handle(dst_data); + } else { + primitive_dst_mem_->set_data_handle(dst_data); + } + } + return Status::OK(); + } + + private: + std::unique_ptr src_md_; + std::vector srcs_memory_; + + std::vector srcs_pd_; + std::unique_ptr src_mpd_; + std::unique_ptr dst_pd_; + std::unique_ptr sum_pd_; +}; +} // namespace mkl_dnn +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/mkldnn/subgraph/subgraph.h b/onnxruntime/core/providers/mkldnn/subgraph/subgraph.h new file mode 100644 index 0000000000..6e3f967dab --- /dev/null +++ b/onnxruntime/core/providers/mkldnn/subgraph/subgraph.h @@ -0,0 +1,73 @@ +// Copyright(C) 2019 Intel Corporation +// Licensed under the MIT License + +#pragma once + +#include +#include +#include +#include "core/framework/op_node_proto_helper.h" +#include "core/graph/graph.h" + +namespace onnxruntime { +namespace mkl_dnn { + +struct MklDnnNode { + std::string name; + int node_index = -1; + int input_start_index = -1; // start index in inputs() + int num_inputs = 0; // and how many inputs + int output_index = -1; // index in output() + std::string weight_name; + std::string output_name; + std::vector parent_nodes; // index to parents in vector mklnodes + + std::string ToString() const { + std::string key; + key.reserve(128); + key.append(name); + key.append("-"); + key.append(std::to_string(input_start_index)); + key.append("-"); + key.append(std::to_string(num_inputs)); + key.append("-"); + key.append(std::to_string(output_index)); + key.append("-"); + key.append(output_name); + key.append("-"); + for (auto& out : parent_nodes) + key.append(std::to_string(out) + ","); + key.append(";"); + return key; + } +}; + +struct Subgraph { + struct SubgraphVariables { + std::vector inputs; + std::vector outputs; + std::vector outputs_as_input_other_node; + std::vector subgraph_node_indexes; + int subgraph_index = 0; + + SubgraphVariables() { + subgraph_index = 0; + } + void Reset() { + subgraph_node_indexes.clear(); + inputs.clear(); + outputs.clear(); + outputs_as_input_other_node.clear(); + } + }; + + Subgraph(const std::string& name) { + graph_name = name; + } + + std::string graph_name; + std::string subgraph_id; + std::vector mkldnn_nodes; +}; +} // namespace mkl_dnn +} // namespace onnxruntime \ No newline at end of file diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index cab2006497..6a23c292a2 100755 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -579,6 +579,39 @@ def run_onnx_tests(build_dir, configs, onnx_test_data_dir, provider, enable_para run_subprocess([exe,'-x'] + cmd, cwd=cwd) +# mkldnn temporary function for running onnx tests and model tests separately. +def mkldnn_run_onnx_tests(build_dir, configs, onnx_test_data_dir): + for config in configs: + cwd = get_config_build_dir(build_dir, config) + if is_windows(): + exe = os.path.join(cwd, config, 'onnx_test_runner') + model_dir = os.path.join(cwd, "models") + else: + exe = os.path.join(cwd, 'onnx_test_runner') + model_dir = os.path.join(build_dir, "models") + cmd_base = ['-e', 'mkldnn', '-c', '1', '-j', '1'] + if os.path.exists(onnx_test_data_dir): + onnxdata_cmd = cmd_base + [onnx_test_data_dir] + # /data/onnx + run_subprocess([exe] + onnxdata_cmd, cwd=cwd) + run_subprocess([exe,'-x'] + onnxdata_cmd, cwd=cwd) + + # models/opset7, models/opset8, models/opset9 + if config != 'Debug' and os.path.exists(model_dir): + opset7_model_dir = os.path.join(model_dir, 'opset7') + opset7_cmd = cmd_base + [opset7_model_dir] + opset8_model_dir = os.path.join(model_dir, 'opset8') + opset8_cmd = cmd_base + [opset8_model_dir] + opset9_model_dir = os.path.join(model_dir, 'opset9') + opset9_cmd = cmd_base + [opset9_model_dir] + run_subprocess([exe] + opset7_cmd, cwd=cwd) + run_subprocess([exe, '-x'] + opset7_cmd, cwd=cwd) + run_subprocess([exe] + opset8_cmd, cwd=cwd) + run_subprocess([exe, '-x'] + opset8_cmd, cwd=cwd) + run_subprocess([exe] + opset9_cmd, cwd=cwd) + run_subprocess([exe, '-x'] + opset9_cmd, cwd=cwd) + + def split_server_binary_and_symbol(build_dir, configs): if is_windows(): # TODO: Windows support @@ -839,7 +872,7 @@ def main(): run_onnx_tests(build_dir, configs, onnx_test_data_dir, None, True, 0) if args.use_mkldnn: - run_onnx_tests(build_dir, configs, onnx_test_data_dir, 'mkldnn', True, 1) + mkldnn_run_onnx_tests(build_dir, configs, onnx_test_data_dir) if args.build_server: split_server_binary_and_symbol(build_dir, configs)