From d342147255d53348b371593fc5d202343a6f2aa9 Mon Sep 17 00:00:00 2001 From: KeDengMS Date: Wed, 9 Jan 2019 22:33:35 -0800 Subject: [PATCH] Add test for truncated sequence inference (#99) * Add test for truncated sequence inference with scan model * Address CR * Update to Scan opset 9 --- .../test/framework/inference_session_test.cc | 156 ++++++++++++++++++ onnxruntime/test/testdata/CNTK/gen.py | 156 ++++++++++++++++-- onnxruntime/test/testdata/scan_1.pb | Bin 0 -> 24197 bytes 3 files changed, 294 insertions(+), 18 deletions(-) create mode 100644 onnxruntime/test/testdata/scan_1.pb diff --git a/onnxruntime/test/framework/inference_session_test.cc b/onnxruntime/test/framework/inference_session_test.cc index 490829b99f..ff7e68406e 100644 --- a/onnxruntime/test/framework/inference_session_test.cc +++ b/onnxruntime/test/framework/inference_session_test.cc @@ -4,11 +4,14 @@ #include "core/session/inference_session.h" #include +#include #include #include #include #include +#include +#include "core/platform/env.h" #include "core/common/logging/logging.h" #include "core/common/profiler.h" #include "core/framework/execution_provider.h" @@ -1111,5 +1114,158 @@ TEST(ExecutionProviderTest, FunctionInlineTest) { VerifyOutputs(fetches, expected_dims_mul_m, expected_values_mul_m); } +TEST(InferenceSessionTests, TestTruncatedSequence) { + // model/data generated by /onnxruntime/test/testdata/CNTK/gen.py GenScan() + static const std::string LSTM_MODEL_URI = "testdata/scan_1.pb"; + // This model is a 4x forward LSTM. Parse it to find out mapping between init_state input/output + ONNX_NAMESPACE::ModelProto model_proto; + int model_fd; + auto status = Env::Default().FileOpenRd(LSTM_MODEL_URI, model_fd); + ASSERT_TRUE(status.IsOK()); + google::protobuf::io::FileInputStream f(model_fd); + f.SetCloseOnDelete(true); + ASSERT_TRUE(model_proto.ParseFromZeroCopyStream(&f)); + GraphProto& graph_proto = *model_proto.mutable_graph(); + + auto find_attr = [&](const NodeProto& node, const std::string& attr_name) -> const AttributeProto* { + for (int i = 0; i < node.attribute_size(); ++i) { + auto& attr = node.attribute(i); + if (attr.name() == attr_name) + return &attr; + } + return nullptr; + }; + + std::unordered_map init_state_map; + for (int i_node = 0; i_node < graph_proto.node_size(); ++i_node) { + auto& node = *graph_proto.mutable_node(i_node); + if (node.op_type() == "Scan") { + // only works in forward, and do not allow bidirection + auto attr_directions = find_attr(node, "scan_input_directions"); + if (attr_directions != nullptr) { + ASSERT_TRUE(attr_directions->ints_size() == 1); + + if (attr_directions->ints(0) == 1) + continue; // skip backward Scan + } + + // input 0 is optional sequence length, 1..N are for initial states + // and N+1..N+num_scan_inputs are actual inputs + // output 0..N-1 are for output states, and N.. are actual outputs + auto attr_num_scan_inputs = find_attr(node, "num_scan_inputs"); + ASSERT_TRUE(attr_num_scan_inputs != nullptr); + int num_scan_inputs = gsl::narrow_cast(attr_num_scan_inputs->i()); + ASSERT_TRUE(node.input_size() - num_scan_inputs < node.output_size()); + for (int i = 0; i < node.input_size() - num_scan_inputs; ++i) { + init_state_map.insert(std::make_pair(node.output(i), node.input(i))); + } + } + } + + // now run the truncated model + SessionOptions so; + InferenceSession session_object(so); + ASSERT_TRUE(session_object.Load(LSTM_MODEL_URI).IsOK()); + ASSERT_TRUE(session_object.Initialize().IsOK()); + + RunOptions run_options; + run_options.run_tag = "one session/one tag"; + + std::vector X_dims = {5, 1, 3}; + std::vector X = {0.5488135f, 0.71518934f, 0.60276335f, + 0.5448832f, 0.4236548f, 0.6458941f, + 0.4375872f, 0.891773f, 0.96366274f, + 0.3834415f, 0.79172504f, 0.5288949f, + 0.56804454f, 0.92559665f, 0.07103606f}; + + std::vector Y_dims = {5, 1, 2}; + std::vector Y_data = {-1.1730184e-04f, -3.1204990e-04f, + -2.9978977e-04f, -1.0602647e-03f, + -3.8115133e-04f, -2.0684483e-03f, + -2.5120965e-04f, -2.9920202e-03f, + 3.0980256e-05f, -3.5933927e-03f}; + + MLValue ml_value; + CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), X_dims, X, &ml_value); + + std::string input_name = "Input13165"; + NameMLValMap feeds = {{input_name, ml_value}}; + + // prepare outputs for whole sequence + std::string final_output_name = ""; + int final_output_index = -1; + for (int i = 0; i < graph_proto.output_size(); ++i) { + if (init_state_map.find(graph_proto.output(i).name()) == init_state_map.end()) { + ASSERT_TRUE(final_output_name.empty()); + final_output_name = graph_proto.output(i).name(); + final_output_index = i; + } + } + + std::vector output_names = {final_output_name}; + std::vector fetches; + + // Now run the full sequence + common::Status st = session_object.Run(run_options, feeds, output_names, &fetches); + if (!st.IsOK()) { + std::cout << "Run returned status: " << st.ErrorMessage() << std::endl; + } + ASSERT_TRUE(st.IsOK()); + ASSERT_EQ(1, fetches.size()); + auto& rtensor = fetches.front().Get(); + TensorShape expected_shape(Y_dims); + ASSERT_EQ(expected_shape, rtensor.Shape()); + for (int i = 0; i < Y_data.size(); ++i) + EXPECT_NEAR(Y_data[i], rtensor.template Data()[i], FLT_EPSILON); + + // run truncated sequence + output_names.clear(); + for (int i = 0; i < graph_proto.output_size(); ++i) { + output_names.push_back(graph_proto.output(i).name()); + } + fetches.clear(); + + std::vector truncated_lengths = {2, 2, 1}; // sums to non-truncated length + auto seq_stride = TensorShape(X_dims).SizeFromDimension(1); // sequence is the first dimension of input shape + int seq_start = 0; + for (auto truncated_len : truncated_lengths) { + std::vector truncated_input_dims = X_dims; + truncated_input_dims[0] = truncated_len; + MLValue truncated_ml_value; + std::vector truncated_input(X.begin() + seq_start * seq_stride, X.begin() + (seq_start + truncated_len) * seq_stride); + CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), truncated_input_dims, truncated_input, &truncated_ml_value); + NameMLValMap truncated_feeds = {{input_name, truncated_ml_value}}; + if (seq_start > 0) { + // continue from truncated sequence + ASSERT_TRUE(fetches.size() == output_names.size()); + for (int i_output = 0; i_output < output_names.size(); ++i_output) { + auto iter = init_state_map.find(output_names[i_output]); + if (iter != init_state_map.end()) + truncated_feeds.insert(std::make_pair(iter->second, fetches[i_output])); + } + } + std::vector truncated_fetches; + st = session_object.Run(run_options, truncated_feeds, output_names, &truncated_fetches); + if (!st.IsOK()) { + std::cout << "Run returned status: " << st.ErrorMessage() << std::endl; + } + ASSERT_TRUE(st.IsOK()); + + // check truncated output + auto& truncated_rtensor = truncated_fetches[final_output_index].Get(); + std::vector truncated_output_dims = Y_dims; + truncated_output_dims[0] = truncated_len; + TensorShape truncated_shape(truncated_output_dims); + ASSERT_EQ(truncated_shape, truncated_rtensor.Shape()); + auto seq_output_stride = truncated_shape.SizeFromDimension(1); + for (int i = 0; i < truncated_shape.Size(); ++i) + EXPECT_NEAR(Y_data[i + seq_start * seq_output_stride], truncated_rtensor.template Data()[i], FLT_EPSILON); + + // prepare for next truncated input + fetches = truncated_fetches; + seq_start += truncated_len; + } +} + } // namespace test } // namespace onnxruntime diff --git a/onnxruntime/test/testdata/CNTK/gen.py b/onnxruntime/test/testdata/CNTK/gen.py index 10f257bee6..adac5e3285 100644 --- a/onnxruntime/test/testdata/CNTK/gen.py +++ b/onnxruntime/test/testdata/CNTK/gen.py @@ -9,40 +9,61 @@ import os model_file = 'model.onnx' data_dir = 'test_data_set_0' -def SaveTensorProto(file_path, variable, data): +def SaveTensorProto(file_path, variable, data, name): tp = onnx.TensorProto() - tp.name = variable.uid + tp.name = name if name else variable.uid + # ONNX input shape always has sequence axis as the first dimension, if sequence axis exists for i in range(len(variable.dynamic_axes)): - tp.dims.append(1) # pad 1 for the each dynamic axis - for d in variable.shape: - tp.dims.append(d) + tp.dims.append(data.shape[len(variable.dynamic_axes) - 1 - i]) + for (i,d) in enumerate(variable.shape): + tp.dims.append(d) if d > 0 else tp.dims.append(data.shape[len(data.shape)-len(variable.shape)+i]) tp.data_type = onnx.TensorProto.FLOAT tp.raw_data = data.tobytes() with open(file_path, 'wb') as f: f.write(tp.SerializeToString()) - -def SaveData(test_data_dir, prefix, variables, data_list): + +def SaveData(test_data_dir, prefix, variables, data_list, name_replacements=None): if isinstance(data_list, np.ndarray): data_list = [data_list] for (i, d), v in zip(enumerate(data_list), variables): - SaveTensorProto(os.path.join(test_data_dir, '{0}_{1}.pb'.format(prefix, i)), v, d) + SaveTensorProto(os.path.join(test_data_dir, '{0}_{1}.pb'.format(prefix, i)), v, d, name_replacements[v.uid] if name_replacements else None) -def Save(dir, func, inputs, outputs): +def Save(dir, func, feed, outputs): if not os.path.exists(dir): os.makedirs(dir) - func.save(os.path.join(dir,model_file), C.ModelFormat.ONNX) - + onnx_file = os.path.join(dir,model_file) + func.save(onnx_file, C.ModelFormat.ONNX) + + # onnx model may have different name for RNN initial states as inputs + cntk_to_actual_names = {} + with open(onnx_file, 'rb') as ff: + sf = ff.read() + model = onnx.ModelProto() + model.ParseFromString(sf) + for actual_input in model.graph.input: + actual_input_name = actual_input.name + for cntk_input in func.arguments: + cntk_name = cntk_input.uid + if actual_input_name.startswith(cntk_name): + cntk_to_actual_names[cntk_name] = actual_input_name + + if type(feed) is not dict: + feed = {func.arguments[0]:feed} + + if type(outputs) is not dict: + outputs = {func.outputs[0]:outputs} + test_data_dir = os.path.join(dir, data_dir) if not os.path.exists(test_data_dir): os.makedirs(test_data_dir) - SaveData(test_data_dir, 'input', func.arguments, inputs) - SaveData(test_data_dir, 'output', func.outputs, outputs) - + SaveData(test_data_dir, 'input', func.arguments, [feed[var] for var in func.arguments], cntk_to_actual_names) + SaveData(test_data_dir, 'output', func.outputs, [outputs[var] for var in func.outputs]) + def GenSimple(): x = C.input_variable((1,3,)) # TODO: fix CNTK exporter bug with shape (3,) y = C.layers.Embedding(2)(x) + C.parameter((-1,)) - data_x = np.random.rand(*x.shape).astype(np.float32) + data_x = np.random.rand(1,*x.shape).astype(np.float32) data_y = y.eval(data_x) Save('test_simple', y, data_x, data_y) @@ -50,10 +71,10 @@ def GenSharedWeights(): x = C.input_variable((1,3,)) y = C.layers.Embedding(2)(x) y = y + y.parameters[0] - data_x = np.random.rand(*x.shape).astype(np.float32) + data_x = np.random.rand(1,*x.shape).astype(np.float32) data_y = y.eval(data_x) Save('test_shared_weights', y, data_x, data_y) - + def GenSimpleMNIST(): input_dim = 784 num_output_classes = 10 @@ -69,12 +90,111 @@ def GenSimpleMNIST(): model = C.softmax(z) - data_feature = np.random.rand(*feature.shape).astype(np.float32) + data_feature = np.random.rand(1,*feature.shape).astype(np.float32) data_output = model.eval(data_feature) Save('test_simpleMNIST', model, data_feature, data_output) +def GenMatMul_1k(): + feature = C.input_variable((1024, 1024,), np.float32) + model = C.times(feature, C.parameter((1024,1024), init=C.glorot_uniform())) + + data_feature = np.random.rand(1,*feature.shape).astype(np.float32) + data_output = model.eval(data_feature) + Save('test_MatMul_1k', model, data_feature, data_output) + +def LSTM(cell_dim, use_scan=True): + # we now create an LSTM_cell function and call it with the input and placeholders + LSTM_cell = C.layers.LSTM(cell_dim) + + @C.Function + def func(dh, dc, input): + LSTM_func = LSTM_cell(dh, dc, input) + if use_scan: + LSTM_func_root = C.as_composite(LSTM_func.outputs[0].owner.block_root) + args = LSTM_func_root.arguments + LSTM_func = LSTM_func_root.clone(C.CloneMethod.share, {args[0]:input, args[1]:dh, args[2]:dc}) + return LSTM_func + + return func + +def GenLSTMx4(use_scan): + feature = C.sequence.input_variable((128,), np.float32) + lstm1 = C.layers.Recurrence(LSTM(512, use_scan))(feature) + lstm2_fw = C.layers.Recurrence(LSTM(512, use_scan))(lstm1) + lstm2_bw = C.layers.Recurrence(LSTM(512, use_scan), go_backwards=True)(lstm1) + lstm2 = C.splice(lstm2_fw, lstm2_bw, axis=0) + lstm3_fw = C.layers.Recurrence(LSTM(512, use_scan))(lstm2) + lstm3_bw = C.layers.Recurrence(LSTM(512, use_scan), go_backwards=True)(lstm2) + lstm3 = C.splice(lstm3_fw, lstm3_bw, axis=0) + lstm4 = C.layers.Recurrence(LSTM(512, use_scan))(lstm3) + model = lstm4 + + postfix = 'Scan' if use_scan else 'LSTM' + + data_feature = np.random.rand(1,64,128).astype(np.float32) + data_output = np.asarray(model.eval(data_feature)) + Save('test_LSTMx4_' + postfix, model, data_feature, data_output) + +def GenScan(): + np.random.seed(0) + feature = C.sequence.input_variable((3,), np.float32) + model = C.layers.For(range(4), lambda : C.layers.Recurrence(LSTM(2, use_scan=True)))(feature) + + data_feature = np.random.rand(1,5,3).astype(np.float32) + data_output = np.asarray(model.eval(data_feature)) + + # print values for test as ground truth + print("Scan input\n", data_feature, "\nScan output\n", data_output) + + Save('test_Scan', model, data_feature, data_output) + +def GenSimpleScan(): + feature = C.sequence.input_variable((128,), np.float32) + param = C.parameter(shape=(1,), dtype=np.float32) + scan = C.layers.Recurrence(lambda h, x: x + h + param)(feature) + model = C.sequence.reduce_sum(scan) + data_feature = np.random.rand(1,64,128).astype(np.float32) + data_output = np.asarray(model.eval(data_feature), dtype=np.float32) + Save('test_SimpleScan', model, data_feature, data_output) + +def GenLCBLSTM(): + nc_len = 16 + nr_len = 16 + in_dim = 128 + cell_dim = 256 + batch_size = 1 + input = C.sequence.input_variable((in_dim,)) + init_h = C.input_variable((cell_dim,)) + init_c = C.input_variable((cell_dim,)) + + fwd_cell = LSTM(cell_dim, use_scan=True) + fwd_hc = C.layers.RecurrenceFrom(fwd_cell, go_backwards=False, return_full_state=True)(init_h, init_c, input) + + fwd_h_nc = C.sequence.slice(fwd_hc[0], -nr_len-1, -nr_len) + fwd_c_nc = C.sequence.slice(fwd_hc[1], -nr_len-1, -nr_len) + + bwd_cell = LSTM(cell_dim, use_scan=True) + bwd = C.layers.Recurrence(bwd_cell, go_backwards=True)(input) + + nr = C.splice(fwd_hc[0], bwd) + # workaround CNTK bug in slice output name by + 0 + model = C.combine([C.sequence.reduce_sum(nr), fwd_h_nc + 0, fwd_c_nc + 0]) + + input_data = np.random.rand(batch_size, nc_len+nr_len, in_dim).astype(np.float32) + init_h_data = np.random.rand(batch_size, cell_dim).astype(np.float32) + init_c_data = np.random.rand(batch_size, cell_dim).astype(np.float32) + feed = {input:input_data, init_h:init_h_data, init_c:init_c_data} + data_output = model.eval(feed) + Save('test_LCBLSTM', model, feed, data_output) + if __name__=='__main__': np.random.seed(0) GenSimple() GenSharedWeights() GenSimpleMNIST() + GenMatMul_1k() + GenLSTMx4(use_scan=True) + GenLSTMx4(use_scan=False) + GenSimpleScan() + GenLCBLSTM() + GenScan() diff --git a/onnxruntime/test/testdata/scan_1.pb b/onnxruntime/test/testdata/scan_1.pb new file mode 100644 index 0000000000000000000000000000000000000000..a1ae04f243195e6311fe8d67b7509596049ce5e5 GIT binary patch literal 24197 zcmcg!eT-ejb@$ud;N^ybc_zdxlx#P#3(MlX``*3xz31%S=Pa>olG>OUY(ztrUGLVc zytfWVPO6#)D%7S4RT>fMHd0UpNKHXiDc!efQZ-*f5wa3KT3V$DsZgXAL`nZZ zrRTn#ne&@7^X}bMnp(2-cFxC~nKS3i`Q4d&`+LiMH|)FR69c`~?a9!_$=U6P=a-IM zQ`&LjCqgOhWoX2K+nq8wt91i_8-Cr^ZTH*q-q8S_7z@-jtsPDkx$@$@@t_b?3PE6lZ zef)~xVDOFEnbb=R(sq)ceut>CTvL4m_NolBb1L);%igta6rBGG}@7 zfw|ek*(!`1EFN&<(0e++hu<6YO`e`v+*vwT+Bp0R!3DIG7nT-x_GEt&ZDxLYac3X> z(Xl=ad}_<(TPEl4qOOt(cXgp@8|uq`cI1mpg$2f@Mg^vBKvOfjJ6#eY)sFFGj=E zdWmx3%=E;sEu1($F|ba45rnsS-EP$Yu+e{9I{S~(l<2)3bFEZ%y&G_lJ%!euOe@!UIu=47 zo9;a3MU~6OMTx`au@N?Dhn?oflp5N^io@m3Ii&J6aAR_plgb`~0CjYF=dvo;db24v zlt_0Y#J1%K4qp)C_wFTT5Alq0GqAnsK3@-0bPpkqbNxnR|Wd79g30APUg(}{jPF43N zg1D#_Cy2$uCV)6ejKbVdw$e|?o3Rb*B5<>)HD?c?w}4OGEf&9P?v;Ytp-ulXo09(gS@>a&prluWI}h42ljP99@tl57YrEt zDoxA24#*2l5GEj_tGXm)kLjw8+PJDSAP+fNB}@~mF0%sP1JXGsAGl9L#aV8OMXSyH{<#jyaJji$|+6GNLxgZZky&w-6 z)&lZ!_yrVWkcV!!Y695$4afuLHG@1f$}TR@Q)1&qp)Ngj19>u6VPzr8K?~$bIm~fV zE?!%qAjBk+gYnDCQ?i0H99BQ*vu4nKKp-`^pbv!|Zd12g3EUp5Rgh!uwb{G{Ib>6( zZw4w>ewA8@jd$;^g*yUMtR|#34`_f**Rhd718&1osSElbwL3apZcO%4xA73Be@CZx zE+y#e;A{oJta{V#6G0z%CI|Fk`6*26DARpnLF(yKx!KwhY&PE-!z3>{2heAweWCRK z5A?;Aqz0fbYU0VAAjV-Ls{$*OF~O4}jtnG_5++E{7ehrLL0?Rj!GY`~uN&wS)@ByC zWn>KwY7FSBRdPTdNMxX|!k`b*CWETTJ~?whAI?~=eE|Aul^oE=_H{rX*atz(_E8az zCygD@S7C$(GP=r3U|%)}4XZo`eZa{oVVaD}2vD0U9FwTVg(HJL;AGHO;fH8^R#Ryd zRDOk^57g=+H4b6HXwVl{yhW$o6l0y2`gn~1gx zT+oN|4Ei|KUC@V5vqb8QDSa5A;DSDol<6Hkt7A6~dFknNKy4hm4Elgm#}i!?;kBtj zADwi1PRc1E7*-SdNdymJ4CssE0%sP2K2Ym8Gl6pjC!UCCJJSRDKwifa&V!7nq+lB~ zzVv`T5cPvTU|19A>(1X5^r73kZT$xH0dp597C0w+I6)TZ!>VIIf<7F{`cC#luCgi% zQMp;5Psmw0PRco;&tE7AF@fZ8{Ic>CtX3T+wRQo0ID_j3eIP7Nv+-^*XtDoeF>iQNn+DE>Fb_;6W-iKUS?qD>~?h)lGS5Ur^=uM`p)ZU0i8_L{QqpkPimPkSejy(Ok#}sV~<$fP8U7LO!o`)u40;ti#$! zKjKJY3-Ij>N>hx@5|+^QU6Me#tg%@Fw=Nm(0Vk+rcN&#-cE2telR6w5GTZ}BQ}ab| zf^RSsH5Sx-MQ{(*>hd(g+qtSdr8aMQ>NOP?vo22~TE+PXOhspWq=8Fec?&RV!0b9$ zqfvOqhw{2S#d(kc8^LWH8L&iSsA0i9k#gu>L|&W%+>2=h_BHE*)lvsE$}ad>&-fxx z>mY;(FNixXs285`p}a2%IdCtkLy!u%M>O_<1NTr;!aek?j@<|@yy)1aHV@oGP90AP z+Nt3l+2@0MzIaONi~&vPQ)59q6~aAG>o_y1b5eBSUOg^^dmyjl3Fkq^6K!%PxMe(L z8(<#X15rQR1BNxhz3%)X2+bLv<#vR75!`*!a1WW+4)?$)yR5+NSi(IJ>e5rUGrqi< zLKK2F+{?*vQqF~Y-XcMW2@Ds(_+>RIS-}|&YZthOGq`TJ2g0&HfO{D8_E;^rhq>2g z^OfS5O`X00qHvLycd^sjdu8XOO~D1LnviTR+`~=RvC+UiYa5nI9k_?7-O=fC!#3`4 z8@Pu7>u`DJG8gV)y=jlff_v%4WpBf#4r}VR5`C|+{-tB-E0tzz*KM0yhmUPd$4iG; z*@KTi+kWvcpH4sDw<+B_zj5VbKWVgQzPqFK>o4pc+p_SD^o5~km;YktU#G9Xb8kC1 zec8%?p8II)BX9rm%4P4yD-S;Xhwbswp5g1(I0$-8tunk<597%nkG5Xg{QK?S{l??z z8~eAmwte%C_W0PYl>=}5QG4i_d)haA;OdphtDa~-y?gu0*KfX}_43{?wEyj)^XcB# zwzWR{z;o$1y(tIk{_cU%^n*WrcWmEHKS+0vJ(wQa_B-jBxo6vNJ#bHZd21^@d*+q& z%_CnMeXicq+O_Yh*7BFeS5AKVx|PW@A6OaeyP`Fo9Urb|C?OA}HE{pft7q!1`{v#m z`;*^DTA$o~QTzG7Nm_S*bEQ2x^nCjEr_L|G_3B;ii=O)P^qI$BY46|j)pqdP*Dp5~ z?r)!cxJi8hSUY-i{ORT>jNZ+I^q< zR{P68I+C7!^u_jHeQYtk936Y=#Y;Ow zt@6x$@7k#K#Ob@!Z@>Hx?Sb*j(rd;R+TY#&)pW;^Eh{hm>tpT5{&`z#=-uUyw7z_A z+H=RXt=_L+-a7cFd(vHdFI~BL_kGKm*Q&$U_m}#6D31Dd7>(z>{kkspB*|r$H;2OJ zy4a&Imt8!frU%IDVW)@4`Rt-1O7d-&g;z8D!;heDUxp;58#pIo4Y zyEL;9#??lBWN=Egi$wyjJvfU5Xg;Nlk-BUl5cYtl(dXket z9e7!>e3vpvlSY9pngnixdst%n_!wV_=>VuxO!03;)aikPtm2EjiZ)@@6>wOj&BXfy zidi@S&Y|Qy;DrQXb%Pd?wz{DglE~nQ42j!*As&fJ_iC;n6sFS#{dBs8mBjsiV42_Ci zl_Lq+Vtz^+_(>=j!;O|E{NjjKiEWx0H(v6=9@Q>9PKB`XxU(FP=$MR_>OMQ!kE$_k z7yZ=Tni7s@w5}JDy}s$gHqvd>r1jY0ONFarHMRMf@jiqjg+wL3y7cj3hF*Hu*0nER z7g94QR3v*Y=s~!g3u^LriAV8H-$>Nrt1hAM#Spjot%zK=(btM->C(4zG-cJva7)d2 z8`^AUxS?Q}p-nTJB^%KxHPwm}W?MxHtuA^#VPs4VhgW;37cK^npv3fe;Z6%r0yH}8 z$fb$)lBA$# zT#aD`(5N-xGNhzW&$S#yD6hqI#25o2o%uvIrYLo0mD-dZJ2k}cYKzPfNYfT}@E_j4 z)GoocZasNAka|I#A}N=_XT*4;#71wY^lU(ON==GYP8+=R<<1`vwYb`h;7v-i^zo$^ z#jaKjyoYTZ32lPe^F#vPBG=4jLDd>{D&(h34C_@4Z=>_df<~Vrs-Eeor=sY%Vd%$B zfR~C)cE_~I6^O{EOfda(8iu&B#eks2R^bwC1$t)jB`d`&oF2m~ZI08CsKr(0g%{_H z(_?6K*pW*UMOLiRl)L6BX4eCN3)n5%VIs=mVUUamDw0j@rVZ1BWDkM)+~}|)mnMoq z*2ru*$eOm=jddO*#JXlz`vSbnr_)CaZQ2kuD0rQa&4osX9k~?IbPW9|(rTE~!h}tA=3$9DvD`IGtnOOpDw=)s(;6SWU&?y3Ny7FX_tcUaiH(CDxumnMot8o2>P<|$@p(4MjSS>cOSjttPc7zJP4 za_0w#T3pQ!eEy3^4vh{wa%rNNpV$MSVhx~F0*5b;>g0i6Bh{L)zFOeY1y*9Ot&u>}jt~B|85ZUcSEv3b;{r zL;C=WDx%2{*{6*mVPt`&Pxfn*w;k;wkcSZ=3VM5fXG z;{%YvZXe_-Y_r)K64&!~?V_F9VF(`iRBnl0HQ=jN5A zJsUctUpk%6)h^iSaYsJ8b2_~)_NHt5rP+6q!*USx)8zHhg!SKk3Ej818s3+Ee;Gx4 zI((b6oBVu5$TaS88atg-T%CATYE_)N%3fHPT070I1s%OrS1(@Z>h0LtQGm^z?!=8l z`Utq$n!$w5_b(+Y?2kpP0Ah<+0cI7j{$IT}XMcASk4y^<;5x=(ZOD7tcw@NSJyLs) sI+FOU^z}*N>xk==gqy8-lH+~BhD{faUmOJVrTG){^QX5TpWD>)zY*HGGynhq literal 0 HcmV?d00001