From 1c8d874412f2991f53cc3f6b16d3f25270f13097 Mon Sep 17 00:00:00 2001 From: Sherlock Date: Wed, 24 Mar 2021 20:42:42 -0700 Subject: [PATCH 001/129] Promote BiasDropout from orttraining to onnxruntime (#7116) * Promote BiasDropout from orttraining to onnxruntime Co-authored-by: Sherlock Huang --- .../contrib_ops/cuda/cuda_contrib_kernels.cc | 2 + .../contrib_ops/cuda/math/bias_dropout.cc | 85 +-------- .../contrib_ops/cuda/math/bias_dropout.h | 31 ++-- .../cuda/math/bias_dropout_impl.cu | 58 +----- .../contrib_ops/rocm/rocm_contrib_kernels.cc | 2 + .../core/graph/contrib_ops/contrib_defs.cc | 53 +++++- onnxruntime/core/providers/cuda/nn/dropout.cc | 63 ++++++- onnxruntime/core/providers/cuda/nn/dropout.h | 75 ++------ .../test/contrib_ops/bias_dropout_op_test.cc | 171 ++++++++++++++++++ .../test/optimizer/graph_transform_test.cc | 26 +++ .../core/graph/training_op_defs.cc | 48 ----- .../test/optimizer/graph_transform_test.cc | 26 --- .../training_ops/cpu/nn/dropout_op_test.cc | 141 --------------- .../cuda/cuda_training_kernels.cc | 4 +- .../training_ops/cuda/nn/dropout_grad.cc | 69 +++++++ .../training_ops/cuda/nn/dropout_grad.h | 24 +++ .../training_ops/cuda/nn/dropout_grad_impl.cu | 81 +++++++++ .../{dropout_impl.h => dropout_grad_impl.h} | 14 -- .../rocm/rocm_training_kernels.cc | 2 - 19 files changed, 523 insertions(+), 452 deletions(-) rename orttraining/orttraining/training_ops/cuda/nn/dropout.cc => onnxruntime/contrib_ops/cuda/math/bias_dropout.cc (54%) rename orttraining/orttraining/training_ops/cuda/nn/dropout.h => onnxruntime/contrib_ops/cuda/math/bias_dropout.h (58%) rename orttraining/orttraining/training_ops/cuda/nn/dropout_impl.cu => onnxruntime/contrib_ops/cuda/math/bias_dropout_impl.cu (70%) create mode 100644 onnxruntime/test/contrib_ops/bias_dropout_op_test.cc create mode 100644 orttraining/orttraining/training_ops/cuda/nn/dropout_grad.cc create mode 100644 orttraining/orttraining/training_ops/cuda/nn/dropout_grad.h create mode 100644 orttraining/orttraining/training_ops/cuda/nn/dropout_grad_impl.cu rename orttraining/orttraining/training_ops/cuda/nn/{dropout_impl.h => dropout_grad_impl.h} (58%) diff --git a/onnxruntime/contrib_ops/cuda/cuda_contrib_kernels.cc b/onnxruntime/contrib_ops/cuda/cuda_contrib_kernels.cc index 1f88aac942..26fd54bc8c 100644 --- a/onnxruntime/contrib_ops/cuda/cuda_contrib_kernels.cc +++ b/onnxruntime/contrib_ops/cuda/cuda_contrib_kernels.cc @@ -34,6 +34,7 @@ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1 class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, float, ComplexMulConj); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, MLFloat16, ComplexMulConj); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, BiasSoftmax); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, BiasDropout); // These ops were experimental ops in onnx domain which have been removed now. We add them here as // contrib ops to maintain backward compatibility @@ -161,6 +162,7 @@ Status RegisterCudaContribKernels(KernelRegistry& kernel_registry) { BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, + BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, diff --git a/orttraining/orttraining/training_ops/cuda/nn/dropout.cc b/onnxruntime/contrib_ops/cuda/math/bias_dropout.cc similarity index 54% rename from orttraining/orttraining/training_ops/cuda/nn/dropout.cc rename to onnxruntime/contrib_ops/cuda/math/bias_dropout.cc index 8d970bdd90..a13974ad99 100644 --- a/orttraining/orttraining/training_ops/cuda/nn/dropout.cc +++ b/onnxruntime/contrib_ops/cuda/math/bias_dropout.cc @@ -1,89 +1,15 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#include "core/framework/random_seed.h" -#include "orttraining/training_ops/cuda/nn/dropout.h" -#include "core/providers/cuda/cuda_common.h" +#include "contrib_ops/cuda/math/bias_dropout.h" +#include "core/providers/cuda/nn/dropout.h" + #include "core/providers/common.h" namespace onnxruntime { +namespace contrib { namespace cuda { -#if defined(CUDA_VERSION) && CUDA_VERSION >= 11000 -#define ALL_IEEE_FLOAT_TENSOR_TYPES {DataTypeImpl::GetTensorType(), \ - DataTypeImpl::GetTensorType(), \ - DataTypeImpl::GetTensorType(), \ - DataTypeImpl::GetTensorType()} -#define ALL_IEEE_FLOAT_DATA_TYPES float, MLFloat16, double, BFloat16 -#else -#define ALL_IEEE_FLOAT_TENSOR_TYPES DataTypeImpl::AllIEEEFloatTensorTypes() -#define ALL_IEEE_FLOAT_DATA_TYPES float, MLFloat16, double -#endif - -#define REGISTER_GRADIENT_KERNEL(OpName) \ - ONNX_OPERATOR_KERNEL_EX( \ - OpName, \ - kMSDomain, \ - 1, \ - kCudaExecutionProvider, \ - KernelDefBuilder() \ - .TypeConstraint("T", ALL_IEEE_FLOAT_TENSOR_TYPES) \ - .TypeConstraint("T1", ALL_IEEE_FLOAT_TENSOR_TYPES) \ - .TypeConstraint("T2", DataTypeImpl::GetTensorType()) \ - .InputMemoryType(2), \ - DropoutGrad); - -REGISTER_GRADIENT_KERNEL(DropoutGrad) - -template -struct DropoutGradComputeImpl { - void operator()(cudaStream_t stream, - const int64_t N, - const Tensor& dY, - const bool* mask_data, - const float ratio_data, - Tensor& dX) const { - typedef typename ToCudaType::MappedType CudaT; - - const CudaT* dY_data = reinterpret_cast(dY.template Data()); - CudaT* dX_data = reinterpret_cast(dX.template MutableData()); - DropoutGradientKernelImpl(stream, N, dY_data, mask_data, ratio_data, dX_data); - } -}; - -// REVIEW(codemzs): Common out this structure because it is also used in Dropout forward op. -template -struct GetRatioDataImpl { - void operator()(const Tensor* ratio, float& ratio_data) const { - ratio_data = static_cast(*(ratio->template Data())); - ORT_ENFORCE(ratio_data >= 0.0f && ratio_data < 1.0f, "ratio_data is outside range [0, 1)"); - } -}; - -Status DropoutGrad::ComputeInternal(OpKernelContext* context) const { - auto dY = context->Input(0); - const TensorShape& shape = dY->Shape(); - const int64_t N = shape.Size(); - - auto mask = context->Input(1); - ORT_ENFORCE(mask->Shape().Size() == N); - const bool* mask_data = mask->template Data(); - - //Get the ratio_data - float ratio_data = default_ratio_; - auto ratio = context->Input(2); - if (ratio) { - utils::MLTypeCallDispatcher t_disp(ratio->GetElementType()); - t_disp.Invoke(ratio, ratio_data); - } - - auto dX = context->Output(0, shape); - - utils::MLTypeCallDispatcher t_disp(dY->GetElementType()); - t_disp.Invoke(Stream(), N, *dY, mask_data, ratio_data, *dX); - - return Status::OK(); -} ONNX_OPERATOR_KERNEL_EX( BiasDropout, @@ -192,4 +118,5 @@ Status BiasDropout::ComputeInternal(OpKernelContext* context) const { } } // namespace cuda -} // namespace onnxruntime +} // namespace contrib +} // namespace onnxruntime \ No newline at end of file diff --git a/orttraining/orttraining/training_ops/cuda/nn/dropout.h b/onnxruntime/contrib_ops/cuda/math/bias_dropout.h similarity index 58% rename from orttraining/orttraining/training_ops/cuda/nn/dropout.h rename to onnxruntime/contrib_ops/cuda/math/bias_dropout.h index 58cf470c50..a6e2166997 100644 --- a/orttraining/orttraining/training_ops/cuda/nn/dropout.h +++ b/onnxruntime/contrib_ops/cuda/math/bias_dropout.h @@ -3,22 +3,30 @@ #pragma once +#include "gsl/gsl" #include "core/providers/cuda/cuda_kernel.h" -#include "orttraining/training_ops/cuda/nn/dropout_impl.h" +#include "core/providers/cuda/cuda_common.h" +#include "core/framework/random_generator.h" + +using namespace onnxruntime::cuda; namespace onnxruntime { +namespace contrib { namespace cuda { -class DropoutGrad final : public CudaKernel { - public: - DropoutGrad(const OpKernelInfo& info) : CudaKernel(info) { - } - - Status ComputeInternal(OpKernelContext* context) const override; - - private: - static constexpr float default_ratio_ = 0.5f; -}; +template +void BiasDropoutKernelImpl( + const cudaDeviceProp& prop, + cudaStream_t stream, + const int64_t N, + const fast_divmod fdm_dim, + const float ratio, + PhiloxGenerator& generator, + const T* X_data, + const T* bias_data, + const T* residual_data, + T* Y_data, + bool* mask_data); class BiasDropout final : public CudaKernel { public: @@ -37,4 +45,5 @@ class BiasDropout final : public CudaKernel { }; } // namespace cuda +} // namespace contrib } // namespace onnxruntime diff --git a/orttraining/orttraining/training_ops/cuda/nn/dropout_impl.cu b/onnxruntime/contrib_ops/cuda/math/bias_dropout_impl.cu similarity index 70% rename from orttraining/orttraining/training_ops/cuda/nn/dropout_impl.cu rename to onnxruntime/contrib_ops/cuda/math/bias_dropout_impl.cu index eed291e1d7..fb6cc7c7b1 100644 --- a/orttraining/orttraining/training_ops/cuda/nn/dropout_impl.cu +++ b/onnxruntime/contrib_ops/cuda/math/bias_dropout_impl.cu @@ -17,66 +17,15 @@ /* Modifications Copyright (c) Microsoft. */ #include "core/providers/cuda/cu_inc/common.cuh" -#include "orttraining/training_ops/cuda/nn/dropout_impl.h" +#include "contrib_ops/cuda/math/bias_dropout.h" + #include #include namespace onnxruntime { +namespace contrib { namespace cuda { -template -__global__ void DropoutGradientKernel( - const int64_t N, - const T* dY_data, - const bool* mask_data, - const float scale, - T* dX_data) { - CUDA_LONG id = NumElementsPerThread * NumThreadsPerBlock * blockIdx.x + threadIdx.x; -#pragma unroll - for (int i = 0; i < NumElementsPerThread; i++) { - if (id < N) { - dX_data[id] = T(float(dY_data[id]) * mask_data[id] * scale); - id += NumThreadsPerBlock; - } - } -} - -template -void DropoutGradientKernelImpl( - cudaStream_t stream, - const int64_t N, - const T* dY_data, - const bool* mask_data, - const float ratio, - T* dX_data) { - if (ratio == 0.0f) { - if (dY_data != dX_data) { - CUDA_CALL_THROW(cudaMemcpyAsync(dX_data, dY_data, N * sizeof(T), cudaMemcpyDeviceToDevice, stream)); - } - } else { - const float scale = 1.f / (1.f - ratio); - const int blocksPerGrid = static_cast(CeilDiv(N, GridDim::maxThreadsPerBlock * GridDim::maxElementsPerThread)); - DropoutGradientKernel - <<>>(N, dY_data, mask_data, scale, dX_data); - } -} - -#define SPECIALIZED_DROPOUT_GRAD_IMPL(T) \ - template void DropoutGradientKernelImpl( \ - cudaStream_t stream, \ - const int64_t N, \ - const T* dY_data, \ - const bool* mask_data, \ - const float scale, \ - T* dX_data); - -SPECIALIZED_DROPOUT_GRAD_IMPL(float) -SPECIALIZED_DROPOUT_GRAD_IMPL(double) -SPECIALIZED_DROPOUT_GRAD_IMPL(half) -#if CUDA_VERSION >= 11000 && (__CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__)) -SPECIALIZED_DROPOUT_GRAD_IMPL(nv_bfloat16) -#endif - constexpr int UNROLL = 4; template @@ -180,4 +129,5 @@ SPECIALIZED_BIAS_DROPOUT_IMPL(nv_bfloat16) #endif } // namespace cuda +} // namespace contrib { } // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/rocm/rocm_contrib_kernels.cc b/onnxruntime/contrib_ops/rocm/rocm_contrib_kernels.cc index 71fa8b3278..a15d9eea03 100644 --- a/onnxruntime/contrib_ops/rocm/rocm_contrib_kernels.cc +++ b/onnxruntime/contrib_ops/rocm/rocm_contrib_kernels.cc @@ -31,6 +31,7 @@ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1 class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float, ComplexMulConj); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, MLFloat16, ComplexMulConj); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, BiasSoftmax); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, BiasDropout); // These ops were experimental ops in onnx domain which have been removed now. We add them here as // contrib ops to maintain backward compatibility @@ -137,6 +138,7 @@ Status RegisterRocmContribKernels(KernelRegistry& kernel_registry) { BuildKernelCreateInfo, // BuildKernelCreateInfo, BuildKernelCreateInfo, + BuildKernelCreateInfo, // BuildKernelCreateInfo, // BuildKernelCreateInfo, diff --git a/onnxruntime/core/graph/contrib_ops/contrib_defs.cc b/onnxruntime/core/graph/contrib_ops/contrib_defs.cc index de257885ed..3248dd688e 100644 --- a/onnxruntime/core/graph/contrib_ops/contrib_defs.cc +++ b/onnxruntime/core/graph/contrib_ops/contrib_defs.cc @@ -275,7 +275,6 @@ void FusedMatMulShapeInference(ONNX_NAMESPACE::InferenceContext& ctx) { updateOutputShape(ctx, 0, resultShape); } - void AttentionTypeAndShapeInference(ONNX_NAMESPACE::InferenceContext& ctx, int past_input_index) { // Type inference ONNX_NAMESPACE::propagateElemTypeFromInputToOutput(ctx, 2, 0); @@ -441,7 +440,7 @@ and present state are optional. Present state could appear in output even when p AttentionTypeAndShapeInference(ctx, past_input_index); }); - static const char* Longformer_Attention_doc = R"DOC( + static const char* Longformer_Attention_doc = R"DOC( Longformer Self Attention with a local context and a global context. Tokens attend locally: Each token attends to its W previous tokens and W succeding tokens with W being the window length. A selected few tokens attend globally to all other tokens. @@ -2377,6 +2376,56 @@ It's an extension of Gelu. It takes the sum of input A and bias input B as the i "Constrain input and output types to float tensors.") .TypeAndShapeInferenceFunction(ONNX_NAMESPACE::propagateShapeAndTypeFromFirstInput); + ONNX_CONTRIB_OPERATOR_SCHEMA(BiasDropout) + .SetDomain(kMSDomain) + .SinceVersion(1) + .SetDoc( + "output, dropout_mask = Dropout(data + bias, ratio) + residual, " + "Intended to specialize the dropout pattern commonly found in transformer models.") + .Attr("seed", "(Optional) Seed to the random generator, if not specified we will auto generate one.", AttributeProto::INT, OPTIONAL_VALUE) + .AllowUncheckedAttributes() + .Input(0, "data", "The input data as Tensor.", "T") + .Input(1, "bias", "The bias input, a vector with the same shape as last dim of data", "T") + .Input(2, "residual", "The residual input, must have the same shape as data", "T", OpSchema::Optional) + .Input(3, "ratio", + "The ratio of random dropout, with value in [0, 1). If this input was not set, " + "or if it was set to 0, the output would be a simple copy of the input. " + "If it's non-zero, output will be a random dropout of input, which is typically " + "the case during training.", + "T1", + OpSchema::Optional) + .Input(4, "training_mode", + "If set to true then it indicates dropout is being used for " + "training. It is an optional value hence unless specified explicitly, it is false. " + "If it is false, ratio is ignored and the operation mimics inference mode where nothing " + "will be dropped from the input data and if mask is requested as output it will contain " + "all ones.", + "T2", + OpSchema::Optional) + .Output(0, "output", "The output.", "T") + .Output(1, "mask", "The output mask of dropout.", "T2", OpSchema::Optional) + .TypeConstraint( + "T", + {"tensor(float16)", "tensor(float)", "tensor(double)", "tensor(bfloat16)"}, + "Constrain input and output types to float tensors.") + .TypeConstraint( + "T1", + {"tensor(float16)", "tensor(float)", "tensor(double)", "tensor(bfloat16)"}, + "Constrain input 'ratio' types to float tensors.") + .TypeConstraint( + "T2", + {"tensor(bool)"}, + "Constrain output 'mask' types to boolean tensors.") + .TypeAndShapeInferenceFunction([](ONNX_NAMESPACE::InferenceContext& ctx) { + propagateShapeAndTypeFromFirstInput(ctx); + if (ctx.getNumOutputs() == 2) { + updateOutputElemType(ctx, 1, ONNX_NAMESPACE::TensorProto::BOOL); + if (hasNInputShapes(ctx, 1)) { + propagateShapeFromInputToOutput(ctx, 0, 1); + } + } + }); + // Register the NCHWc schemas if supported by the platform. if (MlasNchwcGetBlockSize() > 1) { RegisterNchwcSchemas(); diff --git a/onnxruntime/core/providers/cuda/nn/dropout.cc b/onnxruntime/core/providers/cuda/nn/dropout.cc index 0293df46a9..42ec6597ac 100644 --- a/onnxruntime/core/providers/cuda/nn/dropout.cc +++ b/onnxruntime/core/providers/cuda/nn/dropout.cc @@ -6,15 +6,6 @@ namespace onnxruntime { namespace cuda { -#if defined(CUDA_VERSION) && CUDA_VERSION >= 11000 -#define ALL_IEEE_FLOAT_TENSOR_TYPES {DataTypeImpl::GetTensorType(), \ - DataTypeImpl::GetTensorType(), \ - DataTypeImpl::GetTensorType(), \ - DataTypeImpl::GetTensorType()} -#else -#define ALL_IEEE_FLOAT_TENSOR_TYPES DataTypeImpl::AllIEEEFloatTensorTypes() -#endif - ONNX_OPERATOR_VERSIONED_KERNEL_EX( Dropout, kOnnxDomain, @@ -41,5 +32,59 @@ ONNX_OPERATOR_KERNEL_EX( .InputMemoryType(2), Dropout); +Status Dropout::ComputeInternal(OpKernelContext* context) const { + //Get X_data + const Tensor* X = context->Input(0); + if (X == nullptr) return Status(common::ONNXRUNTIME, common::FAIL, "X Input is not available."); + const TensorShape& shape = X->Shape(); + const int64_t N = shape.Size(); + + //Get Y_data + auto Y = context->Output(0, shape); + + //Get mask_data + auto mask = context->Output(1, shape); + ORT_ENFORCE(!mask || mask->Shape().Size() == N); + + //Get the ratio_data + float ratio_data = default_ratio_; + auto ratio = context->Input(1); + if (ratio) { + utils::MLTypeCallDispatcher t_disp(ratio->GetElementType()); + t_disp.Invoke(ratio, ratio_data); + } + + const Tensor* training_mode = context->Input(2); + //Check for inference mode. + if ((0 == ratio_data) ||(training_mode == nullptr || *(training_mode->Data()) == false)) { + const void* X_data = X->DataRaw(); + void* Y_data = Y->MutableDataRaw(); + if (Y_data != X_data) { + CUDA_RETURN_IF_ERROR(cudaMemcpyAsync(Y_data, X_data, X->SizeInBytes(), cudaMemcpyDeviceToDevice, Stream())); + } + + // If mask is requested, return all 1s. + if (mask != nullptr) { + CUDA_RETURN_IF_ERROR(cudaMemsetAsync(mask->MutableData(), true, N * sizeof(bool), Stream())); + } + + return Status::OK(); + } + + IAllocatorUniquePtr temp_mask_buffer{}; // buffer to use if mask is not provided + bool* const mask_data = [this, N, mask, &temp_mask_buffer]() { + if (mask) return mask->MutableData(); + temp_mask_buffer = GetScratchBuffer(N); + return temp_mask_buffer.get(); + }(); + + PhiloxGenerator& generator = generator_ ? *generator_ : PhiloxGenerator::Default(); + + utils::MLTypeCallDispatcher t_disp(X->GetElementType()); + t_disp.Invoke(GetDeviceProp(), Stream(), N, ratio_data, generator, *X, *Y, mask_data); + + return Status::OK(); +} + } // namespace cuda } // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/nn/dropout.h b/onnxruntime/core/providers/cuda/nn/dropout.h index bea0bfbb42..2a14f08cfa 100644 --- a/onnxruntime/core/providers/cuda/nn/dropout.h +++ b/onnxruntime/core/providers/cuda/nn/dropout.h @@ -6,11 +6,22 @@ #include "core/providers/cuda/cuda_kernel.h" #include "core/providers/cuda/nn/dropout_impl.h" #include "core/providers/common.h" -#include "core/framework/random_seed.h" namespace onnxruntime { namespace cuda { +#if defined(CUDA_VERSION) && CUDA_VERSION >= 11000 +#define ALL_IEEE_FLOAT_TENSOR_TYPES \ + { DataTypeImpl::GetTensorType(), \ + DataTypeImpl::GetTensorType(), \ + DataTypeImpl::GetTensorType(), \ + DataTypeImpl::GetTensorType() } +#define ALL_IEEE_FLOAT_DATA_TYPES float, MLFloat16, double, BFloat16 +#else +#define ALL_IEEE_FLOAT_TENSOR_TYPES DataTypeImpl::AllIEEEFloatTensorTypes() +#define ALL_IEEE_FLOAT_DATA_TYPES float, MLFloat16, double +#endif + template struct GetRatioDataImpl { void operator()(const Tensor* ratio, float& ratio_data) const { @@ -54,67 +65,5 @@ class Dropout final : public CudaKernel { static constexpr float default_ratio_ = 0.5f; }; -Status Dropout::ComputeInternal(OpKernelContext* context) const { - //Get X_data - const Tensor* X = context->Input(0); - if (X == nullptr) return Status(common::ONNXRUNTIME, common::FAIL, "X Input is not available."); - const TensorShape& shape = X->Shape(); - const int64_t N = shape.Size(); - - //Get Y_data - auto Y = context->Output(0, shape); - - //Get mask_data - auto mask = context->Output(1, shape); - ORT_ENFORCE(!mask || mask->Shape().Size() == N); - - //Get the ratio_data - float ratio_data = default_ratio_; - auto ratio = context->Input(1); - if (ratio) { - utils::MLTypeCallDispatcher t_disp(ratio->GetElementType()); - t_disp.Invoke(ratio, ratio_data); - } - - const Tensor* training_mode = context->Input(2); - //Check for inference mode. - if ((0 == ratio_data) ||(training_mode == nullptr || *(training_mode->Data()) == false)) { - const void* X_data = X->DataRaw(); - void* Y_data = Y->MutableDataRaw(); - if (Y_data != X_data) { - CUDA_RETURN_IF_ERROR(cudaMemcpyAsync(Y_data, X_data, X->SizeInBytes(), cudaMemcpyDeviceToDevice, Stream())); - } - - // If mask is requested, return all 1s. - if (mask != nullptr) { - CUDA_RETURN_IF_ERROR(cudaMemsetAsync(mask->MutableData(), true, N * sizeof(bool), Stream())); - } - - return Status::OK(); - } - - IAllocatorUniquePtr temp_mask_buffer{}; // buffer to use if mask is not provided - bool* const mask_data = [this, N, mask, &temp_mask_buffer]() { - if (mask) return mask->MutableData(); - temp_mask_buffer = GetScratchBuffer(N); - return temp_mask_buffer.get(); - }(); - - PhiloxGenerator& generator = generator_ ? *generator_ : PhiloxGenerator::Default(); - - using SupportedTypes = onnxruntime::TypeList< -#if defined(CUDA_VERSION) && CUDA_VERSION >= 11000 - float, MLFloat16, double, BFloat16 -#else - float, MLFloat16, double -#endif - >; - - utils::MLTypeCallDispatcherFromTypeList t_disp(X->GetElementType()); - t_disp.Invoke(GetDeviceProp(), Stream(), N, ratio_data, generator, *X, *Y, mask_data); - - return Status::OK(); -} - } // namespace cuda } // namespace onnxruntime diff --git a/onnxruntime/test/contrib_ops/bias_dropout_op_test.cc b/onnxruntime/test/contrib_ops/bias_dropout_op_test.cc new file mode 100644 index 0000000000..f28e313be1 --- /dev/null +++ b/onnxruntime/test/contrib_ops/bias_dropout_op_test.cc @@ -0,0 +1,171 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#ifdef _MSC_VER +#pragma warning(disable : 4389) +#endif + + +#include +#include +#include +#include + +#include "gtest/gtest.h" + +#include "test/common/tensor_op_test_utils.h" +#include "test/providers/provider_test_utils.h" +#include "test/util/include/default_providers.h" + +namespace onnxruntime { +namespace contrib { +namespace test { + +using namespace onnxruntime::test; + +enum TrainingMode { TrainingFalse, TrainingTrue, NoTraining }; + +// BiasDropout kernel is only implemented for CUDA +#ifdef USE_CUDA +namespace { +void RunBiasDropoutTest(const bool use_mask, const std::vector& input_shape, float ratio = -1.0f, + TrainingMode training_mode = TrainingTrue, bool use_float16_ratio = false, bool has_residual = true) { + OpTester t{"BiasDropout", 1, kMSDomain}; + const int64_t seed = 42; + t.AddAttribute("seed", seed); + + const auto input_size = std::accumulate( + input_shape.begin(), input_shape.end(), static_cast(1), std::multiplies<>{}); + const std::vector input = ValueRange(input_size, 1.0f, 1.0f); + t.AddInput("data", input_shape, input); + + std::vector bias_shape{input_shape.back()}; + const auto bias_size = input_shape.back(); + const std::vector bias = ValueRange(bias_size, 2.0f, 1.0f); + t.AddInput("bias", bias_shape, bias); + + float residual_value = 0.0f; + if (has_residual) { + residual_value = 1.0f; + const auto residual_size = input_size; + const std::vector residual(residual_size, residual_value); + t.AddInput("residual", input_shape, residual); + } else { + t.AddMissingOptionalInput(); + } + + if (ratio == -1.0f) { + if (use_float16_ratio) { + t.AddMissingOptionalInput(); + } else { + t.AddMissingOptionalInput(); + } + // set ratio to default value + ratio = 0.5f; + } else { + if (use_float16_ratio) { + t.AddInput("ratio", {}, {MLFloat16(math::floatToHalf(ratio))}); + } else { + t.AddInput("ratio", {}, {ratio}); + } + } + + if (training_mode != NoTraining) { + if (training_mode == TrainingTrue) { + t.AddInput("training_mode", {}, {true}); + } else { + t.AddInput("training_mode", {}, {false}); + } + } + + t.AddOutput("output", input_shape, input); // we'll do our own output verification + + std::unique_ptr mask_buffer{}; + if (use_mask) { + mask_buffer = onnxruntime::make_unique(input_size); + t.AddOutput("mask", input_shape, mask_buffer.get(), input_size); + } else { + t.AddMissingOptionalOutput(); + } + + auto output_verifier = [&](const std::vector& fetches, const std::string& provider_type) { + ASSERT_GE(fetches.size(), 1); + const auto& output_tensor = FetchTensor(fetches[0]); + auto output_span = output_tensor.DataAsSpan(); + + const auto num_dropped_values = std::count(output_span.begin(), output_span.end(), residual_value); + + if (ratio == 1.0f) { + ASSERT_EQ(num_dropped_values, static_cast(output_span.size())) << "provider: " << provider_type; + } else { + ASSERT_NEAR(static_cast(num_dropped_values) / static_cast(output_span.size()), training_mode == TrainingTrue ? ratio : 0.0f, 0.1f) + << "provider: " << provider_type; + + for (decltype(output_span.size()) i = 0; i < output_span.size(); ++i) { + if (output_span[i] == residual_value) continue; + const auto expected_value = (bias[i % bias_size] + i + 1.0f) / (1 - (training_mode == TrainingTrue ? ratio : 0.0f)) + residual_value; + ASSERT_NEAR(output_span[i], expected_value, 0.01f) + << "unexpected output value at index " << i << ", provider: " << provider_type; + } + } + + if (use_mask) { + ASSERT_GE(fetches.size(), 2); + const auto& mask_tensor = FetchTensor(fetches[1]); + auto mask_span = mask_tensor.DataAsSpan(); + ASSERT_EQ(mask_span.size(), output_span.size()) << "provider: " << provider_type; + + const auto num_mask_zeros = std::count(mask_span.begin(), mask_span.end(), false); + ASSERT_EQ(num_dropped_values, num_mask_zeros) << "provider: " << provider_type; + + for (decltype(mask_span.size()) i = 0; i < mask_span.size(); ++i) { + ASSERT_TRUE( + (mask_span[i] && output_span[i] != residual_value) || (!mask_span[i] && output_span[i] == residual_value)) + << "output and mask mismatch at index " << i << ", output[i]: " << output_span[i] + << ", mask[i]: " << mask_span[i] << ", provider: " << provider_type; + } + } + }; + + t.SetCustomOutputVerifier(output_verifier); + + t.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); +} +} // namespace + +TEST(BiasDropoutTest, Basic) { + RunBiasDropoutTest(false, {10, 10, 10}, 0.75f); +} +TEST(BiasDropoutTest, BasicTrainingFalse) { + RunBiasDropoutTest(false, {10, 10, 10}, 0.75f, TrainingFalse); +} +TEST(BiasDropoutTest, BasicNoTraining) { + RunBiasDropoutTest(false, {10, 10, 10}, 0.75f, NoTraining); +} + +TEST(BiasDropoutTest, BasicWithoutResidual) { + RunBiasDropoutTest(false, {10, 10, 10}, 0.75f, TrainingTrue, false, false); +} + +TEST(BiasDropoutTest, Mask) { + RunBiasDropoutTest(true, {3, 5, 768}, 0.25f); +} +TEST(BiasDropoutTest, MaskTrainingFalse) { + RunBiasDropoutTest(true, {3, 5, 768}, 0.25f, TrainingFalse); +} +TEST(BiasDropoutTest, MaskNoTraining) { + RunBiasDropoutTest(true, {3, 5, 768}, 0.25f, NoTraining); +} + +TEST(BiasDropoutTest, RatioLimit) { + RunBiasDropoutTest(true, {4, 8, 1024}, 0.0f, TrainingFalse); +} + +TEST(BiasDropoutTest, EmptyRatio) { + RunBiasDropoutTest(true, {2, 7, 1024}); +} +#endif + +} // namespace test +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/test/optimizer/graph_transform_test.cc b/onnxruntime/test/optimizer/graph_transform_test.cc index 049616570b..19dacf1b1c 100644 --- a/onnxruntime/test/optimizer/graph_transform_test.cc +++ b/onnxruntime/test/optimizer/graph_transform_test.cc @@ -17,6 +17,7 @@ #include "core/optimizer/attention_fusion.h" #include "core/optimizer/bias_gelu_fusion.h" #include "core/optimizer/bias_softmax_fusion.h" +#include "core/optimizer/bias_dropout_fusion.h" #include "core/optimizer/computation_reduction.h" #include "core/optimizer/cast_elimination.h" #include "core/optimizer/common_subexpression_elimination.h" @@ -2690,6 +2691,31 @@ TEST_F(GraphTransformationTests, BiasSoftmaxFusionTest_NoLeadingOnes) { tester.TestFusionOccurs(0); } +static void TestBiasDropoutFusion(const PathString& file_path, const logging::Logger& logger, const int add_count = 0) { + std::shared_ptr p_model; + ASSERT_TRUE(Model::Load(file_path, p_model, nullptr, logger).IsOK()); + Graph& graph = p_model->MainGraph(); + + onnxruntime::GraphTransformerManager graph_transformation_mgr{5}; + graph_transformation_mgr.Register(onnxruntime::make_unique(), TransformerLevel::Level2); + auto ret = graph_transformation_mgr.ApplyTransformers(graph, TransformerLevel::Level2, logger); + ASSERT_STATUS_OK(ret); + + std::map op_to_count = CountOpsInGraph(graph); + + ASSERT_EQ(op_to_count["Add"], add_count); + ASSERT_EQ(op_to_count["Dropout"], 0); + ASSERT_EQ(op_to_count["com.microsoft.BiasDropout"], 1); +} + +TEST_F(GraphTransformationTests, BiasDropoutFusionTest) { + TestBiasDropoutFusion(MODEL_FOLDER "fusion/bias_dropout_fusion1.onnx", *logger_); + TestBiasDropoutFusion(MODEL_FOLDER "fusion/bias_dropout_fusion2.onnx", *logger_); + TestBiasDropoutFusion(MODEL_FOLDER "fusion/bias_dropout_residual_fusion1.onnx", *logger_); + TestBiasDropoutFusion(MODEL_FOLDER "fusion/bias_dropout_residual_fusion2.onnx", *logger_); + TestBiasDropoutFusion(MODEL_FOLDER "fusion/bias_dropout_residual_fusion_mismatch.onnx", *logger_, 1); +} + TEST_F(GraphTransformationTests, LayerNormFusionTest) { auto model_uri = MODEL_FOLDER "fusion/layer_norm.onnx"; std::shared_ptr p_model; diff --git a/orttraining/orttraining/core/graph/training_op_defs.cc b/orttraining/orttraining/core/graph/training_op_defs.cc index 2aae8b1524..cf78dc7e50 100644 --- a/orttraining/orttraining/core/graph/training_op_defs.cc +++ b/orttraining/orttraining/core/graph/training_op_defs.cc @@ -1120,54 +1120,6 @@ Example 4: }) .SetDoc(R"DOC(SoftmaxCrossEntropyLossGrad)DOC"); - ONNX_CONTRIB_OPERATOR_SCHEMA(BiasDropout) - .SetDomain(kMSDomain) - .SinceVersion(1) - .SetDoc("BiasDropout") - .Attr("seed", "(Optional) Seed to the random generator, if not specified we will auto generate one.", AttributeProto::INT, OPTIONAL_VALUE) - .AllowUncheckedAttributes() - .Input(0, "data", "The input data as Tensor.", "T") - .Input(1, "bias", "The bias input, a vector with the same shape as last dim of data", "T") - .Input(2, "residual", "The residual input, must have the same shape as data", "T", OpSchema::Optional) - .Input(3, "ratio", - "The ratio of random dropout, with value in [0, 1). If this input was not set, " - "or if it was set to 0, the output would be a simple copy of the input. " - "If it's non-zero, output will be a random dropout of input, which is typically " - "the case during training.", - "T1", - OpSchema::Optional) - .Input(4, "training_mode", - "If set to true then it indicates dropout is being used for " - "training. It is an optional value hence unless specified explicitly, it is false. " - "If it is false, ratio is ignored and the operation mimics inference mode where nothing " - "will be dropped from the input data and if mask is requested as output it will contain " - "all ones.", - "T2", - OpSchema::Optional) - .Output(0, "output", "The output.", "T") - .Output(1, "mask", "The output mask of dropout.", "T2", OpSchema::Optional) - .TypeConstraint( - "T", - {"tensor(float16)", "tensor(float)", "tensor(double)", "tensor(bfloat16)"}, - "Constrain input and output types to float tensors.") - .TypeConstraint( - "T1", - {"tensor(float16)", "tensor(float)", "tensor(double)", "tensor(bfloat16)"}, - "Constrain input 'ratio' types to float tensors.") - .TypeConstraint( - "T2", - {"tensor(bool)"}, - "Constrain output 'mask' types to boolean tensors.") - .TypeAndShapeInferenceFunction([](ONNX_NAMESPACE::InferenceContext& ctx) { - propagateShapeAndTypeFromFirstInput(ctx); - if (ctx.getNumOutputs() == 2) { - updateOutputElemType(ctx, 1, ONNX_NAMESPACE::TensorProto::BOOL); - if (hasNInputShapes(ctx, 1)) { - propagateShapeFromInputToOutput(ctx, 0, 1); - } - } - }); - ONNX_CONTRIB_OPERATOR_SCHEMA(ReduceSumTraining) .SetDomain(kMSDomain) .SinceVersion(1) diff --git a/orttraining/orttraining/test/optimizer/graph_transform_test.cc b/orttraining/orttraining/test/optimizer/graph_transform_test.cc index bd9d99272c..b33408bd73 100644 --- a/orttraining/orttraining/test/optimizer/graph_transform_test.cc +++ b/orttraining/orttraining/test/optimizer/graph_transform_test.cc @@ -13,7 +13,6 @@ #include "core/optimizer/bias_gelu_fusion.h" #include "core/optimizer/gelu_fusion.h" #include "core/optimizer/dropout_elimination.h" -#include "core/optimizer/bias_dropout_fusion.h" #include "orttraining/core/optimizer/gist_encode_decode.h" #include "orttraining/core/optimizer/nonzero_shape_setter.h" #include "orttraining/core/optimizer/megatron_transformer.h" @@ -74,31 +73,6 @@ TEST_F(GraphTransformationTests, GistEncodeDecode) { ASSERT_TRUE(op_to_count["GistBinarizeEncoder"] == op_to_count["GistBinarizeEncoder"]); } -static void TestBiasDropoutFusion(const PathString& file_path, const logging::Logger& logger, const int add_count = 0) { - std::shared_ptr p_model; - ASSERT_TRUE(Model::Load(file_path, p_model, nullptr, logger).IsOK()); - Graph& graph = p_model->MainGraph(); - - onnxruntime::GraphTransformerManager graph_transformation_mgr{5}; - graph_transformation_mgr.Register(onnxruntime::make_unique(), TransformerLevel::Level2); - auto ret = graph_transformation_mgr.ApplyTransformers(graph, TransformerLevel::Level2, logger); - ASSERT_STATUS_OK(ret); - - std::map op_to_count = CountOpsInGraph(graph); - - ASSERT_EQ(op_to_count["Add"], add_count); - ASSERT_EQ(op_to_count["Dropout"], 0); - ASSERT_EQ(op_to_count["com.microsoft.BiasDropout"], 1); -} - -TEST_F(GraphTransformationTests, BiasDropoutFusionTest) { - TestBiasDropoutFusion(MODEL_FOLDER "fusion/bias_dropout_fusion1.onnx", *logger_); - TestBiasDropoutFusion(MODEL_FOLDER "fusion/bias_dropout_fusion2.onnx", *logger_); - TestBiasDropoutFusion(MODEL_FOLDER "fusion/bias_dropout_residual_fusion1.onnx", *logger_); - TestBiasDropoutFusion(MODEL_FOLDER "fusion/bias_dropout_residual_fusion2.onnx", *logger_); - TestBiasDropoutFusion(MODEL_FOLDER "fusion/bias_dropout_residual_fusion_mismatch.onnx", *logger_, 1); -} - Node* GetNodeByName(Graph& graph, std::string node_name) { GraphViewer graph_viewer(graph); const auto& node_topology_list = graph_viewer.GetNodesInTopologicalOrder(); diff --git a/orttraining/orttraining/test/training_ops/cpu/nn/dropout_op_test.cc b/orttraining/orttraining/test/training_ops/cpu/nn/dropout_op_test.cc index ded07dc7c5..71ca986628 100644 --- a/orttraining/orttraining/test/training_ops/cpu/nn/dropout_op_test.cc +++ b/orttraining/orttraining/test/training_ops/cpu/nn/dropout_op_test.cc @@ -151,147 +151,6 @@ TEST(DropoutTest, EmptyRatio) { RunDropoutTest(true, {1000}); } -// BiasDropout kernel is only implemented for CUDA -#ifdef USE_CUDA -namespace { -void RunBiasDropoutTest(const bool use_mask, const std::vector& input_shape, float ratio = -1.0f, - TrainingMode training_mode = TrainingTrue, bool use_float16_ratio = false, bool has_residual = true) { - OpTester t{"BiasDropout", 1, kMSDomain}; - const int64_t seed = 42; - t.AddAttribute("seed", seed); - - const auto input_size = std::accumulate( - input_shape.begin(), input_shape.end(), static_cast(1), std::multiplies<>{}); - const std::vector input = ValueRange(input_size, 1.0f, 1.0f); - t.AddInput("data", input_shape, input); - - std::vector bias_shape{input_shape.back()}; - const auto bias_size = input_shape.back(); - const std::vector bias = ValueRange(bias_size, 2.0f, 1.0f); - t.AddInput("bias", bias_shape, bias); - - float residual_value = 0.0f; - if (has_residual) { - residual_value = 1.0f; - const auto residual_size = input_size; - const std::vector residual(residual_size, residual_value); - t.AddInput("residual", input_shape, residual); - } else { - t.AddMissingOptionalInput(); - } - - if (ratio == -1.0f) { - if (use_float16_ratio) { - t.AddMissingOptionalInput(); - } else { - t.AddMissingOptionalInput(); - } - // set ratio to default value - ratio = 0.5f; - } else { - if (use_float16_ratio) { - t.AddInput("ratio", {}, {MLFloat16(math::floatToHalf(ratio))}); - } else { - t.AddInput("ratio", {}, {ratio}); - } - } - - if (training_mode != NoTraining) { - if (training_mode == TrainingTrue) { - t.AddInput("training_mode", {}, {true}); - } else { - t.AddInput("training_mode", {}, {false}); - } - } - - t.AddOutput("output", input_shape, input); // we'll do our own output verification - - std::unique_ptr mask_buffer{}; - if (use_mask) { - mask_buffer = onnxruntime::make_unique(input_size); - t.AddOutput("mask", input_shape, mask_buffer.get(), input_size); - } else { - t.AddMissingOptionalOutput(); - } - - auto output_verifier = [&](const std::vector& fetches, const std::string& provider_type) { - ASSERT_GE(fetches.size(), 1); - const auto& output_tensor = FetchTensor(fetches[0]); - auto output_span = output_tensor.DataAsSpan(); - - const auto num_dropped_values = std::count(output_span.begin(), output_span.end(), residual_value); - - if (ratio == 1.0f) { - ASSERT_EQ(num_dropped_values, static_cast(output_span.size())) << "provider: " << provider_type; - } else { - ASSERT_NEAR(static_cast(num_dropped_values) / static_cast(output_span.size()), training_mode == TrainingTrue ? ratio : 0.0f, 0.1f) - << "provider: " << provider_type; - - for (decltype(output_span.size()) i = 0; i < output_span.size(); ++i) { - if (output_span[i] == residual_value) continue; - const auto expected_value = (bias[i % bias_size] + i + 1.0f) / (1 - (training_mode == TrainingTrue ? ratio : 0.0f)) + residual_value; - ASSERT_NEAR(output_span[i], expected_value, 0.01f) - << "unexpected output value at index " << i << ", provider: " << provider_type; - } - } - - if (use_mask) { - ASSERT_GE(fetches.size(), 2); - const auto& mask_tensor = FetchTensor(fetches[1]); - auto mask_span = mask_tensor.DataAsSpan(); - ASSERT_EQ(mask_span.size(), output_span.size()) << "provider: " << provider_type; - - const auto num_mask_zeros = std::count(mask_span.begin(), mask_span.end(), false); - ASSERT_EQ(num_dropped_values, num_mask_zeros) << "provider: " << provider_type; - - for (decltype(mask_span.size()) i = 0; i < mask_span.size(); ++i) { - ASSERT_TRUE( - (mask_span[i] && output_span[i] != residual_value) || (!mask_span[i] && output_span[i] == residual_value)) - << "output and mask mismatch at index " << i << ", output[i]: " << output_span[i] - << ", mask[i]: " << mask_span[i] << ", provider: " << provider_type; - } - } - }; - - t.SetCustomOutputVerifier(output_verifier); - - t.Run(); -} -} // namespace - -TEST(BiasDropoutTest, Basic) { - RunBiasDropoutTest(false, {10, 10, 10}, 0.75f); -} -TEST(BiasDropoutTest, BasicTrainingFalse) { - RunBiasDropoutTest(false, {10, 10, 10}, 0.75f, TrainingFalse); -} -TEST(BiasDropoutTest, BasicNoTraining) { - RunBiasDropoutTest(false, {10, 10, 10}, 0.75f, NoTraining); -} - -TEST(BiasDropoutTest, BasicWithoutResidual) { - RunBiasDropoutTest(false, {10, 10, 10}, 0.75f, TrainingTrue, false, false); -} - -TEST(BiasDropoutTest, Mask) { - RunBiasDropoutTest(true, {3, 5, 768}, 0.25f); -} -TEST(BiasDropoutTest, MaskTrainingFalse) { - RunBiasDropoutTest(true, {3, 5, 768}, 0.25f, TrainingFalse); -} -TEST(BiasDropoutTest, MaskNoTraining) { - RunBiasDropoutTest(true, {3, 5, 768}, 0.25f, NoTraining); -} - -TEST(BiasDropoutTest, RatioLimit) { - RunBiasDropoutTest(true, {4, 8, 1024}, 0.0f, TrainingFalse); -} - -TEST(BiasDropoutTest, EmptyRatio) { - RunBiasDropoutTest(true, {2, 7, 1024}); -} -#endif - namespace { void RunDropoutGradTest(float ratio, const std::vector& input_dims, bool default_ratio = true) { const auto input_shape = TensorShape(input_dims); diff --git a/orttraining/orttraining/training_ops/cuda/cuda_training_kernels.cc b/orttraining/orttraining/training_ops/cuda/cuda_training_kernels.cc index 78b0cf0bd7..21fde3adcf 100644 --- a/orttraining/orttraining/training_ops/cuda/cuda_training_kernels.cc +++ b/orttraining/orttraining/training_ops/cuda/cuda_training_kernels.cc @@ -65,7 +65,6 @@ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1 class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, float, BatchNormalizationGrad); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, double, BatchNormalizationGrad); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, GatherGrad); -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, BiasDropout); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCudaExecutionProvider, kMSDomain, 1, DropoutGrad); // TODO: decprecate GatherND-1 after updating training models to opset-12 @@ -214,8 +213,7 @@ Status RegisterCudaTrainingKernels(KernelRegistry& kernel_registry) { BuildKernelCreateInfo, BuildKernelCreateInfo, - - BuildKernelCreateInfo, + BuildKernelCreateInfo, // TODO: decprecate GatherND-1 after updating training models to opset-12 diff --git a/orttraining/orttraining/training_ops/cuda/nn/dropout_grad.cc b/orttraining/orttraining/training_ops/cuda/nn/dropout_grad.cc new file mode 100644 index 0000000000..5bbfefb80e --- /dev/null +++ b/orttraining/orttraining/training_ops/cuda/nn/dropout_grad.cc @@ -0,0 +1,69 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "orttraining/training_ops/cuda/nn/dropout_grad.h" +#include "core/providers/cuda/nn/dropout.h" +#include "core/providers/cuda/cuda_common.h" +#include "core/providers/common.h" + +namespace onnxruntime { +namespace cuda { + +#define REGISTER_GRADIENT_KERNEL(OpName) \ + ONNX_OPERATOR_KERNEL_EX( \ + OpName, \ + kMSDomain, \ + 1, \ + kCudaExecutionProvider, \ + KernelDefBuilder() \ + .TypeConstraint("T", ALL_IEEE_FLOAT_TENSOR_TYPES) \ + .TypeConstraint("T1", ALL_IEEE_FLOAT_TENSOR_TYPES) \ + .TypeConstraint("T2", DataTypeImpl::GetTensorType()) \ + .InputMemoryType(2), \ + DropoutGrad); + +REGISTER_GRADIENT_KERNEL(DropoutGrad) + +template +struct DropoutGradComputeImpl { + void operator()(cudaStream_t stream, + const int64_t N, + const Tensor& dY, + const bool* mask_data, + const float ratio_data, + Tensor& dX) const { + typedef typename ToCudaType::MappedType CudaT; + + const CudaT* dY_data = reinterpret_cast(dY.template Data()); + CudaT* dX_data = reinterpret_cast(dX.template MutableData()); + DropoutGradientKernelImpl(stream, N, dY_data, mask_data, ratio_data, dX_data); + } +}; + +Status DropoutGrad::ComputeInternal(OpKernelContext* context) const { + auto dY = context->Input(0); + const TensorShape& shape = dY->Shape(); + const int64_t N = shape.Size(); + + auto mask = context->Input(1); + ORT_ENFORCE(mask->Shape().Size() == N); + const bool* mask_data = mask->template Data(); + + //Get the ratio_data + float ratio_data = default_ratio_; + auto ratio = context->Input(2); + if (ratio) { + utils::MLTypeCallDispatcher t_disp(ratio->GetElementType()); + t_disp.Invoke(ratio, ratio_data); + } + + auto dX = context->Output(0, shape); + + utils::MLTypeCallDispatcher t_disp(dY->GetElementType()); + t_disp.Invoke(Stream(), N, *dY, mask_data, ratio_data, *dX); + + return Status::OK(); +} + +} // namespace cuda +} // namespace onnxruntime diff --git a/orttraining/orttraining/training_ops/cuda/nn/dropout_grad.h b/orttraining/orttraining/training_ops/cuda/nn/dropout_grad.h new file mode 100644 index 0000000000..8155329221 --- /dev/null +++ b/orttraining/orttraining/training_ops/cuda/nn/dropout_grad.h @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/providers/cuda/cuda_kernel.h" +#include "orttraining/training_ops/cuda/nn/dropout_grad_impl.h" + +namespace onnxruntime { +namespace cuda { + +class DropoutGrad final : public CudaKernel { + public: + DropoutGrad(const OpKernelInfo& info) : CudaKernel(info) { + } + + Status ComputeInternal(OpKernelContext* context) const override; + + private: + static constexpr float default_ratio_ = 0.5f; +}; + +} // namespace cuda +} // namespace onnxruntime diff --git a/orttraining/orttraining/training_ops/cuda/nn/dropout_grad_impl.cu b/orttraining/orttraining/training_ops/cuda/nn/dropout_grad_impl.cu new file mode 100644 index 0000000000..c8780b7b26 --- /dev/null +++ b/orttraining/orttraining/training_ops/cuda/nn/dropout_grad_impl.cu @@ -0,0 +1,81 @@ +/** +* Copyright (c) 2016-present, Facebook, Inc. +* +* Licensed under the Apache License, Version 2.0 (the "License"); +* you may not use this file except in compliance with the License. +* You may obtain a copy of the License at +* +* http://www.apache.org/licenses/LICENSE-2.0 +* +* Unless required by applicable law or agreed to in writing, software +* distributed under the License is distributed on an "AS IS" BASIS, +* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +* See the License for the specific language governing permissions and +* limitations under the License. +*/ + +/* Modifications Copyright (c) Microsoft. */ + +#include "core/providers/cuda/cu_inc/common.cuh" +#include "orttraining/training_ops/cuda/nn/dropout_grad_impl.h" +#include +#include + +namespace onnxruntime { +namespace cuda { + +template +__global__ void DropoutGradientKernel( + const int64_t N, + const T* dY_data, + const bool* mask_data, + const float scale, + T* dX_data) { + CUDA_LONG id = NumElementsPerThread * NumThreadsPerBlock * blockIdx.x + threadIdx.x; +#pragma unroll + for (int i = 0; i < NumElementsPerThread; i++) { + if (id < N) { + dX_data[id] = T(float(dY_data[id]) * mask_data[id] * scale); + id += NumThreadsPerBlock; + } + } +} + +template +void DropoutGradientKernelImpl( + cudaStream_t stream, + const int64_t N, + const T* dY_data, + const bool* mask_data, + const float ratio, + T* dX_data) { + if (ratio == 0.0f) { + if (dY_data != dX_data) { + CUDA_CALL_THROW(cudaMemcpyAsync(dX_data, dY_data, N * sizeof(T), cudaMemcpyDeviceToDevice, stream)); + } + } else { + const float scale = 1.f / (1.f - ratio); + const int blocksPerGrid = static_cast(CeilDiv(N, GridDim::maxThreadsPerBlock * GridDim::maxElementsPerThread)); + DropoutGradientKernel + <<>>(N, dY_data, mask_data, scale, dX_data); + } +} + +#define SPECIALIZED_DROPOUT_GRAD_IMPL(T) \ + template void DropoutGradientKernelImpl( \ + cudaStream_t stream, \ + const int64_t N, \ + const T* dY_data, \ + const bool* mask_data, \ + const float scale, \ + T* dX_data); + +SPECIALIZED_DROPOUT_GRAD_IMPL(float) +SPECIALIZED_DROPOUT_GRAD_IMPL(double) +SPECIALIZED_DROPOUT_GRAD_IMPL(half) +#if CUDA_VERSION >= 11000 && (__CUDA_ARCH__ >= 800 || !defined(__CUDA_ARCH__)) +SPECIALIZED_DROPOUT_GRAD_IMPL(nv_bfloat16) +#endif + +} // namespace cuda +} // namespace onnxruntime diff --git a/orttraining/orttraining/training_ops/cuda/nn/dropout_impl.h b/orttraining/orttraining/training_ops/cuda/nn/dropout_grad_impl.h similarity index 58% rename from orttraining/orttraining/training_ops/cuda/nn/dropout_impl.h rename to orttraining/orttraining/training_ops/cuda/nn/dropout_grad_impl.h index 8dbf3f9655..e36878e259 100644 --- a/orttraining/orttraining/training_ops/cuda/nn/dropout_impl.h +++ b/orttraining/orttraining/training_ops/cuda/nn/dropout_grad_impl.h @@ -17,19 +17,5 @@ void DropoutGradientKernelImpl( const float ratio, T* dX_data); -template -void BiasDropoutKernelImpl( - const cudaDeviceProp& prop, - cudaStream_t stream, - const int64_t N, - const fast_divmod fdm_dim, - const float ratio, - PhiloxGenerator& generator, - const T* X_data, - const T* bias_data, - const T* residual_data, - T* Y_data, - bool* mask_data); - } // namespace cuda } // namespace onnxruntime diff --git a/orttraining/orttraining/training_ops/rocm/rocm_training_kernels.cc b/orttraining/orttraining/training_ops/rocm/rocm_training_kernels.cc index 616a98964d..fece1d6ece 100644 --- a/orttraining/orttraining/training_ops/rocm/rocm_training_kernels.cc +++ b/orttraining/orttraining/training_ops/rocm/rocm_training_kernels.cc @@ -63,7 +63,6 @@ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1 class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, float, BatchNormalizationGrad); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, double, BatchNormalizationGrad); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, GatherGrad); -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, BiasDropout); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kRocmExecutionProvider, kMSDomain, 1, DropoutGrad); // TODO: decprecate GatherND-1 after updating training models to opset-12 @@ -174,7 +173,6 @@ Status RegisterRocmTrainingKernels(KernelRegistry& kernel_registry) { BuildKernelCreateInfo, BuildKernelCreateInfo, - BuildKernelCreateInfo, BuildKernelCreateInfo, // TODO: decprecate GatherND-1 after updating training models to opset-12 From fda0470683318dad8bea56e6608276b9d3790e85 Mon Sep 17 00:00:00 2001 From: Vincent Wang Date: Thu, 25 Mar 2021 15:18:51 +0800 Subject: [PATCH 002/129] Add New AllocKind for YieldOp Outputs, Run YieldOp with InferenceSession in UT (#7125) * new allockind, add ut * change macro * fix win build * rename alloc kind * fix mem leak --- .../onnxruntime/core/framework/alloc_kind.h | 3 +- .../core/framework/allocation_planner.cc | 5 +- onnxruntime/core/framework/execution_frame.cc | 10 ++++ onnxruntime/core/framework/execution_frame.h | 6 +++ onnxruntime/core/framework/memory_info.cc | 1 + .../test/framework/allocation_planner_test.cc | 2 +- .../test/framework/execution_frame_test.cc | 53 ++++++++++++++++++- .../training_ops/cpu/controlflow/ort_tasks.h | 4 +- 8 files changed, 78 insertions(+), 6 deletions(-) diff --git a/include/onnxruntime/core/framework/alloc_kind.h b/include/onnxruntime/core/framework/alloc_kind.h index 4534d08470..c7a953a44b 100644 --- a/include/onnxruntime/core/framework/alloc_kind.h +++ b/include/onnxruntime/core/framework/alloc_kind.h @@ -28,7 +28,8 @@ enum class AllocKind { kPreExisting = 2, kAllocateStatically = 3, kAllocateOutput = 4, - kShare = 5 + kShare = 5, + kAllocatedExternally = 6 }; std::ostream& operator<<(std::ostream& out, AllocKind alloc_kind); diff --git a/onnxruntime/core/framework/allocation_planner.cc b/onnxruntime/core/framework/allocation_planner.cc index 8c72823f88..0c308faf72 100644 --- a/onnxruntime/core/framework/allocation_planner.cc +++ b/onnxruntime/core/framework/allocation_planner.cc @@ -40,6 +40,9 @@ std::ostream& operator<<(std::ostream& out, AllocKind alloc_kind) { case AllocKind::kShare: out << "Share"; break; + case AllocKind::kAllocatedExternally: + out << "AllocatedExternally"; + break; case AllocKind::kNotSet: out << "NotSet"; break; @@ -717,7 +720,7 @@ class PlannerImpl { } } else if (external_outputs) { ORT_ENFORCE(!IsNonTensor(*node_output), "Only tensors are supported for external outputs for now."); - AllocPlan(current).alloc_kind = AllocKind::kPreExisting; + AllocPlan(current).alloc_kind = AllocKind::kAllocatedExternally; #if !defined(ORT_MINIMAL_BUILD) && defined(ORT_MEMORY_PROFILE) AllocPlan(current).life_interval.second = execution_plan.size(); #endif diff --git a/onnxruntime/core/framework/execution_frame.cc b/onnxruntime/core/framework/execution_frame.cc index fc7241ea2b..f3e9118ec9 100644 --- a/onnxruntime/core/framework/execution_frame.cc +++ b/onnxruntime/core/framework/execution_frame.cc @@ -51,6 +51,11 @@ Status IExecutionFrame::SetOutputMLValue(int index, const OrtValue& ort_value) { return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "invalid index ", ort_value_idx); } + if (!IsAllocatedExternally(ort_value_idx)) { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "SetOutputMLValue() is not allowed for OrtValue index ", ort_value_idx, + " as its allocation kind is not kAllocatedExternally."); + } + all_values_[ort_value_idx] = ort_value; return Status::OK(); } @@ -632,6 +637,11 @@ const AllocPlanPerValue& ExecutionFrame::GetAllocationPlan(int ort_value_idx) { return alloc_plan[ort_value_idx]; } +bool ExecutionFrame::IsAllocatedExternally(int ort_value_idx) { + const auto& allocation_plan = GetAllocationPlan(ort_value_idx); + return allocation_plan.alloc_kind == AllocKind::kAllocatedExternally; +} + void ExecutionFrame::TraceAllocate(int ort_value_idx, size_t size) { if (planner_) { // don't trace the output tensors. diff --git a/onnxruntime/core/framework/execution_frame.h b/onnxruntime/core/framework/execution_frame.h index 39473f0ea3..01f5e5680e 100644 --- a/onnxruntime/core/framework/execution_frame.h +++ b/onnxruntime/core/framework/execution_frame.h @@ -98,6 +98,10 @@ class IExecutionFrame { virtual Status CopyTensor(const Tensor& src, Tensor& dest) const = 0; + virtual bool IsAllocatedExternally(int /*ort_value_idx*/) { + return false; + } + const NodeIndexInfo& node_index_info_; // All the intermediate values for the entire graph. @@ -185,6 +189,8 @@ class ExecutionFrame final : public IExecutionFrame { const AllocPlanPerValue& GetAllocationPlan(int ort_value_idx); + bool IsAllocatedExternally(int ort_value_idx) override; + const SessionState& session_state_; // map of index to custom allocator diff --git a/onnxruntime/core/framework/memory_info.cc b/onnxruntime/core/framework/memory_info.cc index 293acf17f3..3b72dbf929 100644 --- a/onnxruntime/core/framework/memory_info.cc +++ b/onnxruntime/core/framework/memory_info.cc @@ -34,6 +34,7 @@ void MemoryInfo::GenerateTensorMap(const SequentialExecutionPlan* execution_plan //If the tensor is using memory outside of the scope, do not store it if (execution_plan->allocation_plan[mem_info.reused_buffer].alloc_kind == AllocKind::kPreExisting) continue; if (execution_plan->allocation_plan[mem_info.reused_buffer].alloc_kind == AllocKind::kAllocateOutput) continue; + if (execution_plan->allocation_plan[mem_info.reused_buffer].alloc_kind == AllocKind::kAllocatedExternally) continue; mem_info.inplace_reuse = (execution_plan->allocation_plan[value_idx].inplace_reuse != -1 && execution_plan->allocation_plan[value_idx].inplace_reuse != value_idx); mem_info.alloc_kind = execution_plan->allocation_plan[value_idx].alloc_kind; mem_info.location = execution_plan->allocation_plan[value_idx].location; diff --git a/onnxruntime/test/framework/allocation_planner_test.cc b/onnxruntime/test/framework/allocation_planner_test.cc index 8b35ca101a..de82e1b54d 100644 --- a/onnxruntime/test/framework/allocation_planner_test.cc +++ b/onnxruntime/test/framework/allocation_planner_test.cc @@ -428,7 +428,7 @@ TEST_F(PlannerTest, ExternalOutputsTest) { // check allocation kind: CheckAllocKind(X1, AllocKind::kPreExisting); - CheckAllocKind(X2, AllocKind::kPreExisting); + CheckAllocKind(X2, AllocKind::kAllocatedExternally); CheckAllocKind(X3, AllocKind::kAllocate); CheckAllocKind(X4, AllocKind::kAllocateOutput); diff --git a/onnxruntime/test/framework/execution_frame_test.cc b/onnxruntime/test/framework/execution_frame_test.cc index 52147f18e2..43ba139a11 100644 --- a/onnxruntime/test/framework/execution_frame_test.cc +++ b/onnxruntime/test/framework/execution_frame_test.cc @@ -16,6 +16,11 @@ #include "gtest/gtest.h" #include "gmock/gmock.h" +#ifdef ENABLE_TRAINING +#include "core/session/IOBinding.h" +#include "orttraining/core/agent/training_agent.h" +#endif + using namespace ONNX_NAMESPACE; using namespace std; @@ -254,7 +259,7 @@ TEST_F(ExecutionFrameTest, MemPatternTest) { ASSERT_EQ(p->GetBlock(4)->offset_, kAllocAlignment); } -#if defined(ENABLE_TRAINING) || defined(ENABLE_TRAINING_OPS) +#ifdef ENABLE_TRAINING TEST_F(ExecutionFrameTest, MemPatternWithExternalOutputsTest) { auto cpu_xp = CreateCPUExecutionProvider(); auto xp_type = cpu_xp->Type(); @@ -327,6 +332,52 @@ TEST_F(ExecutionFrameTest, MemPatternWithExternalOutputsTest) { ASSERT_EQ(pattern.patterns.size(), 1u); auto p = pattern.GetPatterns(cpu_allocator->Info()); ASSERT_EQ(p->PeakSize(), 0u); // Peak size is 0. + + SessionOptions so; + so.session_logid = "MemPatternWithExternalOutputsTest"; + InferenceSession session_obj{so, GetEnvironment()}; + std::stringstream buffer; + model.ToProto().SerializeToOstream(&buffer); + ASSERT_STATUS_OK(session_obj.Load(buffer)); + ASSERT_STATUS_OK(session_obj.Initialize()); + + { + // Run with original InferenceSession::Run, it should fail due to the YieldOp. + NameMLValMap feeds; + feeds.insert(std::make_pair("X", x_value)); + + // prepare outputs + std::vector output_names; + output_names.push_back("Y"); + std::vector fetches; + + RunOptions run_options; + auto st = session_obj.Run(run_options, feeds, output_names, &fetches); + EXPECT_FALSE(st.IsOK()); + EXPECT_THAT(st.ErrorMessage(), testing::HasSubstr("Non-zero status code returned while running YieldOp node.")); + } + + { + // Run with new RunForward/RunBackward. + training::TrainingAgent training_agent(&session_obj); + unique_ptr io_binding; + ASSERT_STATUS_OK(session_obj.NewIOBinding(&io_binding)); + io_binding->BindInput("X", x_value); + OrtValue output; + io_binding->BindOutput("Y", output); + RunOptions run_options; + std::vector user_outputs; + int64_t run_id; + ASSERT_STATUS_OK(training_agent.RunForward(run_options, *io_binding, user_outputs, run_id)); + const std::vector yield_input_expected{2.0f, 2.0f, 2.0f, 2.0f}; + EXPECT_THAT(user_outputs[0].Get().DataAsSpan(), + ::testing::ContainerEq(gsl::make_span(yield_input_expected))); + ASSERT_STATUS_OK(training_agent.RunBackward(run_id, {t_value})); + // The output is MatMul(x_value, t_value); + const std::vector output_expected{4.0f, 4.0f, 4.0f, 4.0f}; + EXPECT_THAT(io_binding->GetOutputs()[0].Get().DataAsSpan(), + ::testing::ContainerEq(gsl::make_span(output_expected))); + } } #endif diff --git a/orttraining/orttraining/training_ops/cpu/controlflow/ort_tasks.h b/orttraining/orttraining/training_ops/cpu/controlflow/ort_tasks.h index bb78e0a935..56993a97d5 100644 --- a/orttraining/orttraining/training_ops/cpu/controlflow/ort_tasks.h +++ b/orttraining/orttraining/training_ops/cpu/controlflow/ort_tasks.h @@ -19,8 +19,8 @@ typedef std::pair> BackwardReturnType; class OrtTasks final { public: static OrtTasks& GetInstance() { - static OrtTasks* instance_ = new OrtTasks; - return *instance_; + static OrtTasks instance_; + return instance_; } void CreateBackgroundTask(int64_t run_id); From 865c67611cb327f88adef614406cdb90d71084a9 Mon Sep 17 00:00:00 2001 From: RandySheriffH <48490400+RandySheriffH@users.noreply.github.com> Date: Thu, 25 Mar 2021 09:06:14 -0700 Subject: [PATCH 003/129] Exclude profiler from minimal build (#7115) * Exclude TP profiler from minimum build * fix typo * remove Clock * fix comments Co-authored-by: Randy Shuai --- .../platform/EigenNonBlockingThreadPool.h | 30 +++++++++++++++++-- onnxruntime/core/common/threadpool.cc | 16 ++++++---- 2 files changed, 38 insertions(+), 8 deletions(-) diff --git a/include/onnxruntime/core/platform/EigenNonBlockingThreadPool.h b/include/onnxruntime/core/platform/EigenNonBlockingThreadPool.h index 2ecaf2ef4f..12784dbd25 100644 --- a/include/onnxruntime/core/platform/EigenNonBlockingThreadPool.h +++ b/include/onnxruntime/core/platform/EigenNonBlockingThreadPool.h @@ -171,7 +171,32 @@ struct PaddingToAvoidFalseSharing { 4. Note LogStart must pair with either LogEnd or LogEndAndStart, otherwise ORT_ENFORCE will fail; 5. ThreadPoolProfiler is thread-safe. */ - +#ifdef ORT_MINIMAL_BUILD +class ThreadPoolProfiler { + public: + enum ThreadPoolEvent { + DISTRIBUTION = 0, + DISTRIBUTION_ENQUEUE, + RUN, + WAIT, + WAIT_REVOKE, + MAX_EVENT + }; + ThreadPoolProfiler(int, const CHAR_TYPE*){}; + ~ThreadPoolProfiler() = default; + ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(ThreadPoolProfiler); + void Start(){}; + std::string Stop() { return "not available for minimal build"; } + void LogStart(){}; + void LogEnd(ThreadPoolEvent){}; + void LogEndAndStart(ThreadPoolEvent){}; + void LogStartAndCoreAndBlock(std::ptrdiff_t){}; + void LogCoreAndBlock(std::ptrdiff_t){}; + void LogThreadId(int){}; + void LogRun(int){}; + std::string DumpChildThreadStat() { return {}; } +}; +#else class ThreadPoolProfiler { public: enum ThreadPoolEvent { @@ -222,8 +247,9 @@ class ThreadPoolProfiler { PaddingToAvoidFalseSharing padding_; //to prevent false sharing }; std::vector child_thread_stats_; - std::string threal_pool_name_; + std::string thread_pool_name_; }; +#endif // Extended Eigen thread pool interface, avoiding the need to modify // the ThreadPoolInterface.h header from the external Eigen diff --git a/onnxruntime/core/common/threadpool.cc b/onnxruntime/core/common/threadpool.cc index bca5d439d4..c91e93c408 100644 --- a/onnxruntime/core/common/threadpool.cc +++ b/onnxruntime/core/common/threadpool.cc @@ -21,6 +21,7 @@ limitations under the License. #include "core/common/eigen_common_wrapper.h" #include "core/platform/EigenNonBlockingThreadPool.h" #include "core/platform/ort_mutex.h" +#if !defined(ORT_MINIMAL_BUILD) #ifdef _WIN32 #include "processthreadsapi.h" #include @@ -30,24 +31,26 @@ limitations under the License. #else #include #endif +#endif namespace onnxruntime { namespace concurrency { -ThreadPoolProfiler::ThreadPoolProfiler(int num_threads, const CHAR_TYPE* threal_pool_name) : +#if !defined(ORT_MINIMAL_BUILD) +ThreadPoolProfiler::ThreadPoolProfiler(int num_threads, const CHAR_TYPE* thread_pool_name) : num_threads_(num_threads) { child_thread_stats_.assign(num_threads, {}); - if (threal_pool_name) { + if (thread_pool_name) { #ifdef _WIN32 using convert_type = std::codecvt_utf8; std::wstring_convert converter; - threal_pool_name_ = converter.to_bytes(threal_pool_name); + thread_pool_name_ = converter.to_bytes(thread_pool_name); #else - threal_pool_name_ = threal_pool_name; + thread_pool_name_ = thread_pool_name; #endif } else { - threal_pool_name_ = "unnamed_thread_pool"; + thread_pool_name_ = "unnamed_thread_pool"; } } @@ -72,7 +75,7 @@ std::string ThreadPoolProfiler::Stop() { std::stringstream ss; ss << "{\"main_thread\": {" << "\"thread_pool_name\": \"" - << threal_pool_name_ << "\", " + << thread_pool_name_ << "\", " << GetMainThreadStat().Reset() << "}, \"sub_threads\": {" << DumpChildThreadStat() @@ -220,6 +223,7 @@ std::string ThreadPoolProfiler::DumpChildThreadStat() { } return ss.str(); } +#endif // A sharded loop counter distributes loop iterations between a set of worker threads. The iteration space of // the loop is divided (perhaps unevenly) between the shards. Each thread has a home shard (perhaps not uniquely From 8e54b76e2d845b0d0f4c71101a021d1b35ec03b4 Mon Sep 17 00:00:00 2001 From: Yufeng Li Date: Thu, 25 Mar 2021 09:17:23 -0700 Subject: [PATCH 004/129] QDQ implementation (#7033) * Add QDQ basic implementation --- cmake/onnxruntime_optimizer.cmake | 2 + onnxruntime/core/common/type_utils.h | 21 + onnxruntime/core/framework/cblas.h | 611 ---------------- onnxruntime/core/graph/graph_utils.cc | 36 + onnxruntime/core/graph/graph_utils.h | 11 + .../core/optimizer/graph_transformer_utils.cc | 5 + .../qdq_transformer/qdq_binary_op.cc | 46 ++ .../optimizer/qdq_transformer/qdq_conv.cc | 49 ++ .../optimizer/qdq_transformer/qdq_matmul.cc | 44 ++ .../qdq_transformer/qdq_op_transformer.cc | 60 ++ .../qdq_transformer/qdq_op_transformer.h | 41 ++ .../qdq_transformer/qdq_simple_ops.cc | 38 + .../qdq_transformer/qdq_transformer.cc | 90 +++ .../qdq_transformer/qdq_transformer.h | 24 + .../optimizer/qdq_transformer/registry.cc | 21 + .../core/optimizer/qdq_transformer/registry.h | 56 ++ .../providers/cuda/tensor/compress_impl.cu | 3 +- onnxruntime/core/util/math.h | 26 +- .../tools/quantization/onnx_quantizer.py | 11 +- .../test/common/tensor_op_test_utils.h | 16 +- .../optimizer/graph_transform_test_builder.cc | 84 +++ .../optimizer/graph_transform_test_builder.h | 246 +++++++ .../test/optimizer/nhwc_transformer_test.cc | 655 ++++++------------ .../test/optimizer/qdq_transformer_test.cc | 237 +++++++ 24 files changed, 1379 insertions(+), 1054 deletions(-) create mode 100644 onnxruntime/core/common/type_utils.h delete mode 100644 onnxruntime/core/framework/cblas.h create mode 100644 onnxruntime/core/optimizer/qdq_transformer/qdq_binary_op.cc create mode 100644 onnxruntime/core/optimizer/qdq_transformer/qdq_conv.cc create mode 100644 onnxruntime/core/optimizer/qdq_transformer/qdq_matmul.cc create mode 100644 onnxruntime/core/optimizer/qdq_transformer/qdq_op_transformer.cc create mode 100644 onnxruntime/core/optimizer/qdq_transformer/qdq_op_transformer.h create mode 100644 onnxruntime/core/optimizer/qdq_transformer/qdq_simple_ops.cc create mode 100644 onnxruntime/core/optimizer/qdq_transformer/qdq_transformer.cc create mode 100644 onnxruntime/core/optimizer/qdq_transformer/qdq_transformer.h create mode 100644 onnxruntime/core/optimizer/qdq_transformer/registry.cc create mode 100644 onnxruntime/core/optimizer/qdq_transformer/registry.h create mode 100644 onnxruntime/test/optimizer/graph_transform_test_builder.cc create mode 100644 onnxruntime/test/optimizer/graph_transform_test_builder.h create mode 100644 onnxruntime/test/optimizer/qdq_transformer_test.cc diff --git a/cmake/onnxruntime_optimizer.cmake b/cmake/onnxruntime_optimizer.cmake index a062d212bb..1486e5e19e 100644 --- a/cmake/onnxruntime_optimizer.cmake +++ b/cmake/onnxruntime_optimizer.cmake @@ -13,6 +13,8 @@ else() "${ONNXRUNTIME_INCLUDE_DIR}/core/optimizer/*.h" "${ONNXRUNTIME_ROOT}/core/optimizer/*.h" "${ONNXRUNTIME_ROOT}/core/optimizer/*.cc" + "${ONNXRUNTIME_ROOT}/core/optimizer/qdq_transformer/*.h" + "${ONNXRUNTIME_ROOT}/core/optimizer/qdq_transformer/*.cc" ) endif() diff --git a/onnxruntime/core/common/type_utils.h b/onnxruntime/core/common/type_utils.h new file mode 100644 index 0000000000..7ca032f4ae --- /dev/null +++ b/onnxruntime/core/common/type_utils.h @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include + +namespace onnxruntime { +namespace utils { +template +struct IsByteType : std::false_type {}; + +template <> +struct IsByteType : std::true_type {}; + +template <> +struct IsByteType : std::true_type {}; + +} // namespace utils +} // namespace onnxruntime diff --git a/onnxruntime/core/framework/cblas.h b/onnxruntime/core/framework/cblas.h deleted file mode 100644 index c036e24d40..0000000000 --- a/onnxruntime/core/framework/cblas.h +++ /dev/null @@ -1,611 +0,0 @@ -/** -* Copyright (c) 2016-present, Facebook, Inc. -* -* Licensed under the Apache License, Version 2.0 (the "License"); -* you may not use this file except in compliance with the License. -* You may obtain a copy of the License at -* -* http://www.apache.org/licenses/LICENSE-2.0 -* -* Unless required by applicable law or agreed to in writing, software -* distributed under the License is distributed on an "AS IS" BASIS, -* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -* See the License for the specific language governing permissions and -* limitations under the License. -*/ -/* Modifications Copyright (c) Microsoft. */ - -// This is the exact cblas.h header file, placed here purely in order to get -// the enums. - -#ifndef CBLAS_H - -#ifndef CBLAS_ENUM_DEFINED_H -#define CBLAS_ENUM_DEFINED_H -enum CBLAS_ORDER { CblasRowMajor = 101, - CblasColMajor = 102 }; -enum CBLAS_TRANSPOSE { - CblasNoTrans = 111, - CblasTrans = 112, - CblasConjTrans = 113 -}; -enum CBLAS_UPLO { CblasUpper = 121, - CblasLower = 122 }; -enum CBLAS_DIAG { CblasNonUnit = 131, - CblasUnit = 132 }; -enum CBLAS_SIDE { CblasLeft = 141, - CblasRight = 142 }; -#endif - -#ifndef CBLAS_ENUM_ONLY -#define CBLAS_H -#define CBLAS_INDEX int - -int cblas_errprn(int ierr, int info, char* form, ...); -void cblas_xerbla(int p, const char* rout, const char* form, ...); - -/* -* =========================================================================== -* Prototypes for level 1 BLAS functions (complex are recast as routines) -* =========================================================================== -*/ -float cblas_sdsdot(int N, float alpha, const float* X, - int incX, const float* Y, int incY); -double cblas_dsdot(int N, const float* X, int incX, const float* Y, - int incY); -float cblas_sdot(int N, const float* X, int incX, - const float* Y, int incY); -double cblas_ddot(int N, const double* X, int incX, - const double* Y, int incY); -/* -* Functions having prefixes Z and C only -*/ -void cblas_cdotu_sub(int N, const void* X, int incX, - const void* Y, int incY, void* dotu); -void cblas_cdotc_sub(int N, const void* X, int incX, - const void* Y, int incY, void* dotc); - -void cblas_zdotu_sub(int N, const void* X, int incX, - const void* Y, int incY, void* dotu); -void cblas_zdotc_sub(int N, const void* X, int incX, - const void* Y, int incY, void* dotc); - -/* -* Functions having prefixes S D SC DZ -*/ -float cblas_snrm2(int N, const float* X, int incX); -float cblas_sasum(int N, const float* X, int incX); - -double cblas_dnrm2(int N, const double* X, int incX); -double cblas_dasum(int N, const double* X, int incX); - -float cblas_scnrm2(int N, const void* X, int incX); -float cblas_scasum(int N, const void* X, int incX); - -double cblas_dznrm2(int N, const void* X, int incX); -double cblas_dzasum(int N, const void* X, int incX); - -/* -* Functions having standard 4 prefixes (S D C Z) -*/ -CBLAS_INDEX cblas_isamax(int N, const float* X, int incX); -CBLAS_INDEX cblas_idamax(int N, const double* X, int incX); -CBLAS_INDEX cblas_icamax(int N, const void* X, int incX); -CBLAS_INDEX cblas_izamax(int N, const void* X, int incX); - -/* -* =========================================================================== -* Prototypes for level 1 BLAS routines -* =========================================================================== -*/ - -/* -* Routines with standard 4 prefixes (s, d, c, z) -*/ -void cblas_sswap(int N, float* X, int incX, - float* Y, int incY); -void cblas_scopy(int N, const float* X, int incX, - float* Y, int incY); -void cblas_saxpy(int N, float alpha, const float* X, - int incX, float* Y, int incY); -void catlas_saxpby(int N, float alpha, const float* X, - int incX, float beta, float* Y, int incY); -void catlas_sset(int N, float alpha, float* X, int incX); - -void cblas_dswap(int N, double* X, int incX, - double* Y, int incY); -void cblas_dcopy(int N, const double* X, int incX, - double* Y, int incY); -void cblas_daxpy(int N, double alpha, const double* X, - int incX, double* Y, int incY); -void catlas_daxpby(int N, double alpha, const double* X, - int incX, double beta, double* Y, int incY); -void catlas_dset(int N, double alpha, double* X, int incX); - -void cblas_cswap(int N, void* X, int incX, - void* Y, int incY); -void cblas_ccopy(int N, const void* X, int incX, - void* Y, int incY); -void cblas_caxpy(int N, const void* alpha, const void* X, - int incX, void* Y, int incY); -void catlas_caxpby(int N, const void* alpha, const void* X, - int incX, const void* beta, void* Y, int incY); -void catlas_cset(int N, const void* alpha, void* X, int incX); - -void cblas_zswap(int N, void* X, int incX, - void* Y, int incY); -void cblas_zcopy(int N, const void* X, int incX, - void* Y, int incY); -void cblas_zaxpy(int N, const void* alpha, const void* X, - int incX, void* Y, int incY); -void catlas_zaxpby(int N, const void* alpha, const void* X, - int incX, const void* beta, void* Y, int incY); -void catlas_zset(int N, const void* alpha, void* X, int incX); - -/* -* Routines with S and D prefix only -*/ -void cblas_srotg(float* a, float* b, float* c, float* s); -void cblas_srotmg(float* d1, float* d2, float* b1, float b2, float* P); -void cblas_srot(int N, float* X, int incX, - float* Y, int incY, float c, float s); -void cblas_srotm(int N, float* X, int incX, - float* Y, int incY, const float* P); - -void cblas_drotg(double* a, double* b, double* c, double* s); -void cblas_drotmg(double* d1, double* d2, double* b1, double b2, double* P); -void cblas_drot(int N, double* X, int incX, - double* Y, int incY, double c, double s); -void cblas_drotm(int N, double* X, int incX, - double* Y, int incY, const double* P); - -/* -* Routines with S D C Z CS and ZD prefixes -*/ -void cblas_sscal(int N, float alpha, float* X, int incX); -void cblas_dscal(int N, double alpha, double* X, int incX); -void cblas_cscal(int N, const void* alpha, void* X, int incX); -void cblas_zscal(int N, const void* alpha, void* X, int incX); -void cblas_csscal(int N, float alpha, void* X, int incX); -void cblas_zdscal(int N, double alpha, void* X, int incX); - -/* -* Extra reference routines provided by ATLAS, but not mandated by the standard -*/ -void cblas_crotg(void* a, void* b, void* c, void* s); -void cblas_zrotg(void* a, void* b, void* c, void* s); -void cblas_csrot(int N, void* X, int incX, void* Y, int incY, - float c, float s); -void cblas_zdrot(int N, void* X, int incX, void* Y, int incY, - double c, double s); - -/* -* =========================================================================== -* Prototypes for level 2 BLAS -* =========================================================================== -*/ - -/* -* Routines with standard 4 prefixes (S, D, C, Z) -*/ -void cblas_sgemv(enum CBLAS_ORDER Order, - enum CBLAS_TRANSPOSE TransA, int M, int N, - float alpha, const float* A, int lda, - const float* X, int incX, float beta, - float* Y, int incY); -void cblas_sgbmv(enum CBLAS_ORDER Order, - enum CBLAS_TRANSPOSE TransA, int M, int N, - int KL, int KU, float alpha, - const float* A, int lda, const float* X, - int incX, float beta, float* Y, int incY); -void cblas_strmv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag, - int N, const float* A, int lda, - float* X, int incX); -void cblas_stbmv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag, - int N, int K, const float* A, int lda, - float* X, int incX); -void cblas_stpmv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag, - int N, const float* Ap, float* X, int incX); -void cblas_strsv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag, - int N, const float* A, int lda, float* X, - int incX); -void cblas_stbsv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag, - int N, int K, const float* A, int lda, - float* X, int incX); -void cblas_stpsv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag, - int N, const float* Ap, float* X, int incX); - -void cblas_dgemv(enum CBLAS_ORDER Order, - enum CBLAS_TRANSPOSE TransA, int M, int N, - double alpha, const double* A, int lda, - const double* X, int incX, double beta, - double* Y, int incY); -void cblas_dgbmv(enum CBLAS_ORDER Order, - enum CBLAS_TRANSPOSE TransA, int M, int N, - int KL, int KU, double alpha, - const double* A, int lda, const double* X, - int incX, double beta, double* Y, int incY); -void cblas_dtrmv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag, - int N, const double* A, int lda, - double* X, int incX); -void cblas_dtbmv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag, - int N, int K, const double* A, int lda, - double* X, int incX); -void cblas_dtpmv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag, - int N, const double* Ap, double* X, int incX); -void cblas_dtrsv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag, - int N, const double* A, int lda, double* X, - int incX); -void cblas_dtbsv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag, - int N, int K, const double* A, int lda, - double* X, int incX); -void cblas_dtpsv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag, - int N, const double* Ap, double* X, int incX); - -void cblas_cgemv(enum CBLAS_ORDER Order, - enum CBLAS_TRANSPOSE TransA, int M, int N, - const void* alpha, const void* A, int lda, - const void* X, int incX, const void* beta, - void* Y, int incY); -void cblas_cgbmv(enum CBLAS_ORDER Order, - enum CBLAS_TRANSPOSE TransA, int M, int N, - int KL, int KU, const void* alpha, - const void* A, int lda, const void* X, - int incX, const void* beta, void* Y, int incY); -void cblas_ctrmv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag, - int N, const void* A, int lda, - void* X, int incX); -void cblas_ctbmv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag, - int N, int K, const void* A, int lda, - void* X, int incX); -void cblas_ctpmv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag, - int N, const void* Ap, void* X, int incX); -void cblas_ctrsv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag, - int N, const void* A, int lda, void* X, - int incX); -void cblas_ctbsv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag, - int N, int K, const void* A, int lda, - void* X, int incX); -void cblas_ctpsv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag, - int N, const void* Ap, void* X, int incX); - -void cblas_zgemv(enum CBLAS_ORDER Order, - enum CBLAS_TRANSPOSE TransA, int M, int N, - const void* alpha, const void* A, int lda, - const void* X, int incX, const void* beta, - void* Y, int incY); -void cblas_zgbmv(enum CBLAS_ORDER Order, - enum CBLAS_TRANSPOSE TransA, int M, int N, - int KL, int KU, const void* alpha, - const void* A, int lda, const void* X, - int incX, const void* beta, void* Y, int incY); -void cblas_ztrmv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag, - int N, const void* A, int lda, - void* X, int incX); -void cblas_ztbmv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag, - int N, int K, const void* A, int lda, - void* X, int incX); -void cblas_ztpmv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag, - int N, const void* Ap, void* X, int incX); -void cblas_ztrsv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag, - int N, const void* A, int lda, void* X, - int incX); -void cblas_ztbsv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag, - int N, int K, const void* A, int lda, - void* X, int incX); -void cblas_ztpsv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - enum CBLAS_TRANSPOSE TransA, enum CBLAS_DIAG Diag, - int N, const void* Ap, void* X, int incX); - -/* -* Routines with S and D prefixes only -*/ -void cblas_ssymv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - int N, float alpha, const float* A, - int lda, const float* X, int incX, - float beta, float* Y, int incY); -void cblas_ssbmv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - int N, int K, float alpha, const float* A, - int lda, const float* X, int incX, - float beta, float* Y, int incY); -void cblas_sspmv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - int N, float alpha, const float* Ap, - const float* X, int incX, - float beta, float* Y, int incY); -void cblas_sger(enum CBLAS_ORDER Order, int M, int N, - float alpha, const float* X, int incX, - const float* Y, int incY, float* A, int lda); -void cblas_ssyr(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - int N, float alpha, const float* X, - int incX, float* A, int lda); -void cblas_sspr(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - int N, float alpha, const float* X, - int incX, float* Ap); -void cblas_ssyr2(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - int N, float alpha, const float* X, - int incX, const float* Y, int incY, float* A, - int lda); -void cblas_sspr2(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - int N, float alpha, const float* X, - int incX, const float* Y, int incY, float* A); - -void cblas_dsymv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - int N, double alpha, const double* A, - int lda, const double* X, int incX, - double beta, double* Y, int incY); -void cblas_dsbmv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - int N, int K, double alpha, const double* A, - int lda, const double* X, int incX, - double beta, double* Y, int incY); -void cblas_dspmv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - int N, double alpha, const double* Ap, - const double* X, int incX, - double beta, double* Y, int incY); -void cblas_dger(enum CBLAS_ORDER Order, int M, int N, - double alpha, const double* X, int incX, - const double* Y, int incY, double* A, int lda); -void cblas_dsyr(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - int N, double alpha, const double* X, - int incX, double* A, int lda); -void cblas_dspr(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - int N, double alpha, const double* X, - int incX, double* Ap); -void cblas_dsyr2(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - int N, double alpha, const double* X, - int incX, const double* Y, int incY, double* A, - int lda); -void cblas_dspr2(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - int N, double alpha, const double* X, - int incX, const double* Y, int incY, double* A); - -/* -* Routines with C and Z prefixes only -*/ -void cblas_chemv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - int N, const void* alpha, const void* A, - int lda, const void* X, int incX, - const void* beta, void* Y, int incY); -void cblas_chbmv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - int N, int K, const void* alpha, const void* A, - int lda, const void* X, int incX, - const void* beta, void* Y, int incY); -void cblas_chpmv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - int N, const void* alpha, const void* Ap, - const void* X, int incX, - const void* beta, void* Y, int incY); -void cblas_cgeru(enum CBLAS_ORDER Order, int M, int N, - const void* alpha, const void* X, int incX, - const void* Y, int incY, void* A, int lda); -void cblas_cgerc(enum CBLAS_ORDER Order, int M, int N, - const void* alpha, const void* X, int incX, - const void* Y, int incY, void* A, int lda); -void cblas_cher(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - int N, float alpha, const void* X, int incX, - void* A, int lda); -void cblas_chpr(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - int N, float alpha, const void* X, - int incX, void* A); -void cblas_cher2(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, int N, - const void* alpha, const void* X, int incX, - const void* Y, int incY, void* A, int lda); -void cblas_chpr2(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, int N, - const void* alpha, const void* X, int incX, - const void* Y, int incY, void* Ap); - -void cblas_zhemv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - int N, const void* alpha, const void* A, - int lda, const void* X, int incX, - const void* beta, void* Y, int incY); -void cblas_zhbmv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - int N, int K, const void* alpha, const void* A, - int lda, const void* X, int incX, - const void* beta, void* Y, int incY); -void cblas_zhpmv(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - int N, const void* alpha, const void* Ap, - const void* X, int incX, - const void* beta, void* Y, int incY); -void cblas_zgeru(enum CBLAS_ORDER Order, int M, int N, - const void* alpha, const void* X, int incX, - const void* Y, int incY, void* A, int lda); -void cblas_zgerc(enum CBLAS_ORDER Order, int M, int N, - const void* alpha, const void* X, int incX, - const void* Y, int incY, void* A, int lda); -void cblas_zher(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - int N, double alpha, const void* X, int incX, - void* A, int lda); -void cblas_zhpr(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - int N, double alpha, const void* X, - int incX, void* A); -void cblas_zher2(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, int N, - const void* alpha, const void* X, int incX, - const void* Y, int incY, void* A, int lda); -void cblas_zhpr2(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, int N, - const void* alpha, const void* X, int incX, - const void* Y, int incY, void* Ap); - -/* -* =========================================================================== -* Prototypes for level 3 BLAS -* =========================================================================== -*/ - -/* -* Routines with standard 4 prefixes (S, D, C, Z) -*/ -void cblas_sgemm(enum CBLAS_ORDER Order, enum CBLAS_TRANSPOSE TransA, - enum CBLAS_TRANSPOSE TransB, int M, int N, - int K, float alpha, const float* A, - int lda, const float* B, int ldb, - float beta, float* C, int ldc); -void cblas_ssymm(enum CBLAS_ORDER Order, enum CBLAS_SIDE Side, - enum CBLAS_UPLO Uplo, int M, int N, - float alpha, const float* A, int lda, - const float* B, int ldb, float beta, - float* C, int ldc); -void cblas_ssyrk(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - enum CBLAS_TRANSPOSE Trans, int N, int K, - float alpha, const float* A, int lda, - float beta, float* C, int ldc); -void cblas_ssyr2k(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - enum CBLAS_TRANSPOSE Trans, int N, int K, - float alpha, const float* A, int lda, - const float* B, int ldb, float beta, - float* C, int ldc); -void cblas_strmm(enum CBLAS_ORDER Order, enum CBLAS_SIDE Side, - enum CBLAS_UPLO Uplo, enum CBLAS_TRANSPOSE TransA, - enum CBLAS_DIAG Diag, int M, int N, - float alpha, const float* A, int lda, - float* B, int ldb); -void cblas_strsm(enum CBLAS_ORDER Order, enum CBLAS_SIDE Side, - enum CBLAS_UPLO Uplo, enum CBLAS_TRANSPOSE TransA, - enum CBLAS_DIAG Diag, int M, int N, - float alpha, const float* A, int lda, - float* B, int ldb); - -void cblas_dgemm(enum CBLAS_ORDER Order, enum CBLAS_TRANSPOSE TransA, - enum CBLAS_TRANSPOSE TransB, int M, int N, - int K, double alpha, const double* A, - int lda, const double* B, int ldb, - double beta, double* C, int ldc); -void cblas_dsymm(enum CBLAS_ORDER Order, enum CBLAS_SIDE Side, - enum CBLAS_UPLO Uplo, int M, int N, - double alpha, const double* A, int lda, - const double* B, int ldb, double beta, - double* C, int ldc); -void cblas_dsyrk(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - enum CBLAS_TRANSPOSE Trans, int N, int K, - double alpha, const double* A, int lda, - double beta, double* C, int ldc); -void cblas_dsyr2k(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - enum CBLAS_TRANSPOSE Trans, int N, int K, - double alpha, const double* A, int lda, - const double* B, int ldb, double beta, - double* C, int ldc); -void cblas_dtrmm(enum CBLAS_ORDER Order, enum CBLAS_SIDE Side, - enum CBLAS_UPLO Uplo, enum CBLAS_TRANSPOSE TransA, - enum CBLAS_DIAG Diag, int M, int N, - double alpha, const double* A, int lda, - double* B, int ldb); -void cblas_dtrsm(enum CBLAS_ORDER Order, enum CBLAS_SIDE Side, - enum CBLAS_UPLO Uplo, enum CBLAS_TRANSPOSE TransA, - enum CBLAS_DIAG Diag, int M, int N, - double alpha, const double* A, int lda, - double* B, int ldb); - -void cblas_cgemm(enum CBLAS_ORDER Order, enum CBLAS_TRANSPOSE TransA, - enum CBLAS_TRANSPOSE TransB, int M, int N, - int K, const void* alpha, const void* A, - int lda, const void* B, int ldb, - const void* beta, void* C, int ldc); -void cblas_csymm(enum CBLAS_ORDER Order, enum CBLAS_SIDE Side, - enum CBLAS_UPLO Uplo, int M, int N, - const void* alpha, const void* A, int lda, - const void* B, int ldb, const void* beta, - void* C, int ldc); -void cblas_csyrk(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - enum CBLAS_TRANSPOSE Trans, int N, int K, - const void* alpha, const void* A, int lda, - const void* beta, void* C, int ldc); -void cblas_csyr2k(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - enum CBLAS_TRANSPOSE Trans, int N, int K, - const void* alpha, const void* A, int lda, - const void* B, int ldb, const void* beta, - void* C, int ldc); -void cblas_ctrmm(enum CBLAS_ORDER Order, enum CBLAS_SIDE Side, - enum CBLAS_UPLO Uplo, enum CBLAS_TRANSPOSE TransA, - enum CBLAS_DIAG Diag, int M, int N, - const void* alpha, const void* A, int lda, - void* B, int ldb); -void cblas_ctrsm(enum CBLAS_ORDER Order, enum CBLAS_SIDE Side, - enum CBLAS_UPLO Uplo, enum CBLAS_TRANSPOSE TransA, - enum CBLAS_DIAG Diag, int M, int N, - const void* alpha, const void* A, int lda, - void* B, int ldb); - -void cblas_zgemm(enum CBLAS_ORDER Order, enum CBLAS_TRANSPOSE TransA, - enum CBLAS_TRANSPOSE TransB, int M, int N, - int K, const void* alpha, const void* A, - int lda, const void* B, int ldb, - const void* beta, void* C, int ldc); -void cblas_zsymm(enum CBLAS_ORDER Order, enum CBLAS_SIDE Side, - enum CBLAS_UPLO Uplo, int M, int N, - const void* alpha, const void* A, int lda, - const void* B, int ldb, const void* beta, - void* C, int ldc); -void cblas_zsyrk(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - enum CBLAS_TRANSPOSE Trans, int N, int K, - const void* alpha, const void* A, int lda, - const void* beta, void* C, int ldc); -void cblas_zsyr2k(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - enum CBLAS_TRANSPOSE Trans, int N, int K, - const void* alpha, const void* A, int lda, - const void* B, int ldb, const void* beta, - void* C, int ldc); -void cblas_ztrmm(enum CBLAS_ORDER Order, enum CBLAS_SIDE Side, - enum CBLAS_UPLO Uplo, enum CBLAS_TRANSPOSE TransA, - enum CBLAS_DIAG Diag, int M, int N, - const void* alpha, const void* A, int lda, - void* B, int ldb); -void cblas_ztrsm(enum CBLAS_ORDER Order, enum CBLAS_SIDE Side, - enum CBLAS_UPLO Uplo, enum CBLAS_TRANSPOSE TransA, - enum CBLAS_DIAG Diag, int M, int N, - const void* alpha, const void* A, int lda, - void* B, int ldb); - -/* -* Routines with prefixes C and Z only -*/ -void cblas_chemm(enum CBLAS_ORDER Order, enum CBLAS_SIDE Side, - enum CBLAS_UPLO Uplo, int M, int N, - const void* alpha, const void* A, int lda, - const void* B, int ldb, const void* beta, - void* C, int ldc); -void cblas_cherk(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - enum CBLAS_TRANSPOSE Trans, int N, int K, - float alpha, const void* A, int lda, - float beta, void* C, int ldc); -void cblas_cher2k(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - enum CBLAS_TRANSPOSE Trans, int N, int K, - const void* alpha, const void* A, int lda, - const void* B, int ldb, float beta, - void* C, int ldc); -void cblas_zhemm(enum CBLAS_ORDER Order, enum CBLAS_SIDE Side, - enum CBLAS_UPLO Uplo, int M, int N, - const void* alpha, const void* A, int lda, - const void* B, int ldb, const void* beta, - void* C, int ldc); -void cblas_zherk(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - enum CBLAS_TRANSPOSE Trans, int N, int K, - double alpha, const void* A, int lda, - double beta, void* C, int ldc); -void cblas_zher2k(enum CBLAS_ORDER Order, enum CBLAS_UPLO Uplo, - enum CBLAS_TRANSPOSE Trans, int N, int K, - const void* alpha, const void* A, int lda, - const void* B, int ldb, double beta, - void* C, int ldc); - -#endif /* end #ifdef CBLAS_ENUM_ONLY */ -#endif diff --git a/onnxruntime/core/graph/graph_utils.cc b/onnxruntime/core/graph/graph_utils.cc index b91c0d4caf..8c63e60eac 100644 --- a/onnxruntime/core/graph/graph_utils.cc +++ b/onnxruntime/core/graph/graph_utils.cc @@ -565,6 +565,26 @@ const Node* FirstChildByType(const Node& node, const std::string& child_type) { return nullptr; } +std::vector FindChildrenByType(const Node& node, const std::string& child_type) { + // find children and sort them by source argument index: + // Create a 2D vector to hold the result. + // 1st dimension index is output index, + // and the 2nd dimension stores the edges from the output. + std::vector> children(node.OutputDefs().size(), std::vector()); + for (auto it = node.OutputEdgesBegin(); it != node.OutputEdgesEnd(); it++) { + if (it->GetNode().OpType().compare(child_type) == 0) { + children[it->GetSrcArgIndex()].push_back(&(it->GetNode())); + } + } + + // aggregate children + std::vector agg_res; + for (size_t output_idx = 0; output_idx < children.size(); output_idx++) { + agg_res.insert(agg_res.end(), children[output_idx].begin(), children[output_idx].end()); + } + return agg_res; +} + const Node* FirstParentByType(const Node& node, const std::string& parent_type) { for (auto it = node.InputNodesBegin(); it != node.InputNodesEnd(); ++it) { if ((*it).OpType().compare(parent_type) == 0) { @@ -574,6 +594,22 @@ const Node* FirstParentByType(const Node& node, const std::string& parent_type) return nullptr; } +std::vector FindParentsByType(const Node& node, const std::string& parent_type) { + // find parents and sort them by destination argument index + // as there is at most one input edge for each input argument, + // there is no need of extra work like FindChildrenByType + std::vector parents(node.InputDefs().size(), nullptr); + for (auto it = node.InputEdgesBegin(); it != node.InputEdgesEnd(); it++) { + if (it->GetNode().OpType().compare(parent_type) == 0) { + parents[it->GetDstArgIndex()] = &(it->GetNode()); + } + } + + // remove unmatched nodes + parents.erase(std::remove(parents.begin(), parents.end(), nullptr), parents.end()); + return parents; +} + NodeArg& AddInitializer(Graph& graph, const ONNX_NAMESPACE::TensorProto& new_initializer) { // sanity check as AddInitializedTensor silently ignores attempts to add a duplicate initializer const ONNX_NAMESPACE::TensorProto* existing = nullptr; diff --git a/onnxruntime/core/graph/graph_utils.h b/onnxruntime/core/graph/graph_utils.h index 19c588caff..bc6e4ad3da 100644 --- a/onnxruntime/core/graph/graph_utils.h +++ b/onnxruntime/core/graph/graph_utils.h @@ -100,9 +100,20 @@ bool GetRepeatedNodeAttributeValues(const Node& node, /** Find the first child of the specified op type. */ const Node* FirstChildByType(const Node& node, const std::string& child_type); + +/** Find node children by op types. + @returns The matched children are sorted by source argument index of their corresponding edge. +**/ +std::vector FindChildrenByType(const Node& node, const std::string& child_type); + /** Find the first parent of the specified op type. */ const Node* FirstParentByType(const Node& node, const std::string& parent_type); +/** Find node parents by op types. + @returns The matched parents are sorted by destination argument index of their corresponding edge. +**/ +std::vector FindParentsByType(const Node& node, const std::string& parent_type); + /** Tests if we can remove a node and merge its input edge (if any) with its output edges. Conditions: Input rules: diff --git a/onnxruntime/core/optimizer/graph_transformer_utils.cc b/onnxruntime/core/optimizer/graph_transformer_utils.cc index 016fff30e4..b9bd4afb42 100644 --- a/onnxruntime/core/optimizer/graph_transformer_utils.cc +++ b/onnxruntime/core/optimizer/graph_transformer_utils.cc @@ -37,6 +37,7 @@ #include "core/optimizer/skip_layer_norm_fusion.h" #include "core/optimizer/slice_elimination.h" #include "core/optimizer/unsqueeze_elimination.h" +#include "core/optimizer/qdq_transformer/qdq_transformer.h" #include "core/session/onnxruntime_session_options_config_keys.h" #include "core/optimizer/matmul_transpose_fusion.h" #include "core/optimizer/bias_dropout_fusion.h" @@ -143,6 +144,10 @@ std::vector> GenerateTransformers(TransformerL transformers.emplace_back(onnxruntime::make_unique(cpu_execution_providers)); transformers.emplace_back(onnxruntime::make_unique(cpu_execution_providers)); + if (enable_quant_qdq) { + transformers.emplace_back(onnxruntime::make_unique()); + } + std::unordered_set cpu_acl_execution_providers = {onnxruntime::kCpuExecutionProvider, onnxruntime::kAclExecutionProvider}; std::unordered_set cpu_cuda_rocm_acl_armnn_execution_providers = {onnxruntime::kCpuExecutionProvider, onnxruntime::kCudaExecutionProvider, onnxruntime::kRocmExecutionProvider, onnxruntime::kAclExecutionProvider, onnxruntime::kArmNNExecutionProvider}; diff --git a/onnxruntime/core/optimizer/qdq_transformer/qdq_binary_op.cc b/onnxruntime/core/optimizer/qdq_transformer/qdq_binary_op.cc new file mode 100644 index 0000000000..c0a8dc6906 --- /dev/null +++ b/onnxruntime/core/optimizer/qdq_transformer/qdq_binary_op.cc @@ -0,0 +1,46 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include + +#include "core/graph/graph.h" +#include "core/optimizer/qdq_transformer/qdq_op_transformer.h" +#include "core/optimizer/qdq_transformer/registry.h" + +namespace onnxruntime { +class QDQBinaryOpTransformer : public QDQOperatorTransformer { + public: + QDQBinaryOpTransformer(Node& node, Graph& graph) : QDQOperatorTransformer(node, graph) {} + + bool Transform(const std::vector& parents, const std::vector& children) override { + if (children.size() != 1 || parents.size() != 2) { + return false; + } + + FillQDQOptionalZeroPoint(parents); + FillQDQOptionalZeroPoint(children); + + std::vector input_defs(graph_.GetNode(parents[0]->Index())->MutableInputDefs()); + Node* b = graph_.GetNode(parents[1]->Index()); + input_defs.insert(input_defs.end(), b->MutableInputDefs().begin(), b->MutableInputDefs().end()); + + Node* q = graph_.GetNode(children[0]->Index()); + input_defs.push_back(q->MutableInputDefs()[1]); + input_defs.push_back(q->MutableInputDefs()[2]); + + Node& qlinear_conv_node = graph_.AddNode(node_.Name(), + "QLinear" + node_.OpType(), + node_.Description(), + input_defs, + q->MutableOutputDefs(), + &node_.GetAttributes(), + kMSDomain); + qlinear_conv_node.SetExecutionProviderType(kCpuExecutionProvider); + return true; + } +}; + +DEFINE_QDQ_CREATOR(Add, QDQBinaryOpTransformer) +DEFINE_QDQ_CREATOR(Mul, QDQBinaryOpTransformer) + +} // namespace onnxruntime diff --git a/onnxruntime/core/optimizer/qdq_transformer/qdq_conv.cc b/onnxruntime/core/optimizer/qdq_transformer/qdq_conv.cc new file mode 100644 index 0000000000..78a110159e --- /dev/null +++ b/onnxruntime/core/optimizer/qdq_transformer/qdq_conv.cc @@ -0,0 +1,49 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include + +#include "core/graph/graph.h" +#include "core/optimizer/qdq_transformer/qdq_op_transformer.h" +#include "core/optimizer/qdq_transformer/registry.h" + +namespace onnxruntime { +class QDQConvTransformer : public QDQOperatorTransformer { + public: + QDQConvTransformer(Node& node, Graph& graph) : QDQOperatorTransformer(node, graph) {} + + bool Transform(const std::vector& parents, const std::vector& children) override { + if (parents.size() < 2 || children.size() != 1) { + return false; + } + + FillQDQOptionalZeroPoint(parents); + FillQDQOptionalZeroPoint(children); + + std::vector input_defs(graph_.GetNode(parents[0]->Index())->MutableInputDefs()); + Node* weight = graph_.GetNode(parents[1]->Index()); + input_defs.insert(input_defs.end(), weight->MutableInputDefs().begin(), weight->MutableInputDefs().end()); + + Node* q = graph_.GetNode(children[0]->Index()); + input_defs.push_back(q->MutableInputDefs()[1]); + input_defs.push_back(q->MutableInputDefs()[2]); + if (parents.size() == 3) { + input_defs.push_back(graph_.GetNode(parents[2]->Index())->MutableInputDefs()[0]); + } + + Node& qlinear_conv_node = graph_.AddNode(node_.Name(), + "QLinearConv", + node_.Description(), + input_defs, + q->MutableOutputDefs(), + &node_.GetAttributes(), + kOnnxDomain); + qlinear_conv_node.SetExecutionProviderType(kCpuExecutionProvider); + + return true; + } +}; + +DEFINE_QDQ_CREATOR(Conv, QDQConvTransformer) + +} // namespace onnxruntime diff --git a/onnxruntime/core/optimizer/qdq_transformer/qdq_matmul.cc b/onnxruntime/core/optimizer/qdq_transformer/qdq_matmul.cc new file mode 100644 index 0000000000..6b7a578556 --- /dev/null +++ b/onnxruntime/core/optimizer/qdq_transformer/qdq_matmul.cc @@ -0,0 +1,44 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include + +#include "core/graph/graph.h" +#include "core/optimizer/qdq_transformer/qdq_op_transformer.h" +#include "core/optimizer/qdq_transformer/registry.h" + +namespace onnxruntime { +class QDQMatMulTransformer : public QDQOperatorTransformer { + public: + QDQMatMulTransformer(Node& node, Graph& graph) : QDQOperatorTransformer(node, graph) {} + + bool Transform(const std::vector& parents, const std::vector& children) override { + if (children.size() != 1) { + return false; + } + + FillQDQOptionalZeroPoint(parents); + FillQDQOptionalZeroPoint(children); + + std::vector input_defs(graph_.GetNode(parents[0]->Index())->MutableInputDefs()); + Node* b = graph_.GetNode(parents[1]->Index()); + input_defs.insert(input_defs.end(), b->MutableInputDefs().begin(), b->MutableInputDefs().end()); + + Node* q = graph_.GetNode(children[0]->Index()); + input_defs.push_back(q->MutableInputDefs()[1]); + input_defs.push_back(q->MutableInputDefs()[2]); + + Node& qlinear_conv_node = graph_.AddNode(node_.Name(), + "QLinearMatMul", + node_.Description(), + input_defs, + q->MutableOutputDefs(), + &node_.GetAttributes(), + kOnnxDomain); + qlinear_conv_node.SetExecutionProviderType(kCpuExecutionProvider); + return true; + } +}; + +DEFINE_QDQ_CREATOR(MatMul, QDQMatMulTransformer) +} // namespace onnxruntime diff --git a/onnxruntime/core/optimizer/qdq_transformer/qdq_op_transformer.cc b/onnxruntime/core/optimizer/qdq_transformer/qdq_op_transformer.cc new file mode 100644 index 0000000000..b5f565c418 --- /dev/null +++ b/onnxruntime/core/optimizer/qdq_transformer/qdq_op_transformer.cc @@ -0,0 +1,60 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "qdq_op_transformer.h" + +#include +#include + +#include "core/graph/graph.h" +#include "core/graph/onnx_protobuf.h" + +namespace onnxruntime { + +void QDQOperatorTransformer::FillQDQOptionalZeroPoint(const std::vector& qdq_nodes) { + for (const Node* p_node_const : qdq_nodes) { + Node& node = *graph_.GetNode(p_node_const->Index()); + std::vector& input_defs = node.MutableInputDefs(); + constexpr size_t max_input_count = 3; + if (input_defs.size() == max_input_count) { + continue; // zero point is not optional. No need to fill. + } + + bool is_default_zp_signed = false; + if (node.OpType() == DQOPTypeName) { + auto input_type = input_defs[0]->TypeAsProto()->tensor_type().elem_type(); + is_default_zp_signed = ONNX_NAMESPACE::TensorProto_DataType_INT8 == input_type; + } + + const ONNX_NAMESPACE::TensorProto& zp_tensor_proto = is_default_zp_signed ? optional_zero_point_int8_ : optional_zero_point_uint8_; + + const ONNX_NAMESPACE::TensorProto* dummy_zp_tensor_proto; + if (!graph_.GetInitializedTensor(zp_tensor_proto.name(), dummy_zp_tensor_proto)) { + graph_.AddInitializedTensor(zp_tensor_proto); + } + + input_defs.push_back(&graph_.GetOrCreateNodeArg(zp_tensor_proto.name(), nullptr)); + } +} + +const ONNX_NAMESPACE::TensorProto QDQOperatorTransformer::optional_zero_point_int8_ = []() { + const char* const name = "855dd0fa-cd7b-4b10-ae5a-df64cabfe1f8"; + ONNX_NAMESPACE::TensorProto tensor_proto; + tensor_proto.set_name(name); + tensor_proto.set_data_type(ONNX_NAMESPACE::TensorProto_DataType_INT8); + tensor_proto.set_raw_data(std::vector{0}.data(), sizeof(int8_t)); + + return tensor_proto; +}(); + +const ONNX_NAMESPACE::TensorProto QDQOperatorTransformer::optional_zero_point_uint8_ = []() { + const char* const name = "35b188f7-c464-43e3-8692-97ac832bb14a"; + ONNX_NAMESPACE::TensorProto tensor_proto; + tensor_proto.set_name(name); + tensor_proto.set_data_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); + tensor_proto.set_raw_data(std::vector{0}.data(), sizeof(uint8_t)); + + return tensor_proto; +}(); + +} // namespace onnxruntime diff --git a/onnxruntime/core/optimizer/qdq_transformer/qdq_op_transformer.h b/onnxruntime/core/optimizer/qdq_transformer/qdq_op_transformer.h new file mode 100644 index 0000000000..0b5a3fe7fb --- /dev/null +++ b/onnxruntime/core/optimizer/qdq_transformer/qdq_op_transformer.h @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include + +#include "core/graph/onnx_protobuf.h" + +namespace onnxruntime { + +static const char* const QOPTypeName = "QuantizeLinear"; +static const char* const DQOPTypeName = "DequantizeLinear"; + +class Node; +class Graph; + +class QDQOperatorTransformer { + public: + QDQOperatorTransformer(Node& node, Graph& graph) : node_(node), graph_(graph) {} + virtual ~QDQOperatorTransformer() {} + virtual bool Transform(const std::vector& dq_nodes, const std::vector& q_nodes) = 0; + + /* Determine whether to keep node_ itself or not. + For operators that support int8, keep node_ and only change its input and output. + Otherwise, node_ will be removed and replaced by a QLinear* version. + */ + virtual bool KeepNode() const { + return false; + } + + void FillQDQOptionalZeroPoint(const std::vector& parents); + + protected: + Node& node_; + Graph& graph_; + + static const ONNX_NAMESPACE::TensorProto optional_zero_point_int8_; + static const ONNX_NAMESPACE::TensorProto optional_zero_point_uint8_; +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/optimizer/qdq_transformer/qdq_simple_ops.cc b/onnxruntime/core/optimizer/qdq_transformer/qdq_simple_ops.cc new file mode 100644 index 0000000000..b0c2edaf9c --- /dev/null +++ b/onnxruntime/core/optimizer/qdq_transformer/qdq_simple_ops.cc @@ -0,0 +1,38 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include + +#include "core/graph/graph.h" +#include "core/optimizer/qdq_transformer/qdq_op_transformer.h" +#include "core/optimizer/qdq_transformer/registry.h" + +namespace onnxruntime { +class QDQSimpleTransformer : public QDQOperatorTransformer { + public: + QDQSimpleTransformer(Node& node, Graph& graph) : QDQOperatorTransformer(node, graph) {} + + bool Transform(const std::vector& parents, const std::vector& children) override { + if (parents.size() != 1 || children.size() != 1) { + return false; + } + + FillQDQOptionalZeroPoint(parents); + FillQDQOptionalZeroPoint(children); + + graph_.RemoveEdge(parents[0]->Index(), node_.Index(), 0, 0); + graph_.RemoveEdge(node_.Index(), children[0]->Index(), 0, 0); + + node_.MutableInputDefs()[0] = graph_.GetNode(parents[0]->Index())->MutableInputDefs()[0]; + node_.MutableOutputDefs()[0] = graph_.GetNode(children[0]->Index())->MutableOutputDefs()[0]; + return true; + } + + bool KeepNode() const override { + return true; + } +}; + +DEFINE_QDQ_CREATOR(MaxPool, QDQSimpleTransformer) +DEFINE_QDQ_CREATOR(Reshape, QDQSimpleTransformer) +} // namespace onnxruntime diff --git a/onnxruntime/core/optimizer/qdq_transformer/qdq_transformer.cc b/onnxruntime/core/optimizer/qdq_transformer/qdq_transformer.cc new file mode 100644 index 0000000000..6a3da050eb --- /dev/null +++ b/onnxruntime/core/optimizer/qdq_transformer/qdq_transformer.cc @@ -0,0 +1,90 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include +#include +#include "core/graph/graph_utils.h" +#include "core/optimizer/initializer.h" +#include "core/optimizer/qdq_transformer/qdq_transformer.h" +#include "core/optimizer/qdq_transformer/qdq_op_transformer.h" +#include "core/optimizer/qdq_transformer/registry.h" +#include "core/optimizer/utils.h" + +using namespace ONNX_NAMESPACE; +using namespace ::onnxruntime::common; +namespace onnxruntime { + +class QDQTransformerImpl { + public: + QDQTransformerImpl(Graph& graph) noexcept : graph_(graph) {} + + void Transform(Node& node) { + // extract DequantizeLinear from parents and QuantizeLinear in children + std::vector parents = graph_utils::FindParentsByType(node, DQOPTypeName); + std::vector children = graph_utils::FindChildrenByType(node, QOPTypeName); + + if (parents.size() == 0) { + return; + } + + // track dq output edges count + for (auto parent_node : parents) { + if (!dq_output_edges_count_.count(parent_node)) { + dq_output_edges_count_[parent_node] = parent_node->GetOutputEdgesCount(); + } + } + + std::unique_ptr op_trans = QDQRegistry::CreateQDQTransformer(node, graph_); + + if (op_trans && op_trans->Transform(parents, children)) { + for (auto parent_node : parents) { + dq_output_edges_count_[parent_node]--; + } + + UpdateNodesToRemove(parents); + UpdateNodesToRemove(children); + if (!op_trans->KeepNode()) { + nodes_to_remove_.insert(node.Index()); + } + } + } + void Finalize(bool& modified) { + for (auto node_idx : nodes_to_remove_) { + graph_utils::RemoveNodeOutputEdges(graph_, *graph_.GetNode(node_idx)); + graph_.RemoveNode(node_idx); + } + modified = true; + } + + private: + void UpdateNodesToRemove(const std::vector& nodes) { + for (auto node : nodes) { + if (dq_output_edges_count_[node] == 0 && !nodes_to_remove_.count(node->Index())) { + nodes_to_remove_.insert(node->Index()); + } + } + } + + Graph& graph_; + + std::unordered_map dq_output_edges_count_; + + std::set nodes_to_remove_; +}; + +Status QDQTransformer::ApplyImpl(Graph& graph, bool& modified, int graph_level, const logging::Logger& logger) const { + QDQTransformerImpl impl(graph); + GraphViewer graph_viewer(graph); + + for (auto index : graph_viewer.GetNodesInTopologicalOrder()) { + auto& node = *graph.GetNode(index); + ORT_RETURN_IF_ERROR(Recurse(node, modified, graph_level, logger)); + if (node.GetExecutionProviderType() == kCpuExecutionProvider) { + impl.Transform(node); + } + } + impl.Finalize(modified); + return Status::OK(); +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/optimizer/qdq_transformer/qdq_transformer.h b/onnxruntime/core/optimizer/qdq_transformer/qdq_transformer.h new file mode 100644 index 0000000000..448b4d9d79 --- /dev/null +++ b/onnxruntime/core/optimizer/qdq_transformer/qdq_transformer.h @@ -0,0 +1,24 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/optimizer/graph_transformer.h" + +namespace onnxruntime { + +/** +@Class QDQTransformer + +Transformer that fuse QDQ and fp32 op to quantized op. +*/ +class QDQTransformer : public GraphTransformer { + public: + QDQTransformer() noexcept : GraphTransformer("QDQTransformer") {} + + private: + Status ApplyImpl(Graph& graph, bool& modified, int graph_level, const logging::Logger& logger) const override; +}; + +} // namespace onnxruntime diff --git a/onnxruntime/core/optimizer/qdq_transformer/registry.cc b/onnxruntime/core/optimizer/qdq_transformer/registry.cc new file mode 100644 index 0000000000..93a2357540 --- /dev/null +++ b/onnxruntime/core/optimizer/qdq_transformer/registry.cc @@ -0,0 +1,21 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "registry.h" + +namespace onnxruntime { +DECLARE_QDQ_CREATOR(Conv, QDQConvTransformer) +DECLARE_QDQ_CREATOR(MaxPool, QDQSimpleTransformer) +DECLARE_QDQ_CREATOR(Reshape, QDQSimpleTransformer) +DECLARE_QDQ_CREATOR(Add, QDQBinaryOpTransformer) +DECLARE_QDQ_CREATOR(Mul, QDQBinaryOpTransformer) +DECLARE_QDQ_CREATOR(MatMul, QDQMatMulTransformer) +std::unordered_map QDQRegistry::qdqtransformer_creators_{ + REGISTER_QDQ_CREATOR(Conv, QDQConvTransformer), + REGISTER_QDQ_CREATOR(MaxPool, QDQSimpleTransformer), + REGISTER_QDQ_CREATOR(Reshape, QDQSimpleTransformer), + REGISTER_QDQ_CREATOR(Add, QDQBinaryOpTransformer), + REGISTER_QDQ_CREATOR(Mul, QDQBinaryOpTransformer), + REGISTER_QDQ_CREATOR(MatMul, QDQMatMulTransformer), +}; +} // namespace onnxruntime diff --git a/onnxruntime/core/optimizer/qdq_transformer/registry.h b/onnxruntime/core/optimizer/qdq_transformer/registry.h new file mode 100644 index 0000000000..cf9c2c8466 --- /dev/null +++ b/onnxruntime/core/optimizer/qdq_transformer/registry.h @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include +#include +#include +#include +#include + +#include "core/graph/graph.h" +#include "core/optimizer/qdq_transformer/qdq_op_transformer.h" + +namespace onnxruntime { + +class QDQRegistry { + public: + using QDQTransformerCreator = std::function(Node&, Graph&)>; + static bool Register(const std::string& op_type, QDQTransformerCreator creator) { + if (qdqtransformer_creators_.count(op_type)) { + return false; + } + + qdqtransformer_creators_[op_type] = creator; + return true; + } + + static std::unique_ptr CreateQDQTransformer(Node& node, Graph& graph) { + auto it = qdqtransformer_creators_.find(node.OpType()); + if (it != qdqtransformer_creators_.end()) + return (it->second)(node, graph); + + return std::unique_ptr(); + } + + private: + static std::unordered_map qdqtransformer_creators_; +}; + +#define QDQ_CREATOR_BUILDER_NAME(op_type, Transformer) Register_##op_type##_qdq_##Transformer + +#define DECLARE_QDQ_CREATOR(op_type, Transformer) \ + std ::pair QDQ_CREATOR_BUILDER_NAME(op_type, Transformer)(); + +#define DEFINE_QDQ_CREATOR(op_type, Transformer) \ + std::pair QDQ_CREATOR_BUILDER_NAME(op_type, Transformer)() { \ + return std::pair( \ + #op_type, \ + [](Node& node, Graph& graph) { return std::make_unique(node, graph); }); \ + } + +#define REGISTER_QDQ_CREATOR(op_type, Transformer) \ + QDQ_CREATOR_BUILDER_NAME(op_type, Transformer) \ + () +} // namespace onnxruntime diff --git a/onnxruntime/core/providers/cuda/tensor/compress_impl.cu b/onnxruntime/core/providers/cuda/tensor/compress_impl.cu index 6f936e5965..a3212cdf9e 100644 --- a/onnxruntime/core/providers/cuda/tensor/compress_impl.cu +++ b/onnxruntime/core/providers/cuda/tensor/compress_impl.cu @@ -1,6 +1,8 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. +#include + #include "core/providers/cuda/cu_inc/common.cuh" #include "core/providers/cuda/cuda_common.h" @@ -10,7 +12,6 @@ #endif #include "core/providers/cuda/tensor/compress_impl.h" -#include namespace onnxruntime { namespace cuda { diff --git a/onnxruntime/core/util/math.h b/onnxruntime/core/util/math.h index 54bc6c29d5..1bacbf2d3a 100644 --- a/onnxruntime/core/util/math.h +++ b/onnxruntime/core/util/math.h @@ -16,18 +16,26 @@ #pragma once -// This is a simple translation from the old Caffe math interfaces. We aim to -// still keep it simple, so all platforms would be able to support it fairly -// easily. - -// We include the cblas header here so that we can obtain the macros from cblas. -extern "C" { -#include "core/framework/cblas.h" -} - #include "core/common/common.h" #include "core/framework/tensor.h" +#ifndef CBLAS_ENUM_DEFINED_H +#define CBLAS_ENUM_DEFINED_H +enum CBLAS_ORDER { CblasRowMajor = 101, + CblasColMajor = 102 }; +enum CBLAS_TRANSPOSE { + CblasNoTrans = 111, + CblasTrans = 112, + CblasConjTrans = 113 +}; +enum CBLAS_UPLO { CblasUpper = 121, + CblasLower = 122 }; +enum CBLAS_DIAG { CblasNonUnit = 131, + CblasUnit = 132 }; +enum CBLAS_SIDE { CblasLeft = 141, + CblasRight = 142 }; +#endif + namespace onnxruntime { namespace concurrency { class ThreadPool; diff --git a/onnxruntime/python/tools/quantization/onnx_quantizer.py b/onnxruntime/python/tools/quantization/onnx_quantizer.py index 5d8657e80a..19f50712f4 100644 --- a/onnxruntime/python/tools/quantization/onnx_quantizer.py +++ b/onnxruntime/python/tools/quantization/onnx_quantizer.py @@ -573,9 +573,16 @@ class ONNXQuantizer: packed_bias_scale_initializer = onnx.numpy_helper.from_array(bias_scale_data, quantized_bias_scale_name) self.model.initializer().extend([packed_bias_scale_initializer]) + # update zero initializer + quantized_bias_zp_name = quantized_bias_name + "_zero_point" + bias_zp_data = np.zeros(bias_scale.shape, dtype=np.int32).reshape(-1) + packed_bias_zp_initializer = onnx.numpy_helper.from_array(bias_zp_data, quantized_bias_zp_name) + self.model.initializer().extend([packed_bias_zp_initializer]) + assert (bias_name not in self.quantized_value_map) - quantized_value = QuantizedValue(bias_name, quantized_bias_name, quantized_bias_scale_name, "", - QuantizedValueType.Initializer, 0 if bias_scale_data.size > 1 else None) + quantized_value = QuantizedValue(bias_name, quantized_bias_name, quantized_bias_scale_name, + quantized_bias_zp_name, QuantizedValueType.Initializer, + 0 if bias_scale_data.size > 1 else None) self.quantized_value_map[bias_name] = quantized_value return quantized_bias_name diff --git a/onnxruntime/test/common/tensor_op_test_utils.h b/onnxruntime/test/common/tensor_op_test_utils.h index 98a0175598..2380a67242 100644 --- a/onnxruntime/test/common/tensor_op_test_utils.h +++ b/onnxruntime/test/common/tensor_op_test_utils.h @@ -10,6 +10,7 @@ #include "core/common/common.h" #include "core/common/optional.h" +#include "core/common/type_utils.h" #include "core/util/math.h" namespace onnxruntime { @@ -52,7 +53,7 @@ class RandomValueGenerator { // Random values generated are in the range [min, max). template typename std::enable_if< - std::is_integral::value, + std::is_integral::value && !utils::IsByteType::value, std::vector>::type Uniform(const std::vector& dims, TInt min, TInt max) { std::vector val(detail::SizeFromDims(dims)); @@ -63,6 +64,19 @@ class RandomValueGenerator { return val; } + template + typename std::enable_if< + utils::IsByteType::value, + std::vector>::type + Uniform(const std::vector& dims, TByte min, TByte max) { + std::vector val(detail::SizeFromDims(dims)); + std::uniform_int_distribution distribution(min, max - 1); + for (size_t i = 0; i < val.size(); ++i) { + val[i] = static_cast(distribution(generator_)); + } + return val; + } + // Gaussian distribution for float template typename std::enable_if< diff --git a/onnxruntime/test/optimizer/graph_transform_test_builder.cc b/onnxruntime/test/optimizer/graph_transform_test_builder.cc new file mode 100644 index 0000000000..11c11cd206 --- /dev/null +++ b/onnxruntime/test/optimizer/graph_transform_test_builder.cc @@ -0,0 +1,84 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include +#include +#include + +#include "core/graph/model.h" +#include "core/session/inference_session.h" +#include "test/compare_ortvalue.h" +#include "test/test_environment.h" +#include "test/util/include/inference_session_wrapper.h" + +#include "graph_transform_test_builder.h" + +namespace onnxruntime { +namespace test { + +void TransformerTester(const std::function& build_test_case, + const std::function& check_transformed_graph, + TransformerLevel baseline_level, + TransformerLevel target_level, + int opset_version) { + // Build the model for this test. + std::unordered_map domain_to_version; + domain_to_version[kOnnxDomain] = opset_version; + domain_to_version[kMSDomain] = 1; + Model model("TransformerTester", false, ModelMetaData(), PathString(), IOnnxRuntimeOpSchemaRegistryList(), + domain_to_version, {}, DefaultLoggingManager().DefaultLogger()); + Graph& graph = model.MainGraph(); + ModelTestBuilder helper(graph); + build_test_case(helper); + ASSERT_TRUE(model.MainGraph().Resolve().IsOK()); + + // Serialize the model to a string. + std::string model_data; + model.ToProto().SerializeToString(&model_data); + + auto run_model = [&](TransformerLevel level, std::vector& fetches) { + SessionOptions session_options; + session_options.graph_optimization_level = level; + InferenceSessionWrapper session{session_options, GetEnvironment()}; + ASSERT_TRUE(session.Load(model_data.data(), static_cast(model_data.size())).IsOK()); + ASSERT_TRUE(session.Initialize().IsOK()); + + RunOptions run_options; + auto status = session.Run(run_options, + helper.feeds_, + helper.output_names_, + &fetches); + if (!status.IsOK()) { + std::cout << "Run failed with status message: " << status.ErrorMessage() << std::endl; + } + ASSERT_TRUE(status.IsOK()); + + if (level == target_level) { + check_transformed_graph(session); + } + }; + + std::vector baseline_fetches; + run_model(baseline_level, baseline_fetches); + + std::vector target_fetches; + run_model(target_level, target_fetches); + + size_t num_outputs = baseline_fetches.size(); + ASSERT_TRUE(num_outputs == target_fetches.size()); + + for (size_t i = 0; i < num_outputs; i++) { + double per_sample_tolerance = 0.0; + double relative_per_sample_tolerance = 0.0; + std::pair ret = + CompareOrtValue(target_fetches[i], + baseline_fetches[i], + per_sample_tolerance, + relative_per_sample_tolerance, + false); + EXPECT_EQ(ret.first, COMPARE_RESULT::SUCCESS) << ret.second; + } +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/optimizer/graph_transform_test_builder.h b/onnxruntime/test/optimizer/graph_transform_test_builder.h new file mode 100644 index 0000000000..f00d5a8cda --- /dev/null +++ b/onnxruntime/test/optimizer/graph_transform_test_builder.h @@ -0,0 +1,246 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. +#pragma once + +#include +#include +#include + +#include "core/common/type_utils.h" +#include "core/graph/graph.h" +#include "core/framework/framework_common.h" +#include "core/optimizer/graph_transformer_level.h" +#include "core/graph/onnx_protobuf.h" +#include "test/framework/test_utils.h" +#include "test/common/tensor_op_test_utils.h" +#include "test/framework/test_utils.h" +#include "test/util/include/inference_session_wrapper.h" + +namespace onnxruntime { +namespace test { +template +struct IsTypeQuantLinearCompatible : utils::IsByteType {}; + +template +struct IsTypeDequantLinearCompatible : utils::IsByteType {}; + +template <> +struct IsTypeDequantLinearCompatible : std::true_type {}; + +class ModelTestBuilder { + public: + ModelTestBuilder(Graph& graph) : graph_(graph) { + } + + template + NodeArg* MakeInput(const std::vector& shape, T min, T max) { + ONNX_NAMESPACE::TypeProto type_proto; + type_proto.mutable_tensor_type()->set_elem_type(utils::ToTensorProtoElementType()); + + for (auto& dim : shape) { + type_proto.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(dim); + } + + OrtValue input_value; + CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), + shape, + rand_gen_.Uniform(shape, min, max), + &input_value); + std::string name = graph_.GenerateNodeArgName("input"); + feeds_.insert(std::make_pair(name, input_value)); + + return &graph_.GetOrCreateNodeArg(name, &type_proto); + } + + NodeArg* MakeOutput() { + std::string name = graph_.GenerateNodeArgName("output"); + output_names_.push_back(name); + return &graph_.GetOrCreateNodeArg(name, nullptr); + } + + NodeArg* MakeIntermediate() { + std::string name = graph_.GenerateNodeArgName("node"); + return &graph_.GetOrCreateNodeArg(name, nullptr); + } + + template + NodeArg* MakeInitializer(const std::vector& shape, const std::vector& data) { + std::string name = graph_.GenerateNodeArgName("constant"); + ONNX_NAMESPACE::TensorProto tensor_proto; + tensor_proto.set_name(name); + tensor_proto.set_data_type(utils::ToTensorProtoElementType()); + tensor_proto.set_raw_data(data.data(), data.size() * sizeof(T)); + + for (auto& dim : shape) { + tensor_proto.add_dims(dim); + } + + graph_.AddInitializedTensor(tensor_proto); + + return &graph_.GetOrCreateNodeArg(name, nullptr); + } + + template + NodeArg* MakeInitializer(const std::vector& shape, T min, T max) { + return MakeInitializer(shape, rand_gen_.Uniform(shape, min, max)); + } + + template + NodeArg* MakeScalarInitializer(T data) { + return MakeInitializer({}, std::vector{data}); + } + + template + NodeArg* Make1DInitializer(const std::vector& data) { + return MakeInitializer({static_cast(data.size())}, data); + } + + Node& AddNode(const std::string& op_type, + const std::vector& input_args, + const std::vector& output_args, + const std::string& domain = "") { + return graph_.AddNode(graph_.GenerateNodeName("node"), + op_type, + "description", + input_args, + output_args, + nullptr, + domain); + } + + Node& AddConvNode(NodeArg* input_arg, + NodeArg* weights_arg, + NodeArg* output_arg) { + std::vector input_args; + input_args.push_back(input_arg); + input_args.push_back(weights_arg); + + return AddNode("Conv", input_args, {output_arg}); + } + + template + typename std::enable_if::value, Node&>::type + AddQuantizeLinearNode(NodeArg* input_arg, + float input_scale, + T input_zero_point, + NodeArg* output_arg) { + std::vector input_args; + input_args.push_back(input_arg); + input_args.push_back(MakeScalarInitializer(input_scale)); + input_args.push_back(MakeScalarInitializer(input_zero_point)); + + return AddNode("QuantizeLinear", input_args, {output_arg}); + } + + Node& AddQuantizeLinearNode(NodeArg* input_arg, + float input_scale, + NodeArg* output_arg) { + std::vector input_args; + input_args.push_back(input_arg); + input_args.push_back(MakeScalarInitializer(input_scale)); + + return AddNode("QuantizeLinear", input_args, {output_arg}); + } + + template + typename std::enable_if::value, Node&>::type + AddDequantizeLinearNode(NodeArg* input_arg, + float input_scale, + T input_zero_point, + NodeArg* output_arg) { + std::vector input_args; + input_args.push_back(input_arg); + input_args.push_back(MakeScalarInitializer(input_scale)); + input_args.push_back(MakeScalarInitializer(input_zero_point)); + + return AddNode("DequantizeLinear", input_args, {output_arg}); + } + + template + typename std::enable_if::value, Node&>::type + AddDequantizeLinearNode(NodeArg* input_arg, + float input_scale, + NodeArg* output_arg) { + std::vector input_args; + input_args.push_back(input_arg); + input_args.push_back(MakeScalarInitializer(input_scale)); + + return AddNode("DequantizeLinear", input_args, {output_arg}); + } + + template + Node& AddQLinearConvNode(NodeArg* input_arg, + float input_scale, + uint8_t input_zero_point, + NodeArg* weight_arg, + float weights_scale, + TWeight weights_zero_point, + NodeArg* output_arg, + float output_scale, + uint8_t output_zero_point) { + std::vector input_args{input_arg}; + input_args.push_back(MakeScalarInitializer(input_scale)); + input_args.push_back(MakeScalarInitializer(input_zero_point)); + input_args.push_back(weight_arg); + input_args.push_back(MakeScalarInitializer(weights_scale)); + input_args.push_back(MakeScalarInitializer(weights_zero_point)); + input_args.push_back(MakeScalarInitializer(output_scale)); + input_args.push_back(MakeScalarInitializer(output_zero_point)); + + return AddNode("QLinearConv", input_args, {output_arg}); + } + + Node& AddQLinearBinaryNode(const std::string& op_type, + NodeArg* input1_arg, + float input1_scale, + uint8_t input1_zero_point, + NodeArg* input2_arg, + float input2_scale, + uint8_t input2_zero_point, + NodeArg* output_arg, + float output_scale, + uint8_t output_zero_point) { + std::vector input_args; + input_args.push_back(input1_arg); + input_args.push_back(MakeScalarInitializer(input1_scale)); + input_args.push_back(MakeScalarInitializer(input1_zero_point)); + input_args.push_back(input2_arg); + input_args.push_back(MakeScalarInitializer(input2_scale)); + input_args.push_back(MakeScalarInitializer(input2_zero_point)); + input_args.push_back(MakeScalarInitializer(output_scale)); + input_args.push_back(MakeScalarInitializer(output_zero_point)); + + return AddNode(op_type, input_args, {output_arg}, kMSDomain); + } + + Node& AddQLinearActivationNode(const std::string& op_type, + NodeArg* input_arg, + float input_scale, + uint8_t input_zero_point, + NodeArg* output_arg, + float output_scale, + uint8_t output_zero_point) { + std::vector input_args; + input_args.push_back(input_arg); + input_args.push_back(MakeScalarInitializer(input_scale)); + input_args.push_back(MakeScalarInitializer(input_zero_point)); + input_args.push_back(MakeScalarInitializer(output_scale)); + input_args.push_back(MakeScalarInitializer(output_zero_point)); + + return AddNode(op_type, input_args, {output_arg}, kMSDomain); + } + + Graph& graph_; + NameMLValMap feeds_; + std::vector output_names_; + RandomValueGenerator rand_gen_{optional{2345}}; +}; + +void TransformerTester(const std::function& build_test_case, + const std::function& check_transformed_graph, + TransformerLevel baseline_level, + TransformerLevel target_level, + int opset_version = 12); + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/optimizer/nhwc_transformer_test.cc b/onnxruntime/test/optimizer/nhwc_transformer_test.cc index 7b383a0f79..474b6f1c7e 100644 --- a/onnxruntime/test/optimizer/nhwc_transformer_test.cc +++ b/onnxruntime/test/optimizer/nhwc_transformer_test.cc @@ -2,17 +2,12 @@ // Licensed under the MIT License. #include -#include "core/graph/model.h" -#include "core/graph/onnx_protobuf.h" -#include "core/mlas/inc/mlas.h" -#include "core/session/environment.h" -#include "core/session/inference_session.h" -#include "test/compare_ortvalue.h" -#include "test/test_environment.h" -#include "test/framework/test_utils.h" -#include "test/util/include/inference_session_wrapper.h" +#include #include "gtest/gtest.h" +#include "graph_transform_test_builder.h" + +#include "core/graph/graph.h" namespace onnxruntime { namespace test { @@ -30,282 +25,25 @@ struct NhwcWeightsRange { static constexpr int8_t max_value = +63; }; -struct NhwcTestHelper { - NhwcTestHelper(Graph& graph) : graph_(graph) { - } - - template - NodeArg* MakeInput(const std::vector& shape, const ONNX_NAMESPACE::TypeProto& type_proto) { - OrtValue input_value; - CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), shape, - FillRandomData(shape, 0, 31), &input_value); - std::string name = graph_.GenerateNodeArgName("input"); - feeds_.insert(std::make_pair(name, input_value)); - - return &graph_.GetOrCreateNodeArg(name, &type_proto); - } - - template - NodeArg* MakeInput(const std::vector& shape) { - ONNX_NAMESPACE::TypeProto type_proto; - type_proto.mutable_tensor_type()->set_elem_type(utils::ToTensorProtoElementType()); - - for (auto& dim : shape) { - type_proto.mutable_tensor_type()->mutable_shape()->add_dim()->set_dim_value(dim); - } - - return MakeInput(shape, type_proto); - } - - NodeArg* MakeOutput() { - std::string name = graph_.GenerateNodeArgName("output"); - output_names_.push_back(name); - return &graph_.GetOrCreateNodeArg(name, nullptr); - } - - NodeArg* MakeIntermediate() { - std::string name = graph_.GenerateNodeArgName("node"); - return &graph_.GetOrCreateNodeArg(name, nullptr); - } - - template - NodeArg* MakeInitializer(const std::vector& shape, const std::vector& data) { - std::string name = graph_.GenerateNodeArgName("constant"); - ONNX_NAMESPACE::TensorProto tensor_proto; - tensor_proto.set_name(name); - tensor_proto.set_data_type(utils::ToTensorProtoElementType()); - tensor_proto.set_raw_data(data.data(), data.size() * sizeof(T)); - - for (auto& dim : shape) { - tensor_proto.add_dims(dim); - } - - graph_.AddInitializedTensor(tensor_proto); - - return &graph_.GetOrCreateNodeArg(name, nullptr); - } - - template - NodeArg* MakeInitializer(const std::vector& shape, int32_t min_value, int32_t max_value) { - return MakeInitializer(shape, FillRandomData(shape, min_value, max_value)); - } - - template - NodeArg* MakeScalarInitializer(T data) { - return MakeInitializer({}, std::vector{data}); - } - - template - NodeArg* Make1DInitializer(const std::vector& data) { - return MakeInitializer({static_cast(data.size())}, data); - } - - template - NodeArg* MakeWeightsInitializer(const std::vector& shape) { - return MakeInitializer(shape, NhwcWeightsRange::min_value, NhwcWeightsRange::max_value); - } - - Node& AddNode(const std::string& op_type, - const std::vector& input_args, - const std::vector& output_args, - const std::string& domain = "") { - return graph_.AddNode(graph_.GenerateNodeName("node"), - op_type, - "description", - input_args, - output_args, - nullptr, - domain); - } - - Node& AddQLinearConvNode(NodeArg* input_arg, - NodeArg* input_scale_arg, - NodeArg* input_zero_point_arg, - NodeArg* weights_arg, - NodeArg* weights_scale_arg, - NodeArg* weights_zero_point_arg, - NodeArg* output_arg, - NodeArg* output_scale, - NodeArg* output_zero_point) { - std::vector input_args; - input_args.push_back(input_arg); - input_args.push_back(input_scale_arg); - input_args.push_back(input_zero_point_arg); - input_args.push_back(weights_arg); - input_args.push_back(weights_scale_arg); - input_args.push_back(weights_zero_point_arg); - input_args.push_back(output_scale); - input_args.push_back(output_zero_point); - - return AddNode("QLinearConv", input_args, {output_arg}); - } - - template - Node& AddQLinearConvNode(NodeArg* input_arg, - float input_scale, - uint8_t input_zero_point, - const std::vector& weights_shape, - float weights_scale, - TWeight weights_zero_point, - NodeArg* output_arg, - float output_scale, - uint8_t output_zero_point) { - return AddQLinearConvNode(input_arg, - MakeScalarInitializer(input_scale), - MakeScalarInitializer(input_zero_point), - MakeWeightsInitializer(weights_shape), - MakeScalarInitializer(weights_scale), - MakeScalarInitializer(weights_zero_point), - output_arg, - MakeScalarInitializer(output_scale), - MakeScalarInitializer(output_zero_point)); - } - - Node& AddQLinearBinaryNode(const std::string& op_type, - NodeArg* input1_arg, - float input1_scale, - uint8_t input1_zero_point, - NodeArg* input2_arg, - float input2_scale, - uint8_t input2_zero_point, - NodeArg* output_arg, - float output_scale, - uint8_t output_zero_point) { - std::vector input_args; - input_args.push_back(input1_arg); - input_args.push_back(MakeScalarInitializer(input1_scale)); - input_args.push_back(MakeScalarInitializer(input1_zero_point)); - input_args.push_back(input2_arg); - input_args.push_back(MakeScalarInitializer(input2_scale)); - input_args.push_back(MakeScalarInitializer(input2_zero_point)); - input_args.push_back(MakeScalarInitializer(output_scale)); - input_args.push_back(MakeScalarInitializer(output_zero_point)); - - return AddNode(op_type, input_args, {output_arg}, kMSDomain); - } - - Node& AddQLinearActivationNode(const std::string& op_type, - NodeArg* input_arg, - float input_scale, - uint8_t input_zero_point, - NodeArg* output_arg, - float output_scale, - uint8_t output_zero_point) { - std::vector input_args; - input_args.push_back(input_arg); - input_args.push_back(MakeScalarInitializer(input_scale)); - input_args.push_back(MakeScalarInitializer(input_zero_point)); - input_args.push_back(MakeScalarInitializer(output_scale)); - input_args.push_back(MakeScalarInitializer(output_zero_point)); - - return AddNode(op_type, input_args, {output_arg}, kMSDomain); - } - - Node& AddDequantizeLinearNode(NodeArg* input_arg, - float input_scale, - uint8_t input_zero_point, - NodeArg* output_arg) { - std::vector input_args; - input_args.push_back(input_arg); - input_args.push_back(MakeScalarInitializer(input_scale)); - input_args.push_back(MakeScalarInitializer(input_zero_point)); - - return AddNode("DequantizeLinear", input_args, {output_arg}); - } - - template - std::vector FillRandomData(size_t count, int32_t min_value, int32_t max_value) { - std::vector random_data; - random_data.resize(count); - std::uniform_int_distribution distribution(min_value, max_value); - for (size_t n = 0; n < count; n++) { - random_data[n] = static_cast(distribution(generator_)); - } - return random_data; - } - - template - std::vector FillRandomData(const std::vector& shape, int32_t min_value, int32_t max_value) { - int64_t num_elements = std::accumulate(shape.begin(), shape.end(), int64_t(1), std::multiplies{}); - return FillRandomData(static_cast(num_elements), min_value, max_value); - } - - Graph& graph_; - NameMLValMap feeds_; - std::vector output_names_; - std::default_random_engine generator_{2345}; -}; - -void NhwcTransformerTester(const std::function& build_test_case, - const std::function& check_nhwc_graph, - int opset_version = 12) { - // Build the model for this test. - std::unordered_map domain_to_version; - domain_to_version[kOnnxDomain] = opset_version; - domain_to_version[kMSDomain] = 1; - Model model("nhwc", false, ModelMetaData(), PathString(), IOnnxRuntimeOpSchemaRegistryList(), - domain_to_version, {}, DefaultLoggingManager().DefaultLogger()); - NhwcTestHelper helper(model.MainGraph()); - build_test_case(helper); - ASSERT_TRUE(model.MainGraph().Resolve().IsOK()); - - // Serialize the model to a string. - std::string model_data; - model.ToProto().SerializeToString(&model_data); - - auto run_model = [&](TransformerLevel level, std::vector& fetches) { - SessionOptions session_options; - session_options.graph_optimization_level = level; - session_options.session_logid = "NhwcTransformerTests"; - InferenceSessionWrapper session{session_options, GetEnvironment()}; - ASSERT_TRUE(session.Load(model_data.data(), static_cast(model_data.size())).IsOK()); - ASSERT_TRUE(session.Initialize().IsOK()); - - RunOptions run_options; - auto status = session.Run(run_options, helper.feeds_, helper.output_names_, &fetches); - if (!status.IsOK()) { - std::cout << "Run failed with status message: " << status.ErrorMessage() << std::endl; - } - ASSERT_TRUE(status.IsOK()); - - if (level == TransformerLevel::Level3) { - check_nhwc_graph(session); - } - }; - - std::vector level2_fetches; - run_model(TransformerLevel::Level2, level2_fetches); - - std::vector level3_fetches; - run_model(TransformerLevel::Level3, level3_fetches); - - size_t num_outputs = level2_fetches.size(); - ASSERT_TRUE(num_outputs == level3_fetches.size()); - - for (size_t i = 0; i < num_outputs; i++) { - double per_sample_tolerance = 0.0; - double relative_per_sample_tolerance = 0.0; - std::pair ret = - CompareOrtValue(level3_fetches[i], - level2_fetches[i], - per_sample_tolerance, - relative_per_sample_tolerance, - false); - EXPECT_EQ(ret.first, COMPARE_RESULT::SUCCESS) << ret.second; - } +template +NodeArg* NhwcMakeInitializer(ModelTestBuilder& builder, const std::vector& shape) { + return builder.MakeInitializer(shape, + NhwcWeightsRange::min_value, + NhwcWeightsRange::max_value); } #ifndef DISABLE_CONTRIB_OPS TEST(NhwcTransformerTests, Conv) { auto test_case = [&](const std::vector& input_shape, const std::vector& weights_shape) { - auto build_test_case = [&](NhwcTestHelper& helper) { - auto* input_arg = helper.MakeInput(input_shape); - auto* output_arg = helper.MakeOutput(); + auto build_test_case = [&](ModelTestBuilder& builder) { + auto* input_arg = builder.MakeInput(input_shape, 0, 31); + auto* output_arg = builder.MakeOutput(); + auto* weight_arg = NhwcMakeInitializer(builder, weights_shape); - helper.AddQLinearConvNode(input_arg, .01f, 135, - weights_shape, .02f, 126, - output_arg, .37f, 131); + builder.AddQLinearConvNode(input_arg, .01f, 135, + weight_arg, .02f, 126, + output_arg, .37f, 131); }; auto check_nhwc_graph = [&](InferenceSessionWrapper& session) { @@ -314,7 +52,10 @@ TEST(NhwcTransformerTests, Conv) { EXPECT_EQ(op_to_count["Transpose"], 2); }; - NhwcTransformerTester(build_test_case, check_nhwc_graph); + TransformerTester(build_test_case, + check_nhwc_graph, + TransformerLevel::Level2, + TransformerLevel::Level3); }; // Test the basic case of a single 1D/2D/3D convolution. @@ -324,15 +65,18 @@ TEST(NhwcTransformerTests, Conv) { } TEST(NhwcTransformerTests, ConvDequantizeLinear) { - auto build_test_case = [&](NhwcTestHelper& helper) { - auto* input_arg = helper.MakeInput({1, 12, 37}); - auto* conv_output_arg = helper.MakeIntermediate(); - auto* output_arg = helper.MakeOutput(); + auto build_test_case = [&](ModelTestBuilder& builder) { + auto* input_arg = builder.MakeInput({1, 12, 37}, 0, 31); + auto* conv_output_arg = builder.MakeIntermediate(); + auto* output_arg = builder.MakeOutput(); + auto* weight_arg = NhwcMakeInitializer(builder, {32, 12, 5}); - helper.AddQLinearConvNode(input_arg, .01f, 135, - {32, 12, 5}, .02f, 126, - conv_output_arg, .37f, 131); - helper.AddDequantizeLinearNode(conv_output_arg, .37f, 131, output_arg); + builder.AddQLinearConvNode(input_arg, .01f, 135, + weight_arg, .02f, 126, + conv_output_arg, .37f, 131); + builder.AddDequantizeLinearNode(conv_output_arg, + .37f, 131, + output_arg); }; auto check_nhwc_graph = [&](InferenceSessionWrapper& session) { @@ -344,28 +88,38 @@ TEST(NhwcTransformerTests, ConvDequantizeLinear) { // QLinearConv followed by only DequantizeLinear will remain as the ONNX // version of the operator to avoid adding unnecessary Transpose nodes to // the graph. - NhwcTransformerTester(build_test_case, check_nhwc_graph); + TransformerTester(build_test_case, + check_nhwc_graph, + TransformerLevel::Level2, + TransformerLevel::Level3); } TEST(NhwcTransformerTests, ConvBlockBinary) { auto test_case = [&](const std::string& binary_op_type) { - auto build_test_case = [&](NhwcTestHelper& helper) { - auto* input_arg = helper.MakeInput({1, 23, 13, 13}); - auto* conv1_output_arg = helper.MakeIntermediate(); - auto* conv2_output_arg = helper.MakeIntermediate(); - auto* output_arg = helper.MakeOutput(); + auto build_test_case = [&](ModelTestBuilder& builder) { + auto* input_arg = builder.MakeInput({1, 23, 13, 13}, 0, 31); + auto* conv1_output_arg = builder.MakeIntermediate(); + auto* conv2_output_arg = builder.MakeIntermediate(); + auto* output_arg = builder.MakeOutput(); + auto* conv1_weight_arg = builder.MakeInitializer({30, 23, 3, 3}, + NhwcWeightsRange::min_value, + NhwcWeightsRange::max_value); - Node& conv1_node = helper.AddQLinearConvNode(input_arg, .01f, 135, - {30, 23, 3, 3}, .02f, 126, - conv1_output_arg, .37f, 131); + auto* conv2_weight_arg = builder.MakeInitializer({30, 23, 1, 1}, + NhwcWeightsRange::min_value, + NhwcWeightsRange::max_value); + + Node& conv1_node = builder.AddQLinearConvNode(input_arg, .01f, 135, + conv1_weight_arg, .02f, 126, + conv1_output_arg, .37f, 131); conv1_node.AddAttribute("pads", std::vector{1, 1, 1, 1}); - helper.AddQLinearConvNode(input_arg, .01f, 135, - {30, 23, 1, 1}, .015f, 129, - conv2_output_arg, .37f, 131); - helper.AddQLinearBinaryNode(binary_op_type, - conv1_output_arg, .37f, 131, - conv2_output_arg, .37f, 131, - output_arg, .43f, 126); + builder.AddQLinearConvNode(input_arg, .01f, 135, + conv2_weight_arg, .015f, 129, + conv2_output_arg, .37f, 131); + builder.AddQLinearBinaryNode(binary_op_type, + conv1_output_arg, .37f, 131, + conv2_output_arg, .37f, 131, + output_arg, .43f, 126); }; auto check_nhwc_graph = [&](InferenceSessionWrapper& session) { @@ -374,7 +128,10 @@ TEST(NhwcTransformerTests, ConvBlockBinary) { EXPECT_EQ(op_to_count["Transpose"], 2); }; - NhwcTransformerTester(build_test_case, check_nhwc_graph); + TransformerTester(build_test_case, + check_nhwc_graph, + TransformerLevel::Level2, + TransformerLevel::Level3); }; std::vector activation_op_types{"QLinearAdd", "QLinearMul"}; @@ -385,15 +142,16 @@ TEST(NhwcTransformerTests, ConvBlockBinary) { TEST(NhwcTransformerTests, ConvMaxPool) { auto test_case = [&](const std::vector& input_shape, const std::vector& weights_shape) { - auto build_test_case = [&](NhwcTestHelper& helper) { - auto* input_arg = helper.MakeInput(input_shape); - auto* conv_output_arg = helper.MakeIntermediate(); - auto* output_arg = helper.MakeOutput(); + auto build_test_case = [&](ModelTestBuilder& builder) { + auto* input_arg = builder.MakeInput(input_shape, 0, 31); + auto* conv_output_arg = builder.MakeIntermediate(); + auto* output_arg = builder.MakeOutput(); + auto* conv_weight_arg = NhwcMakeInitializer(builder, weights_shape); - helper.AddQLinearConvNode(input_arg, .01f, 135, - weights_shape, .02f, 126, - conv_output_arg, .37f, 131); - Node& pool_node = helper.AddNode("MaxPool", {conv_output_arg}, {output_arg}); + builder.AddQLinearConvNode(input_arg, .01f, 135, + conv_weight_arg, .02f, 126, + conv_output_arg, .37f, 131); + Node& pool_node = builder.AddNode("MaxPool", {conv_output_arg}, {output_arg}); std::vector pads((weights_shape.size() - 2) * 2, 1); pool_node.AddAttribute("pads", pads); std::vector kernel_shape(weights_shape.size() - 2, 3); @@ -407,7 +165,10 @@ TEST(NhwcTransformerTests, ConvMaxPool) { EXPECT_EQ(op_to_count["Transpose"], 2); }; - NhwcTransformerTester(build_test_case, check_nhwc_graph); + TransformerTester(build_test_case, + check_nhwc_graph, + TransformerLevel::Level2, + TransformerLevel::Level3); }; // Test the basic case of a single 1D/2D/3D convolution. @@ -417,16 +178,17 @@ TEST(NhwcTransformerTests, ConvMaxPool) { } TEST(NhwcTransformerTests, ConvMaxPoolIndexTensor) { - auto build_test_case = [&](NhwcTestHelper& helper) { - auto* input_arg = helper.MakeInput({1, 16, 17, 17}); - auto* conv_output_arg = helper.MakeIntermediate(); - auto* index_output_arg = helper.MakeOutput(); - auto* output_arg = helper.MakeOutput(); + auto build_test_case = [&](ModelTestBuilder& builder) { + auto* input_arg = builder.MakeInput({1, 16, 17, 17}, 0, 31); + auto* conv_output_arg = builder.MakeIntermediate(); + auto* index_output_arg = builder.MakeOutput(); + auto* output_arg = builder.MakeOutput(); + auto* conv_weight_arg = NhwcMakeInitializer(builder, {16, 16, 3, 3}); - helper.AddQLinearConvNode(input_arg, .01f, 135, - {16, 16, 3, 3}, .02f, 126, - conv_output_arg, .37f, 131); - Node& pool_node = helper.AddNode("MaxPool", {conv_output_arg}, {output_arg, index_output_arg}); + builder.AddQLinearConvNode(input_arg, .01f, 135, + conv_weight_arg, .02f, 126, + conv_output_arg, .37f, 131); + Node& pool_node = builder.AddNode("MaxPool", {conv_output_arg}, {output_arg, index_output_arg}); pool_node.AddAttribute("kernel_shape", std::vector{3, 3}); }; @@ -438,32 +200,37 @@ TEST(NhwcTransformerTests, ConvMaxPoolIndexTensor) { }; // Test that MaxPool using the optional index tensor is not converted to NhwcMaxPool. - NhwcTransformerTester(build_test_case, check_nhwc_graph); + TransformerTester(build_test_case, + check_nhwc_graph, + TransformerLevel::Level2, + TransformerLevel::Level3); } TEST(NhwcTransformerTests, ConvGlobalAveragePool) { - auto build_test_case = [&](NhwcTestHelper& helper) { - auto* input_arg = helper.MakeInput({1, 23, 13, 13}); - auto* conv1_output_arg = helper.MakeIntermediate(); - auto* conv2_output_arg = helper.MakeIntermediate(); - auto* gavgpool1_output_arg = helper.MakeIntermediate(); - auto* gavgpool2_output_arg = helper.MakeIntermediate(); - auto* output_arg = helper.MakeOutput(); + auto build_test_case = [&](ModelTestBuilder& builder) { + auto* input_arg = builder.MakeInput({1, 23, 13, 13}, 0, 31); + auto* conv1_output_arg = builder.MakeIntermediate(); + auto* conv2_output_arg = builder.MakeIntermediate(); + auto* gavgpool1_output_arg = builder.MakeIntermediate(); + auto* gavgpool2_output_arg = builder.MakeIntermediate(); + auto* output_arg = builder.MakeOutput(); + auto* conv1_weight_arg = NhwcMakeInitializer(builder, {30, 23, 3, 3}); + auto* conv2_weight_arg = NhwcMakeInitializer(builder, {16, 30, 1, 1}); - Node& conv1_node = helper.AddQLinearConvNode(input_arg, .01f, 135, - {30, 23, 3, 3}, .02f, 126, - conv1_output_arg, .37f, 131); + Node& conv1_node = builder.AddQLinearConvNode(input_arg, .01f, 135, + conv1_weight_arg, .02f, 126, + conv1_output_arg, .37f, 131); conv1_node.AddAttribute("pads", std::vector{1, 1, 1, 1}); - helper.AddQLinearActivationNode("QLinearGlobalAveragePool", - conv1_output_arg, .37f, 131, - gavgpool1_output_arg, .43f, 111); - helper.AddQLinearConvNode(gavgpool1_output_arg, .43f, 111, - {16, 30, 1, 1}, .015f, 129, + builder.AddQLinearActivationNode("QLinearGlobalAveragePool", + conv1_output_arg, .37f, 131, + gavgpool1_output_arg, .43f, 111); + builder.AddQLinearConvNode(gavgpool1_output_arg, .43f, 111, + conv2_weight_arg, .015f, 129, conv2_output_arg, .37f, 131); - helper.AddQLinearActivationNode("QLinearGlobalAveragePool", - conv2_output_arg, .37f, 131, - gavgpool2_output_arg, .37f, 131); - helper.AddDequantizeLinearNode(gavgpool2_output_arg, .37f, 131, output_arg); + builder.AddQLinearActivationNode("QLinearGlobalAveragePool", + conv2_output_arg, .37f, 131, + gavgpool2_output_arg, .37f, 131); + builder.AddDequantizeLinearNode(gavgpool2_output_arg, .37f, 131, output_arg); }; auto check_nhwc_graph = [&](InferenceSessionWrapper& session) { @@ -472,35 +239,42 @@ TEST(NhwcTransformerTests, ConvGlobalAveragePool) { EXPECT_EQ(op_to_count["Transpose"], 2); }; - NhwcTransformerTester(build_test_case, check_nhwc_graph); + TransformerTester(build_test_case, + check_nhwc_graph, + TransformerLevel::Level2, + TransformerLevel::Level3); } TEST(NhwcTransformerTests, ConvSplit) { for (int64_t axis = -4LL; axis < 4; axis++) { - auto build_test_case = [&, axis](NhwcTestHelper& helper) { - auto* input_arg = helper.MakeInput({2, 23, 16, 16}); - auto* conv_output_arg = helper.MakeIntermediate(); - auto* split_output1_arg = helper.MakeIntermediate(); - auto* split_output2_arg = helper.MakeIntermediate(); - auto* qladd_output_arg = helper.MakeIntermediate(); - auto* output_arg = helper.MakeOutput(); + auto build_test_case = [&, axis](ModelTestBuilder& builder) { + auto* input_arg = builder.MakeInput({2, 23, 16, 16}, 0, 31); + auto* conv_output_arg = builder.MakeIntermediate(); + auto* split_output1_arg = builder.MakeIntermediate(); + auto* split_output2_arg = builder.MakeIntermediate(); + auto* qladd_output_arg = builder.MakeIntermediate(); + auto* output_arg = builder.MakeOutput(); const int64_t conv1_output_channels = 32; - Node& conv_node = helper.AddQLinearConvNode(input_arg, .01f, 135, - {conv1_output_channels, 23, 3, 3}, .02f, 126, - conv_output_arg, .37f, 131); + auto* conv1_weight_arg = NhwcMakeInitializer(builder, {conv1_output_channels, 23, 3, 3}); + + Node& conv_node = builder.AddQLinearConvNode(input_arg, .01f, 135, + conv1_weight_arg, .02f, 126, + conv_output_arg, .37f, 131); conv_node.AddAttribute("pads", std::vector{1, 1, 1, 1}); - Node& split_node = helper.AddNode("Split", {conv_output_arg}, {split_output1_arg, split_output2_arg}); + Node& split_node = builder.AddNode("Split", {conv_output_arg}, {split_output1_arg, split_output2_arg}); split_node.AddAttribute("axis", static_cast(axis)); - helper.AddQLinearBinaryNode("QLinearAdd", - split_output1_arg, .37f, 131, - split_output2_arg, .37f, 131, - qladd_output_arg, .43f, 126); + builder.AddQLinearBinaryNode("QLinearAdd", + split_output1_arg, .37f, 131, + split_output2_arg, .37f, 131, + qladd_output_arg, .43f, 126); const int64_t channels_after_split = - (axis == 1 || axis == -3) ? conv1_output_channels / 2 : conv1_output_channels; - helper.AddQLinearConvNode(qladd_output_arg, .43f, 126, - {17, channels_after_split, 3, 3}, .02f, 126, - output_arg, .37f, 131); + (axis == 1 || axis == -3) ? conv1_output_channels / 2 : conv1_output_channels; + + auto* conv2_weight_arg = NhwcMakeInitializer(builder, {17, channels_after_split, 3, 3}); + builder.AddQLinearConvNode(qladd_output_arg, .43f, 126, + conv2_weight_arg, .02f, 126, + output_arg, .37f, 131); }; auto check_nhwc_graph = [&](InferenceSessionWrapper& session) { @@ -509,32 +283,38 @@ TEST(NhwcTransformerTests, ConvSplit) { EXPECT_EQ(op_to_count["Transpose"], 2); }; - NhwcTransformerTester(build_test_case, check_nhwc_graph); + TransformerTester(build_test_case, + check_nhwc_graph, + TransformerLevel::Level2, + TransformerLevel::Level3); } } TEST(NhwcTransformerTests, ConvPad) { std::vector pad_modes{"constant", "reflect", "edge"}; for (const auto& mode : pad_modes) { - auto build_test_case = [&](NhwcTestHelper& helper) { - auto* input_arg = helper.MakeInput({1, 23, 13, 13}); - auto* conv1_output_arg = helper.MakeIntermediate(); - auto* pads_const = helper.MakeScalarInitializer(131); - auto* pads_arg = helper.Make1DInitializer({0, 0, 1, 2, 0, 0, 3, 4}); - auto* pad_output_arg = helper.MakeIntermediate(); - auto* conv2_output_arg = helper.MakeIntermediate(); - auto* output_arg = helper.MakeOutput(); + auto build_test_case = [&](ModelTestBuilder& builder) { + auto* input_arg = builder.MakeInput({1, 23, 13, 13}, 0, 31); + auto* conv1_output_arg = builder.MakeIntermediate(); + auto* pads_const = builder.MakeScalarInitializer(131); + auto* pads_arg = builder.Make1DInitializer({0, 0, 1, 2, 0, 0, 3, 4}); + auto* pad_output_arg = builder.MakeIntermediate(); + auto* conv2_output_arg = builder.MakeIntermediate(); + auto* output_arg = builder.MakeOutput(); - Node& conv1_node = helper.AddQLinearConvNode(input_arg, .01f, 135, - {30, 23, 3, 3}, .02f, 126, - conv1_output_arg, .37f, 131); + auto* conv1_weight_arg = NhwcMakeInitializer(builder, {30, 23, 3, 3}); + Node& conv1_node = builder.AddQLinearConvNode(input_arg, .01f, 135, + conv1_weight_arg, .02f, 126, + conv1_output_arg, .37f, 131); conv1_node.AddAttribute("pads", std::vector{1, 1, 1, 1}); - Node& pad_node = helper.AddNode("Pad", {conv1_output_arg, pads_arg, pads_const}, {pad_output_arg}); + Node& pad_node = builder.AddNode("Pad", {conv1_output_arg, pads_arg, pads_const}, {pad_output_arg}); pad_node.AddAttribute("mode", mode); - helper.AddQLinearConvNode(pad_output_arg, .37f, 131, - {16, 30, 3, 3}, .015f, 129, - conv2_output_arg, .37f, 131); - helper.AddDequantizeLinearNode(conv2_output_arg, .37f, 131, output_arg); + + auto* conv2_weight_arg = NhwcMakeInitializer(builder, {16, 30, 3, 3}); + builder.AddQLinearConvNode(pad_output_arg, .37f, 131, + conv2_weight_arg, .015f, 129, + conv2_output_arg, .37f, 131); + builder.AddDequantizeLinearNode(conv2_output_arg, .37f, 131, output_arg); }; auto check_nhwc_graph = [&](InferenceSessionWrapper& session) { @@ -543,59 +323,65 @@ TEST(NhwcTransformerTests, ConvPad) { EXPECT_EQ(op_to_count["Transpose"], 2); }; - NhwcTransformerTester(build_test_case, check_nhwc_graph); + TransformerTester(build_test_case, + check_nhwc_graph, + TransformerLevel::Level2, + TransformerLevel::Level3); } } TEST(NhwcTransformerTests, ConvBlockActivation) { auto test_case = [&](uint32_t extra_edges) { - auto build_test_case = [&](NhwcTestHelper& helper) { - auto* input1_arg = helper.MakeInput({1, 10, 13, 13}); - auto* input2_arg = helper.MakeInput({1, 13, 13, 13}); - auto* concat_arg = helper.MakeIntermediate(); - auto* conv1_output_arg = helper.MakeIntermediate(); - auto* conv2_output_arg = helper.MakeIntermediate(); - auto* act1_output_arg = helper.MakeIntermediate(); - auto* act2_output_arg = helper.MakeIntermediate(); - auto* output_arg = helper.MakeOutput(); + auto build_test_case = [&](ModelTestBuilder& builder) { + auto* input1_arg = builder.MakeInput({1, 10, 13, 13}, 0, 31); + auto* input2_arg = builder.MakeInput({1, 13, 13, 13}, 0, 31); + auto* concat_arg = builder.MakeIntermediate(); + auto* conv1_output_arg = builder.MakeIntermediate(); + auto* conv2_output_arg = builder.MakeIntermediate(); + auto* act1_output_arg = builder.MakeIntermediate(); + auto* act2_output_arg = builder.MakeIntermediate(); + auto* output_arg = builder.MakeOutput(); // Create a convolution input that isn't directly a graph input. - Node& concat_node = helper.AddNode("Concat", {input1_arg, input2_arg}, {concat_arg}); + Node& concat_node = builder.AddNode("Concat", {input1_arg, input2_arg}, {concat_arg}); concat_node.AddAttribute("axis", static_cast(1)); - Node& conv1_node = helper.AddQLinearConvNode(concat_arg, .01f, 135, - {30, 23, 3, 3}, .02f, 126, - conv1_output_arg, .37f, 131); + auto* conv1_weight_arg = NhwcMakeInitializer(builder, {30, 23, 3, 3}); + Node& conv1_node = builder.AddQLinearConvNode(concat_arg, .01f, 135, + conv1_weight_arg, .02f, 126, + conv1_output_arg, .37f, 131); conv1_node.AddAttribute("pads", std::vector{1, 1, 1, 1}); - helper.AddQLinearActivationNode("QLinearSigmoid", - conv1_output_arg, .37f, 131, - act1_output_arg, .37f, 131); - helper.AddQLinearConvNode(concat_arg, .01f, 135, - {30, 23, 1, 1}, .015f, 129, - conv2_output_arg, .37f, 131); - helper.AddQLinearActivationNode("QLinearLeakyRelu", - conv2_output_arg, .37f, 131, - act2_output_arg, .37f, 131); - helper.AddQLinearBinaryNode("QLinearAdd", - act1_output_arg, .37f, 131, - act2_output_arg, .37f, 131, - output_arg, .39f, 126); + builder.AddQLinearActivationNode("QLinearSigmoid", + conv1_output_arg, .37f, 131, + act1_output_arg, .37f, 131); + + auto* conv2_weight_arg = NhwcMakeInitializer(builder, {30, 23, 1, 1}); + builder.AddQLinearConvNode(concat_arg, .01f, 135, + conv2_weight_arg, .015f, 129, + conv2_output_arg, .37f, 131); + builder.AddQLinearActivationNode("QLinearLeakyRelu", + conv2_output_arg, .37f, 131, + act2_output_arg, .37f, 131); + builder.AddQLinearBinaryNode("QLinearAdd", + act1_output_arg, .37f, 131, + act2_output_arg, .37f, 131, + output_arg, .39f, 126); // Create extra uses of the various NodeArgs to exercise the transformer. if ((extra_edges & 1) != 0) { - helper.AddDequantizeLinearNode(concat_arg, .01f, 135, helper.MakeOutput()); + builder.AddDequantizeLinearNode(concat_arg, .01f, 135, builder.MakeOutput()); } if ((extra_edges & 2) != 0) { - helper.AddDequantizeLinearNode(conv1_output_arg, .37f, 131, helper.MakeOutput()); + builder.AddDequantizeLinearNode(conv1_output_arg, .37f, 131, builder.MakeOutput()); } if ((extra_edges & 4) != 0) { - helper.AddDequantizeLinearNode(conv2_output_arg, .37f, 131, helper.MakeOutput()); + builder.AddDequantizeLinearNode(conv2_output_arg, .37f, 131, builder.MakeOutput()); } if ((extra_edges & 8) != 0) { - helper.AddDequantizeLinearNode(act1_output_arg, .37f, 131, helper.MakeOutput()); + builder.AddDequantizeLinearNode(act1_output_arg, .37f, 131, builder.MakeOutput()); } if ((extra_edges & 16) != 0) { - helper.AddDequantizeLinearNode(act2_output_arg, .37f, 131, helper.MakeOutput()); + builder.AddDequantizeLinearNode(act2_output_arg, .37f, 131, builder.MakeOutput()); } }; @@ -604,7 +390,10 @@ TEST(NhwcTransformerTests, ConvBlockActivation) { EXPECT_EQ(op_to_count["com.microsoft.QLinearConv"], 2); }; - NhwcTransformerTester(build_test_case, check_nhwc_graph); + TransformerTester(build_test_case, + check_nhwc_graph, + TransformerLevel::Level2, + TransformerLevel::Level3); }; // Add extra uses of the edges that cause the transformer to insert additional @@ -615,24 +404,27 @@ TEST(NhwcTransformerTests, ConvBlockActivation) { } TEST(NhwcTransformerTests, ConvMixTensorRanks) { - auto build_test_case = [&](NhwcTestHelper& helper) { - auto* input1_arg = helper.MakeInput({1, 10, 7}); - auto* input2_arg = helper.MakeInput({1, 12, 7, 7}); - auto* conv1_output_arg = helper.MakeIntermediate(); - auto* conv2_output_arg = helper.MakeIntermediate(); - auto* output_arg = helper.MakeOutput(); + auto build_test_case = [&](ModelTestBuilder& builder) { + auto* input1_arg = builder.MakeInput({1, 10, 7}, 0, 31); + auto* input2_arg = builder.MakeInput({1, 12, 7, 7}, 0, 31); + auto* conv1_output_arg = builder.MakeIntermediate(); + auto* conv2_output_arg = builder.MakeIntermediate(); + auto* output_arg = builder.MakeOutput(); - helper.AddQLinearConvNode(input1_arg, .01f, 135, - {1, 10, 3}, .02f, 126, - conv1_output_arg, .37f, 131); - helper.AddQLinearConvNode(input2_arg, .01f, 135, - {1, 12, 3, 3}, .02f, 126, - conv2_output_arg, .37f, 131); + auto* conv1_weight_arg = NhwcMakeInitializer(builder, {1, 10, 3}); + builder.AddQLinearConvNode(input1_arg, .01f, 135, + conv1_weight_arg, .02f, 126, + conv1_output_arg, .37f, 131); + + auto* conv2_weight_arg = NhwcMakeInitializer(builder, {1, 12, 3, 3}); + builder.AddQLinearConvNode(input2_arg, .01f, 135, + conv2_weight_arg, .02f, 126, + conv2_output_arg, .37f, 131); // Broadcast add {1, 1, 5} to {1, 1, 5, 5}. - helper.AddQLinearBinaryNode("QLinearAdd", - conv1_output_arg, .37f, 131, - conv2_output_arg, .37f, 131, - output_arg, .39f, 126); + builder.AddQLinearBinaryNode("QLinearAdd", + conv1_output_arg, .37f, 131, + conv2_output_arg, .37f, 131, + output_arg, .39f, 126); }; auto check_nhwc_graph = [&](InferenceSessionWrapper& session) { @@ -643,10 +435,13 @@ TEST(NhwcTransformerTests, ConvMixTensorRanks) { // Generate a graph with QLinearAdd that broadcasts adds a 1D tensor to a // 2D tensor and verify that the transformer handles the mixed tensor ranks. - NhwcTransformerTester(build_test_case, check_nhwc_graph); + TransformerTester(build_test_case, + check_nhwc_graph, + TransformerLevel::Level2, + TransformerLevel::Level3); } -#endif // DISABLE_CONTRIB_OPS +#endif // DISABLE_CONTRIB_OPS } // namespace test } // namespace onnxruntime diff --git a/onnxruntime/test/optimizer/qdq_transformer_test.cc b/onnxruntime/test/optimizer/qdq_transformer_test.cc new file mode 100644 index 0000000000..d056a4b7b8 --- /dev/null +++ b/onnxruntime/test/optimizer/qdq_transformer_test.cc @@ -0,0 +1,237 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/graph/model.h" +#include "core/graph/onnx_protobuf.h" +#include "core/mlas/inc/mlas.h" +#include "core/session/environment.h" +#include "core/session/inference_session.h" +#include "test/compare_ortvalue.h" +#include "test/test_environment.h" +#include "test/framework/test_utils.h" +#include "test/util/include/inference_session_wrapper.h" + +#include "gtest/gtest.h" +#include "graph_transform_test_builder.h" + +namespace onnxruntime { +namespace test { + +template +typename std::enable_if::value, NodeArg*>::type +AddQDQNodePair(ModelTestBuilder& builder, NodeArg* q_input, float scale, T zp) { + auto* q_output = builder.MakeIntermediate(); + auto* dq_output = builder.MakeIntermediate(); + builder.AddQuantizeLinearNode(q_input, scale, zp, q_output); + builder.AddDequantizeLinearNode(q_output, scale, zp, dq_output); + return dq_output; +} + +template +typename std::enable_if::value, NodeArg*>::type +AddQDQNodePair(ModelTestBuilder& builder, NodeArg* q_input, float scale) { + auto* q_output = builder.MakeIntermediate(); + auto* dq_output = builder.MakeIntermediate(); + builder.AddQuantizeLinearNode(q_input, scale, q_output); + builder.AddDequantizeLinearNode(q_output, scale, dq_output); + return dq_output; +} + +#ifndef DISABLE_CONTRIB_OPS + +TEST(QDQTransformerTests, Conv) { + // TODO: enable fully use_default_zp tests after fixing inference bug in QuantizeLinear + auto test_case = [&](const std::vector& input_shape, const std::vector& weights_shape, bool use_default_zp) { + auto build_test_case = [&](ModelTestBuilder& builder) { + auto* input_arg = builder.MakeInput(input_shape, -1.f, 1.f); + auto* output_arg = builder.MakeOutput(); + + auto* conv_output = builder.MakeIntermediate(); + auto* weight = builder.MakeInitializer(weights_shape, 0, 255); + + auto* dq_w_output = builder.MakeIntermediate(); + auto* dq_output = AddQDQNodePair(builder, input_arg, .004f, 129); + + if (use_default_zp) { + builder.AddDequantizeLinearNode(weight, .003f, dq_w_output); + } else { + builder.AddDequantizeLinearNode(weight, .003f, 118, dq_w_output); + } + + builder.AddConvNode(dq_output, dq_w_output, conv_output); + builder.AddQuantizeLinearNode(conv_output, .0039f, 135, output_arg); + }; + + auto check_conv_graph = [&](InferenceSessionWrapper& session) { + auto op_to_count = CountOpsInGraph(session.GetGraph()); + EXPECT_EQ(op_to_count["QLinearConv"], 1); + EXPECT_EQ(op_to_count["QuantizeLinear"], 1); + EXPECT_EQ(op_to_count["DequantizeLinear"], 0); + }; + + TransformerTester(build_test_case, check_conv_graph, TransformerLevel::Level1, TransformerLevel::Level2); + }; + + // Test the basic case of a single 1D/2D/3D convolution. + test_case({1, 12, 37}, {32, 12, 5}, true); + test_case({1, 12, 37}, {32, 12, 5}, false); + test_case({1, 23, 13, 13}, {30, 23, 3, 3}, true); + test_case({1, 23, 13, 13}, {30, 23, 3, 3}, false); + test_case({1, 22, 11, 13, 15}, {30, 22, 5, 3, 3}, true); + test_case({1, 22, 11, 13, 15}, {30, 22, 5, 3, 3}, false); +} + +TEST(QDQTransformerTests, ConvMaxPoolReshape) { + auto test_case = [&](const std::vector& input_shape, const std::vector& weights_shape) { + auto build_test_case = [&](ModelTestBuilder& builder) { + auto* input_arg = builder.MakeInput(input_shape, -1.f, 1.f); + auto* output_arg = builder.MakeOutput(); + auto* weight = builder.MakeInitializer(weights_shape, 0, 255); + + // add QDQ + Conv + auto* dq_w_output = builder.MakeIntermediate(); + auto* conv_output = builder.MakeIntermediate(); + auto* dq_conv_output = AddQDQNodePair(builder, input_arg, .004f, 129); + builder.AddDequantizeLinearNode(weight, .003f, 118, dq_w_output); + builder.AddConvNode(dq_conv_output, dq_w_output, conv_output); + + // add QDQ + MaxPool + auto* dq_maxpool_output = AddQDQNodePair(builder, conv_output, .0039f, 135); + auto* maxpool_output = builder.MakeIntermediate(); + Node& pool_node = builder.AddNode("MaxPool", {dq_maxpool_output}, {maxpool_output}); + std::vector pads((weights_shape.size() - 2) * 2, 1); + pool_node.AddAttribute("pads", pads); + std::vector kernel_shape(weights_shape.size() - 2, 3); + pool_node.AddAttribute("kernel_shape", kernel_shape); + + // add QDQ + Reshape + auto* dq_reshape_output = AddQDQNodePair(builder, maxpool_output, .0039f, 135); + auto* reshape_shape = builder.Make1DInitializer({-1}); + auto* reshape_output = builder.MakeIntermediate(); + builder.AddNode("Reshape", {dq_reshape_output, reshape_shape}, {reshape_output}); + + // add Q + builder.AddQuantizeLinearNode(reshape_output, .0039f, 135, output_arg); + }; + + auto check_mp_reshape_graph = [&](InferenceSessionWrapper& session) { + auto op_to_count = CountOpsInGraph(session.GetGraph()); + EXPECT_EQ(op_to_count["QLinearConv"], 1); + EXPECT_EQ(op_to_count["MaxPool"], 1); + EXPECT_EQ(op_to_count["Reshape"], 1); + EXPECT_EQ(op_to_count["QuantizeLinear"], 1); + EXPECT_EQ(op_to_count["DequantizeLinear"], 0); + }; + + TransformerTester(build_test_case, check_mp_reshape_graph, TransformerLevel::Level1, TransformerLevel::Level2); + }; + + // Test the basic case of a single 1D/2D/3D convolution. + test_case({1, 12, 37}, {32, 12, 5}); + test_case({1, 23, 13, 13}, {30, 23, 3, 3}); + test_case({1, 22, 11, 13, 15}, {30, 22, 5, 3, 3}); +} + +TEST(QDQTransformerTests, Add) { + auto test_case = [&](const std::vector& input_shape) { + auto build_test_case = [&](ModelTestBuilder& builder) { + auto* input1_arg = builder.MakeInput(input_shape, -1.f, 1.f); + auto* input2_arg = builder.MakeInput(input_shape, -1.f, 1.f); + auto* output_arg = builder.MakeOutput(); + + // add QDQ + Add + auto* add_output = builder.MakeIntermediate(); + auto* dq_add_output1 = AddQDQNodePair(builder, input1_arg, .004f, 129); + auto* dq_add_output2 = AddQDQNodePair(builder, input2_arg, .004f, 129); + builder.AddNode("Add", {dq_add_output1, dq_add_output2}, {add_output}); + + // add Q + builder.AddQuantizeLinearNode(add_output, .0039f, 135, output_arg); + }; + + auto check_add_graph = [&](InferenceSessionWrapper& session) { + auto op_to_count = CountOpsInGraph(session.GetGraph()); + EXPECT_EQ(op_to_count["com.microsoft.QLinearAdd"], 1); + EXPECT_EQ(op_to_count["QuantizeLinear"], 2); + EXPECT_EQ(op_to_count["DequantizeLinear"], 0); + }; + + TransformerTester(build_test_case, check_add_graph, TransformerLevel::Level1, TransformerLevel::Level2); + }; + + // Test the basic case of a single 1D/2D/3D convolution. + test_case({1, 12, 37}); + test_case({1, 23, 13, 13}); + test_case({1, 22, 11, 13, 15}); +} + +TEST(QDQTransformerTests, Mul) { + auto test_case = [&](const std::vector& input_shape) { + auto build_test_case = [&](ModelTestBuilder& builder) { + auto* input1_arg = builder.MakeInput(input_shape, -1.f, 1.f); + auto* input2_arg = builder.MakeInput(input_shape, -1.f, 1.f); + auto* output_arg = builder.MakeOutput(); + + // add QDQ + Mul + auto* mul_output = builder.MakeIntermediate(); + auto* dq_mul_output1 = AddQDQNodePair(builder, input1_arg, .004f, 129); + auto* dq_mul_output2 = AddQDQNodePair(builder, input2_arg, .004f, 129); + builder.AddNode("Mul", {dq_mul_output1, dq_mul_output2}, {mul_output}); + + // add Q + builder.AddQuantizeLinearNode(mul_output, .0039f, 135, output_arg); + }; + + auto check_mul_graph = [&](InferenceSessionWrapper& session) { + auto op_to_count = CountOpsInGraph(session.GetGraph()); + EXPECT_EQ(op_to_count["com.microsoft.QLinearMul"], 1); + EXPECT_EQ(op_to_count["QuantizeLinear"], 2); + EXPECT_EQ(op_to_count["DequantizeLinear"], 0); + }; + + TransformerTester(build_test_case, check_mul_graph, TransformerLevel::Level1, TransformerLevel::Level2); + }; + + // Test the basic case of a single 1D/2D/3D convolution. + test_case({1, 12, 37}); + test_case({1, 23, 13, 13}); + test_case({1, 22, 11, 13, 15}); +} + +TEST(QDQTransformerTests, MatMul) { + auto test_case = [&](const std::vector& input1_shape, const std::vector& input2_shape) { + auto build_test_case = [&](ModelTestBuilder& builder) { + auto* input1_arg = builder.MakeInput(input1_shape, -1.f, 1.f); + auto* input2_arg = builder.MakeInput(input2_shape, -1.f, 1.f); + auto* output_arg = builder.MakeOutput(); + + // add QDQ + MatMul + auto* matmul_output = builder.MakeIntermediate(); + auto* dq_matmul_output1 = AddQDQNodePair(builder, input1_arg, .004f, 129); + auto* dq_matmul_output2 = AddQDQNodePair(builder, input2_arg, .004f, 129); + builder.AddNode("MatMul", {dq_matmul_output1, dq_matmul_output2}, {matmul_output}); + + // add Q + builder.AddQuantizeLinearNode(matmul_output, .0039f, 135, output_arg); + }; + + auto check_matmul_graph = [&](InferenceSessionWrapper& session) { + auto op_to_count = CountOpsInGraph(session.GetGraph()); + EXPECT_EQ(op_to_count["QLinearMatMul"], 1); + EXPECT_EQ(op_to_count["QuantizeLinear"], 2); + EXPECT_EQ(op_to_count["DequantizeLinear"], 0); + }; + + TransformerTester(build_test_case, check_matmul_graph, TransformerLevel::Level1, TransformerLevel::Level2); + }; + + // Test the basic case of a single 1D/2D/3D convolution. + test_case({12, 37}, {37, 12}); + test_case({23, 13, 13}, {13, 13}); + test_case({22, 11, 13, 15}, {15, 13}); +} + +#endif // DISABLE_CONTRIB_OPS + +} // namespace test +} // namespace onnxruntime From 53c123dcee3660f3a44ce9519f51904bd045e763 Mon Sep 17 00:00:00 2001 From: Tianlei Wu Date: Thu, 25 Mar 2021 11:32:36 -0700 Subject: [PATCH 005/129] Add session option configuration to enable GeluApproximation (#7131) --- .../onnxruntime_session_options_config_keys.h | 3 ++ .../core/optimizer/graph_transformer_utils.cc | 19 ++++++------ .../test/optimizer/graph_transform_test.cc | 29 +++++++++++++++++++ .../optimizer/graph_transform_utils_test.cc | 22 -------------- 4 files changed, 41 insertions(+), 32 deletions(-) diff --git a/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h b/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h index 7212e119fb..fadac84688 100644 --- a/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h +++ b/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h @@ -47,6 +47,9 @@ static const char* const kOrtSessionOptionsConfigSetDenormalAsZero = "session.se // Its default value is "1" static const char* const kOrtSessionOptionsEnableQuantQDQ = "session.enable_quant_qdq"; +// Enable or disable gelu approximation in graph optimization. "0": disable; "1": enable. The default is "0". +static const char* const kOrtSessionOptionsEnableGeluApproximation = "optimization.enable_gelu_approximation"; + // Configure whether to allow the inter_op/intra_op threads spinning a number of times before blocking // "0": thread will block if found no job to run // "1": default, thread will spin a number of times before blocking diff --git a/onnxruntime/core/optimizer/graph_transformer_utils.cc b/onnxruntime/core/optimizer/graph_transformer_utils.cc index b9bd4afb42..94fbc62f89 100644 --- a/onnxruntime/core/optimizer/graph_transformer_utils.cc +++ b/onnxruntime/core/optimizer/graph_transformer_utils.cc @@ -119,7 +119,11 @@ std::vector> GenerateTransformers(TransformerL const std::vector& transformers_and_rules_to_enable) { std::vector> transformers; std::unique_ptr rule_transformer = nullptr; - bool enable_quant_qdq = session_options.GetConfigOrDefault(kOrtSessionOptionsEnableQuantQDQ, "1") == "1"; + bool enable_quant_qdq = session_options.GetConfigOrDefault(kOrtSessionOptionsEnableQuantQDQ, "1") == "1"; +#ifndef DISABLE_CONTRIB_OPS + bool enable_gelu_approximation = session_options.GetConfigOrDefault(kOrtSessionOptionsEnableGeluApproximation, "0") == "1"; +#endif + switch (level) { case TransformerLevel::Level1: { std::unordered_set l1_execution_providers = {}; @@ -169,6 +173,10 @@ std::vector> GenerateTransformers(TransformerL transformers.emplace_back(onnxruntime::make_unique(cpu_cuda_rocm_execution_providers)); + if (enable_gelu_approximation){ + transformers.emplace_back(onnxruntime::make_unique(cpu_cuda_rocm_execution_providers)); + } + transformers.emplace_back(onnxruntime::make_unique(cpu_cuda_rocm_execution_providers)); #endif } break; @@ -198,15 +206,6 @@ std::vector> GenerateTransformers(TransformerL return transformers; } -// Some transformers have side-effect like result is not exactly same. -// These transformers could only be enabled by custom transformer list. -#ifndef DISABLE_CONTRIB_OPS - if (level == TransformerLevel::Level2) { - std::unordered_set cuda_rocm_execution_providers = {onnxruntime::kCudaExecutionProvider, onnxruntime::kRocmExecutionProvider}; - transformers.emplace_back(onnxruntime::make_unique(cuda_rocm_execution_providers)); - } -#endif - std::vector> filtered_list; // If the rule-based transformer is not empty, it should be included in the custom transformer list below. if (rule_transformer != nullptr) { diff --git a/onnxruntime/test/optimizer/graph_transform_test.cc b/onnxruntime/test/optimizer/graph_transform_test.cc index 19dacf1b1c..4d5453a047 100644 --- a/onnxruntime/test/optimizer/graph_transform_test.cc +++ b/onnxruntime/test/optimizer/graph_transform_test.cc @@ -2324,6 +2324,35 @@ TEST_F(GraphTransformationTests, BiasGeluSwitchedInputOrder) { EXPECT_EQ(ret.first, COMPARE_RESULT::SUCCESS) << ret.second; } +static void VerifyGeluApproximation(bool is_enabled, SessionOptions& session_options) { + std::unique_ptr e = + onnxruntime::make_unique(CPUExecutionProviderInfo()); + + bool has_gelu_approximation = false; + auto transformers = optimizer_utils::GenerateTransformers(TransformerLevel::Level2, session_options, *e.get(), {}); + for (auto& transformer : transformers) { + if (transformer->Name() == "GeluApproximation") { + has_gelu_approximation = true; + } + } + + EXPECT_EQ(has_gelu_approximation, is_enabled); +} + +// Test session option configuration for GeluApproximation +TEST_F(GraphTransformationTests, GeluApproximation_SessionOptionConfig) { + SessionOptions session_options; + + // GeluApproximation is not enabled by default. + VerifyGeluApproximation(false, session_options); + + session_options.AddConfigEntry(kOrtSessionOptionsEnableGeluApproximation, "1"); + VerifyGeluApproximation(true, session_options); + + session_options.AddConfigEntry(kOrtSessionOptionsEnableGeluApproximation, "0"); + VerifyGeluApproximation(false, session_options); +} + // Test Gelu -> FastGelu TEST_F(GraphTransformationTests, GeluApproximation_Gelu) { auto model_uri = MODEL_FOLDER "approximation/gelu.onnx"; diff --git a/onnxruntime/test/optimizer/graph_transform_utils_test.cc b/onnxruntime/test/optimizer/graph_transform_utils_test.cc index 92e36a455c..46e5848413 100644 --- a/onnxruntime/test/optimizer/graph_transform_utils_test.cc +++ b/onnxruntime/test/optimizer/graph_transform_utils_test.cc @@ -63,27 +63,5 @@ TEST(GraphTransformerUtilsTests, TestGenerateGraphTransformers) { #endif } -TEST(GraphTransformerUtilsTests, TestCustomOnlyTransformers) { - // Transformers that are disabled by default. They can only be enabled by custom list. - std::string l2_transformer = "GeluApproximation"; - std::unique_ptr cpu_execution_provider = - onnxruntime::make_unique(CPUExecutionProviderInfo()); - - std::vector default_list = {}; - auto default_transformers = optimizer_utils::GenerateTransformers(TransformerLevel::Level2, {}, *cpu_execution_provider.get(), default_list); - for (auto& transformer : default_transformers) { - ASSERT_TRUE(transformer->Name() != l2_transformer); - } - - std::vector custom_list = {l2_transformer}; - auto custom_transformers = optimizer_utils::GenerateTransformers(TransformerLevel::Level2, {}, *cpu_execution_provider.get(), custom_list); -#ifndef DISABLE_CONTRIB_OPS - ASSERT_TRUE(custom_transformers.size() == 1); - ASSERT_TRUE(custom_transformers[0]->Name() == l2_transformer); -#else - ASSERT_TRUE(custom_transformers.size() == 0); -#endif -} - } // namespace test } // namespace onnxruntime From cc0e7bee76bbd4fece212182c8412ebbe28c425f Mon Sep 17 00:00:00 2001 From: "G. Ramalingam" Date: Thu, 25 Mar 2021 11:34:06 -0700 Subject: [PATCH 006/129] Add function-body to SoftmaxGrad (#6988) * Add function body to SoftmaxGrad schema * Add type context and cleanup * Add test case with symbolic dimensions * Add opset specification to function * handle opset dependence * Exclude from minimal build --- cmake/onnxruntime_graph.cmake | 3 +- .../graph/contrib_ops/onnx_function_util.cc | 52 ++++ .../graph/contrib_ops/onnx_function_util.h | 25 ++ onnxruntime/core/graph/function.cc | 7 +- onnxruntime/core/graph/graph.cc | 19 +- .../core/graph/training_op_defs.cc | 52 +++- .../test/gradient/function_ops_test.cc | 276 ++++++++++++++++++ 7 files changed, 427 insertions(+), 7 deletions(-) create mode 100644 onnxruntime/core/graph/contrib_ops/onnx_function_util.cc create mode 100644 onnxruntime/core/graph/contrib_ops/onnx_function_util.h create mode 100644 orttraining/orttraining/test/gradient/function_ops_test.cc diff --git a/cmake/onnxruntime_graph.cmake b/cmake/onnxruntime_graph.cmake index 315a66edd1..37ab9ac9ff 100644 --- a/cmake/onnxruntime_graph.cmake +++ b/cmake/onnxruntime_graph.cmake @@ -17,7 +17,8 @@ if (onnxruntime_MINIMAL_BUILD) "${ONNXRUNTIME_ROOT}/core/graph/schema_registry.cc" "${ONNXRUNTIME_ROOT}/core/graph/contrib_ops/*defs.h" "${ONNXRUNTIME_ROOT}/core/graph/contrib_ops/*defs.cc" - + "${ONNXRUNTIME_ROOT}/core/graph/contrib_ops/onnx_function_util.h" + "${ONNXRUNTIME_ROOT}/core/graph/contrib_ops/onnx_function_util.cc" ) # no Function support initially diff --git a/onnxruntime/core/graph/contrib_ops/onnx_function_util.cc b/onnxruntime/core/graph/contrib_ops/onnx_function_util.cc new file mode 100644 index 0000000000..dc4925fb0d --- /dev/null +++ b/onnxruntime/core/graph/contrib_ops/onnx_function_util.cc @@ -0,0 +1,52 @@ +#include "core/graph/contrib_ops/onnx_function_util.h" +#include "core/util/math.h" + +namespace ONNX_NAMESPACE { + +TensorProto ToTensor(double value, TensorProto_DataType elem_type) { + TensorProto t; + t.set_data_type(elem_type); + switch (elem_type) { + case TensorProto_DataType::TensorProto_DataType_FLOAT: + t.add_float_data((float)value); + break; + case TensorProto_DataType::TensorProto_DataType_DOUBLE: + t.add_double_data(value); + break; + case TensorProto_DataType::TensorProto_DataType_FLOAT16: + t.add_int32_data(onnxruntime::math::floatToHalf((float)value)); + break; + default: + assert(false); + } + + return t; +} + +void BuildNodes(FunctionProto& functionProto, const std::vector& node_defs) { + for (size_t i = 0; i < node_defs.size(); i++) { + const FunctionBodyHelper::NodeDef& node = node_defs[i]; + auto* np = functionProto.add_node(); + + np->set_op_type(node.op_type); + for (const auto& inp : node.inputs) { + np->add_input(inp); + } + for (const auto& o : node.outputs) { + np->add_output(o); + } + for (const auto& attr : node.attributes) { + *(np->add_attribute()) = attr.proto; + } + } +} + +bool BuildFunctionProto(FunctionProto& functionProto, const OpSchema& schema, + const std::vector& node_defs, + const std::vector& relied_opsets) { + BuildNodes(functionProto, node_defs); + schema.BuildFunction(functionProto, relied_opsets); + return true; +} + +} // namespace ONNX_NAMESPACE \ No newline at end of file diff --git a/onnxruntime/core/graph/contrib_ops/onnx_function_util.h b/onnxruntime/core/graph/contrib_ops/onnx_function_util.h new file mode 100644 index 0000000000..babe1f6537 --- /dev/null +++ b/onnxruntime/core/graph/contrib_ops/onnx_function_util.h @@ -0,0 +1,25 @@ +#pragma once + +// Utility functions for building the body of a context-dependent function. +// Temporary placeholder for utilities to be moved into ONNX repo. TODO. + +#include +#include + +#include "onnx/onnx-operators_pb.h" +#include "onnx/defs/schema.h" +#include "onnx/defs/function.h" + +namespace ONNX_NAMESPACE { + +// For floating-value constants of different precision: +TensorProto ToTensor(double value, TensorProto_DataType elem_type); + +// Utility function to construct a FunctionProto from an opschema (for the signature information), +// a sequence of NodeDefs (for the function body), and the relied opsets. +bool BuildFunctionProto(FunctionProto& functionProto, + const OpSchema& schema, + const std::vector& node_defs, + const std::vector& relied_opsets = {}); + +} // namespace ONNX_NAMESPACE \ No newline at end of file diff --git a/onnxruntime/core/graph/function.cc b/onnxruntime/core/graph/function.cc index 4da98bf0de..6f538d3c58 100644 --- a/onnxruntime/core/graph/function.cc +++ b/onnxruntime/core/graph/function.cc @@ -180,11 +180,12 @@ static std::unordered_map CreateOpsetImportsForFunction(const std::unordered_map function_opset_imports{graph_opset_imports}; // merge with opset imports in function proto for (const auto& opset_import : func_proto.opset_import()) { - auto result = function_opset_imports.insert({opset_import.domain(), static_cast(opset_import.version())}); - ORT_ENFORCE(result.second, + auto opset_version = static_cast(opset_import.version()); + auto result = function_opset_imports.insert({opset_import.domain(), opset_version}); + ORT_ENFORCE((result.first->second == opset_version), "ONNX model does not support multiple opset versions for a domain. Model imports opset version ", result.first->second, " for domain ", result.first->first, " and function is trying to import opset version ", - opset_import.version(), " for the same domain"); + opset_version, " for the same domain"); } return function_opset_imports; diff --git a/onnxruntime/core/graph/graph.cc b/onnxruntime/core/graph/graph.cc index 70eddde7cf..5f3de37128 100644 --- a/onnxruntime/core/graph/graph.cc +++ b/onnxruntime/core/graph/graph.cc @@ -2371,12 +2371,29 @@ void Graph::InitFunctionBodyForNode(Node& node) { if (node.op_->HasContextDependentFunction()) { NodeProto node_proto; node.ToProto(node_proto); - onnx::FunctionBodyBuildContextImpl function_body_ctx(node_proto); + std::vector input_types; + for (size_t i = 0, n = node.InputDefs().size(); i < n; i++) { + auto p_node_arg = node.InputDefs().at(i); + if ((nullptr != p_node_arg) && p_node_arg->Exists()) { + auto& type = *(p_node_arg->TypeAsProto()); + input_types.emplace_back(type); + } else + input_types.emplace_back(); + } + onnx::FunctionBodyBuildContextImpl function_body_ctx(node_proto, input_types); node.op_->BuildContextDependentFunction(function_body_ctx, onnx_function_proto); } else { onnx_function_proto = *(node.op_->GetFunction()); } + // Check function's opset requirements are compatible with model's opset. + auto& graphImports = DomainToVersionMap(); + for (const auto& fn_import : onnx_function_proto.opset_import()) { + auto it = graphImports.find(fn_import.domain()); + if ((it != graphImports.end()) && (it->second != fn_import.version())) + return; // Incompatible. Do not use this function expansion. + } + auto func_ptr = onnxruntime::make_unique(*this, node.Index(), onnx_function_proto, logger_); diff --git a/orttraining/orttraining/core/graph/training_op_defs.cc b/orttraining/orttraining/core/graph/training_op_defs.cc index cf78dc7e50..c2bcce3eef 100644 --- a/orttraining/orttraining/core/graph/training_op_defs.cc +++ b/orttraining/orttraining/core/graph/training_op_defs.cc @@ -3,6 +3,7 @@ #include "core/graph/op.h" #include "core/graph/contrib_ops/contrib_defs.h" +#include "core/graph/contrib_ops/onnx_function_util.h" #include "core/providers/common.h" #include "orttraining/core/graph/training_op_defs.h" #include "orttraining/core/framework/distributed_run_context.h" @@ -343,7 +344,7 @@ void RegisterTrainingOpSchemas() { .SetDomain(kMSDomain) .SinceVersion(1) .Input(0, "dY", "Gradient of output Y", "T") - .Input(1, "X", "Input tensor", "T") + .Input(1, "Y", "Input tensor", "T") .Output(0, "dX", "Gradient of input X", "T") .Attr( "axis", @@ -356,7 +357,54 @@ void RegisterTrainingOpSchemas() { "T", {"tensor(float16)", "tensor(float)", "tensor(double)", "tensor(bfloat16)"}, "Constrain input and output types to float tensors.") - .TypeAndShapeInferenceFunction(propagateShapeAndTypeFromFirstInput); + .TypeAndShapeInferenceFunction(propagateShapeAndTypeFromFirstInput) + .SetContextDependentFunctionBodyBuilder( + [](const FunctionBodyBuildContext& ctx, const OpSchema& schema, FunctionProto& functionProto) { + // SoftmaxGrad computes dX = Y * ( dY - dot(Y, dY)) + // ONNX does not have a dot product, which can be simulated as a pointwise-multiplication ("Mul"), + // followed by a "ReduceSum". Unfortunately, the treatment of "axis" is different in "SoftmaxGrad" + // and "ReduceSum". If axis=k for SoftmaxGrad, we need to specify [k, ..., n-1] as the axes of + // reduction for "ReduceSum", after accounting for negative-axis specification. + // An alternative solution would be to Flatten inputs to 2D and then reshape output back to original shape. + // Hopefully, many of these ops can be optimized away in the common-case of statically-known shapes. + + auto* axis_attr = ctx.getAttribute("axis"); + int64_t axis = (axis_attr != nullptr) ? axis_attr->i() : 1; + auto zero1d = ToTensor(std::vector({0})); + zero1d.add_dims(1); + + // nodes: {outputs, op, inputs, attributes} + + // First, convert axis specification k to reduction axes [k, k+1, ..., n-1] + std::vector body{ + FunctionBodyHelper::Const("one", 1), + FunctionBodyHelper::Const("k", axis), + {{"axis_zero"}, "Constant", {}, {{"value", zero1d}}}, + {{"shape"}, "Shape", {"dY"}}, + {{"n_as_vector"}, "Shape", {"shape"}}, + {{"n"}, "Squeeze", {"n_as_vector", "axis_zero"}}, + }; + + // For negative axis, add n to axis-value k; then use Range(...). + if (axis >= 0) { + body.push_back({{"reduction_axes"}, "Range", {"k", "n", "one"}}); + } else { + body.push_back({{"n_plus_k"}, "Add", {"n", "k"}}); + body.push_back({{"reduction_axes"}, "Range", {"n_plus_k", "n", "one"}}); + } + + // compute dX = Y * ( dY - dot(Y, dY)) = Y * ( dY - ReduceSum(Y * dY)) + body.push_back({{"a"}, "Mul", {"Y", "dY"}}); + body.push_back({{"b"}, "ReduceSum", {"a", "reduction_axes"}}); + body.push_back({{"c"}, "Sub", {"dY", "b"}}); + body.push_back({{"dX"}, "Mul", {"Y", "c"}}); + + OperatorSetIdProto onnx_opset_13; + onnx_opset_13.set_domain(""); + onnx_opset_13.set_version(13); + + return ONNX_NAMESPACE::BuildFunctionProto(functionProto, schema, body, {onnx_opset_13}); + }); ONNX_CONTRIB_OPERATOR_SCHEMA(LogSoftmaxGrad) .SetDomain(kMSDomain) diff --git a/orttraining/orttraining/test/gradient/function_ops_test.cc b/orttraining/orttraining/test/gradient/function_ops_test.cc new file mode 100644 index 0000000000..6eb06b2820 --- /dev/null +++ b/orttraining/orttraining/test/gradient/function_ops_test.cc @@ -0,0 +1,276 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include +#include +#include + +#include "gtest/gtest.h" +#include "core/graph/model.h" +#include "core/graph/contrib_ops/contrib_defs.h" +#include "orttraining/core/graph/training_op_defs.h" +#include "test/test_environment.h" + +#include "core/session/inference_session.h" +#include "core/providers/cpu/cpu_execution_provider.h" + +#include "test/framework/test_utils.h" + +using namespace ::onnxruntime::common; + +namespace onnxruntime { +namespace test { + +typedef std::vector ArgMap; + +static void RegisterSchemas() { + static bool registered = false; + if (!registered) { + onnxruntime::training::RegisterTrainingOpSchemas(); + registered = true; + } +} + +static ONNX_NAMESPACE::TypeProto TensorType(int32_t elem_type, std::vector dims) { + ONNX_NAMESPACE::TypeProto typeProto; + typeProto.mutable_tensor_type()->set_elem_type(elem_type); + auto* shape = typeProto.mutable_tensor_type()->mutable_shape(); + for (auto dim : dims) + shape->add_dim()->set_dim_value(dim); + return typeProto; +} + +static ONNX_NAMESPACE::TypeProto TensorType(int32_t elem_type, std::vector dims) { + ONNX_NAMESPACE::TypeProto typeProto; + typeProto.mutable_tensor_type()->set_elem_type(elem_type); + auto* shape = typeProto.mutable_tensor_type()->mutable_shape(); + for (auto dim : dims) { + uint64_t dimval; + std::istringstream s(dim); + if (s >> dimval) { + shape->add_dim()->set_dim_value(dimval); + } else { + shape->add_dim()->set_dim_param(dim); + } + } + return typeProto; +} + +static std::vector +Run(onnxruntime::Model& model, NameMLValMap& feeds, std::vector output_names) { + SessionOptions session_options; + InferenceSession session_object{session_options, GetEnvironment()}; + + std::string serialized_model; + const bool serialization_status = model.ToProto().SerializeToString(&serialized_model); + EXPECT_TRUE(serialization_status) << "Failed to serialize proto to string"; + std::stringstream sstr(serialized_model); + auto status = session_object.Load(sstr); + EXPECT_TRUE(status.IsOK()); + status = session_object.Initialize(); + EXPECT_TRUE(status.IsOK()) << status.ErrorMessage(); + + RunOptions run_options; + run_options.run_tag = session_options.session_logid; + + std::vector fetches; + + status = session_object.Run(run_options, feeds, output_names, &fetches); + EXPECT_TRUE(status.IsOK()) << "Session Run failed."; + + return fetches; +} + +// Restricted to float tensors +static void AssertEqual(const Tensor& tensor1, const Tensor& tensor2) { + auto size = tensor1.Shape().Size(); + auto* data1 = tensor1.template Data(); + auto* data2 = tensor2.template Data(); + + float threshold = 0.001f; + + for (int i = 0; i < size; ++i) { + ASSERT_NEAR(data1[i], data2[i], threshold) << "as position i:" << i; + } +} + +static void AssertEqual(const std::vector& results1, const std::vector& results2) { + ASSERT_EQ(results1.size(), results2.size()); + for (int i = 0; i < results1.size(); i++) { + auto& value1 = results1[i].Get(); + auto& value2 = results2[i].Get(); + AssertEqual(value1, value2); + } +} + +struct FunctionTestCase { + const char* opname; + + std::vector input_args; + std::vector> input_values; + NameMLValMap input_value_map; + + std::vector output_names; + std::vector output_args; + + NodeAttributes attributes; + std::unique_ptr provider; + + std::unordered_map opsets; + + FunctionTestCase(const char* _opname) : opname(_opname), provider(new CPUExecutionProvider(CPUExecutionProviderInfo())) {} + + void AddInput(std::string input_name, std::vector shape, std::vector data, std::vector symshape = {}) { + auto arg_type = (symshape.size() > 0) ? TensorType(ONNX_NAMESPACE::TensorProto_DataType_FLOAT, symshape) : TensorType(ONNX_NAMESPACE::TensorProto_DataType_FLOAT, shape); + input_args.emplace_back(input_name, &arg_type); + + OrtValue ort_value; + CreateMLValue(provider->GetAllocator(0, OrtMemTypeDefault), shape, data, &ort_value); + input_values.push_back(std::make_pair(input_name, ort_value)); + input_value_map.insert(std::make_pair(input_name, ort_value)); + } + + void AddOutput(std::string output_name) { + output_names.emplace_back(output_name); + output_args.emplace_back(output_name, nullptr); + } + + void AddAttribute(const char* attr_name, int64_t attr_val) { + ONNX_NAMESPACE::AttributeProto axis_attr; + axis_attr.set_name(attr_name); + axis_attr.set_type(ONNX_NAMESPACE::AttributeProto_AttributeType::AttributeProto_AttributeType_INT); + axis_attr.set_i(attr_val); + attributes[attr_name] = axis_attr; + } + + onnxruntime::Node& AddCallNodeTo(onnxruntime::Graph& graph) { + std::vector input_arg_ptrs; + + for (auto& arg : input_args) + input_arg_ptrs.push_back(&arg); + + std::vector output_arg_ptrs; + for (auto& arg : output_args) + output_arg_ptrs.push_back(&arg); + + return graph.AddNode("fncallnode", opname, "function call node", input_arg_ptrs, output_arg_ptrs, &attributes, onnxruntime::kMSDomain); + } + + std::unique_ptr CreateModel(bool inline_call = false) { + RegisterSchemas(); + if (opsets.size() == 0) { + // Default opsets + opsets[kOnnxDomain] = 13; + opsets[kMSDomain] = 1; + } + + std::unique_ptr model(new Model("test", false, ModelMetaData(), PathString(), IOnnxRuntimeOpSchemaRegistryList(), + opsets, {}, DefaultLoggingManager().DefaultLogger())); + + onnxruntime::Graph& graph = model->MainGraph(); + auto& call_node = AddCallNodeTo(graph); + + auto status = graph.Resolve(); + EXPECT_TRUE(status.IsOK()) << status.ErrorMessage(); + + if (inline_call) { + graph.InlineFunction(call_node); + status = graph.Resolve(); + EXPECT_TRUE(status.IsOK()) << status.ErrorMessage(); + } + + return model; + } + + void RunTest() { + auto model1 = CreateModel(false); + auto results1 = Run(*model1, input_value_map, output_names); + + auto model2 = CreateModel(true); + auto results2 = Run(*model2, input_value_map, output_names); + + AssertEqual(results1, results2); + } +}; + +static void InitSoftmaxGradTestCase(FunctionTestCase& testCase, std::vector shape) { + int64_t size = 1; + for (auto dim : shape) + size *= dim; + + std::vector value(size); + for (int64_t i = 0; i < size; i++) + value[i] = float(i); + + testCase.AddInput("dY", shape, value); + testCase.AddInput("Y", shape, value); + testCase.AddOutput("dX"); +} + +TEST(SoftmaxGradExpansionTest, DefaultAxis) { + FunctionTestCase testCase("SoftmaxGrad"); + InitSoftmaxGradTestCase(testCase, {3, 2}); + testCase.RunTest(); +} + +TEST(SoftmaxGradExpansionTest, NegativeAxis) { + FunctionTestCase testCase("SoftmaxGrad"); + InitSoftmaxGradTestCase(testCase, {3, 2}); + testCase.AddAttribute("axis", -1); + testCase.RunTest(); +} + +TEST(SoftmaxGradExpansionTest, PositiveAxis) { + FunctionTestCase testCase("SoftmaxGrad"); + InitSoftmaxGradTestCase(testCase, {3, 2}); + testCase.AddAttribute("axis", 1); + testCase.RunTest(); +} + +TEST(SoftmaxGradExpansionTest, 3D) { + FunctionTestCase testCase("SoftmaxGrad"); + InitSoftmaxGradTestCase(testCase, {3, 2, 2}); + testCase.RunTest(); +} + +TEST(SoftmaxGradExpansionTest, SymbolicShape) { + FunctionTestCase testCase("SoftmaxGrad"); + std::vector shape{3, 2, 2}; + std::vector sym_shape{"BatchSize", "SeqSize", "2"}; + int size = 12; + std::vector value(size); + for (int64_t i = 0; i < size; i++) + value[i] = float(i); + + testCase.AddInput("dY", shape, value, sym_shape); + testCase.AddInput("Y", shape, value, sym_shape); + testCase.AddOutput("dX"); + testCase.RunTest(); +} + +// Test (unexpanded) versions for both opset 12 and opset 13 models to ensure +// function-schema does not impact handling of opset 12 models. The current +// expansion requires opset 13, and no expansion should happen in opset 12 +// models. Test is required since ORT currently generates function-expansion +// even when op is dispatched to a kernel. + +TEST(SoftmaxGradExpansionTest, OpsetTest) { + FunctionTestCase testCase("SoftmaxGrad"); + testCase.opsets[kOnnxDomain] = 12; + testCase.opsets[kMSDomain] = 1; + InitSoftmaxGradTestCase(testCase, {3, 2, 2}); + + auto model1 = testCase.CreateModel(); + auto results1 = onnxruntime::test::Run(*model1, testCase.input_value_map, testCase.output_names); + + testCase.opsets[kOnnxDomain] = 13; + testCase.opsets[kMSDomain] = 1; + + auto model2 = testCase.CreateModel(); + auto results2 = onnxruntime::test::Run(*model1, testCase.input_value_map, testCase.output_names); + + AssertEqual(results1, results2); +} + +} // namespace test +} // namespace onnxruntime \ No newline at end of file From 2bf54bcaa2e299af56f8d4eeb8b94621cac14ac1 Mon Sep 17 00:00:00 2001 From: Dmitri Smirnov Date: Thu, 25 Mar 2021 14:53:52 -0700 Subject: [PATCH 007/129] Fix bugs in sparsify script (#7134) Fix type and check. --- tools/python/sparsify_initializers.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/tools/python/sparsify_initializers.py b/tools/python/sparsify_initializers.py index 61e9476dcf..c81d25c78a 100644 --- a/tools/python/sparsify_initializers.py +++ b/tools/python/sparsify_initializers.py @@ -16,7 +16,7 @@ from onnx import ModelProto, SparseTensorProto, TensorProto, numpy_helper logger = logging.getLogger(__name__) -real_types = set((np.float32, np.float64, np.double)) +real_types = set((int(TensorProto.FLOAT), int(TensorProto.DOUBLE))) def parse_arguments(): @@ -67,7 +67,7 @@ def convert_tensor_to_sparse(tensor, tolerance): # type: (TensorProto) -> Tuple else: for index in range(data_len): el = tensor_data[index] - if el == 0: + if el != 0: values.append(el) indicies.append(index) nnz_count += 1 From c9b29fbd06aad25ba49f7e31c52a37f19b2f0977 Mon Sep 17 00:00:00 2001 From: KeDengMS Date: Thu, 25 Mar 2021 17:16:58 -0700 Subject: [PATCH 008/129] Disable MatmulTransposeFusion for CPU EP (#7135) It causes convergence issue in BERT on CPU --- .../orttraining/core/optimizer/graph_transformer_utils.cc | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/orttraining/orttraining/core/optimizer/graph_transformer_utils.cc b/orttraining/orttraining/core/optimizer/graph_transformer_utils.cc index dd04537928..4b5353690c 100644 --- a/orttraining/orttraining/core/optimizer/graph_transformer_utils.cc +++ b/orttraining/orttraining/core/optimizer/graph_transformer_utils.cc @@ -171,7 +171,7 @@ std::vector> GenerateTransformers( //transformers.emplace_back(onnxruntime::make_unique(l1_execution_providers)); transformers.emplace_back(onnxruntime::make_unique(l1_execution_providers)); transformers.emplace_back(onnxruntime::make_unique(free_dimension_overrides)); - transformers.emplace_back(onnxruntime::make_unique(l1_execution_providers)); + transformers.emplace_back(onnxruntime::make_unique(cuda_rocm_execution_providers)); transformers.emplace_back(onnxruntime::make_unique(cuda_rocm_execution_providers)); transformers.emplace_back(onnxruntime::make_unique(l1_execution_providers)); transformers.emplace_back(onnxruntime::make_unique(l1_execution_providers, weights_to_train)); From 3771e0bf10cde8f3739dda1b1cde60ca578c4583 Mon Sep 17 00:00:00 2001 From: Yufeng Li Date: Thu, 25 Mar 2021 18:12:53 -0700 Subject: [PATCH 009/129] update bert quantization notebook (#7137) --- .../Bert-GLUE_OnnxRuntime_quantization.ipynb | 235 +++++++++--------- 1 file changed, 114 insertions(+), 121 deletions(-) diff --git a/onnxruntime/python/tools/quantization/notebooks/Bert-GLUE_OnnxRuntime_quantization.ipynb b/onnxruntime/python/tools/quantization/notebooks/Bert-GLUE_OnnxRuntime_quantization.ipynb index 0beda4e611..17be273324 100644 --- a/onnxruntime/python/tools/quantization/notebooks/Bert-GLUE_OnnxRuntime_quantization.ipynb +++ b/onnxruntime/python/tools/quantization/notebooks/Bert-GLUE_OnnxRuntime_quantization.ipynb @@ -33,7 +33,7 @@ "Otherwise, you can setup a new environment. First, install [AnaConda](https://www.anaconda.com/distribution/). Then open an AnaConda prompt window and run the following commands:\n", "\n", "```console\n", - "conda create -n cpu_env python=3.6\n", + "conda create -n cpu_env python=3.8\n", "conda activate cpu_env\n", "conda install jupyter\n", "jupyter notebook\n", @@ -46,7 +46,7 @@ "metadata": {}, "source": [ "### 0.1 Install packages\n", - "Let's install nessasary packages to start the tutorial. We will install PyTorch 1.6, OnnxRuntime 1.5.1, latest ONNX, OnnxRuntime-tools, transformers, and sklearn." + "Let's install nessasary packages to start the tutorial. We will install PyTorch 1.8, OnnxRuntime 1.7, latest ONNX, OnnxRuntime-tools, transformers, and sklearn." ] }, { @@ -61,89 +61,71 @@ "output_type": "stream", "text": [ "Looking in links: https://download.pytorch.org/whl/torch_stable.html\n", - "Requirement already up-to-date: torch==1.6.0+cpu in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (1.6.0+cpu)\n", - "Requirement already up-to-date: torchvision==0.7.0+cpu in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (0.7.0+cpu)\n", - "Requirement already satisfied, skipping upgrade: numpy in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (from torch==1.6.0+cpu) (1.18.1)\n", - "Requirement already satisfied, skipping upgrade: future in /home/yufeng/.local/lib/python3.6/site-packages (from torch==1.6.0+cpu) (0.18.2)\n", - "Requirement already satisfied, skipping upgrade: pillow>=4.1.1 in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from torchvision==0.7.0+cpu) (7.0.0)\n", - "Collecting onnxruntime==1.5.1\n", - " Using cached onnxruntime-1.5.1-cp36-cp36m-manylinux2014_x86_64.whl (3.8 MB)\n", - "Requirement already satisfied, skipping upgrade: numpy>=1.16.6 in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (from onnxruntime==1.5.1) (1.18.1)\n", - "Requirement already satisfied, skipping upgrade: protobuf in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from onnxruntime==1.5.1) (3.9.1)\n", - "Requirement already satisfied, skipping upgrade: six>=1.9 in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from protobuf->onnxruntime==1.5.1) (1.12.0)\n", - "Requirement already satisfied, skipping upgrade: setuptools in /home/yufeng/.local/lib/python3.6/site-packages (from protobuf->onnxruntime==1.5.1) (41.0.1)\n", - "Installing collected packages: onnxruntime\n", - " Attempting uninstall: onnxruntime\n", - " Found existing installation: onnxruntime 1.4.0\n", - " Uninstalling onnxruntime-1.4.0:\n", - " Successfully uninstalled onnxruntime-1.4.0\n", - "Successfully installed onnxruntime-1.5.1\n", - "Requirement already up-to-date: onnxruntime-tools in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (1.5.1)\n", - "Requirement already satisfied, skipping upgrade: py-cpuinfo in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (from onnxruntime-tools) (5.0.0)\n", - "Requirement already satisfied, skipping upgrade: packaging in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from onnxruntime-tools) (19.1)\n", - "Requirement already satisfied, skipping upgrade: onnx in /home/yufeng/project/onnx (from onnxruntime-tools) (1.6.0)\n", - "Requirement already satisfied, skipping upgrade: coloredlogs in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (from onnxruntime-tools) (14.0)\n", - "Requirement already satisfied, skipping upgrade: py3nvml in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (from onnxruntime-tools) (0.2.6)\n", - "Requirement already satisfied, skipping upgrade: psutil in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from onnxruntime-tools) (5.6.3)\n", - "Requirement already satisfied, skipping upgrade: numpy in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (from onnxruntime-tools) (1.18.1)\n", - "Requirement already satisfied, skipping upgrade: six in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from packaging->onnxruntime-tools) (1.12.0)\n", - "Requirement already satisfied, skipping upgrade: attrs in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from packaging->onnxruntime-tools) (19.1.0)\n", - "Requirement already satisfied, skipping upgrade: pyparsing>=2.0.2 in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from packaging->onnxruntime-tools) (2.4.2)\n", - "Requirement already satisfied, skipping upgrade: protobuf in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from onnx->onnxruntime-tools) (3.9.1)\n", - "Requirement already satisfied, skipping upgrade: typing-extensions>=3.6.2.1 in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages/typing_extensions-3.7.4-py3.6.egg (from onnx->onnxruntime-tools) (3.7.4)\n", - "Requirement already satisfied, skipping upgrade: humanfriendly>=7.1 in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (from coloredlogs->onnxruntime-tools) (8.2)\n", - "Requirement already satisfied, skipping upgrade: xmltodict in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (from py3nvml->onnxruntime-tools) (0.12.0)\n", - "Requirement already satisfied, skipping upgrade: setuptools in /home/yufeng/.local/lib/python3.6/site-packages (from protobuf->onnx->onnxruntime-tools) (41.0.1)\n", - "Requirement already up-to-date: transformers in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (3.3.1)\n", - "Requirement already satisfied, skipping upgrade: tokenizers==0.8.1.rc2 in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (from transformers) (0.8.1rc2)\n", - "Requirement already satisfied, skipping upgrade: tqdm>=4.27 in /home/yufeng/.local/lib/python3.6/site-packages (from transformers) (4.45.0)\n", - "Requirement already satisfied, skipping upgrade: dataclasses; python_version < \"3.7\" in /home/yufeng/.local/lib/python3.6/site-packages (from transformers) (0.7)\n", - "Requirement already satisfied, skipping upgrade: sentencepiece!=0.1.92 in /home/yufeng/.local/lib/python3.6/site-packages (from transformers) (0.1.85)\n", - "Requirement already satisfied, skipping upgrade: sacremoses in /home/yufeng/.local/lib/python3.6/site-packages (from transformers) (0.0.38)\n", - "Requirement already satisfied, skipping upgrade: regex!=2019.12.17 in /home/yufeng/.local/lib/python3.6/site-packages (from transformers) (2020.4.4)\n", - "Requirement already satisfied, skipping upgrade: requests in /home/yufeng/.local/lib/python3.6/site-packages (from transformers) (2.23.0)\n", - "Requirement already satisfied, skipping upgrade: numpy in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (from transformers) (1.18.1)\n", - "Requirement already satisfied, skipping upgrade: packaging in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from transformers) (19.1)\n", - "Requirement already satisfied, skipping upgrade: filelock in /home/yufeng/.local/lib/python3.6/site-packages (from transformers) (3.0.12)\n", - "Requirement already satisfied, skipping upgrade: six in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from sacremoses->transformers) (1.12.0)\n", - "Requirement already satisfied, skipping upgrade: joblib in /home/yufeng/.local/lib/python3.6/site-packages (from sacremoses->transformers) (0.14.1)\n", - "Requirement already satisfied, skipping upgrade: click in /home/yufeng/.local/lib/python3.6/site-packages (from sacremoses->transformers) (7.1.1)\n", - "Requirement already satisfied, skipping upgrade: chardet<4,>=3.0.2 in /home/yufeng/.local/lib/python3.6/site-packages (from requests->transformers) (3.0.4)\n", - "Requirement already satisfied, skipping upgrade: idna<3,>=2.5 in /home/yufeng/.local/lib/python3.6/site-packages (from requests->transformers) (2.9)\n", - "Requirement already satisfied, skipping upgrade: certifi>=2017.4.17 in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from requests->transformers) (2019.6.16)\n", - "Requirement already satisfied, skipping upgrade: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /home/yufeng/.local/lib/python3.6/site-packages (from requests->transformers) (1.25.8)\n", - "Requirement already satisfied, skipping upgrade: attrs in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from packaging->transformers) (19.1.0)\n", - "Requirement already satisfied, skipping upgrade: pyparsing>=2.0.2 in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from packaging->transformers) (2.4.2)\n", - "Requirement already satisfied: onnx in /home/yufeng/project/onnx (1.6.0)\n", - "Requirement already satisfied: sklearn in /home/yufeng/.local/lib/python3.6/site-packages (0.0)\n", - "Requirement already satisfied: protobuf in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from onnx) (3.9.1)\n", - "Requirement already satisfied: numpy in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (from onnx) (1.18.1)\n", - "Requirement already satisfied: six in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages (from onnx) (1.12.0)\n", - "Requirement already satisfied: typing-extensions>=3.6.2.1 in /home/yufeng/anaconda3/envs/onnx/lib/python3.6/site-packages/typing_extensions-3.7.4-py3.6.egg (from onnx) (3.7.4)\n", - "Requirement already satisfied: scikit-learn in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (from sklearn) (0.23.1)\n", - "Requirement already satisfied: setuptools in /home/yufeng/.local/lib/python3.6/site-packages (from protobuf->onnx) (41.0.1)\n", - "Requirement already satisfied: joblib>=0.11 in /home/yufeng/.local/lib/python3.6/site-packages (from scikit-learn->sklearn) (0.14.1)\n" - ] - }, - { - "name": "stdout", - "output_type": "stream", - "text": [ - "Requirement already satisfied: threadpoolctl>=2.0.0 in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (from scikit-learn->sklearn) (2.1.0)\r\n", - "Requirement already satisfied: scipy>=0.19.1 in /home/yufeng/anaconda3/envs/pytorch/lib/python3.6/site-packages (from scikit-learn->sklearn) (1.5.1)\r\n" + "Requirement already up-to-date: install in /home/yufeng/anaconda3/lib/python3.8/site-packages (1.3.4)\n", + "Requirement already up-to-date: torch==1.8.1+cpu in /home/yufeng/anaconda3/lib/python3.8/site-packages (1.8.1+cpu)\n", + "Requirement already up-to-date: torchvision==0.9.1+cpu in /home/yufeng/anaconda3/lib/python3.8/site-packages (0.9.1+cpu)\n", + "Requirement already up-to-date: torchaudio===0.8.1 in /home/yufeng/anaconda3/lib/python3.8/site-packages (0.8.1)\n", + "Requirement already satisfied, skipping upgrade: numpy in /home/yufeng/anaconda3/lib/python3.8/site-packages (from torch==1.8.1+cpu) (1.19.2)\n", + "Requirement already satisfied, skipping upgrade: typing-extensions in /home/yufeng/anaconda3/lib/python3.8/site-packages (from torch==1.8.1+cpu) (3.7.4.3)\n", + "Requirement already satisfied, skipping upgrade: pillow>=4.1.1 in /home/yufeng/anaconda3/lib/python3.8/site-packages (from torchvision==0.9.1+cpu) (8.0.1)\n", + "Requirement already up-to-date: onnxruntime==1.7.0 in /home/yufeng/anaconda3/lib/python3.8/site-packages (1.7.0)\n", + "Requirement already satisfied, skipping upgrade: numpy>=1.16.6 in /home/yufeng/anaconda3/lib/python3.8/site-packages (from onnxruntime==1.7.0) (1.19.2)\n", + "Requirement already satisfied, skipping upgrade: protobuf in /home/yufeng/anaconda3/lib/python3.8/site-packages (from onnxruntime==1.7.0) (3.15.6)\n", + "Requirement already satisfied, skipping upgrade: six>=1.9 in /home/yufeng/anaconda3/lib/python3.8/site-packages (from protobuf->onnxruntime==1.7.0) (1.15.0)\n", + "Requirement already up-to-date: onnxruntime-tools in /home/yufeng/anaconda3/lib/python3.8/site-packages (1.6.0)\n", + "Requirement already satisfied, skipping upgrade: psutil in /home/yufeng/anaconda3/lib/python3.8/site-packages (from onnxruntime-tools) (5.7.2)\n", + "Requirement already satisfied, skipping upgrade: numpy in /home/yufeng/anaconda3/lib/python3.8/site-packages (from onnxruntime-tools) (1.19.2)\n", + "Requirement already satisfied, skipping upgrade: packaging in /home/yufeng/anaconda3/lib/python3.8/site-packages (from onnxruntime-tools) (20.4)\n", + "Requirement already satisfied, skipping upgrade: coloredlogs in /home/yufeng/anaconda3/lib/python3.8/site-packages (from onnxruntime-tools) (15.0)\n", + "Requirement already satisfied, skipping upgrade: onnx in /home/yufeng/anaconda3/lib/python3.8/site-packages (from onnxruntime-tools) (1.8.1)\n", + "Requirement already satisfied, skipping upgrade: py-cpuinfo in /home/yufeng/anaconda3/lib/python3.8/site-packages (from onnxruntime-tools) (7.0.0)\n", + "Requirement already satisfied, skipping upgrade: py3nvml in /home/yufeng/anaconda3/lib/python3.8/site-packages (from onnxruntime-tools) (0.2.6)\n", + "Requirement already satisfied, skipping upgrade: pyparsing>=2.0.2 in /home/yufeng/anaconda3/lib/python3.8/site-packages (from packaging->onnxruntime-tools) (2.4.7)\n", + "Requirement already satisfied, skipping upgrade: six in /home/yufeng/anaconda3/lib/python3.8/site-packages (from packaging->onnxruntime-tools) (1.15.0)\n", + "Requirement already satisfied, skipping upgrade: humanfriendly>=9.1 in /home/yufeng/anaconda3/lib/python3.8/site-packages (from coloredlogs->onnxruntime-tools) (9.1)\n", + "Requirement already satisfied, skipping upgrade: typing-extensions>=3.6.2.1 in /home/yufeng/anaconda3/lib/python3.8/site-packages (from onnx->onnxruntime-tools) (3.7.4.3)\n", + "Requirement already satisfied, skipping upgrade: protobuf in /home/yufeng/anaconda3/lib/python3.8/site-packages (from onnx->onnxruntime-tools) (3.15.6)\n", + "Requirement already satisfied, skipping upgrade: xmltodict in /home/yufeng/anaconda3/lib/python3.8/site-packages (from py3nvml->onnxruntime-tools) (0.12.0)\n", + "Requirement already up-to-date: transformers in /home/yufeng/anaconda3/lib/python3.8/site-packages (4.4.2)\n", + "Requirement already satisfied, skipping upgrade: tqdm>=4.27 in /home/yufeng/anaconda3/lib/python3.8/site-packages (from transformers) (4.50.2)\n", + "Requirement already satisfied, skipping upgrade: tokenizers<0.11,>=0.10.1 in /home/yufeng/anaconda3/lib/python3.8/site-packages (from transformers) (0.10.1)\n", + "Requirement already satisfied, skipping upgrade: regex!=2019.12.17 in /home/yufeng/anaconda3/lib/python3.8/site-packages (from transformers) (2020.10.15)\n", + "Requirement already satisfied, skipping upgrade: numpy>=1.17 in /home/yufeng/anaconda3/lib/python3.8/site-packages (from transformers) (1.19.2)\n", + "Requirement already satisfied, skipping upgrade: packaging in /home/yufeng/anaconda3/lib/python3.8/site-packages (from transformers) (20.4)\n", + "Requirement already satisfied, skipping upgrade: filelock in /home/yufeng/anaconda3/lib/python3.8/site-packages (from transformers) (3.0.12)\n", + "Requirement already satisfied, skipping upgrade: sacremoses in /home/yufeng/anaconda3/lib/python3.8/site-packages (from transformers) (0.0.43)\n", + "Requirement already satisfied, skipping upgrade: requests in /home/yufeng/anaconda3/lib/python3.8/site-packages (from transformers) (2.24.0)\n", + "Requirement already satisfied, skipping upgrade: pyparsing>=2.0.2 in /home/yufeng/anaconda3/lib/python3.8/site-packages (from packaging->transformers) (2.4.7)\n", + "Requirement already satisfied, skipping upgrade: six in /home/yufeng/anaconda3/lib/python3.8/site-packages (from packaging->transformers) (1.15.0)\n", + "Requirement already satisfied, skipping upgrade: click in /home/yufeng/anaconda3/lib/python3.8/site-packages (from sacremoses->transformers) (7.1.2)\n", + "Requirement already satisfied, skipping upgrade: joblib in /home/yufeng/anaconda3/lib/python3.8/site-packages (from sacremoses->transformers) (0.17.0)\n", + "Requirement already satisfied, skipping upgrade: urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 in /home/yufeng/anaconda3/lib/python3.8/site-packages (from requests->transformers) (1.25.11)\n", + "Requirement already satisfied, skipping upgrade: chardet<4,>=3.0.2 in /home/yufeng/anaconda3/lib/python3.8/site-packages (from requests->transformers) (3.0.4)\n", + "Requirement already satisfied, skipping upgrade: idna<3,>=2.5 in /home/yufeng/anaconda3/lib/python3.8/site-packages (from requests->transformers) (2.10)\n", + "Requirement already satisfied, skipping upgrade: certifi>=2017.4.17 in /home/yufeng/anaconda3/lib/python3.8/site-packages (from requests->transformers) (2020.6.20)\n", + "Requirement already up-to-date: onnx in /home/yufeng/anaconda3/lib/python3.8/site-packages (1.8.1)\n", + "Requirement already up-to-date: sklearn in /home/yufeng/anaconda3/lib/python3.8/site-packages (0.0)\n", + "Requirement already satisfied, skipping upgrade: numpy>=1.16.6 in /home/yufeng/anaconda3/lib/python3.8/site-packages (from onnx) (1.19.2)\n", + "Requirement already satisfied, skipping upgrade: six in /home/yufeng/anaconda3/lib/python3.8/site-packages (from onnx) (1.15.0)\n", + "Requirement already satisfied, skipping upgrade: protobuf in /home/yufeng/anaconda3/lib/python3.8/site-packages (from onnx) (3.15.6)\n", + "Requirement already satisfied, skipping upgrade: typing-extensions>=3.6.2.1 in /home/yufeng/anaconda3/lib/python3.8/site-packages (from onnx) (3.7.4.3)\n", + "Requirement already satisfied, skipping upgrade: scikit-learn in /home/yufeng/anaconda3/lib/python3.8/site-packages (from sklearn) (0.23.2)\n", + "Requirement already satisfied, skipping upgrade: joblib>=0.11 in /home/yufeng/anaconda3/lib/python3.8/site-packages (from scikit-learn->sklearn) (0.17.0)\n", + "Requirement already satisfied, skipping upgrade: threadpoolctl>=2.0.0 in /home/yufeng/anaconda3/lib/python3.8/site-packages (from scikit-learn->sklearn) (2.1.0)\n", + "Requirement already satisfied, skipping upgrade: scipy>=0.19.1 in /home/yufeng/anaconda3/lib/python3.8/site-packages (from scikit-learn->sklearn) (1.5.2)\n" ] } ], "source": [ - "# Install or upgrade PyTorch 1.6.0 and OnnxRuntime 1.5.1 for CPU-only.\n", + "# Install or upgrade PyTorch 1.8.0 and OnnxRuntime 1.7.0 for CPU-only.\n", "import sys\n", - "!{sys.executable} -m pip install --upgrade torch==1.6.0+cpu torchvision==0.7.0+cpu -f https://download.pytorch.org/whl/torch_stable.html\n", - "!{sys.executable} -m pip install --upgrade onnxruntime==1.5.1\n", + "!{sys.executable} -m pip install --upgrade install torch==1.8.1+cpu torchvision==0.9.1+cpu torchaudio===0.8.1 -f https://download.pytorch.org/whl/torch_stable.html\n", + "!{sys.executable} -m pip install --upgrade onnxruntime==1.7.0\n", "!{sys.executable} -m pip install --upgrade onnxruntime-tools\n", "\n", "# Install other packages used in this notebook.\n", "!{sys.executable} -m pip install --upgrade transformers\n", - "!{sys.executable} -m pip install onnx sklearn" + "!{sys.executable} -m pip install --upgrade onnx sklearn" ] }, { @@ -172,26 +154,26 @@ "name": "stdout", "output_type": "stream", "text": [ - "--2020-10-06 22:44:47-- https://raw.githubusercontent.com/huggingface/transformers/f98ef14d161d7bcdc9808b5ec399981481411cc1/utils/download_glue_data.py\n", - "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 151.101.188.133\n", - "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|151.101.188.133|:443... connected.\n", + "--2021-03-25 20:48:36-- https://raw.githubusercontent.com/huggingface/transformers/f98ef14d161d7bcdc9808b5ec399981481411cc1/utils/download_glue_data.py\n", + "Resolving raw.githubusercontent.com (raw.githubusercontent.com)... 185.199.109.133, 185.199.108.133, 185.199.111.133, ...\n", + "Connecting to raw.githubusercontent.com (raw.githubusercontent.com)|185.199.109.133|:443... connected.\n", "HTTP request sent, awaiting response... 200 OK\n", "Length: 8209 (8.0K) [text/plain]\n", "Saving to: ‘download_glue_data.py.2’\n", "\n", "download_glue_data. 100%[===================>] 8.02K --.-KB/s in 0s \n", "\n", - "2020-10-06 22:44:47 (26.9 MB/s) - ‘download_glue_data.py.2’ saved [8209/8209]\n", + "2021-03-25 20:48:36 (79.6 MB/s) - ‘download_glue_data.py.2’ saved [8209/8209]\n", "\n", "Processing MRPC...\n", "Local MRPC data not specified, downloading data from https://dl.fbaipublicfiles.com/senteval/senteval_data/msr_paraphrase_train.txt\n", "\tCompleted!\n", "cached_dev_bert-base-uncased_128_mrpc msr_paraphrase_test.txt\t train.tsv\n", - "dev_ids.tsv\t\t\t msr_paraphrase_train.txt\n", - "dev.tsv\t\t\t\t test.tsv\n", + "dev.tsv\t\t\t\t msr_paraphrase_train.txt\n", + "dev_ids.tsv\t\t\t test.tsv\n", " % Total % Received % Xferd Average Speed Time Time Time Current\n", " Dload Upload Total Spent Left Speed\n", - "100 386M 100 386M 0 0 32.8M 0 0:00:11 0:00:11 --:--:-- 35.3M\n", + "100 386M 100 386M 0 0 224M 0 0:00:01 0:00:01 --:--:-- 224M\n", "Archive: MPRC.zip\n" ] } @@ -245,7 +227,7 @@ "text": [ " % Total % Received % Xferd Average Speed Time Time Time Current\n", " Dload Upload Total Spent Left Speed\n", - "100 386M 100 386M 0 0 77.7M 0 0:00:04 0:00:04 --:--:-- 83.1M\n", + "100 386M 100 386M 0 0 190M 0 0:00:02 0:00:02 --:--:-- 190M\n", "Archive: MPRC.zip\n" ] } @@ -287,7 +269,15 @@ "name": "stdout", "output_type": "stream", "text": [ - "1.6.0+cpu\n" + "1.8.1+cpu\n" + ] + }, + { + "name": "stderr", + "output_type": "stream", + "text": [ + "/home/yufeng/anaconda3/lib/python3.8/site-packages/transformers/data/processors/glue.py:175: FutureWarning: This processor will be removed from the library soon, preprocessing should be handled with the 🤗 Datasets library. You can have a look at this example script for pointers: https://github.com/huggingface/transformers/blob/master/examples/text-classification/run_glue.py\n", + " warnings.warn(DEPRECATION_WARNING.format(\"processor\"), FutureWarning)\n" ] } ], @@ -434,8 +424,7 @@ "name": "stderr", "output_type": "stream", "text": [ - "\r", - "Evaluating: 0%| | 0/408 [00:00 Date: Fri, 26 Mar 2021 11:26:18 -0700 Subject: [PATCH 010/129] icnrease timeout (#7145) --- .../orttraining-linux-gpu-docker-release-pipeline.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-docker-release-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-docker-release-pipeline.yml index 9078b7d21a..d4a466639b 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-docker-release-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-docker-release-pipeline.yml @@ -13,7 +13,7 @@ variables: name: $(Date:yyyyMMdd)$(Rev:.r) jobs: - job: Linux_py_GPU_Build_Test_Release_Dockerfile - timeoutInMinutes: 90 + timeoutInMinutes: 110 workspace: clean: all pool: Onnxruntime-Linux-GPU From a01f15198c03ba4af47fe268ae1f65a1b9373f77 Mon Sep 17 00:00:00 2001 From: Thiago Crepaldi Date: Fri, 26 Mar 2021 14:08:46 -0700 Subject: [PATCH 011/129] Add support for large models (#7113) * Add support for large models * Handle models with registered buffers --- .../core/framework/gradient_graph_builder.h | 2 +- .../orttraining/python/training/ortmodule.py | 15 +++-- .../python/orttraining_test_ortmodule_api.py | 64 ++++++++++++++++++- .../python/orttraining_test_ortmodule_poc.py | 1 - 4 files changed, 73 insertions(+), 9 deletions(-) diff --git a/orttraining/orttraining/core/framework/gradient_graph_builder.h b/orttraining/orttraining/core/framework/gradient_graph_builder.h index a6dd6b4fe9..25967e0c77 100644 --- a/orttraining/orttraining/core/framework/gradient_graph_builder.h +++ b/orttraining/orttraining/core/framework/gradient_graph_builder.h @@ -134,7 +134,7 @@ class GradientGraphBuilder { NodeSet BFSWithStopGradient(const std::unordered_set& x_node_arg_names) const; /** - Perferms a ReverseBFS on the graph with STOP_GRADIENT_EDGES constrain + Performs a ReverseBFS on the graph with STOP_GRADIENT_EDGES constrain It will skip traversing over the edges defined in STOP_GRADIENT_EDGES map. The resulting node set contains all the nodes that are differentiable wrt the input nodes @param Starting nodes for ReverseBFS diff --git a/orttraining/orttraining/python/training/ortmodule.py b/orttraining/orttraining/python/training/ortmodule.py index 26e9e1bf7c..29d4387b16 100644 --- a/orttraining/orttraining/python/training/ortmodule.py +++ b/orttraining/orttraining/python/training/ortmodule.py @@ -354,10 +354,6 @@ class ORTModule(torch.nn.Module): if self._is_training(): initializer_names_to_train = [name for name, param in self._flattened_output_module.named_parameters() if param.requires_grad] - onnx_initializer_names = { - p.name for p in self._onnx_inference.graph.initializer} - initializer_names_to_train = [ - p for p in initializer_names_to_train if p in onnx_initializer_names] # Build full training graph grad_builder_config = C.ModuleGradientGraphBuilderConfiguration() @@ -436,6 +432,7 @@ class ORTModule(torch.nn.Module): ''' # User inputs non_none_inputs = [inp for inp in inputs if inp is not None] + named_buffers_iter = iter(self._flattened_output_module.named_buffers()) result = [] for input_idx, name in enumerate(self._onnx_graphs_info.user_input_names): inp = None @@ -443,6 +440,12 @@ class ORTModule(torch.nn.Module): inp = non_none_inputs[input_idx] elif name in kwargs and kwargs[name] is not None: inp = kwargs[name] + elif input_idx >= len(non_none_inputs): + # Registered buffers are translated to user_input+initializer in ONNX + # TODO: Check what happens when the number of inputs change form one call to the next + buffer_name, inp = next(named_buffers_iter) + assert buffer_name == name, f'Input name {name} expected, but {buffer_name} found!' + if inp is not None: result.append(inp) else: @@ -492,7 +495,9 @@ class ORTModule(torch.nn.Module): do_constant_folding=False, training=torch.onnx.TrainingMode.TRAINING, dynamic_axes=dynamic_axes, - verbose=self._verbosity < Verbosity.WARNING) + verbose=self._verbosity < Verbosity.WARNING, + export_params=False, + keep_initializers_as_inputs=True) except RuntimeError as e: raise RuntimeError( 'There was an error while exporting the PyTorch model to ONNX: {}'.format(e)) diff --git a/orttraining/orttraining/test/python/orttraining_test_ortmodule_api.py b/orttraining/orttraining/test/python/orttraining_test_ortmodule_api.py index 44646be72b..d386836aa7 100644 --- a/orttraining/orttraining/test/python/orttraining_test_ortmodule_api.py +++ b/orttraining/orttraining/test/python/orttraining_test_ortmodule_api.py @@ -1347,8 +1347,7 @@ def test_nested_return_value_module(device): N, D_in, H, D_out = 64, 784, 500, 10 model = NeuralNetNestedOutput(D_in, H, D_out).to(device) model = ORTModule(model) - model._save_onnx = True - model._save_onnx_prefix = 'nested_model_output' + x = torch.randn(N, D_in, device=device) y = torch.randn(N, D_in, device=device) z = torch.randn(N, D_in, device=device) @@ -1506,3 +1505,64 @@ def test_model_initializer_requires_grad_changes_from_one_forward_to_next(): assert training_session1 != training_session2 assert torch.equal(weight_grad_2, weight_grad_3) assert torch.equal(bias_grad_2, bias_grad_3) + +def test_model_with_registered_buffers(): + class NeuralNetWithRegisteredBuffer(torch.nn.Module): + def __init__(self, input_size, hidden_size, num_classes): + super(NeuralNetWithRegisteredBuffer, self).__init__() + + self.fc1 = torch.nn.Linear(input_size, hidden_size) + self.relu = torch.nn.ReLU() + self.fc2 = torch.nn.Linear(hidden_size, num_classes) + self.register_buffer("buffer1s", torch.ones(num_classes)) + self.register_buffer("buffer2s", 1+torch.ones(num_classes)) + + def forward(self, input1): + out = self.fc1(input1) + out = self.relu(out) + out = self.fc2(out) + out += self.buffer1s + out += self.buffer2s + return out + device = 'cuda' + + N, D_in, H, D_out = 64, 784, 500, 10 + model = NeuralNetWithRegisteredBuffer(D_in, H, D_out).to(device) + ort_model = ORTModule(model) + # Check that the original forward signature is preserved. + assert signature(model.forward) == signature(ort_model.forward) + x = torch.randn(N, D_in, device=device) + # Make sure model runs without any exception + output = ort_model(x) + assert output is not None + +def test_model_with_constant_and_registered_parameters(): + class NeuralNetWithRegisteredParamsWithConstant(torch.nn.Module): + def __init__(self, input_size, hidden_size, num_classes): + super(NeuralNetWithRegisteredParamsWithConstant, self).__init__() + + self.fc1 = torch.nn.Linear(input_size, hidden_size) + self.relu = torch.nn.ReLU() + self.fc2 = torch.nn.Linear(hidden_size, num_classes) + self.register_parameter("param1", torch.nn.Parameter(torch.ones(num_classes))) + self.register_parameter("param2", torch.nn.Parameter(1+torch.ones(num_classes))) + + def forward(self, input1): + out = self.fc1(input1) + out = self.relu(out) + out = self.fc2(out) + out += self.param1 + out += self.param2 + out += torch.tensor([3.], device=next(self.parameters()).device) + return out + device = 'cuda' + + N, D_in, H, D_out = 64, 784, 500, 10 + model = NeuralNetWithRegisteredParamsWithConstant(D_in, H, D_out).to(device) + ort_model = ORTModule(model) + # Check that the original forward signature is preserved. + assert signature(model.forward) == signature(ort_model.forward) + x = torch.randn(N, D_in, device=device) + # Make sure model runs without any exception + output = ort_model(x) + assert output is not None diff --git a/orttraining/orttraining/test/python/orttraining_test_ortmodule_poc.py b/orttraining/orttraining/test/python/orttraining_test_ortmodule_poc.py index 1c31f16bbd..8fa062f78e 100644 --- a/orttraining/orttraining/test/python/orttraining_test_ortmodule_poc.py +++ b/orttraining/orttraining/test/python/orttraining_test_ortmodule_poc.py @@ -1,6 +1,5 @@ import argparse import logging -import os import torch import time from torchvision import datasets, transforms From ab86634c365c74847497a02d3d020062b67a8ee9 Mon Sep 17 00:00:00 2001 From: Sherlock Date: Fri, 26 Mar 2021 16:26:42 -0700 Subject: [PATCH 012/129] Address comments from ORTModule master merge (#7101) * Address ortmodule merge master comments Co-authored-by: Sherlock Huang --- cmake/CMakeLists.txt | 4 ---- cmake/onnxruntime_python.cmake | 7 ++++++- .../core/framework/kernel_def_builder.h | 5 +++-- .../core/framework/op_kernel_context.h | 3 +++ .../core/framework/allocation_planner.cc | 12 +++++------ onnxruntime/core/framework/execution_frame.cc | 2 ++ onnxruntime/core/framework/execution_frame.h | 2 ++ onnxruntime/core/framework/op_kernel.cc | 12 ++++++++--- .../framework/op_kernel_context_internal.h | 2 ++ .../dlpack_converter.cc} | 20 +++++++++---------- .../dlpack_converter.h} | 4 ++-- .../python/onnxruntime_pybind_state.cc | 2 +- .../test/framework/execution_frame_test.cc | 2 +- .../orttraining/core/agent/training_agent.cc | 13 ++++++------ .../orttraining/core/agent/training_agent.h | 10 +++++----- .../python/orttraining_pybind_state.cc | 2 +- .../training_ops/cpu/controlflow/ort_tasks.cc | 4 ++-- .../training_ops/cpu/controlflow/ort_tasks.h | 7 +++++-- 18 files changed, 67 insertions(+), 46 deletions(-) rename onnxruntime/python/{dlpack_convertor.cc => dlpack/dlpack_converter.cc} (92%) rename onnxruntime/python/{dlpack_convertor.h => dlpack/dlpack_converter.h} (76%) diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index bdfe589107..e0d99eb40a 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -1111,10 +1111,6 @@ include_directories( ${REPO_ROOT}/include/onnxruntime/core/session ) -# DLPack is a header-only dependency -set(DLPACK_INCLUDE_DIR ${PROJECT_SOURCE_DIR}/external/dlpack/include) -include_directories(${DLPACK_INCLUDE_DIR}) - if(onnxruntime_USE_OPENVINO) add_definitions(-DUSE_OPENVINO=1) diff --git a/cmake/onnxruntime_python.cmake b/cmake/onnxruntime_python.cmake index 9c7f2b12ae..7cb6fb69fa 100644 --- a/cmake/onnxruntime_python.cmake +++ b/cmake/onnxruntime_python.cmake @@ -38,6 +38,8 @@ set(onnxruntime_pybind_srcs_pattern if (onnxruntime_ENABLE_TRAINING) list(APPEND onnxruntime_pybind_srcs_pattern + "${ONNXRUNTIME_ROOT}/python/dlpack/*.cc" + "${ONNXRUNTIME_ROOT}/python/dlpack/*.h" "${ORTTRAINING_ROOT}/orttraining/python/*.cc" "${ORTTRAINING_ROOT}/orttraining/python/*.h" ) @@ -66,12 +68,15 @@ if (MSVC AND NOT CMAKE_SIZEOF_VOID_P EQUAL 8) #TODO: fix the warnings target_compile_options(onnxruntime_pybind11_state PRIVATE "/wd4244") endif() + target_include_directories(onnxruntime_pybind11_state PRIVATE ${ONNXRUNTIME_ROOT} ${PYTHON_INCLUDE_DIR} ${NUMPY_INCLUDE_DIR} ${pybind11_INCLUDE_DIRS}) if(onnxruntime_USE_CUDA) target_include_directories(onnxruntime_pybind11_state PRIVATE ${onnxruntime_CUDNN_HOME}/include) endif() if (onnxruntime_ENABLE_TRAINING) - target_include_directories(onnxruntime_pybind11_state PRIVATE ${ORTTRAINING_ROOT}) + # DLPack is a header-only dependency + set(DLPACK_INCLUDE_DIR ${PROJECT_SOURCE_DIR}/external/dlpack/include) + target_include_directories(onnxruntime_pybind11_state PRIVATE ${ORTTRAINING_ROOT} ${DLPACK_INCLUDE_DIR}) endif() if(APPLE) diff --git a/include/onnxruntime/core/framework/kernel_def_builder.h b/include/onnxruntime/core/framework/kernel_def_builder.h index 9a220feec6..30b9815891 100644 --- a/include/onnxruntime/core/framework/kernel_def_builder.h +++ b/include/onnxruntime/core/framework/kernel_def_builder.h @@ -82,7 +82,7 @@ class KernelDef { bool AllocateInputsContiguously() const { return allocate_inputs_contiguously_; } - bool ExternalOutputs() const { return external_outputs_; } + bool HasExternalOutputs() const { return external_outputs_; } OrtMemType OutputMemoryType(size_t output_index) const { auto it = output_memory_type_args_.find(output_index); @@ -264,7 +264,8 @@ class KernelDefBuilder { } /** - Specify that this kernel's outputs are passed from external. + Specify that this kernel's output buffers are passed from external, + i.e. not created or managed by ORT's memory allocator. */ KernelDefBuilder& ExternalOutputs() { kernel_def_->external_outputs_ = true; diff --git a/include/onnxruntime/core/framework/op_kernel_context.h b/include/onnxruntime/core/framework/op_kernel_context.h index a87781b00c..5e233fcc29 100644 --- a/include/onnxruntime/core/framework/op_kernel_context.h +++ b/include/onnxruntime/core/framework/op_kernel_context.h @@ -175,7 +175,10 @@ class OpKernelContext { const OrtValue* GetInputMLValue(int index) const; const OrtValue* GetImplicitInputMLValue(int index) const; OrtValue* GetOutputMLValue(int index); + +#ifdef ENABLE_TRAINING Status SetOutputMLValue(int index, const OrtValue& ort_value); +#endif // Creates the OrtValue* based on the shape, if it does not exist // The parameter nnz is used only for sparse-tensors and indicates the diff --git a/onnxruntime/core/framework/allocation_planner.cc b/onnxruntime/core/framework/allocation_planner.cc index 0c308faf72..d414f55641 100644 --- a/onnxruntime/core/framework/allocation_planner.cc +++ b/onnxruntime/core/framework/allocation_planner.cc @@ -430,13 +430,13 @@ class PlannerImpl { plan_.allocation_plan.resize(num_ml_values); } - bool ExternalOutputs(const Node& node) const { + bool HasExternalOutputs(const Node& node) const { const KernelCreateInfo& ci = GetKernelCreateInfo(kernel_create_info_map_, node.Index()); if (ci.kernel_def == nullptr) { return false; } - return ci.kernel_def->ExternalOutputs(); + return ci.kernel_def->HasExternalOutputs(); } Status ComputeUseCounts() { @@ -523,14 +523,14 @@ class PlannerImpl { auto outputs = pnode->OutputDefs(); auto num_outputs = outputs.size(); - bool external_outputs = ExternalOutputs(*pnode); + bool has_external_outputs = HasExternalOutputs(*pnode); for (size_t i = 0; i < num_outputs; ++i) { auto* node_output = outputs[i]; if (!node_output->Exists()) continue; OrtValueIndex index = Index(node_output->Name()); ProcessDef(index, node_output); // Ensures external outputs will not be reused. - UseCount(index) += (external_outputs ? 2 : 1); + UseCount(index) += (has_external_outputs ? 2 : 1); auto allocator = exec_provider->GetAllocator(0, p_kernel_def->OutputMemoryType(i)); ORT_ENFORCE(allocator); plan_.SetLocation(static_cast(index), @@ -662,7 +662,7 @@ class PlannerImpl { // node outputs. const auto& output_defs = pnode->OutputDefs(); // External outputs flag. - bool external_outputs = ExternalOutputs(*pnode); + bool has_external_outputs = HasExternalOutputs(*pnode); // output_arg_def_index is the index of ArgDefs in pnode's output list. // At the i-th iteration, we build the allocation plan for the i-th // NodeArg in pnode's output list. Allocation plan remains untouched for @@ -718,7 +718,7 @@ class PlannerImpl { } } } - } else if (external_outputs) { + } else if (has_external_outputs) { ORT_ENFORCE(!IsNonTensor(*node_output), "Only tensors are supported for external outputs for now."); AllocPlan(current).alloc_kind = AllocKind::kAllocatedExternally; #if !defined(ORT_MINIMAL_BUILD) && defined(ORT_MEMORY_PROFILE) diff --git a/onnxruntime/core/framework/execution_frame.cc b/onnxruntime/core/framework/execution_frame.cc index f3e9118ec9..2bbba5a783 100644 --- a/onnxruntime/core/framework/execution_frame.cc +++ b/onnxruntime/core/framework/execution_frame.cc @@ -45,6 +45,7 @@ OrtValue* IExecutionFrame::GetMutableNodeInputOrOutputMLValue(int index) { return const_cast(GetNodeInputOrOutputMLValue(index)); } +#ifdef ENABLE_TRAINING Status IExecutionFrame::SetOutputMLValue(int index, const OrtValue& ort_value) { int ort_value_idx = GetNodeIdxToMLValueIdx(index); if (ort_value_idx == NodeIndexInfo::kInvalidEntry || static_cast(ort_value_idx) >= all_values_size_) { @@ -59,6 +60,7 @@ Status IExecutionFrame::SetOutputMLValue(int index, const OrtValue& ort_value) { all_values_[ort_value_idx] = ort_value; return Status::OK(); } +#endif // TO DO: make it thread safe // This method is not thread safe! diff --git a/onnxruntime/core/framework/execution_frame.h b/onnxruntime/core/framework/execution_frame.h index 01f5e5680e..c495b11a3d 100644 --- a/onnxruntime/core/framework/execution_frame.h +++ b/onnxruntime/core/framework/execution_frame.h @@ -49,8 +49,10 @@ class IExecutionFrame { const OrtValue* GetNodeInputOrOutputMLValue(int index) const; OrtValue* GetMutableNodeInputOrOutputMLValue(int index); +#ifdef ENABLE_TRAINING // Override the index-th output with ort_value Status SetOutputMLValue(int index, const OrtValue& ort_value); +#endif // TO DO: make it thread safe // This method is not thread safe! diff --git a/onnxruntime/core/framework/op_kernel.cc b/onnxruntime/core/framework/op_kernel.cc index eb77efc365..284c92cc94 100644 --- a/onnxruntime/core/framework/op_kernel.cc +++ b/onnxruntime/core/framework/op_kernel.cc @@ -191,15 +191,21 @@ OrtValue* OpKernelContext::GetOutputMLValue(int index) { return execution_frame_->GetMutableNodeInputOrOutputMLValue(output_arg_index); } +#ifdef ENABLE_TRAINING Status OpKernelContext::SetOutputMLValue(int index, const OrtValue& ort_value) { if (index < 0 || index >= OutputCount()) { - return Status(common::ONNXRUNTIME, common::FAIL, "Index out of range. " + - std::to_string(index) + " was specified, but " + - "range is (0, " + std::to_string(OutputCount()) + ")"); + return Status(common::ONNXRUNTIME, common::FAIL, + "Index out of range. " + std::to_string(index) + + " was specified, but " + "range is (0, " + std::to_string(OutputCount()) + ")"); } + ORT_ENFORCE(kernel_->KernelDef().HasExternalOutputs(), + GetOpType() + " is trying to use SetOutputMLValue(), but its kernel_def doesn't have " + ".ExternalOutputs() declared."); + auto output_arg_index = GetOutputArgIndex(index); return execution_frame_->SetOutputMLValue(output_arg_index, ort_value); } +#endif } // namespace onnxruntime diff --git a/onnxruntime/core/framework/op_kernel_context_internal.h b/onnxruntime/core/framework/op_kernel_context_internal.h index d414334f48..4f889a23d7 100644 --- a/onnxruntime/core/framework/op_kernel_context_internal.h +++ b/onnxruntime/core/framework/op_kernel_context_internal.h @@ -53,9 +53,11 @@ class OpKernelContextInternal : public OpKernelContext { return OpKernelContext::GetOutputMLValue(index); } +#ifdef ENABLE_TRAINING Status SetOutputMLValue(int index, const OrtValue& ort_value) { return OpKernelContext::SetOutputMLValue(index, ort_value); } +#endif OrtValue* OutputMLValue(int index, const TensorShape& shape) { return OpKernelContext::OutputMLValue(index, shape); diff --git a/onnxruntime/python/dlpack_convertor.cc b/onnxruntime/python/dlpack/dlpack_converter.cc similarity index 92% rename from onnxruntime/python/dlpack_convertor.cc rename to onnxruntime/python/dlpack/dlpack_converter.cc index eb92960494..2cc6e3bc0b 100644 --- a/onnxruntime/python/dlpack_convertor.cc +++ b/onnxruntime/python/dlpack/dlpack_converter.cc @@ -1,7 +1,7 @@ // Copyright (c) Microsoft Corporation. All rights reserved. // Licensed under the MIT License. -#ifdef ENABLE_TRAINING -#include "python/dlpack_convertor.h" + +#include "python/dlpack/dlpack_converter.h" namespace onnxruntime { namespace python { @@ -193,16 +193,17 @@ bool IsContiguousTensor(const DLTensor& tensor) { } // namespace -// This function returns a shared_ptr to memory managed DLpack tensor -// constructed out of OrtValue. -DLManagedTensor* OrtValueToDlpack(const OrtValue& ort_value) { +// This function returns a pointer to DLManagedTensor constructed from an OrtValue +// The OrtValue inside OrtDLManagedTensor will increase its own buffer's ref count by one +// When the consumer of DLManagedTensor is done with the tensor, it should invoke the deleter. +DLManagedTensor* OrtValueToDlpack(OrtValue& ort_value) { ORT_ENFORCE(ort_value.IsTensor(), "Only tensor type OrtValues are supported"); OrtDLManagedTensor* ort_dlmanaged_tensor(new OrtDLManagedTensor); - const Tensor& tensor = ort_value.Get(); + Tensor& tensor = *ort_value.GetMutable(); ort_dlmanaged_tensor->handle = ort_value; ort_dlmanaged_tensor->tensor.manager_ctx = ort_dlmanaged_tensor; ort_dlmanaged_tensor->tensor.deleter = &DlpackDeleter; - ort_dlmanaged_tensor->tensor.dl_tensor.data = const_cast(tensor.DataRaw()); + ort_dlmanaged_tensor->tensor.dl_tensor.data = (tensor.MutableDataRaw()); ort_dlmanaged_tensor->tensor.dl_tensor.ctx = GetDlpackContext(ort_value, tensor.Location().device.Id()); ort_dlmanaged_tensor->tensor.dl_tensor.ndim = static_cast(tensor.Shape().NumDimensions()); ort_dlmanaged_tensor->tensor.dl_tensor.dtype = GetDlpackDataType(ort_value); @@ -213,12 +214,12 @@ DLManagedTensor* OrtValueToDlpack(const OrtValue& ort_value) { return &(ort_dlmanaged_tensor->tensor); } -OrtValue DlpackToOrtValue(const DLManagedTensor* dlpack, bool is_bool_tensor) { +OrtValue DlpackToOrtValue(DLManagedTensor* dlpack, bool is_bool_tensor) { // ORT only supports contiguous tensor for now. ORT_ENFORCE(IsContiguousTensor(dlpack->dl_tensor), "ORT only supports contiguous tensor for now."); OrtDevice device = GetOrtDevice(dlpack->dl_tensor.ctx); MLDataType data_type = GetOrtValueDataType(dlpack->dl_tensor.dtype, is_bool_tensor); - std::function deleter = [dlpack](void*) { dlpack->deleter(const_cast(dlpack)); }; + std::function deleter = [dlpack](void*) { dlpack->deleter((dlpack)); }; OrtMemoryInfo info(GetOrtDeviceName(device), OrtDeviceAllocator, device, device.Id()); std::unique_ptr p_tensor = onnxruntime::make_unique( data_type, TensorShape(dlpack->dl_tensor.shape, static_cast(dlpack->dl_tensor.ndim)), @@ -231,4 +232,3 @@ OrtValue DlpackToOrtValue(const DLManagedTensor* dlpack, bool is_bool_tensor) { } // namespace python } // namespace onnxruntime -#endif diff --git a/onnxruntime/python/dlpack_convertor.h b/onnxruntime/python/dlpack/dlpack_converter.h similarity index 76% rename from onnxruntime/python/dlpack_convertor.h rename to onnxruntime/python/dlpack/dlpack_converter.h index 98f1c8663c..c0cce9ee8e 100644 --- a/onnxruntime/python/dlpack_convertor.h +++ b/onnxruntime/python/dlpack/dlpack_converter.h @@ -11,11 +11,11 @@ namespace onnxruntime { namespace python { -DLManagedTensor* OrtValueToDlpack(const OrtValue& ort_value); +DLManagedTensor* OrtValueToDlpack(OrtValue& ort_value); // DLPack uses same config for both bool and unit8. Parameter is_bool_tensor is to // tell ORT the data type when creating OrtValue. -OrtValue DlpackToOrtValue(const DLManagedTensor* dlpack, bool is_bool_tensor = false); +OrtValue DlpackToOrtValue(DLManagedTensor* dlpack, bool is_bool_tensor = false); } // namespace python } // namespace onnxruntime diff --git a/onnxruntime/python/onnxruntime_pybind_state.cc b/onnxruntime/python/onnxruntime_pybind_state.cc index 5a26cfd4de..e077548845 100644 --- a/onnxruntime/python/onnxruntime_pybind_state.cc +++ b/onnxruntime/python/onnxruntime_pybind_state.cc @@ -27,7 +27,7 @@ #include "core/session/abi_session_options_impl.h" #ifdef ENABLE_TRAINING -#include "python/dlpack_convertor.h" +#include "python/dlpack/dlpack_converter.h" #endif // execution provider factory creator headers diff --git a/onnxruntime/test/framework/execution_frame_test.cc b/onnxruntime/test/framework/execution_frame_test.cc index 43ba139a11..86a1907179 100644 --- a/onnxruntime/test/framework/execution_frame_test.cc +++ b/onnxruntime/test/framework/execution_frame_test.cc @@ -359,7 +359,7 @@ TEST_F(ExecutionFrameTest, MemPatternWithExternalOutputsTest) { { // Run with new RunForward/RunBackward. - training::TrainingAgent training_agent(&session_obj); + training::TrainingAgent training_agent(session_obj); unique_ptr io_binding; ASSERT_STATUS_OK(session_obj.NewIOBinding(&io_binding)); io_binding->BindInput("X", x_value); diff --git a/orttraining/orttraining/core/agent/training_agent.cc b/orttraining/orttraining/core/agent/training_agent.cc index 88f877edd0..6ab2929494 100644 --- a/orttraining/orttraining/core/agent/training_agent.cc +++ b/orttraining/orttraining/core/agent/training_agent.cc @@ -8,7 +8,7 @@ namespace onnxruntime { namespace training { -TrainingAgent::TrainingAgent(InferenceSession *session) : inference_session_(session) {} +TrainingAgent::TrainingAgent(InferenceSession& session) : inference_session_(session) {} TrainingAgent::~TrainingAgent() { // TODO: Properly cancel outstanding background tasks @@ -39,7 +39,7 @@ common::Status TrainingAgent::RunForward(const RunOptions& run_options, onnxrunt // wait until task is properly setup setup_future.get(); - common::Status status = inference_session_->Run(run_options, io_binding); + common::Status status = inference_session_.Run(run_options, io_binding); onnxruntime::contrib::OrtTasks::GetInstance().SetStatus(status); @@ -52,7 +52,8 @@ common::Status TrainingAgent::RunForward(const RunOptions& run_options, onnxrunt // signal main thread for background thread completion onnxruntime::contrib::OrtTasks::GetInstance().SetForwardOutputs(status, {}); } - }, std::move(setup_future), std::cref(run_options), std::ref(io_binding)); + }, + std::move(setup_future), std::cref(run_options), std::ref(io_binding)); run_id = std::hash()(bg_thread.get_id()); { @@ -62,7 +63,7 @@ common::Status TrainingAgent::RunForward(const RunOptions& run_options, onnxrunt onnxruntime::contrib::OrtTasks::GetInstance().CreateBackgroundTask(run_id); - LOGS(*inference_session_->GetLogger(), VERBOSE) << "InferenceSession::Forward() call created a task with run_id " << run_id; + LOGS(*inference_session_.GetLogger(), VERBOSE) << "InferenceSession::Forward() call created a task with run_id " << run_id; // background task is setup, unblock background thread to continue setup_promise.set_value(); @@ -92,7 +93,7 @@ common::Status TrainingAgent::RunForward(const RunOptions& run_options, onnxrunt } common::Status TrainingAgent::RunBackward(int64_t run_id, const std::vector& backward_output_grads) { - LOGS(*inference_session_->GetLogger(), VERBOSE) << "Running TrainingAgent::Backward() with run_id " << run_id; + LOGS(*inference_session_.GetLogger(), VERBOSE) << "Running TrainingAgent::Backward() with run_id " << run_id; // resume background thread onnxruntime::contrib::OrtTasks::GetInstance().SetBackwardInputs(run_id, backward_output_grads, false); @@ -115,7 +116,7 @@ common::Status TrainingAgent::RunBackward(int64_t run_id, const std::vectorGetLogger(), INFO) << "Canceling background task with run_id " << run_id; + LOGS(*inference_session_.GetLogger(), INFO) << "Canceling background task with run_id " << run_id; // resume background thread with terminate = true onnxruntime::contrib::OrtTasks::GetInstance().SetBackwardInputs(run_id, {}, true); diff --git a/orttraining/orttraining/core/agent/training_agent.h b/orttraining/orttraining/core/agent/training_agent.h index 8918650705..ad9ef73c21 100644 --- a/orttraining/orttraining/core/agent/training_agent.h +++ b/orttraining/orttraining/core/agent/training_agent.h @@ -21,10 +21,10 @@ class IOBinding; class TrainingAgent { public: - explicit TrainingAgent(InferenceSession* session); - virtual ~TrainingAgent(); + explicit TrainingAgent(InferenceSession& session); + ~TrainingAgent(); // For ORTModule.forward() - virtual common::Status RunForward(const RunOptions& run_options, onnxruntime::IOBinding& io_binding, + common::Status RunForward(const RunOptions& run_options, onnxruntime::IOBinding& io_binding, std::vector& user_outputs, int64_t& run_id) ORT_MUST_USE_RESULT; // For ORTModule.backward() @@ -34,10 +34,10 @@ class TrainingAgent { private: // mutex for accessing bg_threads_ std::mutex bg_threads_mutex_; - // background threads for RunInBackgroundAndWaitForYield and ContinueRunInBackground + // background threads for RunForward and RunBackward std::unordered_map bg_threads_; // TrainingAgent runs on a InferenceSession under the hood - InferenceSession* inference_session_; + InferenceSession& inference_session_; }; } // namespace training diff --git a/orttraining/orttraining/python/orttraining_pybind_state.cc b/orttraining/orttraining/python/orttraining_pybind_state.cc index 61d7d993cb..c458a130aa 100644 --- a/orttraining/orttraining/python/orttraining_pybind_state.cc +++ b/orttraining/orttraining/python/orttraining_pybind_state.cc @@ -479,7 +479,7 @@ py::class_(m, "TrainingAgent", R"pbdoc(This is the main class use // In Python3, a Python bytes object will be passed to C++ functions that accept std::string or char* // without any conversion. So this init method can be used for model file path (string) and model content (bytes) .def(py::init([](PyInferenceSession * session) { - return onnxruntime::make_unique(session->GetSessionHandle()); + return onnxruntime::make_unique(*session->GetSessionHandle()); })) .def("run_forward", [](TrainingAgent* agent, SessionIOBinding& io_binding, RunOptions& run_options) -> py::tuple { std::vector module_outputs; diff --git a/orttraining/orttraining/training_ops/cpu/controlflow/ort_tasks.cc b/orttraining/orttraining/training_ops/cpu/controlflow/ort_tasks.cc index 767bede9da..b94c7dfdc6 100644 --- a/orttraining/orttraining/training_ops/cpu/controlflow/ort_tasks.cc +++ b/orttraining/orttraining/training_ops/cpu/controlflow/ort_tasks.cc @@ -39,7 +39,7 @@ ForwardReturnType OrtTasks::WaitForForwardOutputs(int64_t run_id) { return task->forward_output_future_.get(); } -bool OrtTasks::ForwardOutputsIsValid() { +bool OrtTasks::ForwardOutputsIsValid() const{ int64_t run_id = hasher_(std::this_thread::get_id()); std::lock_guard lock(mutex_); @@ -76,7 +76,7 @@ void OrtTasks::SetStatus(const Status& status) { iter->second->status_promise_.set_value(status); } -bool OrtTasks::TaskIsCompleted(int64_t run_id) { +bool OrtTasks::TaskIsCompleted(int64_t run_id) const { std::lock_guard lock(mutex_); auto iter = bg_tasks_.find(run_id); ORT_ENFORCE(iter != bg_tasks_.end()); diff --git a/orttraining/orttraining/training_ops/cpu/controlflow/ort_tasks.h b/orttraining/orttraining/training_ops/cpu/controlflow/ort_tasks.h index 56993a97d5..91583ad823 100644 --- a/orttraining/orttraining/training_ops/cpu/controlflow/ort_tasks.h +++ b/orttraining/orttraining/training_ops/cpu/controlflow/ort_tasks.h @@ -13,7 +13,10 @@ namespace onnxruntime { namespace contrib { +// the pair is typedef std::pair> ForwardReturnType; + +// the pair is typedef std::pair> BackwardReturnType; class OrtTasks final { @@ -28,14 +31,14 @@ class OrtTasks final { void SetForwardOutputs(Status s, const std::vector& forward_outputs); ForwardReturnType WaitForForwardOutputs(int64_t run_id); - bool ForwardOutputsIsValid(); + bool ForwardOutputsIsValid() const; void SetBackwardInputs(int64_t run_id, const std::vector& backward_inputs, bool terminate); BackwardReturnType WaitForBackwardInputs(); void SetStatus(const Status& status); Status WaitForStatus(int64_t run_id); - bool TaskIsCompleted(int64_t run_id); + bool TaskIsCompleted(int64_t run_id) const; private: OrtTasks() = default; From 63d9d5afd3b8172cb13915b8385d8a5246a4ac72 Mon Sep 17 00:00:00 2001 From: Edward Chen <18449977+edgchen1@users.noreply.github.com> Date: Fri, 26 Mar 2021 17:36:31 -0700 Subject: [PATCH 013/129] Fix Pad and Gather incorrect usage of HasType helpers. (#7146) --- onnxruntime/core/providers/cpu/tensor/gather.cc | 4 ++-- onnxruntime/core/providers/cpu/tensor/pad.cc | 17 +++++------------ 2 files changed, 7 insertions(+), 14 deletions(-) diff --git a/onnxruntime/core/providers/cpu/tensor/gather.cc b/onnxruntime/core/providers/cpu/tensor/gather.cc index 5f31fa57ad..7ed3cf2f22 100644 --- a/onnxruntime/core/providers/cpu/tensor/gather.cc +++ b/onnxruntime/core/providers/cpu/tensor/gather.cc @@ -148,12 +148,12 @@ Status Gather::Compute(OpKernelContext* context) const { concurrency::ThreadPool* tp = context->GetOperatorThreadPool(); - if (utils::HasTypeWithSameSize() && + if (utils::HasType() && p.indices_tensor->IsDataType()) { return GatherCopyData(p.indices_tensor, src_base, dst_base, is_string_type, element_bytes, block_size, M, N, data_batch_bytes, gathered_batch_bytes, input_data_shape, p.axis, tp); } - if (utils::HasTypeWithSameSize() && + if (utils::HasType() && p.indices_tensor->IsDataType()) { return GatherCopyData(p.indices_tensor, src_base, dst_base, is_string_type, element_bytes, block_size, M, N, data_batch_bytes, gathered_batch_bytes, input_data_shape, p.axis, tp); diff --git a/onnxruntime/core/providers/cpu/tensor/pad.cc b/onnxruntime/core/providers/cpu/tensor/pad.cc index 2cd3830831..0438783307 100644 --- a/onnxruntime/core/providers/cpu/tensor/pad.cc +++ b/onnxruntime/core/providers/cpu/tensor/pad.cc @@ -244,7 +244,7 @@ static Status PadImpl(OpKernelContext* ctx, const std::vector& slices, const Mode& mode, T value) { - if (!utils::HasType()) { + if (!utils::HasTypeWithSameSize()) { return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Input data type not supported in this build."); } @@ -517,22 +517,15 @@ Status Pad::Compute(OpKernelContext* ctx) const { slices_to_use = &slices_; } - Status pad_status{}; switch (element_size) { case sizeof(uint32_t): - pad_status = PadImpl(ctx, *pads_to_use, *slices_to_use, mode_, value.u32); - break; + return PadImpl(ctx, *pads_to_use, *slices_to_use, mode_, value.u32); case sizeof(uint64_t): - pad_status = PadImpl(ctx, *pads_to_use, *slices_to_use, mode_, value.u64); - break; + return PadImpl(ctx, *pads_to_use, *slices_to_use, mode_, value.u64); case sizeof(uint8_t): - pad_status = PadImpl(ctx, *pads_to_use, *slices_to_use, mode_, value.u8); - break; + return PadImpl(ctx, *pads_to_use, *slices_to_use, mode_, value.u8); default: - pad_status = ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Unsupported input data type of ", data_type); - break; + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Unsupported input data type of ", data_type); } - - return pad_status; } }; // namespace onnxruntime From f365f1d967edfaa942be86653c5b838d3d99e9ff Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Fri, 26 Mar 2021 18:29:21 -0700 Subject: [PATCH 014/129] Resize_impl.cu: Change _Round to roundf (#7140) This is to keep the change minimal, make it work exactly like what it worked before. --- onnxruntime/core/providers/cuda/tensor/resize_impl.cu | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/onnxruntime/core/providers/cuda/tensor/resize_impl.cu b/onnxruntime/core/providers/cuda/tensor/resize_impl.cu index d0a5c79a58..a73a3e7598 100644 --- a/onnxruntime/core/providers/cuda/tensor/resize_impl.cu +++ b/onnxruntime/core/providers/cuda/tensor/resize_impl.cu @@ -20,11 +20,11 @@ __device__ int NearestPixel_ROUND_PREFER_FLOOR(float x_original, bool) { if (x_original == static_cast(x_original) + 0.5f) { return static_cast(_Floor(x_original)); } - return static_cast(_Round(x_original)); + return static_cast(roundf(x_original)); } __device__ int NearestPixel_ROUND_PREFER_CEIL(float x_original, bool) { - return static_cast(_Round(x_original)); + return static_cast(roundf(x_original)); } __device__ int NearestPixel_FLOOR(float x_original, bool) { From f27835c4dee32e83ac8a4c947d64825b32580079 Mon Sep 17 00:00:00 2001 From: Suffian Khan Date: Fri, 26 Mar 2021 22:32:39 -0500 Subject: [PATCH 015/129] Disable batch size test for AMD CI pipeline after agent upgrade to Rocm 4.1 (#7153) * disable batch size test for rocm 4.1 until resolved * Update orttraining-pai-ci-pipeline.yml Forgot to modify both pipelines --- ...training-linux-gpu-amd-e2e-test-ci-pipeline.yml | 14 +++++++------- .../orttraining-pai-ci-pipeline.yml | 14 +++++++------- 2 files changed, 14 insertions(+), 14 deletions(-) diff --git a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-amd-e2e-test-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-amd-e2e-test-ci-pipeline.yml index c16ae5971f..8decc907fe 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-amd-e2e-test-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-linux-gpu-amd-e2e-test-ci-pipeline.yml @@ -50,13 +50,13 @@ jobs: ../../tools/ci_build/github/pai/pai_test_launcher.sh displayName: 'Run unit tests' - - script: |- - python orttraining/tools/ci_test/run_batch_size_test.py \ - --binary_dir build/RelWithDebInfo \ - --model_root training_e2e_test_data/models \ - --gpu_sku MI100_32G - displayName: 'Run batch size test' - condition: succeededOrFailed() # ensure all tests are run +# - script: |- +# python orttraining/tools/ci_test/run_batch_size_test.py \ +# --binary_dir build/RelWithDebInfo \ +# --model_root training_e2e_test_data/models \ +# --gpu_sku MI100_32G +# displayName: 'Run batch size test' +# condition: succeededOrFailed() # ensure all tests are run - script: |- python orttraining/tools/ci_test/run_convergence_test.py \ diff --git a/tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml b/tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml index fe891611b8..0fcaf4d35c 100644 --- a/tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/orttraining-pai-ci-pipeline.yml @@ -44,13 +44,13 @@ steps: ../../tools/ci_build/github/pai/pai_test_launcher.sh displayName: 'Run unit tests' -- script: |- - python orttraining/tools/ci_test/run_batch_size_test.py \ - --binary_dir build/RelWithDebInfo \ - --model_root training_e2e_test_data/models \ - --gpu_sku MI100_32G - displayName: 'Run batch size test' - condition: succeededOrFailed() # ensure all tests are run +#- script: |- +# python orttraining/tools/ci_test/run_batch_size_test.py \ +# --binary_dir build/RelWithDebInfo \ +# --model_root training_e2e_test_data/models \ +# --gpu_sku MI100_32G +# displayName: 'Run batch size test' +# condition: succeededOrFailed() # ensure all tests are run - script: |- python orttraining/tools/ci_test/run_convergence_test.py \ From 65ce5f07b3761f441fb8e98740f39e69cd89cc58 Mon Sep 17 00:00:00 2001 From: Jeff Daily Date: Fri, 26 Mar 2021 21:40:10 -0700 Subject: [PATCH 016/129] add Dockerfile.rocm4.1.pytorch (#7152) --- .../tools/amdgpu/Dockerfile.rocm4.1.pytorch | 197 ++++++++++++++++++ 1 file changed, 197 insertions(+) create mode 100644 orttraining/tools/amdgpu/Dockerfile.rocm4.1.pytorch diff --git a/orttraining/tools/amdgpu/Dockerfile.rocm4.1.pytorch b/orttraining/tools/amdgpu/Dockerfile.rocm4.1.pytorch new file mode 100644 index 0000000000..b63d8b73a0 --- /dev/null +++ b/orttraining/tools/amdgpu/Dockerfile.rocm4.1.pytorch @@ -0,0 +1,197 @@ +# docker build --network=host --file Dockerfile.rocm4.1.pytorch --tag ort:rocm4.1-pytorch . + +FROM rocm/pytorch:rocm4.1_ubuntu18.04_py3.6_pytorch + +RUN apt-get -y install gpg-agent +RUN wget -q -O - http://repo.radeon.com/rocm/rocm.gpg.key | apt-key add - +RUN echo 'deb [arch=amd64] http://repo.radeon.com/rocm/apt/4.1/ xenial main' | tee /etc/apt/sources.list.d/rocm.list + +RUN apt-get -y update +RUN apt-get -y install apt-utils +RUN apt-get -y install build-essential autotools-dev \ + make git curl vim wget rsync jq openssh-server openssh-client sudo \ + iputils-ping net-tools ethtool libcap2 \ + automake autoconf libtool flex doxygen \ + perl lsb-release iproute2 pciutils graphviz \ + bc tar git bash pbzip2 pv bzip2 unzip cabextract \ + g++ gcc \ + && apt-get autoremove + +# sh +RUN rm /bin/sh && ln -s /bin/bash /bin/sh + +# Labels for the docker +LABEL description="This docker sets up the environment to run ORT Training with AMD GPU" + +# CMake +ENV CMAKE_VERSION=3.18.2 +RUN cd /usr/local && \ + wget -q -O - https://github.com/Kitware/CMake/releases/download/v${CMAKE_VERSION}/cmake-${CMAKE_VERSION}-Linux-x86_64.tar.gz | tar zxf - +ENV PATH=/usr/local/cmake-${CMAKE_VERSION}-Linux-x86_64/bin:${PATH} + +ENV WORKSPACE_DIR=/workspace +RUN mkdir -p $WORKSPACE_DIR +WORKDIR $WORKSPACE_DIR + +ENV OLD_PATH=${PATH} +ENV PATH=/usr/bin:${PATH} +# Infiniband setup, openmpi installed under /usr/mpi/gcc/openmpi-4.0.4rc3 doesn't support multi-thread +ENV MOFED_VERSION=5.1-0.6.6.0 +ENV MOFED_OS=ubuntu18.04 +ENV MOFED_FILENAME=MLNX_OFED_LINUX-${MOFED_VERSION}-${MOFED_OS}-x86_64 +RUN curl -fSsL https://www.mellanox.com/downloads/ofed/MLNX_OFED-${MOFED_VERSION}/${MOFED_FILENAME}.tgz | tar -zxpf - +RUN cd MLNX_OFED_LINUX-${MOFED_VERSION}-${MOFED_OS}-x86_64 && \ + ./mlnxofedinstall --force --user-space-only --without-fw-update --hpc && \ + cd .. && \ + rm -r MLNX_OFED_LINUX-${MOFED_VERSION}-${MOFED_OS}-x86_64 + +ENV PATH=${OLD_PATH} +ENV unset OLD_PATH + +# python env +RUN pip3 install --upgrade setuptools +ARG NUMPY_VERSION=1.18.5 +ARG ONNX_VERSION=1.7.0 +RUN pip3 install --no-cache-dir wheel tqdm boto3 requests six ipdb h5py html2text nltk progressbar pyyaml \ + git+https://github.com/NVIDIA/dllogger \ + numpy==${NUMPY_VERSION} \ + onnx=="${ONNX_VERSION}" + +ENV GITHUB_DIR=$WORKSPACE_DIR/github +RUN mkdir -p $GITHUB_DIR + +# UCX +WORKDIR $GITHUB_DIR +RUN apt-get -y update && apt-get -y --no-install-recommends install libnuma-dev +ARG UCX_VERSION=1.9.0-rc3 +ENV UCX_DIR=$WORKSPACE_DIR/ucx-$UCX_VERSION +RUN git clone https://github.com/openucx/ucx.git \ + && cd ucx \ + && git checkout v$UCX_VERSION \ + && ./autogen.sh \ + && mkdir build \ + && cd build \ + && ../contrib/configure-opt --prefix=$UCX_DIR --without-rocm --without-knem --without-cuda \ + && make -j"$(nproc)" \ + && make install + +# OpenMPI +# note: require --enable-orterun-prefix-by-default for Azure machine learning compute +# note: disable verbs as we use ucx middleware and don't want btl openib warnings +WORKDIR $GITHUB_DIR +ARG OPENMPI_BASEVERSION=4.0 +ARG OPENMPI_VERSION=${OPENMPI_BASEVERSION}.5 +ENV OPENMPI_DIR=$WORKSPACE_DIR/openmpi-${OPENMPI_VERSION} +RUN git clone --recursive https://github.com/open-mpi/ompi.git \ + && cd ompi \ + && git checkout v$OPENMPI_VERSION \ + && ./autogen.pl \ + && mkdir build \ + && cd build \ + && ../configure --prefix=$OPENMPI_DIR --with-ucx=$UCX_DIR --without-verbs \ + --enable-mpirun-prefix-by-default --enable-orterun-prefix-by-default \ + --enable-mca-no-build=btl-uct --disable-mpi-fortran \ + && make -j"$(nproc)" \ + && make install \ + && ldconfig \ + && test -f ${OPENMPI_DIR}/bin/mpic++ + +ENV PATH=$OPENMPI_DIR/bin:${PATH} +ENV LD_LIBRARY_PATH=$OPENMPI_DIR/lib:${LD_LIBRARY_PATH} + +# Create a wrapper for OpenMPI to allow running as root by default +RUN mv $OPENMPI_DIR/bin/mpirun $OPENMPI_DIR/bin/mpirun.real && \ + echo '#!/bin/bash' > $OPENMPI_DIR/bin/mpirun && \ + echo 'mpirun.real --allow-run-as-root "$@"' >> $OPENMPI_DIR/bin/mpirun && \ + chmod a+x $OPENMPI_DIR/bin/mpirun + +# install mpi4py (be sure to link existing /opt/openmpi-xxx) +RUN CC=mpicc MPICC=mpicc pip install mpi4py --no-binary mpi4py + +ARG CACHE_DATA=2020-12-06 + +# ONNX Runtime +WORKDIR $GITHUB_DIR +ENV ORT_DIR=$GITHUB_DIR/onnxruntime +RUN git clone --recursive https://github.com/microsoft/onnxruntime.git \ + && cd onnxruntime \ + && python3 tools/ci_build/build.py \ + --cmake_extra_defines ONNXRUNTIME_VERSION=`cat ./VERSION_NUMBER` \ + --build_dir build \ + --config RelWithDebInfo \ + --parallel \ + --skip_tests \ + --build_wheel \ + --use_rocm --rocm_home /opt/rocm \ + --mpi_home $OPENMPI_DIR \ + --nccl_home /opt/rocm \ + --enable_training \ + && test -f $ORT_DIR/build/RelWithDebInfo/onnxruntime_training_bert \ + && pip install $ORT_DIR/build/RelWithDebInfo/dist/*.whl \ + && ldconfig + +# ONNX Runtime Training Examples +WORKDIR $GITHUB_DIR +ARG GPT2_DATASET=wikitext-103 +RUN git clone -b wezhan/amdgpu https://github.com/microsoft/onnxruntime-training-examples.git \ + && cd onnxruntime-training-examples \ + # Nvidia BERT + && git clone --no-checkout https://github.com/NVIDIA/DeepLearningExamples.git \ + && cd DeepLearningExamples \ + && git checkout cf54b787 \ + && cd .. \ + && mv DeepLearningExamples/PyTorch/LanguageModeling/BERT ${WORKSPACE_DIR} \ + && rm -rf DeepLearningExamples \ + && cp -r ./nvidia-bert/ort_addon/* ${WORKSPACE_DIR}/BERT \ + # GPT2 fine-tuning + && cd huggingface-gpt2 \ + && git clone https://github.com/huggingface/transformers.git \ + && cd transformers \ + && git checkout 9a0a8c1c6f4f2f0c80ff07d36713a3ada785eec5 \ + && cd .. \ + && mkdir -p ${WORKSPACE_DIR}/GPT2 \ + && cp -r transformers ${WORKSPACE_DIR}/GPT2 \ + && cd ${WORKSPACE_DIR}/GPT2/transformers \ + && git apply $GITHUB_DIR/onnxruntime-training-examples/huggingface-gpt2/ort_addon/src_changes.patch \ + && cp -r $GITHUB_DIR/onnxruntime-training-examples/huggingface-gpt2/ort_addon/ort_supplement/* ./ \ + && python3 -m pip install --no-cache-dir -e . \ + && python3 -m pip install --no-cache-dir -r examples/requirements.txt \ + && python3 -m pip install cerberus sympy \ + && cd .. \ + && wget https://s3.amazonaws.com/research.metamind.io/wikitext/${GPT2_DATASET}-v1.zip \ + && unzip ${GPT2_DATASET}-v1.zip + +ENV BERT_DIR=${WORKSPACE_DIR}/BERT +ENV GPT2_DIR=${WORKSPACE_DIR}/GPT2 +ENV TRAIN_FILE=${WORKSPACE_DIR}/GPT2/${GPT2_DATASET}/wiki.train.tokens +ENV TEST_FILE=${WORKSPACE_DIR}/GPT2/${GPT2_DATASET}/wiki.test.tokens + +# Enable ssh access without password needed +RUN sed -i 's/#PermitRootLogin prohibit-password/PermitRootLogin yes/g' /etc/ssh/sshd_config +RUN sed -i 's/#StrictModes yes/StrictModes no/g' /etc/ssh/sshd_config +RUN sed -i 's/#PubkeyAuthentication yes/PubkeyAuthentication yes/g' /etc/ssh/sshd_config +RUN sed -i 's/#PermitEmptyPasswords no/PermitEmptyPasswords yes/g' /etc/ssh/sshd_config + +# Start or Restart sshd service +ENTRYPOINT service ssh restart && /bin/bash + +# Add model and scripts +ADD model ${WORKSPACE_DIR}/model +ADD script ${WORKSPACE_DIR}/script +RUN chmod a+x ${WORKSPACE_DIR}/script/run_bert.sh + +# add locale en_US.UTF-8 +RUN apt-get install -y locales +RUN locale-gen en_US.UTF-8 + +# Workaround an issue in AMD compiler which generates poor GPU ISA +# when the type of kernel parameter is a structure and “pass-by-value†is used +ENV HSA_NO_SCRATCH_RECLAIM=1 + +# Distributed training related environment variables +ENV HSA_FORCE_FINE_GRAIN_PCIE=1 +ENV NCCL_DEBUG=INFO +ENV RCCL_ALLTOALL_KERNEL_DISABLE=1 +# ENV NCCL_DEBUG_SUBSYS=INIT,COLL + +WORKDIR ${WORKSPACE_DIR}/script From 90294b9c433f8d2319bfa38e0a34caa707725b15 Mon Sep 17 00:00:00 2001 From: satyajandhyala Date: Sun, 28 Mar 2021 09:24:12 -0700 Subject: [PATCH 017/129] =?UTF-8?q?Fix=20Transpose=20and=20MatMul=20fusion?= =?UTF-8?q?=20code=20to=20check=20the=20input=20datatypes=20as=20=E2=80=A6?= =?UTF-8?q?=20(#7147)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Fix Transpose and MatMul fusion code to check the input datatypes as FusedMatMul only supports floating point datatypes. * Added testcases to make sure that the int32/int64 datatypes prevent Transport-MatMul fusion. --- .../core/optimizer/matmul_transpose_fusion.cc | 17 ++++++++++ .../test/optimizer/graph_transform_test.cc | 4 ++- ...tmul_4d_fusion_invalid_datatype_int32.onnx | Bin 0 -> 287 bytes ...tmul_4d_fusion_invalid_datatype_int64.onnx | Bin 0 -> 287 bytes .../transform/fusion/transpose_matmul_gen.py | 31 ++++++++++++++++++ 5 files changed, 51 insertions(+), 1 deletion(-) create mode 100644 onnxruntime/test/testdata/transform/fusion/transpose_matmul_4d_fusion_invalid_datatype_int32.onnx create mode 100644 onnxruntime/test/testdata/transform/fusion/transpose_matmul_4d_fusion_invalid_datatype_int64.onnx diff --git a/onnxruntime/core/optimizer/matmul_transpose_fusion.cc b/onnxruntime/core/optimizer/matmul_transpose_fusion.cc index 6fac01a033..703babffe9 100644 --- a/onnxruntime/core/optimizer/matmul_transpose_fusion.cc +++ b/onnxruntime/core/optimizer/matmul_transpose_fusion.cc @@ -168,6 +168,15 @@ static Node* ReorderCastAndTranspose(Graph& graph, Node* cast, return &new_transpose; } +// Check whether the element_type is an allowed FusedMatMul data type or not. +static bool IsAllowedFusedMatMulDataType(ONNX_NAMESPACE::TensorProto_DataType element_type) +{ + return element_type == ONNX_NAMESPACE::TensorProto_DataType_FLOAT || + element_type == ONNX_NAMESPACE::TensorProto_DataType_FLOAT16 || + element_type == ONNX_NAMESPACE::TensorProto_DataType_DOUBLE || + element_type == ONNX_NAMESPACE::TensorProto_DataType_BFLOAT16; +} + /********************************************************************************************* Case I: The followin is a scenario where Transpose output feeds MatMul. The Transpose input can be either on the left or right. @@ -277,9 +286,17 @@ Status MatmulTransposeFusion::ApplyImpl(Graph& graph, bool& modified, int graph_ } NodeArg* left_input = node.MutableInputDefs()[0]; + auto left_type = left_input->TypeAsProto()->tensor_type().elem_type(); + if (!IsAllowedFusedMatMulDataType(static_cast(left_type))) { + continue; + } auto left = GetTransposeNodeFromOutput(graph, *left_input); NodeArg* right_input = node.MutableInputDefs()[1]; + auto right_type = right_input->TypeAsProto()->tensor_type().elem_type(); + if (!IsAllowedFusedMatMulDataType(static_cast(right_type))) { + continue; + } auto right = GetTransposeNodeFromOutput(graph, *right_input); if (!left) { diff --git a/onnxruntime/test/optimizer/graph_transform_test.cc b/onnxruntime/test/optimizer/graph_transform_test.cc index 4d5453a047..2a0872e48b 100644 --- a/onnxruntime/test/optimizer/graph_transform_test.cc +++ b/onnxruntime/test/optimizer/graph_transform_test.cc @@ -918,10 +918,12 @@ TEST_F(GraphTransformationTests, TransposeMatmulFusionOnThreeTranspose) { ASSERT_TRUE(static_cast(node.GetAttributes().at("transB").i())); } -TEST_F(GraphTransformationTests, TransposeMatmulNoFusionOnInvalidPerm) { +TEST_F(GraphTransformationTests, TransposeMatmulNoFusionOnInvalidInput) { const std::vector model_uris = { MODEL_FOLDER "fusion/transpose_matmul_4d_fusion_invalid_perm.onnx", MODEL_FOLDER "fusion/transpose_matmul_4d_fusion_invalid_default_perm.onnx", + MODEL_FOLDER "fusion/transpose_matmul_4d_fusion_invalid_datatype_int32.onnx", + MODEL_FOLDER "fusion/transpose_matmul_4d_fusion_invalid_datatype_int64.onnx", }; for (const auto& model_uri : model_uris) { std::shared_ptr p_model; diff --git a/onnxruntime/test/testdata/transform/fusion/transpose_matmul_4d_fusion_invalid_datatype_int32.onnx b/onnxruntime/test/testdata/transform/fusion/transpose_matmul_4d_fusion_invalid_datatype_int32.onnx new file mode 100644 index 0000000000000000000000000000000000000000..76386889014baba434990fb10f0945bf2938bd15 GIT binary patch literal 287 zcmd;J7ZS+N%d03V%`3^wP1P+)EiSQo&&XxX#h#g0P+AgiAS6^$l$cjskYAjd5)Tzq z;tYX_X$f+%6r>jAIxsjeIxsshEnsBl(&fUU2WFt55Li&==#8$|db6dB@Fwx1+@^H8efn>eoic0 a3<5k(f?T}G`MG+znaM@@#rbI^0*n9%EJkVo literal 0 HcmV?d00001 diff --git a/onnxruntime/test/testdata/transform/fusion/transpose_matmul_4d_fusion_invalid_datatype_int64.onnx b/onnxruntime/test/testdata/transform/fusion/transpose_matmul_4d_fusion_invalid_datatype_int64.onnx new file mode 100644 index 0000000000000000000000000000000000000000..63728e2ff920a6b2711a77ca1d63ed460f18498c GIT binary patch literal 287 zcmd;J7ZS+N%d03V%`3^wP1P+)EiSQo&&XxX#h#g0P+AgiAS6^$l$cjskYAjd5)Tzq z;tYX_X$f+%6r>jAIxsjeIxsshEnsBl(&fUU2WFt55Li&==#8$|db6dB@Fwx1+@^H8efn>eoic0 a3<5k(f?T}G`MG+znaM@@#rbI^0*n9&BSvii literal 0 HcmV?d00001 diff --git a/onnxruntime/test/testdata/transform/fusion/transpose_matmul_gen.py b/onnxruntime/test/testdata/transform/fusion/transpose_matmul_gen.py index dda0efb108..1f2bc05551 100644 --- a/onnxruntime/test/testdata/transform/fusion/transpose_matmul_gen.py +++ b/onnxruntime/test/testdata/transform/fusion/transpose_matmul_gen.py @@ -221,3 +221,34 @@ def gen_transpose_fusion_with_cast(model_path): gen_transpose_fusion_with_cast( "transpose_cast_matmul_4d_fusion") + +def gen_transpose_fusion_invalid_datatype(model_path, datatype): + nodes = [ + helper.make_node( + "Transpose", + ["input_0"], + ["transposed_input_0"], + perm = [0, 1, 3, 2]), + helper.make_node( + "MatMul", + ["transposed_input_0", "input_1"], + ["output"]) + ] + + inputs = [ + helper.make_tensor_value_info( + "input_0", datatype, [2, 3, 'K', 'M']), + helper.make_tensor_value_info( + "input_1", datatype, [2, 3, 'K', 'N']) + ] + + outputs = [ + helper.make_tensor_value_info( + "output", datatype, [2, 3, 'M', 'N']) + ] + + save(model_path, nodes, inputs, outputs, []) + + +gen_transpose_fusion_invalid_datatype("transpose_matmul_4d_fusion_invalid_datatype_int32.onnx", TensorProto.INT32) +gen_transpose_fusion_invalid_datatype("transpose_matmul_4d_fusion_invalid_datatype_int64.onnx", TensorProto.INT64) From 9297527b7ad14661480ac29cd9b855e0163d79a4 Mon Sep 17 00:00:00 2001 From: Scott McKay Date: Mon, 29 Mar 2021 18:39:48 +1000 Subject: [PATCH 018/129] Enable NHWC transformer when generating ORT format model (#7126) * Allow specific optimizers to be disabled. - replace unused ability to specify just the optimizers to run - never used so not needed Allow the disabled list to be specified via the python bindings - expected usage is internal, so using kwargs for that so as not to pollute the documentation with stuff no user is likely to need Update the ORT format model conversion script to disable NCHWc transformer when level is 'all' - currently there aren't any known use cases where we'd want the NCHWc transformations to run as they create a device specific model and aren't used on ARM - the ORT format model is not expected to be generated on the target device (e.g. generate on Windows/Linux/macOS to deploy to Android/iOS so there's a good chance we'd generate a useless/invalid model - default to 'all' as ARM and MLAS prefer NHWC and the NHWC transformer runs at that level * Add matching changes to optimizer generation in training code --- .../core/optimizer/graph_transformer_utils.h | 28 ++-- .../onnxruntime_session_options_config_keys.h | 1 + .../core/optimizer/graph_transformer_utils.cc | 149 ++++++++++-------- onnxruntime/core/session/inference_session.cc | 40 ++--- onnxruntime/core/session/inference_session.h | 20 +-- .../onnxruntime_inference_collection.py | 17 +- .../python/onnxruntime_pybind_state.cc | 20 ++- .../python/onnxruntime_pybind_state_common.h | 3 +- .../test/optimizer/graph_transform_test.cc | 47 ++++-- .../optimizer/graph_transform_utils_test.cc | 51 +++--- .../core/optimizer/graph_transformer_utils.cc | 72 ++++----- .../core/optimizer/graph_transformer_utils.h | 4 +- .../core/session/training_session.cc | 46 ++---- .../core/session/training_session.h | 42 +++-- .../optimizer/graph_transformer_utils_test.cc | 31 ++-- tools/python/convert_onnx_models_to_ort.py | 26 +-- 16 files changed, 324 insertions(+), 273 deletions(-) diff --git a/include/onnxruntime/core/optimizer/graph_transformer_utils.h b/include/onnxruntime/core/optimizer/graph_transformer_utils.h index 9c9ff16360..121d20420f 100644 --- a/include/onnxruntime/core/optimizer/graph_transformer_utils.h +++ b/include/onnxruntime/core/optimizer/graph_transformer_utils.h @@ -18,24 +18,26 @@ namespace optimizer_utils { /** Generates all predefined rules for this level. If rules_to_enable is not empty, it returns the intersection of predefined rules and rules_to_enable. TODO: This is visible for testing at the moment, but we should rather make it private. */ -std::vector> GenerateRewriteRules(TransformerLevel level, - const std::vector& rules_to_enable = {}); - -/** Generates all predefined (both rule-based and non-rule-based) transformers for this level. - If transformers_and_rules_to_enable is not empty, it returns the intersection between the predefined transformers/rules - and the transformers_and_rules_to_enable. */ -std::vector> GenerateTransformers(TransformerLevel level, - const SessionOptions& session_options, - const IExecutionProvider& execution_provider /*required by constant folding*/, - const std::vector& rules_and_transformers_to_enable = {}); +std::vector> GenerateRewriteRules( + TransformerLevel level, + const std::unordered_set& rules_to_disable = {}); /** Given a TransformerLevel, this method generates a name for the rule-based graph transformer of that level. */ std::string GenerateRuleBasedTransformerName(TransformerLevel level); /** Generates all rule-based transformers for this level. */ -std::unique_ptr GenerateRuleBasedGraphTransformer(TransformerLevel level, - const std::vector& rules_to_enable, - const std::unordered_set& compatible_execution_providers); +std::unique_ptr GenerateRuleBasedGraphTransformer( + TransformerLevel level, + const std::unordered_set& rules_to_disable, + const std::unordered_set& compatible_execution_providers); + +/** Generates all predefined (both rule-based and non-rule-based) transformers for this level. + Any transformers or rewrite rules named in rules_and_transformers_to_disable will be excluded. */ +std::vector> GenerateTransformers( + TransformerLevel level, + const SessionOptions& session_options, + const IExecutionProvider& execution_provider /*required by constant folding*/, + const std::unordered_set& rules_and_transformers_to_disable = {}); } // namespace optimizer_utils } // namespace onnxruntime diff --git a/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h b/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h index fadac84688..09689b4740 100644 --- a/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h +++ b/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h @@ -48,6 +48,7 @@ static const char* const kOrtSessionOptionsConfigSetDenormalAsZero = "session.se static const char* const kOrtSessionOptionsEnableQuantQDQ = "session.enable_quant_qdq"; // Enable or disable gelu approximation in graph optimization. "0": disable; "1": enable. The default is "0". +// GeluApproximation has side effects which may change the inference results. It is disabled by default due to this. static const char* const kOrtSessionOptionsEnableGeluApproximation = "optimization.enable_gelu_approximation"; // Configure whether to allow the inter_op/intra_op threads spinning a number of times before blocking diff --git a/onnxruntime/core/optimizer/graph_transformer_utils.cc b/onnxruntime/core/optimizer/graph_transformer_utils.cc index 94fbc62f89..14de60cfe9 100644 --- a/onnxruntime/core/optimizer/graph_transformer_utils.cc +++ b/onnxruntime/core/optimizer/graph_transformer_utils.cc @@ -51,8 +51,9 @@ std::string GenerateRuleBasedTransformerName(TransformerLevel level) { return "Level" + std::to_string(static_cast(level)) + "_RuleBasedTransformer"; } -std::vector> GenerateRewriteRules(TransformerLevel level, - const std::vector& rules_to_enable) { +std::vector> GenerateRewriteRules( + TransformerLevel level, + const std::unordered_set& rules_to_disable) { std::vector> rules; switch (level) { case TransformerLevel::Level1: @@ -80,25 +81,27 @@ std::vector> GenerateRewriteRules(TransformerLevel ORT_ENFORCE(false, "Unsupported level" + std::to_string(static_cast(level))); } - if (!rules_to_enable.empty()) { + if (rules_to_disable.empty()) { + return rules; + } else { std::vector> filtered_list; - for (const auto& rule_name : rules_to_enable) { - std::for_each(rules.begin(), rules.end(), [&](std::unique_ptr& item) { - if ((item != nullptr) && (item->Name() == rule_name)) { - filtered_list.push_back(std::move(item)); - } - }); - } + const auto end = rules_to_disable.cend(); + std::for_each(rules.begin(), rules.end(), + [&](std::unique_ptr& item) { + if ((item != nullptr) && (rules_to_disable.find(item->Name()) == end)) { + filtered_list.push_back(std::move(item)); + } + }); + return filtered_list; } - - return rules; } -std::unique_ptr GenerateRuleBasedGraphTransformer(TransformerLevel level, - const std::vector& rules_to_enable, - const std::unordered_set& compatible_execution_providers) { - auto rewrite_rules_to_register = GenerateRewriteRules(level, rules_to_enable); +std::unique_ptr GenerateRuleBasedGraphTransformer( + TransformerLevel level, + const std::unordered_set& rules_to_disable, + const std::unordered_set& compatible_execution_providers) { + auto rewrite_rules_to_register = GenerateRewriteRules(level, rules_to_disable); if (rewrite_rules_to_register.empty()) { return nullptr; } @@ -113,71 +116,82 @@ std::unique_ptr GenerateRuleBasedGraphTransformer(Tra return rule_transformer; } -std::vector> GenerateTransformers(TransformerLevel level, - const SessionOptions& session_options, - const IExecutionProvider& execution_provider, /*required by constant folding*/ - const std::vector& transformers_and_rules_to_enable) { +std::vector> GenerateTransformers( + TransformerLevel level, + const SessionOptions& session_options, + const IExecutionProvider& execution_provider, /*required by constant folding*/ + const std::unordered_set& rules_and_transformers_to_disable) { std::vector> transformers; std::unique_ptr rule_transformer = nullptr; - bool enable_quant_qdq = session_options.GetConfigOrDefault(kOrtSessionOptionsEnableQuantQDQ, "1") == "1"; -#ifndef DISABLE_CONTRIB_OPS + bool enable_quant_qdq = session_options.GetConfigOrDefault(kOrtSessionOptionsEnableQuantQDQ, "1") == "1"; +#ifndef DISABLE_CONTRIB_OPS bool enable_gelu_approximation = session_options.GetConfigOrDefault(kOrtSessionOptionsEnableGeluApproximation, "0") == "1"; #endif switch (level) { case TransformerLevel::Level1: { - std::unordered_set l1_execution_providers = {}; + // no filtering on execution provider for L1 optimizations as they only use official ONNX operators + transformers.emplace_back(onnxruntime::make_unique()); + transformers.emplace_back(onnxruntime::make_unique(execution_provider, enable_quant_qdq)); + transformers.emplace_back(onnxruntime::make_unique()); + transformers.emplace_back(onnxruntime::make_unique()); + transformers.emplace_back(onnxruntime::make_unique( + session_options.free_dimension_overrides)); - transformers.emplace_back(onnxruntime::make_unique(l1_execution_providers)); - transformers.emplace_back(onnxruntime::make_unique(execution_provider, enable_quant_qdq, l1_execution_providers)); - transformers.emplace_back(onnxruntime::make_unique(l1_execution_providers)); - transformers.emplace_back(onnxruntime::make_unique(l1_execution_providers)); - transformers.emplace_back(onnxruntime::make_unique(session_options.free_dimension_overrides)); - - rule_transformer = GenerateRuleBasedGraphTransformer(level, transformers_and_rules_to_enable, l1_execution_providers); + rule_transformer = GenerateRuleBasedGraphTransformer(level, rules_and_transformers_to_disable, {}); } break; case TransformerLevel::Level2: { - std::unordered_set cpu_execution_providers = {onnxruntime::kCpuExecutionProvider}; + std::unordered_set cpu_ep = {onnxruntime::kCpuExecutionProvider}; // create rule based transformer consisting of all the level2 rewrite rules - rule_transformer = GenerateRuleBasedGraphTransformer(level, transformers_and_rules_to_enable, cpu_execution_providers); + rule_transformer = GenerateRuleBasedGraphTransformer(level, rules_and_transformers_to_disable, cpu_ep); #ifndef DISABLE_CONTRIB_OPS - transformers.emplace_back(onnxruntime::make_unique(cpu_execution_providers)); - transformers.emplace_back(onnxruntime::make_unique(cpu_execution_providers)); - transformers.emplace_back(onnxruntime::make_unique(cpu_execution_providers)); + const std::unordered_set cuda_rocm_eps = {onnxruntime::kCudaExecutionProvider, + onnxruntime::kRocmExecutionProvider}; + const std::unordered_set cpu_cuda_rocm_eps = {onnxruntime::kCpuExecutionProvider, + onnxruntime::kCudaExecutionProvider, + onnxruntime::kRocmExecutionProvider}; + const std::unordered_set cpu_cuda_rocm_acl_armnn_eps = {onnxruntime::kCpuExecutionProvider, + onnxruntime::kCudaExecutionProvider, + onnxruntime::kRocmExecutionProvider, + onnxruntime::kAclExecutionProvider, + onnxruntime::kArmNNExecutionProvider}; if (enable_quant_qdq) { transformers.emplace_back(onnxruntime::make_unique()); } - std::unordered_set cpu_acl_execution_providers = {onnxruntime::kCpuExecutionProvider, onnxruntime::kAclExecutionProvider}; - std::unordered_set cpu_cuda_rocm_acl_armnn_execution_providers = {onnxruntime::kCpuExecutionProvider, onnxruntime::kCudaExecutionProvider, onnxruntime::kRocmExecutionProvider, onnxruntime::kAclExecutionProvider, onnxruntime::kArmNNExecutionProvider}; + transformers.emplace_back(onnxruntime::make_unique(cpu_ep)); + transformers.emplace_back(onnxruntime::make_unique(cpu_ep)); + transformers.emplace_back(onnxruntime::make_unique(cpu_ep)); - transformers.emplace_back(onnxruntime::make_unique(cpu_cuda_rocm_acl_armnn_execution_providers)); + transformers.emplace_back(onnxruntime::make_unique(cpu_cuda_rocm_acl_armnn_eps)); - const std::unordered_set cuda_rocm_execution_providers = {onnxruntime::kCudaExecutionProvider, onnxruntime::kRocmExecutionProvider}; - const std::unordered_set cpu_cuda_rocm_execution_providers = {onnxruntime::kCpuExecutionProvider, onnxruntime::kCudaExecutionProvider, onnxruntime::kRocmExecutionProvider}; - transformers.emplace_back(onnxruntime::make_unique(cpu_cuda_rocm_execution_providers)); - transformers.emplace_back(onnxruntime::make_unique(cpu_cuda_rocm_execution_providers)); - transformers.emplace_back(onnxruntime::make_unique(cpu_cuda_rocm_execution_providers)); - transformers.emplace_back(onnxruntime::make_unique(cpu_cuda_rocm_execution_providers)); - transformers.emplace_back(onnxruntime::make_unique(cpu_cuda_rocm_execution_providers)); + transformers.emplace_back(onnxruntime::make_unique(cpu_cuda_rocm_eps)); + transformers.emplace_back(onnxruntime::make_unique(cpu_cuda_rocm_eps)); + transformers.emplace_back(onnxruntime::make_unique(cpu_cuda_rocm_eps)); + transformers.emplace_back(onnxruntime::make_unique(cpu_cuda_rocm_eps)); + transformers.emplace_back(onnxruntime::make_unique(cpu_cuda_rocm_eps)); - transformers.emplace_back(onnxruntime::make_unique(cuda_rocm_execution_providers)); - transformers.emplace_back(onnxruntime::make_unique(cpu_cuda_rocm_execution_providers)); - transformers.emplace_back(onnxruntime::make_unique(cpu_cuda_rocm_execution_providers)); - transformers.emplace_back(onnxruntime::make_unique(cpu_cuda_rocm_execution_providers)); - transformers.emplace_back(onnxruntime::make_unique(cpu_cuda_rocm_execution_providers)); + transformers.emplace_back(onnxruntime::make_unique(cuda_rocm_eps)); + transformers.emplace_back(onnxruntime::make_unique(cpu_cuda_rocm_eps)); + transformers.emplace_back(onnxruntime::make_unique(cpu_cuda_rocm_eps)); + transformers.emplace_back(onnxruntime::make_unique(cpu_cuda_rocm_eps)); + transformers.emplace_back(onnxruntime::make_unique(cpu_cuda_rocm_eps)); - transformers.emplace_back(onnxruntime::make_unique(cpu_cuda_rocm_execution_providers)); + transformers.emplace_back(onnxruntime::make_unique(cpu_cuda_rocm_eps)); - if (enable_gelu_approximation){ - transformers.emplace_back(onnxruntime::make_unique(cpu_cuda_rocm_execution_providers)); + transformers.emplace_back(onnxruntime::make_unique(cpu_cuda_rocm_eps)); + + // GeluApproximation has side effects which may change results. It needs to be manually enabled, + // or alternatively the model can be updated offline using a model conversion script + // e.g. fusion_gelu_approximation function used by onnxruntime/python/tools/transformers/onnx_model_bert.py + if (enable_gelu_approximation) { + transformers.emplace_back(onnxruntime::make_unique(cpu_cuda_rocm_eps)); } - transformers.emplace_back(onnxruntime::make_unique(cpu_cuda_rocm_execution_providers)); #endif } break; @@ -197,30 +211,25 @@ std::vector> GenerateTransformers(TransformerL break; } - // if the custom list to enable transformers\rules is empty then return the default generated transformers and rules - // otherwise generate a filtered list based on the provided custom list. - if (transformers_and_rules_to_enable.empty()) { - if (rule_transformer != nullptr) { - transformers.emplace_back(std::move(rule_transformer)); - } - return transformers; + if (rule_transformer != nullptr) { + transformers.emplace_back(std::move(rule_transformer)); } - std::vector> filtered_list; - // If the rule-based transformer is not empty, it should be included in the custom transformer list below. - if (rule_transformer != nullptr) { - filtered_list.emplace_back(std::move(rule_transformer)); - } - // pick custom transformers enabled for this session - for (const auto& t_name : transformers_and_rules_to_enable) { + if (rules_and_transformers_to_disable.empty()) { + return transformers; + } else { + // filter out any disabled transformers + std::vector> filtered_list; + auto end = rules_and_transformers_to_disable.cend(); std::for_each(transformers.begin(), transformers.end(), [&](std::unique_ptr& item) { - if ((item != nullptr) && (item->Name() == t_name)) { + if ((item != nullptr) && (rules_and_transformers_to_disable.find(item->Name()) == end)) { filtered_list.push_back(std::move(item)); } }); + + return filtered_list; } - return filtered_list; } } // namespace optimizer_utils diff --git a/onnxruntime/core/session/inference_session.cc b/onnxruntime/core/session/inference_session.cc index 4aef0d8c1c..0e203d40de 100644 --- a/onnxruntime/core/session/inference_session.cc +++ b/onnxruntime/core/session/inference_session.cc @@ -251,7 +251,7 @@ void InferenceSession::ConstructorCommon(const SessionOptions& session_options, concurrency::CreateThreadPool(&Env::Default(), to, concurrency::ThreadPoolType::INTRA_OP); } if (session_options_.execution_mode == ExecutionMode::ORT_PARALLEL) { - bool allow_inter_op_spinning = + bool allow_inter_op_spinning = session_options_.GetConfigOrDefault(kOrtSessionOptionsConfigAllowInterOpSpinning, "1") == "1"; OrtThreadPoolParams to = session_options_.inter_op_param; // If the thread pool can use all the processors, then @@ -510,9 +510,8 @@ common::Status InferenceSession::RegisterGraphTransformer( return graph_transformation_mgr_.Register(std::move(p_graph_transformer), level); } -common::Status InferenceSession::AddCustomTransformerList(const std::vector& transformers_to_enable) { - std::copy(transformers_to_enable.begin(), transformers_to_enable.end(), std::back_inserter(transformers_to_enable_)); - +common::Status InferenceSession::FilterEnabledOptimizers(const std::unordered_set& optimizers_to_disable) { + optimizers_to_disable_ = optimizers_to_disable; return Status::OK(); } @@ -1155,7 +1154,7 @@ common::Status InferenceSession::Initialize() { // Read shared allocators from the environment and update them in the respective providers. // // The reason for updating the providers is so that when the session state is created the allocators - // are setup appropariately keyed by OrtMemoryInfo with delegates going to the respective providers. + // are setup appropriately keyed by OrtMemoryInfo with delegates going to the respective providers. // Secondly, the GetAllocator() method inside IExecutionProvider is still used in various places, hence // it doesn't make sense to just update the allocator map inside session state with these shared allocators; doing // so would cause inconsistency between the allocator map inside session sate and that inside the providers. @@ -1213,8 +1212,7 @@ common::Status InferenceSession::Initialize() { #if !defined(ORT_MINIMAL_BUILD) if (!loading_ort_format) { // add predefined transformers - AddPredefinedTransformers(graph_transformation_mgr_, session_options_.graph_optimization_level, - transformers_to_enable_); + AddPredefinedTransformers(graph_transformation_mgr_, session_options_.graph_optimization_level); // apply any transformations to the main graph and any subgraphs ORT_RETURN_IF_ERROR_SESSIONID_(TransformGraph(graph, graph_transformation_mgr_, @@ -1906,27 +1904,17 @@ void InferenceSession::InitLogger(logging::LoggingManager* logging_manager) { // Registers all the predefined transformers with transformer manager void InferenceSession::AddPredefinedTransformers(GraphTransformerManager& transformer_manager, - TransformerLevel graph_optimization_level, - const std::vector& custom_list) { - auto add_transformers = [&](TransformerLevel level) { - // Generate and register transformers for level - auto transformers_to_register = - optimizer_utils::GenerateTransformers(level, session_options_, - *execution_providers_.Get(onnxruntime::kCpuExecutionProvider), - custom_list); - for (auto& entry : transformers_to_register) { - transformer_manager.Register(std::move(entry), level); - } - }; - - ORT_ENFORCE(graph_optimization_level <= TransformerLevel::MaxLevel, - "Exceeded max transformer level. Current level is set to " + - std::to_string(static_cast(graph_optimization_level))); - + TransformerLevel graph_optimization_level) { + const auto& cpu_ep = *execution_providers_.Get(onnxruntime::kCpuExecutionProvider); for (int i = static_cast(TransformerLevel::Level1); i <= static_cast(TransformerLevel::MaxLevel); i++) { TransformerLevel level = static_cast(i); - if ((graph_optimization_level >= level) || !custom_list.empty()) { - add_transformers(level); + if (graph_optimization_level >= level) { + // Generate and register transformers for level + auto transformers_to_register = optimizer_utils::GenerateTransformers(level, session_options_, cpu_ep, + optimizers_to_disable_); + for (auto& entry : transformers_to_register) { + transformer_manager.Register(std::move(entry), level); + } } } } diff --git a/onnxruntime/core/session/inference_session.h b/onnxruntime/core/session/inference_session.h index 7cf966771f..1303a4f12e 100644 --- a/onnxruntime/core/session/inference_session.h +++ b/onnxruntime/core/session/inference_session.h @@ -178,12 +178,17 @@ class InferenceSession { TransformerLevel level = TransformerLevel::Level2) ORT_MUST_USE_RESULT; /** - * Enable a custom set of transformers. Call this before invoking Initialize(). + * Filter the enabled optimizers (either transformer or rewrite rule) using optimizers_to_disable. + * For an optimizer to be enabled, it must be allowed at the current optimization level (as specified in + * session options), and NOT in optimizers_to_disable. + * This allows finer grained control of the enabled/disabled optimizations. + * Must be called before Initialize() to take effect. + * * Calling this API is optional. - * When this list is provided ORT ignores the levels set in session options. * @return OK if success. */ - common::Status AddCustomTransformerList(const std::vector& transformers_to_enable) ORT_MUST_USE_RESULT; + common::Status FilterEnabledOptimizers(const std::unordered_set& optimizers_to_disable) + ORT_MUST_USE_RESULT; #endif // !defined(ORT_MINIMAL_BUILD) @@ -530,8 +535,7 @@ class InferenceSession { #if !defined(ORT_MINIMAL_BUILD) virtual void AddPredefinedTransformers(GraphTransformerManager& transformer_manager, - TransformerLevel graph_optimization_level, - const std::vector& custom_list); + TransformerLevel graph_optimization_level); common::Status TransformGraph(onnxruntime::Graph& graph, const onnxruntime::GraphTransformerManager& graph_transformer_mgr, @@ -544,10 +548,8 @@ class InferenceSession { InsertCastTransformer insert_cast_transformer_; - // List of transformers to run. When this list is not empty only the transformers in this list - // will be run regardless of the level set. - // .i.e This list overrides both SessionOptions.graph_optimization_level and predefined transformers. - std::vector transformers_to_enable_; + // Any GraphTransformer/RewriteRule name in this set will not be enabled. + std::unordered_set optimizers_to_disable_; #endif #if !defined(ORT_MINIMAL_BUILD) || defined(ORT_EXTENDED_MINIMAL_BUILD) diff --git a/onnxruntime/python/onnxruntime_inference_collection.py b/onnxruntime/python/onnxruntime_inference_collection.py index c93520175e..3708e9bf33 100644 --- a/onnxruntime/python/onnxruntime_inference_collection.py +++ b/onnxruntime/python/onnxruntime_inference_collection.py @@ -233,7 +233,7 @@ class InferenceSession(Session): """ This is the main class used to run a model. """ - def __init__(self, path_or_bytes, sess_options=None, providers=None, provider_options=None): + def __init__(self, path_or_bytes, sess_options=None, providers=None, provider_options=None, **kwargs): """ :param path_or_bytes: filename or serialized ONNX or ORT format model in a byte string :param sess_options: session options @@ -276,8 +276,11 @@ class InferenceSession(Session): self._enable_fallback = True self._read_config_from_model = os.environ.get('ORT_LOAD_CONFIG_FROM_MODEL') == '1' + # internal parameters that we don't expect to be used in general so aren't documented + disabled_optimizers = kwargs['disabled_optimizers'] if 'disabled_optimizers' in kwargs else None + try: - self._create_inference_session(providers, provider_options) + self._create_inference_session(providers, provider_options, disabled_optimizers) except ValueError: if self._enable_fallback: print("EP Error using {}".format(providers)) @@ -288,7 +291,7 @@ class InferenceSession(Session): else: raise - def _create_inference_session(self, providers, provider_options): + def _create_inference_session(self, providers, provider_options, disabled_optimizers=None): available_providers = C.get_available_providers() # Tensorrt can fall back to CUDA. All others fall back to CPU. @@ -308,8 +311,14 @@ class InferenceSession(Session): else: sess = C.InferenceSession(session_options, self._model_bytes, False, self._read_config_from_model) + if disabled_optimizers is None: + disabled_optimizers = set() + elif not isinstance(disabled_optimizers, set): + # convert to set. assumes iterable + disabled_optimizers = set(disabled_optimizers) + # initialize the C++ InferenceSession - sess.initialize_session(providers, provider_options) + sess.initialize_session(providers, provider_options, disabled_optimizers) self._sess = sess self._sess_options = self._sess.session_options diff --git a/onnxruntime/python/onnxruntime_pybind_state.cc b/onnxruntime/python/onnxruntime_pybind_state.cc index e077548845..0ed51d6346 100644 --- a/onnxruntime/python/onnxruntime_pybind_state.cc +++ b/onnxruntime/python/onnxruntime_pybind_state.cc @@ -513,7 +513,7 @@ static void RegisterExecutionProviders(InferenceSession* sess, const std::vector } else { ORT_THROW("[ERROR] [TensorRT] The value for the key 'trt_max_workspace_size' should be a number in byte i.e. '1073741824'.\n"); } - } else if (option.first == "trt_fp16_enable") { + } else if (option.first == "trt_fp16_enable") { if (option.second == "True" || option.second == "true") { params.trt_fp16_enable = true; } else if (option.second == "False" || option.second == "false") { @@ -572,7 +572,7 @@ static void RegisterExecutionProviders(InferenceSession* sess, const std::vector return info; }(); - // This variable is never initialized because the APIs by which is it should be initialized are deprecated, however they still + // This variable is never initialized because the APIs by which is it should be initialized are deprecated, however they still // exist are are in-use. Neverthless, it is used to return CUDAAllocator, hence we must try to initialize it here if we can // since FromProviderOptions might contain external CUDA allocator. external_allocator_info = info.external_allocator_info; @@ -732,7 +732,8 @@ static void RegisterCustomOpDomainsAndLibraries(PyInferenceSession* sess, const #endif void InitializeSession(InferenceSession* sess, const std::vector& provider_types, - const ProviderOptionsVector& provider_options) { + const ProviderOptionsVector& provider_options, + const std::unordered_set& disabled_optimizer_names) { ProviderOptionsMap provider_options_map; GenerateProviderOptionsMap(provider_types, provider_options, provider_options_map); @@ -742,6 +743,11 @@ void InitializeSession(InferenceSession* sess, const std::vector& p } else { RegisterExecutionProviders(sess, provider_types, provider_options_map); } + + if (!disabled_optimizer_names.empty()) { + OrtPybindThrowIfError(sess->FilterEnabledOptimizers(disabled_optimizer_names)); + } + OrtPybindThrowIfError(sess->Initialize()); } @@ -1785,8 +1791,9 @@ including arg name, arg type (contains both type and shape).)pbdoc") "initialize_session", [](PyInferenceSession* sess, const std::vector& provider_types = {}, - const ProviderOptionsVector& provider_options = {}) { - InitializeSession(sess->GetSessionHandle(), provider_types, provider_options); + const ProviderOptionsVector& provider_options = {}, + const std::unordered_set& disabled_optimizer_names = {}) { + InitializeSession(sess->GetSessionHandle(), provider_types, provider_options, disabled_optimizer_names); }, R"pbdoc(Load a model saved in ONNX or ORT format.)pbdoc") .def("run", @@ -1873,8 +1880,7 @@ including arg name, arg type (contains both type and shape).)pbdoc") status = sess->GetSessionHandle()->Run(*run_options, *io_binding.Get()); if (!status.IsOK()) throw std::runtime_error("Error in execution: " + status.ErrorMessage()); - }) - ; + }); py::enum_(m, "ArenaExtendStrategy", py::arithmetic()) .value("kNextPowerOfTwo", onnxruntime::ArenaExtendStrategy::kNextPowerOfTwo) diff --git a/onnxruntime/python/onnxruntime_pybind_state_common.h b/onnxruntime/python/onnxruntime_pybind_state_common.h index 16c8556740..99b3d4f730 100644 --- a/onnxruntime/python/onnxruntime_pybind_state_common.h +++ b/onnxruntime/python/onnxruntime_pybind_state_common.h @@ -129,7 +129,8 @@ Environment& GetEnv(); // Any provider_options should have entries in matching order to provider_types. void InitializeSession(InferenceSession* sess, const std::vector& provider_types = {}, - const ProviderOptionsVector& provider_options = {}); + const ProviderOptionsVector& provider_options = {}, + const std::unordered_set& disabled_optimizer_names = {}); // Checks if PyErrOccured, fetches status and throws. void ThrowIfPyErrOccured(); diff --git a/onnxruntime/test/optimizer/graph_transform_test.cc b/onnxruntime/test/optimizer/graph_transform_test.cc index 2a0872e48b..941fae0b76 100644 --- a/onnxruntime/test/optimizer/graph_transform_test.cc +++ b/onnxruntime/test/optimizer/graph_transform_test.cc @@ -66,6 +66,8 @@ #include "test/providers/provider_test_utils.h" #include "test/test_environment.h" #include "test/util/include/default_providers.h" +#include "test/util/include/asserts.h" +#include "test/util/include/inference_session_wrapper.h" using namespace std; using namespace ONNX_NAMESPACE; @@ -847,20 +849,20 @@ TEST_F(GraphTransformationTests, TransposeMatmulFusion) { TEST_F(GraphTransformationTests, TransposeCastMatmulFusion) { const std::vector model_uris = { - MODEL_FOLDER "fusion/transpose_cast_matmul_4d_fusion0.onnx", // Test fusion from the right input - MODEL_FOLDER "fusion/transpose_cast_matmul_4d_fusion1.onnx", // Test fusion from the left input - MODEL_FOLDER "fusion/transpose_cast_matmul_4d_fusion2.onnx", // Test fusion both from the left and right inputs - MODEL_FOLDER "fusion/transpose_cast_matmul_4d_fusion3.onnx", // Cast nodes feed multiple MatMul nodes. - MODEL_FOLDER "fusion/transpose_cast_matmul_4d_fusion4.onnx", // Cast nodes feed one MatMul node and - // the Transpose nodes feed another MatMul node. - MODEL_FOLDER "fusion/transpose_cast_matmul_4d_fusion5.onnx" // One Cast node and one Transpose node feed each - // MatMul nodes. + MODEL_FOLDER "fusion/transpose_cast_matmul_4d_fusion0.onnx", // Test fusion from the right input + MODEL_FOLDER "fusion/transpose_cast_matmul_4d_fusion1.onnx", // Test fusion from the left input + MODEL_FOLDER "fusion/transpose_cast_matmul_4d_fusion2.onnx", // Test fusion both from the left and right inputs + MODEL_FOLDER "fusion/transpose_cast_matmul_4d_fusion3.onnx", // Cast nodes feed multiple MatMul nodes. + MODEL_FOLDER "fusion/transpose_cast_matmul_4d_fusion4.onnx", // Cast nodes feed one MatMul node and + // the Transpose nodes feed another MatMul node. + MODEL_FOLDER "fusion/transpose_cast_matmul_4d_fusion5.onnx" // One Cast node and one Transpose node feed each + // MatMul nodes. }; for (const auto& model_uri : model_uris) { std::shared_ptr p_model; ASSERT_TRUE(Model::Load(model_uri, p_model, nullptr, *logger_).IsOK()); Graph& graph = p_model->MainGraph(); - std::map orig_op_to_count = CountOpsInGraph(graph); // Original op count + std::map orig_op_to_count = CountOpsInGraph(graph); // Original op count onnxruntime::GraphTransformerManager graph_transformation_mgr{5}; graph_transformation_mgr.Register(onnxruntime::make_unique(), TransformerLevel::Level1); @@ -2591,7 +2593,6 @@ TEST_F(GraphTransformationTests, FastGeluFusionWithCastsTest3) { ASSERT_TRUE(op_to_count["com.microsoft.FastGelu"] == 1); } - struct BiasSoftmaxFusionTester { std::shared_ptr p_model_; Status model_load_; @@ -3777,5 +3778,31 @@ TEST_F(GraphTransformationTests, MatMulScaleFusionUnsupportedInputType) { } #endif +TEST_F(GraphTransformationTests, FilterEnabledOptimizers) { + auto model_uri = MODEL_FOLDER "fusion/constant_folding_with_scalar_shape_to_initializer.onnx"; + + SessionOptions so; + so.session_logid = "GraphTransformationTests.FilterEnabledOptimizers"; + InferenceSessionWrapper session_object{so, GetEnvironment()}; + + ASSERT_STATUS_OK(session_object.Load(model_uri)); + + const auto& graph = session_object.GetGraph(); + + // check the ops that should go away if the constant folding transformer or ShapeToInitializer rewrite rule run + std::map op_to_count = CountOpsInGraph(graph); + ASSERT_TRUE(op_to_count["Shape"] == 1); + ASSERT_TRUE(op_to_count["ConstantOfShape"] == 1); + ASSERT_TRUE(op_to_count["Add"] == 1); + + ASSERT_STATUS_OK(session_object.FilterEnabledOptimizers({"ConstantFolding", "ShapeToInitializer"})); + ASSERT_STATUS_OK(session_object.Initialize()); // Initialize runs the transformers + + op_to_count = CountOpsInGraph(graph); + ASSERT_TRUE(op_to_count["Shape"] == 1); + ASSERT_TRUE(op_to_count["ConstantOfShape"] == 1); + ASSERT_TRUE(op_to_count["Add"] == 1); +} + } // namespace test } // namespace onnxruntime diff --git a/onnxruntime/test/optimizer/graph_transform_utils_test.cc b/onnxruntime/test/optimizer/graph_transform_utils_test.cc index 46e5848413..55d2051f5d 100644 --- a/onnxruntime/test/optimizer/graph_transform_utils_test.cc +++ b/onnxruntime/test/optimizer/graph_transform_utils_test.cc @@ -20,19 +20,13 @@ TEST(GraphTransformerUtilsTests, TestGenerateRewriterules) { auto rewrite_rules = optimizer_utils::GenerateRewriteRules(TransformerLevel::Level1); ASSERT_TRUE(rewrite_rules.size() != 0); - // Rule name match test - std::vector custom_list = {"EliminateIdentity", "ConvAddFusion", "ConvMulFusion", "abc", "def"}; - rewrite_rules = optimizer_utils::GenerateRewriteRules(TransformerLevel::Level1, custom_list); - // validate each rule returned is present in the custom list + // Disable rule name match test + std::unordered_set disable_list = {"EliminateIdentity", "ConvAddFusion"}; + rewrite_rules = optimizer_utils::GenerateRewriteRules(TransformerLevel::Level1, disable_list); + // validate each rule returned is not present in the custom list for (const auto& rule : rewrite_rules) { - ASSERT_TRUE(std::find(custom_list.begin(), custom_list.end(), rule->Name()) != custom_list.end()); + ASSERT_TRUE(std::find(disable_list.begin(), disable_list.end(), rule->Name()) == disable_list.end()); } - - // Rule name no match test. Test to validate empty rules list is returned when - // there is no match in custom list - custom_list = {"abc"}; - rewrite_rules = optimizer_utils::GenerateRewriteRules(TransformerLevel::Level1, custom_list); - ASSERT_TRUE(rewrite_rules.size() == 0); } TEST(GraphTransformerUtilsTests, TestGenerateGraphTransformers) { @@ -40,28 +34,37 @@ TEST(GraphTransformerUtilsTests, TestGenerateGraphTransformers) { std::string l1_rule1 = "EliminateIdentity"; std::string l1_transformer = "ConstantFolding"; std::string l2_transformer = "ConvActivationFusion"; - std::vector custom_list = {l1_rule1, l1_transformer, l2_transformer}; - std::unique_ptr cpu_execution_provider = - onnxruntime::make_unique(CPUExecutionProviderInfo()); - auto transformers = optimizer_utils::GenerateTransformers(TransformerLevel::Level1, {}, *cpu_execution_provider.get(), custom_list); - ASSERT_TRUE(transformers.size() == 2); + std::unordered_set disabled = {l1_rule1, l1_transformer, l2_transformer}; + CPUExecutionProvider cpu_ep(CPUExecutionProviderInfo{}); + auto all_transformers = optimizer_utils::GenerateTransformers(TransformerLevel::Level1, {}, cpu_ep); + auto filtered_transformers = optimizer_utils::GenerateTransformers(TransformerLevel::Level1, {}, cpu_ep, disabled); + + // check ConstantFolding transformer was removed + ASSERT_TRUE(filtered_transformers.size() == all_transformers.size() - 1); + + // check EliminateIdentity rule was removed from inside the rule based transformer auto l1_rule_transformer_name = optimizer_utils::GenerateRuleBasedTransformerName(TransformerLevel::Level1); RuleBasedGraphTransformer* rule_transformer = nullptr; - for (const auto& transformer : transformers) { + for (const auto& transformer : filtered_transformers) { if (transformer->Name() == l1_rule_transformer_name) { - rule_transformer = static_cast(transformers[0].get()); + rule_transformer = static_cast(transformer.get()); + + // get the full set of rules and check EliminateIdentity was correctly removed + auto l1_rewrite_rules = optimizer_utils::GenerateRewriteRules(TransformerLevel::Level1); + ASSERT_TRUE(rule_transformer->RulesCount() == l1_rewrite_rules.size() - 1); + break; } } - ASSERT_TRUE(rule_transformer && rule_transformer->RulesCount() == 1); - transformers = optimizer_utils::GenerateTransformers(TransformerLevel::Level2, {}, *cpu_execution_provider.get(), custom_list); + ASSERT_TRUE(rule_transformer) << "RuleBased transformer should have been added by GenerateTransformers"; + #ifndef DISABLE_CONTRIB_OPS - ASSERT_TRUE(transformers.size() == 1); -#else - ASSERT_TRUE(transformers.size() == 0); + // check that ConvActivationFusion was removed + all_transformers = optimizer_utils::GenerateTransformers(TransformerLevel::Level2, {}, cpu_ep); + filtered_transformers = optimizer_utils::GenerateTransformers(TransformerLevel::Level2, {}, cpu_ep, disabled); + ASSERT_TRUE(filtered_transformers.size() == all_transformers.size() - 1); #endif } - } // namespace test } // namespace onnxruntime diff --git a/orttraining/orttraining/core/optimizer/graph_transformer_utils.cc b/orttraining/orttraining/core/optimizer/graph_transformer_utils.cc index 4b5353690c..f6c6ec3f67 100644 --- a/orttraining/orttraining/core/optimizer/graph_transformer_utils.cc +++ b/orttraining/orttraining/core/optimizer/graph_transformer_utils.cc @@ -55,7 +55,7 @@ std::vector> GeneratePreTrainingTransformers( const std::unordered_set& weights_to_train, const TrainingSession::TrainingConfiguration::GraphTransformerConfiguration& config, const IExecutionProvider& execution_provider, - const std::vector& transformers_and_rules_to_enable) { + const std::unordered_set& rules_and_transformers_to_disable) { std::vector> transformers; std::unique_ptr rule_transformer = nullptr; @@ -94,7 +94,8 @@ std::vector> GeneratePreTrainingTransformers( if (config.enable_gelu_approximation) { transformers.emplace_back(onnxruntime::make_unique(compatible_eps)); } - transformers.emplace_back(onnxruntime::make_unique(execution_provider, false /*skip_dequantize_linear*/, compatible_eps, weights_to_train)); + transformers.emplace_back(onnxruntime::make_unique( + execution_provider, false /*skip_dequantize_linear*/, compatible_eps, weights_to_train)); transformers.emplace_back(onnxruntime::make_unique(compatible_eps)); transformers.emplace_back(onnxruntime::make_unique(compatible_eps)); transformers.emplace_back(onnxruntime::make_unique(compatible_eps)); @@ -126,45 +127,43 @@ std::vector> GeneratePreTrainingTransformers( break; } - // if the custom list to enable transformers\rules is empty then return the default generated transformers and rules - // otherwise generate a filtered list based on the provided custom list. // Note that some rule-based transformers are depending on some custom transformers, // e.g., ExpandElimination and CastElimination are depending on ConstantFolding to fold the constant first, - // so we should always push the rule-based transformer to the end, this is expecially important when transformation step is 1. - if (transformers_and_rules_to_enable.empty()) { - if (rule_transformer != nullptr) { - transformers.emplace_back(std::move(rule_transformer)); - } - return transformers; + // so we should always push the rule-based transformer to the end, this is especially important when the number of + // transformation steps is 1. + if (rule_transformer != nullptr) { + transformers.emplace_back(std::move(rule_transformer)); } - std::vector> filtered_list; - // pick custom transformers enabled for this session - for (const auto& t_name : transformers_and_rules_to_enable) { + + if (rules_and_transformers_to_disable.empty()) { + return transformers; + } else { + // filter out any disabled transformers + std::vector> filtered_list; + auto end = rules_and_transformers_to_disable.cend(); std::for_each(transformers.begin(), transformers.end(), [&](std::unique_ptr& item) { - if ((item != nullptr) && (item->Name() == t_name)) { + if ((item != nullptr) && (rules_and_transformers_to_disable.find(item->Name()) == end)) { filtered_list.push_back(std::move(item)); } }); + + return filtered_list; } - // If the rule-based transformer is not empty, it should be included in the custom transformer list below. - if (rule_transformer != nullptr) { - filtered_list.emplace_back(std::move(rule_transformer)); - } - return filtered_list; } std::vector> GenerateTransformers( TransformerLevel level, const std::unordered_set& weights_to_train, gsl::span free_dimension_overrides, - const std::vector& transformers_and_rules_to_enable) { + const std::unordered_set& rules_and_transformers_to_disable) { std::vector> transformers; std::unique_ptr rule_transformer = nullptr; switch (level) { case TransformerLevel::Level1: { std::unordered_set l1_execution_providers = {}; - std::unordered_set cuda_rocm_execution_providers = {onnxruntime::kCudaExecutionProvider, onnxruntime::kRocmExecutionProvider}; + std::unordered_set cuda_rocm_execution_providers = {onnxruntime::kCudaExecutionProvider, + onnxruntime::kRocmExecutionProvider}; // TODO hack - constant folding currently doesn't work after mixed precision transformation so it's disabled for now // ORT uses CPU kernels to evaluate constant values but some of them don't support fp16 @@ -176,14 +175,16 @@ std::vector> GenerateTransformers( transformers.emplace_back(onnxruntime::make_unique(l1_execution_providers)); transformers.emplace_back(onnxruntime::make_unique(l1_execution_providers, weights_to_train)); - rule_transformer = optimizer_utils::GenerateRuleBasedGraphTransformer(level, transformers_and_rules_to_enable, l1_execution_providers); + rule_transformer = optimizer_utils::GenerateRuleBasedGraphTransformer(level, rules_and_transformers_to_disable, + l1_execution_providers); } break; case TransformerLevel::Level2: { std::unordered_set cpu_execution_providers = {onnxruntime::kCpuExecutionProvider}; // create rule based transformer consisting of all the level2 rewrite rules - rule_transformer = optimizer_utils::GenerateRuleBasedGraphTransformer(level, transformers_and_rules_to_enable, cpu_execution_providers); + rule_transformer = optimizer_utils::GenerateRuleBasedGraphTransformer(level, rules_and_transformers_to_disable, + cpu_execution_providers); transformers.emplace_back(onnxruntime::make_unique(cpu_execution_providers)); transformers.emplace_back(onnxruntime::make_unique(cpu_execution_providers)); @@ -201,29 +202,26 @@ std::vector> GenerateTransformers( break; } + if (rule_transformer != nullptr) { + transformers.emplace_back(std::move(rule_transformer)); + } + // if the custom list to enable transformers\rules is empty then return the default generated transformers and rules // otherwise generate a filtered list based on the provided custom list. - if (transformers_and_rules_to_enable.empty()) { - if (rule_transformer != nullptr) { - transformers.emplace_back(std::move(rule_transformer)); - } + if (rules_and_transformers_to_disable.empty()) { return transformers; - } - std::vector> filtered_list; - // If the rule-based transformer is not empty, it should be included in the custom transformer list below. - if (rule_transformer != nullptr) { - filtered_list.emplace_back(std::move(rule_transformer)); - } - // pick custom transformers enabled for this session - for (const auto& t_name : transformers_and_rules_to_enable) { + } else { + std::vector> filtered_list; + auto end = rules_and_transformers_to_disable.cend(); std::for_each(transformers.begin(), transformers.end(), [&](std::unique_ptr& item) { - if ((item != nullptr) && (item->Name() == t_name)) { + if ((item != nullptr) && (rules_and_transformers_to_disable.find(item->Name()) == end)) { filtered_list.push_back(std::move(item)); } }); + + return filtered_list; } - return filtered_list; } } // namespace transformer_utils diff --git a/orttraining/orttraining/core/optimizer/graph_transformer_utils.h b/orttraining/orttraining/core/optimizer/graph_transformer_utils.h index d6f932742e..94ea67f13e 100644 --- a/orttraining/orttraining/core/optimizer/graph_transformer_utils.h +++ b/orttraining/orttraining/core/optimizer/graph_transformer_utils.h @@ -20,7 +20,7 @@ std::vector> GeneratePreTrainingTransformers( const std::unordered_set& weights_to_train, const TrainingSession::TrainingConfiguration::GraphTransformerConfiguration& config, const IExecutionProvider& execution_provider, // required for constant folding - const std::vector& rules_and_transformers_to_enable = {}); + const std::unordered_set& rules_and_transformers_to_disable = {}); /** Generates all predefined (both rule-based and non-rule-based) transformers for this level. If transformers_and_rules_to_enable is not empty, it returns the intersection between the predefined transformers/rules @@ -29,7 +29,7 @@ std::vector> GenerateTransformers( TransformerLevel level, const std::unordered_set& weights_to_train, gsl::span free_dimension_overrides, - const std::vector& rules_and_transformers_to_enable = {}); + const std::unordered_set& rules_and_transformers_to_disable = {}); } // namespace transformer_utils } // namespace training diff --git a/orttraining/orttraining/core/session/training_session.cc b/orttraining/orttraining/core/session/training_session.cc index 2f3d15d8e6..56e9e7d7db 100644 --- a/orttraining/orttraining/core/session/training_session.cc +++ b/orttraining/orttraining/core/session/training_session.cc @@ -740,51 +740,39 @@ void TrainingSession::AddPreTrainingTransformers(const IExecutionProvider& execu GraphTransformerManager& transformer_manager, const std::unordered_set& weights_to_train, const TrainingConfiguration::GraphTransformerConfiguration& config, - TransformerLevel graph_optimization_level, - const std::vector& custom_list) { - auto add_transformers = [&](TransformerLevel level) { - // Generate and register transformers for level - - auto transformers_to_register = transformer_utils::GeneratePreTrainingTransformers( - level, weights_to_train, config, execution_provider, custom_list); - for (auto& entry : transformers_to_register) { - transformer_manager.Register(std::move(entry), level); - } - }; - + TransformerLevel graph_optimization_level) { ORT_ENFORCE(graph_optimization_level <= TransformerLevel::MaxLevel, "Exceeded max transformer level. Current level is set to " + std::to_string(static_cast(graph_optimization_level))); for (int i = static_cast(TransformerLevel::Level1); i <= static_cast(TransformerLevel::MaxLevel); i++) { TransformerLevel level = static_cast(i); - if ((graph_optimization_level >= level) || !custom_list.empty()) { - add_transformers(level); + if ((graph_optimization_level >= level)) { + auto transformers_to_register = transformer_utils::GeneratePreTrainingTransformers( + level, weights_to_train, config, execution_provider); + for (auto& entry : transformers_to_register) { + transformer_manager.Register(std::move(entry), level); + } } } } // Registers all the predefined transformers with transformer manager void TrainingSession::AddPredefinedTransformers(GraphTransformerManager& transformer_manager, - TransformerLevel graph_optimization_level, - const std::vector& custom_list) { - auto add_transformers = [&](TransformerLevel level) { - // Generate and register transformers for level - auto transformers_to_register = transformer_utils::GenerateTransformers( - level, weights_to_train_, GetSessionOptions().free_dimension_overrides, custom_list); - for (auto& entry : transformers_to_register) { - transformer_manager.Register(std::move(entry), level); - } - }; - + TransformerLevel graph_optimization_level) { ORT_ENFORCE(graph_optimization_level <= TransformerLevel::MaxLevel, "Exceeded max transformer level. Current level is set to " + std::to_string(static_cast(graph_optimization_level))); for (int i = static_cast(TransformerLevel::Level1); i <= static_cast(TransformerLevel::MaxLevel); i++) { TransformerLevel level = static_cast(i); - if ((graph_optimization_level >= level) || !custom_list.empty()) { - add_transformers(level); + if ((graph_optimization_level >= level)) { + // Generate and register transformers for level + auto transformers_to_register = transformer_utils::GenerateTransformers( + level, weights_to_train_, GetSessionOptions().free_dimension_overrides, {}); + for (auto& entry : transformers_to_register) { + transformer_manager.Register(std::move(entry), level); + } } } } @@ -1002,8 +990,8 @@ static Status UpdateWeightsBeforeSaving( return Status::OK(); } -Status TrainingSession::SaveWithExternalInitializers(const PathString& model_uri, - const std::string& external_file_name, +Status TrainingSession::SaveWithExternalInitializers(const PathString& model_uri, + const std::string& external_file_name, size_t initializer_size_threshold) { // Delete the old files before saving. std::remove(ToMBString(model_uri).c_str()); diff --git a/orttraining/orttraining/core/session/training_session.h b/orttraining/orttraining/core/session/training_session.h index 8b1dc501cf..86ec33dfe9 100644 --- a/orttraining/orttraining/core/session/training_session.h +++ b/orttraining/orttraining/core/session/training_session.h @@ -45,7 +45,7 @@ class TrainingSession : public InferenceSession { TrainingSession(const SessionOptions& session_options, const Environment& env) : InferenceSession(session_options, env) {} - virtual ~TrainingSession() {}; + virtual ~TrainingSession(){}; /** * The training configuration options. @@ -488,13 +488,11 @@ class TrainingSession : public InferenceSession { GraphTransformerManager& transformer_manager, const std::unordered_set& weights_to_train, const TrainingConfiguration::GraphTransformerConfiguration& config, - TransformerLevel graph_optimization_level = TransformerLevel::MaxLevel, - const std::vector& custom_list = {}); + TransformerLevel graph_optimization_level = TransformerLevel::MaxLevel); /** override the parent method in inference session for training specific transformers */ void AddPredefinedTransformers(GraphTransformerManager& transformer_manager, - TransformerLevel graph_optimization_level, - const std::vector& custom_list) override; + TransformerLevel graph_optimization_level) override; /** Perform auto-diff to add backward graph into the model. @param weights_to_train a set of weights to be training. @@ -522,16 +520,16 @@ class TrainingSession : public InferenceSession { std::string& loss_name, const optional& loss_function_config, optional& loss_scale_input_name); - + virtual common::Status BuildLossAndLossScaling( - const int32_t pipeline_stage_id, - const optional& external_loss_name, - const optional& mixed_precision_config, - const optional& distributed_config, - const optional& loss_function_config, - std::string& loss_name, - optional& loss_scale_input_name, - optional& mixed_precision_config_result); + const int32_t pipeline_stage_id, + const optional& external_loss_name, + const optional& mixed_precision_config, + const optional& distributed_config, + const optional& loss_function_config, + std::string& loss_name, + optional& loss_scale_input_name, + optional& mixed_precision_config_result); /** Enable mixed precision training @param weights_to_train a set of weights to be training. @@ -615,14 +613,14 @@ class PipelineTrainingSession final : public TrainingSession { optional& pipeline_config_result) override; common::Status BuildLossAndLossScaling( - const int32_t pipeline_stage_id, - const optional& external_loss_name, - const optional& mixed_precision_config, - const optional& distributed_config, - const optional& loss_function_config, - std::string& loss_name, - optional& loss_scale_input_name, - optional& mixed_precision_config_result) override; + const int32_t pipeline_stage_id, + const optional& external_loss_name, + const optional& mixed_precision_config, + const optional& distributed_config, + const optional& loss_function_config, + std::string& loss_name, + optional& loss_scale_input_name, + optional& mixed_precision_config_result) override; // Set some PipelineContext fields based on configuration result // returned by TrainingSession::ConfigureForTraining. diff --git a/orttraining/orttraining/test/optimizer/graph_transformer_utils_test.cc b/orttraining/orttraining/test/optimizer/graph_transformer_utils_test.cc index c652908b5b..4fcad57f3d 100644 --- a/orttraining/orttraining/test/optimizer/graph_transformer_utils_test.cc +++ b/orttraining/orttraining/test/optimizer/graph_transformer_utils_test.cc @@ -20,25 +20,36 @@ TEST(GraphTransformerUtilsTestsForTraining, TestGenerateGraphTransformers) { std::string l1_rule1 = "EliminateIdentity"; std::string l1_transformer = "ConstantFolding"; std::string l2_transformer = "ConvActivationFusion"; - std::vector custom_list = {l1_rule1, l1_transformer, l2_transformer}; + std::unordered_set disabled = {l1_rule1, l1_transformer, l2_transformer}; + CPUExecutionProvider cpu_ep(CPUExecutionProviderInfo{}); - auto transformers = training::transformer_utils::GenerateTransformers(TransformerLevel::Level1, {}, {}, custom_list); - ASSERT_TRUE(transformers.size() == 1); + auto all_transformers = optimizer_utils::GenerateTransformers(TransformerLevel::Level1, {}, cpu_ep); + auto filtered_transformers = optimizer_utils::GenerateTransformers(TransformerLevel::Level1, {}, cpu_ep, disabled); + // check ConstantFolding transformer was removed + ASSERT_TRUE(filtered_transformers.size() == all_transformers.size() - 1); + + // check EliminateIdentity rule was removed from inside the rule based transformer auto l1_rule_transformer_name = optimizer_utils::GenerateRuleBasedTransformerName(TransformerLevel::Level1); RuleBasedGraphTransformer* rule_transformer = nullptr; - for (const auto& transformer : transformers) { + for (const auto& transformer : filtered_transformers) { if (transformer->Name() == l1_rule_transformer_name) { - rule_transformer = dynamic_cast(transformers[0].get()); + rule_transformer = static_cast(transformer.get()); + + // get the full set of rules and check EliminateIdentity was correctly removed + auto l1_rewrite_rules = optimizer_utils::GenerateRewriteRules(TransformerLevel::Level1); + ASSERT_TRUE(rule_transformer->RulesCount() == l1_rewrite_rules.size() - 1); + break; } } - ASSERT_TRUE(rule_transformer && rule_transformer->RulesCount() == 1); - transformers = training::transformer_utils::GenerateTransformers(TransformerLevel::Level2, {}, {}, custom_list); + ASSERT_TRUE(rule_transformer) << "RuleBased transformer should have been added by GenerateTransformers"; + #ifndef DISABLE_CONTRIB_OPS - ASSERT_TRUE(transformers.size() == 1); -#else - ASSERT_TRUE(transformers.size() == 0); + // check that ConvActivationFusion was removed + all_transformers = optimizer_utils::GenerateTransformers(TransformerLevel::Level2, {}, cpu_ep); + filtered_transformers = optimizer_utils::GenerateTransformers(TransformerLevel::Level2, {}, cpu_ep, disabled); + ASSERT_TRUE(filtered_transformers.size() == all_transformers.size() - 1); #endif } diff --git a/tools/python/convert_onnx_models_to_ort.py b/tools/python/convert_onnx_models_to_ort.py index fd8099654c..6b8afd5d5b 100644 --- a/tools/python/convert_onnx_models_to_ort.py +++ b/tools/python/convert_onnx_models_to_ort.py @@ -71,6 +71,14 @@ def _convert(model_path_or_dir: pathlib.Path, optimization_level: ort.GraphOptim # providers are priority based, so register NNAPI first providers.insert(0, 'NnapiExecutionProvider') + # if the optimization level is 'all' we manually exclude the NCHWc transformer. It's not applicable to ARM + # devices, and creates a device specific model which won't run on all hardware. + # If someone really really really wants to run it they could manually create an optimized onnx model first, + # or they could comment out this code. + optimizer_filter = None + if optimization_level == ort.GraphOptimizationLevel.ORT_ENABLE_ALL: + optimizer_filter = ['NchwcTransformer'] + for model in models: # ignore any files with an extension of .optimized.onnx which are presumably from previous executions # of this script @@ -88,14 +96,16 @@ def _convert(model_path_or_dir: pathlib.Path, optimization_level: ort.GraphOptim so = _create_session_options(optimization_level, optimized_target_path, custom_op_library) print("Saving optimized ONNX model {} to {}".format(model, optimized_target_path)) - _ = ort.InferenceSession(str(model), sess_options=so, providers=providers) + _ = ort.InferenceSession(str(model), sess_options=so, providers=providers, + disabled_optimizers=optimizer_filter) # Load ONNX model, optimize, and save to ORT format so = _create_session_options(optimization_level, ort_target_path, custom_op_library) so.add_session_config_entry('session.save_model_format', 'ORT') print("Converting optimized ONNX model to ORT format model {}".format(ort_target_path)) - _ = ort.InferenceSession(str(model), sess_options=so, providers=providers) + _ = ort.InferenceSession(str(model), sess_options=so, providers=providers, + disabled_optimizers=optimizer_filter) # orig_size = os.path.getsize(onnx_target_path) # new_size = os.path.getsize(ort_target_path) @@ -110,12 +120,9 @@ def _get_optimization_level(level): # Constant folding and other optimizations that only use ONNX operators return ort.GraphOptimizationLevel.ORT_ENABLE_BASIC if level == 'extended': - # Optimizations using custom operators, excluding NCHWc optimizations + # Optimizations using custom operators, excluding NCHWc and NHWC layout optimizers return ort.GraphOptimizationLevel.ORT_ENABLE_EXTENDED if level == 'all': - # all optimizations, including NCHWc (which has hardware specific logic) - print('WARNING: Enabling layout optimizations is not recommended unless the ORT format model will be executed ' - 'on the same hardware used to create the model.') return ort.GraphOptimizationLevel.ORT_ENABLE_ALL raise ValueError('Invalid optimization level of ' + level) @@ -139,12 +146,13 @@ def parse_args(): 'NNAPI execution provider takes, in order to preserve those nodes in the ORT format ' 'model.') - parser.add_argument('--optimization_level', default='extended', + parser.add_argument('--optimization_level', default='all', choices=['disable', 'basic', 'extended', 'all'], help="Level to optimize ONNX model with, prior to converting to ORT format model. " "These map to the onnxruntime.GraphOptimizationLevel values. " - "NOTE: It is NOT recommended to use 'all' unless you are creating the ORT format model on " - "the device you will run it on, as the generated model may not be valid on other hardware." + "If the level is 'all' the NCHWc transformer is manually disabled as it contains device " + "specific logic, so the ORT format model must be generated on the device it will run on. " + "Additionally, the NCHWc optimizations are not applicable to ARM devices." ) parser.add_argument('--enable_type_reduction', action='store_true', From b22e60bd442cdd9a96138a5705418a76ef46e467 Mon Sep 17 00:00:00 2001 From: Ashwini Khade Date: Mon, 29 Mar 2021 11:00:38 -0700 Subject: [PATCH 019/129] pull onnx latest commit (#7102) * update onnx commit * fix test scripts to remove deprecated call * update filters * add registration for relu and cumsum ver 14 * add promote trilu to onnx domain * update onnx-tensorrt submodule * update flag * update flag * update dependencies * fix android ci failure --- cgmanifests/submodules/cgmanifest.json | 4 +- cmake/CMakeLists.txt | 1 + cmake/external/onnx | 2 +- cmake/external/onnx-tensorrt | 2 +- cmake/external/onnx_minimal.cmake | 1 + cmake/onnxruntime_providers.cmake | 7 +- csharp/testdata/test_input_BFLOAT16.py | 5 +- csharp/testdata/test_input_FLOAT16.py | 6 +- .../providers/cpu/activation/activations.cc | 6 +- .../providers/cpu/cpu_execution_provider.cc | 56 ++-- onnxruntime/core/providers/cpu/math/cumsum.cc | 45 +++- .../core/providers/cpu/tensor/identity_op.cc | 2 +- .../providers/cpu/tensor}/trilu.cc | 12 +- .../cpu => core/providers/cpu/tensor}/trilu.h | 2 - .../tools/featurizer_ops/create_test_model.py | 5 +- onnxruntime/test/onnx/gen_test_models.py | 12 +- .../cpu/tensor}/shape_inference_test_helper.h | 23 +- .../providers/cpu/tensor/trilu_op_test.cc | 242 ++++++++++++++++++ .../cpu/tensor}/trilu_shape_inference_test.cc | 15 ++ .../onnx_backend_test_series_filters.jsonc | 18 +- .../docker/scripts/manylinux/requirements.txt | 2 +- .../linux/docker/scripts/requirements.txt | 2 +- 22 files changed, 412 insertions(+), 58 deletions(-) rename onnxruntime/{contrib_ops/cpu => core/providers/cpu/tensor}/trilu.cc (92%) rename onnxruntime/{contrib_ops/cpu => core/providers/cpu/tensor}/trilu.h (91%) rename onnxruntime/test/{contrib_ops => providers/cpu/tensor}/shape_inference_test_helper.h (84%) create mode 100644 onnxruntime/test/providers/cpu/tensor/trilu_op_test.cc rename onnxruntime/test/{contrib_ops => providers/cpu/tensor}/trilu_shape_inference_test.cc (89%) diff --git a/cgmanifests/submodules/cgmanifest.json b/cgmanifests/submodules/cgmanifest.json index fb5856a401..33449e2882 100644 --- a/cgmanifests/submodules/cgmanifest.json +++ b/cgmanifests/submodules/cgmanifest.json @@ -242,7 +242,7 @@ "component": { "type": "git", "git": { - "commitHash": "237926eab41de21fb9addc4b03b751fd6a3343ec", + "commitHash": "fe2433d3dd677ed5c582e194a49a632e707d0543", "repositoryUrl": "https://github.com/onnx/onnx" }, "comments": "git submodule at cmake/external/onnx" @@ -272,7 +272,7 @@ "component": { "type": "git", "git": { - "commitHash": "dc22bb323ece3c65419717be8a0d3d0f318a61fa", + "commitHash": "99296a4462ffc0a8a4a9635e51382277bec68979", "repositoryUrl": "https://github.com/onnx/onnx-tensorrt.git" }, "comments": "git submodule at cmake/external/onnx-tensorrt" diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt index e0d99eb40a..c7c57c7676 100644 --- a/cmake/CMakeLists.txt +++ b/cmake/CMakeLists.txt @@ -343,6 +343,7 @@ if(onnxruntime_DISABLE_EXCEPTIONS) add_compile_definitions("ORT_NO_EXCEPTIONS") add_compile_definitions("MLAS_NO_EXCEPTION") + add_compile_definitions("ONNX_NO_EXCEPTIONS") if(MSVC) string(REGEX REPLACE "/EHsc" "/EHs-c-" CMAKE_CXX_FLAGS "${CMAKE_CXX_FLAGS}") diff --git a/cmake/external/onnx b/cmake/external/onnx index 237926eab4..fe2433d3dd 160000 --- a/cmake/external/onnx +++ b/cmake/external/onnx @@ -1 +1 @@ -Subproject commit 237926eab41de21fb9addc4b03b751fd6a3343ec +Subproject commit fe2433d3dd677ed5c582e194a49a632e707d0543 diff --git a/cmake/external/onnx-tensorrt b/cmake/external/onnx-tensorrt index dc22bb323e..99296a4462 160000 --- a/cmake/external/onnx-tensorrt +++ b/cmake/external/onnx-tensorrt @@ -1 +1 @@ -Subproject commit dc22bb323ece3c65419717be8a0d3d0f318a61fa +Subproject commit 99296a4462ffc0a8a4a9635e51382277bec68979 diff --git a/cmake/external/onnx_minimal.cmake b/cmake/external/onnx_minimal.cmake index c691965daf..33df1f5a31 100644 --- a/cmake/external/onnx_minimal.cmake +++ b/cmake/external/onnx_minimal.cmake @@ -48,6 +48,7 @@ endif() # ) # list(REMOVE_ITEM onnx_src ${onnx_exclude_src}) file(GLOB onnx_src CONFIGURE_DEPENDS +"${ONNX_SOURCE_ROOT}/onnx/common/common.h" "${ONNX_SOURCE_ROOT}/onnx/defs/data_type_utils.*" ) diff --git a/cmake/onnxruntime_providers.cmake b/cmake/onnxruntime_providers.cmake index 4a93738e9b..ac5b1e244c 100644 --- a/cmake/onnxruntime_providers.cmake +++ b/cmake/onnxruntime_providers.cmake @@ -416,6 +416,7 @@ endif() if (onnxruntime_USE_TENSORRT) add_definitions(-DUSE_TENSORRT=1) + set(BUILD_LIBRARY_ONLY 1) add_definitions("-DONNX_ML=1") add_definitions("-DONNX_NAMESPACE=onnx") include_directories(${PROJECT_SOURCE_DIR}/external/protobuf) @@ -446,14 +447,8 @@ if (onnxruntime_USE_TENSORRT) unset(OLD_CMAKE_CXX_FLAGS) unset(OLD_CMAKE_CUDA_FLAGS) set_target_properties(nvonnxparser PROPERTIES LINK_FLAGS "/ignore:4199") - target_sources(onnx2trt PRIVATE ${ONNXRUNTIME_ROOT}/test/win_getopt/mb/getopt.cc) - target_sources(getSupportedAPITest PRIVATE ${ONNXRUNTIME_ROOT}/test/win_getopt/mb/getopt.cc) - target_include_directories(onnx2trt PRIVATE ${ONNXRUNTIME_ROOT}/test/win_getopt/mb/include) - target_include_directories(getSupportedAPITest PRIVATE ${ONNXRUNTIME_ROOT}/test/win_getopt/mb/include) target_compile_options(nvonnxparser_static PRIVATE /FIio.h /wd4100) target_compile_options(nvonnxparser PRIVATE /FIio.h /wd4100) - target_compile_options(onnx2trt PRIVATE /FIio.h /wd4100) - target_compile_options(getSupportedAPITest PRIVATE /FIio.h /wd4100) endif() include_directories(${ONNXRUNTIME_ROOT}/../cmake/external/onnx-tensorrt) include_directories(${TENSORRT_INCLUDE_DIR}) diff --git a/csharp/testdata/test_input_BFLOAT16.py b/csharp/testdata/test_input_BFLOAT16.py index c9929898cb..b10636292e 100644 --- a/csharp/testdata/test_input_BFLOAT16.py +++ b/csharp/testdata/test_input_BFLOAT16.py @@ -22,6 +22,9 @@ graph_def = helper.make_graph(nodes=[node_def], name='test_types_BLOAT16', model_def = helper.make_model(graph_def, producer_name='AIInfra', opset_imports=[make_opsetid('', 13)]) -final_model = onnx.utils.polish_model(model_def) +onnx.checker.check_model(model_def) +onnx.helper.strip_doc_string(model_def) +final_model = onnx.shape_inference.infer_shapes(model_def) +onnx.checker.check_model(final_model) onnx.save(final_model, 'test_types_BFLOAT16.onnx') diff --git a/csharp/testdata/test_input_FLOAT16.py b/csharp/testdata/test_input_FLOAT16.py index 1de04a1a8a..e77406cc71 100644 --- a/csharp/testdata/test_input_FLOAT16.py +++ b/csharp/testdata/test_input_FLOAT16.py @@ -25,6 +25,8 @@ graph_def = helper.make_graph(nodes=[node_def], name='test_input_FLOAT16', model_def = helper.make_model(graph_def, producer_name='AIInfra', opset_imports=[make_opsetid('', 7)]) -final_model = onnx.utils.polish_model(model_def) +onnx.checker.check_model(model_def) +onnx.helper.strip_doc_string(model_def) +final_model = onnx.shape_inference.infer_shapes(model_def) +onnx.checker.check_model(final_model) onnx.save(final_model, 'test_types_FLOAT16.onnx') - diff --git a/onnxruntime/core/providers/cpu/activation/activations.cc b/onnxruntime/core/providers/cpu/activation/activations.cc index 1c2dfb79dc..db14b7c677 100644 --- a/onnxruntime/core/providers/cpu/activation/activations.cc +++ b/onnxruntime/core/providers/cpu/activation/activations.cc @@ -45,8 +45,10 @@ REGISTER_UNARY_ELEMENTWISE_KERNEL(HardSigmoid, 6); REGISTER_UNARY_ELEMENTWISE_KERNEL(LeakyRelu, 6); REGISTER_VERSIONED_UNARY_ELEMENTWISE_TYPED_KERNEL(Relu, 6, 12, float); REGISTER_VERSIONED_UNARY_ELEMENTWISE_TYPED_KERNEL(Relu, 6, 12, double); -REGISTER_UNARY_ELEMENTWISE_TYPED_KERNEL(Relu, 13, float); -REGISTER_UNARY_ELEMENTWISE_TYPED_KERNEL(Relu, 13, double); +REGISTER_VERSIONED_UNARY_ELEMENTWISE_TYPED_KERNEL(Relu, 13, 13, float); +REGISTER_VERSIONED_UNARY_ELEMENTWISE_TYPED_KERNEL(Relu, 13, 13, double); +REGISTER_UNARY_ELEMENTWISE_TYPED_KERNEL(Relu, 14, float); +REGISTER_UNARY_ELEMENTWISE_TYPED_KERNEL(Relu, 14, double); REGISTER_UNARY_ELEMENTWISE_KERNEL(Selu, 6); REGISTER_VERSIONED_UNARY_ELEMENTWISE_TYPED_KERNEL(Sigmoid, 6, 12, float); REGISTER_VERSIONED_UNARY_ELEMENTWISE_TYPED_KERNEL(Sigmoid, 6, 12, double); diff --git a/onnxruntime/core/providers/cpu/cpu_execution_provider.cc b/onnxruntime/core/providers/cpu/cpu_execution_provider.cc index 5f4188c763..23353b59a4 100644 --- a/onnxruntime/core/providers/cpu/cpu_execution_provider.cc +++ b/onnxruntime/core/providers/cpu/cpu_execution_provider.cc @@ -322,10 +322,10 @@ class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 10, Re // opset 11 class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 11, Clip); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, float, CumSum); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, double, CumSum); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, int32_t, CumSum); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, int64_t, CumSum); +class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 13, float, CumSum); +class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 13, double, CumSum); +class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 13, int32_t, CumSum); +class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 13, int64_t, CumSum); class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, bool, Equal); class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, int32_t, Equal); class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, int64_t, Equal); @@ -570,8 +570,8 @@ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, float, Ceil); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, float, Sqrt); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, double, Sqrt); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, float, Relu); -class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, double, Relu); +class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, 13, float, Relu); +class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, 13, double, Relu); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, float, Sigmoid); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, double, Sigmoid); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, float, Tanh); @@ -648,6 +648,15 @@ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, float, Softmax); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 13, double, Softmax); +//Opset 14 +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 14, float, CumSum); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 14, double, CumSum); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 14, int32_t, CumSum); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 14, int64_t, CumSum); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 14, float, Relu); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 14, double, Relu); +class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 14, Trilu); + // !!PLEASE READ BELOW!! Following that, add new entries above this comment /* *** IMPORTANT! *** @@ -1164,14 +1173,14 @@ Status RegisterOnnxOperatorKernels(KernelRegistry& kernel_registry) { BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, - BuildKernelCreateInfo, - BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, @@ -1693,6 +1704,19 @@ Status RegisterOnnxOperatorKernels(KernelRegistry& kernel_registry) { Softmax)>, BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, }; for (auto& function_table_entry : function_table) { diff --git a/onnxruntime/core/providers/cpu/math/cumsum.cc b/onnxruntime/core/providers/cpu/math/cumsum.cc index 0371d18ddc..5a121ae460 100644 --- a/onnxruntime/core/providers/cpu/math/cumsum.cc +++ b/onnxruntime/core/providers/cpu/math/cumsum.cc @@ -74,8 +74,45 @@ Status GetAxis(const Tensor* axis_tensor, int64_t input_rank, int64_t& axis_out) } // namespace cumsum_op -ONNX_CPU_OPERATOR_TYPED_KERNEL(CumSum, +ONNX_CPU_OPERATOR_VERSIONED_TYPED_KERNEL(CumSum, 11, + 13, + float, + KernelDefBuilder() + .TypeConstraint("T", DataTypeImpl::GetTensorType()) + .TypeConstraint("T2", std::vector{DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType()}), + CumSum); + +ONNX_CPU_OPERATOR_VERSIONED_TYPED_KERNEL(CumSum, + 11, + 13, + double, + KernelDefBuilder() + .TypeConstraint("T", DataTypeImpl::GetTensorType()) + .TypeConstraint("T2", std::vector{DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType()}), + CumSum); + +ONNX_CPU_OPERATOR_VERSIONED_TYPED_KERNEL(CumSum, + 11, + 13, + int32_t, + KernelDefBuilder() + .TypeConstraint("T", DataTypeImpl::GetTensorType()) + .TypeConstraint("T2", std::vector{DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType()}), + CumSum); + +ONNX_CPU_OPERATOR_VERSIONED_TYPED_KERNEL(CumSum, + 11, + 13, + int64_t, + KernelDefBuilder() + .TypeConstraint("T", DataTypeImpl::GetTensorType()) + .TypeConstraint("T2", std::vector{DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType()}), + CumSum); + +// Opset 14 kernels +ONNX_CPU_OPERATOR_TYPED_KERNEL(CumSum, + 14, float, KernelDefBuilder() .TypeConstraint("T", DataTypeImpl::GetTensorType()) @@ -83,7 +120,7 @@ ONNX_CPU_OPERATOR_TYPED_KERNEL(CumSum, CumSum); ONNX_CPU_OPERATOR_TYPED_KERNEL(CumSum, - 11, + 14, double, KernelDefBuilder() .TypeConstraint("T", DataTypeImpl::GetTensorType()) @@ -91,7 +128,7 @@ ONNX_CPU_OPERATOR_TYPED_KERNEL(CumSum, CumSum); ONNX_CPU_OPERATOR_TYPED_KERNEL(CumSum, - 11, + 14, int32_t, KernelDefBuilder() .TypeConstraint("T", DataTypeImpl::GetTensorType()) @@ -99,7 +136,7 @@ ONNX_CPU_OPERATOR_TYPED_KERNEL(CumSum, CumSum); ONNX_CPU_OPERATOR_TYPED_KERNEL(CumSum, - 11, + 14, int64_t, KernelDefBuilder() .TypeConstraint("T", DataTypeImpl::GetTensorType()) diff --git a/onnxruntime/core/providers/cpu/tensor/identity_op.cc b/onnxruntime/core/providers/cpu/tensor/identity_op.cc index 6673793c13..4a94313b09 100644 --- a/onnxruntime/core/providers/cpu/tensor/identity_op.cc +++ b/onnxruntime/core/providers/cpu/tensor/identity_op.cc @@ -34,7 +34,7 @@ ONNX_CPU_OPERATOR_VERSIONED_KERNEL( ONNX_CPU_OPERATOR_KERNEL( Identity, 13, - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::AllTensorTypes()).Alias(0, 0), + KernelDefBuilder().TypeConstraint("V", DataTypeImpl::AllTensorTypes()).Alias(0, 0), IdentityOp); } // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cpu/trilu.cc b/onnxruntime/core/providers/cpu/tensor/trilu.cc similarity index 92% rename from onnxruntime/contrib_ops/cpu/trilu.cc rename to onnxruntime/core/providers/cpu/tensor/trilu.cc index e53d886a33..dff188b855 100644 --- a/onnxruntime/contrib_ops/cpu/trilu.cc +++ b/onnxruntime/core/providers/cpu/tensor/trilu.cc @@ -11,6 +11,7 @@ using namespace onnxruntime::common; namespace onnxruntime { +#ifndef DISABLE_CONTRIB_OPS namespace contrib { ONNX_OPERATOR_KERNEL_EX( @@ -20,6 +21,16 @@ ONNX_OPERATOR_KERNEL_EX( kCpuExecutionProvider, KernelDefBuilder().MayInplace(0, 0).TypeConstraint("T", BuildKernelDefConstraints()), Trilu); +} // namespace contrib +#endif + +ONNX_OPERATOR_KERNEL_EX( + Trilu, + kOnnxDomain, + 14, + kCpuExecutionProvider, + KernelDefBuilder().MayInplace(0, 0).TypeConstraint("T", BuildKernelDefConstraints()), + Trilu); template static Status TriluImpl(const Tensor* X, Tensor* Y, int64_t k_val, bool up) { @@ -103,5 +114,4 @@ Status Trilu::Compute(OpKernelContext* ctx) const { return status; } -} // namespace contrib } // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cpu/trilu.h b/onnxruntime/core/providers/cpu/tensor/trilu.h similarity index 91% rename from onnxruntime/contrib_ops/cpu/trilu.h rename to onnxruntime/core/providers/cpu/tensor/trilu.h index bc767c239a..89afcc4d73 100644 --- a/onnxruntime/contrib_ops/cpu/trilu.h +++ b/onnxruntime/core/providers/cpu/tensor/trilu.h @@ -2,7 +2,6 @@ // Licensed under the MIT License. namespace onnxruntime { -namespace contrib { class Trilu final : public OpKernel { public: @@ -17,5 +16,4 @@ class Trilu final : public OpKernel { bool upper_; }; -} // namespace contrib } // namespace onnxruntime diff --git a/onnxruntime/python/tools/featurizer_ops/create_test_model.py b/onnxruntime/python/tools/featurizer_ops/create_test_model.py index be5f1b9435..6678186feb 100644 --- a/onnxruntime/python/tools/featurizer_ops/create_test_model.py +++ b/onnxruntime/python/tools/featurizer_ops/create_test_model.py @@ -113,7 +113,10 @@ def create_model(): initializer=[tensor_float32]) model_def = helper.make_model(graph_def, producer_name='feed_inputs_test') - final_model = onnx.utils.polish_model(model_def) + onnx.checker.check_model(model_def) + onnx.helper.strip_doc_string(model_def) + final_model = onnx.shape_inference.infer_shapes(model_def) + onnx.checker.check_model(final_model) onnx.save(final_model, args.output_file) diff --git a/onnxruntime/test/onnx/gen_test_models.py b/onnxruntime/test/onnx/gen_test_models.py index 503c5f9c11..31effae815 100644 --- a/onnxruntime/test/onnx/gen_test_models.py +++ b/onnxruntime/test/onnx/gen_test_models.py @@ -30,6 +30,12 @@ def write_tensor(f, c, input_name=None): body = tensor.SerializeToString() f.write(body) +def infer_shapes(model_def): + onnx.checker.check_model(model_def) + onnx.helper.strip_doc_string(model_def) + final_model = onnx.shape_inference.infer_shapes(model_def) + onnx.checker.check_model(final_model) + return final_model def generate_abs_op_test(type, X, top_test_folder): for is_raw in [True, False]: @@ -53,7 +59,7 @@ def generate_abs_op_test(type, X, top_test_folder): graph_def = helper.make_graph([node_def], 'test-model', [X_INFO], [Y], [tensor_x]) # Create the model (ModelProto) model_def = helper.make_model(graph_def, producer_name='onnx-example') - #final_model = onnx.utils.polish_model(model_def) + #final_model = infer_shapes(model_def) final_model = model_def if is_raw: onnx.external_data_helper.convert_model_to_external_data(final_model, True) @@ -78,7 +84,7 @@ def generate_size_op_test(type, X, test_folder): graph_def = helper.make_graph([node_def], 'test-model', [X_INFO], [Y], [tensor_x]) # Create the model (ModelProto) model_def = helper.make_model(graph_def, producer_name='onnx-example') - final_model = onnx.utils.polish_model(model_def) + final_model = infer_shapes(model_def) onnx.save(final_model, os.path.join(test_folder, 'model.onnx')) expected_output_array = np.int64(X.size) expected_output_tensor = numpy_helper.from_array(expected_output_array) @@ -101,7 +107,7 @@ def generate_reducesum_op_test(X, test_folder): graph_def = helper.make_graph([node_def], 'test-model', [X_INFO], [Y], [tensor_x]) # Create the model (ModelProto) model_def = helper.make_model(graph_def, producer_name='onnx-example') - final_model = onnx.utils.polish_model(model_def) + final_model = infer_shapes(model_def) onnx.save(final_model, os.path.join(test_folder, 'model.onnx')) expected_output_array = np.sum(X) expected_output_tensor = numpy_helper.from_array(expected_output_array) diff --git a/onnxruntime/test/contrib_ops/shape_inference_test_helper.h b/onnxruntime/test/providers/cpu/tensor/shape_inference_test_helper.h similarity index 84% rename from onnxruntime/test/contrib_ops/shape_inference_test_helper.h rename to onnxruntime/test/providers/cpu/tensor/shape_inference_test_helper.h index 23691051b2..89bbbab825 100644 --- a/onnxruntime/test/contrib_ops/shape_inference_test_helper.h +++ b/onnxruntime/test/providers/cpu/tensor/shape_inference_test_helper.h @@ -11,8 +11,6 @@ namespace test { auto schema_registry = ONNX_NAMESPACE::OpSchemaRegistry::Instance(); -const std::string MS_DOMAIN = "com.microsoft"; - void CheckShapeEquality(ONNX_NAMESPACE::TensorShapeProto* shape1, ONNX_NAMESPACE::TensorShapeProto* shape2) { EXPECT_NE(shape1, nullptr); EXPECT_NE(shape2, nullptr); @@ -50,17 +48,19 @@ inline void CreateValueInfo( } } -inline void TestShapeInference( - const std::string& op_type, - const std::vector& inputs, - const std::vector& attributes, - ONNX_NAMESPACE::ValueInfoProto& output) { +inline void TestShapeInference(const std::string& op_type, + const std::string& op_domain, + int op_version, + int ir_version, + const std::vector& inputs, + const std::vector& attributes, + ONNX_NAMESPACE::ValueInfoProto& output) { ONNX_NAMESPACE::ModelProto model; // Set opset (domain + version) ONNX_NAMESPACE::OperatorSetIdProto* op_set_id = model.add_opset_import(); - op_set_id->set_domain(MS_DOMAIN); - op_set_id->set_version(1); - model.set_ir_version(6); + op_set_id->set_domain(op_domain); + op_set_id->set_version(op_version); + model.set_ir_version(ir_version); model.set_producer_name("onnx"); // Set model graph @@ -71,7 +71,7 @@ inline void TestShapeInference( // Set add operator node to graph auto& node = *graph->add_node(); node.set_op_type(op_type); - node.set_domain(MS_DOMAIN); + node.set_domain(op_domain); node.set_name("test_node"); // Add node inputs and graph inputs @@ -102,6 +102,5 @@ inline void TestShapeInference( auto inferred_shape = inferred_output.mutable_type()->mutable_tensor_type()->mutable_shape(); CheckShapeEquality(shape, inferred_shape); } - } // namespace test } // namespace onnxruntime \ No newline at end of file diff --git a/onnxruntime/test/providers/cpu/tensor/trilu_op_test.cc b/onnxruntime/test/providers/cpu/tensor/trilu_op_test.cc new file mode 100644 index 0000000000..c4e12c5bbd --- /dev/null +++ b/onnxruntime/test/providers/cpu/tensor/trilu_op_test.cc @@ -0,0 +1,242 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "test/providers/provider_test_utils.h" +#include "core/util/math.h" + +namespace onnxruntime { +namespace test { + +TEST(TriluOpTest, two_by_two_float_upper) { + OpTester test("Trilu", 14, kOnnxDomain); + int64_t up = 1; + test.AddAttribute("upper", up); + test.AddInput("X", {2, 2}, {4.f, 7.f, 2.f, 6.f}); + test.AddOutput("Y", {2, 2}, {4.f, 7.f, 0.f, 6.f}); + test.Run(); +} + +TEST(TriluOpTest, two_by_two_float_lower) { + OpTester test("Trilu", 14, kOnnxDomain); + int64_t up = 0; + test.AddAttribute("upper", up); + test.AddInput("X", {2, 2}, {4.f, 7.f, 2.f, 6.f}); + test.AddOutput("Y", {2, 2}, {4.f, 0.f, 2.f, 6.f}); + test.Run(); +} + +TEST(TriluOpTest, two_by_two_double_upper) { + OpTester test("Trilu", 14, kOnnxDomain); + test.AddInput("X", {2, 2}, {4, 7, 2, 6}); + test.AddInput("k", {1}, {1}); + test.AddOutput("Y", {2, 2}, {0, 7, 0, 0}); + test.Run(); +} + +TEST(TriluOpTest, two_by_two_double_lower) { + OpTester test("Trilu", 14, kOnnxDomain); + int64_t up = 0; + test.AddAttribute("upper", up); + test.AddInput("X", {2, 2}, {4, 7, 2, 6}); + test.AddInput("k", {1}, {1}); + test.AddOutput("Y", {2, 2}, {4, 7, 2, 6}); + test.Run(); +} + +TEST(TriluOpTest, two_by_two_long_upper) { + OpTester test("Trilu", 14, kOnnxDomain); + int64_t up = 1; + test.AddAttribute("upper", up); + test.AddInput("X", {2, 2}, {4, 7, 2, 6}); + test.AddOutput("Y", {2, 2}, {4, 7, 0, 6}); + test.Run(); +} + +TEST(TriluOpTest, two_by_two_long_lower) { + OpTester test("Trilu", 14, kOnnxDomain); + int64_t up = 0; + test.AddAttribute("upper", up); + test.AddInput("X", {2, 2}, {4, 7, 2, 6}); + test.AddOutput("Y", {2, 2}, {4, 0, 2, 6}); + test.Run(); +} + +TEST(TriluOpTest, three_dim_float_upper) { + OpTester test("Trilu", 14, kOnnxDomain); + test.AddInput("X", {2, 3, 4}, + {4.f, 1.f, 5.f, 8.f, + 4.f, 3.f, 2.f, 4.f, + 6.f, 1.f, 2.f, 3.f, + 1.f, 6.f, 2.f, 1.f, + 4.f, 1.f, 5.f, 8.f, + 4.f, 3.f, 2.f, 4.f, + }); + test.AddInput("k", {1}, {1}); + test.AddOutput("Y", {2, 3, 4}, + {0.f, 1.f, 5.f, 8.f, + 0.f, 0.f, 2.f, 4.f, + 0.f, 0.f, 0.f, 3.f, + 0.f, 6.f, 2.f, 1.f, + 0.f, 0.f, 5.f, 8.f, + 0.f, 0.f, 0.f, 4.f, + }); + test.Run(); +} + +TEST(TriluOpTest, three_dim_float_lower) { + OpTester test("Trilu", 14, kOnnxDomain); + int64_t up = 0; + test.AddAttribute("upper", up); + test.AddInput("X", {2, 3, 4}, + {4.f, 1.f, 5.f, 8.f, + 4.f, 3.f, 2.f, 4.f, + 6.f, 1.f, 2.f, 3.f, + 1.f, 6.f, 2.f, 1.f, + 4.f, 1.f, 5.f, 8.f, + 4.f, 3.f, 2.f, 4.f, + }); + test.AddInput("k", {1}, {1}); + test.AddOutput("Y", {2, 3, 4}, + {4.f, 1.f, 0.f, 0.f, + 4.f, 3.f, 2.f, 0.f, + 6.f, 1.f, 2.f, 3.f, + 1.f, 6.f, 0.f, 0.f, + 4.f, 1.f, 5.f, 0.f, + 4.f, 3.f, 2.f, 4.f, + }); + test.Run(); +} + +TEST(TriluOpTest, neg_k_float_upper) { + OpTester test("Trilu", 14, kOnnxDomain); + int64_t up = 1; + test.AddAttribute("upper", up); + test.AddInput("X", {2, 3, 4}, + {4.f, 1.f, 5.f, 8.f, + 4.f, 3.f, 2.f, 4.f, + 6.f, 1.f, 2.f, 3.f, + 1.f, 6.f, 2.f, 1.f, + 4.f, 1.f, 5.f, 8.f, + 4.f, 3.f, 2.f, 4.f, + }); + test.AddInput("k", {1}, {-1}); + test.AddOutput("Y", {2, 3, 4}, + {4.f, 1.f, 5.f, 8.f, + 4.f, 3.f, 2.f, 4.f, + 0.f, 1.f, 2.f, 3.f, + 1.f, 6.f, 2.f, 1.f, + 4.f, 1.f, 5.f, 8.f, + 0.f, 3.f, 2.f, 4.f, + }); + test.Run(); +} + +TEST(TriluOpTest, neg_k_float_lower) { + OpTester test("Trilu", 14, kOnnxDomain); + int64_t up = 0; + test.AddAttribute("upper", up); + test.AddInput("X", {2, 3, 4}, + {4.f, 1.f, 5.f, 8.f, + 4.f, 3.f, 2.f, 4.f, + 6.f, 1.f, 2.f, 3.f, + 1.f, 6.f, 2.f, 1.f, + 4.f, 1.f, 5.f, 8.f, + 4.f, 3.f, 2.f, 4.f, + }); + test.AddInput("k", {1}, {-1}); + test.AddOutput("Y", {2, 3, 4}, + {0.f, 0.f, 0.f, 0.f, + 4.f, 0.f, 0.f, 0.f, + 6.f, 1.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, + 4.f, 0.f, 0.f, 0.f, + 4.f, 3.f, 0.f, 0.f, + }); + test.Run(); +} + +TEST(TriluTest, small_k_float_upper) { + OpTester test("Trilu", 14, kOnnxDomain); + test.AddInput("X", {2, 3, 4}, + {4.f, 1.f, 5.f, 8.f, + 4.f, 3.f, 2.f, 4.f, + 6.f, 1.f, 2.f, 3.f, + 1.f, 6.f, 2.f, 1.f, + 4.f, 1.f, 5.f, 8.f, + 4.f, 3.f, 2.f, 4.f, + }); + test.AddInput("k", {1}, {-5}); + test.AddOutput("Y", {2, 3, 4}, + {4.f, 1.f, 5.f, 8.f, + 4.f, 3.f, 2.f, 4.f, + 6.f, 1.f, 2.f, 3.f, + 1.f, 6.f, 2.f, 1.f, + 4.f, 1.f, 5.f, 8.f, + 4.f, 3.f, 2.f, 4.f, + }); + test.Run(); +} + +TEST(TriluOpTest, small_k_float_lower) { + OpTester test("Trilu", 14, kOnnxDomain); + int64_t up = 0; + test.AddAttribute("upper", up); + test.AddInput("X", {2, 3, 4}, + {4.f, 1.f, 5.f, 8.f, + 4.f, 3.f, 2.f, 4.f, + 6.f, 1.f, 2.f, 3.f, + 1.f, 6.f, 2.f, 1.f, + 4.f, 1.f, 5.f, 8.f, + 4.f, 3.f, 2.f, 4.f, + }); + test.AddInput("k", {1}, {-5}); + test.AddOutput("Y", {2, 3, 4}, + {0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, + 0.f, 0.f, 0.f, 0.f, + }); + test.Run(); +} + +TEST(TriluOpTest, zero_dim_upper) { + OpTester test("Trilu", 14, kOnnxDomain); + test.AddInput("X", {2, 3, 0}, {}); + test.AddInput("k", {1}, {0}); + test.AddOutput("Y", {2, 3, 0}, {}); + test.Run(); +} + +TEST(TriluOpTest, zero_dim_lower) { + OpTester test("Trilu", 14, kOnnxDomain); + int64_t up = 0; + test.AddAttribute("upper", up); + test.AddInput("X", {2, 3, 0}, {}); + test.AddInput("k", {1}, {0}); + test.AddOutput("Y", {2, 3, 0}, {}); + test.Run(); +} + +TEST(TriluOpTest, zero_dim_2_upper) { + OpTester test("Trilu", 14, kOnnxDomain); + test.AddInput("X", {2, 0, 0}, {}); + test.AddInput("k", {1}, {-5}); + test.AddOutput("Y", {2, 0, 0}, {}); + test.Run(); +} + +TEST(TriluOpTest, zero_dim_2_lower) { + OpTester test("Trilu", 14, kOnnxDomain); + int64_t up = 0; + test.AddAttribute("upper", up); + test.AddInput("X", {2, 0, 0}, {}); + test.AddInput("k", {1}, {-5}); + test.AddOutput("Y", {2, 0, 0}, {}); + test.Run(); +} + +} // namespace test +} // namespace onnxruntime diff --git a/onnxruntime/test/contrib_ops/trilu_shape_inference_test.cc b/onnxruntime/test/providers/cpu/tensor/trilu_shape_inference_test.cc similarity index 89% rename from onnxruntime/test/contrib_ops/trilu_shape_inference_test.cc rename to onnxruntime/test/providers/cpu/tensor/trilu_shape_inference_test.cc index 561f0bb0cc..99c3fcf0f6 100644 --- a/onnxruntime/test/contrib_ops/trilu_shape_inference_test.cc +++ b/onnxruntime/test/providers/cpu/tensor/trilu_shape_inference_test.cc @@ -9,6 +9,21 @@ namespace onnxruntime { namespace test { +void TestShapeInference( + const std::string& op_type, + const std::vector& inputs, + const std::vector& attributes, + ONNX_NAMESPACE::ValueInfoProto& output) { + +#ifndef DISABLE_CONTRIB_OPS + // test trilu contrib op for maintaining backward compatibility + TestShapeInference(op_type, kMSDomain, 1, 6, inputs, attributes, output); +#endif + + // test trilu onnx domain op + TestShapeInference(op_type, kOnnxDomain, 14, 7, inputs, attributes, output); +} + TEST(ShapeInferenceTests, tri_upper_float) { std::vector shape = {4, 7}; ONNX_NAMESPACE::ValueInfoProto input; diff --git a/onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc b/onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc index 82bc10d077..37acd93e86 100644 --- a/onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc +++ b/onnxruntime/test/testdata/onnx_backend_test_series_filters.jsonc @@ -51,7 +51,23 @@ "^test_resize_upsample_sizes_nearest_ceil_half_pixel_cpu", "^test_resize_upsample_sizes_nearest_cpu", "^test_resize_upsample_sizes_nearest_floor_align_corners_cpu", - "^test_resize_upsample_sizes_nearest_round_prefer_ceil_asymmetric_cpu" + "^test_resize_upsample_sizes_nearest_round_prefer_ceil_asymmetric_cpu", + // Following tests are for opset 14 ops and are not yet implemented in ORT + "^test_gru_batchwise_cpu", + "^test_gru_defaults_cpu", + "^test_gru_seq_length_cpu", + "^test_gru_with_initial_bias_cpu", + "^test_identity_sequence_cpu", + "^test_lstm_batchwise_cpu", + "^test_lstm_defaults_cpu", + "^test_lstm_with_initial_bias_cpu", + "^test_lstm_with_peepholes_cpu", + "^test_reshape_allowzero_reordered_cpu", + "^test_rnn_seq_length_cpu", + "^test_simple_rnn_batchwise_cpu", + "^test_simple_rnn_defaults_cpu", + "^test_simple_rnn_with_initial_bias_cpu", + "^test_tril_*" ], "current_failing_tests_x86": [ "^test_vgg19", diff --git a/tools/ci_build/github/linux/docker/scripts/manylinux/requirements.txt b/tools/ci_build/github/linux/docker/scripts/manylinux/requirements.txt index 9587e53554..8a47b1e55b 100644 --- a/tools/ci_build/github/linux/docker/scripts/manylinux/requirements.txt +++ b/tools/ci_build/github/linux/docker/scripts/manylinux/requirements.txt @@ -3,7 +3,7 @@ mypy pytest setuptools>=41.4.0 wheel -git+http://github.com/onnx/onnx.git@237926eab41de21fb9addc4b03b751fd6a3343ec#egg=onnx +git+http://github.com/onnx/onnx.git@fe2433d3dd677ed5c582e194a49a632e707d0543#egg=onnx protobuf sympy==1.1.1 flake8 diff --git a/tools/ci_build/github/linux/docker/scripts/requirements.txt b/tools/ci_build/github/linux/docker/scripts/requirements.txt index b5eec6ca4a..9bdd27b96b 100644 --- a/tools/ci_build/github/linux/docker/scripts/requirements.txt +++ b/tools/ci_build/github/linux/docker/scripts/requirements.txt @@ -4,7 +4,7 @@ mypy pytest setuptools>=41.4.0 wheel -git+http://github.com/onnx/onnx.git@237926eab41de21fb9addc4b03b751fd6a3343ec#egg=onnx +git+http://github.com/onnx/onnx.git@fe2433d3dd677ed5c582e194a49a632e707d0543#egg=onnx argparse sympy==1.1.1 flake8 From aeca7c29408f03ad9e19837085156cba3c117263 Mon Sep 17 00:00:00 2001 From: RandySheriffH <48490400+RandySheriffH@users.noreply.github.com> Date: Mon, 29 Mar 2021 12:04:36 -0700 Subject: [PATCH 020/129] Cuda Profiler (#7110) * implement cuda profiler * add counters * downgrade cupti kernel version * move mutex * add cupti to path * fix win gpu build err * add path for cuda10 * fix linux com err * extend include path * add init flag * fix test case * fix tensorrt pipeline * add UT Co-authored-by: Ubuntu --- cmake/onnxruntime_common.cmake | 6 + .../onnxruntime/core/common/logging/logging.h | 4 +- onnxruntime/core/common/profiler.cc | 154 +++++++++++++++++- .../test/framework/inference_session_test.cc | 68 ++++---- .../test/python/onnxruntime_test_python.py | 6 +- .../github/windows/setup_env_cuda.bat | 2 +- .../github/windows/setup_env_cuda_11.bat | 2 +- .../ci_build/github/windows/setup_env_trt.bat | 2 +- 8 files changed, 207 insertions(+), 37 deletions(-) diff --git a/cmake/onnxruntime_common.cmake b/cmake/onnxruntime_common.cmake index c9dd4326cc..d414dd1f51 100644 --- a/cmake/onnxruntime_common.cmake +++ b/cmake/onnxruntime_common.cmake @@ -79,6 +79,12 @@ source_group(TREE ${REPO_ROOT} FILES ${onnxruntime_common_src}) add_library(onnxruntime_common ${onnxruntime_common_src}) +if (onnxruntime_USE_CUDA) + target_include_directories(onnxruntime_common PUBLIC ${onnxruntime_CUDA_HOME}/include ${onnxruntime_CUDA_HOME}/extras/CUPTI/include) + target_link_directories(onnxruntime_common PUBLIC ${onnxruntime_CUDA_HOME}/extras/CUPTI/lib64) + target_link_libraries(onnxruntime_common cupti) +endif() + if (onnxruntime_USE_TELEMETRY) set_target_properties(onnxruntime_common PROPERTIES COMPILE_FLAGS "/FI${ONNXRUNTIME_INCLUDE_DIR}/core/platform/windows/TraceLoggingConfigPrivate.h") endif() diff --git a/include/onnxruntime/core/common/logging/logging.h b/include/onnxruntime/core/common/logging/logging.h index 96c07f09c4..95b6a4aa6d 100644 --- a/include/onnxruntime/core/common/logging/logging.h +++ b/include/onnxruntime/core/common/logging/logging.h @@ -55,6 +55,7 @@ namespace profiling { enum EventCategory { SESSION_EVENT = 0, NODE_EVENT, + KERNEL_EVENT, EVENT_CATEGORY_MAX }; @@ -63,7 +64,8 @@ Event descriptions for the above session events. */ static constexpr const char* event_categor_names_[EVENT_CATEGORY_MAX] = { "Session", - "Node"}; + "Node", + "Kernel"}; /* Timing record for all events. diff --git a/onnxruntime/core/common/profiler.cc b/onnxruntime/core/common/profiler.cc index ee752a6a62..3679501a47 100644 --- a/onnxruntime/core/common/profiler.cc +++ b/onnxruntime/core/common/profiler.cc @@ -2,11 +2,148 @@ // Licensed under the MIT License. #include "profiler.h" +#include + +#ifdef USE_CUDA +#include +#endif namespace onnxruntime { namespace profiling { using namespace std::chrono; +class DeviceProfiler { + public: + static DeviceProfiler* GetDeviceProfiler(); + virtual void StartProfiling(TimePoint start_time, int pid, int tid) = 0; + virtual std::vector EndProfiling() = 0; + virtual ~DeviceProfiler() = default; +}; + +#ifdef USE_CUDA +#define BUF_SIZE (32 * 1024) +#define ALIGN_SIZE (8) +#define ALIGN_BUFFER(buffer, align) \ + (((uintptr_t)(buffer) & ((align)-1)) ? ((buffer) + (align) - ((uintptr_t)(buffer) & ((align)-1))) : (buffer)) +#define DUR(s, e) std::lround(static_cast(e - s) / 1000) + +class CudaProfiler final: public DeviceProfiler { + public: + friend class DeviceProfiler; + ~CudaProfiler() = default; + ORT_DISALLOW_COPY_ASSIGNMENT_AND_MOVE(CudaProfiler); + void StartProfiling(TimePoint start_time, int pid, int tid) override; + std::vector EndProfiling() override; + private: + CudaProfiler() = default; + static void CUPTIAPI BufferRequested(uint8_t**, size_t*, size_t*); + static void CUPTIAPI BufferCompleted(CUcontext, uint32_t, uint8_t*, size_t, size_t); + struct KernelStat { + std::string name_ = {}; + uint32_t stream_ = 0; + int32_t grid_x_ = 0; + int32_t grid_y_ = 0; + int32_t grid_z_ = 0; + int32_t block_x_ = 0; + int32_t block_y_ = 0; + int32_t block_z_ = 0; + int64_t start_ = 0; + int64_t stop_ = 0; + }; + static OrtMutex mutex_; + static std::vector stats_; + bool initialized_ = false; + TimePoint start_time_; + int pid_ = 0; + int tid_ = 0; + static std::atomic_flag enabled_; +}; + +OrtMutex CudaProfiler::mutex_; +std::vector CudaProfiler::stats_; +std::atomic_flag CudaProfiler::enabled_; + +void CUPTIAPI CudaProfiler::BufferRequested(uint8_t** buffer, size_t* size, size_t* maxNumRecords) { + uint8_t* bfr = (uint8_t*)malloc(BUF_SIZE + ALIGN_SIZE); + ORT_ENFORCE(bfr, "Failed to allocate memory for cuda kernel profiling."); + *size = BUF_SIZE; + *buffer = ALIGN_BUFFER(bfr, ALIGN_SIZE); + *maxNumRecords = 0; +} + +void CUPTIAPI CudaProfiler::BufferCompleted(CUcontext, uint32_t, uint8_t* buffer, size_t, size_t validSize) { + CUptiResult status; + CUpti_Activity* record = NULL; + if (validSize > 0) { + std::unique_lock lock(mutex_); + do { + status = cuptiActivityGetNextRecord(buffer, validSize, &record); + if (status == CUPTI_SUCCESS) { + if (CUPTI_ACTIVITY_KIND_KERNEL == record->kind) { + CUpti_ActivityKernel4* kernel = (CUpti_ActivityKernel4*)record; + stats_.push_back({kernel->name, kernel->streamId, + kernel->gridX, kernel->gridY, kernel->gridZ, + kernel->blockX, kernel->blockY, kernel->blockZ, + static_cast(kernel->start), + static_cast(kernel->end)}); + } + } else if (status == CUPTI_ERROR_MAX_LIMIT_REACHED) { + break; + } + } while (1); + } + free(buffer); +} + +void CudaProfiler::StartProfiling(TimePoint start_time, int pid, int tid) { + if (!enabled_.test_and_set()) { + start_time_ = start_time; + pid_ = pid; + tid_ = tid; + if (cuptiActivityEnable(CUPTI_ACTIVITY_KIND_KERNEL) == CUPTI_SUCCESS && + cuptiActivityRegisterCallbacks(BufferRequested, BufferCompleted) == CUPTI_SUCCESS) { + initialized_ = true; + } + } +} + +std::vector CudaProfiler::EndProfiling() { + std::vector events; + if (enabled_.test_and_set()) { + if (initialized_) { + cuptiActivityFlushAll(1); + std::unique_lock lock(mutex_); + int64_t profiling_start = std::chrono::duration_cast(start_time_.time_since_epoch()).count(); + for (const auto& stat : stats_) { + std::initializer_list> args = {{"stream", std::to_string(stat.stream_)}, + {"grid_x", std::to_string(stat.grid_x_)}, + {"grid_y", std::to_string(stat.grid_y_)}, + {"grid_z", std::to_string(stat.grid_z_)}, + {"block_x", std::to_string(stat.block_x_)}, + {"block_y", std::to_string(stat.block_y_)}, + {"block_z", std::to_string(stat.block_z_)}}; + events.push_back({EventCategory::KERNEL_EVENT, pid_, tid_, stat.name_, DUR(profiling_start, stat.stop_), DUR(stat.start_, stat.stop_), {args.begin(), args.end()}}); + } + stats_.clear(); + } else { + std::initializer_list> args; + events.push_back({EventCategory::KERNEL_EVENT, pid_, tid_, "not_available_due_to_cupti_error", 0, 0, {args.begin(), args.end()}}); + } + } + enabled_.clear(); + return events; +} +#endif //USE_CUDA + +DeviceProfiler* DeviceProfiler::GetDeviceProfiler() { +#ifdef USE_CUDA + static CudaProfiler cuda_profiler; + return &cuda_profiler; +#else + return nullptr; +#endif +} + std::atomic Profiler::global_max_num_events_{1000 * 1000}; #ifdef ENABLE_STATIC_PROFILER_INSTANCE @@ -16,7 +153,8 @@ profiling::Profiler::~Profiler() { instance_ = nullptr; } #else -profiling::Profiler::~Profiler() {} +profiling::Profiler::~Profiler() { +} #endif ::onnxruntime::TimePoint profiling::Profiler::StartTime() const { @@ -43,6 +181,10 @@ void Profiler::StartProfiling(const logging::Logger* custom_logger) { profile_with_logger_ = true; custom_logger_ = custom_logger; profiling_start_time_ = StartTime(); + DeviceProfiler* device_profiler = DeviceProfiler::GetDeviceProfiler(); + if (device_profiler) { + device_profiler->StartProfiling(profiling_start_time_, logging::GetProcessId(), logging::GetThreadId()); + } } template @@ -51,6 +193,10 @@ void Profiler::StartProfiling(const std::basic_string& file_name) { profile_stream_.open(file_name, std::ios::out | std::ios::trunc); profile_stream_file_ = ToMBString(file_name); profiling_start_time_ = StartTime(); + DeviceProfiler* device_profiler = DeviceProfiler::GetDeviceProfiler(); + if (device_profiler) { + device_profiler->StartProfiling(profiling_start_time_, logging::GetProcessId(), logging::GetThreadId()); + } } template void Profiler::StartProfiling(const std::basic_string& file_name); @@ -101,6 +247,12 @@ std::string Profiler::EndProfiling() { std::lock_guard lock(mutex_); profile_stream_ << "[\n"; + DeviceProfiler* device_profiler = DeviceProfiler::GetDeviceProfiler(); + if (device_profiler) { + std::vector device_events = device_profiler->EndProfiling(); + std::copy(device_events.begin(), device_events.end(), std::back_inserter(events_)); + } + for (size_t i = 0; i < events_.size(); ++i) { auto& rec = events_[i]; profile_stream_ << R"({"cat" : ")" << event_categor_names_[rec.cat] << "\","; diff --git a/onnxruntime/test/framework/inference_session_test.cc b/onnxruntime/test/framework/inference_session_test.cc index 4db59b3bc8..1e20226232 100644 --- a/onnxruntime/test/framework/inference_session_test.cc +++ b/onnxruntime/test/framework/inference_session_test.cc @@ -590,6 +590,11 @@ TEST(InferenceSessionTests, CheckRunProfilerWithSessionOptions) { so.profile_file_prefix = ORT_TSTR("onnxprofile_profile_test"); InferenceSession session_object(so, GetEnvironment()); +#ifdef USE_CUDA + CUDAExecutionProviderInfo epi; + epi.device_id = 0; + EXPECT_TRUE(session_object.RegisterExecutionProvider(onnxruntime::make_unique(epi)).IsOK()); +#endif ASSERT_STATUS_OK(session_object.Load(MODEL_URI)); ASSERT_STATUS_OK(session_object.Initialize()); @@ -602,25 +607,29 @@ TEST(InferenceSessionTests, CheckRunProfilerWithSessionOptions) { std::ifstream profile(profile_file); ASSERT_TRUE(profile); std::string line; + std::vector lines; - std::vector tags = {"pid", "dur", "ts", "ph", "X", "name", "args"}; - int count = 0; while (std::getline(profile, line)) { - if (count == 0) { - ASSERT_TRUE(line.find("[") != string::npos); - } else if (count <= 7) { - for (auto& s : tags) { - ASSERT_TRUE(line.find(s) != string::npos); - } - } else { - ASSERT_TRUE(line.find("]") != string::npos); - } - - if (count == 1) { - ASSERT_TRUE(line.find("model_loading_uri") != string::npos); - } - count++; + lines.push_back(line); } + + auto size = lines.size(); + ASSERT_TRUE(size > 1); + ASSERT_TRUE(lines[0].find("[") != string::npos); + ASSERT_TRUE(lines[1].find("model_loading_uri") != string::npos); + ASSERT_TRUE(lines[size-1].find("]") != string::npos); + std::vector tags = {"pid", "dur", "ts", "ph", "X", "name", "args"}; + + bool has_kernel_info = false; + for (size_t i = 1; i < size - 1; ++i) { + for (auto& s : tags) { + ASSERT_TRUE(lines[i].find(s) != string::npos); + has_kernel_info = has_kernel_info || (lines[i].find("Kernel") != string::npos); + } + } +#ifdef USE_CUDA + ASSERT_TRUE(has_kernel_info); +#endif } TEST(InferenceSessionTests, CheckRunProfilerWithStartProfile) { @@ -641,24 +650,23 @@ TEST(InferenceSessionTests, CheckRunProfilerWithStartProfile) { std::ifstream profile(profile_file); std::string line; + std::vector lines; - std::vector tags = {"pid", "dur", "ts", "ph", "X", "name", "args"}; - int count = 0; while (std::getline(profile, line)) { - if (count == 0) { - ASSERT_TRUE(line.find("[") != string::npos); - } else if (count <= 5) { - for (auto& s : tags) { - ASSERT_TRUE(line.find(s) != string::npos); - } - } else { - ASSERT_TRUE(line.find("]") != string::npos); - } + lines.push_back(line); + } - if (count == 1) { - ASSERT_TRUE(line.find("mul_1_fence_before") != string::npos); + auto size = lines.size(); + ASSERT_TRUE(size > 1); + ASSERT_TRUE(lines[0].find("[") != string::npos); + ASSERT_TRUE(lines[1].find("mul_1_fence_before") != string::npos); + ASSERT_TRUE(lines[size - 1].find("]") != string::npos); + std::vector tags = {"pid", "dur", "ts", "ph", "X", "name", "args"}; + + for (size_t i = 1; i < size - 1; ++i) { + for (auto& s : tags) { + ASSERT_TRUE(lines[i].find(s) != string::npos); } - count++; } } diff --git a/onnxruntime/test/python/onnxruntime_test_python.py b/onnxruntime/test/python/onnxruntime_test_python.py index fd711f3537..545cd6e561 100644 --- a/onnxruntime/test/python/onnxruntime_test_python.py +++ b/onnxruntime/test/python/onnxruntime_test_python.py @@ -535,11 +535,13 @@ class TestInferenceSession(unittest.TestCase): tags = ['pid', 'dur', 'ts', 'ph', 'X', 'name', 'args'] with open(profile_file) as f: lines = f.readlines() + lines_len = len(lines) + self.assertTrue(lines_len > 8) self.assertTrue('[' in lines[0]) - for i in range(1, 8): + for i in range(1, lines_len-1): for tag in tags: self.assertTrue(tag in lines[i]) - self.assertTrue(']' in lines[8]) + self.assertTrue(']' in lines[-1]) def testProfilerGetStartTimeNs(self): def getSingleSessionProfilingStartTime(): diff --git a/tools/ci_build/github/windows/setup_env_cuda.bat b/tools/ci_build/github/windows/setup_env_cuda.bat index 9e51d68256..b89dbb16db 100644 --- a/tools/ci_build/github/windows/setup_env_cuda.bat +++ b/tools/ci_build/github/windows/setup_env_cuda.bat @@ -1,2 +1,2 @@ -set PATH=C:\azcopy;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.2\bin;C:\local\cudnn-10.2-windows10-x64-v8.0.3.33\cuda\bin;%PATH% +set PATH=C:\azcopy;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.2\bin;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v10.2\extras\CUPTI\lib64;C:\local\cudnn-10.2-windows10-x64-v8.0.3.33\cuda\bin;%PATH% set GRADLE_OPTS=-Dorg.gradle.daemon=false diff --git a/tools/ci_build/github/windows/setup_env_cuda_11.bat b/tools/ci_build/github/windows/setup_env_cuda_11.bat index 1ab8e7ee73..a10f5d7b68 100644 --- a/tools/ci_build/github/windows/setup_env_cuda_11.bat +++ b/tools/ci_build/github/windows/setup_env_cuda_11.bat @@ -1,2 +1,2 @@ -set PATH=C:\azcopy;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.0\bin;C:\local\cudnn-11.0-windows-x64-v8.0.2.39\cuda\bin;%PATH% +set PATH=C:\azcopy;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.0\bin;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.0\extras\CUPTI\lib64;C:\local\cudnn-11.0-windows-x64-v8.0.2.39\cuda\bin;%PATH% set GRADLE_OPTS=-Dorg.gradle.daemon=false diff --git a/tools/ci_build/github/windows/setup_env_trt.bat b/tools/ci_build/github/windows/setup_env_trt.bat index f5ff7efb45..41524a3465 100644 --- a/tools/ci_build/github/windows/setup_env_trt.bat +++ b/tools/ci_build/github/windows/setup_env_trt.bat @@ -1,2 +1,2 @@ -set PATH=C:\azcopy;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.1\bin;C:\local\cudnn-11.1-windows-x64-v8.0.5.39\cuda\bin;%PATH% +set PATH=C:\azcopy;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.1\bin;C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.1\extras\CUPTI\lib64;C:\local\cudnn-11.1-windows-x64-v8.0.5.39\cuda\bin;%PATH% set GRADLE_OPTS=-Dorg.gradle.daemon=false From 77c19436c0bfb33518406f52d8854d589209205c Mon Sep 17 00:00:00 2001 From: Yufeng Li Date: Mon, 29 Mar 2021 13:24:14 -0700 Subject: [PATCH 021/129] add a notebook for mobilenetv2 quantization (#7164) * add a notebook for quant mobilenetv2 --- .../Bert-GLUE_OnnxRuntime_quantization.ipynb | 0 .../imagenet_v2/calibration_imagenet/cat.jpg | Bin 0 -> 5869 bytes .../calibration_imagenet/n01440764_3198.png | Bin 0 -> 214158 bytes .../imagenet_v2/calibration_imagenet/rose.jpg | Bin 0 -> 52997 bytes .../notebooks/imagenet_v2/cat.jpg | Bin 0 -> 5869 bytes .../notebooks/imagenet_v2/mobilenet.ipynb | 351 ++++++++++++++++++ 6 files changed, 351 insertions(+) rename onnxruntime/python/tools/quantization/notebooks/{ => bert}/Bert-GLUE_OnnxRuntime_quantization.ipynb (100%) create mode 100644 onnxruntime/python/tools/quantization/notebooks/imagenet_v2/calibration_imagenet/cat.jpg create mode 100644 onnxruntime/python/tools/quantization/notebooks/imagenet_v2/calibration_imagenet/n01440764_3198.png create mode 100644 onnxruntime/python/tools/quantization/notebooks/imagenet_v2/calibration_imagenet/rose.jpg create mode 100644 onnxruntime/python/tools/quantization/notebooks/imagenet_v2/cat.jpg create mode 100644 onnxruntime/python/tools/quantization/notebooks/imagenet_v2/mobilenet.ipynb diff --git a/onnxruntime/python/tools/quantization/notebooks/Bert-GLUE_OnnxRuntime_quantization.ipynb b/onnxruntime/python/tools/quantization/notebooks/bert/Bert-GLUE_OnnxRuntime_quantization.ipynb similarity index 100% rename from onnxruntime/python/tools/quantization/notebooks/Bert-GLUE_OnnxRuntime_quantization.ipynb rename to onnxruntime/python/tools/quantization/notebooks/bert/Bert-GLUE_OnnxRuntime_quantization.ipynb diff --git a/onnxruntime/python/tools/quantization/notebooks/imagenet_v2/calibration_imagenet/cat.jpg b/onnxruntime/python/tools/quantization/notebooks/imagenet_v2/calibration_imagenet/cat.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f1e32c44819e20ef93c496fc20635b29ada4e350 GIT binary patch literal 5869 zcmYjR2QVDm+FllmM7JcXt+G1Ndy60zS)EwqQ=<2>y681pB&+wjR!Q`ps3Cf+)l2wv zAw*BO`F;1^x${3~&b-e%&)d$tGw09zpKkzaEe*H^fPer1AhfiZ!>?4s;2J^^82VMaD_8F4{rULj!t0&;S4 zN(xE{H8n(tg$*Y3KRo&$`+r2>|MCCQKRo~%QUHcP5J@QAcDKhJC&AQ^s&A*2_1)5#2b1s)#Cczug3;p zPVpy(Hr|ozCk%JF4+;K*|JA)~M0l5aCvnr<)gt@{cdh@U6A;qSvIFTv9~0?Yb8vb^ z)YJc21l$MyyB!Tc8SsvSlu)Ft2^!B%8c9eRNgdBm%0sFO4S!pVh`?1KBBs@KDByAu zjBE(tSkJj$!RdDRQ})Vwi^fyiSY+LNfcJzu3=U*v=3-m+WvAT}Z1fr#6|S1rNyqrU z<%;0BFDhEX(a;3h8Gh9kbQ?P2h6wi;cAmP3e41YK2_Up z5OxmbLkJMXZXbF$FDXvZ8HB87htlv#cUF;WCn!1=;g-Rmr8y&3(dKiaZdIUYO zEH9bLKoN}Q&Is$vlzQnaVAwH6j;p1QvE)!Al@eloUfu|O!gzYx$UN`^zTU@NCBz!! zo_d<(f%=h&#vr~eqDo?-nbuXA;F_Gw`xXD%ad4}-S$<-YONp&LY|OPhQsS18oX85_ zT{GT3{elnr#3nCIhPPs5=%JZ?s@lGc?97ADQ!;B7PW{W|Tx)PKVjY;h9C#*B(2d#I z@r25zZ%Fw9V75c@9$zwbkiK~bu}$)u58F=WsSvzE;*jE^G`pVcJp+eKDwO*PP7A$6 zQoLZvoIogdL{)5z}78FRM1-Y{?RPjRt2;7V9d z;Zcv%@%QU1_DC#ORY>qsj(k6bLEgM=I|k`h%#9IJ_W&(kihJMzcOGigl zhFKKh;mXFwXrQkb&g~3&n@vGj{d9cY|5hhvk}c~n!!#LZDN&|3a@O@!RZFcHM!#7M z6Xi=#N%*%%2qWO5jebXpe#rMlMTqEA=?_^AO26u{VAz14b0{|yrfHDdg*Xtpnbdb=rU zT%TB1^?bUr^l3_ID(mAH{c9`Dlj8LYURhgP24Xl^MRl`hwh-_H7I^aspOJqPYTL3E zqOlsqestnE?@9}w0+1|zK1i>P|7xdaG1Sy4^^q>SBTic*`qh^;CmnqhC;P-m@Nhaa zuEKGhAKmZVDUh#{DM3AlV;@&So^EGp1`7YB-?2pUT8~d&k^}yesIZObH*w+0)!Edp z6kWw)P@rR`oyYPbn%C$!4v6mD;tZRrCLwmc;kNz$)_cL^k~)o6>LFDn# zQYOD?yMcEN_sC}fzRJR^y~rFY<2J6n$P|Zvk3jQ!+S*X&yt$C$WwO@O4cInxDHX96 z#vIE$C9dI`bKU9w1JKr=YjTd=%18$`bb2Xu)KX5&935E-td%@RSBYrda9YcUkSA(5l<8B2*C_|%Tv`rc^ zoyXPC;VW6gZmWDzT2aoxVa#zTjnrhx3O$`-8X0b0(2mo`UAzfh+`R+`g(~L?GMF{--i0*Vtb=?QxXaz+d>$UP6Oyo`6~M~!L;dMYw-JO?q=G{;=|z~ z#X}~>X|Gd)@6e%QJ+awp79m*IHyY)0fwFpQ>bV&Q5s2jZNS)0E@uBVF26T)Z6%zDa zH+^h!&s|t58I)L2@f>rCZfpao2Jxr1L-QvpBk+Fjj_YpIe3}orn!m}W1txG68c)ph zVYjc$6fbT!Nx$T77X>PDy0(3}m}oBBQ802mAp?zCA7UGp_;cC_cOHt(5jL+ zUAkaa)Y@Mt^7qGerwsin0T|u#lwPYIScNZ)NSgo?b=sL{ zPi~SPjoDv`6@rl!Y+u!H3rz5kasHkn@^c-L?xBL^N`&hIR0@VtCY9yIn#Y6&RPt|s zBp_?PSA=SaEYI#~xW!)nl(c`#uk4Q`D9@w$A;oyVzIkhF< zWH6nFVTe(k(vAmGU#Em&prq8yaSB>}SXn0?Q>dn~;M$Ufa3oel2Was0mmiOsoq_$Q z_=BDL_{Ac#%uC(G=!Js+D6u6 zOTJF3T<+agMHc<%iek<_8AgS}O?FrPfzFN;J6&`c**0RF4HZn&TxnbjmT*6;Sl}wK zm%I(hoGcuj-xg$r_|$KG2O(30hpC`S4xRcHd5oskVTht(3)`wF&OFvq__W$V$!2{b z6%h&bg9YMoh@l7d?CE;0j|h~-syGDl#0J0~dw+{!ru7FX+owaw`327ZaP5XJM*&Lh zzy70L06Z^g{a5#7Pg1)t=TFSMtu#Q}k~2V_#dzl;<2}T!An141%+-sUP4O0m=b*7Dxf& z=zudHE>(T(_A?pirBjAqEEu|rZ8O&y;A-Z?p_~G0WLW~hhjrKVXCsm{ksW}h`B59-vFXtEC{^_2keIV!uvSVfw9#UND6+hzb4*fTE_G9bF&QwLh4faMF*{_t# z;u5buIc>yS00-@vxKCBxWMyRypLrQ$2EZ}PM&Guagp#9%BtbA)=o1?`!6^)L73>)PMjyeQ?g}j+qYZh^vItXaCT4EFVPam3cLzanhoqwB?QM_zBOn5Ya)&t$JqC{4&+$ zYx^n-^Wn6{3Hw@z@z1>S=b?=j%`K=FMQNFQ#wQWxc|GVCF(l0|8n92wZRK$Rh?T?q zQ@#cnEMq{^uC$p@pgg8vgZwA*T*ub9t@c?P-=2-9=P!Wh2S`=w43CAd6X!`~pP_<^E@j zje;NL=S@Gn?9;63kQ`z!J$r{ucfPGYQ~)3q-HjGjGtd7z}O`WsT}fWa=9uIq7)n!i71`d;~q za$g$w#`vIP({|i7pqg~Xw=ag{RgZ8)pl{ZzKl~kb&D8J6vbyqovHF?{a zr>vOVWI}IhWB{;YNl-skHciJ-^5Ff5(w7OzXMX_l8qpHI(}<5+K|hvV@_yCMT1un= zwgU%}mDPcHd*KA`g;j++Ff=?#u6}?FqH760;E_#%X5UsQo|*k!p6zBGYvD_Si%oAh zkuS!8o;A<2YAr4Ac{)CqPH}1SDe}IZJsxWD87=#ZK{qCnsh%g%f0+&4@$1#z>)ikPV@PbV8g=%FKR&UI#U z793aJ&UMCn-TY`qVanz3{dndpy-x8Y3ECjSF!Dh`j>CR!|6lPpb?LHp=`nbP5%gNj zhgOz_gS9i3qqP--&fdD%l#(Bs$}zL1O!J=iHF5W<`+w^(Q6!Ii?(Gyg+-EM$7<62* zHtA@7e-bf9%P83Jc{Mj{i?gj|ASwqM!(!5wKe81p8)s~tsr&^68$<-}1?(+@iakM* z5tyjV(00ds$NoNgME_GlJKcPlFr$a~TAES zJc}<{_nl6OQD7H^P`WKr`32hNq0MIXcHXFmhBu?0M>SCw{NzDwM_2~=Uu??Q@ABC* zgt!^O(eF~#t+uRdHuYXE>9v)P+N5@VL-&`du=uu6>4wLUnDJF4-YRA5hXX?zobALP zoH%-^r^HC7c`U}xoExJq*ia)N?s(ZNFbZI>pHOQF?rNsTN zE2yJfN-U|hmEcRD$iJ76Z(NO>*#{a$9DkQ}`rx{uRLWwQk^cAD2A`jOvAxwak`zE7 zAMXEmuVHmebE%%jW|jgG+sN5Tm`U~T4!8RN2QN;Wr_ltHlU5~dw$4DUPUJWLQ6XT3 z(@79d>&FSU%AA&Pr!*^yclh_8EB9vtC;k9B+LOMac@;7Ue>vzOxfMx9$s5C`PJRn_ zpp*B4CaNgY_U00YyDC_WS9FG8<#A?s>~6*MhGO@ecC?>r2l=8&u{_UJ1eJo za{U3NH)Xh|FuK*`wa~E3641V_*QI%iu#lm%bDX`MFB#x83alh7Sfd6rrB@U$Qa=Wq zvrVsmwOI;wcIG&8GM0Vr{i$#|RAZ=wq_O+IB`&ZvU`*3lVD=oAmh|?N*;qhf4s3$ z87gIICm-ZsaG}`6b7u0p!;2tU5lWPHnCwvRr4XGXv+`7~-hM1CD zomZGLF;Ayp>ueIrP9HlCDZeDXDF4k zrF!njh*2Wvgv!N2Rt9*jvRGa>yaU}VsI-!t8 zojWQEV4A*fwE+~{ho>#E@<)^AGt^8OvX0F|`m%2Qt&PdTW1~3;Yv@&Z3TvEh$M@r4 zmmmr{<9tm|*y=Ka+uF-`e1UM(lM5qC+U?zMC;J9VaL)G~hw9u(O`qL6i6eX#AxFP1 zzkQ`HY3UFDMRv(|c$^qBKyEKX%dW2LkvPlBqflsxUwSpA@T1sx-%4`g1K=A>#q%lLk$SZO%dn>V;NerwNkZIxW_azF;W@5#awB`!E!!1k_f)0crvc0 z=ta-uF+@72QfD;g^b8I|>uSEUTmK|dD3NYZBirRj3!$IA^y#C8i}D@J>NQ6!MW>sM zZdqZG@iV-9dX6H5aGpViWGLa-`|O;u^ga8(>B=%LBbLxsvW?KP^ntyy?)B}MaX8G4 I?a#vh0o3x;v;Y7A literal 0 HcmV?d00001 diff --git a/onnxruntime/python/tools/quantization/notebooks/imagenet_v2/calibration_imagenet/n01440764_3198.png b/onnxruntime/python/tools/quantization/notebooks/imagenet_v2/calibration_imagenet/n01440764_3198.png new file mode 100644 index 0000000000000000000000000000000000000000..15d84dd3038629ece3b915ce580a2e1606749634 GIT binary patch literal 214158 zcmV)vK$X9VP)6NW=yQi3kYNL5dX7BgG>M96^vGMvqc7 zAo>LXNeDGSfDlMRCSfv3rq7;T?|rvVeV41R^TT+)@5l8othL_j^?I%s`|Q`=i%fF~ z65>%mWCzF<x$1+J!Erk@u9L-me zaRtcK$PWXelzIr!^lMibk$KMKjZw~03>+W}joHN@IHXqUDrKM?bIw8tAp{`fTDi>- zV>G&0x82~W0I3RI%1j(=9|I#M!lym@>TB0;-L*f>$(=nrzJ7S4t}ARxM+fl zL%#~9Yj!mRvX)~u-Z?UXl}=){JoJ!hJgEc_AnoobLf8ASxKWIhDNRu#i_t437ItVi zZq!VmcjGNC1V3C*@0^ipa2*MQAi2t!Q5mA<=pk2S$pNn3I=phvy_?M@=5%&T+s(kR zh`@YUF(yMqm(uoD!D>C4Aqs+-aT;7W#>;HeBhItt*sk} ztE1y=^~y!&16z_ehq?P0lo-KV~G zI^Jg>UB4b&I9;_z$H%89M*@;8%1al|Tz~e4(V8(@t(HZdFWvL#wP&wQcXk(D%ff>3 z&E5CE@#OZ>Fo(_Fru9DFdflgd#jpR5x8)q8%cBaf+;{)A>sODD?_9jN)A#-M_BJ(p zU-H#I@UuVqFITIh;Fm_2veK9Ce(2*L{mWs~QFdSTP2c((|Lb>;?_7P&D<6CQ#phO= z;gcV{_xkl09Sz zfQ%5)S_>rFwX;1-5izk8BL-O6;EDGx6I1iv_AWu>IRFLLXvJdi&H@UTfKk?w+Xcw2oQNH6x6sDwWPc zB^L)FR9Tj$kX#TKnQ^~tHz6lOC`^R}Ix{3Fl+Gc>7&oi6>v{-*p&TM3j?-~jjAdm+ zRaL;zCOa9IJG05OiIrMsI$dnm>&1MlQjp;J8^^Mmb$!b)&vvF-RJrGMF}b*ZPbr&9 zH+G%0G-ga&=S{O+fn+Hok9@TX1Y$C(r$sd`YsL794;I*{FlTpnnx<;owol1gW>>Dz zY%dz{8+vcUdbJw*C4*FI04ceYki&F3RYcocy%rLK-tlDc<+c#iQdp+zq?cn4{U-J0lfe+VDkR@(-3v`widtT_67F zU+J=NSoGeh!hHPWA7_kJRlQtTXDLPgUElNXuiaSxlkfW>Ol&%ytlQOcX>Y!GgSmlI zQnI}H(v9W(NNe*oU;DM&lZ&QkK6Y@;5nnBPl&WebaduZXtOatB9fRmXL?#%)?$%T) zt|=fP2LMhH1&mDzz=cqigw=WhDNZN#q$yD{gKSn7TTPuq)Y?dn4l`3crJPdEl!S=e zwVsVlQJ@rpaiO?EPTVkoET>?U1QtUG!{CadN>MOIMb)sv5K&~Aa~#!`1R%~3F{c1I zDJhXK6Z0wco$o^Af-;LCCzi7E$Zh5>CWe^U&|776p$$vK7!yJiMIoi6m@=~9EQ1`m zxqY-+YzC$DnkZFumqSdHA+?=nlvN?5E;*xMT@IwU5C}%0RtlAjG4jL(N^K-$#+fK| z$+@gdMK&SRDJ3o0v>6phTN7;*&p89&Y*f`eRw*66ba2nv3;n83DG^rf+?}jk=GD;I zsxYSFJ|_)^dNFMBw|~pGMIYMbB3hOMfQ(&O?d*(algd(Z*|%;;DN};d=*SXkeg@04 z+B6wUEQ7ea2@z z{MhTqqn*7o`&<^fXmn8|pBQ9-RA_nS^2Po07nL$fnh>o(Y*)+O?P<#L{U7|uv(LX| z6y9}lk1M%<_AUne=FRJ$@ut`P*pGc*+rIddKk=hJbSaTR==$Zsog<&};~#mV8yqs3 z65Y9TJEw4TzD>188Q*zUAl+mTw2bMgG$tTtkkuiob z#yJN7%CKH9Q%W(pqvdk#olBWPgybkXl~Ycx;G`7K6p+UuP(-zHIiw6imPRr~F%31B zk<%jkl@!EKGSpi8-b!M%)H!F)xpOZ1fWQ)ErbNhTF?eliU|cEFFf9U0waVF)^5<+yxf3-kI~1W?OPj1MNtSaOiYcEIdC8% zQbre?mn`44z4OM`zVhsjIdirtRdH}QEL(B3avWu? z3k>B*a*U-Zl+j$MjMBE7WbLG`cBd1~WutXfjzT6`%1iJk77=seX>id8lBl^PA`f}7Smuy@ z=QxwAdH=pYeSfX8(R_RVLN(ia{inR?>|OVoqDIcw?IycW2vsRLs;ixyoeLK)Q-;3p zN25_8>QPnCk8it>UiHW)88fYlVlu7ndGz7^vzLWXqf!0Q4?Ta+<@@Y<`QZKgU;5U+ zMJ_;hdTwvDwYw8ydgRf^j!)+Rxowx);P?HLuPopsu5WKm3L(UY9{>2)f7^GCrZZ3O z!B;;#KC`Dsqe~ah{q?{0H>6gp=bi?;afQBfx&q?=_3gj#{>PuJ^LjmaJ=zL{ZOA@` zzVD4GheY!(9M9Lt6A>ksF+14>hcYJc%fV9ioFQc%+`z1iZXG429xB7om?T&ss&*Zm z^FY#&)J0?CfH@ZorIZ{au>_vUPzVGFi6RA`q|`ZQpD5;-tQb7Q#50ge*qBp zbOI4YC3Vx(!VpWTM8g=F95FEkTuI{$&rxDxI@74&OeG``m?i}nZDl6%8A;5V5|DES zk|kuhVl0!-3~b5@Sx&HFC^+l=;K6S)E6a)Im;#)?u+t9n&e@o68iVk81k)@^#n~Y7V&QA?857|*9=X=RX0h}L? zN|J?8ZE7R)B8Ey9ad0lUst|%f!b}-C8r%86g@WuhtD*DTJ59_s0db>cGcJpn4?|fR zDYduV_Gsc$D;aT@KluI+)P|H&o5frRA&Ipc+xG2{hD0>@jx(Sf6kt)68;Fi&VTy7# zjXcb@r`tPY9mCysU8x$zL>Nu?2*k4=|43!PwZlnoB{zYK$3OI^(Yw$4g1`OD^M_(I z`uJZx_0D(x#>Q`4Xus{d{@FthK5}7ys~~vr?3JdfX3h3Qmj_S()nC5v_eUjLo*o{} z7bjhO`+xa049BidQSLtY+Ry!)Z~c}m_J8N`r!HK$C*uUP{=zT*x>-}ON|ds=up3ax zktx@kVa-UH_x8KEjRz8k0qfJT{-&>GMflYxd1B0!8OWNrz4!m21KWuvv=&~;s; zED$9ZN=S|>MF8Stp+Ko9hDbhpC1WLGA(17fm>ic<$&hk}oP+m;jEW`2l0e1)kTFw; zNHHTwofu}u*vVpvh>|lQwBQ9IM#>>ZpJK}rNX7^f13^qjNG^xyGYjK_0rP4sBts^6 zGD2&eQu5x%%vZ~e5Gp22nLAI}XJ9Fl6T~zl0okSDbZuiSoC`!L<*-_JL+@%O51x7A z&T5mXSg+c)?+A%9W<%v}?V;z2rIZE*a}$q{`8d zPllc^PAvy$3T+C-Q?%Z#b8b6o*P&axN*ATk(ulR~m%ZI0xDFy>l)z$T{bn8DpLTFErQ1dfoakD5W^(F-EPmK=gq=`E{=c z(c3UYB35sqX_kVBbWAEd-A2NjyTv?W;DTU5i!Uz81 zy^x0MFFq9=rL;uiw{9P_!y*P-6@^i{+q7=zDKa2(tAnb*mu|kKdAT@UOs18f<>gCz zho`4U>*23H^!Q)C@4X*>-v`G6d=#wBDX=oxOJR;PRz=wstQ4#81BM zi6@?@jjE?rC|I9@8qJP2>CCw+Pk!_xZP&f^%f9pn-u8eMtPf=^FQB<`S0y*c_T13!V<(P`HNlZ}Y zj4`4hlu3}35-GA25daidoC%2%05hh9pt)d~7-Nv4w~jG(v`8O*;`VYuS}M+Y2Ff|d z7>lBi0_##UKM3ZH%v+Pf2&l_S7q!&t{FyVl6vSlfL&z9H5<(zml!eyCxDd;yA73o5 zO_Sf*Ia7@%mZD{SImIq-BKKS+pO=)JW=Jti>I!18q32uzDXj`XnS+>(wg)$?hK^~) zwN43|LP-IXTp_X0CNtFz4j3YX96hC=46AA*wH#7DI&E_nEVE28##q;*lv2BK!RH${ zZsiPRRYae(LM?^$PHL4X17Oa;xulfc;6q5W=}ujj!!UU7i3k8R=S4X_JUk}GSF5mT z{lzQiF5Pvd)U^^a`rrn8dT`+T4nlbRFFp{4zUvqJ`)91RCnxiioZ-1`*OjjR;CFxj zrKg@=p5A)*Z~y+spLpV#k3appzxlt;Olkx+KRw-s_B+4i(we*YRm;oiPl{^o}E)gl_~lRn?bj>IGqD;%JuTJkP64bD8@qw zLI~?a%rx|&;r!y(R$ZuSR2=tw9CK~8eP@fRoG(wyZJud>5ZZo|SntB= zj?q)5L{&3Ll0#XV7*pF0-g_a0kt5AV)kqQM^%??M!brjA=&c=)Lkx~5Mo5HIR*Dfw z$%SHC0tZ)>P>i-$eH=VFzt%>1mmCo>bGja;lNk{KWQh#21G0><48R!kE-=8H2tul~ z;SsI%*7e9407xlQN|2L}`S|$s(MLbUc~O-0YI8c9Zo6S{>vp>t_lp$_7DFav6QkS3 z@#65-@yTJ%nKDQs!)aWVCx^>8B!lYt=RbIM>z>2gx7yR?-nfbD&7Io^P1D>tILOhD zs?oDgKRtMU>-I4XLs{{&`+Ij?yw@haUiGtTlwHeGUbWWdH3(UZW-q+>T-TmfRT^yA z-P?cRg=^%TO7ZgEtRGt6cP~8oB!w&)t4iTLYwAKO6-oJP9~)za@u=B(?0xTj-+lMp z_pV?6m2UnC%1eFb9w~|+|Cj$F8#_IF?{vE5EGtZd#dRx#da4QFhAc!&YFZ~;fA? z7)&KP%LFf$E{7I7Ps)hagNr<+)TI<-w(E7rI-yNjmMM6jfl@A5uDSBg5ywQxVg?S1 zhy(xy4!mR~;}FSJrLL7Ch(4!2bTUQ7<7T}a)=S79dGetHRp_Ds$dHt>JkY>_)w04& z5IG~v6p)9MLXZJ;%t(pl9LXTHd2o~l0!v93GLz$x&5EL5=Qeo7B(g9nVY@DJXUmZ~ zv(=~xYAM>qrWWj;^W#|s#1eQv3>~wq%zng^U>23k8SRiprQT`e>U4g49>$Ja7#3mm z(fQS7LmskZU{eZ-yA%&k0|8Vc5$$~0ZR%n&4q{q1tJSiZ)S1PNUmu;Ea4u_sv!)^5 z4G~yY$zvpO)US=85|a^F8tr`495{i<3I&bF4G7l9WeBO#h64KD1v{W+1ID=vr|UuS zk`m+6o2qtRbTM)%1YxC1&eY(@5hIR~thdf>h$jx25mBJ%y=PpMrHavkfsI{{w|0aU z&%OBkN1psZ+b!Swr++LVL$ZYwn||}LkG_y|#7zfG0t-CFkm2UR>3TIZJKG1xZL!s41VC}lr+ooxB zX}B+kp*@)&8CiYKn?H9Ic7Nc>i? z3?U?sv#}giU-=c^{QH0OCrr@We(0Zm!J9v)jpn!w^E*#pdg$TnNABJW_ka55|ATv9 z^%)l~?~pFLuDkpE?%RI+hen0GdFSB%`|qD0%^M?2#YVgmq85chOe{Jkh;Igtgk!~2 zjaQG4u5ZF(VZ24fBm+B{7(&~-BU!@9$hlMgTVWkZev66ZjAle|23e5wne`zbFZ<_iop!7_>{idUi$8h(sh2iOn;heb3wE;HESEMo(Y0A%^ThKv z@3fADDb@J2U8GtP#8OF_1!1uL+I4LXJ`)c)0plsrGW#AO5+Lvhy!XD+hI1}T<8ths z4c>=7W|uHgS%{((F63Hkjx2IYL^;x`9h{3HI%_*Z#&SYPrB*p6LNJvXqGgN_ND5Sf zW2UtY>zJJjDUm?woXdc# zY*scVN>o=xpcMS(o(o&&E;P=ava+~!WUJ9ma+bv`m}UX};O7S?M>p<}OXPxY+tcIq zYPB&154l*ke(2V9BQIPyw_dqg)|>V6!Fw)f0fOR4%Qw94lmE-l{^Ga%!+-Yn|MX*< z&0#&MZXDcx%in%W58dwU?64Vzc5~sbtr&gub~H23zWB_@@ZDKO*_*BH*L=no-gVEt zT;$YU|AXKAmFT>+mYLBPeEB!N`SZWme>&Tq)}=0W-w&=&n1HoybSZ1Co4RO9QANLPs+!3t9I;`0irb@6 z^o}Sq#!lP0kw#{5fn$oncFwxqIp<8FR_iVX@V&Fvwwpf0%uyUKH-x$miCEu^L|voS;-u{lR+|+c zPT3%4RTL0%>VmU_cRtFhNI8_HS+AF(te?AjaB}K^E1%g5w~i7)DNE>8p{k-Tc)*o54 zH?4D$3)}ZU^R{cfw-J^$oiJ^9h6o_zeh*{%1-yWQboMeJO&y{((L_MiHuzy1%u>6`!J z-M^|f*MH}ozwjBK@mU)y&Rx74b#v+N2X5ZF0Xd&;Hs|hspe!mPIUY@O^tf_;K<+3B za*1qYF;^UP26TW@^Du<%ftlXRS5ooD)z+k~{zmlsp4tMD$5O zkx~KxrIfP?KuBRIC88{(Y%L5amQ~|Y^29Pq!I_WF`WPY)mg;&uPz1?OSBpNd)6Tjq z2hTNYL@>?W+x$KLXNtxTO1Hzb60;-udOr=ChXzO!KBm@c|4ZxKoB9>`L6p=?p6l4k!Sn49{ zEK!usIjvNPR96jAQkr+Ji_Qi=oSq!9oDd+066TNxMwP)Z#g&KK)&?Zn+Y zrq#x#G>i&V3RgoH7@e#)TW8Pp(K|}2Dh`g0_Ag#gm5FIMJw0|dt-JMjI_bNVGpp;7 z^S!Ri#o$lpJ}}P7DBOO1n+1kT`c7GbkUL6(S-q&KL)lQfw4&V>jezRLIKb z#O}Xqzh4JY#d)$ZV_!ERjhWGbdIGhAxH> z0&*?{Ps7INK$>%A*PCUsezGM)9yYcGPN+H8=I~^Byj)|>StTtQ67Z0jkS>5j@{~sP zm~mx5rJT*U@DVUD!BrZD@vH(AQlgTA7>hY)hMn`6D7k2@eg4JgLY4!Az|_&kuDhrz zg_0pNn8{n##u${IuQo#hQI==U?r%-2!xK>|CZ(<_e)+=XljSP7;pV}MZh*mVCR>Kb zP#1h~c2qP)GYg(}X4T2*jUhRf#mRa(_>OXWe*Y{AiYTh05)2o|?a#dZ=jQ9p?(Ug^ z7%NItkQ2MLyrV_RM4n_!a@B@azZs-y+apfdWO(ciul>1y`(2tZe*4#dc{R9S`=z&k z{nvf7@IZ^@kNu}#3n6XR$HiDV$Io54@8aGYe)cE7C%fn0__<&59pCuXslWN7Klj@o z`1rLKZiV&XotsDYu}5FAb9SHUDaL3ea-QX6GOB8=B#{dkGWh5R2bppVc8J-t+8D-l zBBhvKo%T|cuYK*Ob=|aX0zdF5C|D1mLo>sq8$t?At&cXQlX+K`(^@w&h*49gl=>m5Ob7`TS(kb{px|7a*?3lvV3^w0 zLe_l*AF>b%00~l2XvHOC@QOztWQ+vK?eVr2JV)Pc7RJc3X5$I6Vc-$F`2a1ZmT508 zZk=r;N&%2l@U9!IwO(;7iYjNYL$Vp$KHNGu>7CVu6o%cnek;c`D%FL(a#}L7>vQMM zO05D>&N+|>KAWmE#>5y4RYV^cCFcffdoEaFQgUJ<#em3k--Vbwuv}G@Qj&2I0iE-! z)h4C**rRX!#794Iac72IV9uR8x33JP>~i)>gDLQ{KI^lKQu@#n#LA2`BOd1C5-KH~ zg~4Kq?Bw|1%~twR)BJ3Ch{U8xK{``k;`#`Vc^=#y3=TDx2f5SRoMJXDor6z;q4o~EAc z@9+D5^YITq!5G^+bMNkb_k70>{paV7zGs;~Ht z-Mh~Hy>I;1&AQ*R`r`JL@A&G!xjK02VAX&7H+{o*{^M_5yzoTBy4zR(^6>U)X(}O5 z#0V^hvgX}-XcTyF0a;2}2*IObL|XFRCawwr8GIX|K%P01UDti!L+@9GNIbJhrqWfV ziI_%~87oVZD2EWI)2ZgbDGRQUvDsLkE?PvCQV{`T+y!4Sri92j58g_Yv&ondB}yET zax9GS#E2uM48WpIbtxc+6qp`W#Bdk_Y7t{Xf|Qxm#<@WXrMb)@3n60kNihJB22qqE z24YCW7|W$L(Fa*7!5L$W1B>LjU@0>oD2qC*R*W$z#~9fZSS<(Rk}L*4pyrr5C9vylj7$ro6lMuPWQ=ixHB!YeG@~g)7`z)* zr3p4eB9OiHIg>UrQ}W)2ZXi;9=Zl6RVo^%l5Bqz&&bH&}GTAS-q@1NV* z-Q8AKuX8Sk!Adi-VW>pb96AnbYcYUNd24be_EsplTrNekb7uRp>xQd`*FX8wUVrxN z{xerUnpO^wsp-zS3zv_NkFVW4c;LQ!j}8y!r|YuZPN2uTd)eCF4sLL&nVcNmXnVUe zDjbZgj}iiAkpn2nV{~OzM4QgO`m6>`R(8Q<@w3M4}Q~^);s6VKlsSe!J!1niphv?{K>UH z`qlrsJf5>?AA0zalY^7>@@TnPVhTb?PGPWJw4FDq;#!fwA$Y9{qm)*XOO-Q9E_O!M z?ZeyEPF-rH3e^rPZ6KkK(XLk=V@${-fEA@4hJglXv~CO+Tq$XYfI7ci9N$?lLI@!Q zrBnzB03<+gktm^nfH^=y!31*3vCxtz6A=&w-?Pk-lz_ywV2w;ErIdsa34JB`RPq>e z2!S!?v*(;EDIo@q5ON@*NH3$tf`|kpp%nlG?;$1uOdGHJ``zJf%d|S|LPZYFQO7 zr<{0M8DN2>fYIiD!?`TWvM36Qq$N*0j3>p}v(45lPe$YYvuDqq*(yy1QA8^o;{NW= zNHasoTojT3q6^-83yi9&dKn%lltd9s0R*0+!{{fanfGna{Crqzq06EeRpzdddc}VA zN%vIy+x7XK=FC>Z7+d#!PK*G&bxmD!;96@21du~;&JBLE+$wa1au_zI5KY#HELcuIcmc|l#HoY%gzlsC;8G#FQq(Ox_Hm1-k~hB z0OIWUc#d3#oIKBjaW2@ck>7yqi0>P?{LyA-LV$-DqWM(?fS(xp^?ZaqD!x z8S2sa=;)|vs_uS52hAWQ0svC{~CP zwc;r{ic*zGaWhd3F{q8%sq&d7&qmzY)Q3JL9R_5g$*fG6>r!Zr0U6gS0WktdkWvmV zN>wmkbUty!QbNL3mQra_BDg5pzzv~q*N`d1mIZ2i+ofC>$-I@4Yf`~E#=(TQEkYJT8t*V5= zz+f|Hyw;;K;2R#kcf?DOY=6Rb%QzE)QDEGbgArOp8Jt!$52_HHBfO;?0zA&nkwH zHz}le`_@Uf8hFfAHQg+FjY2E6NwE)YVYDen!C?vnIWbZxljqjP^#Go8wOTAi6!X=f zdCdmf6xn%_3T;n?RMLpfrVM9iIhtprc3-9W%*v=cl|ntH3)t_!(1K4QG*5;&`G zT|Z=msSnHb;UllQOUd!kd;vamlwY`Um016Bvnv&j3cPgr%DJ=qRRdb_gB#Z)Fo8@L z=H~4iee{#bq;8tY_BP729I9293vIMv2qkeP3-yU-U%39%i%);}FWq7(wd!MB93TAM zzyIZ(TNbk5QapOs?2ML$SODa!FFidr`AlI(QnGHhClvhJt*$cGk8}z&6XFYrOztJKKB5E8~L|NR`66s<;AG!>B1rQC6lA zGG`!Eh{F0{+f5&1wu4WJfOw3qD50D*yfhEL`jflm@w6~!M!Q<83zzTjt!RpL{!FEr)(dMmtt}MOl+iv&lRym$pYyx4|uY??Sw`RoE zeGk3D_e1P`$#82rIX*s)nI>D!*4Eajp3E1k!;|CnYU!=h8gdFs*zN5t4+J#y-NtqP z#iOH>^|EfXKsjzE!Npg<_6*xEEXKCnXz=&3b+0 zLuw|?t6ufcJAU<@!~F0YzwE7*d$yL{T^FVg-}l&W{Lf$8J-1WUYH#dMPz2*LHq%~wkX$rP-fj8Pb*T#vV6hOW0ZL`cxCT8PL5Yo@wr zn1|M9n|QFcU$ypVvL0gyP0L@8#b zlv2_N{xaRiiAWb^1{yqdy+N-4EuLMy>p zL}1YeZcvuZWHL(8bAS|rK;(=OkwA_F#+aP5QYuCd#Fs7~p3XUk?b*(BDiMd2S0QK4 zWe7ms&6{(yfVT?jx&bcgWp*e_bLvIQ>8rLZ$!nj;7UiaEh zJ3c-=yLU!_v@w48$&Wwt(hZ{Y55DqCs&ah)D;`5LE9!!Co{{y|6{Z>bFyE|BmWx&2 zGR78*#d^IyySL97Ub=98JgxxHxuCfymFAqE-PsZX*K4<4tX=P>qw&^sYP7V@jhe0L z)@E4u6Y{~i1T00b1W1)jQ%OM_bKlv`x>aSh zduC5+-48=m6~>qlLP+bOU%&CSuh~1ZMFgu=XS*e^j9exPy=`Omrce(&cz@rwUGE3$ zD0|AH@7veC{x#>X+%?%fd)NIBW#TE)Lk~Uh@S~5s^07zTVHm8fnnnujhPLzhcYo)1 zmYel@(alefyzNp-*2m?l6GB{j;W-V+G}rDR+XLhd{ZnXlH}ouk9A|KczFga7`M z>h!5AQxbeo#TY;wFAq4IzxcxYm1adzG);wH@nv5+8jb$wkN(t0KAG&k{*9l0=OugN zMtAeLXH-jGmP}#7%<7R;t#Tr~JPM7a}OpG-DYdQv{X+1(t^xV`Pdk27o#JZ}M;$ zhG7*>7MtUZO(ZDG5E#C^+6nRUYF7}Z!~ld4hI5`$Ld2MS@DwbwLoWrg%p^mbd{APs zSV+MQmz*(ym}4Arvfe9{`2T%UD~4hdND$}1(%_YtNLJaNla$DWVM-|>IB3Djf^!su zi_5_-hgB4VWeyYVO=g?5RnicpNIjv;%rZli4cBfdmPRsXRRDbx{!j7lLzc`!-WF zQiVLMQ#!TLyMQPd@QMS6P&R_Glwn{P4XQ%x<$SeigA2i>0W|f=dce`Blu};5_Tu*T zwo#LuIJ&iDIWT10SB>B@Dy5E(j}bAla^3cm+9=VOLNz0`cj4US`|f?s8$MM8mV8{T zR`d1d_U)UmfAkgGd)u?wc4?+r3zxE8b$#@MkLNF*n@z`$J@P0+q4o3=&pcgLLMa_% z-rd;}LI6t_E}So>=ejO+ixo1dl(KFk!@z;O&jH1d(sCGX9~~$bbR2%{Km39j-z}!E ze*Wg|F)VwOy@O}39v+;ueQ!72+IQ=idQY)DbI$ScIyBEdb8xawM%IleMx*lKhhKT+ z$`v$K{npR_8&dL}{q3eOfAmN1i4vxh^AB8p{o#$_;Q7VLv0FL_y%?7HaB+HbwO}Qi zjBAj2T-MC+h}g}8_j&L`7Op^4Dhi!R_j_lc;hbk7N~6((W{wSt83Jw;DRk3WJ#Ho( z3XB4=!d(Avh&X@L5aU2<&nI zu9#q4BqK$kB|;;vhlyBHad4#xfI{oy=HZdihd+vVnsi%r*eX{Q7ZI>uZj*~_SS&(C7mti=T?zBinHLj5< zpR%=nXggY@ZBwK!j7n&#P?ag>&GxJsk1CB3gjn_i>eBnj5dzO$2C12M5UrD<%xvVi z-uQl8b31g5GRo+}l2UT(m>JqgU@FE)X--*`SWK#^6vb>Rx29Ti)Cz0Pnj$N%h>Dhs zDrYfKv`!l=3e#xqoUPQPNmMFvI$!MX?L7MW*I&PNZCEcuPC!wkQIZvkuiANX{d8LO zH1rYHyQt{-Af!@!n@^KN4wdGNlwAH4se*L-SnLNX++7?mWG zZ(9~pu(oPaG zQZ5YtH*b4;96hGSyZ`71{{8R&!JmKI_kRCB{=pwTbLO!-ZL&Rv#J0D#a_}|)OO<7k zDDK?z(CM;U9~>Vq4}?j(<58`vv-?{Y?|zVRI(OFv{KMaW@8|rr&l8&0)#%=P?|t~; zSB$3<>+_Qz``F;(`HOeo|KOv?$6d-s>82TNcdchxO4;w9x$pd47ldHru`EY=ee6Uo z`(+$*?&1)66haX4(N>e07cp*E1w$_+c;9P=NGvDr2JgM+k~1wqiHKkm4WNDEA%vIh zzLaH3gh)Pylz|I@zyxw7Bta<4vMfufAZ44fwKO;ss0xOVQUb!vFar|fNHPV-L}F4X zA(;)RB@x*aY=|~_ZbVTOLI?mbQU&J`SR!VF6n zW)y6-Tzl_BNOlMzq>OSkY_=}V=6)WB#2A}QCW4t@K`Fx`DkdALKms#-Zs!2FT7=hq z(&sMLt0A;3ZI<1l8*DwADZ!mfLYYl}!^m|3Lurs`U~W@OnAGL&OwC4otFGz_UB3|w z&RyJ9hHbjJuD83@YQ%{ZY_>h&86%_9^?K2+%bGJp7ksxmniguJ7<>vrljF{&X(=SS)p}ij@;R7#~t-utd^ zwbo@>a?X!$-?(_`yc$(oXD@P5{Kb3!h*FH+MenBDys8RMG@4GHeg4I7`Ic|GxV`uA zgRi=Myco}B+L*!hvz_|N-S-y7_`>ckIiH5X_1!BUc{SFB4gUOH_wJtAA8WC_y>s#6 z1DN7xzxnfx)hQM?ut?$-?`fM#~tf(?%(j( z!x#5=UiYbQJa^B7LMUU5HVQADJOA^)_={imMKAFHA{|DRrfkQi!N^6_fQTFIo$ca?YLYfKm*Oa?CmU%vdG>$T_E!Vur5w zK7cYMAu=Ht5*JJxX$m!#Bs%=PnThqHPT|h+dhDRQKWm&0px2ozfmuj_w!^0t^ z>c+vH*`%J0&2*%59)j~o{KEc)p|e`%$PzjrN_ zOffFoA%%n~Db67~rK#KW&PK}2#gtM`8KqW2W5~ncJ49HoTggD1`rvfP(YnFTTYEHL zjVCjP_($*lovq1ah?!@mkQXB(kSkdzSt%_TWY;fQ4g^VS#TXEgwO&e%2ttULNqW;5 zF4<%t(3s+-E%hN0C~0U-~J0f_loY>!s%bTj5^Je>$t z1IHI%e4#W-NnTFs2zbwZ_ulu=D?HN=Jn(Re`235{|MTzuc3R#3gFpEFAODf>|H}`* z|LWDN&tE%~)Xr)(ZR+v3sfw~Li=Be+)#mcv_N9wg&h5^u+n7>sjYnMK{SQ9`oG0RC zQCn-B8^rJa&b#J*eSYuKyWa8c|KETA`5*a_fBE{?zy9pmb6@mDf9;j8df@xM_xqoH z_JvK`FBYeF4sNbD_RId>7q3oFKJdOLR-ye{f9s1MedNlpx}7pBOwLwNM3xC;%Gtl1 zP{I@|P)3_xnja}BAxQz8E4j9=o=i9+Q-IBxziiKQ4vMY%jO2lHAp}7nmP5?IP2X*d zDL6_bQc4LS5+;etIv)cVQ^pt*5#=md8Uhj%LMT~?5K>A>@Ssq4J}Dwu=tzn6uHZa{ zRB#E%d>Cx9l5^|4l)4`VvVK5TmL;%^BmvU6F^cCdNam=I9K~eVTQxzP*6A_Zki8B^E9|D8LdLdy_s|mz(aQo>NV_}dh79XF*R*L6Q3&Cj ztH(71P8pB`fy@bZ>#=1al4l5naG6bP80VxzCG{dGnYzHW&ndO4v z&RzE&AJ31EPctNmX|r5Q!9_|wyVf4p30fONwHqV2z8i0=(YU^IGS?;-y6*eFFf-2i zda;~Lcm3duF%P}w6)Mp67p~oN&v{DNIs&&od-f~fQ6}Ro9aXIY9OxnVD3})ut&+D61~!`RO8ndDW{QrtD%BPEJob z%kO>XPmd>;zUDjrHTvbG#DbZ2gM|U++8_oqn_PVORqNX^hu%2{SqtLL_I6TbcDX53 zN?8cC=~l=T7{-_LJI}rF^!=}V$jdZ#w<^|#~yjj#dGK1{JEc1G-iA6 z{JmH1diT43@4bKV=Zkjzmd||i>tFlITeoiAI&_&~*NfnRGZaV)SZ<4>Th6wLC5$l! z+$g9Kg#yoIOq8|ccIjeHCU^mvV-`Hcz_j38#-w=0(7RsBtxca8SK7#NQwhd6BhEO_ zjwxRlk;0HHq|AdG$Yx+kp$?=%!kAMAz?d<5)CmzYPc_SlQ}!{3ghsbKM~Wa21p#T6 z8ATs+LPR4xljp^uz$a8_#`!8JuG5@iO~+EV1*5f7P_5QJ?}a~t|;ZK{S%#zjcL zWz3G%qg0eq5@%3y2w71i;kq_ED(czxncc$hNRIRL>%aF~cTetq^wBrYmu>Es^;W$K zo|{q3sTtRZVzU-Uck)5k{0$X_*LgvHf1s5h81-=-tOChGh~@MuiHUQXS*E1C$~E3O0L>rqeSTk zdhx}N*|Hxz)JinCXG}Se}|KoR#mW$SX z`#=4zzxjn<@PGf|)N#(MXgzxf+Pq?D3e{Mn!V`3L{<@q^{&i@)qEcDByn zS)SfFI_XkkQL$uvpJZ;RRcRx0&*O%ZrJcecc&wb_h68cTx}l98%yq1!y

=fQ(%DV)g5XoN59zds1u7tfy6pkal*n2IxJ z*D+D++q>7EdYVg)ur^IvHhDOR4V8TQqfe#tF@!o>C7v6kc~O+CD2i6A6@_6KhOJgC zC1(YLm=sxRBVJ^W;?NEKR^WDyh9jrv^m?5|p04fe@TQoi$^ZDz|Jmtux=N;@;kJBt z`0&<`{rT^C?wJdNet3Aek5vX(X<}kSfRtFR7M>s5u7CT^qqUZ|(Q8pm5H&Wpx43M$ zJrd%PYK{ZKq)fj?Dj<-xwDYTM+NT|6o zq7V=YgoIFA5Xvf(MWZPup3R{yxUPj+2~cyWpv?fvid<{0g+z`4mjV)z_DFDlc<=ni z*1V*nNfJQ1$N_XB+l-1;%7H=L_Jf0kXrzmr3W=sj9VptamoI8zB2lEU$}4Um%gD86 zL^QGq2i-yF{L!C$^WO1%wyF-2^x?@V#$>vxs5H{DK>}i}2)5>{rE5D}){EJ6x?*A5 zE{hQWfSOj36tji+&?i1$<|~Y8H|Rkwse!Y3arfY$zjx-|gBxY3l3BGqIMEz0A`vdZ%5 za;A~KduxmkiY#eUH9R=+{r274!)_Rw7*apL8bm>`s6^B7WR_4@JB|}Yey5AiY;P^o z8oLBo_?>s(R~4TQkILEHA%Uoi$?*8-WRd5&Mc~DATg1bh^GX0-Dqbqq0KDx_ryf#;E4o3u^V^<4;|>CxT2ovO*KjwckHkMsT6Vw?%Dvt2ZeZK7_= zO=kDvXb^jQKlX#)Z#l4FDNC}z3oJxlf92)abvtp#!ptxEz-R7bykZWK45xSnKaJphh zY02hyhezi(*CGT4t_J{k^~POHG=Lmou9T#vaeR9E7>J9Bk!D5BxNAEYA|<2{;_*{l z6{Y8grfC~k`L1nSZk6W9pa>z(YAuyPOsyaQ076|hTmq;uO6DcTUaub@z@>#6%`mk_ zjB#D1^&(eHXvP%coI3(+s zJg=lf%(^F-?-3g?ud{vYG#zVnxSY(Yac`sjlmEqEsVikuo07UaU~*N&j^FNfv29wW zL8LS(-W&{O!^5hAd8x*us_nXed7j!dZDeEmBB5`eVjkX2phMRInh^YodEU;phdQ|>K_ zQ@2fK#VM;*p68nzJBIC?xzG(;yIRY*C>uqKDmQFvmMjN@-ei6pwIeQp?a`VSopwNgYP3{luvn#~ z1a}UncMrz>wRI)AmQB0sQCo$rmWE(Do;bGc1%9{RGpH51ZLDp?k#89kG+Z$?Ac zeGCAGVQ8(Tl!Oq|uoz>GYZ-sjPifz}jwUQJ9 zshZd}DYQGbkC~EbY379ldLp4_G7ej%;Uf>>guJ%e5nlM-o1P0*F7$*s$QBFO;`C~Fj!w-fBTI$hpQ&9lmgl$Dt65E zjV=(}v~iM6mlH(^vMh6N&|XeQStA(q5HT>VS&j0VfBGXYy!Ou5lp>9QQD8sLs*zG_ zY}O4}?*`Mw96-Dn&-ZpWsj14M&Wm)T+f$k_Rs~+{+W6MJ`$8hu3a9}aRugJDHtM#c z;do&fMo|=oPiN^$A-onw9zqzv3~0fXht-3_(;%<{C*n;55ylqn1VOP%t9pTn%M~8a z7iTs%+nwnB8@GyzZ*H7Del+!B7gMp>p$!8se&izn4MFn0Uw`%WRkE}k8%h?oTI1u> zN;Iu7j3ckE#mRUagzd|_YmI`Bj+eLYpJ<36Aj+^LKt!o(e3u&Fg%4fpb&1l^c)CPf z8i5HVhZq=+edUP{|ITmzC*KW3qj**Pz5o7a|L{-0{Q8?W%0&^nzHc}EwRH?32WGZf zZfvwA%cY`{pvcruJ$Jo3=zrz+zbu7#{>7KLqVK))_GmJ0ggq`YA7aK>omH!nm&8)C zKDV)b@AmuM^$z3q;W$B*3V_$bD9f^@X*5QU(?kJ+j|V~oa8VS#?*~zsB=e?eT-&MZ z`f*s!+LE8$7WWfBoCIef!q@XtH_b z+M92@a`w`8?DgJz|7{?6JJ{g5w6v&*#e|;EmYq&V%8J@XKW>XW7pj?*bv9r1x3((I zUBg9;k+Z647{(Yw$nvs*h*AWB6$O?> zn%Ik$X)#N4V!(D7%;trO4F!aPf(pF0-oAf$2$6-9G7wZsH5d^HqR`J)>EmBDBpOnB zd@}S6+KF9&&4i<}7HffN1u?5KlUf!4qv0V6t;qCEfRZwYz*wd$YRSOz8m3R@1&0s- zRL@siXSpy%9hGPy?5%yoSng{d84sZFMy( zi4i^f{0qPEi@#cxqGIfsYrA_F+gX;yt#&b672Kv8G_oqBlw8@i{eJ=jAJ;&}7=#cT&~x0yZ0=BpQ>$YO17oPUMR0H6 z4Qx^YIb88%UQz@3ez?d}3?PJ%2*9{xN)(*?hHC|(fl*Uc8LK_VBv=_HVNy|!(BJ=u zKLatcP0O%UvS=oAAp&=At#5jU?^8R9CBr9=#)rf4pcnSKTkUomSKOz;!@GAE$@p|W z#h9ig{^fuD%NVGQ^>(^qJN;hEuIq+q(~VtxGAWpV2DNmRVys)f$5I&u(Ycm!`9i1P zG9Mm{uI@hBuo`J@IeK_Jf9Jgiwi!Eq2RjaSP|Ad*RD?m`J7u{FGN>EbxZ|ty7tS6hb6NAWY!+4C z*%EEmlS&%D6XxybU+2`tnM`t6hH z426Cpq|z{KIc2p9JKbKZZD5ovriTY_|KJaO-^tN&%kSF6aV=8=)i8~Kf3T0niRy25 znNWV*wQO&(%soFshE*<3`Hnlu>AI4YsDo)A3?;bTV62^7LRNIkOz|8!x|sO?z!) z%dlyYEytsiwSI5A<5Am@f@d!3F_ni5PQ^exXo7Ewl$rNjBEtm(<&(dcA!@bJ-Owo2<}Sy%Zu zjcohO<_0e_*5u1F*LHa0_6NRaNQgGN9aF25gTso|e!B%V7@iy>0uZ)M$4SyuYham9 zk!3*;lx3MDNi88Zti#cyVojYVjs*sTsMiglZ5C26t+#re@A~v7FP}eWMVo)=ul&^E z;mz1FZAwhbVRhNmnHNA}BhN-{hbo}gHr6?pl+v0v2vP_UYXsK_V7*N9d9swO1QG~@ zq3$$|-S6)#=6RB&tX4plMX_=NA0okd(~1X*#uqO=2Z1#@yx;HI>n$4g0^73)Hk8av z1F?clriaVrLJ9x?!Wb{J{LzD(N(qG^M4%{&UU#50c3of8#6YMO1X~+xowj9ou&x@H zo9(SFtWcScN5@M-07S&1Bvj)1`Ssm?lxK5+4NNR371L4(5jmbqfK&%aP}K zj4|7Bgiwre)`*_p+HUtvscT&;stUz~gajE`QHRj*ZOb%lRIM`0@h8hvBWw^*3n@97 zCd2V~%C&m#>bX{ISw-^P-dR`B&wTVb3faxO_i(w`>xa5bR(Y|?%C_%aIJ1RSz256b zEs9KZa(wIBg}^e4FeVrZ$%U3cYCr*DTohGi*~Yw-O@j!v|JYm69h!VM8zD3VAx=VAaVB}4R<%s$DKASI9K@6 z*^930HcU%pO((1E^*xBu#fulqG<*KTFZwoJrStFq*hemP<7(_3smm-&7ry0VM3-q^R7`UL zrI;_LT#9)zw;ZElQZxzyU6y5)M2XOBc^o4SRI3zf6mZ*Pg7n+CMmc z@9lR^kB6q|PDaJ>VE>5=JFOtrT$DxT+mv86nTvEEQ;Gj zCDSy85Iy_ci{~z$?{wE}*Y|wev>ig7vPm5mpN{u;cX#8c9R$H8CtjaV)-Y6L>7v%6=3gk@RHlNA@NRw7MPB-DI9)uQ~{f8(cp*G{u#QCCW^i(5U9 zu;lOO3Wdls2d!^KP}Eh@$ufdHwBRhs5VA~jhh4WW!n1hs^)6r!l@&U&vO8Sn2G{oc-h^_&CdwQsnZrZ@rSP*ldzTerJ8_++i}^aNDxUEDtw8_^u6vGEKXxQ_y4`R17xS z<8-o0C)4@z&wtNnL6Li|ZBTga;#u3ai!8?iP)dPRP)XTjCIvt+mzsf#6iW@ms;h=l z!$xp#dyPQhnG1n+bSU@e)a1r&cUfAso*76$XRUJ5fwaRQ_T)A*AY6nad z*bs}=C|wljyBI;xwnRvYx9var(%0G@4n3_i2Gynj}g-J}tcv05vb~}n9&+|OX zRFVtPlvO39luA8@q5%M;CdWtPJS(eQSq`hpndidQ{#%x%C?ObQsnuXGn9Qb33D5CD z%jpIIlFTs;-?3-O$TqMWm@CPUV<3d#_dI`Tl+3k7BdC1aDJ8EMz?Ppbrd!*C%{WR& zqe`*`C$XVi(>ed-Q(yk(x2|1!qSuP12e(JFMG(40mh5d`n2d&QZ@bQQ7{b|j|I(#1 z@4x%b*7^p67Gnjgb;g!wHa1t~;^4UG`2InXUEOto^c4Vg)r?j%j~JXc0s+Ug0AuGb zU*Z}hb2dx!jef7lhs9!H#e-fqm=AAXzjFOM?|u+_-kHtKufP0`X|+-fFGOa=OUL)h zG9NB8lR%Bkt_9~AvAVFyh2txe)4X84wE+;4VB>gxR14k6My1MaIiGp)ilIq}&7`&_ z;~Tr@pE{mRUU}<{&5g68;oZOd*M1!2=)HGuJ$$flMJ?O1E?>B)Li^>feXGvq#A-_k z0RRPC_1D@*N2i@u8!-WdTnf#(v<)+7ndSL;F(+j{=thuQCL(1m^ITC63zkNXYuipE zl;e0sHGAPBANuki{DI+gYNi1e>w`gEmWyOY8@0LB(@u2d{3rk5PyQnpiZ;M(+k{Vi z?6u?8pwrKRdhLS;)oT9Klg~9$r1_9SS*=RsMnTxEgz!)u2EEbglnLg9{%Sh5iI)_a zVH-tJ^gCV4K!zV596SiTxE;l=V-&@lmrYgaM%8UAs8orl+`HIGm(9%&j&0j@t+-5A zy-qkDjtA?gQD;}#sMQXNYylPPZ(RAKSC0lBx_^Aqi+ux@CLxexJ2W`}mSF&)e&7c_ z3kfW)C~ zJm~DT*K=sU`qpcPx%p^+k^)WJ=%@_-9 zw`pdROhF7bvTm62&D(jQs>@WPlCogP=o>&#lvdrf!pAij3%|89f zPqo@EHUy-l%G^2y9`5{=zw%cd$HACZb-h?7h!T@aUufE~MU^*&Tr8#IWP}_%`}DPB ze1ASUfe=@D`eQ%%!&u{-vuT=iHa5b@Uo2+tzWesq{_syIHhP=828F!I^DJ>qnrAE1 zG=^b;Sy;Nb>Ui_n0+ZC}3n6eO-2xPYkcAfl9NNU3E-p_kX?Jj*a4 zcW>TZZw*4jHFUGti9y3O7w2}*efqn;=kRo8TIl%?KN+~>qaXP&1WE{T_2S2s49kMv zJ$ZEZ-t@CCefr6B+xc`Tvs@QaF+{0%CJJm;(PBPIX5;DPxhJ0{T1-z5n<{Vh+k_yM zS0OT*ENg|)?BShDTU%Wx9uKEM*hy-AFg)4V+Y`W)j$@d1Hw@Oi?bz*8VtE#&SoR04 z*WP|PY_}Y*`#sx-?s$pFe1u@Lu z{rx}Ef=8j>>$Dmd4bER~Z$6Qx>HhIh0?2W_wzG9Ip82tTx=3!{ACDiL$gJ3C8yz3| zrgXj5^Ur=hSpm(Rri>S(<0mfC&9;lQ#FQA+&kF_#1_~L};f)qTJkC3e!d6+-b=5Qt z&$7(242TQh zuqsPxTU^O1FXqcfUIfZ!-0M?}7MzWqzc9FVX7JLL&0fa_qTb%x@VsC;OMde=zeFiT zkYGe{J$FE9N%7K4FNJZ$YMzuzAzIXx>6m4@l0YfI+r79mXubXJd&y)btJ;JzwB5jg zaZ3WDPO~E@8dd=(>>M2&rzKzJ^;h3~?<7enH61Hn>+HHTwhj37wM)~JgHL_pi<_HI zNND*HX!yM0#jpO_zgw)**?eI-?lR4k1RI037oK~m=5=7;jn3fJx8Fm^yKwHp>#yBj zEEbmOX@PmoDY72k9~-9qUElLrq2y#TG7M7H1yTw_RV*`#q-!giLKkZ&MK^8*u9L2k z`FuJ(TF%D#-8%=7?+m(eD+oLN-dk_J({B$nBAOsOaQAk1ijw(3lw|pT`*;5C(aG`E zC$2S2?QWl~%VMn^h&;_!DFDV~mc4uH*2eCI(NXc}&Qav_zWn>Y8%AE#if`ThKvG%p zMXw(djHf5Vs~0bg$0H!cj!&2AYCK<7jAj{J&NV`4k!D#X zW{Xrvr~qku9g(58zW#b6MFWUwd61Ifc%ICsk#7*&V~QGBScH0(xwU!T!M5P&kN@QN zv{BFJj|`f`K8)<{&Y;ze?54`FVy)N*S|TNF$3zBtoJv?#RbCdBW81FNRFV?Ho4VhP zm{eLS1+h{}3+_5bJN6NR5Fja~QXtPt2~oj0!iMXGSy4ZxwKPpbF=9rq(gi@;JNCLATgdRp!l2)s1 z(C>4MPe+q>&?a2}|_tce3)oMxk+#=jH$znBIBgQo~+mY%y9E$#^R=BUcZ${FB+e&;_2Orjv&sd~oAra!MT#v^~Rh>xKeMEZezs z`Rd&}_b{f7kltYPwcGdlX8b*$`P7|T??heiPrvorY}&+YUM}o|!?}fgK<%_b$x;%8 z+pVC;a?_*;1IMx2y}+izl+rQL#vs}lw6@p!u4yVEYf;;joZH&!t+$~`XIW8es%^Ir zv}ja6j*cJREg5jUNCG?=jyBeJ4v$VW@c;2Y`X^VsfVN{W-CwpS&^^qVi=Kz=)^ zi<|d8V4UfuY_(w=5!Ry|&geA19~*flb4f zyR*GfPFH~)&d1~3tuCQbHl^i;Hx3WqxpNaDUUD^>FDH4PmGaTyaF#5Q6Psbr@9ezt zr(a$zXTT5~!BRJ0_`c6y*}GtwUfhc=ZHGu@TvT-<0K`dNe&f}5mX*Hs;G}}y;Uvk` zqAD36_H2<9D^{-RUhGmSI&sUfkln?@WWj-V|Ag!x<@5Q{G~Hymf>M3>Y=k|mB z{i8#Rf^(a#eu#9H#;zX)_Q~nK2F+tZ4ClNND$UD^@ouYYQ3C^g^Tr)F>;g@thDs^N zwqAPrvTGZQykLxHX^t_ixyZ^&VN%s%JWU#|wbq0X2;m@#l@Qv*-JTqEhZ?YyqG+pdMgvJ_?Q2M$PCy2xt@sBJ(B(Xzp;F1NPN z9GpI)5I8_XB!Jew9SzpWjJJ~2ENE7}R+tHT_1v>JK6nFJUN44ez8WT}VLPkEjFMJp zfxK9qKQ~yers*u->UNR2e*KA!hj-qe4YSVH`ooilUhDzXzV|SxWx*NQ-5PK}kgQHl zPHd`-pw~1)$dXdiGN9M-V+0Uc8hCG9?k0LL*pK6@)MhLsryWi^y`bJaN3XXPbcs##w1w zf#j9ob>!Ja#V~R6l-UOEw7sTiP}|#>B?U2ze&k^QF*UNXnJkjBEXy?2kYdX%O;fEZ z4*9gAm>L+1umvqc3)x7^DQWQPjbYT^!d5Uz(zK`_oX)2!mJ2zqR#VWVj9Zq|R3$V` zfYf%+?b`Igoev1s*ub_$TV7y74cO_&o_L9jPd5oV6Gt^i4%PGFJ0d?4Y;kaWWMe#t zBUToMZJj;W&yu6L`F2 z<0iGvUEXkA__5EtaOT|JwJW2_oAMTtBFJEg*e3T(B&|<m>&WV$CefvNSD&-(KI|_~9S? zbMwV1va}mHr>Cb|TiX!eZfEmn|LHFjN{%0mw2InZd#`0&+&lBbKlmdx$BN0jx8F5s zxyY-ntu5CvrlWm>pgbuEL=dBoe)5x}X{t1hI~$F__m7SY&&&jG4F)<$_M=|sI5Gcr?dH6 z@7`G7-jOVsj}FI2Cwbz}Cz%t}MNzJ=uiF+K&X%pg=7Z7X?*8(3e)sn_x^CR}C1&fJ zYpc~N&&sSwld2K`7B#P>&4E$Z4OB4jW5@Lchmq&6Z}jqV{MpZb(e_-!r=EWt!Kl6BvVT3<UcgG&X&`(`QX9m^0lWJ z&`G+g8y<&2x|%yyTM0XCoq6i%7uwyJn2IaOrLa6#DrFJaYXzo-1!uihY*Q5)`Xe8H z5<_WF)QLNR=f3#dvzBYcUSOJL5CojF`}ZGuUUd2Tm43f_xc{&fcZgwzVRU#pzJ2>n zr_=kL-~LU2fz|ddU4Ek19n8kF)6> K7!)Zgry(MP`UeKl&oc4I$d`r@VSU;Fp} z?*!0(u3jp+%u4pm zb1z_M-o1T~RaL4O46T)_87tm<^Ns)R@Bi%Pb{8So>#hT!AcRShcp9Xu1W{C10sy?Z zezxWi6Ki}rT<`ZdYk19Urzx@pQXWF~@xk%KySH|?<9FYGV{Kx<3Il6 zXV0udQSNS?F${y)Mpm!VRdPHWt(J?`atWZ=zc8-6AcaX>cZ-V%avz|M0JWxo)bOHf5(KhUvYI_o)wrqi(PX-qPqzl`WHoslegULFKnu=%+p-8IK${kHY?Cnt zBv48bf;<-@lr{C(F^#IK0K%r}OU14~c?B7y;)-j9cs1MD-p(uOt)F*-KBaWLSPB3M zLHnbV{ex3gsrA4+WyNAv_Iz)E%Kb#tQ&b-gAO$fq2$Ky znrCy50$wXj42}R0vejPK+ME};>H7J+Fr{9lQ{q~vo*mpjsNj${fHQJ1%nptpRt;Dw z!)^7`vf0`^Gn}kW$4f8jd)RpH`49V@?N%?oe>7r>wAcDUTc2A$(~I4}F*iGQ;JbC^ zcltdSnODz0X$AE^`e*;-cs8{hvr4Dy8!e`@B1^UD706w#lw&&mmM28BvA!EQn~gFp z(|_jbGsDwk1f)_Grci127k>1MZ@m2-&oOeAF;1WR*yld<=`RSa>bmv}OR`iotMC8r zPcl{#k#?y3zA8vHT6|<0WcJGSU+C&XBg2iY%R;GW+V(-qtU7Dm_in$wX_27D}VHrEkBQGwRdKh z*lTGCB}4O+VcT!C*<0UJTx@RkP8Z8kDqtX_G&blTe&tWO66Y?Qtytw+v=utu8MK)iEvNYYEWBR}POaIDsUEB6tr-cx*ZPWK%uhnzg?X%~$ zZO3eyrYI`e!12jsv$tL~7-1`|3JNU@i4yfCQ`W`=8;RCxp%2pG)_HgkxYFMj6Pi`#44{g@icByxB<8}!a>?X=#z zaW8V)>0)ticQ-Hi-8;Ac`v2ueY+QWe$No~#+6^qwuyQz`1HnoK8iE&Nc5^oSjW2yA z6D%+0JUeNbY|yqnqBUbJ&m&58TK?w1_dK*(9A)WvJRNE+l@fK`#I4A7EJ83+$|A(G zESrj6;Q4jOJo?RDQZr~#xFHnI#8EvtkpndK}ia^#ubumARc7TB7f ze(=;@&vo71Uik56&If@Zv1`Y%>$;7Nve}F=PE9{aR-JZ$6~M0LS?1GEY_4}69l7gzea*RaKOEq2um=^Xhb&XoXaR949t`(el@dS^{jkQ81oW z$x3F+Mc0X%LI9gpHMLvM?)H0YduIhCNZ@wd+r9MsSH6CuF4*De^5T_q3k+77wsD*! zNmVdzc1ztkbN>2xK4%Rts@cQCgGr(_FO^zK19S)7n&p~OrTKU!wX?SsxYRdXgV6D02qErvdkvJKNg>j+C9~`}wCvckj3)o==Rf_? z&wcN3G~Ma1U4Q<$@BXnL***8nxoa0&9Xn&yi!VJF2aK5D&%W_Z+6Ziz1~qlbE$X^f zxSUt_ZrwaRegrHmm}&>T*WNqr^tv}6++|82glaD0?)Ka7etSCIH(kebT%ly0=11cM z;*iS5A*4CHUrXFE*wsycZ*O2AiiF7X2{izes3|K#?IKf4*-*z1d~35wX*doxu;rrH-g~90SDZ7?@muSI*l^Nhx@4=}bDJmkKghG?t()&4H*$iw$;%}_ zQJ?}qV280LZ3I(|jJvil@vWN=b!~S&x3z8Mt88O^aDKP585k%I4s&$<%;tuzkB%SS zKOJIhUbuRB&2zr<>O0SU{0W{f&qU70o@k$6SLysT?)Jj|=5E}&UUTyZ%}qhXEl@@b+7;Q;HvqCha(YrB#--Y6t?*rV=*hY*GlWY==0GgYJ$- zn}fc^>%&Lsqy5=pUO>aWdz@L}9$OWQ`Sf@)sg~&crSqm>H}KDIT{+jYe)ZRX>CI8`jqkkEu+@6sS&Z(!@z%>8 ztnR)0ZOMvG*cDl$F?1ZqvMk4S@7#Fnl~-Q*rQi8urE8g=Ak&$r%Y(xQr;9>s*K_*! zj>`q3qf`=XI;O=KyMH=4o##iBWbe$*)hEw?>bqZBX3ODd|Ki1sqkFH;#wTf-*OEVZ z?Foy}es7JMR?`TLNyADc%kA@B$)!6sLH{W<)LFM^f$LS$q^w!V6 z_^I#v_@`bft2|rfT1&046xcF-gw*btzSfv4jG##lDUpakQVTg6j*pLzO_O#)w<@#y`$v11 zt^z``JQb_TA*8$JTNG}s_2STb{`_+xa%0PFB(P1VA*)_wsZpAfn3}wR(MCX0xM_RFNbWBtl6|5CRTo!RV2R;iI2Q= z=0Yp0#v5JgxyE9-!gjb=CYxRV`cqGK9a%u~Q-A+oKy0-9Ft%;ib*W|FyMLg|RWTi3 z*j!^=OePc8G^Ez9;~=TWFO1s6u<-m zjEEtn41BlOX0MQ9na34yE0!a%TgIe1VFLK<{T@=g>4bX zwjm%0k_Z>^l;^a)AZW+&YIJBIU}JQ4tzWae%nJ|OhH06$(Ue)T zT3M!@30{>+lP~r*HY^H0_qosX2RlFW5B^^pJKG=n?5FMwkEd0UkM1r# zuPZYo^|;pG{pruNYyzRQef{X@gRR}QuYK!{*^fzCJyqImQubISF3LcNAbz>{x zcup^Dw;mn7mn0AF??1|m1W*%UGmhiPa~-2;+1LOebiGO-KiqKacs?8gpvI?12GXsT zmuHFMz@*kjw;OwoZpxNx?rpBMqM#K8QUKF5F-01xOP9`t@!(@0|K6f(g3yH!w!&z4 zcfh&)m;dU2rIyR2ZuPop)0|Ale&oB36}S$;2n{0FwVg%NU_|acIG#=FY?W4`$?I9( zG?sxpVjVrqA3hugzA2SZnnBc9mT5v<*A3&15G2o4m9N?z4}cmmQRTc*&KvLFdHd#l zE9|FbVH!?VsKqj`3(*wHje2dz?#JHFX55Nx8a;)e%q578UdMR%&if*;-gx7k&wSgsNtdxYQbhp?07S+jk^1<70f9D%t z*D#n)mM_2jYMu*Xm@3j1_?LkrH!_%1o5H*`uU;D=C z=^-`XayI3vF-#lUR>_*k4g3AI;c)2tzTN8QjNg3k?dAM*mP{W_W}|Aktg7RaB(DTl z*pAwxc?y+bHhQlgt;ZeHw5Mt98sG;$^UT>bN0g0iRGS03d#>9*yY|5cCzu9lI{A;k z_nYs%_nN5ZgF)xgh0Dv~{EOfJ{MPoRx8A<zi#s<^yq z?6zH#lBgTE*Pb0F>Bm3)k#l?NK((SfgGcsU^n6Lr7ytiXn)@+u~7YbU& zB3adG3N+!+eEZJHxM=?HD_;*x7eP9jEho3VNC{;h|}==QCH zuIoQ@;nEWqFWz|P8>~FB2ukyW33NJ5h}GU41SiM07iC8Lmc+2BnvHe{Koth~{@n-L z7cMEJA_6I*oxwS1nv_TzOToEM{TpxIW4<>@iq&k9vZmV$cvAtXki>7i@-~+U05Bg< zq?C(U^6`&9|KtnT+HJ2F``vEqJFk9Au&P|1j*lJ@n>gL*55D@!?c29kfbg zJ7g8#?6iH)I-JaBMRPn(R+ZM7@dtnQM~5l++SmWsc0*}|w~v>T2ETLrwryi%P$qdR z*zugMYIx*Z)W(YfPEUEiztQbQP7w067DCvBIFUCvyS0LP5X4E|+`0X*Smp=`AB?kgW~R)YR(_sKRFp5Updq5?OjeLr+1DfmVy50YX`5q_12j)T^HeB`PcvKM?e44 z>Ga;;_y<3?dG6^78H0{f6{VEcyZ84UEZvqPrM3cR@A9q_VAmFYh<)FUVs8+KrK%V) zO{7keog){=k8iY=!Jq80ijO$$7x%CYj7cYP98~gL)$#k*ajW@R2?RM<>W)Sd3 ztisTin4Ki4w!$R`XSTwI>t4H~fgxm-mrV&^o>dPI?{LV35ELLJ#K~xJa5^h=Js*yu zPI&u+yPMsOb35lDAXq5_Bgq@|@BjDTosGv;Gp|`wv#Q(bB7hv*21+)Kn5{Aisq5I( zz@FzT1eaAI5D-&lv|7DRygxd5``vfG@I9aV=u6K5j(813Sr6KNS+RrD*=$vD z4NoW2d$&%MG%#@=98BF-R~vG2a^RYDRpez^&Xp1p|I73VlP?3rMhnKtt%bp;XxOI*leD$K>>cXQtZ?A@@G82|%DWyEiE~`9G zm$znm*Zf^9p&h912{G%WJ z1=iEz!rH{h9yi`V$|^lccC>C-z(yMv((GS?dp2StN_a%isRi zcs#UC$21Ji`Fybi(6g>&$m3^M7Ys~!VR8{7LxXWHH6hSI+rB$V!tsP0cJ5 zBgGf1CB_(oh9GP?4hB#GjZB&rx#u~)@9dpFS2fHq42S_T9L4L4=e9odp% z0D=I3qtk;Tt;XZ=##(=SH@JR%fMf!-Sk6xs1g&=W=FQvVlPN&ivKj?pBY0C)3=xx9 zuGY%2@%p;g?%VVHWc$p;4?X`PQ_MjU1EwiP229dPmZYL$v&nQio7KF|SQQ81Vlj=n zapYUB>mq~zg12tp?)UqaK^wt6-)For9S0!8c4HyIYp=almw6!4nCQ*6_sws8TM50I zXMs($mc%v*K#>TszlR>Zvy=bR$r zxNUw!?RcW*AU{DWI}h-sD$`J=DBHd>^}^j^Mw zcrY2d9{7b{{1=nyc$TIO&x6p^fI$dV!`fjG*tTmMf#=zVWk6!vmT8CM)%fEdIcL+n z*XwO(7%y@z^lUt#23an0)rhi4+K!v&d6K2`q!`U6C&S^x2Zy|2 z*yd7|bv*%s4-ZfBg8Q!5)cM+=yVmsxsp2+rOaeH_k^~42DTYXwc~O=Ntc7V|Rh6D? zguYc5>1r`o4Tn)HT_DyW4F zeZ!;l((Z-vJkbC*wG4ePO{%R<7bALq>Fhr|P)NB!?09`su~oW6*hZ$lb^os4>KT6P z_Q6B~${Fi)I#H{YX4QBycO6qp!*|>>TW6HEwX|Apz-q8u<(=N(!O@A~xTD4Fop%o{ zi};~|OuW0%HxL|-C&!1$e8g%=TZ1#*{`#scmEeFg6Cs=E?S4!NIys(D8uKds{V)Aq z;MiC(&CAp2I1GIOMZU_fU%j}oIr!}7K2@+8b?CLLPX$4A``+!RpE>J>EiDC-YQpy@KpDpt&;i^uuVzFvodG+SM{|~>r)9swu+3gQHduIm?OKm^8 zeVlIYoqy)}XO{~;ovuuba$djo_6?1qM@Px8{F{F}nT%k`!bn7GgUJ#+cr=+7s%-G> zTX%}0@O$yv&JJi8MU)~8RE?B0D0WSg3#O$gvXWJe(n2Z-L3FS`I&<+QCjis&aVLs= zM<~fORu!KqRBmkchvUP*bur>y*U_vYmbpw<0MprG`44{X9|1tFo!c%7n9OUSWnF8- zuvXLQ|NKw?7a;L;G8#9ZFdZhL8wW0442KP?2!uqlelKDg+ZI)#&UNLOBn~}_ zu;3+c3d4}d2CLBG{eB-pETzDh3tc2)vH+F=BtQ^3ZeaNy#d%%M^W5}8 z1CT1MEz4-?ii>Q0GsvqF>)eYvozU*M*8Rid!P*8;dR~-1wKg2`tv7Ca?Twm=G|#C` zE!Qhb(Ex-YJb3T+qr0~qtL+)i>A}NS-+FD9WaGssSKyty%P)TMiwKC=U3dNVEOlDl zGmAprcsR1$sA-yZ=*1ymg?_Mae*5M)ilUZ@TqjVdtm@o0G!aegk#>xIpC-!@s!JpE zw)WQl;tzcG;+bt!WrL23G4#DK&Do`kmt6YK~7gn&{m^2O3KT}r9rB1A!1v(+kvT4Suw zU$}^riekUD*$!IXaxwkf=e~>8x^6%Ph}JT8TJy=c5i8e#{n&+~tT<3!A28?|UccY> zJ%?Atwd?2Ky!k$&-s$Av5C8Z*ZMw6?p)}mA)@S?P>Crw=Fqvfr76@h~#Ccigd0sUI zXH8_F-PpU@Yo7`HEID*cvMO>2@w@^z??3V^~v@3tQu+-Pe4+kf=#H(%SI4u?D)_hM^jcd&QvtQXLx zoG#M&`g(MDd~b8GbNgVtnm&B?xl4cRzy9lPyTvI_D>xc0AvGS&>e0!<>)7A_!#~+u z3(M*L!L1Ks+m>7nM@uNcde_&)uA6Gz4?;i6>Ks862;K^fpodR1zB4(Vt*Q$>Jz9<5 zd-G1tn&pah?Y3#a|MI{7TUgoNF)xw}Yw}%L+nc@6A}&yp z67ipY?>}#xyBxcIQ*#Uvf=XHd`zDeQT{*M!6F>2jlgR`_2>@ugZU9P(%8G+qk7f!` z1{%eI?FI&Rf#OZa`3gs&#O z*b_o5%d}Lo-|=62`D=h8gq@})FTeiwjr;dbkM?t+-oRCsA0V-Cy!hwqNWkHg8&af z@V=IB0;IZ|ThD*!#imY^*-^4uY5XdNCuI&)$7Q=k1*vRLl!oYx9} z@ZeD{JZLDzQ$G%hM&vae9?z@1Bp7(6UF54Mirki=%woLU|L}9qcOri|n{0IZv(r&F zORiu0P?57Z4%Y{JetYnpx8Ir0(uALDdbzQ)u_{13xa5Xs?;kH79nIRE^|%kKf?U1&Ic)er+;6Y#+`2t0Doljl2x(wS ztt$Ws=d3@7gkrz^OTW zSyr4>q70+hLdd{I!D^wh(e$Ac+o07NO@?3lvp*V-PFR5t58m=E=lt$D)3B2goWFbp zQZh@5{-8CRCcJ^?&s{k^J) zw?FZTkN?Dv{g7a}Vnwr>C|D_!OE4?T3J|l~ZeKaO-5<26X+UIb?`@7oqs6RZ+9=BJA3numZfMP zeDJXla5NhELArB%!Y~GejHWY1hE+p>Wm3fBC<+}3&Q+|cV;H)2vh@p#A3XT@o!f7(udMCwAKkmR&$CK8&PJny-Ny&LUT->0Vt=r>Tvf3D!(aNM z0)%eJX%w28iXn@lq|vOr@$EksvVMPicQg%}wdO3GP)=;ut2MlCx2w5wT!wLxH0PIY zKiGZYwJ#i>oL#$gSwQHvTF7WIiq_AqudJ*Rk;+0J%(80lpZW?8Y-u?L9 z=29&w%3u38zqYu1PFA(LTPezvP%Z$HWL1(?2_ixW6CCaK5U&r#ss{IvWm7?riUVdi%~`*dv9Xmt>Yo{aLiP@An4XrfUP1iac7I zUxFNFN$waj0Kj&tm1=WgapS2MUe5*IJ3TY2^^b4g#!`Iq;?w6ZKcPx;8Af%ZuoPZq z;Zrv+LRhF8$}*^_7J>poC`l4yEYI^+tHl_bMQKt{-S!x?_V@Q?o+5-00=l7u>4;M5 zIQ1NhR1^gVL6WLM@Tf3UWi}miPATP|eRjvPoHz~)Q^GRUP0gt`>{^E~>Nw5gQ-3&! z9L>t2n9y88$dL7tkk-ON6h)l#xw$zc@`bsE<$7!9E;EQ5jcU$H%aAPuEGHoulO*r* z`3t6EEYHuau66)qnrun{a+Twi;gv(dEeY6{6U$Yj+BCt;o>S1(>V*gb??o{i(Xk9P;7$?@L()rGc#rAnvc z7wPxj{_Nzq*E>CHd7du7G@1Evz(Dc8{m1{+aW%`<9v`oi)?O_Md-=%^^=IdNR0T~)JqBI{NmRDYIA z0`JaZE+OCduiv_5VM{J~@6k?0Rl->|7!)Okrr|}!?7{9XQtUaa;i!&hcuRBZ6Q68e zyyV9T1T@JLtk{54RtOBasw=d}We5;tMG=&xzyEl*=6bb8yCiY~lqkzN!zF-}<*dY- zYRoyvs~AX^=3AH{pqXBjr&&CkYk37P6{MG$5Lt`!{c1Gj-X~3>DB?WlpCwn{=6EAgA7+gh7@~N8>0AA&?whYr3kTyBiy8@4o$= z)8X;e%jZl500B1_<|!+QUe5~Rhm#k+@be$s-YJ=U+7G5@LC1C#1q?=`SwV-rsi~Me z&Cd4EU?~JLd$Y1tudT1njfbZQfix>j7FMcmj&KYlpm>juXAk=6{k!{5Tzl@l&vu3p zs&?$#_dm*g;^;C1XPvF}0OD&at)U+#5^UBb%EVwaJ(v_pGLwK3Pp6iJAt5KjSrSpV zVY0yQICa1?OV^q%)Mdm0Ra8AGQYuK4gmE0-dhQiMh^m9LZqPqC#!_Vzu*1Q$q#R?+ zfm(Ad(`xST9dlV;t~J6q=%4zgR!RLZ4fBojt(}A0p64kVQ8cwyu}{arT(wo^B^KPZ zaOodkyEOmCJ3A*2?%Nt>ZY3zeFwcZplxYNe1R|j!gqDXgQSCtAvv8*6jQw*okYS(L)ER$-@l4aX5 z41*TqS$^-)DFrIV^8VgwqtRfT8-@Xdz%Z61TGC=?XGaFozxaRszfD<}FeIX&P)3?P ziwlA|7L*ezNc!VH`IFsoG@FW}lm2krTU=dH3|-Qcieryw)3g4#*Y#&(7Wo3pWxqef zoNE}U5F|avnjwtDVTwdNij-k!|-N867F{h=f& zobxP85dw;egaDWW4nYaHqG%Wzs_Y#de=68n9zSx-j76ad4bwCcfSmI(%M(AY)an(- z4Kp@NfFCEL{=mT6;^HDGIgSg{wknOr`HPoi4W{L&(vTPzl(X?@+HC7c2G2hKa=p5! zYhIEu-86zIGc83mbV*hbGZeUR+&i+qI?#bcJ1g@+!t)G99jMtah56hue=0 z(^d_&(W<}pxtGJibUF+w_4yCp{st|8V{7e(Qm<(;Vzy?sD(;-;=CYU;VP)QRU3t0f z);+nT*}3x>>B`lmN=>gfY!fI| zO+VjWeC_*Qi}KK|G@Gu=0a9&qtvT0yyx*}ZG6bCSl#_Sg`|~87Dk^U@Oxe)h`SkXu zpWWZt-9J4&h8&u6-+kuelVrPoQ|`*cki24b3BbRNwF4j^X3oDnc zYE`dDvvE%b)YBCVDG+ot8QNAg@FPXhP`w#Nv-I?srI{(KPEjZ|+f4IPkwrmCkrr7z z%ah2hXjz$H)zB2BRr3~1tma8YWeBT&kmhAc1`ock9-}-A73oKXK^-$;qWlS8COkp|>cN0F*nO=4YSX5rW7FuPiV3PR}l0+KQs6 zVD$I@`26uOlK6~(pFnDlqQshLIaY<6jN&!k$(REFulrqkXob2uG<$3<{i(mTE z7k}vczW;|`e)XA7)nG*~7==K%j*THe0&pNPCLod+SDLN1t{TmHb2yqBrUMxZlJKn8 z-8nb{!iggYp=d&+AY8B%=uBquE4KElH)DYxPbY zc^#M8FhYe46kZyC^oQ?lo;$a?y0(6P6AFN2el=QRJ6~w z4}1NwhNQVp^V*H;!{Kn2jGw)Aez~Pd>1e6mWL(!R`|iWzySpbPEiRpJ7MXPKlgACG z)o4^c{NN3e&Z>r3UFiI~-}ue`X-|W0znfAXtgU+zh|7|imL^F`kw&`Zy#A#x=ylI% zR6l$#E=FWfa zm6s$%mNa8BDf3ilik{@mv1@xrM}9Ds5SI|HsidROXD%%c!eiSu{@#E0F9!XoX6W7i znP(eWo(v~LNmHXNWfF{{SjY12&ZC_9QP$I3vam4s+UH*0Tv=I}pI0SiIt_c>DFEQ? ztcMy_)l?)IhcKT(RwOx{hAGCl?Wq_L4Z((^Dp0X>Eew({nw|{%lQavm9OzEK_$-J3 zQYW*(@f=-Mi#+?o|NMK?aqoZp$A5ow?Z!X+AO7d{D^ESz-=#%fGjs_sP6|PbB8fNF z7XHcq{Ht*((ll??J;HL^wFzg-?KvIGj%HlG@m$4mU$}W0BX;lhy$f5L_YVhtAcKAxVEgCmTuY2w$o_Vk|g!R&@`>d)c3=gq`=?(!`}^NVVov} zk@@w_#m*cMS*_JL=?xiCgJHPbSQzg256Cr@DVtb}{R9hFq>V7mj?Q|z zX{|1-EzPe3uI8!WoC^++Zm9E%_LWs@W3^s$C9h!(qs*;nKvvs^I=|iu7{wM&vQpPn zOLtY(7T8EKnb6GA5yTwfqF!~QD4vF)Q?C^ukEmcs9Zn}j?Ei~j{r`w0A5CHjxfhlj z5+;fa4a2$ju(vhWdivH)86cRGgp-UAR*1=XASse5DZ%K3i*oyL+-ToCIPISehGAUf zluUxSBx3)t2Z7;drEW-#*81UTzhN4@46CX^Ma)zsr%6VOR<#Z&Xc%mEdYB~1@lh|F zp3NqMC{0Gaq3=(_;hu^V&Oo(VZ?CPjYVPrlN1j3~Nnu#6PyL$O)z__Yad;62`z5TdEmTJnpU{MlAWn>$=XPJ~ik|c<# ziswQ@T07r#HFY)^EVMhSVvLSXi+CFO;D+LII^0U&ag=7nDp#Na}^-l{7E<+q-?2ua3CC`3?}u?$MfP$f-2e0Vsw zP;GYf$H&D-_wL7e1|ZznTIkeFT9#~<0{|W!pERm->s6H^03=J~ei0W*$&Y%|XP&)T zu_`;C9rX^fZ+v6tt-F1yXoe#xo(y!^sW&uBMblxJ6S=@mBx_kIpPl#$dWq;?`QmtS6=l}Ba zFTQR!Jl)pZW@R{wR8>_0FU$P!Xcx)i?wvcs*<_H$7?4q~FSsPjwqY0>7uU{?&(cgV z;HEV1&4Lu6TD^iGuegTe8oeM$1=b|Avd}p`*f~4gRW&Ua>YRtRmYN{);{s162?Ux~ zb(0LZwI$P%WFzl(XN^V!X<8y^r+NO_=by3NJj;{A-brtKMyN73f9{hH@9Bz#7z)Cq z+H5P>P`S(~Vid2hTyPvu2uKS`2`CE=fb`8jeOtkHt=?o1ed}v~)IUD7nXf`}`N9?z zLm4ZWTa@5g9*pvyZZ)u`A9arfBd7aWk)cZ48wCMFD$6d?21|;kst*@+0 zlI17t{P~+)G9nH~qlxFLg84~Sl$0~U4)z~68Y-WyR`JYkjGDbzRMMX>`v%qb%9v_VuZX1Rb#3hjJ$NPtSho?)c=g$04cAf5|&!h6|fBNmE zt;@CsOjW2rUYoB|nr4h_Z|}|gVXI><&(&8Knum|?{pN4{?+{s4yN+0%j0P^?NW!X& zn4#K!7bK6vPT?Yc?8Jlah9m|!WB1tl1 z?C|jT+uwO>It`bXSI>I8ghlVZ_vYSVzuIig{1H;rsUN7SDr;&%7ze;oq=khA3a|oD ztx{DaHO))FL3cO+SW7eU{(GP89_keM>WimZbW4PDXIHG#4*Gin5brI+3VkbfpG#M6hb&|b$)Ke zbKKp-qs{XdJWEfKK7lj0j>+h>m~``edg=Q0-NU_;?ulwzhU4Z%G2gCm633B07z}^} z3+k8L$7*TUYMhG0z5Uq?{?gC?A6Oy|j}K3~Cxg+@HhEU=o!@xcF#yKUtJl*su3DC= z%4r_lfAp~8x(mw-&8mCh{F+;X&%bopwDUY848mr`JA3p<0ixY%jz_&8`|%%*;zM z#WrlyL=XvCWT1!$kdSU*jmnyhB(+{|DvC^s0#bo7)-=N~tfEAo+o)F?X_}TLl_bqH zoU#yMP-Y3J*PEs#-PpQBvNTQux4!YYFZ}es{I`Gj#aDm&ul~|sFXKqXpjvOl6qO~Y zH|8EbI=~QutW+t)9H(J4^M?gZmll?Tm^7DHBnj0_g%oA8F>ja-2dZGWOiER;5d#({ zHP3c6Rgioz@g)oZV`%|Kli_H5hN!eYw<;C<#>I>AY^=$oNX2vlG!uRD>FtYa8*R-y zIO@Os-n+7t$D#j)7hn9#KlCGcNo}u2%Y13E^(%kpZ*y4ID;DKtj`B1P$|7YY>Gpe# zTFuaON#=}afA-C{Y6&>{mmtMOJWG{^!5`@9XET z9F4})lr`qp2jhvPYJ?V5*N_qG%sX)!r8$XHlINmQZ9m+*YZ)pc+^JRGe*3$I24~Y) z)tw+p4^Lu%Ni-Qpk|w82 zU;OCf+j&{cwc8CF^-m5C_l_OSxOZ>Ys4wa5mFDV_q`-WZp?M%#7kj-MwCO8@PD_Yb{lXPi!;A_*!ZS!8+4C|y|CG7kS?#(gCf^tVC&V>=_pH6({NYT zR;!KnY%u9g&Zg7T^PSb%#7B_!;$&fd{*zB0wHInj^K*m2M3GdOg_3P?S)PVt$8=hEHtE-!8D9DZio zooVbRWo}@_F|cBo&)Y?&`)$Sz-tc z4Uu==`g2kO$u#G^g)3LiPvVhocub(#Y-(9nS(fj;|4FYqW+H0V+8bLN9D+ai{XcO` z8vwAmaqjfs-qeo?}VNV>?}8&}+q-@dOjz?xl^5 zul&Rhe|YEPpijcVw&l&uCPU3|>lGA7>02M}cNP{&nOGWJ?kv-i^@h_ln(52Ses35g zxnUcUf)O#|c(`h)m#=M({n=Wxoe6q08cIy=4*Cl#^I_=Qs%doUyGMHsrE&Yw;WHO% zjL@Cqp>9~iVIFd{vfvgh@2qb0y8}Qp!HXoOScC1FHW?3|+FI!iqE~XI7sL@7%}wXrPq zdc9Vwr3!le#@0vM_iNSp{z(^b$h*DrP7F!DRL^B zq{IE*erMhT%x5CDRK$_Mst1x}b9t>I!?WGfilI3=*EPv7ZAFq*RaJFUGxYh*^9w7B zvWgtN_6tA%H~zst`sa!StF@L}Ya}7$Ol1^qY~1+Q|MCwGPIj?Dt1Y`#vvzh5uU=T6 zb2LAgG6-g6$!!BGvLK}mt$L-?9mh^hHCEK5^lczW866LRa>m;chw|DtUg&1Un? zy}Jj;djJ3`gymF#kdM0krH!?+psMWb9h@8=4K&5X5LD{5sK_$JyZw>l*3NpPk3YJX z!~`SF_d^Nmu2ZL_#5v$1Ps-%N)~2qa>1debv1vKAYDdw`A}5aH2q8wJk&Y9DBPg_4 z7}YA(qn%xpL&3E1nBBhrV4Rd+{oXsf$Kyt8nU*9NP1aWynr#cxLd8IZLY3u4wPre| zZRi4{B;thtN5k$oj!n<9t7b_kMDot=QKi{-U9at$nv%W$?gwjY%TGLY>+$x(H^29$ z;Wo})e7Zl$Zr|HmTHd(v7^-8@j1-TTmX=6TJbmM-X0;aQ%4m?1ls)&{^OjdJbW7F}RjD&jaI9L@`YcL- z#1u)cyUo3$6N05UE1@VXQ{p1AIIs*|QKhYo^=7qFBr(9~&Ye3|*DGmOFj`PN3(FFz zqe*$(J0*fZ3|@NiDbrNbj1wYr0R+s^NuL~bbJJNlJMleesd5z|I8A0jnE8da-Anor zMO;8uTAD&e2g>rVIgy)BpH8zws-7@2^W(o{U4wGSjpi4#t1~ z@Bafq#c%%lf6x^*OW-8ry3&#rD^7eJ%frEx3Jzru6$ON_Q?I`G+-B$>dUb4@&5E-; z8E{T$vs%>T|mTp}*2N9o}o68Gw)E!LX;&c=oot{{xN;&c>RYP~qZ*2a^ z54;v7p<(G|nMYCCA5QN-*clDNqr+1mkZpOBaa6A^U}Ta~D2k>VR;AGdh#@E#plP0G zc}XbEiWFmf@7}$3yZ!XjHG?_Ec^=>bW9-?MCQGWWjVFD9(PDccO>zQwZEijY$Ew6tLz{)OOINNH91n*f zFUutAJ$2)G%@7ZF4{NpB3(r2|&!YYP>EhZo-7=xzfA+0!udH3MjoR++?$zthoeYN4 zFpkn>7NtQD$TIxqx4!Yh^UqAfAWI|7&`Lx9fUMdnl<`bNZR1+OV?i%Cv>v2`}w zcI$RVvmz|ADE6FcRxsQ3M&0=2VQfj2x4wJF)V)tW`HU0#)KiyR9ZS*Zhwpzh3IZf6 zU;O+JMv29ViB+XP98JeFLWruWrefHJS#oxEb|x#5=Nh&tXHi5s!5BV#aL=|3T~i37 zx~{J*FPB+}A=eDWw(U_c8IMB8vs-utE#MlSuksL+A^eZnw4p0n)c$-vSOI7rZikPOej*TKl8J{ ztV(F;=a%kjuvGP;KRjr)>!O@(tY{1Md0Ukdnx2gNQIaBAKOUxvMKc_e&d^82Z;HTDU=meGeZul&H3eKrznCb z4g}`?!RecCe47;G%U3UDGrytgqw#FepC)lp6a^OyVKEp?FjCTl6-7x2OcNH*@{FJ- zUVKqeWB{pYyJ6}VW!##pQA)2~y(&qP>@J-@ceCmU74eTgy>~K7&qg6(B`XU6z)Ghg zm9vI}F(i`8GsfZ^&0=ucoe-c5C$S)e6?qs3l~$b;nZnAt=hB?zdD*E}Rm3Hk84W978}sH6d>aDHL=+i$%TaN~vkcd??AF43tKD|L{PcR=2!H7pfAH#+4v-rGce=f)@8^^NA8BuY{O(Wv z=xdc)ZRSVyW=p9Wr6f`z$*waUPoCP?ETY)5oRVZ8eEb1ose^#7>)R*C!<=L(achkS zM~_+?jplr7HlFG-^(+Ml3CVOe8!yhSBh~iYRfyB>U}RUShrP4!e(meDfQ##wk|eP! zl{_oI^PTVY!<|Vma%|)L>Qcp2tCg05r9AQ17CV3!7dBUAq()(;Nb>7H^8IGLf&s+3 z0u;P{VN*49&-1b*-B?(xIb5RlymY~BSaTPicRQEbt%bkxU;e^04nFws&7nUgl36g}$7!=#@oC9a z6{^B(8dz~>lTkq#m5pz|`EHU@3`?CsmKCjLQ&KHS)`S2Y;ELzebj>gfo&v0xi(9w$ zcXkniVVY^0;Yy%hF=oC$n1x6-7S=ChMX9UmbUHbFcz|U^kx(!^y>;_BkqR9dVGd|i z$fD?-OpbPsN=>OF_zT&;=k*b3y$-Tb(8BX1!sAk`WPnZ<+{M0kYtLtgc?zIJfdMKk?=8 z?9bl(^RNBpzx%goctpdKY;-DuV@azmUAbykidhiMwJR{zx}(YUmtJYjt(&f<(GVGt zRx|g{jyko4X4S!rKKr?6pL^z&x9|0eA}N!pRIrDizPY_Si~L~w(e0O>d*!|N-vUOh zT4^8bJWg2Tm{pO@7F}cO+{T^#9s{+;{Q4`e{(xiIp4Czn%Tgt+h_7FqTc0x@J@|kT zM#3>wk>XXxqppS!q8TU=%G2{}*9gatjviL3#@ga)!KJ$ok1d%kwCCQs^LB4E`t1Jg z#YSss9*LrGObfY68Ws<4-~H*I`(@ow;v^i7gN^pabFaMifBBVv0)#rZur!&CW*Jvp zV>I(Omd`0N+`6>kn#Sqb;PJ=zqqrESL0-@-n||{C8(4(|D*fYWe>}Bo?iYXf$7BuE zu)}4+GG?k0<;*lm#dBN;GzF=sJnT;4_4R6oR9cpomggl!-`U%DUHkm{YA_zi3N$pG zQD|Gr&;8s_{q}$Q_0w+r13&T;w?F&<0Q0Ng_>i_|YkmcVA+E?}O=WzCdr#{k@DnYC&)MMbec8_Z@ai|azk=Q`8TbiTDtAQBK~dFdFsteU>>S1psW ztUn&rDhqbiqj?4}mk~~rgfi&5_Wk>JZ-4gKtF?9ywmr=RPzj<;t5&A7VOeH#)oPyS zrs0tyr+{aiz>-0L83Kk3%e=rCH=30w7`AH7yqrwpaypY$ES!pYHuFW6LdJF7EQ-u< z&4pIy^z1-YqHYeKY zNpGREt=XzXahCW~NAqS`sbi_p_ICGAJ=+>gX11yn7_TfWeE7*Dw6d}!%d%ma{r+Gy z8mX$PNr)rbuGd(RxfL}@gCq*ydgIM@tJPUtzIOAa$A^8PI7X{cpehp9}?ES%1x+@M$>t8-L?(Rx1t8Fz3811jaZFL!Ks^8%r38 zI85?15u848{pw^q+}}U6ZL`Sf-qF!`JhXL%j0#Oc*PR_Ibg8B0h2)zEcCb!?j>Xd0>^-YO>D~o_q z-EbCHR%YQWEP{HKwd%5M)$ZN?tln^LilU5Cioi&rmoPfvhcj3-3md~5UCV0b!+QDfm+vu=L!>4&-ioy8VX z8j_^lx%XD1(FnUcu30+@b461xtiVMQ)~^;iE%-ImKxlo*VY5@FhOJnLa-z$2C8YAk|bdX6-BbNc>a^ycRI})0t6$V8Tx26>Q9UFOC7_Oi#!QueF`kb zn5Ie+R>LT>D>X~S2p7XqXgSp=$vNkYF${!i>Yes-7)&4wB(717QX(kj!OUNrTkQ3Y z2yl|3G*xu*;zs}Ez)Af zVM$gRmW?@p6qQ9Wm`#>f*CB~N`0PM23>oJ)uHRxMF?Ay$MM;VziQahQdz$PKLO%cc zGg*>OhNIv6<8OD`OGQEQ#1}$TE3HxLcd8AL&zR8fFB22vN-ngvNB*26%q*A~hUTs(KF zw|hJa2SH4x!(6W5maCQp2Usp6zgBIlt{#WuB$X%)f9#9j{~N#kd(DR4Y0XV%QLR>+ zOeT_|r)e6eG35kOMOPqYx#d-ow7^(aB<$Ga$y;A|>wDkk8kmNqrDMny9V-Zv-IKnl zTU-^MYNHbH3rlrb*QRm#^wo=!jK`EZSPi0Sn5Qe9^&6{Owr!XBcs!l-hQrwJ9UbqR zh}S#r*2U|#2fb!hw&%94Ut4K)TnABBEN@;fii)Wj5AJ^Qn_v6Gr=GYxU#T|gPL$@1 z3(u=!iK(ic%`)PHmT7yYZh5j|dO`>Q07NlKL_t&_L}7U{p3J8CgU2VHZIuO|l*9ng zNE4Qq71!*Bfj{Y!oNlhIb{eW>%h>T|SzN6*B+lB6W|$(VsFN^m&NXQ2yAAW^t*h5B zolk?oc+}U`8oUi>^T-R;q&(pld-8~+zS_raNC7Um-RpJ@U9Q(`UDl$+zjkX=$oP|w?+v=ekN?OICjNn;6|UPb&5ETd8WM#l z3e8M|SkWLis9)qIC|MplPPN-j=Djt548oW}P?6@fdJ94ClSjuBKf8YO#?@<=+O2A( zQp=jMpLzC)&wb(hWWkFlM8H%frL}hT(}#EbFvzpftU$f9(@*Z~D=Kzt zj;hd?UVQST*B|*=DNyX6U4G@3v#_YDww#iTgh^OjSls%VpZ@tu-I%W_!Du|cWN)^r zPhNXoK~RF6%kpgcxFRVBcW;04(f7_GGLF*j{?5bW>A(8*e^p<0&jyc?1SBLktLG@; zGFC^Y{iVi=r|2?g1uwQgySvy}RCFmKu!tDZ%Xi*;^TyN9D-hRowbjr!n+?p-xs5Hu zgqv%tvSnpKVp@)Aa9OudYp%cm2q}&7xXe%byQ+c|m=i%QTPd?VEprWXMOC{eU0GHk z6j%{hv%mW7KSYvF3*2^GL-nwX{B&p<06_^z5K=iKC2BdkpVF6EGdfOg%@6KdR0hR!>u4ekt7XfN#M^oBM9)r zgOhq=E)KK%d;2>lN2)E)x81Y|g{)k?cB{;bvh?e%1*hJQ;zW_OoHAr8AHMTO_vqBZ zPEkg(qWt==|4xwj%WE!1roh;?&5CUiCIG^I6o)}F98C82j^BItqv5dM9~>PV9NfKo zmr}|owJZw?E^#mmX9IuOA51984tDMqIh%~bB8RFfOBg+Vu&-emOQbt^1W5cdKmOCE zY78b_j#y4oTQ@w-%%(Ax<>_GL${NPFJvSdG`FJwp0&Slh%dSmiB*{QQ0FyFHV+CP< z8fUbw>F^ktm@s0fYR-8|a>~K2XK&bbyAXhLy1cx+v9ZQzu1GkE zqE34`$-tMs?}v2TOJW+0{KfUTB#(SQ=${6E^ap>8Wp$M0pFFr*snur3M^`r24XLcE z%hPVuURXRlJ@Nf%5=}G<#aXnvu}D&}eQ&!U+W4Rl3F__ar4k;+GO#Qt!^4BFW3?Y2 z#fsiMIygSq+m41OOEow3$EQb!QOcM?Dvj!J8ieEfXD9b+bG5o+o?Dzx^B77(FfPk- znoVh$Mp0DhR7*$*XY-8~6mrVJX|G$g8+AiBYg9GC(#G5y@4W@FDg#Br#xzJ3S=-y+ zjr`!k#mjOy{o=DvDCnTJu`#Y!e^K(oPkP}U>DT)@S1J|r@guRMgaUE6?3<0V+p03I@M>7?4|IrSk zqFHlBlVC7OcTXmM&K{rkcTNu|rVNohF9(C3>sg%EbzQf-V7@Kizx^kkHg$C6;r5}V zxdk#euipI1&-~@jcJBh0fBSoXDywRVWQuTB3JR_xo)UVrG0YRn@vX~OBF@YO{3%lVL30UX*r`wba>qN!wf>iIUkKiMNtGn zaPQu|A}v%!qGgH%Xx1zBnhPi~b;UHbrKKeZaGsH^tt}Z*6$9I{Zr-}_;>$Pcm8zm> zpZomhWepXiq+D>$XVa+!lswPt)moVsoo4H-H?X{#q^S&XoPzLdtd@|GvRSYGMHiI? ztXCRYPF2m^ICq%=If#qPSFXGD7Ac4!Yi`ZX;&Gh#pMCPtER9RfDpmL3VDIeg)G{^8 z(2FdGP=c!c_x_*%h>!wHjzIe1$pB%^H0&%HRvXCl#57FTH`hzb!pX4P?>m;^)K2nj z$1uH0)kX+03JtyG&(O}}agYvI*Imn%4|+ks=~R%bPrpL^ss9JQ$4zh&0L~iIM^cRu-A-Sh}X5@hnwHax^R3^(GO3 zh)hMLWovs!$G3M6AMKt*VKyCPK`8mDon*YAa5(YDle3+pK|cm@M1WwVD0lC@4@$qL zy^n2(u@`n7b1>)BufOVK9d~PL>vZw9J z0HnFNF3I5W;WVd}cDprKuT?eG(%Z6RCP_A#O@)M1(;!eLWjQ%Jy0EoX zt+=u*6=}CWed96DNQ;tv0dI=$vQeBa+T)*+x z{?^|Z`h7)GZPz?^e#5dlTj!oxSa8O@QNbv7)ilqA5VKj37qr_O9v+=$c}xgHPz}RS zRn^18!|jKUNBx1WsaR8rB$jAt$k2}yKPQ9X@xw>=YIV=CtBsmvXgmv3)9@6{VL;0m z$YG(#hK!9cEHg$FRizYF9XBrvE}_|E$|x{QhoVAA%xgA>en^n0S+yX{V+OJ$ZdPjJ z;RInmn@x|7jwD&@_9ujyI}Z*G$qB>K)RCuynx!a0?H>25o_*)eXIxO*F=?3sNCjoH z*(5E>)%6ukC6cBGT*(R}E0ioFm$t4-^XEqqeYD@N)Z2S|hh-c;ezgDmQ_oonGTjR2U^W^@fNWe^Z`3`vX2}Q(A!tc# zqxRPKKAcX5wjs5v^?&p4{sF%^K*obAOIV6>c8YXqoXsGkE!%0h5YRBoqXK00mURE| z{Vrzq`)BN>1a!f44kZPw-iH;xX6*(5e~ZLA@L znJxhbK*yFRqdwr|qi?@)?Zzd)hy}MT%W5_CYTX!3r-I@%jS&JuFqDuo=QNG8-dVra zFpy+=PPNSZrTIo0n<#2o~;y&*XQR`&ahfc;#r!_bbx13EUOAM z~DiJ-0+_Wj-Q^(&X_9$s49$g{8%M3yw&s&PPK4ssIKJT=PFEHBHH z%0RExDo&-IQ6vjeGwf1Giw*UOD;E_R`TlfuWAWrq?}(}p3NpDqyA(%8VtVn z`@dha?RveD_!CJKTdS?@gPrToya;4m&&#t=Le6ozB85FMTeH^GdyL*{!Y1S1^__(iGD!c!^|LhPG+d z8NeA#HPxMErR*Wh>0+g(atL^c5DJqRFg83sIz2h^Di%jDhnhfU7G<&lW_f}og;SL< zX&NTFWzOQ_dvCtia1mh+Y|*HzPs%bb+?cs4A!k5DOz zfUzV?^136kcq-|-Y@4yH-rhf5J$JD+*P8hgs;H6<+7&jp)Cv445;O>c*q>5OJ3N}2 z2AFTPe&k2L&~AISp?jXXus9EkwBt(gY!c@Y)b;j-&1j5FOM3P7AG&v#?siWV!~U&b z|23qR?Zx(NG8xYLjsq6j#^~g}*PP3eP{xiYNy+Z6VyY3RWx%VAx-2;in7faA4aWdn z=S;e@y<@71KbJfvXS$B8nQmoqT8KC*Xvt>%uzPrBsb0x)#uySx$teW6 z48n-y(-_=8eJC8glw`YBxp)7uXIOOw-@0*gGW8di=5vxF8Kp%CAphi}j{pM%8$puF zilAIMI6M}D$%5o*@`bN_g^^jyOAqhgml3I%%A#vA!csrxRM?JDFuK3Lf8pk>z5Pdj z{QJKhrb*67Qsx4pG%p~=Ns=gvLI^QbLkO5esckxgeqS&V@#2k-KbXe!+~xBE%?xEU zx3Lb9w6S(B4U*l*2bSq6G+$d8wh>C`4idS(PXT9SzO*KVL^N|20W5P1*lcQr@R}rNWr{6KTer6I2dYj^uu3%b!D!7`;*%b zb`I`upB{CkN5^BsZkncfe!1${IG&cH<1=BU!%6y$Kl{VYw%fGjvS9&U$q^9m}O+nvyCDAW987?MBluWlpCNkVm5vjPh39 zKG_>!U9Z+FQ8;VOH%MLxuB|Vv8>+)4W1586a9Z={iXwY-xHF+)90rP}5-v_oPU0w+ zWpxsBv%26mmlXrW$rNFtiF`cjRqK_U(BWii=$79-gERrA}^}3yvmDy~pXlh9W2fW#sYc-lh95}Ye80inYQNaqr z+nvSb#g)Zo`|Y>iJh!&^>I*lkj?FX+nO=7mJBG0`H)j|c6EwyB)d;8r_9zGhcpS$s;SAR%W zHO>J-xZ2dXOh(~gkgM2Rdw3jk29A#RDwT?=wt#RaLxvI16o42-fLy<{@q=G}_2rkp z@8ws%+-xi-S*$Az(L|A^o168ewRwP$U`){jfW$QQ#7_XHK^)K`Z8xf}X$V@ZE-zIo z6{LB0cDg&q1Ixp1P2;ej44Jm0DhLXj7si#X7Z~T2YSl2!YgaEDsyUmEZ#=VRx}eok z9oMi-Rd6z!#)P85H0({Mi*t*XW<#0CXtuIu8J6ud79Sm)9qb=_&HAUGe&#rytm+%* zuV87^YS$iaN2h}kVf@O)3&X+4GBn4~u$&`=JjZraXiDPT%0fDMkWBZhE@d=p)t%ue zda!fk=S(QNSE)yNqFEZ2ly-YgLrSN)065ka{obdK#-o9(N?3!AZ8aU;)nuB_RG<}p zfvHp`xoLTVPyn>;?d_*;J$*8q0tsKZeEFm|#EL2DM#ib4;-{W^ia}%;hHX`S-v?NE z_;~mDtnVk0kf7rlbFFr@S&NfMlGSFbEo*v;^?&%!{ta~&RU{dvR8ZozssPHQ$R!|N zd*MZ^Qq?t&F`>v1pxA5RoX5yE-C#1VRB*FqRlFvYwYvwyDCZbSKq#uJDk`)L&9Q8v z;J7S-gr~7zcdL*C4J!n~5<-S>G%J$~#!O-uCxoPwr!Mu^CkWo1H~ zgfrE^Kll?rWmIe14RNZQ2I12n}ZaVEU zDo7N`Kt9~_OIdEH0+lnz0Q2n%%>u$=3F|>5I8`6t{unZn#-*zopdgrX%3z)qQJj`# z8AVZ^=V_9!EUxIPktE64Y0tK8gDX6a!@+QG=OIH{Z&<{GNgAg`S!(9I=B#}0-R-Y` z>wDk)<3GOr?%N-K{PDsES_tChIz2xCkkyaD8*NQCl6H#jdq@dF%Yj(wr>og=FxvotEcllD@pq zV0jts1r~5BPJ1v;s;(D?q1CV-J-pkh*P>~3d~i_n^xiNEr0wpG8GZZf~E4l&_szQ*>=SotTzm8T{2(pWWZzr6sv|@w@=wwbx&j z6y>weKKaR?{=U)iwhoHNcXkny?7DmTiI)k%I<^sFqSE$c1tSTDSvX5#&sN5>lyH)S zneRt&TBbQGn9u;05JnI#R2zm5a}h2WKFi*-pjqYLwSZ z*C~pqK;pF9heFDzlpu^akRh1Gd9&Ske6YjI3@EZyaRo+PNRue6Hr*u4KKStC6u@S^ zHSLZ{3?A&CeD!O88jVNUDoqVfXZn@4hjaOoR|0zxyr7!sl;2&p00U zdyJMJeehnzQWqEMw$stA#W&u$9i%+Xv$e&JL~=(r6F-v_H;f}s&d!}*RCP7aCyJ&b zgt9o-EeFaNbIDXRx8j(p6i3;ge&f5-SsF!b><5TpMUYvqerJt-Fi_*IF5P;M_nv=1RDJ;%>j=5G)#g_WS<;F-+b>Q z&eLk8GtJXkepYYSB-;?_*(_vS$UQ%s6(t6!-w%(*vxD8KDX?i8<59#YSkUAze&yA1 zn$a|cS$c8FSgW|ZpLHL7*xUa2WbbfN6hS^q2LUkL#&TyZ_D3o-!$~0P5&)T}=#O{0 zjyFF|X~c8LQAKBU)1gV}2jfiOB#Ujkf-yYUd-(b1o&ijr3{OVA)Ah~eJRc$npX_Yg zhWygYKen=Q(=)wqe&btTdi{mD&JBbl%+s<&pMCrh5qLJ5Ax&2&qd=A#e!!ZoI->6y1nyRIxB+ZK=!rWQzOv-c=#jaPgvhKB{CGV9PJOw zkM}0qj}A+uV%rKszv4B3Q2X7Z@!2-a2m8DCqm&rx-15d8LlKprs$o%NFMr|tb03E*Gb;I;Hg@jR3 zlpF~3`0-;~(;n^hqD1t1Jww(JmI&iq(2|lkO$0;&fFwN*PM)vB@3s{i}n_+OC>A!mdZfC%4@2ZKRx+%L;=adA=C zbwyDS;`7p-G;rPQvNJ!or2kbybro%eg2S6D&x2yN6O#NQ!Le zmLeN6h5!Hr08(VtFchp3LJ=2dM~4p|J$9=Ve;V%X?wXneNb1ee>S;$_hGeKi?TE>%jwjI93lzIs$^J>X?vQYX0cCb&KaqBO$2dS zkl|oNay^&=8MWfn7NU86>$%7`*H@mH`K4posw@Zo`0~~&6wz?lkNg28xozA1gRW^g zKvMEdNn^9syrk%wVHh%&yoyHghu3M_3;F_jNT!m&*H#R#(I&L);nkTkp z?H`-k+up0! z+C}M060cUYiQmJ#nwTvrfa~huHt|)9!0T+)I1#9 z6|aC&nG*z=BHMWqVYY8p&~5l9OSNI3!B)3ZEBj~ z0xX%T)t$vf8mp>~G!1IFOt@8PNUEAn`lc=^mZweyFj1b3GOQ}5EaFUPQhzd#Rg{*D z0yr8D=9ES}i@x!#H!p6kdmWJFG!2uB7cM{Ex!-OqB8Ck~AzbI6D04a+O%~=V!|qvs zI5?XPE^b{o8;y!&#HH}>{GGq6DDJ}i#XFDRymsrAYuB#*qksNCJ#%5hmNo7vBr-Y+ zor#||YBfJ0FF*gxG$%B{-$!K2vJ`Qq{>LyF~9rxU-^sShIG4+2@ik?)^h zU&`YkOHi1G3~{VUvfwI(aTHPtUOK;|8`{Y@qO?r2NK>UW&l^_7FLG3na>m>e<~krz zmX;_d30JYqibOG>q6&F1qRB-Z?)Qg= zxiCoN&YEM|mI6?j_*l}eY%WN$>Daa-YlIYG9B7srB|H^b9EJ0X3lJe$F{`dd0R$jD z?(ShuZe71A6_MjuWgZtqPXP5QcFpt}_3GilX+KC-mpUD%-d?uaP5u1FWhBXN(^)^i zG@hM0mU6hi4^uCtq&FE2hCz`N1`DgIIId1Jkr%8eX-NyAS!$&!u!xfkE7)`m(@?7o z11kb4aw*vGY~bovn1o3b%CaiaBBu-)mRH+&>)l74^?6N~vXt!Yomx=DQ(u;43dxs# z=!@5%y|uVpl|*rVVMPKF%O?^oNd`GnWwo)pf9krfttwym%2(PO7sFEY2T_`Wv(bnV z>Zlqj$vBDwLW(jXBDZZ_qtmKMu3o)F8QR_5?j4_|ah61>t|`a;zNDHMvMd=;TIOlI z``H`8bfjsHrI)IP>$OU4vGLr~&rAm=bM1KrGnGRPrK1VqGJfvXkB0d;i2V{_&#;=U zS~McMCNIs|oWvwcY3U!H?fP+YVP$?{sS;=Wr+(qTfKn!qo?$?JeqnQKp-F{yAJrSHVUAF>ffQ#J__J^_ z>K?gBSzDM}n7^p2)em=$Z#{Qy90pNNn~NKJ!)S0AB@tY1uQ&=4WlZxZo(U39F?0}v zpL_N7THT(Vb_cU=Zx|`A{@H`w!_)rm(E(JqHFhG&C7Fp!9pK^0(w z75OA)8NX%Drca`xeD=9*RMMU> zzWp&m&Sa7(vQ=hz5@e&HpG0)#j|C;;{-8UaZrymIrCM2*Ikx`f6W37&`rXi~uO1$s zVgoCB!3)L&$RV9yXfIUfY^D_yBUveUNlKBX#bh!`v%E-H-SSvT1m~mi7*b#=T0D!A zB>d#lJG$Y;Nf<>DWrAUq(=v%Cu7+qjO{XIS+az$h$ot$-#y4~(*G|~h&RXiDN>YlErcK6!ANZlK1mTnw zInBNPSxyPQaq(Hmq!*uiDo9QiR$3*=xI)ffTqB%6KKS;z&6`M>3xibGb%oQ-_4dWB zrDn|v{oeVF#migka|_FH=#M6o_Tm%O#=^J1`Mol&H0q8nV{1Ww)H_ZjZP51t!+-v* zuP?PKe!uT&YFdzx;h)k~AGo#w+z|&9guExi95O z&PAp`*l=_OW(7$l7Oyuv$km5;K9xn#sz`^&k1W@o^J+;HQ_K;-xZm$mf*C}VGcG8W zz+bp24MVdmLsl>p$e&JDmKOvkM~7WiH{vXmSkbCF728yxECjI~eKr|Fz&ov~XBu%5 z%9<=`>R>V!Sk#&=fD~1;u%HkF%d^rf&eM2(b*dPUqt0 zxs0G6{;{9>@(=!yZZ|Hi&0kuySa=jifoZ82O0&QxlrjO1509B(6~|H(31d7SPtzzY z6QWA8tWe98C@#{xNTMu>eNKhz)CS{erQUhs#;q*N6h(P(@6lwE>>ZvZY1;2ir{g_Y z${s!VAWHQMmtRaGR^~)-X=P<;8u*ssV8O=IiK@w#V-`iJD!Kp}DGLe9S(e68H0clP zt!7b_eiYYgjUtBH`IowRco&Aj}1c~_Rkg<+LoFq-B?1Yq{PZ@%rvX-U}i>(^`b&cfni7ze%6Jub+9_jmu<6IWkS zjiS@2`r(WbVwsHx4^9kl42K1d=;^K#VhRc-zJ z@?}(NITN~RKt}IBdguq_)6X6Y0TdakC@b?YD>K3XDw(Nj1u1b{r~)lk8ka9#Ls*1i z0D;(keE;6V`z|8+r2pMN{zJm2n`_Pg)8GCptyU|_vYKs@f?~#w4|g!eLWqi2>yKyu z>G!_c8x7|ga>Ju(-I;4>x&#@OA#zRY&f6co{f+M$B@!yXaN|mC zZSnWM_9t_#w(FVoiX$O$jri$MUoZU)MY4;5F+Kp zD9gub8E2`hBFIUn(U74809Q3xQ6vQ5Ur0;ZwN|wYdt_;b$z1qgJ(AGp#bqr{hTy#h@TG$r_HQ zz#=7CE`Vy-B`4o}@6GSN@%~r8`a7+Px_5FECS%!_+toVHGfP$_P?j9zLd_y_BQ!VZM<<0q(If~<|%fJ7nYn~C_yga|Txgs<(C7ENAOXoLhwW{fu?d1il zT5T^aoX&#t*Drf@XTDi0NivR;-QjG%KZJ@2goL1acrZA3aWjJe>$Zg37}N-pNRB*B zcNF*&KmSv2y!D4<76BSlOo^->506upoJ{&aFAl4 zUS3$%WT;h5SLpV=odGl|{d9UXJfi?!e(Dv!(4KhiOEQ;b%Fit{Z(Y1-s7g4A6r>A? zb4Z{pl`8tyN4LNE_U*VJj1gOs+lwM4j6zm$gtVGsSB`d%Pj-(-{Yl2x2^YF$x|Tc6U_OZ; z&Jk2(EM`GMgg~4N1zM`wa8*rf4##J4UTB)(he@@r2T^Fr8WW&abt8XfsEVq&XHnt; zY#NHH7D90jj}KgXuH7*1K6r?*a&B>@h({OKw}RkIH?-4k)E)PK>`Si{jQ5ArVXr?3 zGpa&A&K*rwb0Jq;MPjog=6PB7JljCsX_*VvV65atnsQA5tLar2=Eui}<4G7LMQ6U* zsu_7o=If~LG~e1eUY}b$J?*KA-n2oQ%_y>`R66bTUw-b%Z+-jE5#di1Nwv*laC$$9uPJV-cVXOV&Fd+^x2o)r#Bg zpH(nY%w{hM*W0z(tSG`+y=_4tGXXe&C4)$nC$nk0X49NOh%L*w{`9Ng__IHDER)8A zI2B&CjxYrP$&zt;(lssJlHk!g=s0Jsw|<5 zcs0$BhEQT>XJ@)<7z)lap)2ZK!?rb{Y4YC5QFG(Mrw6@KF}zCS-s6Mq-D6Dw7cXr< zp1Ba%s-o#iV`G^y;+R%VwyLI`Co!c(-7?IwkePpRwR!nmLuk`1N&Aln(~uipQ#LK3 zK*g=M7uI+8lQ%wkq~&-z85lZhx4nrU%rC7ex?TFgTz%nR|1bY}zTy}voP|Tf(rV3m zDVXY1YK?l`vlY3Prs{*83Dn?VxVQ7Li#2OJ2z6*BX+HHQmSZManH7w1cHzQmk!6!v z%o*TJA|-t7^{;&AJAZE1D$5(UCR5;5774UJ`1C=Pk;jj=D_%WIDUgj4&;pc$-q7_b zrswFY=Qth-0X6fwo z^uh3C`*Sb7eA12b0`Ke|W@&LWn&zzVD(0g*_uhZ&&k@T`#(`oPc@c6LHS1ERWmM+W zNj6R)bE~yRt9{|ZrTW66YM46YDpG7&5hT=WRjz2e`=^?+-t4r)5bYhFt*)MH)SAJJ zYD$IS3IQo01%Ps=qg*`SoNqdog`RryimC{jXA)yo%c_{QoX~``tPuAfY$JgYO*s&X zW(0YZQvTonkN*rvxII@H_V$_1zVLmo>(J9g?rLTh78R>D^;5Uz$~3As8z0^M_|m2I zvWQ5MrD2XGH4Taj6=|;KajYW8Fac$*>z+SkiNS z$-M#KB;A5(Gh(7Dd6BaN>ZJ8QOhx|FnCO$B71oqr*nUL6TByRGaO& zz2k1RKDWBENqGZPkC%FHJg9j3G!A9W)-?0N*4ER{K3nGH+3}HUz&zucswZgt2jBee zr`w@GFfV7ptlO#E$A^1iT69M zs5Y5os#<&I`7e)yupms4wMKK{#IiZUsJ1h|Nc&Ee+BsVDXLoTh1h{p-JF8@VLl!Trxk zB)$6ai7NP$o~23h_03IFHXnWZxwpQ#z1{U=V%fbP4-a{sZ<@+5EFOyz;`axm)hhN# zg%Cnb$q-tfpUz}ic6tV2t0-wO#3dkx0B{rWjPm5^47KS z%pt@wL0o?H_=D;3Nn9kH$RbHD;sUZY2U;KlQ4(q1#&yx6x<+MRP(`iH2t2)fTjZ6+ zhzP>z^|gYT3MrGp=$+lWUKrJW!UJx!X=g2;Ef%P4`;P5cdRs#QOCx7^a&?2TROaRE z^a9a}*VNs~!_zm1#HU5m>Xx068Z>mdSn?vXic7x`wkF$rN6jmcvq}&pEzinK@!S9S zcZaulLswflih?)}5tj0T38KJ@eXivv-Oe=jHQg#njd&&qNTYIHOK;!NXT;5h6#@`WqI>{f1U)9MYUbW!fLdYn{%9F$)uK6}Axb;L*qLuA>st0zwwR)ENyVfcoQ6mR9qtv!DI^lZ&hA z;&Lv?c1L3(KNxqd-Q7E1{o&j3#Ei>wwBgV)PFG8g$z!jmT6KCk15B|y=zGys%`!mO z2AiXApM0-tJfM^p_&W!KG)+|7I9UcfMope3rGz!UBXV1#O_E3BkzK{B(L_JLUaKgy zJCXpk(eAC^|LG;G0xyrA{`fu3)`Hsm{`pzo+yqi*?nf=xk~~aXM#bDTLSYIkTC!nL zU`5DDl%~^`E#}Mc>^fmtq>FV`6&UL!&)>KnMo_SkEX#=>y0b+ZCS}%=I>0=q>w3eq z`v=?8^_8F)Wm>0cTjZqcNC+{UD9D25ds)o*bxo2mY9SwmUYL@sSa)T`lCdtTJwxGT zB=Q5X;n^mi&aX34rfHRtczd#2$7_Hw5@e^VB}LJ)4XA{TjZK*Yb%QyUHbu#_93bWS z`Lu#Y)hZ$@Qp~fGkQLvxRaFNlBT~E)w zw5TAGcnT|0aIm!5h{=mICbKxBQOf}?4E#LybyX5oK{gEt8Fzija8kq8!!9^{@!iG6 z%7hwhXiZttw1FGohG`Pgv9;;VRTUYBo9hGx`dZX29os%;~wrupE#$FDC= z8%|<@kQEW)8`DyL_Vf4e-yMJP)AvZ;G=Pbl#tRAJM9c12PTv~#0BErUmaea2jD@YH zg$H-Wp%-rGrt7+^d8LlGo1B+xE{g&|0VBs+xtI4p{LmO!$LCj1U%oL!tnr+p+xH(l z6hSRu6}Eh*GpdSYG}?8vZsdl&$pb@%5<|V^${eXJp2#e#+v@Gxx4RwZ`SB_&sLHT) z9@nw|!NU*zm{JJcXhqzRsb~|IErmmEn3weJ&3svt3ILY2b=v|-QUR+Jgh`XK?>J>y z?%vuPST-#R5+ztNvLx391F%R_C0$o(ONt`LNCC7}Wmy(P86wXMUYs2hK#Qv60jyf! zMnO{KapEz+Eixu1Da*7?N}#G|1 zU)|anOg60|3ClEY@)Q7|s?uyWvpOnCJyGHv6{ca-)Cd;7%B1n~Bx{HyqAK&9!9Wyv zP2?@jc=uqNqbtZZj$?-bQFx(P&HD-}>msZ2C|-HjmphY5Q85AnO+h~Ui;n~bjz)uq zX#px2mKvs6lQIqBo6GByi%Y%J`{3~xYtNm}mYr^YG`6<4dsz|Lc8`Q{SJeiCk!d)p z-X&G-h5>0?Q!^1@O-;{YSQFvm#G`F}a&lsK%*gj_T~;C3wFf$C(_Oe(`NSq{;*djo2P*DWy<@sQ83k#?vk>&L6 zKYFJIRY-YG;syq@w`ECub9-YX%2d^aI8IOA9(61uPQr6Pd2updu9}#@%jHTP^}l^~ z{rcE--CSX^%D)H8vvtK&QPx7bUS^U;n>mEh?BJn zfTRIM3(~sD>XwBNlDwSH)~=tikneV!%hMB%<6gZvp_KBN5k)ZwB7tW@*DLCVrIjD~ zot}dQRu&`yh=fC&`k~9cgK|~y4&oAaIBi_0FAdbqOW`6qq9aY4% zDQ>27$bew&nx>8=Y*;-Ji=RAxw7fduc>#ddvh1(F`exkk&8{vVJ$!d_XS{cC8$d=< zl$thmT}!gcw(S3X^~YQLdr_PxF{z1o`g(rv{?F#CV1MrxCE1I^=c~1cMPq3z#ZU^{yj&U@vo*9aha^D&E&Yez{M~H6wz`9y!w)|F?5Dr{RZaxk5eH5YB*`~_ zenu%%(VF80O*9lyDe6iwI;5$#Ztd}+F&OWp1uaYI^YyC=`0Ac8F;agrjb zy4htBpXB6fegin3Lqc6PVVZHG=(;`tK$3)pCXQuukbn8=$(z$xWs^vp#9AY zqug=&!|tdl%CqA)AAEFM(U_Hcx$>_AvJiw)SB2Tl#qs%B+f*##TgVHVwwNxGbnRGx zLke43W4!)o$aBGoYzh01?p?cn0VP@MonCtiFNE&;^7MFXXHU}f?|ytrY27r9uA8E! zb^4AVqRGasBrPnv^Zdop$B!Q=nx5r=g}fm0dpiT$P*?`0amljm?0UVN$F|;abWxXu zje#?CI#DL*ABuV1+;=0!#E~Avk=w`N3qzdD9w>NzK_9Z}#C}9MpToB^KOG%UG zxg?8i%bI3a6a9)2U!1T0=9hmJd#l$kpXK5F?(X309}fGn5hN6|_)mX2lvGLJ!N|7K zD1j7~Rod&@qw!|w%P!^8JiE33MJ8oe=WigWERiFkotL#?kEZh*Rg|xf-Zqq;o;(>0 zdqoMH(RdMh;mU8z1lHMl&FH4tv3qe6IK7S+hi*bN9`}g1H5OZ^K#0t2os>#44MMMD=6x~Z>sgX<`)$v*p zG=a|k`Y-Q+j1AIeytVn||9&MhRH%Vf=M1Vu5ln*xmhd;vP8mjOii|}S5AkB|K?TQQ z3Mrd9rDfR**Yinwbp8rd2zB)HtJynSg96l5NiZ)yy0goGFmuUhXD0}+*|hXyhU0|e oi?g1iOS%-NSq5=kBv|18A1*7PqW$+~&j0`b07*qoM6N<$g7ASY8UO$Q literal 0 HcmV?d00001 diff --git a/onnxruntime/python/tools/quantization/notebooks/imagenet_v2/calibration_imagenet/rose.jpg b/onnxruntime/python/tools/quantization/notebooks/imagenet_v2/calibration_imagenet/rose.jpg new file mode 100644 index 0000000000000000000000000000000000000000..5080a1c7ec43d600b5a984db5bcc62735834f1ac GIT binary patch literal 52997 zcmb5V1yo#3w=KGHhoHgTEx1E)cW4}fHNoB839gO1Lx2Fmg9Z2C9$W$ef(8$-^L^)@ z^WQh_9dEqq-o0n9HEYdPt9tb4U0wTS`DF`ur6{W)3&6kt01Wg2y!-*sC7hk@!7661 zcBn5q2m%UHQt#9?)npZvW&WWdFq%RAu#^Dc;OOqEDJMy(r*A-sGz%aA7yvo|ZDtm3 z&JybC%Ku;}_wV_CZ@(A+Y5~9;+uvCKyYK(=9fm7Z5SlI}l!^ELU(E9toBxaD|6(t9 zXLl&a=r6XgHnW6cXDDWM)z*}P;#?@keE%QV{6DZI*v$gUsey9dS~$BxLCxv1GOE(J#D2B#_jr&^*s14fOl>k7D3;^K9|Ho$F1^_kX0Pvdr ze{76|0D!Lx0Bzj=W4qG)r#>(Q-`uRYx&8?UgDwdG2pcaiSM&gYlmYLZQt;s(u0RSk1&Vw#004PNOK=w8OFw_A6ZVLdYV*r4*AOIT-E!V$? zJJ=gqR{)T5a`thxd2j7b$;H9JMalN}#7+6nr;$?1$-&vf9qdY};Ap|lN@-?qPx%kk zjnWnD26puXTe1TV_8!oA`=?<+U(vyiAT8(_S}#~Du$#<(VHjIWGim7D0PN5*&02z` zWuTZ9ij%BtWEG&85qb<*xypk6#V|NF?h1eBA8LmcbF^1g{x1wTI;pBbF*H7K;_NQ@ zw?0q|W9#Or^*81}c3U$=btoo=+J)?#l>TxEpm7*|t+k{uk~9^Z|PS z4A=mU&{s!*63_(X07-xia0XlfC%_8ov4Jvdpw_>BFw|-TxB+(mN2WcX{@=aqp$s|T z2-@f9pFS>t2ekL!_&R{3DiF&0-w`M|mN-s17C4GHN;!Te5qmKHk0;diosGq}#B*re zzsi5J`5X7&9RJ4uo72DXEuh&-K)v?Bd#Kd`di*;kxBup|{bzPoj!DG-?V z|KFqfx6*${^`Diu_z&KH)*Ae>21o&Lu&{rh&;}2EA|N9mz{4Y;A|W9nqobmuqoJaq zVPN9C!obAFL_>Q;@Cq9j4<8>N9qTnA0UjX^9zNdRATV&yKJW-A2nZ;67-$%H|G(Qy zFMy2FK!C(Wh*f4O|FfRiD^ot1=4(4xze)9flh)B>3;ZR^<0H|C06#xqZ3l9&6 z0FQ`(jD&y!z`(-6V*?0?R7g0S61W;>$avH)Av9dcMfj4M4PVc>&0S}<326DG+(J_t zQFwT@+>0;h=siC7320k{g{L<4&qAY;!u*%$|I!5D5#SJ!ppwW?uM{>^1Rf3^837h* zMu27p)db+NsSu!w8fGrI)FH`5P$@(_u9~EjwvVlGciq~zENn~kq^WO=}a8&h9ch|Ku%(v zi#{Ew_WGpOljJoJr;#WaOahByi$XdYsPEGS_Ah4iD3fI%CtlQ#%WHPOX7BD>QG;nU z8>nY-s^K&!l2xk>_(@>=OoR8$g>(dTHRqNFwYGNGbM6h%dSUTT7$HdH4 z?PvLrGeA)-+0IX5gb|65){irPtUX!`QkUbrE!B6vzg#;qcCBt|iONirsopp0&`prD zpy6j>JH2=plo9kj5n`bzyR)h^3U7bA21EP;h#wGVFx1qI!)x*Q_jxOYu=?*DZ{c3B zeMMYLXrbO?A2@#jvP^3cujd?O(=M>B`#3!<-$`-fs6BFY4&!~B*_#+`jr&5pw8vX5Y_WIAxGgpq^+{?icpwBl*)olp`IujhM*$T~00 zY&+Gtec7V-RhpKS-W?)wj*Dk9lTn!(haneqRDyK+f}t{kdF8n(%GzZl+@BS*#-(#m zPhk5xZQMFKsSN8}KY5H?5mkmsbnb zsK$mBC8)<0N2Fd4t`UDeEC$}Ic4?$RQn>ha5C;sWcV{6y)r99FS;;w0b<;jlekzB1 zLipx*u`oZwazIq}{gkDP)IS60dC~lSt89NYKw6VL-7f92cba}{JT=_au{B2bt8)6+ zvlMBX&Wk|kc@9=rPB za$OvKv@A>>`O>?Atsg--SVE{Fc&3r_~HDmG~ERdYuqA+#gX4Dk-=h&=jL`beF4y9SZv+X z)SmE=DXyz&G%Wo^Z>uaTE9bMCa&Pr4yw@%Hjh%9moGEqPA2f}@?-ZzrAFoDR3+@RwuG|o zp(JXd$D6J>JrWEn8;pv%jm%Cne<$<;P`cWa%Z9Z6ba9Dr!{a6G+_`10iDQTsd-uWP zhyV4gw}h?8Yv)GS+OyWz`%fbEJIYA_JyT|9x3j#8%~{vQp%(#9y4feu;$|Hs2pl!( z;L>0cxt>MW=v!*l?+OBqwL@iwQx1k#^cVMIc9=mpL*x((^LW*Pkh8VlvM0a2x6?s= z-x(0!Nuk?t>&KwNSvcQ_i>f&md;IZ{TBzJq9 zV(H60a=l#>Cn~c!(%0Gz z$_t0E$}N4GM)E7J9-!0F3#pBz^J7=4`Wf193 zw$-`Pdx#Khr6CJb%ETcyz-1%y?PsFEy;1rU8*Ry9#N2F`X|%qPEpP12_1+l|5?l`vgpXv@f(ilU_%`7)h)dfA4f6W zVK>SuO8ZYegrGjet4R>KVfMJf7~y8YpjBg%%Kb$e2`koC@4Vt5nnqkrL{Y=G$ zOP-~>6#P5Id$=d((RL47b%t(Hl*&%&SWy|%WJEpR{PzbmIO(35*ObUvzs`a)_YR|n z2l8BA%ks<3ue`Di&eGQrbD>UajLjROlc2ttrB!jqKF&b7k&$hj*4MA;%|u_{s$)cgLAxiPYbv3oEUiOl@{)jn|* zT=P+Rh29?72il~-$jyBGb`JG@l-8aJ<2gop%Sj|X969G!d*lm-u%zROk@r5d$r^|7 z2JR3i1cNhqJB*wqXTSR$e%iHM!IYZV&HAg|VR?tJ-H%Bx02U87?@r$QcE84Fcim~r zPbpuw-?!UyR>!^oIkUY|FncEASOv=d;7yB%t&iyvCUfr?olj=1Ao&V&t7YCZ3{zsv zZE;xvq-laGyzIIp_6xX5gyzEUQg0FdJTYYzWO*aPU{p8@CO&uEAs=<(b<|(a#)jL& zXkXlL4l7&gmzPh1s&=wltiBMBge<453jZoOc`8GK| z@>wP+`MEH(q;E512)$Vs=8AQV0SD|e$ZnPyA#$CF9YHc8M`e!k^EPzT;e1q2iCKln z{a^<`f-uWD_)(`X4b&FRW@ItR_F<^mTCQL41rGs_7_-779U3r5Y#cKKR zUro8>VrrN{D%-O2;ZoW@_LnU9ApTz#hmUgOi1kwA$Qs=vbr#56mqVr$6(+L1w|A1d zQG&Gr1TCX(ih7IHNgy`XOR6@RjFVBC%2omX<`B4V=xO2mTA<=8jg_FF4@HV z-)W~WL0!_qyf^0R;2Ty44|t=~o}j}y{9`L)c>AKcl+s}=i<+(0dwuOMcaY?Y31*5H zV3Ab~c1%41TlVXLn1`F1{S=b((@n)dj-7lx-n6E`du|+1nB(Hn(+j}g z%g(r%EYfkAO<0;DIfB1@vm9$=5nODRZM>R!pd;Bj_M;m*+O^3 zyEBU~K(YXe-GqvlOf4H^)WZXwf}$acK2p}V@=1=`Dh|yxbiDr872G%`odV*F@qm4H zsXuu*On9>1Kuz;>pJcN*z}FI-_+~WYwQ!&P_hgWGFhGl`gn?FM#BPxcK;zt3#`5 z%bjp=Erf?%$;@nwLl?;<;gTzSZgpg~!r6w>8CDPKZuKRIDFv+mm!8^SxQQ*i zZ)I~!M2KU@0Z0KBTWKLp1u(|}RLN;`c;|~ed?umG1zOf=n!Jv3FMyoRr||w?2IERI zV|8!N+&$tXZ`K^*}z*br6E(A2Kd9vS!(ASn* zdYDjOf6#6?;M6ZkzPy$030Tw-Gci6-egPtBztUxB3x*usM%= zshAbbzMuCsyzW+YM6^p^5Y>ipa5c1`74>b?XlM}7?qqgU2BFojGmDCd*ZshM;@~$L zq4awd`Axv1tT~s1RX@*4ii=V2pz_0loWmco4|IXLqMpKAE369lpri-Ml%Q6Pp`4x|D<5!=poa{H=EUJ!GrzjAt@58F@p5c(Hj)XB)eaBxq@+Gxy+d4vA@5P~6^ zlLlH^LS1Cp7N-K~*!m~jDsm z$ituM6e`w8t7?;ZTN@xBN1DFJysIu;z@6rgY;>dRAYaCa zO3Trq&4Nfq)5^rlw$xN? zt@Q`}>s<0kN--b@v%?b|rXWJqiD=8BG@O_PInV9QfSFwP-zCJxv_GE39qV|TZD-ujiRVjg+Q0SQ&zB=lPbtrWlw`uv zB0lxT*Z7^rWH*;SAfQDFM3adb_sTeJ zoy+3S7A!VXP#qDsvY+pxu0N%d-1Oq37R6e`QVVJKh=x#J-u?;9BwCeguPT*MFO5Cz z`lA;Ib0b}3B{rBc>6e2ecXc(;RHI0sQgWN6pb5Y((I&oSb+h}B%GaQut7F2%>UkCv zNzJEXdgOhGUf{EwK|+7NH5yam z*iY)v05(6bdP0OO_%Lljb>0S~&+2|#!<;$MX7W%*+MGY_m4Zgw`s zydYUgNXxM_tOlz{RujX~O8p6G!f7s*Lk<|92*aoe(olmtY!`B?@5hyY>_>6Y#}o}# zVuf>d7k)lnH|tMja-p*3CBF&pA>)>^q{2eGEdlep#vM_{LZTUK2TIb4HG5KbmDSu;W zpyr?|)DUWY!^e|X2k?R`QPxvU5uQfjett+%pH(OWLX9+BKpLG64oZ-3`LG(#y{Uno zNR?ON{|<9f6$F*U5+>P*sh&zFMLP4Wuh@? z&{{o7r5ppPlfFi7n56Mse&Tz5hVd-46}hOLa?)4TKmT|dgUB}!H5-opIM~@7{vJm_ zhsV)NlwGch>r_n*_mX8_lfRS3jq6f`*3AvmcmllS(ni00tQXx(*CL^>6PkS)1Dh=v zBh4-lmIUfj&a$c-e_uSYfP7KC&2y6+Du#Oj=m&arC4R$L#TpTW@Zvqw?`|@}EaJ4m z7j0n{8^v?`*D12@|Dk`TK(>O+lNbz2rnfu=Zw%}S|9H$jZw7U;m8l>$G`#>n z`X(^n55Ap&jEsZIf3RF3&uUWz=k$fH-Sd39PP#YE^0B-=Q!U?RRPm|w)gc+BDQ8k~ zEMQzRG8!Ss7um=rY3$pnW4YO5$i(Wce=x&j+`OEZ9Upo;zh2R>8FV00%UHz*HH#7T z@b!8QsxVhk^Ssv3wD`^1KNJVBFP<))_XCney0afM+vipT9i%d8B{~_YSZFV?ovQ*O zzNAP10e+^DbPpi>>QU9s-!Nx=7vrBp(r^$IW-;j}o6h7&-4=R3(Po$$OYk98(HgC9 zjyPNo*Qs$bNu)nKuG)2XC3WG;hFa$jIo?FMRm}U<9e;(Ac4oTE`ivJ-%=m+ z^{M+aWogaW&!dC#^nBU#S)D^2NhL(tGcm62Aj~bZw0e*9lqdwQd8ajvL;1=l$tx|# zax(}l9>R5ZICl5K=_2x7M3Jj1)m|;+7`={kEOUJTOyIrzr^ykEi{Gz`ymb=;A2N_u zqwNy2UNWBHEHu{mF{*=H7T=d9DoYLPGTPNBue_=pUri)+0iSDP5{&QxcvV>)W>!zX zfY@~JT zPLG3HukVbj=3kdi=^}oc^~O3CZgE}2C6#o0i+s7yDT2=_o~$iJ*DuFvDm~VFO<6?n z=ZV3LjxZ%;!`}(BfdTKfpL(7O&~j0MK}trf_d97&XS$9? z&cKe|OOhB9XVcly?$1lTUOWHJCD=`!(7y=RGKjD-3W37THz-g*ebV}h7vUzs`ee~O z@bXL~QCS)SqHb7pQIwlcfe=s{0jz$$fAmh7S6b*KTZWk`P}$|z{5*!4dDh4f>_a+V z03KL+)Ige_)8+`(p@clvPN;DSkSlf;@YE+8*of%UMjS)8EAyW7-96$${^8SoI5xSv z1`jV3+x}+#FGu8=Y<%y^k0KCQ12GNLVPq6gzUR^pu1#hpSTR&FyuRgXerhPTr=d{r zwZ5(+uAOR3#QmH!jWMH~r&=|-I-!4A_Eb>%dUn&*yo*K_*l$JB4D~uWDa%mNe*3}F?yoD1<3P607oHsQ;;1s871|!KKM)T# z<`x<@Qu)DWvV1epN?iTEHGI85&MeX-w?Vab_K8JWIBfG%K)tyUX58d0;XYf)g70we zu&7*8%a;+uHx-=d2Ldm^?*Iut)U6=t!^XEtzxfGH8<1XyUBZ}(Gvey7?8@h^|auOrqUIZSW}j2Fyn zoXM}f0HBd=@9_qk5P=_*3rus-Z=HlCdtz2Q>=Rm)J0jIT>(k`xh@2R`;a+_fV`zPB zvj`hU!lBS5_RPkD*~K5t;JQKTv0|_ZTAbQ? z>trpAL-k|ZBwH^jESj=!SJm=sZz7nOkRJuBY4>nKzRPKrOrT*2Zx{9*km2hiH#^nx zC)c}GAeV`TSY=44#MG}q~c^m%Epx}gk(e(lbvi+toW3jF#BM(Z2HzR8cFV{O;+9Mzr< zwK!6maSs(9EF+u7lf7Svellv~JEI#y5ytE9&UywZ30^z1MYwvWF1vN1jv5-m%wv+a z#Pe(7?0UZ*|HNw_1QHqkta3i}0+ig+eY^I0-CT=VQ6~SHcW&mf+1;Luxd=ZQ=f|$q z>%F5J!Y=tQ>|i6K#*Wg`InR35Nb;xLX{DnE0`)fatpMI_5(#hGmm$J4x)Y%0u5AZ@7^hMNdDl7c!Q=xhIfOvf`xIpLMcC6U7iLKUftkM zSz?&#hsYJjdh~b7{DPQL3FBaSSBZ>XhBrE}+j>Y?6%xA=w0;s$vT8rI0_!SYG}u-b z8XMR6Gp6C_`2aDK*DZ`ym~5sxdsZ52in` zKm+MGWNWdpIS4GZbO_Te@ZA6QAW>JPtqhLJj!$H*EQOnNTGYhacun%lqJ^zSx#_Bu za@`i=i+2J|%SGIzd^_`3Ecy@w7IklD8f*UV8m}LAc@l`>NHtwMchk4dV1C|PRHfjW zwGf!aF{y$i`(D>8kJ5;?cT_dB25o=PP^Vrm4Q0Z$s6tdV?H&5rW70G7m2&l*Nv5hv zhR-6MO|e5LR1c@3?TYZirY^~rw2X=9<0?(Cl>R<`isd$wo1i({$R@f6{ddCO4z-he z2)`SEU=k1Zyky!nSx=P}r!#oZ0`rRk(E`hlb1Lj=Sz4n~nKP=>^IEE%b&=1=_Z0oF z%+qj*zBX2x>Gq8oa9l8-7q8LLI=uc=V{_6{sz40s`WD3fM?gP^qeef&j6gxFXQP2| zuX;XYtX2vq#SYH1Z8_GHp=bYo2R^^OZutXow%m3Sg2;r^VKknDUV(@%2$ZAuJ^1Q+ znEo~)rlps!>=O*3o58c!fMpnb%N@4E$@daF-WLFmIwpbd)~K61QwPLc(2&5d30?27 zS$3^FQwkhy@ODp4<+f3O(5kv^VHOx-rsI5tQl$l^bkPtxhrOuwFe&kv$VQ>HV2F%c zuc$JcP}@-@`__{@o8tYI3&%|5J?w%Q0k&CxHr8d1NU$qnS$r5rJJknaEaO{&P<5gx zKnNL^&}*-3A+X^*rjGO2Kxl)iOn8ilN0nk2ra*r3>W-~P}9Ln&Tt)f`hI&ab-xi=$V-L=I~A$Al6l;M2kqAfJ=r=D6HoX`QRKJE z-0GF=jqWSIW9Fx^f>?hxz9L`F0JU*VAHEOmwlf;Qaz*!!Y)15#u&%L6wDm?5r<2Ys=&MC}gWWu3m_q5IMso+z zEc_FL|9v@s%^oX@aL4!kKA~2aF%pjZ<0T?mayqvgik;Plp~WnjV6hF8>#8+Vv7XJg zbfkc0FEt}-A(lAOrHi^vc1-o5zilnp9v()VCyLEit8oxP0BjJ{cKbc2%3^wRX+H|H zN=-12@B8>hLm_{DnccI&`SuuFv1 zOM5uY*Nwx>L%HZm6W(YQ+}}K=ths&1jIywz>F|w|c(j;rKoDe?j^$Y16Bmxu3<$Mt z%WK$XQY^CT8fsZyLG}{XE7MQjw2W?uYj%~1xnuRCJXKS)kP&rtS7u6K#!I;IIh1Z2 zEyQR(`qDgwN2y5Su_PODVW)0(E(4D92&Wdi?@IO({gIW7(lA{6_!`#jybiE!`w6mb z)WSKd&%f?X`;x?A3Bs+)3;lkZNAZk5QpvX-zLChreRVp5_4RtBL?|f zkh`QZCd$9pvb>MOnFTi=riT`r{k9)2^}nt6u#4r5CZN9MD^*bxN+XUf-%UJYn<7-3 z7*}orZ-7vy3rh2m%(iS^1r&J4)921($Qdb)l4{OGIu`QQl)eBCuYX<*r*Kn)`dH?j zy?^fFwZGQR9;;b4l1&?JuJI}5Y%rrVbNIF&3rjEgO`w~r*HB`hFC+Gx4YMMlh)=N+ ziQ9`kD(wZ>-+SWJ%Ttr#kbR&~lkYGaXHI7n#tP}lGxKA6r^zqsyE{+`r)iNff_VP@ zndVn6^Q?;k@mHD|?A=tqDo<%A@z=?T6Wbf?#Xs5jWSE16udhR#Eh(}+rfp&B_XB4g zku2GpHdL}~JBK1;nV1Lyq;|xa_VdiZ`*e_gPQVrav)7#ha6Ra#slQTzJ5xz#3fS3# z%N|Rl^77J?B$s$KXMa=&Ya{bfhXpuH=#UY;g`*4r%jAz5BZz{?1k=l7StaMYnGpOn zXb>p#&CbIc>~exJP8}4+i}saqK`_3k^uO19^QCNrOSrS{lE0;DXDYB&8Ef`@GMEjn z?#qT~qhgqDvXsWs)J}LPr$v;+vA=^WT+3XEd4>(SX`z=uFLO(KNYGeuTd8oT3>Z|Q z>f~kVTJ(WlN^z=X*$UOHdS#nC(8WZE@}1tBzS_yv0TY@J-m@YE$-IfBc=`D)#+*H=AeHek~HAR+P6GlKQ?@qAP8CP zZjOl4dQW8UN>q~pq}TINuD$@9ccxzOI8hit&3iFk=Uz2|d3_^FtNB8=8jC;8ua2W0 z`dxdXoFywlm`2nXt-d1o3@;qw2^nhCs-U};xK#mUrxs6tP8bzO1eNjjx1zaW=z9|! z|MZB+LkwV;0Ug#KpAnJ}SlKI*V$a^pz5wqwKg@rLun)oevP!F@;nB(=rg!pD0tewF z3leb4_4U46xmRBJkwVBAdz8=$@0_W6-s)3cKVwx+IT-Ki3Zx=Kx#c7 zwJ}7jKHP5?b+bUajh%5>u0-V3)+rXew}I;i)J3aiv|swoz8J?0`|EJ?vSHgYCn%!& z(bJYI(j9O%D{e#Ftp!GVhZDj##4x=WNL3qVpaDgny@jUp^->&7*@?xox?{O-+k_Ps zK*qF82zf(;>9}~RK}!V1D|BH2s?-|16jgP`ct?`GH}{5^=jjnFM6CmJ2ujFx4Vm6$ z*`t|D+8`ctHQ$=^f`BZz*C2*RrdRifSA)W@-8+m;#^&k^3^e&B5cOGGB7Z7ZeR2%H zg?NjLHO8`z=I5{z#pE8;$8`2pqIDDz$cvx6)k{1sU)JIbXg##2;#EXxTKh$m z`N`f`X)VoL6tAVcK_!U|tN-&r{)51#oIy{0f#r4Umk(FB2~Uhz`Bv@B+)8 zYUeRy?|QH}X?}^=NBFsMV}@>3`6NenYrFt`9gIO0{Do~IAG_a*DUS)V{)}sXzz{q; z?=*R2Ir465$tvKlOibmTq-FUXYn1>`3xEA3bod)Yq=B6F1yJKoBd->js^|#o@i#bE zFkM5~VgE#RA1|8pK4OOwC%w6I@^tQJ0+B{-@$#y*j9Gb&j^BF2+ZsRLb!u+p_LB=$N}dGW3+9my~>m4?tR=RMlxsCBf2s zh%H@$K1xN5oZIp%%bv9&0%tE;@~Hb?VN%VrxK#*cMtZ~Y)zyN?niZ~% zn>SxfgA2vWlgE-`mO79;R5QkIRPbf>bM{A#rQTjQL6V~kFECsdnHVJLa!!vDn1j~RB-#o{gnO2{vH`&e~dopD!8YaB-I9>v=4lvGbs5U9mO&J z2fKh|dlo|F(?HwFF3gkI&S*mW<#81`YUTQs6yqM)>@#eIv!f8%*KQa&`Dp|Vr(Oob zEX7RMF%rJ`bDu%deQx77X$RCwnTMRbJvCh~z&1ssK!GTwRM>}0PG^XkG)pIx{Vn=O71A;Q^V$#Av+;Qx8 zcV_Q;x2L!u6N+WMDz2e^J+BbqcU`rP?dhXyTj9*v*5G{K8CZ1$2^XfPN4p_^F2Ibm zlGhzlJcn&`Gt4>lkuGv-_pEQTgJFN_yUcRYt_hZVNSN}dg7Zvv$W4L-pn-4p*sRp6 zc-W4S@hh}F6o}PzHHhu#MiM}Z^#_pt4cjuE{DkOJR~Xbt)n4OIV<_v(vF`{XSL`*K zKXTG<*}M~i8PB!E)iM?u+n-sG=h2LORy97DuPT2yBIP~s=SHr&tfVopuM0NFM^0iq zpxJR-ZxH)}=CPjiKHu;o3W20O;;#v%2t{`~Ef&FClhTbfBtY~8#)li<&*ZBoT-~w> zMCKDOynIpZBe!SsnLS3mtFE2+Cp+ubk{TO2ZGtxsUqajhdW6lLKclVJhp_PvM&B$p z`=?TeN~D3~F0FiW%Lm8O^|qoN8d4-&RsWD`>S5aLW_gWQgS4b$-`PCbZTBu}@kK9< zD{I3iYaOkj3msT-jO^*BArfr70Lt=Z@UbV6M}_bhN%&Pqnu?AZ7Iq;&Eu40P^NI+< zh>Y>|rbOh%ab=eR(@WJ*Q>w+AHDJyoEtZL};9|rHzS7_f*sV)|_MAlK< zv3MZDqR5&_86}{wi@00%>+nNAq$nEa1jSBNf#97I&1d^+gu6KRlRHF#BNk{yfgZbmEh<-e-LFf^WDjFI;Hr%9v1~Ew%k9cY-<JJwLt3-#gyP@l9tj${nu^M>$S$diIxtSTk^X%8NVM948-} zSJ2Oh7Ufhbm~p5@G;n{-1!_Gxf5T9hbUlqx8p_-Jzmx@9oU_n}*Q}NpYk&^;1_dv2TBb zTK0ZyRinbI`RUotg|s?*Tk1sDBe&bWD!wyO*;{_+(jC`P)@*@1e&*1ox_%XbRmMz5 zF}jUE6aqpY)lqbRL+nLRcCV;C z+!@(*tivfTe)6U%!(3gk&q-LHP1S0?m4^8#!H>y9u&Y>RrE;5gAAgwY<6L*wh1Ol2 zUdvBAiccmMYH;$PBj)l+ZXsV(E;M-`nlG67?|+~?m^#Da22c%@NH9tnSX1RyyOL=n zSxOourZ)szRi2mN6h< zq@mt6mw)7FgOz9qdfvUMBVaS;WT9qpvoTn&j4@jLw82csg+M0fXpg4)YBU^i_6;_I zS_IvUN)MXp&}2tB{dlGcTjRr#Rx;4B9Cmm`@jYz1BX0=Cr>VLe<)~W3BhENB;Ae=l zL!`==#qc^BD&b;PvmGV;fnWg=?6nGil^n)W$tmreSQ!HIa_{&~Zq3r90fXVTYb3ZT zabo+(xZId7*T>87TRd-~3h$JFhtF>eL}m~oEyOo4l8MUegtGjT@z=Pgw1Vr>O2HNPYC8mO5nF zZkwR^xDPgYiN*uE)eXVNOHv2JA3c)kz8K=~SE&tDi8&-2`do@W%eDCniDf~K+jF*ce|FWr$z~wn z%#XUWJxy`ay)L_{%bQF%9*G9)8JiAHA!py?`rb2Rp67fu-PYTzFxx}q>xnj_D;85j z&qf=ZTS8fSMwUy|2g$(*##E`wl~&_N(-UnrR4Z_CMYcw_d0B{}mZtpL7eyUm>!&cm zkpwg2d@Jo{2p3aKjKIcIL+XY6VWB4hj7(pGT~0l$APQK4PRVCt3>3kK#hjySN>Doy z@Ih5^%H4jnMKjrh?9h}Hv2?XOy;)g#rvW2v`TmvPZSSvFCDzQwjnpwVi|Q)3msh=M zMK#KWTG&d2zZ^5##Jo@GGq7>u%|UEIq9^h!^({lc{{*&WF%yKO=Qf3$3S2mEYFn8w zL}5x5EXF{pVdOd%1%Q6~+r2kh)a32!Ig>}r zoPP>qBs)>roRE~O$wkOwYN_gpl$Jd6;SB4nryMv6W^%Ry7|652CLpUp-3*M%#17*m(hSYWQ9)q&)yO@+lfW7rSek;>?P8-+vp;65!?ZdO zuxJt|GJMtQF9{xxuBOV&uceG+r;)mKLUg72GBCT4jOr#7dxgp)X!6a8yj07z{boOvAyOql+I`X;1&q3t21_Wik z0K(=|vGh3IZX^T7Zi>ymF5h8nY?kGbpEahR`JW<%bd3?H>H2jHvw9M?Bf)%Ank9~tx8T3Hj02-iOT%GtBrd0R~?mC~w>H>ufo&ZDnVhjf0VU1%a zL`o*aK87I}QkqX~(pvu({7KPVhdq-a{NJ z)5`9D1e2aVieuiL9!ZaN?^1f`2vxKclC6D7NCx*`NKXjOKhz*oNj=(GEF+)oqpprH z=TOdRG_huN$%41#(B9BAz28FbT@IcgM7!B@?7LH^>)^|i9GvW3GYgV$uJ$SIJPxf6 z4@OxroX5DvtL2ZsC~k`w@O60o^`xKm5j@3>QWi<1k6PoNFo~TScoxge_?)lvL+AH^ zSPh?wJvM>=GT%bx~IpoMFFx{4{MrUek+6|t7_`(YQE&JIv%oZRdt47&3t-l*| zrDV+-Z5~3dIKK_@Xf=+!02*TIY17%Etm)p|29bA^5J~2Z{gwk^PmQ&9Fu8a9H8EbP z34WJQmVLF(#dqR0McWwrp={!`s`wob^pv5euPw{5zNYl~sK*5p#$_?a;#yXPUx#YT z^bYSVSZQB_Kzuz;n^B)yYF5OnVh_l-H67h!F(}&yjnR{J3QLPa2pDw)#+rgte_UZxOxs4M_;6)^M=i zv0z^hGd9YYQC}U<#z>P6h3cB(rw{nE1@>@m;t&k_mZLh%hYxsEq~KxDMK`xkZw6_! zET7Ox)lk#1SA4{^CcL~eAyOc5#ynm;!$gs+W-7WZ#cW=@u){l*AN(F08vy-Vg-hhz z0Horv`6`L@_`?~uXC4u;u;AtZ*;9jn{%Jy>5$!zrNJ!RvRxwL!bOSmcZTt%mGE65< z9M3PI%)o@Y0#|>i!ZJ#F(WqE}lfp{qbbsp9kIf2XpF}z-g_klR4#9dXnq*CIT`@PO zMaofB`x=hiK5SczHXw?c4RTwLt;=ph= z1BNI8yjP?N#n|_GVny3~Da{-elh#-pF?G4ZfRH1rOy_+3u(F7Q=G-+}H346kT0leE zPNR1^d8pny#pz`^J-0IG9XRTv+b=*#aPn9{KYTmJoZ+Wr9_L9|^wzV$@F6tnN7PTwxg)?nD3?AeP)z9cR?+D$d(^ zlcTAqUK@a(E~<}`9Mp5{GU_8w#m>;NL(4VUt{T$oZEfzrURqv5%ldJta3JJhp|}vq za$UBVfi{+ZfROyqF~2QCRlPxhvB}K|2Q7(JVSPu#47(Bm`O^ZCzfca{$1&UWK@4pf z!6c)hzKI%<5wO;E%hE7>L=2;3bd~!=V>zUCGhE&I^E(}ormoGzR^*hcmvy%K`i6+@ zdh6RoEmN;`d{LhU62~fD`(r9ZGOp%>bibc6?eR~$bwvyLZ(i|qYsLrRFt6TYyZ~oa zPdqQcq#(^ax`NT@w~nyx9&5TK`7E=dR<1q z#)LJ9*f`8!&MOoqCa zR$q0ozybkdL})Oej%11u+^6#aF*|eCxY5B3-B2DY{c(k(k&0yPGX*6My@p z@T3+TxrMUP6%Ljb1t&wmQYLhN!&kjXa(w+GGv5WN{4x=~v|@ zcO}#ppgN5NqBu?b;o;M5fSejyzJz8B5>M#%a58yb;;#WwYWgJDcz!ZJ%nNefS%u1w z>%2A5(^n*U;29g``2%+YkprU96*t0dpOU!Mr8lm)c7H}2#Jn-KxmlUVspP!L-Fwo+ z%%rgGc{zC@4G#)HUJF6({RZ~Bc%t?NgK#a+_2+r@X~|iT6+ZH`Ru<~7F_T{AT8Qjn zm2i8nC~1H2>yRqmHcG=0tO+BITUw6(WK<%Egt!EChYqAl=wq9n1O$Q2d~a1(a(||? zz1B@mRNETkv0m%QuU#};R{)+ai0$$urgrxQa5@NUq%o|eJBl-~oN0o^oml;zP71$Pm$y??+e%4MW9eXp65j8<-&CY*$xyj5U{*#=i{15uP@2!eX;?LV_}? zE2_ry)REJ6X_$RMco72+k+}zW*SY&Ys8}#&LJ1`7>X&0v-8NzUJH}AIEo`pfzo*x{ z+7eJTxbr(CW{B+Y;`AmUEn%#5_rwGzrLpZfI<(F%SF*8p?%QY)4-xj`Sfg0=ZoXUV z;B#L|Ff30l|M&t3RDMAoR7}LA^kd0?EEu*oIyeBa$Hm9swmR@*V!NXE2)XiFt~SwE zWZn+Z>}8#T+;6y5gy-%~I+NPk_D#}TdP(lLY3wakK(3AfzvVu2#Ww+pb}F6srzZ*R zgSz2pn}hVXB)<|sorK$IVGfj{wiclbco%HYHJ%k54j6MUTA$sR#o(hA^jv0!Iy(sh zeOa@!(vJi=S?x7L$K7GSl%2fYE^8&$c-d&+u^VyVtNm~x61-8HmDrp zU;t>SY_S_}34LFb#Ui&drL4;0+XMXn031XO4Pi9Phyapwgf^nmf|E2_+Io+2DBd7= z_F%x=6(gY@ewedMPG@hUP|aI*MmdYh%c=gAl~a^H_&vu;+Bj*ct7=2gJf815HLV1l zLl5dO-ra~`Ht31wK3U3a)N=ce5ymW-4l$9$w)v>g9i#AfY>hcfm_rwdGJx(DnTAI?|PWLaXSzpJr}udj z!7)`kcye#~`F!v`Qk@KTNO*O}X!qq7RP!wG*qw^(y~n0KOMqGux`L8v6`8m+#K;Er zxVM%&Q8cUyx)jnH@a~A}z0N?QN}rRJ^MTjPDX~ z%3>kpZ2+D-2<{#6X<3?nk>X~S&Zy^(S_aFjsmrMNSQ8A2M^3gqe`Jr5VPl|LHs&6p z9F+&BSbYYmDOmC_KgPC$HOncg7vh$Ui68-Tjc##&_f-8bJ1RNy*I!}=#v3baEPYA1 zjBBj2*>e2iUl0~a323FYgRS=ZkGg(X(PW)VjSRar9Dh2-MKpya)0=JLiwD4wuDyXW z6*;DVoN_6bni@oqjn#K|0Q#Q~{t@YlmNepvjkKO8)9)9bhMFJDbnqU(X!OHK&LgIc zLb9@eNag&`_uy$|B{l*s&Z4P8icbKNg}lHBhLqE_yl-GQ#9~fH10gqcqt-`n4SY06KGv_6IhZ zHKe*|4^;5ndp7Werj^YQR7OF;6LkY5 z!gG!${q!~M1o423-$iS=U@eg9Jf5ndMZ{9nNta4Lvhpv6PyYazFsWE<%Y*qd0=tMK z#^NlJ_^u3-Ny%k|?*T3j&<=0=VN)**c(18-F&d1`?lf*CYP{<|ji|YY-L!^hMgWVR zK*wuaeZW=^i9Vi*)q)PBs1OFz_=`!$J{^{5E=3ek27wI_zEsL7U-eLSL+gXQI-$_< zSLZc3shYmBGHZ2ZTxOmh1a8q$B$i_su)NJ>Jfhs(zYWI^!&{YBboG^{ZG-)_)bZ0; z%CX#S+4?r(075iu8qog$bA(#I>C|n@Q(T$tOYMP^YaHz9odB^WjnfQgeC`JOD$2zM z!Z5?_5MS>fyw>UnEJi5WR?=k=e8g0C-23#5>C9nE3srL~QnS2h5;Ui=ZE=qdbk?Hc z#J0Om6NC`!eZ`2N#TD{Uw?NQrV4UYRtA9_(Il=-x0zEsdHCw+i=sje{XP$ ztd1JoxXcAry!vj9Bc}B3Uqc9XjU!+=fmPq<@;1cHwBFGc6r`I%I*W|a(CL=QHO)Ou zY*gaZx#S7sxKaADsQ9isJ&pVBE<}2Hsm6eKWmwy={f_;Se3nY2U+I}N8%5}R;4L`4 zr}Ak$Y>!;!w2Ew6hOS3nm)VuLKOyxPS@N?e>mb!9nTuh@uxYc=PAF&Q0=)?kW0!GM zXY_E?He;1AlI3KEw&>p?RDK`79IdxSlt4WH01@ZQNR($~M5O3IuZS@s4M81SlND_X z^|1HY_da`(jvHmrYi{^~Y_gF;PjQP_>nq_=BT}=+{{VXdSx4f8;IbBhRU^Cc0I7+0 zEQOBSdBwh0JoTkmK~X9Wpu$MMZrG72T8JekoS+vNOLY|;@qbHKEIa$G@qxcEMg)4| ze2W*#L`;!QGYib9%X7O&F3R7R!hf4$BUFkG?|MWC=ZKt~YLr=yet8TGt}E~Ng&wKbb4?Q^G!jgzZ}1O>-7S8Y zULSG8L<$Y(?G@MYd4~x^n>+oa=?Ae7JkrQ-yBlx6VTY3QgHQkvx$F%INjZv-AGhWPw9QZG z4BlEf*d-lu_$gDCAb7}fk5;%sJ)n<>p$ zA?FnF$j9zW+B3IC*slF~m!~c3B{^ynroEqt)Nov3P9!$9QCdThGcY7jJ{0V3E&Klf zh9FVaG&Y^go16RhtV&w+0`Dh{eTe8c^T6hW-U~~F*a`s>>GdXGEjdMca)?UvLdtna z^&Q3_aHNV-ks7+!tz(>;f(rU&R**A+*8N9Ja)(J7POz2CsaXOLoKfbBQ&Gthv-5qL zs$8)o?4S<2n^;_%{yzrGgrOp&?1~S>D5BeNaUT3(i^Jb$okdTY$6c!?e3vGtW~fZQ z8I;sW%%pNT7ZEL+a4yICo18ilNol&wU0P6V=3tB?4w)-ToK+AFK-z0w;d{bw+SfCq z;qGqvR#Q~SQ7nr0mi52P3y;#q{r650*aJGMr2 zq^+mbIb3v=K&?~);!D_H&(IyO^2LQ{T%(1=Xnpu>jO8UWXbD!L7@Q>bJ=eTxq-?_> z%c7;p!xdc9Gn~T%k+I~mv&mz(!lZiPfOdctnn$wuT*T4-T5VlPpzn=^?CJIm1->r+o|ph;YUF7ag)pjK3hDF3c+?Ybn6Vz0!Dx9TYYSB>L8yrNhH2rW zGB|sWp5D0Zx=6H6(*%$RqzG1L&kW&+Xo&cgfxiBj2~Qf{AGa!%CYr#0pwqDCGo05W zmN@E|-m5Qm{iV%nUChSXsx4v*!l2_1f2TVPs>gMQa+Vy*hajM zXA%37YuC$UmWr$F=ZTs&Qbx?C$M<8#CREYLy^#dmf*k?o?hX6kjO)l|kK7D*a#Wayk4ZEf$6fS(s@t%=Rj3FtUI6Q zd_v^dZ~A8NQ&MWKXH7k02m4!b>o?+lrH4kvomJIkxom~A30`(_H9)v3Zg&xngbwUm z(-ui2xXT?YPhNuPhcl)A%v=7?R`G`yP}}ti=`~IE$i>^8$W@%Q!e4gNe*33M711(YMOwy zGf84t{DrIsVd_pQt~{xSORjCuZW{-^y^(ptbl2n}Rpk{J|sXf{^7vA`J-$v}A1KFnDBTvE(}& zZZS{`&02`OIYomC&0~^f^^(V16mdsa8*{HPuw&#$t`3)dNa_P)6-m}xBrca47)hO3 zo#?FErLx-Bq>e`{(0%xvUi_?v;QAh*cg5YdS(<1Ciuw1(UNL+}1({#UI#E+vYLm?- zp!20^!Wr5t+5(d@6X$_c481eGwxhDDixwpuwb{4tULpeZ9*nG>P_b1E_7)7qb1qnF zfZD*wiX)SI?#ivU~IKdXj#2uM?DMpFdp%xtQRB>n)O#|O6PTuz*Llh;Rg+5@w<|s%Aw+5gD%eI)K&E} z=Jf?>Rzk4S^XyZw{;2b|=Y8;%U{otuHxFgZS@StQqmUk|D?v@>pSk4)v1ajjDFhPH zLr96ru`)fBc?CvL2c{xfO0}smp((|hC8`11e29wb>YCCIxYbh1RPz8@NXXl21{0CCJ~nTd@9P=Zap^EvWO!boPd%s?Yp0 z$kR|p(s;yt9r21|;pgU*xrL^uU^U7Hg1UEB z9~EB7+yM4S$%Rf~r)l-prl&v_vsBCP3ez%zs~sAF5$XvHt;#nnd;N%FEy$piT*E(x zJ`!^ueF;T!ldxXt zK$BSgNRNVK5ZX{1r3TG=_3?{br`k(6N}soqhNc#^;&`J_exYxCc${izALd+g{6MoG z1j;Et%ghe?lj{e$Ul{US;uSAT?v*6&(H?cbt+)KJ>8xF+MHOQeH~t!&mRD6-lg8e5 z@q&D-J({6IQ%EZzJFg^w6$jLi4_}@brml6G`$8AxQj0h;o?mMHqrHA|td1lD#9pA0 zar3t05f$8mE+Q7#p{jI%JpTYzh-7$_m0SEmL2!P*4`;E0WH8Zg+Cd1 z+PVWKYCOKBMxv^Wh*b|UrBXkpK1aHG*|z{)<{2T=EI6^%9w+g)x+q@}f4kt2HfY9d-@P;$ETsu9O)7ny2l4#2oF|aX`H|=h|qWG?b8iHd?muw_n z8QC3M8aS3hd_{F&<8XE(`>`rl)hRIDwC#oln$3)IGFV)a9YOoRp1(cD_#q1blkCK* zVxqB`bt9SNasajYKpn85NfnD4)jI({)FiR?W{?u$ z%D)15AG{Um&LFm1Dn&DgQSmIJ;#T_O)GXHahwQvFb&UasEUNm&MgbFD{6IQwZk~VRj)~ZIN6VK;6;b+aPkSOv zxsmaez*5Qy@;CDGz{*q!M3mFtI1ASl(dw+8YI9**7&B8yru>e*y)V%G@Umq&g+=KI zv3xh?Wt4%g&Ty2c&r+y(b4TGy2@J=r$52KRu3Vz&H4%xDGi6Yrjy2Vj!xWwicyhJJ zJb!-H=nr9kF1-#2W|~qAL*;~?j9l&8*}4LAeBBlmgEFFnF{T%~$svkm2liwekEkOL z7--isN+~_#w0($xoLF-13c&SdgvmARMKtDZB|c`Lx9HsGl(P@&PP-q#k21a6w>a?Q-yrQB>+p@lwclvd<*($Uw`Nl8@^<#?rA5sOn!q4WnEH3ks4 z+S*Y<#R!XiE0{+`m{CfQC~9)=Be}R{vE{JsZNCf$HrZQhNjlT>21>07T1_;qWBRXA z*3svclG4@9OwBVt*|{5&ewVg9M6+vUA!!ECVkh5a!_h3K-t8jdJ}ByVVF;PA>1(nn zKf3}PE=@-0hbj1n5=v;7d;}ZtmbZwUTGf=gb0VpQ*c?o9yJB zT1jP?8*~ipw*z3Omgjqmh`3mmX{N<0DdJ3Nm?j?E^NuOfNJY;RKn9v^w+Ll>HE>ra z$s>YPp^7Mwf~XeT`s3U(P-f{|xq-eX#fuG@X>O?D8V8BIOUw9Skix1KT;8k|qorGR z>mzw8?eio7iv--$a)_z9Ita}$Y#ID=VC-!_Qn~CLPnF}$pJ+WnnQN_InN-PDB=1h? z0&^DIK&`h&|Ebll;7Vy@oN`u_mo>@hMpa!|)qtB2`LhpFv={XN1g+c|j0 zj$>yw1DBZM1g&oxHi&BT?MbIH+&&Oxm~cKUnt^0Go{kEk`jt|uf_$tu7$I`Tl~RX% z0WFU$Ys=9njo^OKLzO{OPf}x!q8D^>qDW>EM#OnA+x*QRsY*8$J{cU3QKm@4_jmX|g(GSZH`1-uObC>f=&wCOoHb zt+?~+g;~y(f~l6qC*|3zG>UnJ{{X7k?Igr2G_*|OLwnsd9(x!cY6U$-x`qScC;tF2 zzn3qTCUpgA3r?fozHl=SbS;+aVGBx}=mTKEK{ng+$B0b?E_!Tig+CMc;ZKl5Ee1^@ z)aMPe$}G{-{Qm%HyqBQw<%=|i9H)qJ(M0&4#AjI7#A%lWnPXM&*gofmgKp@}_4U;m zon+NjQ^`|PGnbNSShG7Gq?6R*-~dRV(lTYOPAIKrr0t7*fup5`rF-)++pw_pz%vz5 z$|K-tA+5uv=1AGB8Hj5STvaE97eN=i8I#c`j6ZZqX=9EgMtK+z6bq7jZTMkbXk$QW zuBe8#QIzHak-?y<3P^Eym|PG+9Y3AElZ2?&(9N&`1ehYAhN6<9KN6l`RC4n)WCA^Z z8-6(R`GDLQWR!xFaT)VComClB5)Rznf8mb{U_mNwf~FVQ)i`3#*HPGC9wZ%*mEv>| z0h>@oSz8n@1hpv6F9ZVeOW%}y`g(3~)rSjdDy=EB=s1|FPc+i5=&F6&)0JcUG}Ah( zSJjbUUer|lp;m&BGx92n1+d!o^*!+;bF!=t8QQa}{aeZ}riOF!`@T^s)Klr)5z1%*mnM`Pl`^V^fknMXIUy zsQXaECr)unY^s(o70qT=#_KdTEQ0<1Uyr61hDcovlr|W@VSg32!Ob!(bWphM_gHsn z{e4YcmqR54%CU0Yq>a0FJ9^+%W=e=U#q>It&UGj(D@~#=sWmuxBy&=YAl}3Pcl@`u zJXHc|vLtY%BqSt(91f(QDm8lmTNlJw&zR`?h@+nN|T&W65 zuhpTU+o3cZ<5_UK*nc9)vU+-sAxOE_cPI||bW_Nbbj$lTLai%#mpq*SO}Uw)QcOz= z3afeaAA6WtEhZE*ItkznJa+bX!!fu|!u>rO!A+&ITPDeTvYgoY+@Uwyt?}xnGFd9_ zp!A3A;c0XazwDtI{etc8#O%JYgI1>IzJ}@<3VoRk2(vu=9-jsB~=>Su1#J zB`Yr0LP~-=Sl{>Il(h4Xi{27FD2DhzK4V1j&m@8Q!yDX-96oKpQUxJW+aQt*My7^( zs)*o;9$BPdh=?xC#Ml5kl1;HOpcRf$ok|H2>esS=0;#w~qTq&DmQ3oh(`FSG-KDqO z2li8-{{VRRIIpR~`4BWJ@TlZQ+6W9m0i7_x{iy zmLkOxOK4a2#hUzxaAHf^*S^Pd^~FYFX-~SPtkyNz3?(>b9@`mmoli5rmXM2~ zR?Tbc9#4vSYgNs=mv%q>U}CdQl}lGIv}z6!D${9Bfh2bi!HG4Rv9qeC%W}z@pD0t| zys$A=sy)eWyn@>=L)71E7`I9k8gI51wTGo`{M5**wE*Zh8gJ*O$PLdl4rQkms_Qb@ ztLWW;HPF>;#GS6Yl_%G~eeg=zT2q`6PEqu%`%XL1r6efvH|1L2wmwl!DnVCCJd;Ib zJkkp|7qC|IAd6qt-LS2|6;#rmhuz}B%@jJ@VX2s_(H$wMVp3zP4M{JNCvGCZrT}>klL2PV){t%0pSAUaB;;R$j|IQXm%y%mV}FF2MS7 z`r-0{c2J$E>jUTSglA0-hI}@!rF~P(3wcwGV>WR+Ba2Sv5N5e_VxKIpavGgM2~RK= zWnU#A77Nf>fYJit_X&skOH0t85(}3 z4NI0(9B_;mk%4h~x7!HEA6|q2J4gAqy#H z5}&)uJcb?sL{!NJIb*ouCdF7%hmlAOrq(^dIE13BsgG-DaqxzkT|-$zOH4w^FtK*~ zr~urbL2Og;G0~{k-D6O}2yv9sHfE2e%k4oPl zBE;~uD_Di5ID4pgou8biPGnPOm4FtIB!VcROZAdgAtPT_BK-x<*dd82iMp1J5;mGp zcArbdUlDVlNl|wwJ17b4-x_N|dj?QtaS1h6Y$dK_9xORLY>`LizAKJ-K76Tuwit4p z9i<95vZLHP2j2XmoDd)D%qXh50!NK44NSM$<4R;Oy*6K$K@4$xGB?B>{du~Z*me9c zTBO@TLPBW|7?^E@0EAMzspA9si-nmSuri#!K67qYj@;*BZ+m(X^!325NwlF-A^QsH zEttZWUGP$po=F681!o+}n;UtLU+}`GWm1AgVAO=EH#G(kTHAq?wLT$waI%m`FdU>G zr(6!(uC#ij#b$2{mzu9uoJ1n?juz(na#`of$f~9eR%%H$xf>9CkN9>Y4cnh-@B_k! zVH+ZmF@rrUrF06+HjjH?QkzibdTy#iLo~G3H-xx~Vfj;1@K`>cP`IFoB5NJ5f2n27C(u&VfTXZG#{1!N z8+Lr<0*MuMvWkDS>xiNhw%?&OfJ8d4oaYc=aP8 z3r9&1I|Z>e!lsoPo?40sGR-pDO-rXM^7)*`^)P*mfLtqsbtmL)^al-AkoziZ!iHXU zPGt`uS1GvRwibFizNK<4LzWU?t}qxvF}EZ0g&_Qm{{RdjVM-0S-r7ai6KS2`#_#oNmY{9O-{i9 zrgD!IlE#vuQ)?A0H|@3XTav3vih&j4K!z>4+s!8_s5%i+JRpZs>pH4EHC+s5DC$fs zsME4Z5NmA4#eoB3-=;Wa?zlp(HjQfuCS*xv$C*}D07^-<4J=JY!+ju6LlmUY)%AI# z#&&upM3KiSno_$sAKD?kEO9%DNtjxxL3(*p$|93nsgjmwK4%h?l^rW;l19pab%vgQ zUmYHc(0LnOvqmQ|o-6MRejg-Zhhp9#Hu3}Zh9`KXygd@<>Qbvsz*e2`zEKK)6q=hK z(*UF$=AfEy0+&1O4qV2)G}i_*DJLaouZ}u6t}IKt-2MoRe+)Z9zFtLZUewdR!`3sx zgEx(~lxTufc>(t{jQ7S&(hR$V)it#N+kzT{h*y0zZ+-vJe9K|AoEr_A}Hn4dRVZo$b z<#_C+i9H8@OhTJOL^??nj9Fy~&5;kjLh&4VAVH6{X*(gO7W zc}4|_skP6(roNJ;9}@sLo)yRTK98gOA4ky2;HG()<+<{%UCL*U?nI0g3LTg+J-ss6k~8tb~wK zQOrGh;!OMCq>y!sxb;LY7_;8{5<6l*;2M}Iut9)c25?VS@k#Rx>prZHA<@}$0asfk zHL5Bfse+YnZgMsZ`%cIiNxomnyD4&)tZfPK#_IH>InLOG{d4t$Llf`9c5)w#3AE!at zv(_fXejUWneqjW?U1{|W&D*J7Q0RGStD$Krxh9U>!Q*hzl-qFIgV)auS7h*ta0_hF zc0IFXz2`o>01s|3HnwUckmmD+zSKm0aG#e=eU9h>K z)gnxTCe3Gv5twq+6<|T|r5wsU$HU9`oC>AoTVZ@Fz?yyH#U$CCK~hf3Q-P+CDa0(+ zmFhCNDM@#CAfDIW$}CTw0GjH|1Rb$pla+9#X?H>HuqCQ=vq>D1xE3loh3-`C)OJ6* z_!U_S%0Z3?MovU(Arr)XHJCuJUz5aJHKh;FNy=Sn;~#l>E;*P02m^910ll!lqe>T? z=yl=8@7Q<6PZ*GWzwy#%`GS=ig$`S3->^c4qfg{qP138H(?F0-c1a-(cc+neDjTU@ z;E~ez?}kLXB;^uq<~c&7%gW=@q0}ad+BSo@AAF$y0NCT$ev6Bmj<-?j%88#w2mP{1~ zsvNT|riPA^o)D2iPah;vqK;Kn01bcu0k=(n!q;XAs)@W^ByjMs0!09@9{umECiQ&v zx{9PrQC&4ISI)7TdSwq&S0h}kN=?G-Hy|;!{_e~v(@z&18LCEsBULM&;kK0KBSe*H zGR<^s036h)=#-65J%YC2bb(Dx#C(T_@g+86A%>wvqsp?PrcivOnYn;(FskHtYvEvd zAw>e}8xcwlY9ROn&K}F*(>RKHWKbjf+^rA$P$)>|Q(e%w%ss1X&10@#qVnd6HVc_G zRPXa+<0wBw1JeZuk_}XZ(}_|40Q3ihJ!j!WIDkycgzrdg1GA{3-b`KO-qSiuJf>PL zWx_1FR&irlkk&~Az#if!Hz)ft50(l`xMPW=!vJwjN3X0XocLoLm0wAPPpFUWUQ^>h zJI8EVW!d!|H2(hp!cAi~V(JW&==|+l-vAr1MY8z74gSn?06yWdY(lw3OAen8b2>F6 zXyz;wIEo%HujW+hzEk2kZjDC4%@9t`w_73LN|JbK2PvQFe`?Mh=2|~HPjaiMgfgII zc`0NE;HZK3457M?;FUJHICF>3w>Ux<^$zjxckdNkf8n>V)RiXft8F|KH|`^tu94-& zso~aV!oE4ClFoTjr!CInrJ3SC3ACz1E&CNJ!_bUfnmZ1nFo#de;US^ z$+P9F%Yjz0!IsQ_(CwjSjH)3~)@esRf?0Zk_bk-)tfHuaq`b zs9!OK0#as(>^Tp@CJXV8s3C{Vys=K6Mpp{Phu@B|s{B7PAC>uKnyJ71Yi{j3x4<5n z%phPQsdh?dwd6(SXh`VbceijiAe$T83Yp54lsTc>yNp`q+gVO_g;X|g;oGqH!Dg+} zl)9R}57Q2EZ26;7xs5W3Y4Zs73nBHjzBRvVs5Zrc?5iOYB8%9qLWcOM$vot$RJkLW ztJMa|LEJ0HVeBR>GM)rvHKHS#SI1ij7AZ~6<-Tnp87Ej{N zzJ)fX$p>Q1zv6%i?NE!&VLpW6L;PZStTPM=VA_-&n_%mWJVw)c%k*BD(?#oB zEvtEI>*7vWONH>nac+E0rtNEN04IDp#im*f{HhIAQC-5DB8~9WjN48wdTPO9i3Hs3 zsoleAykuW&?QKU6L6T_7NAYTegE6hMTp;8t5I&?cH_G04Yk@FYXt=GqMtT1L^ju9Y zDY*oikaHdupz|k{BPwMxR#H^>v=YfrR-YP43l(8*pb`A-iz}fDPFDp%jA4~H+OxE1 zp`j3emU~BQjIS=Mq>nO%nrOTvscKXcLvdor$_1Ll{6^$;7CYg3aH&MCD@tijCXn=e zMTmyf*;u%xXev1oOYyucFy1RM`l?vi0#A^~m85+W>2ezC&*pO_+AWyvMyK))DiY3+P!zo$oE{{Rw}OM&xOzTfW*RvwZFHa`9Ei)5}i z;<~C$c|q1qtSP29k1%B$S!yZSZa=i8$KnUiwihm3jkJfXd_gT*IZ*|4I?2HZDj;%t za>zf4z=SC=WS7P;_Xr&@2aUYrVsQ^%<@FgdY5gfh4qK8@p8_bUVen7^xo%TpZ{3@7 za!DlP+(vWwlj#UFMQA*<*~%?AHWI028JWdqaaT1p*JXVLyQ7v{!#T40cBq+4uPAdX z6Y*SMYj*zZ6mb59H9Cknc)Myn8+ye}Mxw3AYR9_ES(cQ0tkiQfTi;@NYz_Tx4g*Z% zvg-*7BgAr~7O7^RMO{i(5BC6k@Aip$mx^<7A46x_Pcx`Yw=&G=q^70K9C&fOhSpha zvWB&}`g~R(Uko3eIE1-x{{WB>Y5mkSAZ{mi*r{S44|E7ql4hg(cXF-B;jCEcoX(e2 zXdIR$PY%B?rGk$!%j7nwqn;?lQY4AHAsnQFKn$#`304HSoLHP!OwqEg2;-%95Ntux znnKJW@P_&_XseGNF%K)R8DBP6T@5;BH06TM6q&y$Dte($|>}p2{g7@ zL?T4@2xn)GAvpcQ-y`772xAaG+={-R`j?2`-^7x%DdNnZMX}M=f`qiW)ba z*<*=N*?h;Z&|jqC!#LhHogN}L8gFq!(v9Ar)l0}NSBdHsPxIL zG~75OXa^m3#Hn=>QB|5Z&a)b<#^p^wRh<#V!4agAs=bIJ>ZftB$Bj!ip`B?79K6T` zj`$JPm90N4+H&s95~IXcKXW-XUQbO1V^Nw#tF+w4@OlLz22jzs@-`s#7Qrb@ zvQ!#%8A;HBXn2EI93aeQGHNJwKcz_k@-(GE-+kgFYF!yq24fMRdEE?t`$1V7GR;a* z2@#e-@dYE5l-}xDY;HFlrC;7P`V{aJ8Qp)HLnp- z!tYk+d7K)1O7UkUSm`rno=RNeDr6BW*Tg`1!mX7|$ibaJ0G?xR_^`}m=^B|wS3Z=r z#?D*2w2H14*Fa|Q8LGIJ?wXpC+O)aalTO{yBig?~Q1N$&oHybISxnH@)}}`-Y+rO# zK_QIUhwUZW&DOz(WdhEeN1((D3pP*(kt4|Y_QoGK@c#hK;pS>pJ$Bq$NdDx7Z5zc+ zI8<(mZY=5wdQLRUQ5~1x5?}?b_a|Zp&`XNDz^o}_KzzsIG|750v^1?sd+GbZo|e{W z6lkpAf;*m{Om{B23KJbX*j2(?%n&vpAW|pw?xBZE6K4=#SIAHPnZW<2+CRJPnu{9_I#%{ zgTkI8%#JsXWbGtv`pSIALvUt8A?mp;tL5`Q=Ejs0k zlIL{iHpmMZ5-wn~(YgNsSx+JRr+wG%wa=Ns3{C{M^7Bc|--zNq(7JmHD|hE4mC?Pl z{*{f&8+aW@WHD$og=VD7fA=p_WV(hG7ZjEDqs(IdqXrf~GC=afWZ~{UkMM3K!N=+x zKX|CQiZvwn{;e#wj`a1LJV)ZNDVcjYaYH?>K0ZMXOO{LU)aGwpP6>?lUl2Jcqz(ED zW0vrYK^N%nvvA`cO?+8_Ej0@&~`;<5OLg9<_y9L0?+c> zrSgqjQ2i+uN=2$j-2ge?6Y?GmQ#}Bvrp$F;Amq!{xnqNGgi6-7}ZekW?)tnME-?9MnGXczMjWH&l#cvFeZ-(aImLHp^P?Q@zJRq*Qq?L-hn>*U(rqwy*9T^8x`{J z)5aaOBtWX|&+V!1gDj?_D9fc-^8Bi!FmCfq1POcgTQghK@}2q&Ds?47uDN}~@B716 zWl~51CY|4(C}nAAqRyAYmn)~I-144jpO@0*_rkK4^W0h-ra`qPibtGu);uEAd4)5a zo7@nT9LrS{uBY*02W^UxhD@QBl~o|qf%bf2Coq-M{5b=G6(0K|KI*KOTIiX1bq-%p zXv5tlOSK~0{-95@UzLU|NJO*ZtdI|ne!ekPn`U-f{K8INA5VpZNy_#8B_>x@P^hxi zLaGsE7aYuO&2Hg$HtucO_+Gp@N^~V!A5ERHEUeQg1#1(ze`)DJDbE@&H%VN#Ux?nr-^h>aW||l zHB9uC=4-bnI+bF>ZTgYr=kx4vhGAjAN=**PeALpAkddS+rmo~AUB1ytM?ttI_9uSF zbBM}ME*F2RO?;{uKcst69C?st^v!vRdQ~>^ov$zK~Tw*Lbyg z@Au9TP2nufD!LP`E#fX9F27M{nJlJy9P+JMHjOIS%z=9^ITz`*%Ack_L(ZnkI{Wve zCU*myZ3pCL3I6~}?mG^nv_u5sPOr+?3W*f47ZP}BBDVMRzfQlp_#iBwrCsq%etv#Y zeG}=SfZb8Vip@tnuU}Esi}-gcppY+RBEt6QFNs4=E|oO`6wS^#I4I(+yFmf7*TFb& z(p^a^w;+EE7?zZpP(p<^OzXx5}NO(MiSX^BU>6QM@#)uBL_B{Vh5pep~%KCHL_IbN7?(sZH! z0JPK1Mhn4^WLS`8dMqpcMJXV5r!XC`1EFB|IvC~)Z}W&2SqRxf_?9I00IXx?$Ik%1 zb1185tvo+5q|K(K44nv3=0FziuXI#rT4J{%q?)RODNM&QHSD!4z_gGCj)b zhfGey!&4MPt`vZgL9m^*gzv3#i)~Wv1@wgGQPdjS!%Eme_Cjl)AbGqFNh_y{no3eW z30~Z#fb4!X->9{>>4k-|rKOabq?>)=qw8!romMDGRd)@0jSZpEu5#L4Kc#8tX;6Qf z)5lC&-yEe;P=4WIgK>PIb#;mk47I&7Tv@Rs@D(J$OqmR|9?zL>RTL%j9M@9Rh$BIM zW+;&;7VGi4kDB=}1dzMj_ zh8Em%(xUPosQ`UO4xa1cIn$y?hJq@XIzn)Th5brbij7QSFK9hqCS9uZ?q|=<;M7R6 zkb0{C82rt#HfS*(kOei?1KF_tQJ_KaN&@Rn4Brd zP-sQs$|jQFs4&CIHDQ_52R9@TPWyR&{{Y7Xp;f$m6e(Cg$+b}s#2Qb8C>J|z>*s@# zzN`qMA(5rHt0q%SX{Mx1!ikO6DVTtVe@>h9?S@RvoShoQL;&ovULjessBIei;}lw^ z4L`0TdMv`8o;X>HM3cE_;wSc80P&wuanobd5W0rn8AyvXjN7uLIE5U;Xt6z)2a%^Z zN9&#nX1a1?Rg(DCR1Dt=R5F%NK_dJTeEtLdTvw)KOGdP`X!w3OUl|>tR||jIZaWDa zVSQ6UpG4(qAopHFVI;q+@+*JK3qWlK#Mg(h55wwpHAoaY?cp4t%SK>rnbuZUMgIWZ z;|!4>n>OTpfcfHcHs~2rY0THdPdHNai*Fpx5y>EV{?X5fx|cZ1^yB4vM=Psb%DT`- zGWb}s57u(PZElw*V#5qDrkY!!$H>Z?-`~1Md-0YbIV&bukwS~k^;1A=O{Tj57T{DN zX{G&`b=5@YFs_Q024YL4O-V8mPgWsVZhX!yQy9Fu(oo$`XWWR!aIb>#X5JGMDM%YZ zRr~?*Fj2&ei!tF&k&0Mo^zA%x06oeo{{R+cXEx^%xV!Jvj-Mg0a?S%I@5>k+hhN@5ED|K@ znwT@?D1(_W$jP+SA!CMU!jZl0zi&(s(`PSQ2TNoM(g~;vYkdMYjniRkk>uf#t^kEr?4srEAvu6vT%EExQjR| zO}_nm;_AWkp$3mDvnx8HU97U5&^W?0oxUIBKX0 zN|E(#(%XCX$LoWZVX%tO3;zBQU0bQ6ng?2Bk`RLSDSyLkcBY5aG=(onlqe}zuNV`n z)paOBDo9rLzcqpSjsE~KkAqpRsE_3kl1bOz$gO<#R*qg*N@RBGqQIMOZ+oAaI3X=1 zGyq50QWPy%pO7<(b6{iyUIbB6A`Ntxy9Tlz?3iNA!z{pGnOOJ3ouZd8&y>qVymk#KbE z)N}pE1F(tp;-NaBr6><4R+W!M5v z-3}#`s@>_=WT)RQQ6$nEkda$TyN*~5Nj|jOtv1Epvd%fOy1d$`v{by#w2{A8D{%4w z*1&>;&4YPhNs_mKaQ>s!95k<_1%|Q`@@3Je0I2L6K<>HWP4H#I+^J;MnI2ri!$xI6 z*havq%_@TzC;gv1hx>6y+W5UnB_Y$7Mv@Re+SjV^yEz&X=aNRyzA=8m+US|e2FVg$;`r^5irqOq9 zXe?5&d6sGx0Z5aW^try@F2ncYR>t@xDz4Zfqt9hfIUSVrHfwK#0ohnJDo)cA&T?js z%_8*uV~982uROk1to|%Caw5~ytYe-; zDIdbF6mQM(9j;C!m|F+|rV&DEN{Uvtt@VmZaXeOFm!}@;jX)xTmjHi2fWM^$f#I(m z$STT%Q&S{`o%xuaX&afXe&@Nxr~m~7W=<_mD6K@2cihJudlA7e4|3(qH7z-H=17Qc zm*2J*PpqMW2H1asO8m^wwL)(iw@5)ZI<#n2RxnT=w>|#=1B%ON8K|wXyGqHl(m++} zZw?H$nV`u<9eGsKp z@A3OVofn0f($m#Y!{fV!<{b*D9e3<~Ju%`N%TyhZolTi_DW^>UufB10msPzo%@UxL z*t959b8pq^Z{>nZBE|xFw5+8>(|Pra%$BxFp(re>#9VXk6VZtM-G4lMSOVZe*_M*s z2>tm-c`|xTh+ywRLV0tWu6q<#(_T3I`7NSkCy(24B2@uoyWoia%ZC?tXSF%QKh*QdLDg zn)3+G;PkT%h{0+}Y|&zyzYz;Bu`!u!tQv8++o#?-zGo6TfiMaeV1$$Mw`_VBYR4tQ zl<9cmCUfIA+BsW|l=UOFBT=Zd2IJ{%Jx=(#$hCzQQGm4;q+E^T~PDB`O4_l^d&Z-yAc&ahF9O#aRnnTma#(^cdqRji~{Z zQm~EUMtPM~QdHC|D;$HPgJxoO8w@m%W!{h_Iz>wuW?2j^=LHQmZ-vJTb#7y(xTU7^ ze733SDxu6G<&jV7vjt^7V5uOFBa6U7LJCNzq-N4?sWXkK2AyH&9=B0yEYCF466QH1 zfiuc*(gGdfm4`wr*!>7aK%e#ujo6-x@HyYe90}9Qx}cJkle*t(!GL2)K~GMK0jWu-9D2#| z5X6dr(q#!fZdx>UoI~P8M92L0t*z88hak1@BLLl$5%DSRHI+H$Q;`Ei>Z8v}dBsdT zA9%Mc=T7mlU+75j&A}k$+M`<)j-=J&JI^AT2mz&|l_R5~g$sC+i+E8Yi<8S>NEar+ zezq2+PE0%&l=Y302XtJSlSGWFkm~7CRj8c-xg_r}+SBs43N$rEK4RnipsA6n^E)tO zPa^;pCT{Mk03d#`ZlGM;5RoRpmPx9I)Oep=k(^8|ILz6dBP$>PfNfNb4L1{hRNEOd zjl4u{Ntp6;{e&yZs13rXJ1^!*J;^xblxHFnVlzP?pmxSa*Lk7L^Vux?@y0n+U#0gr zzsNPwGA(l$Mlkt)ZAP{hiW+*EDP5{5O3`?{;rP;3M7U^Sy^!k2CXIg8T-(nHOKM5HU0Iz;(7|mMq=i&jl#bVK zxZ|8L6CmEPJvBrFe-M3oW8$sxIGIj_$MDtE{R1tBmc#gConm1&o$(7b;;vDV#9pel z2rN1?f_{U)(-Jn)-84c;jvd6Mm`-Pw5;&Mn>%KH+Ii$W`1Pyq~f? zPX7QbI$}|#4tbS|jh}t-OvJt*nw%XD7TU6()k+dLZs)H73wX5Nr>BM1nPHLKs2UAlWUPwe0Y2i-< zrPMn1AT`$F=EmL6o<4qEGpHFHxX|18fmwOXJw+}zK%qhnN1dSGPSEKdAW_Liq7zas zVWUJ^TE*M_UsaRLFEJ+Hbald&qY;!=blh#dLGQTkj?x*LG!z?#hwOXdRj73A*%fr~ zku%2=aVE5bT;j&d!PMTzb!!UWXf(2vt1#a}JPa7{s*ozlC>+l#a`|oi z-|@nnK~-2u*&0W}Qs@a&s@i;`R_Qu;=1DnFGk9st$8Ll1JvPGTk`zX0)Fs6VGaFRW z8JS!S5+pI8F0My2zgA>cz0Xb552hM2fHuRnP?B>f*pC26@vI#~66)kJi$#%w6LV> zx;42=5xt1tap-aB1Xx1(>1SCz7@^DGPT0H4Xer`b)APqr$~;LYBEtLf>@U*T`9z&z zYtFc%Mcixy&M=lo#7a#eMGi|-m(5p`)2T{!iGi9RNhbWn2NxvT;@9%nV`!KYiLA|* zC@KYr78M)?tF|xgwen~PZ)8Y)aieqFqX`PZMWr% z2*ahGPG*T7F->#gTuw!O9NGea+B&E0;3DHM;-Bd$BaIuvsgZ&rNWEC>H}BU1xrV8` zn}G+n@ieRGxhXr3xXLo2Fx8`F{2DDy1hr!#N1kh0&P z>4TbO4SgaLP0znM{ZnXw!y?GPFm69~2!LBzZDC_!ta`b@O;HysJ}Q*T3TRvvDop!?tV5xU9DB%myPDg(yg_C;?IGA!GLnuc9trj=*O z>pA`E31Si=a~sN%CwQ2Pvi_@YVPHiW`E}vX6oFJK)H*0`K8I1T)-_x*a$6Rb5;MP2 zK9vJPH&E;&Kz567fa9O=zdxy3xwL*~k<~xNBzZP@$wYey1eTm8>}qWtc}%Syl(1jN{?G zzfd6b#LYaV46@}7aVGubc|sgZ51xG1d^s`)=>;iI{8MiFo-kvVaJx$JB1vkiIHg~x z%9HOYr?R379li#rp7Em|pcTEhwXka#3sVQCef<2cJO2PZky@JgpA*7}mzBek$vS5> zJo2W7oK3bzJwrZ<;RQ2NY0S9g6^Mp8Gbw9LPH7@{PaKO=t8dI8RbzVxLwp{yD<+bO zWYI@cTk@{X%hPl&O7SdJ!83}y7Ljm@Q2_7D`iR+{qPm3(MpEMc02<^GdxZibs3iDd ztdTG6H($==_b09aVS?$sl0DFR$2h(!Ws*xnGQ7Bx`*%iZ)_TqT*p-=HS%Lev#ga-& zs_P2~%V)8bEwX=6-J~o-c#`uy&CS}!d|GKiK@>uB8kn%payjyB(x#P>ipEJ+>vX=y z5Hj*!Y%Fc_i5x~%l&46o3gr>aRZDt=U$pV}#+1qUt6) zUOxJ)gn`~-Wjg9C>Yb#hu9_&XVvw&Va5p38h!&|<6-1_|j7RfRkB+;P94+~a{UO6- z&NFn1MQMuJ<#{(d^jrD#KP&|?h$thfXgc#MF+g!0r2pqfh)gMOP|&525?_ z#a^7^RMiy`#*f1xh707ha1k%h5SC$FU4c*rQ=k1m^F*40oAV_6_L&~Ylgi0y{f zl7XMz1I8|Ajl!Mzwz0ANFx4u!Fx|(J65dm=-*gnb(n(XAs!Ae?qn23tO@Iq)>@Cxl z-k!KHSky$!2)W{SMt!N_Ly_W6oXic%gN(5h- zE}pn#${3h5bvNEDd?6g`udvxIDzx8t0nD|wU#JpK(z<09NX?AY6Y`V(>jV4n?YR{( zPJ!cnhW`L)&gLWWDybF8?~OFV(pEP;WtW%%yWMo`*{lkH9T^<7RvjOuq8(G$#2ToF*7eL08)IQLfdISR8iIC^sI%TogKkD4Ugf2kmG>$ zgbt*qWNFW1o-PVtbd7xMlR`|CMUUP~+^_ql=j2b8BARdi0F^PId^?CGZG4-bQgq@s zc6mnO&N!U9KSutOtjmUNl*KBV_#>CXmNY*Il^({#5-td_9#*(ZG~*!w52G?wYicW{ zjfgv~c)GBHTqU*TP!=RqawEHxDYYDY4AmYbeH@ynBdULh(^2KIuqU~{jp23mTlKaO zIVB)QW~L}N6sY^}i)6Mpi?9M#ge-5*_f32tZ!U|6Sw!n7%QLL(Cj6P6V;m^Q)nQBh zIsT^}p_w!q(5sz?bx)iS_A`k*2HiCNA?%OEt+Q zu1WdeP@K`*vnG@l}}{A%g+`01u$T&{@HIJJqn9=KWjMa3%GM5V2DJ$YeimBs5Twj#2RR?b-CgbIS7PTT>ZZvKy>U>0Y-d>O;S{Q-Go)%GX z{yI9UDe4HRm7{oO0?KWDk6pJL`e2+B7IP8IhXCq;K@l9}sAOkJu{uJl7PwU(z!C|^ z(exg0yFgTG`kCq!nBs%rN9<%H_YJS}!dymhOMroLb>$AQ`4csnX25yFYCSg8ba7J{ zL7W53sbTT%qzl`><$o+w3*~OA@ZE+K;=@~E!NUF@{*TN2cR**Ai?dsIc8nW-7yRvxx;@bXjic=8 zg^U|St~!>E!2TQlcE@3>Rw7e*#r$HsG}6?WjVna(sxbt7TW|FJ;`kslz9JIKtzi+w z?g3Hf)Uvw3yyE3$O9n>tBe$nR^T$kwC6$@8*|G$k!KLvQ`Zq-m!Zja`5P+x1^+yPE z(Jefa)6$lP36g_wDZ;MZJgZ3i&xZuh8+)@KcS27Dp(=4_v+d$H%) z8Ye(%N__DWvNVRCtC!;NW3m%*v5xj$zp6$niI#bzS!ECQoj(zLk&&2Xu&gogTQ|zB zI|$Q`klV|3C%Pc0nwm%oxF^F-Wj!x`XTYH2l-R&2ty27;T-DcQ~mDPAg?C8mN^s7H_?lI*;|*sC{9fWKcOh6zXR2 zfRXTyw>J7;*4Su_LnDK229cHGEjQCCDKzul=A`(>R9#qTMT8IKw6Nil+)cH zM^r*VO@a5U3#3z3$*^f$_v?m4r2}*flqqJyEg;i1vPKIjKz9pnxD``PgJ?Yx4=7fV zb3wx#*D#7nRx*(y7bS;aFM*YaHZ;5taJK?Vjj0rQXv{Kf!j7_rX(_9saWpcGQSm6wpq9r~@<8-A(V z%N6;?D1~%-YB&*b;~K6Xh&PG0QmZPIBb}-^c}6l_eW`eTo!4a*6MNC2sCsuwfO&yN zDtGJKz3@t~=~>3o>PB3`w&N8hvFt++S!HU>5@~&H$M0AtqvP_(pXW64!zWT!%s(E# zmKQ4paVMytL_BZew2{|I9x?e<#`xzkq+x)+R|Eh*Ki!WLgSo1s<$O%qhyZ)>f$aA- z&$DyS8`88aE(~*qBzlX22h#>H%(F+WTJnW6c(n84h@cz{QDwR&yEdz4sgy>%)-8T! zbL234v&}M^AXXI1VUNxzxZ$AMVa=53II?`c7BHQxBa9u11Fk6VPAT^YuQ-)NG$Sfj^1)|{W@C+l7%Jppy^{0Ve;Tltve}L zcWK=wuuwx1lM@95ToL~Ot}Kl-02CIBiW8L9mV!zRFUmpi_NBwov=VgZDFDt&blt3Z}+kVYVs*1a?ysJNeIg?-;DWK z0dQ^W`?1+bHim>YQ$<+t(EXrGRPe%|I;EDLrkN@ubs^(*wa1s8`hHjqZPTonug|c< z?WrmiNwj8c!z`}hcDSg?qmhT%iK&`vj~WsUfj!6|t;bsrJTG__XztCinafS91Dq^4 zYSnu{JADi}bcVf)Hp|@%YNVm(RvUs>t*^M-^1;&X1yb>=N-U{KUT$2V;*T&jG}MsE zDuxC#D)uB;^IrR0TLv})Q`G|TXKh(jwIo{f?Sib5hF=afMkB`LRU_dn+^j|a01eOg zoDvS$@?v2mn(V3Hx)|AYB$O*6_^j0Owb*jl?grz1tZ(Vl5+v5~S9R38Lraf+ut^-! z!$9h?d(#$s2VSSC>FIl7WZo2%rA`7yT|UqS#lA99@cNz#Dl1Q@>LWa`lkn0Z=_`-g z?yKLa+wXAusg`S{b8drw{{ZA|J%SIA!__#mpZZ|j@8CUt`Yrf-ox|e2eN@jZ?#4+a zklmS$$t3*oRY7&Zs`vWoWSQz3W%vY-YXUp}08|_9dT)l3As_@&lsmoO&JfHn zJxauIH0-rhsVV)V%*RK+4-g088r7%|B5`E^E_%ksz| zW?1AeEW{fP&6IvS_UVPQ4hA1CR`iDwa%!7s0y?AJp@F8jSKy*3rsk8?dSO$O3v}GY zsfh;K009kMHNjMrqtwC)JKLA0AqgayI5ZMyAo^Y&#gk=HOkz~0i3_&f02}^T<;4mAToKTMLFj1nxWw7)z%tS>xTaD*665yH#4lmjs2E(A4A~_njg7ncaO1Wb2Gc7{!mb?# z?tFN6!&8Uc7pjt-bA06*Y{FsU2;~>>{D~*^zjiBg*pXNArR;&bZ{yB2JP+Z-w2?JC z)^y&6^!uMJka?Hr(H(G_V9Nv|{u!}S-G@)F&e*F=O%x+2)WqKsKABd7DK}S_=T7+V zB|FvDGDRJ%dVmODr}NVhve{B%gDW8P>WYA(cm24`%%&nBe-z(~*}%)bWw#>;cK zu(0}Van+2fb7m9Mz^%lon7)a6&ZEZH3))Alh_3ZG*nyXT2 z>U2t&ftI^2g>B5D_mvcMJ6wUW+>P&cl2eLchF6n2`Mfkc&PR9^Ni#Gu+y84Q|aD3 zqtX5lWoccMQOJq{ga>86KtH(Ru`B|GV`6PI5KsO@$*8#0DItg{8FW(21Gq>Z5P-2uIS{k=D z)@$wIUjY$JbdmE#AeISJ^^|klZT|26o_On`CtE?E%vV`7Aoh90Dv2qinc}0WhFNwY zQb0q2v9N0c*Pz^E;D9Yqc|iyY04davIeYKYDe9iCoHB|@9Z9vJR5>395#hR8LzhkVm&ayZB5SpSV_#Mg{t^w~@fFHU1q-rA5CcC^(Rp zoSAA>#e}U@p*nB}bxxdNKcs6PEvr{@)bbJW#%?z0%s*?r?dgi*ALv(qr+?lVVOWCj zrT+j9#M6;A>Zs^A6T5NQp@)u|s%fYu@f3v|g_p?yApS({{ur?~;8C{d)li|G84YR; z>rhGK4jiNfAcR9M+YmXIt^Iq98iEK07O2!xw(vfXQ%xL|5Yn>;Ge)5aQ*K~>A-A2+ z=j(z3s32leg(wP8pB+1(2a0~t6dWjqe2*?@vrSY{Rm(7VC+xlV)iyRJTc z_N4}Au1<;Cc%8$-oIg)CLpPAi> z+>&uva$$2Wbgr780l*F0gk6x*8bEEaKsG@>U8s;KRdAO6wXgNW=pi~l>S@7oe#qK& z8Ejvpsq=k1R=(j=8p^OaB^)0y*#7KuG|RGTE&WLwKF612P<~*kW?peUblST|-T*ZP zTGz!nP*QlW;=;$`Q~XD+6cq&kq?(U*JYj)lOpsNf(Cj~+5V6(NPnqSlbj3k082}cx z{YLil7U_XsPe_{`wef-9Yo%#zkJJn-^ZKaryy0Z14`W6mjnXC|iEe8iRoL=Iy={A( zaK%5FN@gT(8!mMgsTxHa2nl5iQ>KL5LUg5c9_b0rBtutO39MM8ju1I~NB9X|pmH7U za!t*zbFcss-c3nL@{@D>Yww=mv4&rLr8pBrjrx(c+qBoZ7Fs(*)$9FGv@l=Y;~}P% zN8MZW9;cxCp4hay_?BxmE_8|{l!E79=>GsnxjvlCYpoH%^rV6qqKQ!u00NHR$i;80 zvsFf#+X!VL0BH^FHNzO;D8evJ_S(Z8^mHJY?LrwjK97zC0LBITGVgp68=`72hbF(q| z4?IPQyV0~6N9ync-<(+f62nd*IA$EUZv4s9)b~cbsME2;3lSHFmR}5>Pfdl1{(54R z>VruuRfKJ`4>ut$U3$O@@2?Ks1P)ha64Vxo1rh^(XSIMm&!=(dF;r=4nR79;V=Xf= zkIogW&>c?UxN+DZhll(!mZllrswo1y7g9j@mcVp7j9aA;Kot~=bErh2ODu~BWND2C zlmbGL#@x)(Murx02{$(+k@;=;VbctRrhq~16ok+M0oH^Kb@99$&}huksFq5kMX@ap zBl8a{{2*KWFmhT**%K+rR>jb3zTa~kDtLa>vlW=g^0@%J!pnR1xMSzm*zB}UnndWC z3mrj3jv)P_qdMX7o}1yBJVYkUzZ#S3HY4YOon%njCq&pr%XfYZV2(I&xj1*kYN}5O zDD>v1iMbgC(wdc>bOnPMELot%Bq)cJvH_xyIn)hMZ?FKTd= z&|Wc57dV4osP$cR83tiJKA)v?9Vr%<3!0X(6E?3N6K=6w{d%8~0A^4DvMx`^Sd&4o*~?FwdQmPS3m9LM)> z>4ViXil}vxdoAHlSV>F`pj6rb;!&7){$uK^Ce#s1&tFrjvKC$ZR}xjizcP%h2d^Q= z37eP(j;%@m0C4pc@)7DEk}0Dtvgq6>liWcS?TOHTmTOHU)ESiyr=wd2Xe3!!Z~DqY zI_^Bl^xF)Zb@YIwoWp_l#8dvr0v(n8KzN&$6dllI(bKUxF22{`Z>10Mke^~t{5pY z^24kZ;$n0h6s=vP4{&z@&4*81b~;nGBp)s`^Dx)Zd}!1fPekR@A%?z{qvn#Ds2c_w zU04uQosW_7$Fn%UK=N358IwUm4G0Hj;sxnlG=|P#GPtK*%AtG4-eL^%y4TMrDYKZ= z(NxU}&B(2D8D$M`QNMn;TRPgLpHXDgSG;aM`9*c4iFFxQI?40d0}GobZxe8Bh#bOjCaHfn zKdL@*RlwEl-kl>mUKw#-F`(-H>2HG)=e z2Kio(k2Kc$=fW;sp{-;_4kXoSdWvbsa5YN2jR7*<*z`DWoL5siTiC z(3J8#WXBysvvS<90Zzm4ALELyG?US`k+k|k97#E)$l@a}>0T9Qnbj)RwGk$z`Ha;T zSkSlRZPML48}2Y5mQ#cT3uquoTqinuX^RtZHulB+Uj|^yNEw{%Q!R;9ZGw^104zIv z{+Kl~r2_Er678}ogi--BZ5Iw@n?*oaQPQYqs;L8LU{G1)9-%?`n|b5fhVC|d@sD09 zLJ1}_j~6v-Yb{|s@j@wRp$kn#M2722PZ;HjA?M1VT=gtio0E%#i+LKU)W&5t>3!0u z-uwXJTIennWfYG{(PlFu)m4>397!O%mhMO#t6(}H^}<<&7hNmt5&9e9oH${UC_?N= z%&PA@I8a*gZJEyoYg08ASY?%|fbrxPDxk5sVyBT?axM3byVwCh#avv{!z_xYQ^f4- z$t+G5*5?FX`ww5O@WG}sUXd9>kZh2B`_eDsr7<8Ph2)8ao}nCEZMDeh z(*^`rWD2DP8fFz558gP}WR#0V%POmzk_5!uc?c{@Tffus>wz9@Lu(QhL0%Q`5t;p@cyM*zlc=Vrh<=fQmad9M{YlsE?mtz7 z6+VrA+ag-Ty2FP}4-AMU%mv5D0X>+inHM+@65r)XTPs?ty18rAE7NLNq#ZyissTw7lAwU5U_grGDL! zyfjc58qGT?4N+I)ODi4OjllB1YvS0$DNbc?F>Rkfc;rs%N8c4Vvjr|#a@dqLMAVZt zRFgoqWr_Uq`4*OOjL42r|+ie{B%I!Dvfo$JaWeQ5eq$7emefP|t#6GrA zBmV%271v=K@G7WuN@<5bRnCwLI*-J_l!v^ePCrmATas{Ly{-AJ1%Y zGl5FJOPG4{*Ji2QK}oJs4$(p&+l>4>$bFzR1to1IY;opP;J#m*R7YVMHWI;7s4f+N z0N89mBW|zdW>S=u15}>zO}C84ANY0Y&>PAU^LkfB1E{X!vNC3U>_4xlDJH6+smSv} zIh`n}Ymbb>uwHaf&2HDjrm++*fk`8~(3r`lei6kP%%FOtDZ#|*;Kqd;Psrqqw9QXc#!2T4&{NmiGE$Jjw=y+6w6wq>vOxX^)({IIGK9W~Z*HhHKnBQ!)@oOBEoD ze5}UN<4-uz3POlH4LqQ~DC3^I)3P-5n$tasM@~*wuaZzlsB*(&>3nl_8-QuN2xg34 zd9`K}3$Oc$^Q=#AXf7gEOtML@GtUus0c#4Vzh8t4epvNRDwN2d_>Z&_oO2SXB4vH? zI!|aWBGo8aHO_Q=ygHDd@~FfeAsQh+@gHcL_~s&Ym{)(|H612XF0Y`Z%5v&p4E1dq zfwJYIEbEPWgIgXw{p zQDsJgpz1foMQK_}7Ahm4)bx6Sf#mTbr>jx+wDK@sQ{0R7UtAXBcFx_>ebDjqgi1(l zRIQ>14)hF~<6bQCMkLGf)(pp}Gac1_U>P7zyiTEjOi$daBOVPe+U_QN3EbcIemQ$ZHTCVo|Lzh@xauTXk# zi3$J|fx2;~(GjX4Xru_kmAf9yZTxW=7QA){QfmmGY0S9SX`HH&)tx2GqKS`dHTEae z+ZAkGHGWk))#>}dbBP2uYZay}{lYjhsA)ov2^1&``nKtB=e8Z?08?Z#P$5M^6&ysQ zf$g*hSO35n44J?N0^2%45S0oz-slSo!(J@K2%)Y5cg#g#z9lRp;uSwHz`y-{vss7fW zCBvJuO6TuN$Emr$pOylcN?QI}bv{1)1YNM1WF?S>{*m<^=kMepy~9lx3^^@16yd5Q z0pygD3o#qq;@Kv^X{3bq#a9-aQGGwd@!kZpZM%;UH{?lJyOY zQdiJbqL^L?_!HqwD2=F@T&0NPO8* zBP=8M^{cW@5a}GtRp!Jx!or%4lJ|PbvaDl_4&!3XdMc|QAXJ%CGOUKcYwzELNr>XZ z50n+Uj)UI8wmj|jb>ihln%DY1ia3Y7O{yS~q&r5iOv@C=JBxOWQ zN5sZiAwDItgpR&&YqFb1Z{9sn)tbvB(it69c5ha+&_ySNv4PDkM7?<<7wfgpVQZTV zIwcG;UU^wkH9muR+yrgXvX9KZhM5&7umQZrbIh}9{co98W?7Li(xklJ0dImZ1;;V{ zW&+>C8J;7FOk%1nH`Qot-^E{T1Z)z`vxhQ>Dcr+S6I(d2K4TXSU6qR5Kk93hev9y-yHnYO=Pb z(z#Uh5=;WNV^R=Kt9zA2TZJaydzg?fZnwv!67stBmB&vS);ej2rfBTSsGVuIwE776 z%krI1m_W?M5eEg!G)2gY`fWLzQfep4^GZ=kOP*0rMdLJQ zlf()ofvz_AWsMJ=iNW&92o#~zSn;v*@P+uOM8R#wh&e&#G$Xn@aQ8BbTDokjF^pAY zxvkOZkxV3|BM%ys4+i(e#E0un~rCxDc#2-c^z`h^(A-36U(HL02)WmsWzTE;j-n1E9YP}mF3+Gz zBP-fyqmDU|y>1on+)3sREo?X)WeP~pQ1kryN9T`^u&cRFI5UDU>a^u=^nhP#y@Cc z^GI=GRSL~RRUPN???C`rwsf_+&b~!rRtTIE*Wx4#eiy}dSrto*J4ha}3$6r9280Rc zQZ-W$E=lcz6ac&=ao~tZYkc~rQs!{!%z+JL)ib3tNfzd58M#(8w_?Oxj=efw_8dA> zDGoWV!Rz+j<&b08;^$ZL5+=mZk+kjycJsGQHT1jL*MStgF3_1~RKPUR$Imp!tdh3! zIrAj+^a?v+s*qh-S?uKv+6a-CM6-XVQ@ou|y_AQA23HgaP^jfS9{n&%8nq$^9Z|T8 zSSh?FRYn94+C6Yk8DJ%^FeJq0szhK1!UEl+w*GkdBp4VVkq~n@{_B%+qZ?|Z)oi$QNztGKB~{)ilbKM zA!(tdX;`fkMT>YU*STVOq?L!?VQYY@F;`1N4`VvTRCnur{_Uptx8X8Mn3?Fa5n7cW zd+Y}7#vsv9)oKji-fEqyDj9*1&fPj~*ju0-FqyO&mouwa(H>h;taMG;KKtWJ=!!8% zmqM52Jfq+u_r?8&$`B}R7~UkZw+V~$OX5_hlj90@?YDjUd10Gi($K4Ph~{%`nn&eQ z&FG+P20jZF$~4#aiVXDAn1iL$$PL5qyub$w%{HFmUXsnTM`*IT9Nvzmt=2OkSW|ttJ#Z*c+ZjaDE4&uti65{JcB%HfCpxAdF4 zBln%8AS31jN znWm_d<4I&BxhlnnF*}~1A488HeQ20VyQ784wepEznx>>zZ7N8wAZU0*S{%Z=A&Q=y z#$oUog-{WOlH?n82Ez6?9dR0x+lnh1oi{@yC14<(2E1+ezZjE{YFzHCHJ)j!B=AgZ zRS6(1l##eMv9*rGZW`o1)ekqZw*4 z`qVs(it|ITI_IZz(z?n((Lt-v|j3%(FI=`kW@iO45h2qT8z# zc?Ll`!IJW+4%Mg@`)(15{lf}Y=3=!CEIZ6uR-}MN;Ts1j$YZY4j@5_36v^*?A#PBH zKZ^J5ups(h7Fl*o!b5vcoMms!^b-|E+jc76u!5IZSMdj0$(z;1PaKo-#mPp5MarbA zunbAxuIIhGW4X7|^Vtw66H%_WiyIQ@l#~pDZNFYHJ?*VULznQcN=EP~%o{f?9HqV` zkKNy#d2UMe>0yh09Ni@xQU3r>nT%`WB38l_bj|2}1Y<&>PdOL1;}XV5M%aTxL)95|W4m6Fw8^?we-KbtxgLnFm4a09J zS8#*hYIOcWZ4d8u#!iZE_`lykFr4}|$+*|edshFcT@^d1lw!?RDSP1)(y zl__Ua{iz+mRPcd~d&G6kDeGyn#$=&iurpYPs4cm$JF4>e?s*~^>d*j+_KZXi389c`1hqj(Ly5rA25{DFTUDD z*!A8`TOvgkWlM2;A$~^b>Ggf=K=`_rq%Nf8WW)0H$rbzXr`rxQ5@XDMdfu^ZWvr2i<}YFLFj&XEvDUQ zRaEc$QVwegSgln-${t?EdS5J-497S*G%@BGePYwWEMn~(Mk4LYdjQ0nTw7z?bBiV) zD%qsdgsEI~2G^$ec`GKwA-r=e3$NZ1(|g6(1I0^i97 z6JjoxW-{y+?TdaC$I8l;s>)sa1!z3aGXEakqB`~z(Ey< zPG3rC;f$zGW3`{H@or}+6pZ2L0+eeO?1>oVdo}t49#w#xsDr2QnZRZ*Ew9So>^;ZBZiTd=vn?d-*5+Bo%(HvmQsP4R)$PVmy{CH zYN4*^%-nPKr^xtym9-*D^JWvRsp+F5-fAvL78W5B&PLm9xsKRM{P0pqXwYx_4X4X& zTiB8e5VefT+%z0X_D;bbZ*hBxIsX6^^#{G@HGJ{JS!wAIp9Yx?t`rk;O@-L*Y&rk{ zSci&}!_4n7Qk|M|Y1*jk;5d7{DO|<+Vi-zZ2=3h0>N;QZ!$F&(D%xN_y3Kkdi zf{3*8ItB~Oq-|^~GX}Ja!Xyx_m4coNJX&{INtf4H<<;R~sg1r7Bt_A${_CV~=Vfep zH1A5-<8DV}AMne6!{=8lvwcStkx#LWeXQss%&2C{-ZBzuCYTPpte`O4`)YAtnQtO; zCr~Tn@4hwpOvzAm>zbY8@d+$~3Y`z1c+F4a#2E8OdkJGB;#>3wp%?Bjs6LeupSSN8 zN1061B`7pH_QB3)k?GA?iBm4UytbYbo^HLhe>A3mz$EsmTyJQjXWGihmO!ZxC z)0IbfiybS&-9L~dnVxl-!m!46Qo*^6y8&|FA$$2_rlg*lBq)TWYHA00*&Vqw_@itf zl^BJZX4C6g+KdC2?|w?sVAry^_Bvt zgj@qscafkR1$EkVu!XbJ!KT)8b&kJpyfZklnZ>1eNt;uKDOH-s`aYZm`^AN%`UDMg z{usGSr83Q_xl_F zW2y>TJES)@^V=MgNcxqJgCVArNE9(sa2aoZ@avChm{L1RvMRAYpDkR<2Y(JKV16?i z6Thh>orWPU6es~x1mzASkO1WbdZrqm($g%g@>Q2`?!>be1cFKWVN|M;TL)#MiJa~{ zO3hSGrsfVc{$ zF$_pKF5HVBN^9AOwN0ZvOyvA(MN@Ex|r}j4D&-Fj^F#Mg86!%evTALcAfHpmQYF88f3JL7372 zvcn~dI`(mHzDN9VWGPGSiI$CyyC`{a*%d5N3XyuLx)rAm5%H*U*QIW8@SA=ZT(Y*+ zVqjLLvnMXt4ydfJiOChSf(FD$cE@R%eL5;QaI&F7kr2PjT;C|DjIL`}8W*{aCb@fW z$^p3E-A=A#P&!<^Ykt=KlbW$)m9N5WyCX0a1j6?$wHxhHw$rxO(X zO5={Gugt{9P}Hi@lgZ%n#*Ez=xdp+$t@g!{r`>f`DIBTq&I_LTd16CC!*2Ld>K;7m z*NK($mNAolUj`uW{{WL=hYiZM?x-Ed?-WVwWyU^(PU@cxBV>C4XIZl8T~?XxOH^06 zhcab#3eiX5zbaUCBu8_$*SA~};BAHpY)}gM5414ACSIJ*m(^7oQ*f!K&T*pj9+{}m zCoIZXxi0PI#N406oKl%#Pt`^=tXg#vsLBqO?1#pj$dfFNH+3A=7YDbP#im)H5-k~Q z+|hWDAg!iIVgxYeW3aW);fF{8SA;<&MutA=W)jN9*<1^3FLm@kk6p1nlN>EMMT9K) zvBey_Ou*ETGD|Ej9$5{Y%LP?Bv1_i_xa@+ z4?e7uG0*0#tEhUKxt)lTS1j^1t;|W>*c>l$N>D03_(JSy*5xHlC<=uDfn465V3BFT z<*2CUum;30bNSoniRH;y&9HgKT*^Y#eNU7KOF*<*){dKSlhTz?M%TM-_^uuWmn9Uv zputU`HH$Rffk~k8^@nCvrsAy1>F0q0!cbX2>9GF*AE-F2)m1gtH6$>Q1vh*9L>)z< z>Ukrqr3c1_{{RU1ScBXjG3D#@$6>&LLbNci6T{{n$n>G>qP+VL2-X>Tq!d{!6%Ef) zGUb8Go%zU&kWc%k9ZWV|HKxx|8XU6S=mvdlUUa{VOoK?|*)1&DhM;6<;zKLNvlktJ z7VoegrwbdydSzAHC&~?SdFM!OS8c2306Km=msjR>MZ{WYl3i)mcABI$K3i zR>c_3B+o15+Bm6S$6`-nK_GADMm~2AeIxu+)El<;0rrj#DK@kc8_1EZZ4vc`tf%3= zp{~o9B%pdKND=Dls%jl9r30`IqSgS~=WkCo49ZH&af((g@jS%p)EMd`{_1*p@q_}! zdln>;Z)4Q={!SxtYFdhvYD`hWNr)B4#FLb7*xRSbdSk&5yJJXkC_K|G zm(e@^Kf~7%*c$Hh{6ydyR6*vi{{T!YfAI(YcExH>{Hl3+2$|pOz@>NkR6pB{H*fs6 zWI&(mj&Z#>DgOZ3Qz8EV!)G7=03v)#GyYv%y(4k>clr!-ANd$S=f8fJ`xws})m*(J z^f1;LoDly2M{EB8@vZ*=BH`M9qMqMKtKuJ5Y#Ud9toaY)h5rDhP%OU4vEhgMWB&l? z~IRSf}Ho^16h~- zTC3)`5r0}@A*=*tf9XU20O4f)%a0SO_qHZq^vopnmstLvyA~-;BQV82fzB0~kNvX$ z0PP3x#jUUD2*~Dt&7N`S&$0LYv95pkJU{bP;cUKhPzVj3juXJ&!=}GfH^5FLMj+6SM@*n!uZ~G?w!xAg$ zj@6@l;}>zK{;uEn&;FLhGGqGA_IZfcF#iDRb`K#L!#w_)r~Q!q*bRD)abtJ-Y3J%8 zJ74~l<=^;eKlzbk#s2`7Uoq<$Z^fVHoBsf6zw%60W?$*bfAIGI0GwmFZ}Nxz-jPg} z{<=9IF%-JL{W#Nq+W!Fi1{uo#0F-y$$U1ZX0C1G-{{YG#+Bf|dqyGSZ{&&TEeQM|E zBUxwYPrb2cUH<@9f0jOT>4MYs2xsQKf8RgD2e105_p&W=zd_~e67`S$lTrTOm;Oe? zxR?2L{{XlD0MLl#{{X5vDiyiU{awGbn9l~i@okm*`wyfN@Vosgp#IEn-_6h$0J7bOAzCt`t z(m6#$ztXlP5G$I;+x_JJ&N$x?on!8w`epf#-ZA2|i4*mRo0)0Y1 beaHFYV;qePS`MrJ%0KxR1CP8Eoss|9jfBBG literal 0 HcmV?d00001 diff --git a/onnxruntime/python/tools/quantization/notebooks/imagenet_v2/cat.jpg b/onnxruntime/python/tools/quantization/notebooks/imagenet_v2/cat.jpg new file mode 100644 index 0000000000000000000000000000000000000000..f1e32c44819e20ef93c496fc20635b29ada4e350 GIT binary patch literal 5869 zcmYjR2QVDm+FllmM7JcXt+G1Ndy60zS)EwqQ=<2>y681pB&+wjR!Q`ps3Cf+)l2wv zAw*BO`F;1^x${3~&b-e%&)d$tGw09zpKkzaEe*H^fPer1AhfiZ!>?4s;2J^^82VMaD_8F4{rULj!t0&;S4 zN(xE{H8n(tg$*Y3KRo&$`+r2>|MCCQKRo~%QUHcP5J@QAcDKhJC&AQ^s&A*2_1)5#2b1s)#Cczug3;p zPVpy(Hr|ozCk%JF4+;K*|JA)~M0l5aCvnr<)gt@{cdh@U6A;qSvIFTv9~0?Yb8vb^ z)YJc21l$MyyB!Tc8SsvSlu)Ft2^!B%8c9eRNgdBm%0sFO4S!pVh`?1KBBs@KDByAu zjBE(tSkJj$!RdDRQ})Vwi^fyiSY+LNfcJzu3=U*v=3-m+WvAT}Z1fr#6|S1rNyqrU z<%;0BFDhEX(a;3h8Gh9kbQ?P2h6wi;cAmP3e41YK2_Up z5OxmbLkJMXZXbF$FDXvZ8HB87htlv#cUF;WCn!1=;g-Rmr8y&3(dKiaZdIUYO zEH9bLKoN}Q&Is$vlzQnaVAwH6j;p1QvE)!Al@eloUfu|O!gzYx$UN`^zTU@NCBz!! zo_d<(f%=h&#vr~eqDo?-nbuXA;F_Gw`xXD%ad4}-S$<-YONp&LY|OPhQsS18oX85_ zT{GT3{elnr#3nCIhPPs5=%JZ?s@lGc?97ADQ!;B7PW{W|Tx)PKVjY;h9C#*B(2d#I z@r25zZ%Fw9V75c@9$zwbkiK~bu}$)u58F=WsSvzE;*jE^G`pVcJp+eKDwO*PP7A$6 zQoLZvoIogdL{)5z}78FRM1-Y{?RPjRt2;7V9d z;Zcv%@%QU1_DC#ORY>qsj(k6bLEgM=I|k`h%#9IJ_W&(kihJMzcOGigl zhFKKh;mXFwXrQkb&g~3&n@vGj{d9cY|5hhvk}c~n!!#LZDN&|3a@O@!RZFcHM!#7M z6Xi=#N%*%%2qWO5jebXpe#rMlMTqEA=?_^AO26u{VAz14b0{|yrfHDdg*Xtpnbdb=rU zT%TB1^?bUr^l3_ID(mAH{c9`Dlj8LYURhgP24Xl^MRl`hwh-_H7I^aspOJqPYTL3E zqOlsqestnE?@9}w0+1|zK1i>P|7xdaG1Sy4^^q>SBTic*`qh^;CmnqhC;P-m@Nhaa zuEKGhAKmZVDUh#{DM3AlV;@&So^EGp1`7YB-?2pUT8~d&k^}yesIZObH*w+0)!Edp z6kWw)P@rR`oyYPbn%C$!4v6mD;tZRrCLwmc;kNz$)_cL^k~)o6>LFDn# zQYOD?yMcEN_sC}fzRJR^y~rFY<2J6n$P|Zvk3jQ!+S*X&yt$C$WwO@O4cInxDHX96 z#vIE$C9dI`bKU9w1JKr=YjTd=%18$`bb2Xu)KX5&935E-td%@RSBYrda9YcUkSA(5l<8B2*C_|%Tv`rc^ zoyXPC;VW6gZmWDzT2aoxVa#zTjnrhx3O$`-8X0b0(2mo`UAzfh+`R+`g(~L?GMF{--i0*Vtb=?QxXaz+d>$UP6Oyo`6~M~!L;dMYw-JO?q=G{;=|z~ z#X}~>X|Gd)@6e%QJ+awp79m*IHyY)0fwFpQ>bV&Q5s2jZNS)0E@uBVF26T)Z6%zDa zH+^h!&s|t58I)L2@f>rCZfpao2Jxr1L-QvpBk+Fjj_YpIe3}orn!m}W1txG68c)ph zVYjc$6fbT!Nx$T77X>PDy0(3}m}oBBQ802mAp?zCA7UGp_;cC_cOHt(5jL+ zUAkaa)Y@Mt^7qGerwsin0T|u#lwPYIScNZ)NSgo?b=sL{ zPi~SPjoDv`6@rl!Y+u!H3rz5kasHkn@^c-L?xBL^N`&hIR0@VtCY9yIn#Y6&RPt|s zBp_?PSA=SaEYI#~xW!)nl(c`#uk4Q`D9@w$A;oyVzIkhF< zWH6nFVTe(k(vAmGU#Em&prq8yaSB>}SXn0?Q>dn~;M$Ufa3oel2Was0mmiOsoq_$Q z_=BDL_{Ac#%uC(G=!Js+D6u6 zOTJF3T<+agMHc<%iek<_8AgS}O?FrPfzFN;J6&`c**0RF4HZn&TxnbjmT*6;Sl}wK zm%I(hoGcuj-xg$r_|$KG2O(30hpC`S4xRcHd5oskVTht(3)`wF&OFvq__W$V$!2{b z6%h&bg9YMoh@l7d?CE;0j|h~-syGDl#0J0~dw+{!ru7FX+owaw`327ZaP5XJM*&Lh zzy70L06Z^g{a5#7Pg1)t=TFSMtu#Q}k~2V_#dzl;<2}T!An141%+-sUP4O0m=b*7Dxf& z=zudHE>(T(_A?pirBjAqEEu|rZ8O&y;A-Z?p_~G0WLW~hhjrKVXCsm{ksW}h`B59-vFXtEC{^_2keIV!uvSVfw9#UND6+hzb4*fTE_G9bF&QwLh4faMF*{_t# z;u5buIc>yS00-@vxKCBxWMyRypLrQ$2EZ}PM&Guagp#9%BtbA)=o1?`!6^)L73>)PMjyeQ?g}j+qYZh^vItXaCT4EFVPam3cLzanhoqwB?QM_zBOn5Ya)&t$JqC{4&+$ zYx^n-^Wn6{3Hw@z@z1>S=b?=j%`K=FMQNFQ#wQWxc|GVCF(l0|8n92wZRK$Rh?T?q zQ@#cnEMq{^uC$p@pgg8vgZwA*T*ub9t@c?P-=2-9=P!Wh2S`=w43CAd6X!`~pP_<^E@j zje;NL=S@Gn?9;63kQ`z!J$r{ucfPGYQ~)3q-HjGjGtd7z}O`WsT}fWa=9uIq7)n!i71`d;~q za$g$w#`vIP({|i7pqg~Xw=ag{RgZ8)pl{ZzKl~kb&D8J6vbyqovHF?{a zr>vOVWI}IhWB{;YNl-skHciJ-^5Ff5(w7OzXMX_l8qpHI(}<5+K|hvV@_yCMT1un= zwgU%}mDPcHd*KA`g;j++Ff=?#u6}?FqH760;E_#%X5UsQo|*k!p6zBGYvD_Si%oAh zkuS!8o;A<2YAr4Ac{)CqPH}1SDe}IZJsxWD87=#ZK{qCnsh%g%f0+&4@$1#z>)ikPV@PbV8g=%FKR&UI#U z793aJ&UMCn-TY`qVanz3{dndpy-x8Y3ECjSF!Dh`j>CR!|6lPpb?LHp=`nbP5%gNj zhgOz_gS9i3qqP--&fdD%l#(Bs$}zL1O!J=iHF5W<`+w^(Q6!Ii?(Gyg+-EM$7<62* zHtA@7e-bf9%P83Jc{Mj{i?gj|ASwqM!(!5wKe81p8)s~tsr&^68$<-}1?(+@iakM* z5tyjV(00ds$NoNgME_GlJKcPlFr$a~TAES zJc}<{_nl6OQD7H^P`WKr`32hNq0MIXcHXFmhBu?0M>SCw{NzDwM_2~=Uu??Q@ABC* zgt!^O(eF~#t+uRdHuYXE>9v)P+N5@VL-&`du=uu6>4wLUnDJF4-YRA5hXX?zobALP zoH%-^r^HC7c`U}xoExJq*ia)N?s(ZNFbZI>pHOQF?rNsTN zE2yJfN-U|hmEcRD$iJ76Z(NO>*#{a$9DkQ}`rx{uRLWwQk^cAD2A`jOvAxwak`zE7 zAMXEmuVHmebE%%jW|jgG+sN5Tm`U~T4!8RN2QN;Wr_ltHlU5~dw$4DUPUJWLQ6XT3 z(@79d>&FSU%AA&Pr!*^yclh_8EB9vtC;k9B+LOMac@;7Ue>vzOxfMx9$s5C`PJRn_ zpp*B4CaNgY_U00YyDC_WS9FG8<#A?s>~6*MhGO@ecC?>r2l=8&u{_UJ1eJo za{U3NH)Xh|FuK*`wa~E3641V_*QI%iu#lm%bDX`MFB#x83alh7Sfd6rrB@U$Qa=Wq zvrVsmwOI;wcIG&8GM0Vr{i$#|RAZ=wq_O+IB`&ZvU`*3lVD=oAmh|?N*;qhf4s3$ z87gIICm-ZsaG}`6b7u0p!;2tU5lWPHnCwvRr4XGXv+`7~-hM1CD zomZGLF;Ayp>ueIrP9HlCDZeDXDF4k zrF!njh*2Wvgv!N2Rt9*jvRGa>yaU}VsI-!t8 zojWQEV4A*fwE+~{ho>#E@<)^AGt^8OvX0F|`m%2Qt&PdTW1~3;Yv@&Z3TvEh$M@r4 zmmmr{<9tm|*y=Ka+uF-`e1UM(lM5qC+U?zMC;J9VaL)G~hw9u(O`qL6i6eX#AxFP1 zzkQ`HY3UFDMRv(|c$^qBKyEKX%dW2LkvPlBqflsxUwSpA@T1sx-%4`g1K=A>#q%lLk$SZO%dn>V;NerwNkZIxW_azF;W@5#awB`!E!!1k_f)0crvc0 z=ta-uF+@72QfD;g^b8I|>uSEUTmK|dD3NYZBirRj3!$IA^y#C8i}D@J>NQ6!MW>sM zZdqZG@iV-9dX6H5aGpViWGLa-`|O;u^ga8(>B=%LBbLxsvW?KP^ntyy?)B}MaX8G4 I?a#vh0o3x;v;Y7A literal 0 HcmV?d00001 diff --git a/onnxruntime/python/tools/quantization/notebooks/imagenet_v2/mobilenet.ipynb b/onnxruntime/python/tools/quantization/notebooks/imagenet_v2/mobilenet.ipynb new file mode 100644 index 0000000000..e1bb201ef2 --- /dev/null +++ b/onnxruntime/python/tools/quantization/notebooks/imagenet_v2/mobilenet.ipynb @@ -0,0 +1,351 @@ +{ + "cells": [ + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Copyright (c) Microsoft Corporation. All rights reserved. \n", + "Licensed under the MIT License." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# Mobilenet v2 Quantization with ONNX Runtime on CPU" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this tutorial, we will load a mobilenet v2 model pretrained with [PyTorch](https://pytorch.org/), export the model to ONNX, and then quantize and run with ONNXRuntime." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 0. Prerequisites ##\n", + "\n", + "If you have Jupyter Notebook, you can run this notebook directly with it. You may need to install or upgrade [PyTorch](https://pytorch.org/), [OnnxRuntime](https://microsoft.github.io/onnxruntime/), and other required packages.\n", + "\n", + "Otherwise, you can setup a new environment. First, install [AnaConda](https://www.anaconda.com/distribution/). Then open an AnaConda prompt window and run the following commands:\n", + "\n", + "```console\n", + "conda create -n cpu_env python=3.8\n", + "conda activate cpu_env\n", + "conda install jupyter\n", + "jupyter notebook\n", + "```\n", + "The last command will launch Jupyter Notebook and we can open this notebook in browser to continue." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 0.1 Install packages\n", + "Let's install nessasary packages to start the tutorial. We will install PyTorch 1.8, OnnxRuntime 1.7, latest ONNX and pillow." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": { + "scrolled": true + }, + "outputs": [], + "source": [ + "# Install or upgrade PyTorch 1.8.0 and OnnxRuntime 1.7 for CPU-only.\n", + "import sys\n", + "!{sys.executable} -m pip install --upgrade torch==1.8.0+cpu torchvision==0.9.0+cpu torchaudio===0.8.0 -f https://download.pytorch.org/whl/torch_stable.html\n", + "!{sys.executable} -m pip install --upgrade onnxruntime==1.7.0\n", + "!{sys.executable} -m pip install --upgrade onnx\n", + "!{sys.executable} -m pip install --upgrade pillow" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 1 Download pretrained model and export to ONNX" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "In this step, we load a pretrained mobilenet v2 model, and export it to ONNX." + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1.1 Load the pretrained model\n", + "Use torchvision provides API to load mobilenet_v2 model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from torchvision import models, datasets, transforms as T\n", + "mobilenet_v2 = models.mobilenet_v2(pretrained=True)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1.2 Export the model to ONNX\n", + "Pytorch onnx export API to export the model." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "import torch\n", + "image_height = 224\n", + "image_width = 224\n", + "x = torch.randn(1, 3, image_height, image_width, requires_grad=True)\n", + "torch_out = mobilenet_v2(x)\n", + "\n", + "# Export the model\n", + "torch.onnx.export(mobilenet_v2, # model being run\n", + " x, # model input (or a tuple for multiple inputs)\n", + " \"mobilenet_v2_float.onnx\", # where to save the model (can be a file or file-like object)\n", + " export_params=True, # store the trained parameter weights inside the model file\n", + " opset_version=12, # the ONNX version to export the model to\n", + " do_constant_folding=True, # whether to execute constant folding for optimization\n", + " input_names = ['input'], # the model's input names\n", + " output_names = ['output']) # the model's output names\n", + " #dynamic_axes={'input' : {0 : 'batch_size'}, # variable lenght axes\n", + " # 'output' : {0 : 'batch_size'}})" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "### 1.3 Sample Execution with ONNXRuntime" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "Run an sample with the full precision ONNX model. Firstly, implement the preprocess." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from PIL import Image\n", + "import numpy as np\n", + "import onnxruntime\n", + "import torch\n", + "\n", + "def preprocess_image(image_path, height, width, channels=3):\n", + " image = Image.open(image_path)\n", + " image = image.resize((width, height), Image.ANTIALIAS)\n", + " image_data = np.asarray(image).astype(np.float32)\n", + " image_data = image_data.transpose([2, 0, 1]) # transpose to CHW\n", + " mean = np.array([0.079, 0.05, 0]) + 0.406\n", + " std = np.array([0.005, 0, 0.001]) + 0.224\n", + " for channel in range(image_data.shape[0]):\n", + " image_data[channel, :, :] = (image_data[channel, :, :] / 255 - mean[channel]) / std[channel]\n", + " image_data = np.expand_dims(image_data, 0)\n", + " return image_data" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Download the imagenet labels and load it" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# Download ImageNet labels\n", + "!curl -o imagenet_classes.txt https://raw.githubusercontent.com/pytorch/hub/master/imagenet_classes.txt\n", + "\n", + "# Read the categories\n", + "with open(\"imagenet_classes.txt\", \"r\") as f:\n", + " categories = [s.strip() for s in f.readlines()]" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "#### Run the example with ONNXRuntime" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "session_fp32 = onnxruntime.InferenceSession(\"mobilenet_v2_float.onnx\")\n", + "\n", + "def softmax(x):\n", + " \"\"\"Compute softmax values for each sets of scores in x.\"\"\"\n", + " e_x = np.exp(x - np.max(x))\n", + " return e_x / e_x.sum()\n", + "\n", + "def run_sample(session, image_file, categories):\n", + " output = session.run([], {'input':preprocess_image(image_file, image_height, image_width)})[0]\n", + " output = output.flatten()\n", + " output = softmax(output) # this is optional\n", + " top5_catid = np.argsort(-output)[:5]\n", + " for catid in top5_catid:\n", + " print(categories[catid], output[catid])\n", + "\n", + "run_sample(session_fp32, 'cat.jpg', categories)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "# 2 Quantize the model with ONNXRuntime \n", + "In this step, we load the full precison model, and quantize it with ONNXRuntime quantization tool. And show the model size comparison between full precision and quantized model. Finally, we run the same sample with the quantized model" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2.1 Implement a CalibrationDataReader\n", + "CalibrationDataReader takes in calibration data and generates input for the model" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "from onnxruntime.quantization import quantize_static, CalibrationDataReader, QuantType\n", + "import os\n", + "\n", + "def preprocess_func(images_folder, height, width, size_limit=0):\n", + " image_names = os.listdir(images_folder)\n", + " if size_limit > 0 and len(image_names) >= size_limit:\n", + " batch_filenames = [image_names[i] for i in range(size_limit)]\n", + " else:\n", + " batch_filenames = image_names\n", + " unconcatenated_batch_data = []\n", + "\n", + " for image_name in batch_filenames:\n", + " image_filepath = images_folder + '/' + image_name\n", + " image_data = preprocess_image(image_filepath, height, width)\n", + " unconcatenated_batch_data.append(image_data)\n", + " batch_data = np.concatenate(np.expand_dims(unconcatenated_batch_data, axis=0), axis=0)\n", + " return batch_data\n", + "\n", + "\n", + "class MobilenetDataReader(CalibrationDataReader):\n", + " def __init__(self, calibration_image_folder):\n", + " self.image_folder = calibration_image_folder\n", + " self.preprocess_flag = True\n", + " self.enum_data_dicts = []\n", + " self.datasize = 0\n", + "\n", + " def get_next(self):\n", + " if self.preprocess_flag:\n", + " self.preprocess_flag = False\n", + " nhwc_data_list = preprocess_func(self.image_folder, image_height, image_width, size_limit=0)\n", + " self.datasize = len(nhwc_data_list)\n", + " self.enum_data_dicts = iter([{'input': nhwc_data} for nhwc_data in nhwc_data_list])\n", + " return next(self.enum_data_dicts, None)" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2.2 Quantize the model" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "As we can not upload full calibration data set for copy right issue, we only demenstrate with some example images. You need to use your own calibration in practice." + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "# change it to your real calibration data set\n", + "calibration_data_folder = \"calibration_imagenet\"\n", + "dr = MobilenetDataReader(calibration_data_folder)\n", + "\n", + "quantize_static('mobilenet_v2_float.onnx',\n", + " 'mobilenet_v2.uint8.onnx',\n", + " dr)\n", + "\n", + "print('ONNX full precision model size (MB):', os.path.getsize(\"mobilenet_v2_float.onnx\")/(1024*1024))\n", + "print('ONNX quantized model size (MB):', os.path.getsize(\"mobilenet_v2.uint8.onnx\")/(1024*1024))" + ] + }, + { + "cell_type": "markdown", + "metadata": {}, + "source": [ + "## 2.3 Run the model with OnnxRuntime" + ] + }, + { + "cell_type": "code", + "execution_count": null, + "metadata": {}, + "outputs": [], + "source": [ + "session_quant = onnxruntime.InferenceSession(\"mobilenet_v2.uint8.onnx\")\n", + "run_sample(session_quant, 'cat.jpg', categories)" + ] + } + ], + "metadata": { + "kernelspec": { + "display_name": "Python 3", + "language": "python", + "name": "python3" + }, + "language_info": { + "codemirror_mode": { + "name": "ipython", + "version": 3 + }, + "file_extension": ".py", + "mimetype": "text/x-python", + "name": "python", + "nbconvert_exporter": "python", + "pygments_lexer": "ipython3", + "version": "3.7.6" + } + }, + "nbformat": 4, + "nbformat_minor": 2 +} From d1acdd4f4b1c9983c2b8064d8d5bf6c5fe1e6b5d Mon Sep 17 00:00:00 2001 From: Ben Niu Date: Mon, 29 Mar 2021 15:35:30 -0700 Subject: [PATCH 022/129] Support building ARM64EC onnxruntime.dll (#6999) --- cmake/onnxruntime_common.cmake | 11 +++++++++++ cmake/onnxruntime_mlas.cmake | 2 +- onnxruntime/core/mlas/inc/mlas.h | 4 ++-- .../core/providers/cpu/tensor/cast_op.cc | 4 ++-- tools/ci_build/build.py | 18 ++++++++++++------ 5 files changed, 28 insertions(+), 11 deletions(-) diff --git a/cmake/onnxruntime_common.cmake b/cmake/onnxruntime_common.cmake index d414dd1f51..222eab1ec6 100644 --- a/cmake/onnxruntime_common.cmake +++ b/cmake/onnxruntime_common.cmake @@ -63,6 +63,8 @@ else() endif() if(onnxruntime_target_platform STREQUAL "ARM64") set(onnxruntime_target_platform "ARM64") +elseif(onnxruntime_target_platform STREQUAL "ARM64EC") + set(onnxruntime_target_platform "ARM64EC") elseif(onnxruntime_target_platform STREQUAL "ARM" OR CMAKE_GENERATOR MATCHES "ARM") set(onnxruntime_target_platform "ARM") elseif(onnxruntime_target_platform STREQUAL "x64" OR onnxruntime_target_platform STREQUAL "x86_64" OR onnxruntime_target_platform STREQUAL "AMD64" OR CMAKE_GENERATOR MATCHES "Win64") @@ -71,6 +73,15 @@ elseif(onnxruntime_target_platform STREQUAL "Win32" OR onnxruntime_target_platfo set(onnxruntime_target_platform "x86") endif() +if(onnxruntime_target_platform STREQUAL "ARM64EC") + if (MSVC) + link_directories("$ENV{VCINSTALLDIR}/Tools/MSVC/$ENV{VCToolsVersion}/lib/ARM64EC") + link_directories("$ENV{VCINSTALLDIR}/Tools/MSVC/$ENV{VCToolsVersion}/ATLMFC/lib/ARM64EC") + link_libraries(softintrin.lib) + add_compile_options("/bigobj") + endif() +endif() + file(GLOB onnxruntime_common_src CONFIGURE_DEPENDS ${onnxruntime_common_src_patterns} ) diff --git a/cmake/onnxruntime_mlas.cmake b/cmake/onnxruntime_mlas.cmake index 15323d6b59..0a3ccd389a 100644 --- a/cmake/onnxruntime_mlas.cmake +++ b/cmake/onnxruntime_mlas.cmake @@ -54,7 +54,7 @@ if(MSVC) ) list(APPEND mlas_platform_srcs ${obj_filename}) endforeach() - elseif(onnxruntime_target_platform STREQUAL "ARM") + elseif((onnxruntime_target_platform STREQUAL "ARM") OR (onnxruntime_target_platform STREQUAL "ARM64EC")) set(mlas_platform_srcs ${ONNXRUNTIME_ROOT}/core/mlas/lib/arm/sgemmc.cpp ) diff --git a/onnxruntime/core/mlas/inc/mlas.h b/onnxruntime/core/mlas/inc/mlas.h index 92f0958be9..2e59364cde 100644 --- a/onnxruntime/core/mlas/inc/mlas.h +++ b/onnxruntime/core/mlas/inc/mlas.h @@ -35,7 +35,7 @@ Abstract: // Define the target architecture. // -#if defined(_M_AMD64) || defined(__x86_64__) +#if (defined(_M_AMD64) && !defined(_M_ARM64EC)) || defined(__x86_64__) #define MLAS_TARGET_AMD64 #endif #if defined(_M_IX86) || defined(__i386__) @@ -47,7 +47,7 @@ Abstract: #if defined(_M_ARM64) || defined(__aarch64__) #define MLAS_TARGET_ARM64 #endif -#if defined(_M_ARM) || defined(__arm__) +#if defined(_M_ARM) || defined(_M_ARM64EC) || defined(__arm__) #define MLAS_TARGET_ARM #endif #if defined(__VSX__) diff --git a/onnxruntime/core/providers/cpu/tensor/cast_op.cc b/onnxruntime/core/providers/cpu/tensor/cast_op.cc index 5522c29b36..2130ad43ef 100644 --- a/onnxruntime/core/providers/cpu/tensor/cast_op.cc +++ b/onnxruntime/core/providers/cpu/tensor/cast_op.cc @@ -22,7 +22,7 @@ #include "Eigen/src/Core/arch/Default/BFloat16.h" #include "Eigen/src/Core/arch/Default/Half.h" -#if defined(_M_AMD64) +#if defined(_M_AMD64) && !defined(_M_ARM64EC) #include "core/mlas/inc/mlas.h" #endif @@ -207,7 +207,7 @@ struct TensorCaster { } }; -#if defined(_M_AMD64) +#if defined(_M_AMD64) && !defined(_M_ARM64EC) // specializations to use optimized and Windows x64-specific // MlasConvertHalfToFloatBuffer() routine for MLFloat16 -> float conversion diff --git a/tools/ci_build/build.py b/tools/ci_build/build.py index f06a18fbff..997cdb6aee 100644 --- a/tools/ci_build/build.py +++ b/tools/ci_build/build.py @@ -268,6 +268,10 @@ def parse_arguments(): "--arm64", action='store_true', help="Create ARM64 makefiles. Requires --update and no existing cache " "CMake setup. Delete CMakeCache.txt if needed") + parser.add_argument( + "--arm64ec", action='store_true', + help="Create ARM64EC makefiles. Requires --update and no existing cache " + "CMake setup. Delete CMakeCache.txt if needed") parser.add_argument( "--msvc_toolset", help="MSVC toolset to use. e.g. 14.11") parser.add_argument("--android", action='store_true', help='Build for Android') @@ -679,7 +683,7 @@ def generate_build_tree(cmake_path, source_dir, build_dir, cuda_home, cudnn_home "-Donnxruntime_MIGRAPHX_HOME=" + (migraphx_home if args.use_migraphx else ""), # By default - we currently support only cross compiling for ARM/ARM64 # (no native compilation supported through this script). - "-Donnxruntime_CROSS_COMPILING=" + ("ON" if args.arm64 or args.arm else "OFF"), + "-Donnxruntime_CROSS_COMPILING=" + ("ON" if args.arm64 or args.arm64ec or args.arm else "OFF"), "-Donnxruntime_DISABLE_CONTRIB_OPS=" + ("ON" if args.disable_contrib_ops else "OFF"), "-Donnxruntime_DISABLE_ML_OPS=" + ("ON" if args.disable_ml_ops else "OFF"), "-Donnxruntime_DISABLE_RTTI=" + ("ON" if args.disable_rtti else "OFF"), @@ -1626,7 +1630,7 @@ def is_cross_compiling_on_apple(args): def build_protoc_for_host(cmake_path, source_dir, build_dir, args): - if (args.arm or args.arm64 or args.enable_windows_store) and \ + if (args.arm or args.arm64 or args.arm64ec or args.enable_windows_store) and \ not (is_windows() or is_cross_compiling_on_apple(args)): raise BuildError( 'Currently only support building protoc for Windows host while ' @@ -1738,7 +1742,7 @@ def main(): args = parse_arguments() cmake_extra_defines = (args.cmake_extra_defines if args.cmake_extra_defines else []) - cross_compiling = args.arm or args.arm64 or args.android + cross_compiling = args.arm or args.arm64 or args.arm64ec or args.android # If there was no explicit argument saying what to do, default # to update, build and test (for native builds). @@ -1837,7 +1841,7 @@ def main(): update_submodules(source_dir) if is_windows(): if args.cmake_generator == 'Ninja': - if args.x86 or args.arm or args.arm64: + if args.x86 or args.arm or args.arm64 or args.arm64ec: raise BuildError( "To cross-compile with Ninja, load the toolset " "environment for the target processor (e.g. Cross " @@ -1847,7 +1851,7 @@ def main(): cmake_extra_args = [ '-A', 'Win32', '-T', 'host=x64', '-G', args.cmake_generator ] - elif args.arm or args.arm64: + elif args.arm or args.arm64 or args.arm64ec: # Cross-compiling for ARM(64) architecture # First build protoc for host to use during cross-compilation if path_to_protoc_exe is None: @@ -1855,8 +1859,10 @@ def main(): cmake_path, source_dir, build_dir, args) if args.arm: cmake_extra_args = ['-A', 'ARM'] - else: + elif args.arm64: cmake_extra_args = ['-A', 'ARM64'] + elif args.arm64ec: + cmake_extra_args = ['-A', 'ARM64EC'] cmake_extra_args += ['-G', args.cmake_generator] # Cannot test on host build machine for cross-compiled # builds (Override any user-defined behaviour for test if any) From bbcf419ac61b1a619e636e0d88419f23c447495b Mon Sep 17 00:00:00 2001 From: Changming Sun Date: Mon, 29 Mar 2021 17:32:03 -0700 Subject: [PATCH 023/129] Move the Windows GPU machine pool of Onnxruntime packaging pipelines to a new one (#7161) --- .../c-api-noopenmp-packaging-pipelines.yml | 4 +- .../nuget/cpu-esrp-noopenmp-pipeline.yml | 239 ------------------ .../nuget/gpu-esrp-pipeline.yml | 2 +- .../azure-pipelines/nuget/gpu-pipeline.yml | 2 +- .../azure-pipelines/nuget/templates/gpu.yml | 10 +- .../nuget/templates/windowsai.yml | 10 +- .../templates/py-packaging-stage.yml | 2 +- .../win-gpu-cuda-10-2-pipeline.yml | 2 +- 8 files changed, 16 insertions(+), 255 deletions(-) delete mode 100644 tools/ci_build/github/azure-pipelines/nuget/cpu-esrp-noopenmp-pipeline.yml diff --git a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml index bb1fd0c907..8d2ccd2ee6 100644 --- a/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml +++ b/tools/ci_build/github/azure-pipelines/c-api-noopenmp-packaging-pipelines.yml @@ -85,7 +85,7 @@ jobs: - template: templates/win-ci.yml parameters: IsReleaseBuild: ${{ parameters.IsReleaseBuild }} - ort_build_pool_name: 'Win-GPU-2019' + ort_build_pool_name: 'onnxruntime-gpu-winbuild' DoCompliance: ${{ parameters.DoCompliance }} DoEsrp: ${{ parameters.DoEsrp }} OrtNugetPackageId: 'Microsoft.ML.OnnxRuntime.Gpu' @@ -160,7 +160,7 @@ jobs: - job: Final_Jar_Testing_Windows_GPU workspace: clean: all - pool: 'Win-GPU-2019' + pool: 'onnxruntime-gpu-winbuild' timeoutInMinutes: 60 variables: - name: runCodesignValidationInjection diff --git a/tools/ci_build/github/azure-pipelines/nuget/cpu-esrp-noopenmp-pipeline.yml b/tools/ci_build/github/azure-pipelines/nuget/cpu-esrp-noopenmp-pipeline.yml deleted file mode 100644 index e679b4edd0..0000000000 --- a/tools/ci_build/github/azure-pipelines/nuget/cpu-esrp-noopenmp-pipeline.yml +++ /dev/null @@ -1,239 +0,0 @@ -parameters: -- name: RunOnnxRuntimeTests - displayName: Run Tests? - type: boolean - default: true - -- name: DoCompliance - displayName: Run Compliance Tasks? - type: boolean - default: true - -- name: DoEsrp - displayName: Run code sign tasks? Must be true if you are doing an Onnx Runtime release. - type: boolean - default: false - -- name: IsReleaseBuild - displayName: Is a release build? Set it to true if you are doing an Onnx Runtime release. - type: boolean - default: false - -variables: - PackageName: 'Microsoft.ML.OnnxRuntime' - -jobs: -- template: ../templates/c-api-cpu.yml - parameters: - RunOnnxRuntimeTests: ${{ parameters.RunOnnxRuntimeTests }} - DoCompliance: ${{ parameters.DoCompliance }} - DoEsrp: ${{ parameters.DoEsrp }} - IsReleaseBuild: ${{ parameters.IsReleaseBuild }} - OrtNugetPackageId: 'Microsoft.ML.OnnxRuntime' - AdditionalBuildFlags: '' - AdditionalWinBuildFlags: '--enable_onnx_tests --enable_wcos' - BuildVariant: 'default' - -- job: Linux_C_API_Packaging_GPU_x64 - workspace: - clean: all - timeoutInMinutes: 120 - pool: 'Onnxruntime-Linux-GPU' - variables: - CUDA_VERSION: '11.0' - steps: - - template: ../templates/set-version-number-variables-step.yml - - template: ../templates/get-docker-image-steps.yml - parameters: - Dockerfile: tools/ci_build/github/linux/docker/Dockerfile.manylinux2014_cuda11 - Context: tools/ci_build/github/linux/docker - DockerBuildArgs: "--build-arg BUILD_UID=$( id -u )" - Repository: onnxruntimecuda11build - - task: CmdLine@2 - inputs: - script: | - mkdir -p $HOME/.onnx - docker run --gpus all -e CC=/opt/rh/devtoolset-8/root/usr/bin/cc -e CXX=/opt/rh/devtoolset-8/root/usr/bin/c++ -e CFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e CXXFLAGS="-Wp,-D_FORTIFY_SOURCE=2 -Wp,-D_GLIBCXX_ASSERTIONS -fstack-protector-strong -fstack-clash-protection -fcf-protection -O3 -Wl,--strip-all" -e NVIDIA_VISIBLE_DEVICES=all --rm --volume /data/onnx:/data/onnx:ro --volume $(Build.SourcesDirectory):/onnxruntime_src --volume $(Build.BinariesDirectory):/build \ - --volume /data/models:/build/models:ro --volume $HOME/.onnx:/home/onnxruntimedev/.onnx -e NIGHTLY_BUILD onnxruntimecuda11build \ - /opt/python/cp37-cp37m/bin/python3 /onnxruntime_src/tools/ci_build/build.py --build_java --build_dir /build --config Release \ - --skip_submodule_sync --parallel --build_shared_lib --use_cuda --cuda_version=$(CUDA_VERSION) --cuda_home=/usr/local/cuda-$(CUDA_VERSION) --cudnn_home=/usr/local/cuda-$(CUDA_VERSION) --cmake_extra_defines CMAKE_CUDA_HOST_COMPILER=/opt/rh/devtoolset-8/root/usr/bin/cc - workingDirectory: $(Build.SourcesDirectory) - - - template: ../templates/java-api-artifacts-package-and-publish-steps-posix.yml - parameters: - arch: 'linux-x64' - buildConfig: 'Release' - artifactName: 'onnxruntime-java-linux-gpu-x64' - version: '$(OnnxRuntimeVersion)' - libraryName: 'libonnxruntime.so' - nativeLibraryName: 'libonnxruntime4j_jni.so' - - - template: ../templates/c-api-artifacts-package-and-publish-steps-posix.yml - parameters: - buildConfig: 'Release' - artifactName: 'onnxruntime-linux-x64-gpu-$(OnnxRuntimeVersion)' - artifactNameNoVersionString: 'onnxruntime-linux-x64-gpu' - libraryName: 'libonnxruntime.so.$(OnnxRuntimeVersion)' - commitId: $(OnnxRuntimeGitCommitHash) - - - template: ../templates/component-governance-component-detection-steps.yml - parameters : - condition : 'succeeded' - - template: ../templates/clean-agent-build-directory-step.yml - - -- template: ../templates/win-ci.yml - parameters: - IsReleaseBuild: ${{ parameters.IsReleaseBuild }} - ort_build_pool_name: 'Win-GPU-2019' - DoCompliance: ${{ parameters.DoCompliance }} - DoEsrp: ${{ parameters.DoEsrp }} - OrtNugetPackageId: 'Microsoft.ML.OnnxRuntime.Gpu' - job_name_suffix: gpu - EnvSetupScript: setup_env_cuda_11.bat - buildArch: x64 - msbuildPlatform: x64 - packageName: gpu-x64 - buildparameter: --use_cuda --cuda_version=11.0 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.0" --cudnn_home="C:\local\cudnn-11.0-windows-x64-v8.0.2.39\cuda" --enable_onnx_tests --enable_wcos --build_java - runTests: ${{ parameters.RunOnnxRuntimeTests }} - buildJava: true - java_artifact_id: onnxruntime_gpu - -- job: Jar_Packaging_GPU - workspace: - clean: all - pool: 'Win-CPU-2021' - dependsOn: - - Linux_C_API_Packaging_GPU_x64 - - Windows_Packaging_CPU_gpu - condition: succeeded() - steps: - - checkout: self - submodules: false - - template: ../templates/set-version-number-variables-step.yml - - - task: DownloadPipelineArtifact@2 - displayName: 'Download Pipeline Artifact - Win x64' - inputs: - buildType: 'current' - artifactName: 'drop-onnxruntime-java-win-gpu-x64' - targetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64' - - - task: DownloadPipelineArtifact@2 - displayName: 'Download Pipeline Artifact - Linux x64' - inputs: - buildType: 'current' - artifactName: 'drop-onnxruntime-java-linux-gpu-x64' - targetPath: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-linux-x64' - - - task: PowerShell@2 - displayName: 'PowerShell Script' - inputs: - targetType: filePath - filePath: $(Build.SourcesDirectory)\tools\ci_build\github\windows\jar_gpu_packaging.ps1 - failOnStderr: true - showWarnings: true - workingDirectory: '$(Build.BinariesDirectory)\java-artifact' - - - - - task: CopyFiles@2 - displayName: 'Copy Java Files to Artifact Staging Directory' - inputs: - SourceFolder: '$(Build.BinariesDirectory)\java-artifact\onnxruntime-java-win-x64' - TargetFolder: '$(Build.ArtifactStagingDirectory)' - - - task: PublishPipelineArtifact@1 - displayName: 'Publish Pipeline Artifact' - inputs: - targetPath: '$(Build.ArtifactStagingDirectory)' - artifact: 'onnxruntime-java-gpu' - - - template: ../templates/component-governance-component-detection-steps.yml - parameters : - condition : 'succeeded' - - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 - displayName: 'Clean Agent Directories' - condition: always() - - -- job: Final_Jar_Testing_Windows_GPU - workspace: - clean: all - pool: 'Win-GPU-2019' - timeoutInMinutes: 60 - variables: - - name: runCodesignValidationInjection - value: false - dependsOn: - Jar_Packaging_GPU - steps: - - template: ../templates/set-version-number-variables-step.yml - - - task: BatchScript@1 - displayName: 'setup env' - inputs: - filename: '$(Build.SourcesDirectory)\tools\ci_build\github\windows\setup_env_cuda_11.bat' - modifyEnvironment: true - workingFolder: '$(Build.BinariesDirectory)' - - - task: DownloadPipelineArtifact@2 - displayName: 'Download Final Jar' - inputs: - buildType: 'current' - artifactName: 'onnxruntime-java-gpu' - targetPath: '$(Build.BinariesDirectory)\final-jar' - - - task: CmdLine@2 - inputs: - script: | - mkdir test - pushd test - jar xf $(Build.BinariesDirectory)\final-jar\testing.jar - popd - powershell -Command "Invoke-WebRequest https://oss.sonatype.org/service/local/repositories/releases/content/org/junit/platform/junit-platform-console-standalone/1.6.2/junit-platform-console-standalone-1.6.2.jar -OutFile junit-platform-console-standalone-1.6.2.jar" - powershell -Command "Invoke-WebRequest https://oss.sonatype.org/service/local/repositories/google-releases/content/com/google/protobuf/protobuf-java/3.9.2/protobuf-java-3.9.2.jar -OutFile protobuf-java-3.9.2.jar" - java -DUSE_CUDA=1 -jar junit-platform-console-standalone-1.6.2.jar -cp .;.\test;protobuf-java-3.9.2.jar;onnxruntime_gpu-$(OnnxRuntimeVersion).jar --scan-class-path --fail-if-no-tests --disable-banner - workingDirectory: '$(Build.BinariesDirectory)\final-jar' - - - template: ../templates/component-governance-component-detection-steps.yml - parameters : - condition : 'succeeded' - - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 - displayName: 'Clean Agent Directories' - condition: always() - -- job: Final_Jar_Testing_Linux_GPU - workspace: - clean: all - pool: 'Onnxruntime-Linux-GPU' - variables: - - name: runCodesignValidationInjection - value: false - timeoutInMinutes: 60 - dependsOn: - Jar_Packaging_GPU - steps: - - checkout: self - submodules: false - - template: ../templates/set-version-number-variables-step.yml - - task: DownloadPipelineArtifact@2 - displayName: 'Download Final Jar' - inputs: - buildType: 'current' - artifactName: 'onnxruntime-java-gpu' - targetPath: '$(Build.BinariesDirectory)/final-jar' - - - task: Bash@3 - displayName: 'Test' - inputs: - targetType: filePath - filePath: 'tools/ci_build/github/linux/java_linux_final_test.sh' - arguments: '-r $(Build.BinariesDirectory) -v $(OnnxRuntimeVersion)' - - - template: ../templates/component-governance-component-detection-steps.yml - parameters : - condition : 'succeeded' - - task: mspremier.PostBuildCleanup.PostBuildCleanup-task.PostBuildCleanup@3 - displayName: 'Clean Agent Directories' - condition: always() diff --git a/tools/ci_build/github/azure-pipelines/nuget/gpu-esrp-pipeline.yml b/tools/ci_build/github/azure-pipelines/nuget/gpu-esrp-pipeline.yml index 2adb919359..2b8dbf34fe 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/gpu-esrp-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/gpu-esrp-pipeline.yml @@ -13,5 +13,5 @@ variables: jobs: - template: templates/gpu.yml parameters: - AgentPool : 'Win-GPU-2019' + AgentPool : 'onnxruntime-gpu-winbuild' DoEsrp: 'true' diff --git a/tools/ci_build/github/azure-pipelines/nuget/gpu-pipeline.yml b/tools/ci_build/github/azure-pipelines/nuget/gpu-pipeline.yml index eb8a2ae4ae..9572abb994 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/gpu-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/gpu-pipeline.yml @@ -5,5 +5,5 @@ variables: jobs: - template: templates/gpu.yml parameters: - AgentPool : 'Win-GPU-2019' + AgentPool : 'onnxruntime-gpu-winbuild' DoEsrp: 'false' diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml index 406cc8487f..f89b07018a 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/gpu.yml @@ -5,7 +5,7 @@ parameters: jobs: - template: ../../templates/win-ci-2019.yml parameters: - AgentPool : 'Win-GPU-2019' + AgentPool : 'onnxruntime-gpu-winbuild' ArtifactName: 'drop-nuget' JobName: 'Windows_CI_GPU_CUDA_Dev' BuildCommand: --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --use_telemetry --cmake_generator "Visual Studio 16 2019" --use_cuda --cuda_version=11.0 --cuda_home="C:\Program Files\NVIDIA GPU Computing Toolkit\CUDA\v11.0" --cudnn_home="C:\local\cudnn-11.0-windows-x64-v8.0.2.39\cuda" @@ -28,7 +28,7 @@ jobs: - template: ../../templates/win-ci-2019.yml parameters: - AgentPool : 'Win-GPU-2019' + AgentPool : 'onnxruntime-gpu-winbuild' ArtifactName: 'drop-nuget-dml' JobName: 'Windows_CI_GPU_DML_Dev' BuildCommand: --build_dir $(Build.BinariesDirectory) --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos --use_telemetry --use_dml --use_winml --cmake_generator "Visual Studio 16 2019" @@ -51,7 +51,7 @@ jobs: - template: ../../templates/win-ci-2019.yml parameters: - AgentPool : 'Win-GPU-2019' + AgentPool : 'onnxruntime-gpu-winbuild' ArtifactName: 'drop-win-dml-x86-zip' JobName: 'Windows_CI_GPU_DML_Dev_x86' BuildCommand: --build_dir $(Build.BinariesDirectory) --x86 --skip_submodule_sync --build_shared_lib --enable_onnx_tests --enable_wcos --use_telemetry --use_dml --use_winml --cmake_generator "Visual Studio 16 2019" @@ -113,7 +113,7 @@ jobs: - job: NuGet_Packaging workspace: clean: all - pool: 'Win-GPU-2019' + pool: 'onnxruntime-gpu-winbuild' dependsOn: - Windows_CI_GPU_CUDA_Dev - Windows_CI_GPU_DML_Dev @@ -232,7 +232,7 @@ jobs: - template: test_win.yml parameters: - AgentPool : 'Win-GPU-2019' + AgentPool : 'onnxruntime-gpu-winbuild' Skipx86Tests: 'true' - template: test_linux.yml diff --git a/tools/ci_build/github/azure-pipelines/nuget/templates/windowsai.yml b/tools/ci_build/github/azure-pipelines/nuget/templates/windowsai.yml index c04f97358c..edefa7e34c 100644 --- a/tools/ci_build/github/azure-pipelines/nuget/templates/windowsai.yml +++ b/tools/ci_build/github/azure-pipelines/nuget/templates/windowsai.yml @@ -4,7 +4,7 @@ jobs: workspace: clean: all pool: - name: 'Win-GPU-2019' + name: 'onnxruntime-gpu-winbuild' demands: [] steps: - template: ../../templates/windowsai-nuget-build.yml @@ -16,7 +16,7 @@ jobs: workspace: clean: all pool: - name: 'Win-GPU-2019' + name: 'onnxruntime-gpu-winbuild' demands: [] steps: - template: ../../templates/windowsai-nuget-build.yml @@ -52,7 +52,7 @@ jobs: workspace: clean: all pool: - name: 'Win-GPU-2019' + name: 'onnxruntime-gpu-winbuild' demands: [ ] steps: - template: ../../templates/windowsai-nuget-build.yml @@ -65,7 +65,7 @@ jobs: workspace: clean: all pool: - name: 'Win-GPU-2019' + name: 'onnxruntime-gpu-winbuild' demands: [ ] steps: - template: ../../templates/windowsai-nuget-build.yml @@ -154,7 +154,7 @@ jobs: - job: NuGet_Packaging workspace: clean: all - pool: 'Win-GPU-2019' + pool: 'onnxruntime-gpu-winbuild' dependsOn: - WindowsAI_DirectML_X64 - WindowsAI_DirectML_X86 diff --git a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml index 3404ac62c1..df13cd3c22 100644 --- a/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml +++ b/tools/ci_build/github/azure-pipelines/templates/py-packaging-stage.yml @@ -481,7 +481,7 @@ stages: - job: Windows_py_GPU_Wheels workspace: clean: all - pool: 'Win-GPU-2019' + pool: 'onnxruntime-gpu-winbuild' timeoutInMinutes: 240 variables: CUDA_VERSION: '11.0' diff --git a/tools/ci_build/github/azure-pipelines/win-gpu-cuda-10-2-pipeline.yml b/tools/ci_build/github/azure-pipelines/win-gpu-cuda-10-2-pipeline.yml index cd47b37d73..3ae61a208d 100644 --- a/tools/ci_build/github/azure-pipelines/win-gpu-cuda-10-2-pipeline.yml +++ b/tools/ci_build/github/azure-pipelines/win-gpu-cuda-10-2-pipeline.yml @@ -1,6 +1,6 @@ jobs: - job: 'build' - pool: 'Win-GPU-2019' + pool: 'onnxruntime-gpu-winbuild' strategy: maxParallel: 2 matrix: From a01334ba5657bce9ed51cebd76272ad223574507 Mon Sep 17 00:00:00 2001 From: Tracy Sharpe <42477615+tracysh@users.noreply.github.com> Date: Mon, 29 Mar 2021 17:56:48 -0700 Subject: [PATCH 024/129] MLAS: activate udot kernel on Windows ARM64 (#7169) --- onnxruntime/core/mlas/lib/platform.cpp | 27 +++++++++++++++++++------- 1 file changed, 20 insertions(+), 7 deletions(-) diff --git a/onnxruntime/core/mlas/lib/platform.cpp b/onnxruntime/core/mlas/lib/platform.cpp index 7e66e0b2f7..e3bb159dd8 100644 --- a/onnxruntime/core/mlas/lib/platform.cpp +++ b/onnxruntime/core/mlas/lib/platform.cpp @@ -17,7 +17,13 @@ Abstract: #include "mlasi.h" -#if defined(MLAS_TARGET_ARM64) && defined(__linux__) +#if defined(MLAS_TARGET_ARM64) +#if defined(_WIN32) +// N.B. Support building with downlevel versions of the Windows SDK. +#ifndef PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE +#define PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE 43 +#endif +#elif defined(__linux__) #include #include // N.B. Support building with older versions of asm/hwcap.h that do not define @@ -26,6 +32,7 @@ Abstract: #define HWCAP_ASIMDDP (1 << 20) #endif #endif +#endif // MLAS_TARGET_ARM64 // // Stores the platform information. @@ -330,19 +337,25 @@ Return Value: this->GemmU8X8Dispatch = &MlasGemmU8X8DispatchNeon; -#if defined(__linux__) - // // Check if the processor supports ASIMD dot product instructions. // - if ((getauxval(AT_HWCAP) & HWCAP_ASIMDDP) != 0) { + bool HasDotProductInstructions; + +#if defined(_WIN32) + HasDotProductInstructions = (IsProcessorFeaturePresent(PF_ARM_V82_DP_INSTRUCTIONS_AVAILABLE) != 0); +#elif defined(__linux__) + HasDotProductInstructions = ((getauxval(AT_HWCAP) & HWCAP_ASIMDDP) != 0); +#else + HasDotProductInstructions = false; +#endif + + if (HasDotProductInstructions) { this->GemmU8X8Dispatch = &MlasGemmU8X8DispatchUdot; } -#endif - -#endif +#endif // MLAS_TARGET_ARM64 } From 4f303412539bcc51f0d8a09e2a510b799afe558d Mon Sep 17 00:00:00 2001 From: Yufeng Li Date: Tue, 30 Mar 2021 09:00:48 -0700 Subject: [PATCH 025/129] Check the count of DequantizeLinear for matmul (#7174) --- .../optimizer/qdq_transformer/qdq_matmul.cc | 2 +- .../test/optimizer/qdq_transformer_test.cc | 33 +++++++++++++++++++ 2 files changed, 34 insertions(+), 1 deletion(-) diff --git a/onnxruntime/core/optimizer/qdq_transformer/qdq_matmul.cc b/onnxruntime/core/optimizer/qdq_transformer/qdq_matmul.cc index 6b7a578556..770b435151 100644 --- a/onnxruntime/core/optimizer/qdq_transformer/qdq_matmul.cc +++ b/onnxruntime/core/optimizer/qdq_transformer/qdq_matmul.cc @@ -13,7 +13,7 @@ class QDQMatMulTransformer : public QDQOperatorTransformer { QDQMatMulTransformer(Node& node, Graph& graph) : QDQOperatorTransformer(node, graph) {} bool Transform(const std::vector& parents, const std::vector& children) override { - if (children.size() != 1) { + if (parents.size() != 2 || children.size() != 1) { return false; } diff --git a/onnxruntime/test/optimizer/qdq_transformer_test.cc b/onnxruntime/test/optimizer/qdq_transformer_test.cc index d056a4b7b8..0cbbeacd24 100644 --- a/onnxruntime/test/optimizer/qdq_transformer_test.cc +++ b/onnxruntime/test/optimizer/qdq_transformer_test.cc @@ -231,6 +231,39 @@ TEST(QDQTransformerTests, MatMul) { test_case({22, 11, 13, 15}, {15, 13}); } +TEST(QDQTransformerTests, MatMul_No_Fusion) { + auto test_case = [&](const std::vector& input1_shape, const std::vector& input2_shape) { + auto build_test_case = [&](ModelTestBuilder& builder) { + auto* input1_arg = builder.MakeInput(input1_shape, -1.f, 1.f); + auto* input2_arg = builder.MakeInput(input2_shape, -1.f, 1.f); + auto* output_arg = builder.MakeOutput(); + + // add QDQ + MatMul + auto* matmul_output = builder.MakeIntermediate(); + auto* dq_matmul_output1 = AddQDQNodePair(builder, input1_arg, .004f, 129); + builder.AddNode("MatMul", {dq_matmul_output1, input2_arg}, {matmul_output}); + + // add Q + builder.AddQuantizeLinearNode(matmul_output, .0039f, 135, output_arg); + }; + + auto check_matmul_graph = [&](InferenceSessionWrapper& session) { + auto op_to_count = CountOpsInGraph(session.GetGraph()); + EXPECT_EQ(op_to_count["MatMul"], 1); + EXPECT_EQ(op_to_count["QLinearMatMul"], 0); + EXPECT_EQ(op_to_count["QuantizeLinear"], 2); + EXPECT_EQ(op_to_count["DequantizeLinear"], 1); + }; + + TransformerTester(build_test_case, check_matmul_graph, TransformerLevel::Level1, TransformerLevel::Level2); + }; + + // Test the basic case of a single 1D/2D/3D convolution. + test_case({12, 37}, {37, 12}); + test_case({23, 13, 13}, {13, 13}); + test_case({22, 11, 13, 15}, {15, 13}); +} + #endif // DISABLE_CONTRIB_OPS } // namespace test From c4ebc608702c001b53d54ae32d05d3deb194932d Mon Sep 17 00:00:00 2001 From: Yufeng Li Date: Tue, 30 Mar 2021 09:01:15 -0700 Subject: [PATCH 026/129] sort quantized nodes in topo logical order (#7172) --- .../python/tools/quantization/onnx_model.py | 39 +++++++++++++++++++ .../test/python/quantization/op_test_utils.py | 8 ++++ .../test/python/quantization/test_qdq.py | 6 +-- 3 files changed, 50 insertions(+), 3 deletions(-) diff --git a/onnxruntime/python/tools/quantization/onnx_model.py b/onnxruntime/python/tools/quantization/onnx_model.py index a38d7e69d7..325a393995 100644 --- a/onnxruntime/python/tools/quantization/onnx_model.py +++ b/onnxruntime/python/tools/quantization/onnx_model.py @@ -1,4 +1,5 @@ import onnx +import itertools from .quant_utils import find_by_name from pathlib import Path @@ -197,6 +198,7 @@ class ONNXModel: ''' Save model to external data, which is needed for model size > 2GB ''' + self.topological_sort() if use_external_data_format: onnx.external_data_helper.convert_model_to_external_data(self.model, all_tensors_to_one_file=True, @@ -254,3 +256,40 @@ class ONNXModel: if output.name == output_name: return True return False + + def topological_sort(self): + deps_count = [0]*len(self.nodes()) # dependency count of each node + deps_to_nodes = {} # input to node indice + for node_idx, node in enumerate(self.nodes()): + deps_count[node_idx] = len(node.input) + for input_name in node.input: + if input_name not in deps_to_nodes: + deps_to_nodes[input_name] = [node_idx] + else: + deps_to_nodes[input_name].append(node_idx) + + + # initialize sorted_nodes + sorted_nodes = [] + for input in itertools.chain(self.initializer(), self.model.graph.input): + for node_idx in deps_to_nodes[input.name]: + deps_count[node_idx] = deps_count[node_idx] - 1 + if deps_count[node_idx] == 0: + sorted_nodes.append(self.nodes()[node_idx]) + + s = 0 + e = len(sorted_nodes) + + while s < e: + for output in sorted_nodes[s].output: + if output in deps_to_nodes: + for node_idx in deps_to_nodes[output]: + deps_count[node_idx] = deps_count[node_idx] - 1 + if deps_count[node_idx] == 0: + sorted_nodes.append(self.nodes()[node_idx]) + e = e + 1 + s = s + 1 + + assert(e == len(self.graph().node)), "Graph is not a DAG" + self.graph().ClearField('node') + self.graph().node.extend(sorted_nodes) \ No newline at end of file diff --git a/onnxruntime/test/python/quantization/op_test_utils.py b/onnxruntime/test/python/quantization/op_test_utils.py index 31c2a32718..27d5e4364d 100644 --- a/onnxruntime/test/python/quantization/op_test_utils.py +++ b/onnxruntime/test/python/quantization/op_test_utils.py @@ -20,6 +20,14 @@ class TestDataFeeds(CalibrationDataReader): def rewind(self): self.iter_next = iter(self.data_feeds) +def check_op_type_order(testcase, model_path, ops): + model = onnx.load(Path(model_path)) + testcase.assertEqual(len(ops), len(model.graph.node), 'op count is not same') + for node_idx, node in enumerate(model.graph.node): + testcase.assertEqual( + ops[node_idx], + node.op_type, + 'op {} is not in order. Expected: {}, Actual: {}'.format(node_idx, ops[node_idx], node.op_type)) def check_op_type_count(testcase, model_path, **kwargs): model = onnx.load(Path(model_path)) diff --git a/onnxruntime/test/python/quantization/test_qdq.py b/onnxruntime/test/python/quantization/test_qdq.py index 895aed8ece..c7f682ee81 100644 --- a/onnxruntime/test/python/quantization/test_qdq.py +++ b/onnxruntime/test/python/quantization/test_qdq.py @@ -11,7 +11,7 @@ import onnx import numpy as np from onnx import helper, TensorProto from onnxruntime.quantization import quantize_static, QuantType, QuantFormat -from op_test_utils import TestDataFeeds, check_model_correctness, check_op_type_count +from op_test_utils import TestDataFeeds, check_model_correctness, check_op_type_count, check_op_type_order class TestQDQFormat(unittest.TestCase): def input_feeds(self, n, name2shape): @@ -163,8 +163,8 @@ class TestQDQFormatConvClip(TestQDQFormat): reduce_range = per_channel ) data_reader.rewind() - qdq_nodes = {'Conv': 1, 'QuantizeLinear': 1, 'DequantizeLinear': 2} - check_op_type_count(self, model_int8_qdq_path, **qdq_nodes) + #topo sort check + check_op_type_order(self, model_int8_qdq_path, ['DequantizeLinear', 'QuantizeLinear', 'DequantizeLinear', 'Conv', 'Clip']) check_model_correctness(self, model_fp32_path, model_int8_qdq_path, data_reader.get_next()) data_reader.rewind() From 07201bac7a68d385b4817c9804398a7bc597f207 Mon Sep 17 00:00:00 2001 From: "Tang, Cheng" Date: Tue, 30 Mar 2021 09:49:45 -0700 Subject: [PATCH 027/129] expose session option and provider options (#7112) * expose session option and provider options * merge provider_names and provider_options * integrate into orttrainer options * fix doc string * fix a typo * Update orttraining/orttraining/python/training/orttrainer.py Co-authored-by: Thiago Crepaldi * Update orttraining/orttraining/python/training/orttrainer.py Co-authored-by: Thiago Crepaldi * Update orttraining/orttraining/python/training/orttrainer_options.py Co-authored-by: Thiago Crepaldi * fix the usage of provider_options * Update orttraining/orttraining/python/training/orttrainer.py Co-authored-by: Thiago Crepaldi * Update orttraining/orttraining/python/training/orttrainer.py Co-authored-by: Thiago Crepaldi * update expected result in tests * fix default provider options * minor update to trigger rebuild * minor update to trigger rebuild Co-authored-by: Thiago Crepaldi --- .../orttraining/python/training/orttrainer.py | 56 ++++++++++++++----- .../python/training/orttrainer_options.py | 38 ++++++++++++- .../orttraining_test_orttrainer_frontend.py | 14 ++++- 3 files changed, 90 insertions(+), 18 deletions(-) diff --git a/orttraining/orttraining/python/training/orttrainer.py b/orttraining/orttraining/python/training/orttrainer.py index f7b3e5e68b..3965c32523 100644 --- a/orttraining/orttraining/python/training/orttrainer.py +++ b/orttraining/orttraining/python/training/orttrainer.py @@ -95,7 +95,6 @@ class ORTTrainer(object): Inputs to the combined PyTorch model are concatenation of the :py:attr:`model`'s input and :py:attr:`loss_fn`'s label input. Outputs of the combined PyTorch model are concatenation of :py:attr:`loss_fn`'s loss output and :py:attr:`model`'s outputs. options (ORTTrainerOptions, default is None): options for additional features. - Example: .. code-block:: python @@ -121,7 +120,9 @@ class ORTTrainer(object): ort_trainer = ORTTrainer(model, model_desc, optim_config, loss_fn) """ - def __init__(self, model, model_desc, optim_config, loss_fn=None, options=None): + def __init__(self, model, model_desc, optim_config, + loss_fn=None, + options=None): assert model is not None, "'model' is required and must be either a 'torch.nn.Module' or ONNX model" assert isinstance(model_desc, dict), "'model_desc' must be a 'dict'" assert isinstance(optim_config, optim._OptimizerConfig),\ @@ -205,7 +206,8 @@ class ORTTrainer(object): self._train_step_info = TrainStepInfo(self.optim_config) self._training_session = None self._load_state_dict = None - self._init_session() + self._init_session(provider_options=self.options._validated_opts['provider_options'], + session_options=self.options.session_options) def eval_step(self, *args, **kwargs): r"""Evaluation step method @@ -564,7 +566,10 @@ class ORTTrainer(object): return onnx_model - def _create_ort_training_session(self, optimizer_state_dict={}): + def _create_ort_training_session(self, + optimizer_state_dict={}, + session_options=None, + provider_options=None): # Validating frozen_weights names unused_frozen_weights = [n for n in self.options.utils.frozen_weights\ if n not in [i.name for i in self._onnx_model.graph.initializer]] @@ -662,7 +667,7 @@ class ORTTrainer(object): ort_parameters.model_with_training_graph_path = self.options.debug.graph_save_paths.model_with_training_graph_path # SessionOptions - session_options = ort.SessionOptions() + session_options = ort.SessionOptions() if session_options is None else session_options session_options.use_deterministic_compute = self.options.debug.deterministic_compute if (self.options.graph_transformer.attn_dropout_recompute or self.options.graph_transformer.gelu_recompute or @@ -676,10 +681,14 @@ class ORTTrainer(object): del self._training_session # Set provider-specific options if needed - def get_providers(): + def get_providers(provider_options): providers = ort.get_available_providers() - - if 'cuda' in self.options.device.id.lower(): + + if provider_options: + for provider_name in provider_options: + providers[providers.index(provider_name)] = (provider_name, provider_options[provider_name]) + #default: using cuda + elif 'cuda' in self.options.device.id.lower(): cuda_ep_options = {"device_id": _utils.get_device_index(self.options.device.id)} cuda_ep_name = ("ROCMExecutionProvider" if self.is_rocm_pytorch else "CUDAExecutionProvider") @@ -700,7 +709,7 @@ class ORTTrainer(object): # TrainingSession self._training_session = ort.TrainingSession(self._onnx_model.SerializeToString(), ort_parameters, - session_options, get_providers()) + session_options, get_providers(provider_options)) # I/O bindings self._train_io_binding = self._training_session.io_binding() @@ -731,9 +740,13 @@ class ORTTrainer(object): if self._load_state_dict: optimizer_state_dict = self._load_state_dict() - self._init_session(optimizer_state_dict) + self._init_session(optimizer_state_dict, + session_options=self.options.session_options, + provider_options=self.options._validated_opts['provider_options']) - def _init_session(self, optimizer_state_dict={}): + def _init_session(self, optimizer_state_dict={}, + session_options=None, + provider_options=None): if self._onnx_model is None: return @@ -742,7 +755,9 @@ class ORTTrainer(object): # Create training session used by train_step # pass all optimizer states to the backend - self._create_ort_training_session(optimizer_state_dict) + self._create_ort_training_session(optimizer_state_dict, + session_options=session_options, + provider_options=provider_options) # Update model description to update dtype when mixed precision is enabled # C++ backend modifies model's output dtype from float32 to float16 for mixed precision @@ -867,9 +882,20 @@ class ORTTrainer(object): # Keep all finite flag on CPU to match backend implementation # This prevents CPU -> GPU -> CPU copies between frontend and backend target_device = 'cpu' + # the self.options.device may be a device that pytorch does not recognize. + # in that case, we temporary prefer to leave the input/output on CPU and let ORT session + # to move the data between device and host. + # so output will be on the same device as input. + try: + test_pt_device = torch.device(target_device) + except: + #in this case, input/output must on CPU + assert(input.device == 'cpu') + target_device = 'cpu' + torch_tensor = torch.zeros(output_desc.shape, device=target_device, dtype=output_desc.dtype_amp if output_desc.dtype_amp else output_desc.dtype) - iobinding.bind_output(output_desc.name, torch_tensor.device.type, _utils.get_device_index(self.options.device.id), + iobinding.bind_output(output_desc.name, torch_tensor.device.type, _utils.get_device_index(target_device), _utils.dtype_torch_to_numpy(torch_tensor.dtype), list(torch_tensor.size()), torch_tensor.data_ptr()) result[output_desc.name] = torch_tensor @@ -1300,7 +1326,9 @@ class ORTTrainer(object): # create a new training session after loading initializer states onto the onnx graph # pass the populated states to the training session to populate the backend graph - self._init_session(optimizer_state_dict) + self._init_session(optimizer_state_dict, + session_options=self.options.session_options, + provider_options=self.options._validated_opts['provider_options']) def save_checkpoint(self, path, user_dict={}, include_optimizer_states=True): """Persists ORTTrainer state dictionary on disk along with user_dict. diff --git a/orttraining/orttraining/python/training/orttrainer_options.py b/orttraining/orttraining/python/training/orttrainer_options.py index 8208ea22be..fab4cba259 100644 --- a/orttraining/orttraining/python/training/orttrainer_options.py +++ b/orttraining/orttraining/python/training/orttrainer_options.py @@ -3,7 +3,7 @@ import torch from .optim import lr_scheduler from .amp import loss_scaler - +import onnxruntime as ort class ORTTrainerOptions(object): r"""Settings used by ONNX Runtime training backend @@ -277,7 +277,18 @@ class ORTTrainerOptions(object): 'default' : True } } - } + }, + 'provider_options':{ + 'type': 'dict', + 'default': {}, + 'required': False, + 'schema': {} + }, + 'session_options': { + 'type': 'SessionOptions', + 'nullable': True, + 'default': None + }, } Keyword arguments: @@ -383,6 +394,11 @@ class ORTTrainerOptions(object): _internal_use.enable_onnx_contrib_ops (bool, default is True) enable PyTorch to export nodes as contrib ops in ONNX. This flag may be removed anytime in the future. + session_options (onnxruntime.SessionOptions): + The SessionOptions instance that TrainingSession will use. + provider_options (dict): + The provider_options for customized execution providers. it is dict map from EP name to + a key-value pairs, like {'EP1' : {'key1' : 'val1'}, ....} Example: .. code-block:: python @@ -459,9 +475,13 @@ class ORTTrainerOptionsValidator(cerberus.Validator): _LOSS_SCALER = cerberus.TypeDefinition( 'loss_scaler', (loss_scaler.LossScaler,), ()) + _SESSION_OPTIONS = cerberus.TypeDefinition( + 'session_options', (ort.SessionOptions,),()) + types_mapping = cerberus.Validator.types_mapping.copy() types_mapping['lr_scheduler'] = _LR_SCHEDULER types_mapping['loss_scaler'] = _LOSS_SCALER + types_mapping['session_options'] = _SESSION_OPTIONS def _check_is_callable(field, value, error): @@ -731,5 +751,17 @@ _ORTTRAINER_OPTIONS_SCHEMA = { 'default': True } } - } + }, + 'provider_options':{ + 'type': 'dict', + 'default_setter': lambda _: {}, + 'required': False, + 'allow_unknown': True, + 'schema': {} + }, + 'session_options': { + 'type': 'session_options', + 'nullable': True, + 'default': None + }, } diff --git a/orttraining/orttraining/test/python/orttraining_test_orttrainer_frontend.py b/orttraining/orttraining/test/python/orttraining_test_orttrainer_frontend.py index f6d347f860..dcd16f1a4a 100644 --- a/orttraining/orttraining/test/python/orttraining_test_orttrainer_frontend.py +++ b/orttraining/orttraining/test/python/orttraining_test_orttrainer_frontend.py @@ -18,6 +18,7 @@ from onnxruntime.training import _utils, amp, checkpoint, optim, orttrainer, Tra model_desc_validation as md_val,\ orttrainer_options as orttrainer_options import _test_commons,_test_helpers +from onnxruntime import SessionOptions ############################################################################### @@ -99,7 +100,9 @@ def testORTTrainerOptionsDefaultValues(test_input): 'extra_postprocess': None, 'onnx_opset_version' : 12, 'enable_onnx_contrib_ops': True, - } + }, + 'provider_options':{}, + 'session_options': None, } actual_values = orttrainer_options.ORTTrainerOptions(test_input) @@ -532,6 +535,15 @@ def testLRSchedulerUpdateImpl(lr_scheduler, expected_values): assert_allclose(lr_list[0], expected_values[optimization_step], rtol=rtol, err_msg="lr mismatch") +def testInstantiateORTTrainerOptions(): + session_options = SessionOptions() + session_options.enable_mem_pattern = False + provider_options = {'EP1': {'key':'val'}} + opts = {'session_options' : session_options, + 'provider_options' : provider_options} + opts = orttrainer.ORTTrainerOptions(opts) + assert(opts.session_options.enable_mem_pattern is False) + assert(opts._validated_opts['provider_options']['EP1']['key'] == 'val') @pytest.mark.parametrize("step_fn, lr_scheduler, expected_lr_values, device", [ ('train_step', None, None, 'cuda'), From 0ccfe6c86a144248b9ec0d6a78efa42bfe37cc16 Mon Sep 17 00:00:00 2001 From: Edward Chen <18449977+edgchen1@users.noreply.github.com> Date: Tue, 30 Mar 2021 11:02:24 -0700 Subject: [PATCH 028/129] Enable type reduction for Scatter/ScatterElements CPU kernels (#7171) Enable type reduction for Scatter/ScatterElements CPU kernels. Some refactoring to reduce binary size. Add MLTypeCallDispatcher methods. Minor cleanup for Pad CPU kernel. --- .../core/framework/data_types_internal.h | 54 ++++++- onnxruntime/core/providers/cpu/tensor/pad.cc | 14 +- .../core/providers/cpu/tensor/scatter.cc | 144 ++++++++++++------ .../operator_type_usage_processors.py | 4 +- 4 files changed, 163 insertions(+), 53 deletions(-) diff --git a/include/onnxruntime/core/framework/data_types_internal.h b/include/onnxruntime/core/framework/data_types_internal.h index b81ed85bf7..7ad932b2b8 100644 --- a/include/onnxruntime/core/framework/data_types_internal.h +++ b/include/onnxruntime/core/framework/data_types_internal.h @@ -350,10 +350,12 @@ class MLTypeCallDispatcher { } /** - * Invokes Fn<..., T> with leading template arguments and the specified arguments. + * Invokes Fn<..., T> with leading template arguments and the specified + * arguments. * * @tparam Fn The function object template. - * @tparam LeadingTemplateArgTypeList A type list of the leading template arguments. + * @tparam LeadingTemplateArgTypeList A type list of the leading template + * arguments. * @tparam Args The argument types. */ template

%!=&1`wL%%5WutE0)QP3ScOVseR8v@JcC8(T6k)ju_;eZ9ga!?XX&`R#H_ubL?4Y z<3}I=!%JuC*`#dOX&Az~-=qxIxlOmu0V9b`$dq~C#mG%h{A9g6IT~th1ucBvaq@3_ z+fN1Ob4+=N`xTBfM#{QrtamxOQiI`;5)DIZZJ!b&B7(439VGH~Q*vS{y6M*Bc(W#= z(P#t!6lJ@#zz{*Ut2QPO#;m*6Ij2e)5(-hT+C^CxRaGU>E{jDMuHU?~J)0H4_a>wJ zE??ekj+dL$t-b9_=Pz?v5sGm$0jB#dFs5CUArNC$6?w8%j%MYkGTr7V`CiKjqI~+9 zCl>87VB$)=<9FZj%(Kt``Fs9kF+V^a?;Op2yI60Qo53dUauPAH?2E0a>nF-#|;VVDc<7%CS?vgXOBpMKAK{^aQ=o=5~& zufCA->f(L(f6e#(VA5MvJ=&aHd*O-SeD32PfANLqCOf+?Z&H^oo*%5|LJih??g)d7 z>CCyUy~`I&Ikwgk5$7B^cQJU#;@5xuSB)`k+kV5>e#0OB$sdOhZrr-wRQ1?fTWfdg4PdhS6wRD%rbq-~AVM_hx_kr+<9s_TlMr@vYzb zt?zlyA3ghtXSCsadwUVqD2h^v5II4P+4^A5o!z^6ye^8udmnuai3(!?(B&+IKn$(* zgbX3GOiD^1h|#A^To}$dM@bZNhCn%6>SH&qwO|1Xg@~M|oXJQQGZR9moV|}ym5c%8 z%(+5FFW)HH97Sd^MlN+#6fwKtNeJK^kV!(8LW(gf$$2JikTUlDS_qL7bvDGDl#&`L zakd@;K?;fXw$E<*MMys7l!+taxP(oRZP$xrd`xY0MldEsCe~Z8C7X@Lm}%2_;EXU8 zx;i>uqp6~|^TX3M#nsAlBWwwnl2k0E(3mO&uVjt5A?Pct>_8k%;eAz`==|k#=jZd) z+5KIB82X{qCdU|K43N#Jtgtky=r*gmD6~+Ec^^WeOx}?|iZMdWpIPG)YRTvm9rEk5(q@bu`xvg?E(?TveH6}TQ_f;vdW2@s&;lslrCJn{QQeg zUc7ME3op&z@EM=Mw0`vAhh2Akr}wuHPpW#?^|^QU!v3WcyWSo%nNJr-J6mTKCmX34 z=X|+bJ8+Y_y!yh^<1=TUdG@*=)&f{Yb^H3Qwp!f%ibquxrf$;h!}Aw*V&{z!E@xyQ zDKt${j+zkLgVWRC>HNh@C&%mHI!1MdTvbJHyKz(Z8^s{EePEanatKLlJ#5y1VrP3N zy5q0>hJW;5fBDx>j_=r&PnI;|*52r@2kwf9nGw9qQ3?qFQpMHD!97>*x%S)(Inv?s zNJujtk14a4Ub;RW)w9_w#3&FM04XJZ?&p3+73Op?|Ht3)o$vggzea^>*WLDbx|uKE z@P;>xn!O+SH$PZb;QUb67kG>Sw0q|4zxb|im8JZbKlmfpUU>dje(9G(2q)|QyT0K& z-}%n})mut&$Ymi)?sIaf@00Eij$^_h#}o;n5JGjXXPMiWcorNj=jGrA$we)t(cHUe zIS3(A3X~EE1OReuh_>RguF7dGB;ZS%P6jqM!g@a&kCyXgW|A=hoM!;Z7!g=&Uv?Bj z_ROg4qgD-Ja=w*Hr`&T!&I2Yb3*=BnpC~iIJ!MZxX@f)*W56r~!%~~R?G*6dFqy^A`jkk&cTyn$dL?B#i&*xFgL#uUn>RKT+ zaivsURVPEp>lQ=?lQ9zx-CA-HAal;^5x+cTyirH|YPPEvxg=Rk3`KGT>3YjKF)3?L!nLXDHk)rL}Eu(MTz9mY-6>aV9pKP2d zrkU7_FFaLZ7qcrSBLs$ml#%n*iS1W)$q_MRM6N&&a&^X?C+59D4^XeH_nbn&Cs`z@lScy=A|MvFf+WF)^^DS>##B}}k z>B(vpf;wI-Jjw)QbJ<4bWlorx;F)+KD&m;c2mmw+pD9DME+h)dC=@(qVq8Uv3B!QB z(Tp3+kdsfxzSAfemqSm;gb%r5MiQe$IWtPh`kbO5$*mt011)8NjAN0r$DBCl!8^fG z@)DVTc@$*wfaH@bn`w+f>YNfoM5D+0#mD>9pEwwp?%n1rz)H7qs5%@rgnLn8O%oe7boW;#8=;c->BJg zURmq6cSe=gby22_=XQ5qxOIEJ35ZzLBSPdQkbo->oaed9K_IY;_uPH&eUEHH`m+x` zF+VzJ^cEtFj9T_SCE6SF-Tf&tTaJw2bkF(o=g#jsA6I?+$P=IF`Xwmgg+4l(x2t7# zK}Pa!bLZ&bbiVXCrr=lIdg%J0Un^NX_2TU(p8UjmevG40ec|G{W~3*D*(&s1Tl;6G zdt0N)sMbI7L;os-@ci@7f6w=PPtJKZn{`L?^eWyFls9bL@WfLezx&DqWjXzs zpZx``%-Q|(|K;t!@Du<3ryhRjf!*=AsmH(lTfe(rwK3zD{r#`(ZFE83Uf4U9*24yn z0CLKpWMQ=A0)Tmx10ZpUF$XOT0irRSQ_enT&lnSknu~E+*JZ^R>x1i}Pn3PmU37t5 zsUe5HG`y@hLoZ~?IReWPDh zDU**%DF8^>_RbRG5c&iGC}8q+skD+gS^y9tLyAL6E}tLkGEVtGb(#XgPUS?!N3O*8Y)%8vY<6YRSyUemQ>R{fXl9O?9 z;q1&vb-3w*#0)gF9m6vCJRTQY)d&DQBacKJgLOeLo1@gKX_^?LbFPa6*M>qaM3X%r zMCXQ~Z%3tS#)=7IN_eZgO$5kW(3e9DIHZB19}Udc)t1XPJjq)aJ0n}#7- zYeR6Rsdy?QjtQB%hTeID&Y_#UixjVP+2(F!D(OH0uf^(tTcD-DkyyjI8r0o0k;)R#) zv>sywXM^wDFxaLlk?`=~HW6PQp8|8!$%G5$;s7K|)3nQtrvy2-C-d1>(>eddv(EtY zQo($2h%v^~)1&dEyl`&k!Ta}X6?xoH*i0w&xGs?LWV-buKm4!#(0A?n`~I(g{xT;S zeOS!rzxj@LzUgz`+_wFVTerXBtG@ctM<4r973i z|ElV7rJ8T}if?dBXCrk(?AC6v=@_V{(wvjEHbus@c=-hegqUL{VkuEd#2AxYa9|4g z%NwmxIy1mb&IQP+t_trx=PX9AC9euy8dVgUF&2}r1n%!m?!LGuk$IOhCm#}JHaM4o zgpwgeWmHCj7`(RtR1_xrz@>`hU7<+Nruo4YwnW@Je{;cE6pXv1ONauuF8}W8s%buOheyu&V39* zh!mo8&RVOKDow#aoZG*^7|S_ZYZ0I>j39r0Z_L3-DHpA6eO`9<=7C)-JHZuVUsY_< z=x%Ykw|6E*+n5Y7MDIkRI0TL=W`RN}$pDZu?7FqJeYaR092`uhTPX)FQc9U(sukZE z*DR;4t*rueBAV`OPp6Z^n^%Dl0NS>lOeXVHtF<2bRUy@8=tD@+=ccKM@u6*vlw9&R zz3EMNKm6dsk3QB+wo%E+&LlIKZBJ*j*~dTr@!4cFtrUx{u4IhaT1(&&p{`8ycGGu^ z1%iBZaBZ+OGdeuDb?er(WQVPt z?LwE36IaZ7|B=W4a=ke6-H<}u+1cS-*2SoiOlxIbzrVka3~6U~>cfCqj%T=YMn81_ z?uGrWQC(@lfHLQNYdS^-L%aFs|M~}y=ckj&venQ)*X(- zFbiR#S1E-M9AmV0F8G82y4FikBGcAdN?8JOViZ$MNdTjmc*u<`M`a~|WKnH=Oej;x zkXiIe2r-$AwAR~`qEP^GGoQ~Dvn0jo5TFvxHb`}{dsYQEpWnP$s-hb@Q|4KX zwdNdxQ5rI(L_XxrrlaJC#d>jk^1{&0fWd)~)f7ouA&CY2F6v?r&A7>3-K zh7fMtdR_|TlwwX*RTV`spU=5esxaF-ljxvsrd3tc#n@Uq8c%c1g7eEeXN^&bqYHHY z{C)f9&xPC%K5p&pzv?xwX{yoT!5t2?JsaPD-<77G-nw;gbbNalhM`}YLf*W615&K2 zs;ElueNpP&z1eJc@7163DfM*cwV(c(36)g(@@wAIIni!5RaLFm8=zFxYFw75C(COu zUj4uaKJdX0ec;dD_a}p0E19zOf??uz@MIzP&Y ze|CEEu6Ms{@^e3b_Uzf8{^1|_=7039#By1he(jrcm;Tn@dh;*+(l1$St?mBD|NPbE zFueLTkG}qOkN(E5y(@*-7_KKQ4{;>TK*1Q~5f7c^nKNsRKpTM=5?~((;0T#@HlP6w z1?s4j8d8dVr-fFG#T2FDedw)?D%nRbob zcjlYSKyd8cS{Awufk$8r4GNJWFSw1HnpK++QkK!XQWQR?s+L_J3y?i@tIejW3MUhH zEU}XxmA%Q$MZ4IvdR)hDnA>g?i{-}4I=%mcA5vP3 zo4OxXTeF695eZPUovlKbRO%T^@??E79?ha{Q^piT+b-9AubQb1gNfRVW^UOVDG&qK zGOa1|&S;i`uy&Oh3?UpWmPJuu?8-_3Qt6tq+`e_vJ@b)|l2C;%_xARJTeL1*9-Z|m z)w9tPfBAT5ds&80`SiCS5F1PHftEw3L z#ooD@=G)g^dVz6V4*tU*{^(}ikE)sN=DINJReR~oq)lvbxZxUZ9&UKitXD4$VOYgz zj}Dre+u(6J)k;Xg3Kc_P6i{%)>SA|(usS+AHcD2dGGjg28XKcBxn@*URfPyb;CtTl zhrj!W?|$~#=imHUZ+_J)9vfA)z*KPQ+y46VFaDh``Qn^&>@7$9_$QuajQz=<{L!!c z%K!MOpZf7H{(EnoOp7a5E^juQU;gjEe01mVm5;qvR?}x*xc$=Y<&BfBPdXImCi79)ecF5lQ4KBI9Utbd+8uLfW=ntTxDm;wr|(r1Uu_M38FUwu!{xGNAUE zat1qipK>1vWhrHmGXQ{4ybxky3Jk0}^5K+=42-hTN()`%bOyVEI7$8$U-Kne%f0Qf z-yBPD$_^@qO}p&6_37#H)oU*@M(a(BKwINcrG*e!RrQ6lm$IkebK4FqW$%4VG6o(Z zQ<+UB7pBHcf{>+%MRouA z^E-x0;~1)gr0~gaZA~|;Bc+6tLMpymp9rB+WW+#9*=^drompvw(IWZH$;pY& z`swGMQ3`Fpx%T|^&1M)}@;+@gn}ZuSkdc&>nfA9A=-@W>QMALO0 z$;o_cYfEc=aOcs{~q!Y};74}S22&prPXqq?f~a()N_`V0@zPEC)N(rS4-~E$yh8PDk~+U-rFM z(f~Mt3>a})`kVwyEK!EMvAr@{0vM@)V>{%Os8NVg#TZS=QltQ|nXja&8DmPR7y=UH zm>dxTF9Z>>)#kKnN((-TD3f^H*QIdGYQ&+pP$Bq07Rk9OAmSgY~P`YF3Y?O}&ZM_f{7Lfzn#8`sMjE zXR>$5foqde$SF7<5rq=GDn*Pjr4-xLd22>Bagjr>QBG$UZFh2bx?FGix*Dz4i}CEt z{Pf19N%PX_st=6S8F^A^G%ZJ)p%wh)0A{H zS9w;5@%E@4Y>ata&rXgH?|EQ<=vu8S!8OQoF<;*~T=v6UDqfk!W^lHx1fNbOA^{gn z8(jEIDOaX&gRe)`Xf$G&V(3}Wul(f8)4h9#-c?l-f~(5OYQ3;2zx(%ok0r`+y*=Jx zs$8}kN;&&bO||W8X^M8pIp<)VbsNTTe}8`%oR863uMCg9-#x#7?WNnJ(WD<1U;Opo z`p1vY!__BK`@$<9e(VDu`T&XaHQ)R#Z~uS(OZ0=awyx{mx{xB{sL0pFXdt5O84_h# zYPIQyOfatMknLb2dKZMiqU1fWjm_<%)0jhyNl=V&Yck!QO;{A6+iX>|HNyR=xwt!e z%`5J@@8UN4<>hI;zddS3CS||BzyE8$@{X%Fu0Q+Y3*Y!H-&{&vRaHc&s_OX{um0Ze z|IWYrssGq_;nP0#O=tEl-8ncpyLaV3{PchLh5!1CG32v5v%4-vK*F#it{^Zr2?ukB+mXAxFwunx<@kNOIXn zIBEOi)oRsQMwom6fCwp3?6VtJwctt^#X>5jW&{L@F+vV5CeCHobveVT?OX`9vxwx2{n~srpFi}_!(56XtYyKKDciPpHfXK4 zMnxe6Fx3UuwynsJB3c`OWxMVb0LHxD_~l~w=wChA_q`9c>$({DFgR1NvaY7nDWyb| za*m8;-i+(fSTnX*4BnCJJ!gp^MDWH`)9EyL7ro<*g_xC5li7H=TnnS;C&&A<>093X zcmMVmeCg0`mSO$jCqGmbQRMg&|K{fv8^%W2euzw>kOau3F3J&PmNR8?J3HHyRNJ=YRI6fA?L#a=PwT z16{v%=f$<_UEUs7CD%+83rkt5O*`}vtP_jDmpo01IssWqEGO$)V7ZSt3{c^iao+`} znAC+r9lGpD7AWfqxpOO(`*VAZ(8PsSfSg#Sn0-d@y2ynV+4ZARBgv*v;M&9qqUQum ziJT!R&LzeWG-sM?VFZ@~01|}m^mNL(0uiW7A{Sithx&z&twPG#hmuv+TEc{a+ID{T zg*}9Jv5CMqgrRM7@1ksAwQP}0+_YWaFWb%9ItV!5tfqA#it=E!4y#q`ts~-!LI{G? z<=s6gWd@uTVp^93tP8N}*Gw_sB|uctltqm>LJqZ&h^iDNg!yc&?%cjXLC;USt$Hi! zjp%V;uo*8WH7oTQ7guQz?FwVYc6B9)k7rY-fNkX^jTgN`} z!r{SjzgYGhyf8eac+Z)7Iw{XzICJma_rC9a?;C7;*SW2l$x3L>E2(&;r3*yJ{$HN{ z^J)Jqzv}}}4&QX;>(2N6iyfwWdb(#uGowjJLeT`9Brh04fy~0%2HA^iU5l$!w%5i4 zyKF;kkjo%!76HNt7FI?wFao5JCU&cPI`=Q%`MR(4%_pCI(60Rh&cjo6&gb)fzh0QT z-N7_q4v{TTn9U2y0!_i`=At#7>zZ`He)vazJkJIgQO;?=ZKt;&vY?b!VKhubw1Baj zRg(ZlF;mV!#G_=MK+gi?NRYLx`gZYu8TmOgl&^r5-`zVz87XH>>4vJfIW_&Kbn1 zj4lAo*=n`TvaG4Q(PZd!!&$n2a^I-x@uSlKy!B2{lBE!%_~dkZvyNroPYZT*IDUFj zx=_z^axj{F_mSI4Eyk=>0Ws>d&l1XMP$nK84_6oFbldd-deumZ1ml${`%V{KR3Kl~Yy?|%Hcw$W2e5|P$z)%WrBZ+{D0@E{iYTKTrR3P`;f9%J9>6iay-*$q~PPLwncIPvzs>Ssc=hR`A&Dz!Dv zAw)3c!a}G+?5zHHpUE)s)}=D>-dk&(b8TNc=i0h$+P>-R;%3{rPMdx(80C4&nS>A{ z3<&2sLWU&5yj2cSf-v6M-zlq2!svFpjYdrd+0CjB5nJtI2+r8P>y(ZNQif=r=L9n$ zMAz4p^7VT2?uT!kpFe{bES5JJW5IMW*yry&m!&C!AVi0Pf9`OGt?rwCEXUAANx}X! z380aQMNtV(LI^~1!C+h0Rb?RJ&KZP|cF1;M* zJPHYgpctb(8?Lvl>f_C_={g|OVZG_7qixmKZCp1IAR+|0wKF)_9gPY;9VG>m2_Y$? zD0o3IL$I#8XV1?6!hiLb0hNclxBkHI_ya>MaD*;|gM*`TTdCe?6`aQhC-=Yo&2OyE zFDGm;-#xgvy1sM&&fBkl`|iQc7e4jE(+}QH)2yue*2Yz(K6-MoT(i8dHQ*?Qz*y@1RA_lhB7g1OTH$)b)K20PrAEO1sV=M6C7N2amBsKnLd$V2m+B z0RSKGzJ(A5L&7B&S>0RfNkr-V@Id5+@n|v`TWbknJfQ)E)pn(|H^u;LyS@)0n66G3 zaeZ3|eEVoG$?_l>V*F(P(0eb0NQGoH$4Ej?2ucZIToBGF1eg%Qu*1M`3}@q!04Qac zI{&@z{8qm%Gt2~I2_$FlyfI~1dR|p3fXI=0_@FCm4tzckz^jU&V7=iCjYgvYQP!kESg(=^8b7DYigBMeLFDfI}`{ez>! z!$aNm&z@C{wxlSkP7y5n7Iv*yO5sXP5gA6x7^74+wWlmy+^o;e7u$N(C+>FZDVxnx$B5i3?b9IVUY@g1>n7nkpr5D@jw9YyNDn_*gEVZAzF`fJBLBDl z$RB_4)i1pD?%RjAZgUz4jhM1!sXlnRzBsQ=FRN{7mBtAbJW}FFbTL9tg!Hy&$^(zC zZ?+*y5=g>$vo=qjEK`JcicwVU>644L556xsPZ1?uD7grEM!gLIV?rSheU5R#Fu(*v zZX?Xn6d{Bulb9zcNP%$Zb4g|cPAoo{j7h{XDsMqb&^W=vAxSvM1;`}L^E}}&Lx_5R zbab>g8BuJcwEKgJRW_A^FqRi7BVjsZwq4EAL_yaFr<%$^fB)zR*dAk|BGs|KxY;-h zB@jN4Ro9LN_nJN-yclH+?MyliG%&r<2vUYH(0U+~BAYqKI;}BI8qlVBcD|^z4@B^j zy9da^Cd8Ufu9at5QU`TWIvr^fedmA!ZmAp)I6x1)#+ZKa(Z!8g6$P3N`R#+@JVDbW zPtyZ}u(GIcSkv9qomDl=D2dhuV|kkCE2wgWNRBn_NGh)q+i<|E7r?m`(P z>AR+0-v|PC_I5HVlqGqRBNidR5OYKb^FaipbSDt_2^8BOo#~*Nksmwlk zck*X`=D+IEk>Egx6baXB{oaEQ7aANL+}S;tQfG|;d6K31C_-^_dPWGj-Znz!v&ndQ z)8{OuPyps2X5`WL9u$-rpfTY9hptyRAqctzQyqYDH0X$6NYm5>)b_m`K+0Y4(O8ct zKRBFupv!F+I@tH&^16L=wy_?<7$m1KxLXIa^?LndU--Vur>9RATOTb%9zr-wb7Q06 z=pX&#U;L+E`(+4d0Lb|OqqEljp?~Wq|J=%pd)?{;lQa;_v;vzvp5A82#;E z_`9Ec^@Tt7Q$P6^|NLL3^IfG(Q@0R8>pc#z>D{CACBYPJ1feq!g%~swQt6JMSXbRk&mXSV zPr9uEks`qA)dnI*C9jv)1t)D=Hv!M*Q-qMIT29H{;fZtJww*KeK*+k;dJsBoq_$k( z$#j+rC{l%xjuF0lWB=M;{kwnekN=5uC}3C--4nnZ$B5vc_?=)9Nvy9)`pAWOl z8UY1n|-0rxBQhc|k)eJ&BdT`cRrb`^W$FANcVf z|F8eMzd@5*QKV8%ww+TJ{K7B%?O~D708@?y6~Uw7aD07rbN}9*gXvUKI2z7~$hOPn z-n5|Bc&FR4>QwK72Om=;0U}HUKoy7phhi}BAtYHAAxg4zT~;2^5KxijokiQGiO5Te z5F&`+q!?V^Tpu6JDCJ$(rJ2Z5=4{`TWyX+ds>7o_YfxD>iQv=em~tYA!);lwH=86A zSuRg*9o@Qfn-%m+zwL*NarJ8BjSi-l2|s^!`sjoA%bS}JO<9|&Y@IXSe{5CsnG~4v zLB@XUCw@FY(KdZsZ_-Rc2v@5Ofc@t_|JhG}`cp4`>XX}gODUeuCU;-Bm!(pY?Dlg< z|KtDg?*UgrL^uO^l8V_p*gen81Gah4ny=lh9Xt64(fVfvb^ySa#{$|<01oc4b`=7ao0Xti$Roy}W z$C(HK^uU@3003MN)hUc<(=_E~{eSxDpT54n7BsuNv-9g;|JASm=C^<0fBx&pV-EuW zNGT%)=9FdAsmHKs+FmJ-DJ0PtvuWINvyz<6r^%S7!IPjo;jC|}rx#&&EUq@&R^zAF z8=y27#6}%iwB86x2?T&JPC3Rlx>`9TaAxht5rjw)4LYPeMbxdT7BL19u--MbN>TwL zF}?4Z8xaIB_Q=ox3z2LZ*S5-eV6^Xvi2*lV$EK6bvS~4GooaPCo)5H-%Hzw$1^_S| z?~QP%>(*$;88K}Om;?!3Zq^V@O;UJ7Qsvkmy?oq2P={2cDZBzg0cFzFGki zUb=T@D8ijvCtrW_9Y_hHUZx46i1-KzR?gZ^krYjaa#bmgi1DGRR+xz4FzYFcRTrS& zZsKr8uh)yad$Y?$$#Qn*V9)g>0(v+cmYZwu?Q}M5H*2L8OEMI_vtfC;nM^YaT<=sd z9iUNGPI3+rv3-+LSvd_6yIyEn z$OIZyR{^3`xuw>@FMRsBw&}FCGLtxZN>C<2h7-@7>j35~EAs2r8u}mz_9hS{aik7ztjk zd@D5VHnU>RfdhdPg%nMxA`@NHdx)0H6&4gC#8FHH)&O>`8mDw8&m1I`3srA8v8`dsW1uX{V#NC8HcfGsh|_16X^~P9uWz1BC$}G+mC!eXeDCzUJRH-*qn)OydbI_7 z{N!4#+J2Dmj6lXEWl@>Xm5K=t2{hV8;2EPJVx#Nb+5XwZW$go_{QCO(_Re5PluU4I z{K4LYMA-J74SxUV_|5koM5~Tw<2(0vo;qAJicY;`12;JJzaQDI0!~Na8 zAODdb?L$b@97PnPfAn zu+~P5R^?4zg9lmMs*%>mEtS)CDr z0c@>>lzXK=hSn*kg9j1@jCrs?rX&%vX%_PorvYp`%_+NHZ^-}`1A#rbT&#A7BWpdx zczLz>%&V_mUYrTSKK$r>IvqbdyU{TaEEn5TmKHF<|4>M6it^VKh3c z18Qvd?DTY83{ipsR4=!(ZW_MPLQw@POW?V?rc0wehVH)ewrN!RK$<56T#pvYt4xt2L#9Gs4 zc^Y8Ux}Qz=l?&71FfO|^VM+rZa0IYy`>`ZA8b&F>p!XhQTGd_Iwxcu;rt`oJG8Tdh zAtZ?yGOD!$7!XF6i<|E07TJC`OIE9PQM*mqzI5j&3!5gedu+{>EQ#n&$8P!rw&%D`o1gCzKM3GA=2lE*J(GM95l;@yGEi zI-iX}#5&lrZ91h8Bo1en8}+sCotd_t4)a%@+pCoV2nH7s@RNcsjn&3D2SUUk#9^Lw zy1|%KRW%>w%2;i!kAWae5yAvQ01>7RL~R`*1YndWQn#IuLfOC|?K?|E1kid%jk8K? znIsTK+qX^Acpx#r6qBlK*?^nq0itc+86Q6S;6nfz4mN_Y-Ym03cB&5%=2?1uaXQ&O z{OLdYGe7q~{B;G_1hdF|v)K@W7!_wvFD8dO#`^hm_rc?fPkrjs@4WGb@@?5PyYb`V zGQPf9xQHJ=IUVH%iv828C8dc`{ZJ0;ay_06>b8u0*!Qj0igUhd8e>eBq=-OBl3poF zZ6bLlD8$6LND;8s2Orm~WD^d3qnXUAs&X#m*`R8hl(XOZ z&UcS*-8R}(mBOTo0W(enXZy)+t3#S3b-8NlUPrHty*@qr-h1!vP6n@j_OqBL45Opt zgVXolB7`If?YfSmknc`y^d!xu2m^|t2{9rJ0E7?#V2p9Tb`hGc1s;TmpFTM|J3Bi$ zI(+`+=c4BT`1|+oD`fx*f~SlN0fEi%DZXE^~t6kbUJVU(&q-rcWljzxf;AJU*Hn?jNfVh!7$Mh%v4jg$T^YqiyB; zpnm%gfAN?9@vk(E$udS1QVvEOcXmcWZ)rf$Zft;^bFo)k^56mBka1jg>kvSUF-?;k z!3WpNH0LaV&GtGY0OIu7v+cHH0`q$(bIJOPO}lOFWRNq2A%x!R1V&7JMmWr}IL(cA z7t7jvKN%F(V{Og<4*H8s&j=ok(P*%)ff;4ESa;Om{|w9AE~uU|7LG0=B^s094l^ik$aCh_0z3 zW&lM9BP9?=;9XA%=M?oia*-&bF~WopETBx1wl|=6F`-2mcmx>A#;EJXa28W{K{9K ztRIS<>Dk3Okf2fJFqavm5P)nns7pJV-1_=AUVrUQcD-x`W$UXe!Sk|h%FPydELi$% zQ3)X+gtvCah*%JjuEsn;A;_-nSIbf+yze^*WsK28iwux3L6ixB8AT-FRo$mNYhAF> z?Hq%cCRwS;+2z$Ro2*(;Yu)rc8e~o1_tt(~y6sh8IU~6QfD{8v=waX2F?y~0 zB!$GV-RZihcXO?|fY z&&uVa41}AD!n$|Hp`oHCi~r8cD%>uvqYJV zbH>{6;{DrFq}D2Bjn;keHpT!E(H@j-w<;@LHciuNV|!~u2oaFZbvSqoDL@jZ*=&D) zaBw&oOoNXxfCvMi>8b_qp02;~#^djPPmp=*=&|^j_*weJ2HIf zlg}BwLBXr_2D_lU*tu8&zbMyb+iiNHy(2h42(7if?pO?g@!mK=85&CM6$duW#Le|0 z&5Q?jRae-zsoo(#&~vl{?@nN)NOfw@nI^n2w@Lwt3Be>`d|(Gu`mD7 zU@%zUteq}by#<)y0MaxK2zxKLRqaubGWW`v5If^!f^E};=%gr~JUM;)!RwEnomy|| zwsNX8P0w=9v*OM7-<93NzP&QOZrk>gpZijy&FM`e3*N1&$LE({{DH3yCSydw{@wr@ zvt2H?UDGIQ`bH4nd;i_H-rHgdAEb*_X`L1{ARNnTg#%!WVNAPD4f4UglLN|0jfjIOiB+>q)TCIY+3@C6)!_k}(8R$)iI!;9vr!)HvHYql`<0 z%+g_{%xE@eDMgS^CKF>jf&++ld3}>4*7^H!tIPRpG#VA3eD%e)v!-eCxT+W5 zD>xpHc6TPjo3qD8_h}OE9vzKFqm=U3Uiu`oo^iO@tYkuhaU{}lmXRc@+v;S0y1#dl z=Q-m5K7VUIoeZ6^O|MVpk_*&n;DQx2Pcy1Q7mX=5f$^j_nq#%bOcW z1()I8@o{g1ZX4ZJmWGghR+raB;jx*DK=G0 zQci+Whj%A+V;gOv1s1X6ljqmvjZrt$2j@xGmjE;GBb6Cc_Ht7hA3($*0EoqQQ(5aB zG!RjMV2qv&%S z5eiwJ9v)1Avs!PYOa*2L831D2b_BB!9T!Qj6{g%;OEM`O=voB;B(i9Yveph+)~nWs z9zo_Ian@g7ZL=&*s7Z#|^rzPUpAVx?`aOJZ6H<$CI|K+P2le zkB3R!xsM*M$J4=VQotns-+%U>{^@`B5BclIR%yan<-E4&15GAJZtu_vFKruy_V3-SF16o)0?VY^0^rnWr(#AXGEQ3;} zj8QPmaIJOKDG5o!MF7zU6r81m7}sM2*NyA?NO4?kY9I8yy+I)WM07TmPcLqAE|u*D zqcI3@G#Zs)P$k^$%;jFPhTdfT+8RXHl-+AD zzY?7=8W2E7`m^v?0&{lm$pp1UTu8 zV1lO*Fc|XzB)r|(=S{VhGA3!*8_I*7xdV}Tz>L`y@+&|1<@I*OF?6<9o!+d}VpSq4 zT_kmDUGF~k+KaQ1m=u^X{wMzUAAtT@hX76Zbd(GeG9IQ`LML;&tuA7$AAa-!#n5SU z{M>Vd=bN%i2XeYImYMLDO-6@(-;H+%omW9p4B~K*&ITF71R?b0*FJgY`0n+y%biq$ z5Fvz2a3K)^ux;Gw!flq8MusRsA!3LuCV`8xZ)1!Qp@3U;(gCQ2yHP$I9 zGmH{Kh>$os-#2Zw*_PYuY*0u!SgaN|%X6b!D)`NIB~z9Vc(S+4pndtJ`(OFe7l*^? z?#@w`j;=1(=Qr0n`p)a7>vs3%S(YjT2IIX(A#Z%$Y?q5`o=NR(B*+3pDGjC%LD{~G zNCbdbHfW4HJ$(vM+~41aKqPXYRltz-jk&zKAxw!nvdSI3^E^8Sl2lx@@N4HqEI0nM^E-hTIux8C^R?cWldMZ$bg#u$tQ8D?qU_Xtr6 z!I+{TbFD%H*wcPfeejJheD+SeeiU>GFhW+zR1kn1_Io)yU(^xP>$*Oi%3k}Xs++nJ zf@Lz<-yd&Q7dVscrfXCW05Y8&B!z~iR~92$TM&I&8@g9Wo?z2UgyR;tU5>x|-QV;E ztV0VykWyj{APNC{6kUX{>zheFsJ304kBl+f^`^L$93N*pF7bfXRd;$*9qgo-#HMkJ zZO0 zwjxP7O0uL%3Rb%s2+-W)pCPfWvKTo;^DK(I5TMhmYPpNGA(~2^m5WL$iubBtuL=Z|Iot2M-@>BvF!HtV$gtz>o_<0BUW= zD8d*yLQD*ywkY+^IKsxtE+HaiqZtMq5q=G*K2w zj6ThHYb_y|+AbR>(Nbe05p3PJRTY2iM}9a`Y>i3>!%7*#iNJ%6R$u@1Z-L%W3TKJ% z#i;F?(P(`B`0Dk`xBtzb{6oL_um0B*7aeW9_tsd9Sl4*U3D0=Hg*y0vgVH@klx4$5 zj~|}gx(5)90aeXL2*E}2?BZeDTEKzwZG`Ib+08Te_!IX(by=#d?S0$bKDpJK<{+I$ zCV%VOZ#jaRa+&30Y=G0ID!aH{|L))WyC0lw-+JSnVWwaEzR#^UTZAaq&KPeS1Fl-N zsjJ`p2Ydm)xd9OqmL)s|5Tf7*gHfJlgCa%9 zctjihG4+hJjZ=NVfbH$=DBpF;EjA^j{N-0(?X|Ak#v2t)I~^rI_QS7!_7ivam;|Ow z7#k!5F3A6Gn+U)MU?Ye=cOhd>jhpoM(gL`1ttv z@Q5+?`OklTG#UY%xX2z~Y~Fpi`i<|MFWUrR7C>lJO$qnjYtu8F5XxI?EpP_Kwd!>Y z1frk}3Q$MuJKme^4b%VS7k;kVUUlo>kyE;B zl@mgarlTwq8bJd<+hR#G6j5Eb02nMd6wiU%Q=k5nosO!0|qOL zmaXa{uoOegIbtIwkpGxQ$6$yQ69SR3)&)-(?bM2MP9Z7s36-geo=PB6PMMqx2B()- z4zU>Hw&}OZJB?hh5typJsp_qBI>9jZJppjQ>Cx_JG)#gw2u8`pKmTX{8y7T5^N$Ua z&beUpbTFYX5bC)M-j9vlg2P1a-*5N<-Cx8F;?URFpz0r7}ESc==BZd=3ql5tcjx@WclKu^nI_-<<~P&9FiVo1{S)J1qcDW^#1>6j!CLWIfr_2P6{+K3XtZynAM zzz88qY1ehu+HsMOvSKWBh2!?>o7tDV{5HaDoGM&x>jk3`2NBCV1G}W?)qlgwTg?xTI+&4JlflwZ=+qj z{8IYW&%bzUFZZ_O$jnFS$=*)E2@Kkrw(t9oee~Km%;;*nJi9!=V|H@=E#t=~=5ayMJjt{>3)?>(z-uj^Vjc?t&_3l~c80KSx zbcD*<5`f?WfRuBVc3OGpt?`0WNlB6<7!u&%AW1M~>&l{Fzx3KCFW!F_x^VvZa zPuDWZ&#%sI?e2_5MKT^YeJhj9dH6#=_`?trnmLFnA4)+B)pqkyrgRMe8V-l|?%w+P zul;(OjLy%<+1dKf{-6KIXFmI7)ze@6#jpMSzyA+7BZS}(<%=)9d~tp8(n~Kn4=Lr( zKY#Ml-CL>PNiN1a_};zowp$v|Szx?#d6sd8VYJShvTpjO%VlPaB^bjHVFXOd0FaN+ z1@DbR(SgvIzG4`ewujcD=u=7op>4nck@0wpqZ3(@?+$4p=R4Ch8AzE>7^R>wMsEVf zfJ$-u5{eLtF+P6$*gC!4l$3$dL{J7UE-of1b-MDV@0!hYGzP&d z)xH1z`|I_($V6~PFs8NUOn^uQVP2cQ4q zSL;UY?CcO6?mhRsb>Z^z3Kb#&#-)^#XfL;I0~n6sfBM^hYpb`0k;UdlCZycC~OB876KY8y2U{`yiTr@VUH`T-Q1*IGZ zw?7$1r(N(zw|0gj+4>qHf)TiX@A+b98ff~L{=5HCQZz_Kw2Dx`1?l=qNW58>5G5%? z)#ehoe!JabMwQZ4Zv#RfJbA3UQ$$3i1avJU{oOmm7w_y(b54BEf*B_4WN*qa(OqwP zg)t5w`d}f7A%v=GmdkZrw@R71uC>+xfB=AVE;z?9dGNu9!MT&2`O)6q-p+243P{mw zuf2BNm(On2s~$W$?|=Op@BeTA>Wz0lSO%Sr=pM%eg$M$S9zaetijh$VA`{ewUIMtA z6-COE!8jcse(CeCsLf(7)4NBvWikz*$nsm;+P(DYg9)Y1+S>Y8KmEz`v+Jux`S*VQ z7rLreDgciKp8TQz!%tzvRoi(V6Pd?A%By8t>E)&qMaT=7GxiUD>1VAqn38;4{P0iw zh<5n#!$pyg(v*AC-9A1lQc!NMMg`>@Mlda#cy_-2`a7FPSFl{e1VkBqhQoNA1Pr=} zGM>VSIFtW>-705NYpb>8ETx2V7C!gN?b%S|QYKJJOnLN@3(Up&<+ANm!rA^Px!hbV z)%u->589?jkSB83_7zJx%MyTvhfoD@`s^`=Kx=z``qX>p14z>pV|?q@t$+YdxWlgL zZoIcm?R8%s?$7$J9u&pdvr8$G>+5S{RZ-;FSD~+L-3;cV#dgiIEI6>X+((m7Yxz1BKLzpb}j)2vt9#uEsrH@b=T)$jW}MyWGa8Edt{ ziu)b~oqX@N-lW6>iaqcLhr4(0?4u+J04~lik@t2m$YbYLGKQ zgrg#wzz0djAh|sew|9oRtr5V4px|i$gdso4SW1YuZHP7%6eVzHketZ;?lg<47AZeC znojZ*MAvjBg~ar($8@o*0U_R!&9;8!Q=j?Z;rFWR&FcDUU0wSC2-G0JMt9D_D6&m& zWJ1ekQ}xc6gMIrwNy$uF8A2 z$MbP=cycRhTbEUcNUH!mKrsXebo9%%s@v}RW?9yawGLo)s50Nz-N~?|Jyg;{70|9{d7@=7@0F68WgPaac7hsjwy`KU_fj6No5V$qWNeEtVb`uFIe7#y4)wheYZ~gkOA5DjngU`Ku zGAS|}xc0?|Pp|*sKlp11*jB}z@!o^e8;7LEXac6J`0#_rEJ*aHnQwYY z1Vqe$;F^A#A8ME0to7lo8N$5Wst|&KD%y5uG!b$z9_CK9P~_H9AZXNTG8oH*M&L$i z3J~%E!GKmxOEBm=EhGuSBZQnqvd9=s&mW)PJbj$!QV7PdlcR&S@~iV_lil&z#p%m0 zzx=_6>r94=>zkXaCtYvyS&E=5#@qz-?BWb#$R+9Za(aAhBXl-oG(()h;3DwEIhRt- z9WvH0*JYAs*y~`8>n&xHauFf&F_0|BoKYz?iVoV^c8%*%LP;Vvn+?>?g;;A10}#0V z{!bt84t&a)?$m0rZFFA@`JjM<%qEFQ_(3x$H%j{06Cut*6Alv3@Ta?yJ~8I5hHx~_{cE|<&IYSnb@wyA&n5B)&X zwwN%?XaImfgfTY8+}zwCgpQAnc6WCV_V*zGAjaT{v&dQW(bM(yvIjsc77Ij)vsQP# zQEegFkVP8(c3ry|<|+T=$zjgrD9vev!y#kL3=)55mhUDavF_$#6+*IYP&$}?_l@r| z4r3q(vx#Zz_2P1WRzO?M#}kC}#}`ebW8e47XHV|!?g|c_CS7medw+R-eg49|-Q8Id zX#9=ud~0#JxX-O2j z6{l>-!TydYvPf`L9yWCeyi=;Dln;|BP{g(MFd>8#Rn=NkL#P1M9=cdpMaFrKdmSQA zuFD(i9i<#Vs)HU6M{T8z@~ITT+vzX|)(%s;w?CCKY1@7<%toU@+xCGIfFO;Kg9HOS zn$5GUI6gjHtya1>&UoN~={ck6cDv=AS5-9{jbIEOfwtGR3O6@57mpu60;{glUZ?XM zfKaf!?G%Fi?Af_$O52&fsuq{)pZ{Bb=gseZ7off@ORYc%$OGT>Dhk#bu&uxw?>@S^ zeAZSg9^v}pCgof;_3{250WtcpS(T^Ho*3I&t4&v>Oh_gUb|>iL5SjNLL}XkrKFGAN zK5VuPV+=%wXwfO>BMJZq1h2{lVx;O8Q>eR|A_%6dlLu>sr0*3 zIVjj@kQ6DmA$DCe8)QWhT-fdnO(M~vEE!{adk4!+B@y`Q7r*qa-}t(qD3=07LI}3T zPiE7@-C33vw-1ku~X z{_wo?qOgELN?0laB0j>Coq_MlzVA~Zhr?l(^1*=bj?-C@E}osuhsmtS_lE;tm)*hv zn_pb@zw(>!z4dgPB`lxF(KvC=t!`G5lOOqkFAq{+Rk=5x&|r|!7)IHXkL2ZLQ?+X! zpm%PVr3oW>LUzXad@$JG8P29d6>%y>ktdXs=q*Jt1UH_fAzDl&!jw~y3D&o@OcPG2 zWE@4%^yc)sRE|ReJEh)!_XD2iy3;q83kqqL@absmgF={HFIEuQq8KtJ7?pM1I%iC; zoO6UxV?&5~zB7L1lb=QyK!~}J+SmYL*XgQik|d#&jYre#>l=dM>GkDkG;>-FivfiA z$?2I$G9RJh=*F~VFf2(n>*np>dKUxlluIRj_V|gEvT9pODPs%(#Hgfova4lt~f6!vaeHEmfpGRa!F;cI;AZa5Jig^K{2SlR}O&6 z>8DRmjSfEI2yil(q~m>pY1gW*HQHHE$fm9aGQ|j;?CqtT3qlK-7BY3t*%&@{Fkpmw z18!~>lgWfr$B74_ulkxH2`sNFA?PTHk@4sV?gFT~_P_lr|1FK}ul=)MMKIc+gSJ|k zwr?<|kdnTwCzIK7adWu0_p$b^u4{?}rx*f5J&usQ`{=21bk!Qng%CmmsEnbM0`PG_ z2OuF9wR!)|H?v_<>6_K{c`E3=TgSWOlm?wAcrqCe2L;F2TKjRw3IL#uX|<~Bx@nqi z)tp|QuQuzpX&{7y!Juth7y$+_5t3qrK(tyFgG5L|C>SOD`@izpul@QjYan1UG*P^F zzWx9Fi*JA9dyn2ZyIxe=rthM4w{IN`M~QN_x5hY|=Xp`&<09W~>d|ME>1ExhYJE!e&5L)&cW=){#)Gx@P7bro zYqzcIz**L}&A7;?qkcH1s}ha8;ww66wKto zbT&eeX{|%3k?oMz!P)IiD;PgLd)9OsfDpWuQVKzY2vh(Q`ljp4daabwotutlX)^E* zK_U>O97{_0Vym`QJic|p1ZP67OLcj^NCfTM-dSy}O-2H<#6|Js>0@jBYPG^J`Q|sj z#RXViTz&7&w^rqrBpflHacHgEZa0+DvTqHHt<}R3Pcv?e;Y=!}31JvRkt82IcrX+c z_#o2)(F70yhyV+HZ*stlDw{fy!{FkB4?p;=@4o)#>u)3pVFYoJc?Z%o>$mhQ8`8<4MW_tebXKs>i2iAH4UhYFVX>L+D^UfT0H_bXEbtmTO}y_JDQ{ zwAx9AAqt^2d*gk`2LYyKjXpSCagn#{7IQHWgNxF*;G9*AT1S&*Z3V$R;Bo5zgP;06 z+s^Qm8Mo#bt@NfeU8JcL>|nOzf|wre`{Sw6~O44sf zkwDQ~t}-^<&2}lia;m$ydHjvMWmjbK>ZWS(_~bBMZL4iZ-hTT@x3*e=wzp?X z!w^mdU99V!G2$4+*cU}swL(RzVyvu!Se-4ei!_%J^nF{m?$PCzf*73-Auyc57_$Py zU|B-Wn+3)=#es2DR*UEEA7KQxTgAA1>l^REzLzpWKrkPrX%rotSzUz?f^|T< zoC{M7lc$d+ zlZnyRdV8{04CN>p?;PAZoQ#LZJNccR-0QxJ>U6OT6kDrO1_a}~ySqN9s%jdgAcPWz z-n5-sm8zM~=S4Pr^Sj^fyZW`yeg@F=gZJL~)F)p%+&e7t$v^vNzc3#a8BH!8Kgys( z0AwI{nAL@+)tD_bucg+-}z-+F{D}$3psQ)!9k`08rH! zqo6l#(=?1=34N_x+uBBZ%+hVsX$P-2Rd2zQXV(P~sbpL-BI(U`*_!T!`!8u7x3%d4 zkXcFs@Ij}7Kt#urf?&3|Dq+Wf#VCaJrYftpZh9_B!AR_tZTo>tb0(le58nN--&SwG z{@tP&S`RSGf8S63VH|}V473Lk@gzwSiu>)Nysj=UH!vBILAJXypXFk`X*-KnrMXyD z>1Yfo+jcF)VKU8L-b)XL&Q;b#Je>@0?~e~B*<_eA#sn59m|+5<>C#LJjK2Bp?{vy& z>#WsUb-L}0*4`Sc)q1_I>Q-y*0o2ZG=ew@Ey1JUpW%c~ZA%IK302cdCBn`P=hRJ{@HdIbf7p zF8Z7EgFHQ%4ePqg({Z)#r}NQjb3=2ns@8q*ufP4_hmW6fjBYNkHf4mE?e32N(uaq) z##7$3X1m@>0Zv}{B)BPi!Ev5L9X-UN-PU=Y z0))@6RvkoGCct2pq-EQ+$`_M#U2Uio7`hNbM9@Y^WjZP3FyoZ7?|kbWt+q|G)?Ggt zO%}`T)%8=tXw@}U+wAU5#*<=qcb+5(hq%b|M4~>J4#29^%bRkxlPy-ynr2yVZ@RYa znmT3pWdHV^Tlc!IcODquQ)xeV|Iv0^I^P3rDWzMbXVYQm-Tjl32o$CGD_{QM!@Z+7 z-gxu)-ksTgdU5^a@slSS+N0ZdC>^9JMF=QkvdJg_sFi>Jy$=at*7%PezReIUw+bm$ zjVD8h2*UWq`?v2MOoDDBWr*?7Y$QYyyc!mg5dm zz0oQt)gc6|)o;G}CYK4%2F3&ffk0 z3F3G%8t#sh0@L#?P9U$^~Ll+ z_U8KZ>9fV%-+#BQo8S0Xzb+Z?`@X80#r3vRy={AIE$4i*S~3Fe-Mde) z==ut}o)Jhn_bvkGY4FiCAiDd{y|BA~Fd9#N3}v|)4u_n{s%r=nTKUVHDmt-eAM^d(f-cU58l4IIAatY9q#sh9j)G-46+1|MuVUHga7v4?!iZopP^*XG+p0m zLRbLa_d0}tMkxgVG&(BhjWJS?qe%iGteRH4-Z@j`6uAaMh-l(Xe>j^AGC9q&l#-NS zhJhqVBB+U>xv)LXn7}?VfG`P=MCBBZxHGD@U795cqvf_TI%15CYXec&+vTcjsgTM# z8)Ds>tL3&M6fuezZB_4x-S#VyA{3nW-UD2B+jX^3s!14R&i>4g-+VloL28 z@=-pR6{&<_e=?lUr~A9RPASTW_Z}gD33JYc2=&2aB%eghwo)Wo@c@ve|&PZKc5MLl?J^E>-91?etlU@M<-l}d{B%g<7v(b0%wbL zZy@2K2femnxh*3MCiDOY>q1%kR^yMJEQ>7jG5833(}CbJihJ)32(@Y?qZr_N)8#ZF zNc7%ef&fCTcGbEOJYz||Sg!%Xh^7Lh43NW}iB^qKdOV$QXzRKj%?|{geDJ}uXp!pu z>9c1|djnxV9E}7^zWw#z^qpE?Et*;-VzAxx5W<7qy{2g>ONLo4qLY$InT8Nfru*A< zd3bUsP1E)Ah7*iYIh$m4-BL=8iy%nGA;zfdG)(x#)pah@S~&-#c4%96OeB~BgdGXy zs7Pn|?%l&Xrl~k4D0CUslW|eiMZQGYY{8A|+uMAQ%8eq}II$GDo6O zGzO9+;%N6^IzCh$HrfhGH|v{Ra&9^xnQ9eh8369~aqa-TzP|R+P(UCCrYVD}t99v3 zumMxdDDrK!mU$KR!Q&3^U%>+vRpc7!?d~fiVIAc^7osw!>nuuFK!}=C5B>fLaGJ z@J1i)?FMTS&YV`5i>BS&o@N)1ZZIap(X@4tjYa`;inAhPFTHZ`<6n9;p-C|wonM|$ zM)~=Z)4nqoXO^&$6uEPOlB}+7jt>V1`w|gduFj5!yIGWbEX^Ufx>D0(`113Ekz|bK z4?cW4ne%6-En(z((e0ib^W9GzzVuqv!P;_*Q96Jws-2xZZM+WT{HpXIw!ZceVoW0p zec#{OJwlX;L|W)WGzlTI(Qq~_AQXTifN0%0=$$vgcqIm-pim+(hEaQnsjIraiVE;n zVkZNdCej7JG!`I8K%B@Fy3pvl(oNa)z3xy1fw;Er42Vr#FRnLL({PeOI&MNlnCHm| zVUd!|TDn-PoCsp<_xevs`f7K_1TJWD3Gj*pTeXNC?2d^VX9EC_{rdwVjOR()BU z=Dm*|7)Y+`Eug_U{q$m``f#{=WO_9{xocxQo=?bNa&dmW?W*l&F`i_)g@|~P3w-Nv z@al{AyzSS@g@`AU*)T7VC(dAFsWOgG9uW)8G#gp8QrAth_w z0)ipIH&ufYEQ@64cn`{CFdb!r0Luf~5Nx`KBa*|( z0ZKC9yAPi}zADw<`Un4VFc|cm%JKv{JsysB2J_K)0H9+`xM-TX&6B}MW@T4hHOtP$ zQIQWB)duzsF3KuEF`JHd1|&kTtX!uDi>jB{+ebcUtQ4+Gw z8;oHtg9~L;o8SKVPyE1Zhlc}kJeptxuNK$YY+48wtcwm&!YLNs;lA&UG2Z*-a!om? z+iJO4fAr|#yC1wCAaF4d&IW_QC>60QZ{5G^APSJCnZy9tzDtITNK)h}%lX68Q_ROZ zhx-7dT*`x!W1banzWd=n`RBj>9o#s5R-}g;ZmCKcs z{ASx33pO{4G$pP#(=31E*T43`>)*USe{i$ea*+fG0`+;Gd+)vXfuhr8`|PZ0syLk| z81vP-85C31y5eZMb8Bb1JCF$MP6o5V0DF70xcZSF{V^`sW>Ze)hZN!6-O<5LIvbDn zr$d1w5|vxKMYYVr8=$p=67 z!(Uk}F0RfW6=^aaOnksuF~%uC9RgxHm2$_#2ZQE0X1mnh7Wnly~Ld+$T zgA^gMF1yk6Xt`+uh{l*%krNPNj9P1B3oTyl{Nsu2ag_7&JaRvZy_iD$}hRVJyrvi_#`geZvN4)n+xjdI3 z5-K>s(K%NP zVt0V+hj0Jws=RUDSZkeEgd*=fDyEKcM3|{I65ycp;#y&p)s>x!@mD`{>xaI7_v4n&GEW9*gJUR|%t7hd^f0O9_vqp!Yp>y>-E zU;gY%w@>yla_bwnUc|TGJR^9rGdjF`^x|{R-P_-t7LpDUQ3yWd;?~Zt1ZZzC{NA_U z7>i;)921s6Bu9hso8SKD|KY#;M>*N#2^LVVmh01trO5L4A3VxZsZ<}mgT{NKF^(AH zj|nS%RdRrQ*Nw6vbYVKorC<;^MxrrFYfTxJ2};tuQ%V^PfkcFBYewTejD+_d0mvBR zQqJe&`6Q1>;o59I@rIr{~We z3l5BThex-&y8gAV{cV}XWPrWA%@#=`0A}-ll*R3X-#sw@H%e==J zOM+7v4J|SeoX4_z(X(qtU?G z`ew6w;ko+*DQDxPZM&v-2m|KbP9hT;>s})|eEi;6 z4jdUFduN9|KMhlcNI>IavVS~hLvmi8MbaVX#1_YY zzx>PJ`qsC+_rLGo`27g8-t^A7wr!iHS+Cb2gb+g4w4+qc@??Kn2pVY&2!!av_2v1_ z?k#OimJ7+#)%7JqfHJm!crYFgW+%5dD)vx*`qjH%`SK?|`-M+I3IftqyDZC#y1v@f zjRTM{J{gWuhQ}NqP4lrNlLQ~+$^FCK>2R>oT6dzpc=!*0_GkBHKjSReDop@SD5l&x zFO#{4z`W8nhi3gM^El#bP`hRok^wUNBBbKnO8TAcRH6Th)6( z0sy_TU9Xp$X0hrVWFX6GZM#N?0JTx$k-T-X8!=g|o8YN_c@z@`|8!QPxOj!76KG{LrzOpxs2;zDaZ7!0EEAV5h%(_O(Z^4_{& zjqOuOYvrD8*028dukIi2(FAVVdRbdz42%7r`cr@E?BXiTMxXxvudFM#-oSw6dmlc; zEV*_2#rfWDZ>v1b2txb&d(WP}x4Sn#Jlx-(O^yzBmgRE2TwGsY4@Tp%UA_6vx4!e; zuaN+wN5%vM2*-#KKS|kqJaHx<70qzQA(zkJf2|jj%}w*<>>?5$1KIYq%%uscQ5EH( zE~{o^uiCm8q#gkQgO9LnG#6>`z=7xqa?WXN5olCzY~Sq9_I~c?f1ZiV_&yQJ;S8gGsSTac%9;IBSIVJ$Wuv)FG4}dUa(O>_6{@4HGU;3Fodg?WfnGbkf-5B3u zheo@TOy(@*3@2TOsGwi_|Nc^2DokQ3Sdw7`#gG5a-!tCb|MXXWe-KlLkk&W`i~yb% zNIUq}2bT#CU8lF}?N}0-5Jz!uy^lnxt)T3)ubzxEf@qfJcs3o4U*`Up^zNRW}_mC;Bt=c-ab6s8-L|XpW7ME|N77UzqC`5WE3-_Y~LyXDS{*h z7$d|mj&{-jaMgxJ13`<6^OcJHqYt03C_5E(-=$ngN}`21krsr5$#8Ecr^p8f3Lz=z z{^%e5*YDh#&!qrfAMDL+vm9|WD@487>b@yPnG~!T71QZ7lPKeGZ;~I3M%G?V^TEiw ze!1)_pCbXRDJ|KI#K zJG;ZP#~=OT|NCzaCu4|}>K(z8s#A|I&))s$>EkDto4)t3sO#qYKL2^^{pHp2?DX-r zyY5;miV@7`eZ2_~MC7~HMFu)!_xAVqX9qp-<3ZYZ*J|@e{>Yz1ROG+%&tHG*qeU|R z)VLpayS@Yn1?QRI0RqYZ zpwQWd%K$*&IN3H%5@uSR0Pl2zSa|&O3`S&(IW3#>^^Iz`Mn}~LjAX{MapojQ6QF?) zR1EsgI~R?wl#JHzLXcV+B2!{7PN>o)W^tL5I#%vP^QG!5gq4sK3;oc$_fb$gbm=n(fC!VEjqgI~B$TSxa@mO^} zVq!fAure?<9ztvgX3!yqBBG?K`>oZzR#EFT#V~pl^|AG`{LgJuH&W#sC>{mO)K8vIY)P@QGKR?+J~O#z1@Jt)||E2yEZ< z0Q#N%gK5Txg9)NCM?~rNi$Cz&)HZCFkFf)7+cdS}oaaM2F4CsbC=QE!G8*ia)#{V4 zzWC02?=qU692{mM8r>}}&StX%AIY|=FHUcCSuuwv#h4{xG#(PfpFLZ4b@$FY-v&?{ zV;Exvm&k!fr`HFQTYHnE@i56UQ51=gq*mMOtD9}*5|I#w|CfLK^JuZH(tLbzaq+Eh ze?y4m_~c-5{xqS;KpO(6s>=JgS#O(0B`KQ>*v=%`ou;#aY*p`^;*3%TldJ$CbY15_ z?3)&12|}PG=9!GaU;+du2n!ox-|3s1Ro&^J{cu>k^vRd+KL5f?&%M-j9fG9m`t^2O zRShQ|fIvCfRwW_C2jyMloCO#n0a-4k~)^`R)2BOUva=m`*jkm|6 zo#k?gF?{~sXizxHx)JF{#CX+UnJ(Am#j-D?9Hxxg;9OA3AVi*>UZzqGv*@)+)8b~e z?u=E@ZyV)7UreZWwbpjMsxV>^vOF28=rWl^U=cz!wi@efEROc)#<&ps+3rr;l{@pH zWDt18aEMkLV<;D?B$5KgdDAq`IRHR!jSeamXu3oGz#sU*=kAZMudgSQNt&iRM@MU= z7OScc>b1{*>iF*DOF!`C;dt_mZ~azPtA`IyDMZz#GD=PJtSE}$L6*py>njkv5P~r# zB~MbBrm}7NtFu$1`@ZXf>869^aBrS4a_9K)WM_PLXR@DXp>LO$t4zp@F^*a9IzlXz4Em8G0mh>s%li%w=SaS;z;tQsl4|QhRJMvaB{4zQ`)BapleM8 z0RoBw$@BEhx84c>UYswC?X+oL`}9kL(JW1q@pzu5>0p?Gh>cYM`imz|oO3avU;DLR zy}UXTBx$-j9nBIsc=Gh@;j=RY>6gCn*@sUaN8B>mrK!7pd_ZtA7){%LrG$-wJ0P>BDf@kP$Hve!Gj`)oM@w0vq=$S%+h2uNJFs3 zdJrN203b+-s9O^~+?no_+x2{BgkcM#F176;+H^hi7*jsV3yksE!v|X1gpYQ2=i0Ps zCelHcWV=M@$XM@mk`hd!5515K^GMUwB9@KPy6-#$o3=DsV;9n7RBJ6q!@XkIY;JOi z7$Z`qc{ap>0?#&UMX2Dzov*)lE|QdS**k<|C=eY0p{y<2PC3tuT$|{OKfPRBFB>e= z+PKTr2Cz}JL2v!m{hnQ4NJiREF&lWqQ9YRwC{j+D!AV@CGPo34{(V)nO zX(o$7RumcIbUK|rcYp8xy(1*pcs%~qfBCEVfVtSG85xcEbNkbmZXJB)#pmzsoNP{4 zs_7+6vShGcl*Gf(SOG=nm2vHAdzIvcHxXOMXznZ^$WMm@2cz}Sb_Qc?gC~#!03h-n z!^kjAgld|jz1@_Pz1es&9L(qQM)eSz{dqo0<;|unTNh0X0am@`QF{RE-c~o&U@+L* zpNOJZRoh#~d#}9q^4{S*%cYPcN!ZopCc1EWeo8qzzu5o)?;P#ypB&t~f13i53}IP= zZCg2KoC_G^liT+=Wk<(5LgMjMyz=V(o5glGnBKqtNvrHPzWcqS7hc=nJ9+Zx;c9!G z7rW2h`{bYas&@`}#8|5~9^%u}Qx75V6hX#< zuWf7$SZ%h}TH`EdVN%SxkQ9@mHag|O1@-X!vW;NVqkJF<)y4tlmJs%Bx0%j`4es3! z9!FY8O0jBe=K&V2>u>a8KAsK}K|ty4`q}vcfh@W}lKAMVQX0+%gKbp<6pPWsLEky} z`bSTnwBdYPJ-)acrsG@$9D~-HXQDGLAU+}tAq)yg!9Mr7&n6UMiaGR!w{s4k-yhxE zA3_I0fY(>+;E`7b108jPthLs?AyiWa#L+O<%~i<{-ZyU6v^-Ty}$GL&fRam z`OtTWiR4qCy0bXQJ(QCh1Gh-G<&pi^|2RZm*#ZFMsm>@zGqj%Uj9x#rZv1OwOvx zSci>N-;tpPbpU-{e@#-p9> z^)lE12)?N5(f<7Q?c2*m4V=pcLjX|S2c=Z-kaOp4A0p+FP3Plcn0@gppBoheK}}Av zY84cS!0_m!x0T=g=$AgfI~*>y=OUlnKfe3q^x4^B`*(ivAI}c&)Y@Ke>M#7ZpLqG@ z?|=Op-=IXyhD^v5L-*M)eeUr`7uQdpz$|_6@bZ(NxP5y1_RjRq^|Jox*^|07cTbKN zr5hf-{NaO-#>4FD^dX{{BuRF0Q5q+#mp3cj2iA593xF_5a4bOAbzA4VEazq4>i`>V z5y-tmQp%6pcL*VXkw?JC5FrR5pa^tXo*W%TfQ68RVgNuu7AIl0lT^*t#|Sz9>7V}T z$#nG9uYPG;*3;P-LFkNCO8bE3vjgvar#6WC)|u62xoI>fQef&`#0ahz7dKZ=){FC< z$;btZh|IF-ci&&6a*zU6mEJ|BJbv)-Y1LF*vddDfou`6Y5DW;l(~BDojhx9H-PKtF zrvDAe?IbGVmkG|>K&N#?AXSQ|Y9ZUeY zGabD4(*0o}@9qqK`17CqpMUoM@rQr!?=@CMVx~p*((RoWjtBSe4aWuC9f(idJ{-uz zC|#Cy+jni#Z?_d8EJo;>ek6{L%gW z&Y2G$ogLlTfAJG{_I782@cZ|lzxU$pANb+l-rN5lN&o%rYnIscojdO*KhHORPrm2o zs=AfCI!Ue25>kUDAz@??;5e2U2LU!?Jma(+mY2t156Na>Fi0Q^TLGagq?TH(?yhcC zS9Qf(>E@HqH$OS-I6w41vDf;nz4z;F-gxiBhmViWPA{a;Tl-f#1D<>N(|`1>Kl<>& z2O4SBq_?kcUD+M%O-5BVKiCfq_}rR%G_cijNaPc z(L#5nsPoLH&gJRRwQF0LM4eEtzwz~d^I!k>7iSj^1~;$XY?|)g$;qM?$xT5U;g?x9zMAcTCJ9w7oLA! zs7wIz6F>3i&mYfwTf6`KSO0I*EfK>EB274x23y1pWI_XKVb^taT{{jR4F|sCx(?f# zj`wyaThlRT3__^2ZR*Ziix3h*P=p!8ZQTf^eBTGgBFdTLT+GkARtJHz*=#z2PiD>R zyc0@C$!=Dw|K)%F-~MJ?Re#LX_~;rqo83WoZp4s!9`12~Vq8FZPg! z4UIAL7wgp^3S!sSjkJJNb(iLq$C&HrAflmziZmW=wuW7o8z8DyoXyib>+amX6?rW5`Sx@Ik%S1XH)UD1uH#IG z@s=An!uB{VOW`n&Yh{E~BHtQ$4k`zM{BQot|Lp5u{Ss`OVd(TcuJi|t?j+E zNT<{3>u=x)vK@~HVHjMyzHbn2YIF12-q3dff`Yy? zIT)_0O^n2qQ84i6wY{y#WwNP7BeXO|8sGN;l*1$fN+0ZvDf~7LFUmA?xnm(vz%s|F zni?2ecVeE^7nkWetuiSk1_)S$MO9}M0t5ksxN0TF)OXy@G=@nx#@e9OI*oj9c5%)q z21XmBJwoP}=l#)e(BIxaI2`o&i_gDM*LB{ZP#_1 zme_sUDp>8_ddBUqKM z{^swt=a=ujd+$@PeE#;0S2|G~-g-uAqOD58WWC9refHVa#q5)xdTBbI>}>5`9zUK= z`-G6K>4-5WihS-koJZ|aIKYu&~GzJ2q7f3gNRSL0oG~*tW6cj+N!9Q2M3473g7n;Z);g3z0qv8 znlIbSW&LA6`e%2xueOcmuIC4i(b_2Ox;|wNM5^EKC-Jn`pK9Pz;$an>9E(y;JLrAz z@R)j$gB_3hb=u~I9t;M}EDo-O9~?jWuJ8Xslj*Z3_ZF8&T~+0Id3JFL5I)@7(y~6> z-#&Ttp3menH+Mew`R_QqwfE+GuW#)hJ~=(FI@v0jcM8#HIls6#dg3#3EgJZp?m6LP z90eR-pG?My=QEVV-mTp$4C1QmS}73LZBrQTU(U~7e(4$PIiBZ*+=UR9WeG9T)?kc* zF^H0~LrQrZ^ei}|Xb^n@8MbCk8f4P1mx9JyJ(IX^k> zx@MhjdU5iJPkdr)y7!sSeD7quzcch1LP0W8T8xH)>l(-J*F{GVav05vbbDtPa%zk* ztr*9{<@u$m+skw{+hiX-zTefoM<=uY;6MHM?mzyx$g42u&&stXAgjgKzVgLk$Z(bO z3E=Ve!}B8x>x;`r-+be%G~oZnU;K~ChBX%c*}w2_?B94MUo4)vdXO&8R(G9`JeyXC%*F)t5dC4{(vD$lvaT#LEBUSpjKETp~etlMX-@t^*zQh z+#1IXpws7mXnN;!!ez3R+%nD}N5E$;fq}GETLqxth-o23Q;Reu8uY{D!J|h`6#0bq zgQ?GX(NzSQD*HG=_Fw$^&pvwmei*qv)r|UCn+Eap&NEN#9_)+89A3FK8cv0R?|<<2 zAhb8P20>!w=A%8QLDFC#AyR2q3)kr-%ylS*pu_0+qQWLr?x2<>MO{;rn^xC^DWP|h z*b9kvnMIsxA!Uawik$fLY*A=+nabTb+D#%24JO{I))1j<< zZc5w=&ZPA{H?vCC-(_7FI>4g8AqIZ zX(k*OAM6gM5pJvHblCgeS3dJgKlgvnHfJ|(KY#CNb?cc=x&AZ>wq>b~9v!t=21K)e zb*sqJizkntxw-=}diKuCO00dyEsHve`u%=#=cTL6v)}*2UjPIuZaM?=)rId-YZP~F zU9HE1pvzXs6bTg1?M<%@x441~(rJGfU?*L#0!kT&5}K;)2E88Ew&_}+G=$J-(pF-Y zH>Gs5v~;0G4z2RWDk&+_P%+=hkj|w#ovpMkED&8@=8JUD>kA1c+gH-%0ti(uHxSpu zy{%*zot&NDzxOatyT^~8q??tILgZEm(&p>=x;#5iYY8I)4#&anAT|h+Abj=BS6}_g ztDAIjWqUUn20-F8J%^D0rJwrCE`VFtZ$hkbHTSqjjo#_UdsnaFIC}1>&lH=JF!X(| z#{ueEu}blO_h0{4K@2vR>)-k9-&!8O(YEOyz5Uj=-n!2m=Rf*S{*z5XzV-GIchJo{ z&xp=*LSA2fYv2kOp^&?%Q?iu;Q8J7|DP<=Vgjj36Sk5a8A%xV4gwbs$SEUUPzvFNH z?SDuxZM39VIYu+iwKW#VZ`V8=$Dzjfln&xZ0P7D1^YtnUyiqSn%XO8ldwwX4_6Us# z8YkR=$^xmNDeEp6ZBK8%kmly=ufJWS8|#o(ip4S=kNi8=wmkqXXnn4%tbgdo|I}zS zGMyxtDPGKgs< z*2H6$7|-XF8b4$h8V#hNAb@;vc0sUvxhRiMvml6#5w%j7qA&?5Mu=d7A+`)yL=f8A znhZvR8;1wN*x5#47Vb~RS_&aV5Crq(T3K{_IX4zUjAq%!Le#8^b(+oBv%FXWit{on z%WO277-M$#_tt5ZR`QK^KN@e1F#%_1XPYK0MJHKcx==aAIlk+D>k>@o{Q#4w4Cg3P^ z`QiS461og3q&tj2w$9U4+BMB+G~%4^?(QN5sWGaqeHiGfc=^taenOvr=CBtzL^oKQ zt>MnwufLgB0uZK!R-#4Ne4BM}9LIHCDJ6uET1XH~%POzCyzYcmE_Z-7R+~;V5L(}N zDfc&J3jpF20{|d_Rcq?Dqr`5~MHqQim35%bs(h0#SDW&kcit&B#gmKq>E)uSyWK0- zypZ_5&9c>Mwa{8OU5Xu`L_;wE25#1Ex@asWAAWe(^T6{jJWV-TWQ~BHX=S_={d|=Q}?2spaXDN#qv!dEn!`SVhrf zy(tKzx1YH-9gctgpZ(loc1E$O%gy`mzIp8q?F~kqe)eKk1o5;H5dPqIJoD`>7jWE2 zl@)nWwFYp8VbUKIn>A!es5<81;U|CWZ~VQ#@w5N)fAiL>zkN9`+pZeTq(=;YXO9M_@iqOyu;DOntdb-LH{msO6PpeSp{aX3Y8eCs>E z|4$x2dFSA2?_d7%zv!&2YC$?12WoFCfzZ#ZLWx>vBrRAj>%^you)3AQ$g6Y(5cJ%w z$qS$T{AWM;bAR{07Pa2qz4py_-UEcWgbjUG8=wGikm)3_23jC6k`YIvAq3Q7Xkm+b z{qW>uR!CGfbeVOBJA2n=?66C=qSitkKE(&((%bX7?R;{UJEt9*&RCK*^HjqVh7T&jhRBj(vI?hO&<_fF!J};$aZ>2K}wUa+U@`;9@>o&mSLMQilev=eevZ*OQ(v420M#icNpe z^C62zzUj0vrenRkckkzUKHl2Gy87Pl`@t|y)_L*8-~BxyMBno@rmJj~1Vhl>viP$r>CcT zdwb<_<$732la+pvKl$|cyyWo7Y_Te;8fmQ&LI@#1@s)$gOV1r{?~E|Uo*TnfMUEFn ziNp@1!Sy?z?g03?U;KqyQ*BvV*QK#}Ss+R%#ospIwbl(X1^|z7Y(cikt<(gex-5iN zT6Pp83qaj!rHmK&LbK41Jc?!A>E`0~ul>`sn1A^ze*lfu5@7(7$eH${e&Ehli>j=R zP>i@kHw^s%06+-goGaN-3?X2Vzy06+mw)3Q{k@+dLOL#f|L%v$XlIo4U55hDa?Zak zti`Uw0AY}EP5{EjKxAA7L!zAXC#CLu)p>!#ajSBx*0m5?ZPMk%{Jg5FuImO#&u5NH ziSN3~C~IvHo52tTv4ViDTk>R~8tqVuT`bz7`gTaPEK6yuvDhJqVdGJNF`liKg=nQR z%1|p^4QNreggB>JyQrZJhG7zW;qY5;KN1QKhtn|V`@wLrT1wmHc^-zb1f-}1Q0j7V zf+*@_TeM{(n^wWHkOcWnZ9rwPz#)#GfB6=s?QqB^+gE~QcyRspU~iZD(QB{Yeec72 z8q%^AkB?SWV-BV}&)m6Xtr|~bV@#gsQ4~?v+q=47G<908nTvxs(hY9%dO5qK5Ps&9 zpTJ0tC%v=d<9f9`+#2pp$JC&2SC+p0Q^9pz2qBBvcq2CP=AJY9`OK^Si+eF5oMDC#lEP$* zMKAQBmc8L&TbXaZ_SQIxfU%62Ygb0!|AXH%*&iKT-FLWmda*8wnIGC~w}wJVA- z?M|!z=&$_cr=ELeaQ$lIP@BZOclpfI(~F0fm-F@Z&i;71Lnw8T(E=EP^yQ<+k^$Rj zA!JRmS{Ml7HdhISR+`IuZ+-JO{pYT4*PD&89jLJlI_R=CSIj30Y6Or}t*xPzkoP#K z+DVLyl_<)llz<_~eJ>2rD@X+Bsc`NY#V|_J1(7fHHrchhrZFZwQzBE?>k@I=eW}fR$bLJ&BI5h z)9s#+%^-32Cqw2~&PhNI+qOnVaqKD6bt?ChgR`?UrMux^GGEMMAUxzYc}F}e*B6{% zZ0d`59`Lw_Dc&1i@sq(~UG+!a(Zy-AS^ePmf44TEtxPXU2EAA`5>=}t*s7``7PQwJ z9-o|0*Y@KMx(ZpoB^Xnf7r|0HW`|tsMgpp-`nKt8;7IyDlZDXgD#ZrJxCItC)^6D#jO{f&zBdj8{T_x--+GOrM>+^v{>rmWVtZr@R+IJoo7*Z$zuc$fs-CH`pV>h7EG>WfWY)tyhy zF6U?8_sJ`ylag9R7>nb?I-Z1jwO9upSZC?A!~Ki%d2RyF^_Yb(-yENxeo*HdMr@ka zzV8!++qQOzS9Jm*Y$%0vP}x8^8M7wywv6 zeyxb(V9x^2o6i}DqIOJy^q*egvQ)3~5xUFmE zJIwI_NUg!?vfdvDqj6GKO=oN=*vV{DwEDT-@%8;NChT~2DuK$gh7bfm=U@=|K0vlw zw%TzZbgk;1y}tL+lcP>Iz=BPdX~Z#cghjp3t%@8{M5(8tFi>}mG17eS;Mn!OLE;89 zmifZ!Hm$PdvThU}B+g=)62_k_E&vN5MaO5Cw+?RP`DVV!;vn2)>#}S|<4EXkQRn;n z`)6lo@c>YQFjQIzhf~L4Cr76M8W2XCY*TMGZR-RAK#2XROIT-=SkLk+2YXs;tp$)G zrq2H1wbi;@uJc|J9-W>g{XT@idd!PLHk+G-2x%SYj;B#kR(V-rLXJ;QuU@@TZpyRc z<6F0GW!2mb!qsxq^P@rJT8b;JW!H7kx^TE$pK=Q8TxdmtAP5Ql@M6w2jpD!pq-y0l zG)As4B9*3_Z*KPZa=D1(J|;@(PS*xNR5x?AS{nl)Voj?tlUB>3taqow@y?d+ypK=b zNIc^Ee%wn`ONeZnbX8@%D;w93FV2@AJv=$wz4F#;Z-t3_^XaRL)0xBA-eht)ou=!} z<>u1o1Y`WhYu~){{GAWqeSdFjt7yw?lV@3BTR{BKb3)(m{fR&M{cnBiZMm7%b-V6P z$I~gmC`+r^`CXyR8?S$HXZs3A<`{%#*&wuIYXpu^538LIkq9L$|5`z##L@HxD)k#em z=9F+hLe!}`_3a4`0NA9}FaOKmlUTbh9}Ydwfeba1krVj@Ko$7(`2La0kmvHP$#gnQ z4tIuH!K#!}Nu=pX&bxxeGs!!zca+xep3EUnI!jB9E>k(*G^4@3 zG(=k53U+d_T<29@*UL1og@n*9L(kyuE)10 z+tv^v#B{PFgxQ+zDd-*^ULW=1Ds4iS>8|!*A_XFv=0yRlU`q8kXLai-t?GPh(ChhL z;yH2XccLa9NvQ7!F2uH~3j`FQsP8yK&qo?oWmmP(;bB>{001ZCLF58x%dEUSzf99S zuhXok9XFuJ5`^p44k!Btn`F?}geXO_tOz2%6Rj2!0&)A+p2Lw4U9T4v<$5~yM4P?y z=4&s!@QEmjA+n$R^eZpC_=@8@5|~DGY#-c0n8xGnKl!iz>85QD z_MXlb&E{frmZz*gR*r-Ha9xT&`LF#O2HUDBLqD)u6-7~X0wB1*vx}7J%Je7y!cYF( zuYCnU3?PI?$*OABX6#i7R~g*vO}SnX&B*HTUj;Qbn|pwwoQ?t z*>SNyy*`Rio~M`w#+bZPiuX2)94e!=hT0VAEO1n>*MktJc_lSri~uOJvapE8y{S@a zK0ojE{g}zCSNHFJ{BT{a7$K^z%WBoOHH4%ED31F5-WUK42+XToW6WJYilWd95oEp> zyPN{85Y&*sx~@adw=$=|>`ca1cgQG)(O@`sJePa^iaBSq^&;v;z44%Ia>wIUqokBV z2neCaIkeUdeU3cQsLAe@=EcYOLg$D<)euqij3sr}vX>h0lp zi}Z$en8X}Hj}XQbv_guO5;Pk0+qT|Rn?7a?0L-xGxr89@vdHuM+&kQ!?oPKXWRvkO zC5}?2RvqzK)Q=J5p4+SQ4nT<@dF$=FUDxjH?A^KZ$u#e__mdzF5a(GTmgnod-fT7- z*YPN^5LK>Yrdz|NEf7|&YlZ_~OMw{ew6e$|ieLT4H>6fdDU7kh`R;fcMJxbFhyPs?+?|Qwe2csdD(oKJ%;w5>re7L=-S15o5J7b=I^+C9J00@wf-D zSJWBMZ4~x}D!K+83@8AwGmu~mwDCPBt4btwqd_lnVvlKqD65<>`PK&)4&x9)iTHSH zpVKToPmhl-+)23OQb=egq*S1&y8d7|>f_6^<+_zUYCVtC(q1{()@3uC^zJ`8Ugrv1 zhXCURyl#!ql}~BV3xsHkO{;L+D77;Qzx&R#H0tQ#NiX)X6Q9jdyV48P+OFMs^;eI8FkR3ATi^yyE3`pMCU z+)WgTjvpOus)9N($6z^Mg+8-D1B9;a?k~coNq!F94 z&8lvd35OA(;b7QPZKDlu8Y`t~+X?^>1Ysjus72_zjN>@)K`Vi5(;^#>$EeOHgMQtW zJ6jWItTJsF1O&6DDjXgJ%B;$(Qh7dq z@11)=94n=E2K_>;S84a)!$+U~%=1udp_LQ}08o@A_x-99!sL}#d?0x9}N-4^K3mC@1&b%(D%0| zi2%|3?0B?09!~a;?mcL$Y-g0r(|UP%KHAnm8X995C6b*IR)(M!Ioo0u8KB? zPd|J0z4t%UAC$-K5-4eZ7Wy0I>>~vbGN@xZ7Iu)(xR@riA z$|>x8>KX~HlB4MW$Rb^)aWJT5DTHLq!vG5doj6Pa%!v#*prC`cN_u=WiMF|!Z8 z)3quq$}Fv#R!C#5G^$fdwNf?^>X5@x%tHomUh6;e^Z|37w?BUHcu?XkR|-o5kui&de+-p=KNx9`0C z$?0@F=nW&D?2Qh`z2NNV0|2FI@_+Z={&#-=8}F~OoN_{B*Dy+;HKq~57>{v`tx$j= zy1jLfrmZo`Xc7iI&pXCQ;6SJVhDIm=k#3p_7;BV@qOdAThydeQb}i)uLBuJn>OAC( z_M)<>pMCoIkDeT5szs&<0~1M`r^Qw;$@7`%l#nc&<9_7Fo^yJ6fiaF<_v*p^V&i!j zZqhVwh37jOLkM&{NLsaq5EfNeDVa;<(ROcxZmlqhh9LV8dob%>rzF}^d zBvDi?EC5ZB83BbCF`r*9SED4vpy~O(=r)7V=y={GaZIc!iw;ppIIUD&r1c=_EoUp} zc&e?EAdrQ?Mi2-pWD8NNj`|eGA)!PLeM$*8+B;91tkk--wXVgw@Htn=K3Q9rIu3If zah9E~fUv41+`7b zt?$xqHVYwF%0P%D0+3rOY~VPXyovj)DC^ywEsB!U^@4NW*7bNeZ0joShZpPotvBzw zP9QCt&vS&eu)ytWVk^4cy~zjfe+xkiq24<d|NGzl{O3P=@BaJG z-MRD6e(vW6J^!cv+F!YH{iXlMum0-ZwHxzG$m zBFm~cAOrx43FYj{_SXIP-sb(JKY|py2%@YkA}_2f=k$2f>rbn46G#1~Dk1VIHIu~Y zN4&|hPFRCA;Xb6{{PGkNssWS)B|+>sbTgazBm`E^*DK0`S_oqdrj9a583e5ovgw4@ z#_4M71mWey=9Q;!KRG>%C)@Aee;9ZEcYp3Pm3Xr;Dgd(Y_#Zr2YS|oY?QWVmhlm<* z|LD^1ZyAfMlE$LhJQuZsAS) zTi10l#^ccrR;Da7h-4VK5ccw}3BmzTTB-&@D<$e=%Q=7UxjV%=w?LFtHX0;tows%2 z_(>yVFAO$W7Gef4lEV64TolFoM<-P&G4>WK27x-*kH#TWnAN7ip7R~gy;9`sKlsMI zzVDrU^SyuNhrhS2T8c>E61>$*yH*ygZCkM^JcpQ4eqwuXuE4xb*LfQBG@{CNNG}Y9 zl3800L;B$Hqw%Cidr6&_KJ&YBBPHDARoD7k!-26Ph*%QrpPxRdiWVWBm&M!fUfzE3 zvyVR9Xp1n@qkiz@$zwu!FAkHQe|~Wsi#|Z4C^x{;IG}Ay(sk*`l7{Y`mp=Wa-}=%f zUqeKqh`e+6twB6_<9jg0d|K0R^h1E+bS!Yay~N0YU=;fa5S_g%^6B8$En*^3iC(-fA|NR{Gk>_Iu}Z+!|ffVE5Gki$Qa|uwikuao$~y!@7Y?T+#Q4vV;T() zy!XELE;Mver~O_paXf%Z>?4D-NeFQ!Lq;`H%dD21j<$Nk!QRKmtFy-!*yR?2GMCey z(`@wRszEF{+K9dz1hUXw2iq)iA_~d66x=~x#2n_SDxJ?#nM8wO810RWs)$d{mKT$V z3vIsir7uxTUjFVM{rO+|g*%g`Q+W;WAyJCbapEtQr6kT5!rw&W;d5)B#Us%bPa`-7@n> z@b-<#Chr!tI6t+mzPL=qTc=09=e=e*3-A-bK=&89A!8ey-MwYB=*haV9XP-O1B zblobASQ{%|WqH?iF%1N=y)YuGyS!L!igG*}=GD3v239MFkeVsgRSrQ2S*UGk$~!lo zd+^bT=rr?!t5*YA1BA%Q`B7Jy>5dmnb{J@@stRxIR^2jm$nH3dTy}MOLz%+$JfnHr zL5xwKyDdTnR0>OourggQ2qJB+2BWo%N(1JVcD!KKEGTyKmZDUThL~BMT`t`yKpF%d zr7S^<*7fS?WScr+CzbKIAH}_J@aXh-d+Y~}e`Rm?sp~Jl{`w!j_wI-5bm{pn?wV(w zdhT>_as2qazh3r}(dp@F%v@wNl!79mDA*mvAAjp>aReDQ9k5-M-MRURx-A~wd!HIi zC^$NQ^3+p@AAR`#R^k`ctf>dcG|LZXS=MarOdmb@5F+eh185Mq>d*Y>Pki;&2l;Xd zpkRZ&XKznPWtz4$m^cP03J|6oBGw7js)`!qVjOYS>Xs00@+|P;D0K3wZW~j!-A>PQ zDYZtcu2$M4VPtehtub1AuG4mvwuuz2wKPZ^4D9>I%Y%6i(i6=?##mFA7t2h@dTW>n zD(XtDO9P2M+!<$!RhFls6%NK#mOYkTX9(s_#JQEE=Xuz)EFc!qv&E`|SSjQQ9E& z3muo0RpU~t0NPY_7={?A<#H*pKU?I_KljY>`)@pVxRqTV?eFir{owe)d^1g4Vnim3 zYx{@B!o^~>%u67(Po2(m77}SBN3h>dgsDS@mBK*6GRtcTbA>t~8FQpkRzYi#l-h;R zLAYp|Wvdl2EC}vDcqj@*-6*-*&&o3k$@MFa>|i@lx2_M~djIj4-hGIXi9MEo?Y$)O zWmT+Qr|^X=@Jlt4a5XchR!ceqIgSn<{PiR#u)x%U-Jbv+miL|3mjbH>Nh;jnJ92Oqy*wlWC&gnFk>j*;7W!PeeW2Qp1A&fmFp{nl#P zBH~r~qTe4y;m{(fTa69#+^8)TAp}_j5u}u1YNa5)gWI+qBw?GC7?a5Jtx`-%*9{5_ zQQyzf8ZfUA4b1JJPhAcW#I0;Oa4s+BWvP&3s&YLFy)YWBDtNgP47R&Fqqg1@ty0RQ zT@DSpdTr0H&w@;7Cd#Lr~;xwR~e1|~a^QhTnySgqNpw?FOO zKA2Wuytup{IAUIP|N6iAm;UzO{vXgLt(&qnLTC$!MXivgle%j;fo@0uf<@I1Jr1SX zjeRAvx-J{3A(HdWCQG}jsGxxa@viI2yr2*PV@1<4N*T2llU|&dt`^b`cAx#kE1!O9 zCn4N#+xFY|Lf+`Cl=F4>=DUy6tm&}R3nD41rmJ_R`~BE?@!8#htCQGOZPhic)QuCk zjAPd$qj7|Qc8SwyhzX<&@9kby0CGQAuA97;>s2m*69x&zTnLp{tpZ3Gpt`oL^Hp6m zm}~A9!q!?tZ2F;Mx}FY)>vd@{`B#79&+Xl~esFkm?_huL%AV&rp5t48R5pNnQCco+ z{v<&a^W3wO^N3Rh@pNlM18;YFb!$2R7&w9Nd5n7mSxFI=M!UY}`>vH@7)8v=fk!+J z2EBd~Mg#zcFh_P2Fwda~A;M_rdl&*km^9dkIzYv9Pu&^~da>gt)0-vkKmYOe*WY<~S%V7=Gsw=%Mq&;eR!E~1HkO`UtkSAmNIc4MUo^(PcB>OLJ|Q07^Ca?+3jmrJlEOT*};f^+aUA;7ZJ!D>M$C4u8VOT zx&0{I9>@J@C}ER~+=y!*HeR!OnmM1E4qo0FZ6}Un(b>teD5@Y%Fn1UY7~}mx6#DUc zvDUH!K-5)*-CiZ#EZZQ+T|Z`yr;Q?RJe$psfkxxWcqa3MVvXv zs?g=3OATzf$CXBpX62&9jrJKEmZi7ZxP(Fr(Q;n2jp{o6?WkH51Pl{ODIvrd6Gh>C zxe5X=aw)=&Mx!76f$#g7zxChe!U6ys$Dx!8Wy(g+7in*Br7SvaQJdGk?>ACdguhP@5O2<2c+2D5bvdW?A8dL0MM>yHXg4cxT-*@0PPA zBed3#I6kn<^BrS!(X^{te0+4;8jyE+SuaMhed=((>qJ(Gpx^)fSHDtQv&pNrZu0f= z{=IvRu&;d24-@8(Mw9va{A`nNg!*_{g<;b7y}i*OFV-jN+BVSj9Y2Y-uI%*VAc=#3 z(IkuLXfey3_{#aJIa|(qxW2jDyMC~>J>BY$w~m+T8xM}& z_~_Bma&vl_oh>s3ovIY4r}O!2(`i7NpXL>%ZdtU};Hu2JrX<)VL8R-J0Q~Ub!$*%E z1wkN%`ShngR~BtrG}5A@vy0U_``MrSXUQ<86fe)(c2gFYOOMx_v%I|QY~_Z?^*zsF zZP!2plQ3e8L{SokF{Si;F|!x}f(OZ<7x#zpz;`??Rp4OP$Y~t(6F=-ZSN6s)-Z~h? zp-WXK*Vo71&AsXV_D&RpWm%R@eSCZ}nM}sxA;FIC#iCQ|%}h(>2ElvpKL!T-_lrYkj7TD68~0 ze)E?_)&2QD`x8I=fBjlj6~jUAqx;85!m@a%+Zd5YnB+V6@}(`e76$ zot6zynA1|X(wZX65XTVKQP>*}gWI>SY^r8i)f>~*Z8a~JWt$;vdi^L$0;NG+mp5+R z)L1upS3UXGKm1$&qgL&t@AX^{Hnrzd%yAUOr#0BDmdqzZT(rWj)_S_Tce+_Up3STZ zV&7MS{_Jo3(YwdTF?Bp_RV%6IRHY)+Hv~k4PvS^cb!Y11v*z&(Nl2oAJD#?>#GE09 zi#A6H8I*H^@1JijJ4_fas@(FXN>8VQ`0Dmve>l#BT{m(#?tS4qzl%1VC+Ip`=j#=! z$COvP4O)p}-U?zN<(FBy&UK!e!T#1}U6tv2ub*&^g*2&w zEs&$Wd%d4bMj#ahf#wJAhhbVZ1K_MY9XvfzK7-EBm}5z zySlCxZW!Gt9xIBBIbKsPP%eedx4VUN4N`CDIp`1mC4jKVN-$Z~UL ztre9V^x|abl-cFK_NV{UXlpB(9IW!vi=6Rp|7bY})HTYiPZxjm%|DnNOylX`;PwmU z2JUa&io(&v*MsewWnFl7>2UPiog3P;rg0>+@N+L+8TOn_UA3L`C}zOv_xswGO2Q7A zs@6?oeIB$`)$hfgC!jQC3xrmIL$NjI^Mz2Z6UyiI$E{RC$j`oXY2`qi;`aS}+!+O4tPZp0T2z)9#~^2i%pqG)4&MNQE|89tB~Ksl=rc zLJbKvAkEfslAK*$G{EhI&{}&=&~?pZG}Ka^uhsxD355kj2-N4$0rha~QOc&HEokj* zetPTXQ|F8FV!j>@dTq4~BY!qu*9Br6hXJ22XAX@vjVMe@p)bp}GkCU1L)Y$35{HIW zQCt~(8!2yki6gBE95)C&B|4X3=3q(?Mj8kQLaUTW1`$RktGkasIv);)f$vLeu51l@ zJ~UQB>HH?M04q0-s6Rt;Z<5T0ut(QjD^s7v09b6!$2H`O<8wchk%#M*0l1arw;~E zJ?u>v7bOqvm4m&ne(9Tl;)rp_4No_#rWMz29%`YVJUZT*j;gM*h-nCSwkL1A^SLOc}2I19eJsoekx)!EpUTBPp9k1&;4Q$(W z^QH+X2gVwU>aMM1M-9~w=ZzQyjC1F~{bS}PJAJoPtL@!U5+qHnnG2gnc9jj|ATQG4 zc*r<<_wGk-FyIsd*$~XS2DVxo+g&b;FzUB?rJZ-A%$o3d>Jhhpj;>|VY5!ADtMq-lBQ_H~yU z)8M&~o9y@tpZjCqeE)IYXk9KLLuZ#6bEt!9=sH)fTnTu0{n{R+q^vstX_YQ1rGx^| zfvm_K*PC3~J3X4Y9A8{K z(wbc^=FstaNuoNDRgDHjT2!_IOVQcQoo8OOz*uLLH+FwJST9bp)k7t49LLe1HyKQW z@$}6%-;_nQ+SJRuWe%lMmesnqbz_yz_eNf^%n?UrXTR`6Km3QUz7-PfdEWW#LbhEL z^f44a`eQ%zoB&>O1fEK@KUTJY21+TDQR1 zS6==EI@sI3^UN)Wf`f4!aO@EPLF+Iy=(&KStSt-$E=Pn~$HjGJw)*{1KX51*M1B~# z`D*_3%~8bT(?yYI^?UEU{ou(7b6lYGe3qV_tmhX^%T+d;fBm0-aTE`H>Z(!_hbXHc zfy)j*UQ`#A!3_FwPaE(4ll-HP*XI``&*5+X=2r=xeEj%{WZwVuzy5vKcaTNpVjYC; zrpWJn;>Ajol~fwiz-Okb29Z~mjYDE=lcs5kyt39h4&#i9wrty`X&MNjs@v%x{^YY) zKmYwdKHMH&eR>ckle_O zO`%-hUo2Lx<2OwMWbG2=ax6tlFi=RHUM^N?rG#XJI*fH?T?++xFyBfhOkskQGsZxThq3!_#_;K)>0?r3oG@!QvTx0n}w^WBH9z5B7| zBpOTG}Kuat9H|3VZb6?jYox=iIjjw?;z@pzB}+?JyaC^R+J}F$b7!@(f~BR!!G+uIp;8IRFmz zFpIzP)o=KrhdJ20ekDwzHZKpJdn%mv4Y05j)Bf+Nf)luJf8IHV&TGo%^X}>?F5)H;9wrwCcLYr{`%= z7Ml$R?#221e7^j)_Fy55lX1G)aNNYvus5K?;b^wreEBP1^c}leKWfT_4-vACgK3>D z|F2*8XHqK+@M?9!7`gx8(ZT-Blk+Mb-u%+5Z~3w&%yr_hvsOabY2f+sdvCuF|NNhP zMRuuZh3D~6Kgbq!)N?6zx>^`y?p=07Ne}QR01Daw0RR9=L_t&rD&la2&6S>42_Te& zGRKT}0*wNCy3SXHEJ|UKj$F9CJs?6Xs**FWX}W{m5kz#gDLlrZu@tkc&J_$+%VlE? zz>H`rtkA{^A&R;p79z)iN$bg&`G0=ZDn27E60$$TtMhbBO^apu5)d0)0RK>$G^it-6vjo;o|I=TX1}_m|f$dCT$pZL;m|DM%yQ{*?VTnBaLM}AC6 z6naHb40=gFc8a20F48M|Q|u5RC1Xw;Mik>bTQzm3rR}1VGpomDbu?%OIasi;K&(!-xdkmF)u$ zX)kiExf&#i;zv$GqASF5U0(nLudM{X)axz5|Rc3p>3dbuf~ zp}y~1D*J=cX0hBZ{oU-}RJ zs+91Z`s%nhP1MZW>ymVYHly;PAS~Z>Y9?zD0+x_L`CX77` zOsh<#mFq$mTZ~E5)WVWR%B$OlWs{4#TI5yQiCPE^sj+xfch$*icd~b7cdKa{j_OCJ zbE$zJBtVNKiX28RGHsFSjfSu+0Q^7$zcs+jk^tMR5j;pngNRtzm=3ortpp$p5!Yf> zo7%Q2@B$>ntZwqM$QvE@`dud&V-^CT84H3&>P|@^H8hN~H0t2auTI+I2#0K?wirj((dAfU8Ct%TZG!YG7g*x=pSxH@U_b8c-Bf9QQKOjR(Wa)j6>) zgf<8QQ5Fpc*}4pap6j|&%v@%>R(qc3d7kXL`uqs7aGq|QL6|qIU_e6>4k-h$aFMur z`?`$T*~5>%_}f2!ZF^5jGo19xR&FYZy5e9kDiCNJ6ZG)A@7?PSMzeK!{i#^C&0sKD z&F6V>;ZWu>4xw=fZlI|0#mIF$pp-G&JH57pMW&5tb`rOd;ElIG-kFRae*Eyt;ht<1 zP#P;ymmBVQ-0{ABHH9%=tuAifzCE3mN~y9DtIM*%tsjM96s;Gh6uG;5W7`R1(JEaz zA&L6~M&0+`xwn6?Ln(F$)>0DH`4rktd&B}QukBCUS~Q)hi#!>O`5iD@aaYutIZ`4HFqPAQ(vmf zi;`gW!FwNUUkQS+*LDyB+G;Qu41WAi{J@vqIxg#?H#o@3<$I?m2fIC&M~g*Wwwu^x z*hu&TpS=SKR<^t{>Ce;k&8^XLQ&3DD>R^se&o`aKwbhg%MzDv_{?NO~>nv*$*9TO! zg{V4FD+MevmX%GuHJ-?}$QuB)@*Nuc-h7j3izuikZ=w?>R{j5@4FDvyAzklXV&z8;5~tYk)x<4HQBglBR32 zG9yUIx~wW$1A>t*vT}Rp%1hTLA!;9=uScViF(w{vzxLWUkIpX;FIHjnw*1E0>05GBxR&yR}TRI3T zVF01Vc9inzFn)5L62N26xL7g=cdd3f5s1{*V6V*2Kk@Q&AKgFBDnqcnzP}qri~z21#}M=# zoWz5&7M;)lfXQTXxj2@Fdaf&^1}J*(jk_;DyVswDt`}7K!T^+)W$5y%Z1b#Up`%)* z5XOkNZ4Cg(%LQU#FHAP;3)kZa0R}k@!^^V^H;l9>R4WZYNLmeHGMKDZE7x_;=cm-| zD`*22&Q}+H>^z#M9WWlFZJiH;p3-(a4*uF-|GWR6pZbr>Y)J@u>ekH%%myj-QV zFa$Wg#8aeW8az6h4}v(D@SEGyPkr`Z`CC8z3whC8Jv^w3%hmj1dy*`cW!FK*1p%(- zdGJqs*K^!r>{w~mi;d$`t?4osJ%{II#r?nyc+AatR#mMv zm7u<6*sEIIG>y+)NT|{^g`5#lv?TAKM+mZNcfWUXdI>Q@hPw_X*gyzF9Aq-zbW#Bo zFy09ajj77CRiGET9qQ^)5&VxYSTs> z>BdM5C^JSwhE3MkDCkjD?QM-$$)=p>F&pVuim~LegE_OfAEdBO4-j%f3?X@&dxVY zd*|kr5Y>G*=(_>qh+-p})?qvq^4S~LiscNpnT5u6UE;=w@vnXLofls`Y%+~OHQL!K zDp{4PXtGfp0VrG&47OdUb-i9)yLPKnudYh{iuPCV*eJJ`d1aPr`zxIYo=%=qnJ{qY_{`0{t9QQZ`@i?Azx69kn_AHI-KeTojfIJy zRC%+hVB5)FLIxB?aZt$SVp&4K*V05x^nGWOS)r}b!eWFC><>aMvsTf(kO+V{aR!_p z&)0@fh5+}Fi!3%I>3LnN=Xq61t5vHtbqIBsUQ_@oNU8KZuWeiI!R_H-wVWUH`0u1X6L!URItN`Wvn zS~Cc_!|Jy2eK*fj1UaQtYwftaEOX8Rf{nI@dv2AM03wWeB?@anj|CkTU8`Kz_Xy7N z!f^rvh3tgZI*t<8p}jaH1~j5zE+G~NRC5X{VFX~(A}HnIRusBwk+vZ-s#S}^5Q4qX zK6sK5H|~sR1&An{Cc(NX2E)*35piS{!T?Jk4)^wtj@FEbYd7~!FN^n|+^b;&DS#z) zJh!5221%WkTf9dVZjEZU@%wL{$27ThbGKVo4=&nQp6ORrb1~lp5vGzTC;!yr$f| z&+~#H$g}d`+N~##&wljJ{P>rCTeB_;s5bVzvB&^>-#&q*TxrTDeUUfcEv^qzDthJ%>KmWtO_O&Mu9^GG5n`?X5mlwwd7@@i-^o;?sQ;?9z;nej>aG6U{2~OzFWXgzjAk77uFBZ8j%eTKW9)>^gz2E)mFZ`MJ?*3Umni3^$Jbm-G zf9KWN3ZBm{0@p{*Fxol%J3sT&NHhWJ$kwo`TB!Bz)vHCBwkGJ5yb$@8k79z3A76|U z>@qkR_v^MI7Uru8>Du%AqUy9&tuV@(dQ)QV5EpOer%;z~eeL)BfI5z|S?8`p9M><3 zLTI2YK-6_PRh+qjU)6aOC7Yt47(*lWr-^L+w;wzzo4kdC#o5V?!+oh5*PS598571R z08GGqt3g0qp>jktW-cegrmj9XpXKfP?#BQ) z=_>EVy|Ss>vN=6F+8!lDvSbpZMfp3w@hea72Q1n85N?_pV<5%`d-S3UPFD5qX2eb+t7fg@G?DM0sZ*#(+@K z=BnECII==X0U?BWkv3i7GsLix8dm~@o=*shLNE5H29i=LTWz$UoOK3t)+&TsjiVrd z$Px^dQeD>tL2$5_+_*82D(947=-q#~IJ-=xMRn5&0ISxXpKt26bGSz_qkwGk?lLbv zzW3zr-FxqUaKB7Tgs@b`sMg^GV?Z1Wk#MLf%Tii{T}}w&gmOa4av227ND%}CBiL4A zIP61cDaBDgY-MecjsxF!oxpe7w#Gn*91^4nwH9>1SYUPFbAT+z;Of?p*oHu5wWin- z%!JfM*GY@dFR~9$HeY=6$ye`h7KK?XODRlMHECYYF6&ioC5gk)!6F0Sc=OS+1Bg1E zf&e=JVr4+vwm@6vCg=02?qu0^h|}3(1u3HxN!fLsXf|2Id@K1T@8)FzeGiL4wPM*^ zeSDnXeY_s-T=Ou97><0Ft?L_y2m8|`^3iC>Jg%fvjgbTg=}Dt%cv@D5mEZWr7yt5q z@KZ&xaXDrj74wr9Z|)rq*ymn)=DC|YRd#&z_yO`&M(d5#_m@Qm{IgZ}_R(yET8~Svp^5Ril@ytf)$Fv|C9-2yti{1PCL~=k)yesLreT zVs&)#;N0)KdPa_n)Il?C_xye2LUe*Dk;q-*dfj`KpDEjJPvSe*aG|L1@5 zH~+!U6xq^B6mpw{QC8GSQ^ctN1Y(9U1_T2Dsw{_p>U(b#RSIx$5K!YeisC$Ldz?v$ zv91b5In-_p??0(Vy(nOECCoJRtF%1W**?3Nl`SETOI)i|4qK}+u9c%rI|=y7d>wjj zFT!afy9P;(5Gbk7L)S$JYprERLPu%r)>Ya_9yqSc4YojnMk?%hs;Zrf#fBj#8n{F> zxst?q6hj`nGOsrCWzmTe5X)f%xz?6*mqD2~oh8V&D?f^>s(~mZ0O?M|<7l?o074wg zFi@U@m=9&6Id!NHW|tYKfszI1TnGWE4ML9+KVZRfvjzxOMeVyR^u4^yAm?@2M1G$; z_~i81LgHb>Aq*2|m3KwcjQZ2E=?p|^Sx`WbMJ@-85Efx$422e3164g33^Rj86^3zJi{ZfO(N^LxWGpLZVKn*R!NVK3u77a% z!-Jiz@BX1ry?*r3qi@{1Hr;K2zSziV&krd+IbT4-Jcl)lvz@)kXdJH#|Go3Oql4|U zyXVC7eH2vLy0yee5p#6q`eY&E7>8haSGBeFoEM(myZgY)dr%`|WyRUx0M%}@5=kev^NxXOa#b=N1-Mx3O1iZ#t86G@-e6+W-rQ6F-efEbx z{^t9&N|Vvvd6Df-dOz}+TVMLtyfpUR*S-u;Kk~XhZqZ? z5yr3=4+n90oMu+r9^nz8(5j-#a*LuU;=Wfk%yr#OqW}$dlEh<_F}7}U?E7xw`61y1 zGe8J}u7e!bdW0d3cL!160`5edx^Coo4!L!B=u&EpD)UMSnihFli($kPFJM?`tu--~ z?69F>;4**$Qv_~cfo_UgL1J1WMC(WXy?akiSNTdn2586Q9VYc=qNlG>z@DK}Z_~hH=;@V9*cF z7mtG^>^7^d#C4gDnAx80q(#{eVxg)cTgS1F3AB_HwlQ9Xrs~ID5;EDP zP20Jy+muCJwGi65DPqL_4g|Jok2@~`~A|NCB$jHA#aEd(IV=0_Ko zJYdS!+uKeU(n5h=%y=)N!LY0=V)5BUF&GW!mk-{3?;Fihc8-@xa_#EVzwqR14HaAn^txfjEW_@=B z(X3<2==a}#;yQ_q1|?t`hS^4(&6+d{z++8`mwKW+>zJrOWIzT$zb#<;A>L%Ji1aKI- z6hQ}(X@4{xj2L50sStrLefsG;FTG#^9`pi>+^tarsNLGzyRr5B^1SYrXm)z})OPYhm-EfZatTqgEWtYuFB<4*8p+nW3>8HtjfRFqv8PPY zDO;6I*L4EGs#A5RyEZ?&IMz^Gi~!b@qJVmi!z_h}f*=ahBA2x&(ykRiH0|MNG$h0r z+H{g~OengTT^3EJ5nv3fssj)jU^p;9gAPJXbQt=}w0eA*ud=)r^2zBL;{+JXG1;2* zt%XHhU~8e!+#+fvAqXRZfZ5#}IidA@+#f~De8GLf1Bx7wL|&OMlaQ60%l++9x>;`{m9e%nXfoIi z85~63)~HuErLj5;eLsj=*)oT#w(9lc{&1im^c^;g{V4E^QCf725eu}G9YDOO?R?$M z)B1dspD&BE%f&O#JQw@k>3lug6zj6hs;+CbX|5AM6Z zgBi7y9-Tev#lb)LyZ>9{6Yk)=YlN`?IdMGj9UA#gS(caUayZ={4SGUWLN;C3VGjW) z33hHg^@=8<(>3UtV!e3tjo1FOzxtOtZPUtJJG{1B%v|nS13T4Dr{lxD?bUq!+|@e= z$wZaS?zP_Zw!8gw812(vef1Zf{xpxT3V5?F#)~W|W&&jZFEqTIEz_!b@BTwKj)k!h zc!)-qn=)_3CaV{#EU&swm)f)*qk-oW47#>1^J0EEYwN}sbar+jMCWi{2+_9nbI(8B zb|r)mgN8fw!SQ^$x8o1SnkL7WmtGVgqUY=E{rgXjF4y09Hwp(YII#G zLS2u$H+By>@5Q~HYPCH6_^6!KiPP(Qlk}o#E-qbbyHHqMr{1hR*>(`&v z2Efn1bj@N?w*q4dt)>_mE1AO+7w_*5s>;-D3$20<-V}LT)1gm&VxbdcAI0}24P3sWFk&GO_?PLj%1x_bhge3;H>@0+^A16vF zYjvwM)aL9mD?8n2NNooIQ3fy$2*D6pY>m$VBfwavX*IwG0nH)Bpp`77*g#p7jVwen z9gYw}N-4*2AVBLP>xc1XnR%fHR52M2PiB|bZrtjc=J?UEGMrGoyK}Hy=Ya=gCw8V= zXYx4`irO--Y?_7;;&N}= zbFS}3A}?9s3*8RFNm=HOE0u&&YCn!NpiPI))(c;&4n|BYF{2iMtW#QOg5mZgGR7#S zgn<;A+t&_Cu@pj$d((G5KJz^9;_NaE!*6`>!R@E_DN!)3zx%V#ee~dxBD$KVo{RSn zZ*|T3^dxs&43ti*v;X41_+Nhh|NO_JB>bJ%{s;ynIdnGFe7Mau0;cT(kG4{33`9*{ zy!K;X{*B*iQ$FHOFX{mxMp0r^)vP!1U~CDnZR@%WNjV($>$>iZdl;z1O|)qsph~xS zp8LK}NbXKQ@=}0Mu;({nqPmWb4i0-j#bt zk3Ba~N_&CNqNpeeLK;K0rg|`(Y|>>f8DE@a_fIdLy|wi{pZdhg_I~MCzPd^u_FY0S z_rhd$ap}2SbP!;Gfeks0`u@Z73<7rL-~d^R7P%-Z3lz5qVTh=OLqFo2`+?KdRhp)C zT}uJ-wL&IxJYOjlh5@|w7aFdaG&W9T>z=RB=a&x6u8e0XmB^VMy7Z>F?-S_OY7witkpwm9K4gwsbt}eF6!>%qoC)7gk zOm|dQ-Mn^fy;+`KUi5|o>N@>6IlegKfwRuDoxSVpO&11!l`YfFqVFa9qy6fh?G(YLTfFh z6k}|yedhK-;G)qWa5={aFhaQ^l(ueI3Js|fK;opEvdGJ-Dv9R<>Xn_&^O7+ZhM_Sa zOABm)ZAFo0gnCM8U}emmkWg$9K#+1D0_=M(!AMFuN&<}F*=+gb@vJW4#cA&NiIT?m z`3QTjT-ot!zYr~PaFreJPrb>=8zi2~+=urcoE{zbNAa`IU*9{NM9Ji5{?Gp_E!U70 zpZnZ*Oa|AE-YBNt4nZvJE7t*z!(B0=+5wjS-W$I-xMp7Yo@Y-Q_SQ-M>b=?5KAwHN zs@K>t7Om4#7z;4A0Ht}+8MZE)MhdO<`T1Gbb(9hStk%kCU=27sJ8SB$uG-gLd+qG( zL>n1Jk+kkNUVDRK1jmu`lfLaltyMZikcbwc^RKaH88sK}HtsJ63~LzJ?Tr9^;&Ap)sb7XdDYRs&U-Gm>8iAClYDDRMol?*o%Rz4P=HmM~{xm zMi@c}qC!c+8L?1GX{|*BD_~trAyNb(f-u20j*yQuL?-I*%+_;afCWe>U39YN^|!a! z{JaRqw8@vVi+p>0kY(9ucT^Us%h@Ka4i2wuF3(>1%rmcj^X>V(_50Js^8DIvf0N5N z8R)j=0c-M7G=MWxcTG5)Km`yGdDA9AsFjd)>x52IlzV%7Z+!4U+~0{k@8a_K+EdRw z_~_%%>k-P+BHtQ>m+Lw&YRU=0tZAC2GqLYhRYj2-CqbE~<4Ih!bzMoI2}U&UioKnk z$45t)5(*KcTq)Ca9c4&}_NAM=+p^?p% z81L?0IX^$u9XMSVDCUYu>pckW%h zeSGrD=WaOznUyEZw*jVQCLsy5X&lxYO`g5~+Ht!8tprV1mv!p^Z+CC%7{yICh=agI zO(~@6JePWbgP`pvzQ?#RLP=D%ZCNg_>>OB)5Hnx-qkAttw-?8wy{oso2EF>VuRi(1@dHnu!FTJ!_EF8zlm+O8K0;8o?Wm9U+AD^dOrHt8~$t2#s z`m4YEr7QbGtxQ=r`_qZG_Thua^VOVlKk$>yW&^M%OgkKJ8{0fy%n<@bTm9vK|1bQn zKl4Qe%tg8)R8r9m`jgE%$BaYDL{qeNodiDdoZ3)qea}Hc6;;{6KlbV8g8+4HT2{be zaN3KT)&SUe0Z!K)r-|n~d0mxd8ATB?RuY-kVi3f{DhB}wb!}kak}9ox!(Jh`X^pk(QV(IDD+sVMT8qZkvx;_Ne`l)B0{ll-l z9>o3SI!nT+Yuf&Jvb;FCelR{+td+JL+0{z)hrP?w-GC--ep;u zM4s!qMOG$Bzcq9|J7qp26uF4ryn1-={zu7V>-79`-1CYS$hzH`OlPx2+3BW}P)pbM zTcPSkCrRY{?rgn|0)MkwfX?X<+&jE-_T;|jbFDxnkW@xlVd{pvP8i2cRapbHwirVQ zp|!SC`lY9DwPm)upLBW3kxBZ4yYGJ(#FIkUN(i5^UgR;;1yCzM+O}bgoo7`pP2hn^ z63ey&S_}ui26B=4E{5C(X>D~|27Z(mtFo~q2I;aI#=YH%8~fquqla-c{)vD0Cx7u5 ze@RK8)8%l>IXg>#_|N>%KmF-{N)6sWm~QV(7|}O^Q9V2{evsl>ehVt@yTX=ap%re8c%Ioy9nk@Rkxx)3X-J9 zIcJRJ>*b)=Z>mZuG29xhQ*rj_9GGJ3U`Ldxu>Spz&!uRgfmgQn4KC`sUC!p$uJ0R- z=d;ZRAHU!4?*&Qk=;DHMmNv~(hugC}JGwXv_;B~e)z?3GmxWZT)^)R15Ugv2u^e(#vI=VyDIkQR4eNv&nqE}7?&v&EnxRe+gIqDjBtVwq zW@+KL8bW9=K6bsu=xiP~gjaLRf(uEaY1D4C$I;9&^f;erR((24>$;$(l@?;5Q%{ol z_LUpA-+9wAX*8LJK}Qg&fQF2DM3jPsQdn^y#o9{C*C~b|%@P1)tKGZ5dn&7S$&*ii>L-e`oR|ZM#QXP$ zlbpnviUpS0uojrQL93E4t^^mimJpHu&HwQC{^no*@18t9;+3G94hRNZ3`Rv!6j_`F zI$B-oK~R|>?=U(_!Fk&0xy_m_k)0&t zOIvFa(fz%nMmuQr7LSjQF~+P)HLUSUlthvzT3Y?F!58r;IBLE?? zG%IIGk;Ip`w;YE|#xurPXJv8k$?5U&aat8f^Ig|1;u(ZsT<|dT4cj^&4yIYcsSq4m zj?X1mP-a;nc;(wxl$Ew0I>eO3==Byq{OE0j5kQE#UaM{!ny$x%zO2%{3HW(|5B0E2uOeYb9cwQ77K#8!bva+Zov8XJnEY7m5Kb|5` zX?Q@85i8WBpRQLbP5FYrZp^jMi6|?tOUg%H^enZgX&U z9(aDM(b#*ktC`Bupkrv7rj=!>q*!0+VWe!GW@&PKBbPaaL9sg7XNAsXaKmQMS-wjo2Gu41=IEWcCNsU|9xrm3cB zVWX90Qo~$|igT%i@I22nY)XA5V4ThxJrf({-47lTGuYWbv|Qrb4hG_-=Wi0-48q!b z?|$^b?Kh|f?7Cf}+6z}N-~R6RW-N_T^Kh@)Sh^WanNYk#)F*FT-sps@i>tO_&d#+j zf932yedVKgAb#o3|Mk@i%Y*<;kz$%T%4o>`vuojc+ksYa7?p24Jp1VI?9tKrU>L`V z7z|@Br~r7L@<~z%A(AW?8XV20hHF!u1VI2~1*8)B)G^U`HXMx3Oq~Xnb98j{v2ZdB z{U8Wz+ewmSHlI(USt%+^Aj=CUY&u~pt7I)~+D>_w=dE@%_rIr_hlu-r8lc8;!LEuTriK#OM zdcAc`)6UNa2qE~B&t5HIr6HtgN&qb`D=M*T(1otkpG-L`oAoeHs)9=eh-Tx_aIO@h zlrk(F1*5F&wc6>tuj>W}5DUeLthk7W)#l{|#AAf4qv0&d%E>%QW(*-j2+0J45F&)I zL=q?gfvy1{5CWJGga|CFTu0i+W^6>s$IB2y2?2r-Q%Y?^6KW11Hz{nIdRj6i#bUQ} zIv!gDP8qMchJ&^7EGK%#xCgKVQXm3V0T^Kw!?G-H+`RVA?GJs^3_^2f?{M?t<+GzB zR`GUVVbdGN(Q++3>c@Fm8L&b+g#dBR7gsvjbZS8IXn4}DwU)ZQ$vhoTWS;J=EUsb1 zrg6O3>+bK|1K8TUw0(Lw3lwXiWyk|N6hwU5X1gR_%`)u3SKt)72+ zSFUyHhNdea7MFwJaJams0WyvT{TY;rOf$iYrFtTjc z4Sc_j!(jEB@9ys(426Wq09D5FA^{vMZ7*!BuUEsFj*$joB$D|wcU@aYMjXY^w(cLE zk}?G{US8TTB?)b5QD|sb!$4u|U=1UvyS8OnX_o1b=c4Ka0VTR_z-X2?IvvEb-qP|s z%awwh3CAQ5(^T}}#*7x@hHdi+vKiNwPMj^nqolFxe2*%M&V%%$W87z!rT$W{;#!^tz(h$_K5~$)h&q^T}lAOtWJWr4NDFlXr4FhV}AOO;;LdVCGS(HJf zB~?*!G3TNvtNwTdsGS!|7i_g&<9S4o3TmdvM8S<8-QGDKWyk$llrUuK97BeHK{ccR z22dytq4F$K!v;YZLj^@eFhCSqwr<(FZV-%tX=)e)Aw*f0000QpcsyiGEp!%=`E)Xm zF~&7Nn2g81sU<}UDOClZjHWu#DC*Bqrmonj%uWHM4R(-pLJ6Ch}N*s$F06t+#vv&?R% zbGkpWw2&9nFsvlcQo&D)DgvfaUwZ4kJGK>;0v`4IT&AK(>b{p{MVSmb4!^qLeB!0; z7hk$Y!RV<=-WNZ$d1+%I&Uk+iBQCesTI;KJ!_j+9^TnHsf9hvHQK#bVAHJOyY%rdk z4#qD$^Bm`#>RM9frjA;cTyB|*J?|$!_i8$w9%R!e=jB|hoy0N`gkwEQGl-BA1QMZ> zqmw8pFwyf$Bza=k4nTA`9Gbo>ATX(xOrt8xrIcl!>xckDpXY3FGEhn_EGz&3Dn%N# zb{eO)-?+HBeQk5O<;dmDdVz&kxe9kc?vtO~bWt8t8JdyL@N= zIOf=(+JpN~sO!eF#P(~PS06XKKRzibCIA2kA%_A3NHC@d5{#6r1b~`Gq+nsaHlNRx zQmhgmFHEXoC1F)@0FbG{s+56;7P`)Kmd<9AZmWe5;6>S8X&xS(ZZ0fFvsnc-YO*+z zjkb4kFfnvuVS#K7B3+ebQX!q2gdg1%` z&!EgTO_Ngku2G4K1AcS;!sDIYvJ}`ZdFg0@=IUlN9bsN9uU>fkcz1bw`|-oK78kBT zD2{iJNz$ zaadu1eA{NSG?9t0Osa}=scS}F;hJNd4@T7RsT1x#Imp;Zpu!@JY&_W5Uc!Fq>)ZG4 z+|{v0DeLs=MKY|_8m*xFd)Zr=6vqO|fEfVX={=<^PNfrIm5&2k{CkQB|PxxBLY z`0-Aj!^5+oAT-NT#&gN3VQQ9vr?U~O_{RELHlIQbVIdS$nr;n;(|ME-UDqvd5DiSt zp3GyQQMcX@O3u<4TUIn5&u0~U_4!M_201JNgJIxRMJa^P4Ns^-!{m4{%NVq&Vw{(p zDFp$Lx~^lT2sJ7I2tX1t!ARFP4VYVQ=V*93jColV9BBeTRb*0OL!%!j<4UQIPs*|^ z2_Z^T6e3meM#HPu{G_NV8aL*p3T|SY%gjyzt)b_j5$bGFs~e zidBtTeQ$r)_Zt1l3_9*r#r_hYu8Q>b~TsYdhXeO_nY7U@y}lS_78RyL22RJ>cg$t)^VK071FetrtOOf zedUDyz?G~c;9TV7k^YO-@U7Xm7YdX4%lPbw*FT`P*BEK9>M zfTD9I5tMl{?fBaE)|LiYn$LAfXTw+!LqG^2>-wJKd*ftox_~GRbBI0QV!P!zC$n-o z8&is-M7TNz05DJtqLOps*tLWHu4gqJG&Xcg09dO#^EjVQbFBHgrh2|nTWCCbe<$O( z$j3pAD#>B#P}SeqSlB(7Nky@S4J~|la(H8{C855$zJ7X=g>40yO2#LbF0WE#fADzn z_Riya=)1c9+*7xH@T1!t%REmlgdha70v57Z3F@=7@Kup3X)bIW9UUd1c_&1RVdLTXeO zDg%TTxgrE2ECu5_p@bqRc$7;3K86d+z%e=Jwoj6H9N0AQY7h60w>B0AgUQx~&DmtH zEI71A&lX{;a<0QB`*`x@}kN9>cW~9%E|fMaKrKN6bR{>Myt2%+VZ?VEwea@ zvJ$syR(6^Lodzw_{n0ybeCo3=+`YfEz0zikB}v*{UORkvJkEyAa3xNr0LZr2%uBv| zer9@g1|&gzHqMfw^!*T2mG8TQX+ogz+*)3gOo}AUbqFoEjOaY{-rLFD5gV~dh9)$)r?Hbjzyehk^3pLZ(Jv_j= zC0H(azS3CoYT^FjzLaWzJj949K&9kqn);5ZX{2KLY%*S6>^Z)d=kdY8!Sl~Q|78FC zc>hpn212x~c#;+MRxq4S+Di-YbI)(v4$U*6F!czLX@(p^s-oR%>>r;*rIs=V002^n zikY?>#dA!FrCEb$q+_I;R6|rFSTG)ydB!wZGRv_g1fnc}kfv@>-EbTWAut*bDa8m& zj7&`n07Pxa8;=H-ffu@ggCNW}thie>j%(6+Nm%;$5gYqh4Ip7vqG(l}yiVHo=UV7|8MN{-^V z%+h(I?q6J3c`_JqWaK<6=cSHBP;c@6yDMv}L1R^x=bc9DjgL-E%k>-nJ8$nKT$N?n z?ku)KNFj(4QSl5@3M4E_k))y&+_AB3I=TU;ae_IwT(8KhQc4|UHOJGaaqn;v&yr@_ z-E1@s%<3-UOfGenpFBF4k4Me6-sv_C&($gARRMTuxVB$!RHYoA_E}yH<8-~##Kf2+ z)%a-SnYv`MCK~|LKgCVB20VYb~(093%li8FS zx@X!fFTB8~6f;&TSU?Dr5>i54!$2{D1d_rwbWDtFK7ppiOIhYfDh?qubPJI(=b54D z1fi;`bWC#&qP(QY)G5%ZWm^D3L^U_`K>zf#5Ukm7u3lZed-nq%EGV>?mCaDg^W3km z9v+?e?FL~p)5V;VDuq>*Aw+BoIkr({Vp2v~9JM`1Xw57G1{UKe(J9e2p3cT$t2>-b z7djyqU=oclU)^}^^+Qv4d~IghO-9@@?tcsdNTS>~&1bG%c=yfkovB8i&et0CmtMT^ z!`Huq-OiJpvzFtJnV^)`>mkdkWZZABEada?c*Y6_(rni2^%la$+aJBtYOJIMtatGbvP^$4N!`F@6))1*w4FFf0VbNJ&F6E&B3;jF+g1@}rfEu56fG+IFjs zAvlTV3(Gwsfvn2Y!MN^wWj2l~JIdMl=%i-(*Ed_Yu5Ro+d|TK3DlcBVbyFbu9(`6 zy>N7Nlx6uWQU3CaR~w-P)tu)MOXITAs>0Agd)S`~ zsg;?)67r(d5U$s25LddTJfk)mq+HV7!x46g+vG&IvotP}IGUV=o}Z@$QV?MM-~Z0n zCbEa~_*MfUQvTL&e%G%pWr@0UAz1CsTL!zi(Kx;P(Du`<3ky$n?>j*g5LYuA8UU9T zJ3CL__w8<{?J78`1)*yi8b)PV&So=$@oXOF6$@*An#L6?k>p5n$F!K#cMnfT(~0Hz zfa1I=S5{hiJb2~#o2%>1vYHbOYf9d{enmGhQ-G_o({fCUIIh0B(n}M5baI~L8Kg!j z)cIuYdzNWfJTD!JUV8qyg@qL%@zO%C)^GzK{g;37uTzPTNQr_~T)gs`Phj0(tX$}I z)|QsL^`O7=F7=uPBc1^V=h^+8<5xa+Yn%+u&W}TrH)}3ISY&W{@y4(GlP@z?M)N6S z3<-rXBnH$7Y99P!ES`T4y^`Qr~xWF=g$YCH9#LG&Nq|M0=% z=zLUUB`9ISP-Xh=Xc(QHj*brdr|0MMS>)@+{A`Ss98V{*sG=xJtKuiV_!GG(=T#B3 zYB!#}xp8sR(23xx$iZ1Oi^{p(cG+Z7j-y$B&Z$*NI!&qv9~`h0`Bw1C)0gWeZ91l< zBPlr&h%-8z?n0Emy;`C{=3Onr%@niz}dcF3#r}v4SXx*#sj?ND^H!ddAzUAt1$wexNp^lDJy8|Z-uQe9V`j1>|WiZ(%sNHJVK&8j0 z$y1k?hNp+K0zv9OIXJD?EX{ES$CHAqYu9ePcjsQ>Ta3LzX5b=$4wm0pn+XVYnwl~gl%Q8AuvY;8X}I>(h@B}Wb%4<_fMn7CG1fFP&| zo==DK&DFMmvP>$fl3O=7-+liCm~O7-?Z&Oq`G?!f3)7@HKRNC-YPwE!gN&mYQM?yA zob%CKl;!A4Kl9n0)9mo@u|dg`bJ%f0sdk-3m?ZNpmHOAzAsCb&EvuG>8)ojR#h>J3y%_@7`64#X&juMu0Qq3H{XA|=GfOS-}vD6 z+fY;&Hr8|j`u+Yqt;$MDKnKH_t!aqM&T>l#7Dcng%?p5wbRKIOnN7!6uUsTR?e6Tx zLQK;FVJ#^b6ht@3Je}zVEemmWez>u*v9!=GvIqg55RJ}xT1Z(6DFG}C>DaY-lxv!X z5YjXaYO(?))-?e^l@%^^kfI9cKsOjyd7iqC9=MKgStcd(@cd0I`Tvn12<+02AlyIza)q}H@e5E*6$fSjF=5Fvi2F&;g+w6%GD zbl&N;yI~_cIi2TVX>Gj~YTH|@SvK0=JNcQP{)zeM@mi|^RJ?zDcyIUc(st+lJ8w%9 zyzuPJ$4~CJ7edK04YG~Zx=pIPAHHo-EXv$-%|_FO5CR1aN&t|Wp&y;4C*w>H>J{;)Kp3s$2Y- zr?2)Wll_AyoTFsgcL>W#H_eYxlQ9`OvVX=$OYZS~qK&DugMbST?eY;3QuT)cSkqrLNg`8%&o z%V1id(R`egx$XLwuU+O6rWq6pCbfwiIJq5g{X;%*{Usxik}?!`@=c>>6caov<1ez$bf=Uf~qJXCPl?7UTQkg zbt*V>Obr2==c%q+yyT}z>e`JYjR262rWq7@j&D28JWWoIj+?Oji5D&+?exQs&K9oU zdjDa38daUT>$w60g=_=l+|(!syw|Xz(LmX58JAf)0Ro2&Z#W#fuB(`s#U&)Bp+g~a zhAGP`!*sH;CI_9GofW_%3n`F-%G*{f$L? zam~AX|0J$ZIxTHzA;L z?e#t1IX!&v+>NcfhalyXx)~hr?^(Syl;#yby|j7r`1El)FQ0qqmDgVXo@NM9Mue)s_p3PS z1kGRiGk@gY{`=n^4RUUfD#>~Q9!x}|u9p>YHIA`xTuP~478Q`9UUSCdabSnK3pL3C z-(Ohj9Uh-OxclI9pZVhbM{k~c8^MAI?Gm|znKi4ih2-PVm8t$H%sx$?|s|IKTAM@KZDj|{^Iy@n3RV!MS@ zWre^pI6?XG*)&PjC}C1c&v7sihOSyomtf{N$~JLTRiZ5Fj%g6+X@p8axug(L2nfJX z%~}gOwexBYO%#S!AfyTAK5&63T{^^2Fcn%!=&1nU-_PRC|zvDI_aIDh}`0|W`@+;I&` zBg4#*9-D^57*pLMrtTQ#tbgwMi@T5JMV3yYK^%>H?cl=YYmfGx3}=(cv}`OcLKE-r z@6U5Kn9Xk9ydo;58MB}IsV}^D_X*dPrGwqOx36z52vD6JjeMhxDdt(KOR?DNQYeS# z2N<$zmo5yBXH|>>yBmgHqtUQ!H%{gX$T&?k%q?4o5O&*50Dwfa(^~AG&nBbX*KAwY zn5YI*QE|<0`c}OWGd`VUlfi+kqQ#askCHS^P16ot%d=>n#5Fzo>TmwX(^*9TEs7%W zTu{vnU{_hyGGWaCp>1f6lU6*fR4(SDQRP@}b3va?b3=D|&X4zYTmyymrFfR#yt2`2 zdq^X@N5eJE_lk6*3v>joZ*PRI)$)B=G?Q{bA^-;lIl95 zBu#UvKq~R0gAc#_ov;4xo3Gva{4c$=m)zb<|JA?xbwc&9Uemom5Mp>Qi81D@P(_3opzfbHkgk4{X#QMN>wEw;f3v3ra_>ngb=)6S(`inaa2`#*nm@hk5ig-dIy!g+dbHd{IdhMqgN&IzCpC?!B83(hQ! zj^~+@rJ;oFnk56hrP_QPGm-~}zJD|bTRo*{&~nnl-FZa>u!#ioBCAS@HR@T=unWwp zssc+Z*QR-qvj|A_(#@-Tdwb8^+WO%B-SPQ&YxB~bogpQbX$irqU;c}K^;dr7{}osw z%Zg5K@$7I{_ZR#9Xk(>ydh$VNbcCsDhCZ3hM=^9gT}nAg_-e1_do+zxKWM#u_rAuN z={Kr!+N>>1=c#SVZYP+H=caAuF>+eo{)1@&2MyPJ_~3ABYuWN>k>rB$RyUl_C|6at z*?fF(QnSnT&5Lin{RpVJX#`B5C;JD`-RQZ@^+=wgtQbNTc9s_I-@UhV!Mn0~@rU1j z8vt+fQuFRdcN)D8=ib3V-_#?j*GG{cmPH6TJw0ba*@lH7na!t-dT?c>K_pW==Kxad zuB--67`xpJ)h*otlRCzQkxA%`}?Q6F(*P>YuQzKW?0@|`ZIt0JKy|$fh5$8 zv(a=kN0WYPnG74QZJ=sCr5b=htJVE>;7q3xG3w*-UM(~iJFV}%{bZr;)h*E(7Not8iaE-@*s7%)tz6mmV|@ZdDG z-Dxz3M0Y(y78Rrfp^7mrPBPc^KZfo}t_;7Kq!Xw(2M}`Qg|#BD;y9fRPCoOg=fxy# z2XcFD<8S`cFMsLt&()T%{-gi!^{mJ-#*8s)&t`_t)IN(m%~5NvL( zmQhl2cyc;|PhVQ<)_sYJgF!^F&3NHxnrZ6Yz%1FEl_<|yiNQ*<6~(jjX^9|F3RS!! zSQ(VCk}C*I-7qy10gM2uBvT?uOE;}D&$10Fy_zYjg0sBcbt_qE04Ttyf=r2N6anI* zN*EgFvf>i2toEkEfh5Y*Syu2MYydORuw=u*=`;~QS7jB}x}$LpkOI8WEb{8hy?Y<* zxvgt*c0OiRv(=kSCN592tVCrl@z9y)>vr1=kacU@bXO=Hv4mEgKl z;}Ycap=YY=*H-S_J#B~D;qifEUyF)U#l^gki@hcmc{G|>VM9_(WHql?x7Rw}pZdO1 z5=ZlKx3S>h=`iywgFM=uXhBT_1w=yA@qF4}YxfXiXY--wuOm$q@xXNI{nOoMt$ubs zzI@?1)8Zd}^g(a2`{1Zp@@a2n;-fTJ3 z*>LmvjfZ#cKYjH|p2i4){k^kkqMy%>l;Izo%xbo=V9&2y>~f*vbZ%*u67KQ&q3sIW z+q(N;_@yuX#Mi&}P1B%`Z8Dinr&$mLm~%o9Ai8bIPP;jquq-eAz&}kNZJFaLo=4PiA08H0R@-Sjk_8Y*JUro!n{#CFvhr=jE7q(BXT#JFEY78<${+}` zB2BW?A{y2;1$dHhOuRfV2q9%zYPvozxbInzRmYPN##FSC{LTw^v+C)vfT! z{*!LEDMcDZF(pRF35LUARaL`&oTP<<1`A8mMBG02b&MixZHE*7m5)pv3qLKi@R;OuOu4~zzZ8(ktAtZ!Mh7$~d z;AlK%8mUc(<>+vKKIogC?Rj3KQLoiKo|i~6-_~7=SdJbgtmK|#gUh$JOhXstj8N4- zn`_j)_s+d(!lSfwO{3Kg5h{d=i9TsGoMzZ^Lno})g7%6*f#j2CpT4MI?X9;TLS^-* zd5KJFrKUwRqS_5#R`Hc<>$R4%xU$HKh+2GY!AFuco9*YGc@{z4c6|oDU}3|xHH?9- z8yE^M%%Y%KYAm)do3=GCCRetWSrKrda>Tv_%8hwPXB_@{5(>Yj~K z-SQk?LwOuxXbXv;>@75Hi#Hk_`4-kUmhC8+wnE?5$=Ts?uf1Sd z7H4oaDFw#}rb?C?#z@08S1d2}j$^rY=|acXS30Y!TdiE?$D=t{s45sQQp+R+$wCyL z`1FeyaRs$mlu^T;XSw0%LI}=fzdzJ8T>@PJWE#|S14Fksmk?SCsgh-wVnmQ>n-V4% z=dK|YV7_a)hNb}$+HS8I>an*h1K$D8ctSYjCbH+4H zs|5eShws1r?rkwkAcQ5y{^HV+a$mnc{Ga~bZ|(LYJ7{r+0BlBcq>%OS;D{rn+t%eP zn=5O*PR|oCv1~0*i`g`r4X1{H(P%zA9e9>$))$}jqr0agL=DJ_GL9;Wb(3gRsVbi6 z6wi|c5lcdnvJ!F`&6B}6jgtf+WSKfth*DiCNFX8ra?S}MVHhHW9Mb@CuImhf!ZbLg z42VpMm=K_-!2r!miHQL)>5s+=>W6Vcu#venWFT+42+6f zX76zdq^ioZkc>dITTKq4VXh>u}mkXXshEqA3? zefziXC1BZe3tSv}fm~>T{^;TRZymq?*5lRg_!CdJe1*-X9kaBzyx{xxuzx-r_Lq8H zUDt=x87rl0dug8KS)@}k91iP_uOnk;?{E-}X4#Co0QyOsXO~vP&poq#Wv$j;Sg86>d0`!#>;a0EA*R!a7uxxFsFboj_tuT4AKu$Z(_%Q;3u?MhGX<(nw=ogo18&_BShU0p^?>9p?;Iqle z!6`+MS8;TXF&pCy0`+Yq`@V9jl`>|`#=lBE@Jb$!Mw zRmt@Q``~Cs;Oy4T%kSSg$VxbxR~oQ3)|->@NgkcGTidhwajVhm52v%tSY8MSR;4h2 zm`o;<3u_w?RfGPa9~gOMvr<+$FO@#%pWS-u$|qlX?jQcMzwbL2!&UFs zFc1LmJ-&Nk^J-b87>ZF+*e11YH=d6j$DK{4fAUZNrQiB@|Ekll5wenWez13b_149~ zY=)GSY2gKanp2^`;ps6pe8CFAtAYUoz;RLOrYi(P1p5w!CV-qUQ7IucgiXyZMP*@F zN?0l>n3PpH9H#As+Pq|hE8Ve@+0?dmP4_300VNhD5&)6a?E)~p5P$ijE zG^g9rkp zWf`I6b{hfDv$8C=J8eItEKTMq?@vm@uq;dnR+=Vt4cHcTO$$n;Do!Z|NK`5lK#V7I zqEV=GOqgi`s=-?53!sRmJD$yDscY0UsAXWJSXptQBM1tj%4WAFBr|lc7PbZUdy5-^ zPyL$R=y;80Q#YmU=!S{&GBI4!vZ+C^VQ5GJCCX-(`JOpVim(0NJ5Xm~ApXoB|CwZb zGMi8197B`aj+N(ZI*Iz{(fBDs? znv34jqD9!iwDm@};d_o>w@kn7`PasibT;F|QLGVvK26+a&DK!OclP$C^?Kd)g1hg< zNVhL-ZLBoirC#UkWDZoL*=|iHCzrRb10<2D6U#g}9_}8@XR`|1Ezj#FF-+rVG>zZ8 zpYH7UkNZ_W0(o-w(sP$jcZdJz*Z+@aU-@iOqT`djg{9>puKK6!)z7W(?0~o4c(S@> z-Mq5AveJ0t_KrfwPe0$H2tT}c|H5jw3PbMBx$d<-+1S}^RsCsWY`M-^q>3_ z&)>SD%3(1%7Oa}gQbTuYjXJW85=bez608~6MXsV*RhGF1F$NGJR6$X4c0P~uY_6n8 zcxjtX=-1}cnT8mmDo>}FB8|W*9tRE$Y!fnQm|747j0r0+CTXIG6?p!9G7ep`(h0(b z+s|h=pSgtqMvB_DZCMtCFqy{~A_7rave|SFAuiLrOmo1cqDGeUyp((WA*pv@Z}Wpb z`;Bk@=->bT>kI@Jt~}dpFIawkw|~w&ea7->8ZRv^ZLD|J*A~XZ0n}Ng%6ZD*pZW2p zX8COQJjVcs9t9i(HVFg3r9MmL!D+07ay`R>+%QSSL|J97(AqDe2RK+|>g&lZE<+9A`t=j&3|@d-uriMuwWWD_NDMUS&lY;CH8Pz?DlO+>pp{BwRjMIVA>DM` zbPiV*^?M&aD3H}&@K?Kw(Rk7y&omND=M&HK=JWa08{2o^zfVlk^!=J2?Cg&;$B}$0 zbCUyARr8i-PUZ<0$h7_5vYw3QhQX%O5<*yT2NM*{Qjab~#~Vxj?vtbQsUU>c zo1y7w8cV2(yVjf+g${vf7}GQ| z6pB+$3{{pT)I$PFf-Qh@AONKl0)w&Ku&pG?@~pHiJcP!kbDw+uox!?wI2^u18mP2cxw zL8t&Ax<+&gFfa|YveZJT2m=RHJQ+=sc&Z@)u>?SJQD_FU$-IBM7u3RD$F$9gmy^k4 zlIMwnv1#hMX_%&7uZOy>BLFR0AD(JR1(VTUrzfjyN=WbBcXvDO&Gp6gFa6=4E@BwZ z^Qx*W2U!j;s%coWG$q#;7OFVT5IBho<%C@5S)qLYW>SIkVRCd9+a`Fja|%?K6?T7+ z>^ywjKb{}~l@N|&#c|wjH{bp6UTAe)ts@Fv=CR{g2G6dpbQc<#uA$CC?U|P^hk-^Y zH!QJ#Sf2L5gU63;+uK}Pkn>3yD^bX+7q&FT)>pfLh$6>Wxvgg7%9SMq&~(TYCg1t~ z>+9RC7jNBq<>{-0kDtEvsf*VZYc25J2e0#N#Z=+WN9W@i^m^-O@sp}ZKKaFPWxK&d zdl*gq(7tl*!r9@uPOR0{wW8v-u4#~azB5abgsbC7`|$pt6|N(Rs)C6!KR&sKln#{H zYt?ohKH`#o@Mx3)buvr!=4K?!^SsIsy1RRP9;>zOs~_I^Xfim{0AJ~LhU0N(abBM%-JWpN2G7te0G7e?Q0)r}0nHDikY#BCZ3JAoxsw9Y5 zSug+yPO5pHryt{0F{hScdX{PE)Nq``;rvJMKR7(knDm2YtJbYSt)P@Dr8G@5P1AMV zvMl%a_ovh8d_HGN=B#2$GRg8XW0EOh0D(~{lIehRUT`^6NN+9`B>1f#eDrtz`EUO2 zYj;|{ePw&?;?>QFYQS-@LHdKyXfie(4Sw&0X69%8-&PAgs_L*r+OwsKu+CU6Ym+9;r z7FG1(voC-9ySH`AYnScO6Es3KvO>4Elb?SK< zX`WfkO51lj3%v(-A65cxUs^{zgFN@^-kY}{2cCs+S_w<#*~a?n?*5Zm309Vtt8AdK zo~2b3anB~}y(LJCb`O8;tB1F)Z`{9o2iSU9RcY)9JYDRyR+^@ahNrXsXvns%Zp_Q_ z@ZS99`cpBS-MKdo8|BGy^0}XW{?0pln4Vs{xb^sO+CMKYJ=Hj$!h<{Kw{96ZYaZ|J z1#bO#fFOu0t2Qj+EUpNUK?T}7DD5$)!!-Y~n01?0X>L-5m zgYOe+C`=A#I(36`l(J}dalHr23=-F|aaBg#Xa-)8B-!-jsL=#~+mq4w*=H}@e=@vx zIH@%jr<3DPJ-2Z9_>==^n^}7B3hYx3^q!biF7zkSsDS`w+oN>u3At*^RPKZ{PrD5s1MtGh(L=Z|5!u?TdxPe4OlZ;6L zAU+w)&iW&2I?bTz`#wS_n&oMkrFp{gOb9_KwQTd_#tFpab6}~6@jSa0tsbMElDQP`Oz=^p)c^+sM%>+j{U~%A34-{)GrXh zM#))H6`o^LP0LH}g^j_q015{t)gbe2$EB30$~Sxy5+GGGtRbLu%~sfAjJd9>l!6Qq z1r3amC_~$HG-4abF^O%Lrp^EtqGV;37D-hPJc?+V%xo9v8njJn7=Ri&#@KUR$FJ3U z8^~Oi+|rP%xK@^Vn#D>9%hYsDg?_+6WoXzY$kmPOH`kUo8n6A}hx-TPkM18F9-cn? z;#F)o8=IH9D=QZ-U#j_zgfvvf*riEvqbHZ_$-3*?V|TDCIcXbksjRI&s~` z_g;J7aeXHBf;njtZ(mvADj^tkJIzNA@3cavY1c8MSFWwut}-2FQ2uDAbem-FXn+lu zGmuOwjIzac4|4Lu*WZRzv)tg-&wT!@KhB2bt!Em@y>oo@QMV;ZHj_LW>>XTMo2nST z{)3%QeCC$WHX3`=ZV2oo(W3DhhOQNj4>bdr=M6P6#eD^O~>kdi~7R|;cl zl@%CHqIsG?Dv&5y5?MMT2BjKAsIoomTG*v1<$RJW%a=B;tgLJ+rRBLS zlwwlFX{Hb)x~7>%6h%=KeT;YbxZiVndde6RLX0QF^Zw~zcs?GFIp-gvLjV9uDa3?@ zbY7_>N-8eqd5*1M<=S(<`@!zF9-jQYfBPSgqcW)Xl;Pjm-GM*$!iBRz8*A+I-9x>V!~>^F)t`k%+hkt3bV3GHOz(LT)A~0aKWVnP*I9?Vwk3p=cz)9AfRj1 zAP@pha3wg95^<5qBDYL8&1V|5kS3@BTwPn}cJJMN*YuiP=oMpOt4=jDnx%lJf^mXC zUJ1px(6FaFwo4pWPt$lj!}Z!~bbjFWjQ;to>H9~=yMOL4{Mq09^6vq}AX>bz**H1h z9iPp&wpWjiPBh|Hf(xP2Y-(5<Ih z6Wi%ZNfuWEijy*};u0%m_m3xr&*swzDiC`0b{n3a?>_g;Pu;t}GnkeXgRH6+78aD4 zJ$3Wa!w37b$+TkWa}%A_+0Z3zO(<}F_CbBgl*`^ zGDsQ&r@@r4?%lon{Il1uUt0LB|M*sYnNH4~Ms0HA`m^8s#&@O6e(6vA{D1oL>-7c^ zIX-?IuV1y?roZ>LT;7g@mik}+^}`oly*%!Zj*bVPdiA--J4c~&Yk&6}odw$tFWtTW zPzbX`Ocv6zf)c5-(OfB5bIpaiuUI8TvAxuCEINvYH?D4tN6F#YycRfBQFe_&)t4Xb z&+0zZ0qPG&x}CYri?G0<5zo_vNnaS#%WPh0x;V#5CS4EiEjFyIUmQmX)#bVJ=36C&9Cj9 z9h*o4gpQ-RR9pyQ>(rn`qZmP*<*80U&T}CRCdl4ko@cCCYc6lBWd$n=rDSz-ax$4r z06_#n4MDf&=Xq{fmPs|G6hf$|7$HOmA%yrC<^J&;B7}6JSAvJNn(ui?Nki9kqH%yv z`{Sd%T?v5%3Sn%RTBFebI`(bLBxrhg2#~R|u>!ANXke*3mevS#mKF01GEnIPtQg2_ zDol7%8F}gkUThL8XFSp%V_XQ`arLHU2u;t6BnZ7^mRpuVC}d@!B&Qf+B+86w8qX5p z`+mtLh@q-fmIDpL$jcn-aIxj-)Y#j*OWlrX)W(yE7g}X9Gq9W^A1i$}oM{x*JYVVr z%Tm*Y0+IrpFjuoXu-L00FC)#=S*?R_KC}Me+oxy8>8ubsmD@MgpM11?>Den!c8*zL zZmzq#ySp1}3wL&AH4jNPpN!M65$Lg+Ru~yth4MUMmToDP-gv6Jb7wc^MKr~_wqpCd z(QvZ)!S?!vyN~vj^i1VPIF>B)9D7!b%QW$5;pyT1<%=tCzlrSdc)8h2i!{%bYgz|S`Z_iKi@)~SU;D>j`Qp!A>7NGQ`QC?1 z3uj9!*WP;f@Wm^u-@pBqX|6O^>iZAw*WEfY$1gv9<-PBYX`PJc)AN35I6>VEclVCn zrgs)qvm7qe!^z2s@A|$CU%avwCnIIR#cnN**oO~KqH(pd6cos5EUnB3Q87q_Fw+xzcd;@1OPE6Lfwryw>0Di z{$!d^9c!wrioDaTw(HWo5E5RiM5CB4@;5m|1*G&vb z0a&lQ<{AjTcx4$Q?z;|w70pCcWKpGBwR(q6{Qx0HpQdzpnqOKEgi=CcrJK1BWyK{T zUdwiD9SFgyLf5Q{GfJrrK~?4$%34UH@hB|;MT$}?xnQ}pOs}kBgesMo>Hx6Id)lQRmIXI3VnApJl4%t ztyViZ9<6Wojt)_lhF7xpM2r}_%}b;y7=_&@q?>R zUH<69`!+T^jdJJ72OAe(?jJ|n7nk0+z29kBbzeU}8RV7pYT+cBEp^)!lfAaq>!ELd z=cBdFAA4~B_UcmWxPM%C;cTK&E46h8Lig}I`PB29AH4f|Qj{@kc)qFFL`gI_*Rflf zm|}>sR8$f$##VcfB3_EhuWdYhd{i>hXwuQq`5*bCKlS+F=<&nBwVPKDcb=RaJ$~WE z&Zy7tKH9&2v2*{!*$~rTc;%UY_2uuFRex!tDBZ)IoS0{*!GszYR#6@Y?zIvv9;B8U1Do0Fo~_pD;L{~ zJybK2r!Uu=Ueaqq&r}#F0yF?h!>;NreKq8r#d?|cd0t8`sfIAd2=c1PmE;I24%AYo zzSM4RTv}`T_0Xf9qhG$b+&}E+xoEW)6wfW2DgbUhfAjR@;CR2kx0iFF@AV^Sv|fJx z6Uihq&Dx*)3x8HfgkX@Ta&75~X@Kcu2&G=Jp>5b0!wZ);eaD;(k9~(6?H^!H2!IAn zxa@!XE8n%Pppw(S@n8Qj0%W_s{Qd7d4%_Xc{!UPHAKcyXT+`4+&amV3upH(^vVYJI zeVk7FK%PQ2e(I?O-2h=d98CMm%h&GQjoyA|FqxYld~jc*)_9V*mVY*qvn-AieP{pl z$-zKD@Y);S4{MfEv2ED>^Et+0vlS3y-1It9s-nyqUDojN<*n}KqI-IL&|6(TK7I#@ ztnQcq9*fGo`}mDF-<^~y9vmHs>iwVo$sc>;&GGuB_AmVW7Y`1Kh0E&3#>Q*!K3Hi# zbLA8t?t-9mgVI1 z#L|s}!~W{x0tC8dcUCVy_doyrf3~u^=^=Sxdoir{SSp~EM5HJo9`|;MI*FojmF??{$zLX z_WK`*A^{-VUJ98=mWq2E*ZS_~_9i#@KS(UkdE)<+g7tj{;jk zsHBuUEi=KYBF{0P$hIvA)nqc#6b=Yz_y&=MZDK*G9fO#*q#Ceo zI}Geg7kjQDU0ZWa8n{NgVXrLO&9GQ#QY^+SJLkoi<^6UYn?wLnSQcqD3{5Lri~R z&8LV!6Iu7({-CWHfBqM~_~NZA^+rN7`8!KzeFTVWAyy*L`1`+?)|KcBO zMD_dqs-Uh5Lm!Ad#|TKlfRxc()Z6QM8Z=wswVRugt5&m@%nH{>6tSRYS~k9Vr878t zKWL%AT{TU!-Ki`OpAS!~VsicZ_2pG_KAF1yQnwX!-E?`WmoiB07B)y2kkut;HsNcV z3!bHGy1a37GYIVpjQl2iFpg=nS55Pa8 zskgna{)a3|o*bRc%53Y(Lei|ktC!QutIO$h7twjAWgYJCUR-Y!$*|FEHx}wp%pjn9 zyQjbRz1#DO#wl#ojhY94_Yc0+tl6G#zVf*r7Z^L9+drQ!cG}VCU~qh}veE^xqE*&t z7`&pMNt0;ey0(t>0-&YkjnK73S(ZsQOW57V2SuJQFRbcN|Mma%H|8Y}N?uqGiv(%_ zD526S@@!)=o-$Txkhr#A=72M#QFA&=xJ0JmMuP}J6X;f&rmpP}2?(VHD1Y>w z?{;hU}@zk(1#008IQw(a%RwUxys%QP8d2qDX|G@WPw z6lK1&+}YSzxVYWz)|`cQ;MoMD%5(KRo1@um)@al$%T!8vp04TGvN6@M1fXO1?r`Jex#|0X3{mcM%8$`UqB z&gU5Cu467Nt}-qu!KUd(=MkUgg%Wq3yz|-zzh^@K^z1OK>rcPX>vnzHw)G0>Oj#0W zT49)k5Wcp)u(8!C3-JB#e|ux=LQ%*tglp@@#Vf{zbsH1aj}oa`-}uHyFTV83Y~BxR z{wpuMdf{@@uo6io7-#XUUvoUm);u>*0-Bmw+-N}TP+h-xZ4D_xHM`l|x^Q_TPtq)j z2gB1p^fRC658nIar#Itx&$R**`|-Tm*t{Ok%KIPf@giSpZ<>bJYq+$|X9RX0_0l?Gzx#soS6YEPnXz&R_iVpG>NW zZlIOr4Nn&X2eTQ#^7N!% z70Dm@*-t&X^Cp%>US+OMKmDnfbtCyaBwqK;vDAtK6)0tz~Zmp}3?ie(l zA3!kvrC<8Y!RhG9?tyPQRA&gyya26jF8$2U{^Y`94`MX#4+j0wbdrsS$vi3`#CcZk zA091tdSCj5KYBKq00aeNHlm#KO7P)yCa@c{)-yr!63im@!G{kD9{1X{x`n>|`Wyb@ z<<~x%Ac+K@D>a(H*+@i{h7L~q|LL25@a3=l!S8(S_rLd}AAWHEqp$t$H@@<-HL^Md8h@u1Sm_-~ks~PEk>s3S>iEUjS*96XFv4LbJD_I4I-yp`ozp zxO$N%d8^(+7OuXa2X&H970scm_k9-K^8 z8T7IukIsMO>Ey}o!N!fP;aL_J6beoZlxOAAO6~fU?XQ0QJ#4_AdS&xF-`gpP(Fz$T zhx5`}Te$r02OrkjCIV>~`bnJ6W_)?6I~kwPle4wuXJ=V88pdHG7#!cP*B3)a*&#hY ziZ5>WZoj+lw;T@rBF#MCY_`eH2fWpBPESuA$MJo=$O;VLJm;(HcA1?$cnFLjc;UG# zZ~WkE2%3)3vwiK!Zsb}_sN}ilp83J;$ARUg@jQ>Y)2wClK{h^Z^j5boto4Wcr@KR_ zxjAAhP3zIap-x$E`Rdv6iD`pDDuoz->8CEg^Tr?~mlVzcXh|95}*^s`rAd+Sl2 z_P_MSKlH!-$2W;q?>47lXz%RJ!a68(^oeJizyEtN(aN9u(x-pp*T3qx6iD>R&tCoJ zf7*NbW%1o_oL+lo-3izK(^u~|7Nrh@CkG=;H4gD~HrncRlqfegI+wQAbvC7NczaiW z`MFoW{Po|wwALQ>=b!!DC*FB?=iv0{rO&Nm=FV zYwj#Yu4&k2HJr&ZDS8WaP!vD;g`fIQ-+Q-;=B5$6v%7n8GWL9TW4Xnu;&fh23qYVq zqI@=+t*xj1o!a00o4@$}y^7L$JbtVQM#zLnD5*UBVeLQrQ~&d%jK%DcW>ZWZ z&uazEuEGS1s#iB#UEgRv{m1{?-*DQ?zx}`c(+ew&EbGsvgECK{N}~v3-6~5y9q#&$ zPO0Vx$S`K}d`uxW^+@qKQe-wQHQfw+rx6e_~Dr>&OUyE9^ieRv$PN{xo; zY&?Q8jb;;`Wt69B7AYm9l;d$9;gZXlrlVnBj{6q2j8?02bUsl<@@N0VPmiV#9LF~_ z(+S&}>A&^zxIY_miHOQ))6CGFpZY_eJQ*A- z1LjI0ND0<|@crp{h`#WLe|#`kZ@&L%b?egE`9Rlo!>Pah<{JjmDCHbAxX_=wb!9L> zr7+W}VLAQ44#(%y=U>_)K>vx)zWSYSee~*&FPP@jR$E-Y^z!-P&NDCC-R9PKI9y+7 znHW-)F9wES`N8?@yFa-9d*A=y?vrzBbc0&!-be5Dy4@(5nKl3Iw_kho;9ZlV58ip# zbu@z2d$->h96iy5mCtgWLdP*qMjTyvX}YB$B1?uT;SBZ9Ps%bzh^ASrX*vd& z5N9+g-Jt#U?GL+4tB($jSUyK-7vR)2iU2Dm8-=2%f%q>&CU~^&tGc@BYE>efQfL z=LV*>X2_BOC0LqX+uk-Dmze%=&Mc@|RL@xkrDA0Xp|D(oLj_m~uWdIhjd+%x^@m@$ zWNW5YNUFrNyHRIZGOP5!U3P2hME8p5JZM(L&NR!>4Pujm0>DMWIinhefFT69r~qd| z0xnFU_&mvyMER~KRH9+5ux?qkX}*f}ADY=&CBm$<%0!!Hh3!aN!?i{@Or-!X>Mdwh zhrZJspC6r$9-9V%Qr}qbCS|k4i^KEdC-=Vh+|%o4$Kz-^SiRK!;G=_jdu?^O0Tp}e z?NPh8)@)IYcnVET4Iex@gqo@<&2gOBG~z0w1ZfmuWsN7tl+vI5`Op0B@4lX8K#8c^ zT0R;~FK&eA{c(wm<+fJy;OU7DO#gh`&!)onm1+Am8~4xW7q%L2{a|M+96FYq4ffu_^~>?xbQ?D01FDB-M>-dWKmW5ocjxYZn9h)C zUZ6>w!lXiaCm`?LzIWsLmE~pn;k$$G68Pa8D-qO-CJ$IH(RQ=BF#{{xGqfWh%kDvIp z?%A|jT--c9AD$oW2iW@MzwtNT{@%ZvPvJAS+Izd_5QC$`)5XPQB^g$UuKO{u+4Pv= z-sYwC?|2Kb-)2e%=;91V&Y<~IjmHT%dng*VZM?_O$qk*6Xk*G?JvA(;1^ztV@ z`@`4XxO(IA2OmCQ+3f0O=huJp8^7|;{$-hf7EMZE2tvO7``@0<`puSOS&q(9!(YUv zM@1S}h+qxN5>m|oV(;FY04ZuO__Z*D0#!moy3{ZQh+5h#iqHCkv*V+eUw+vz3|{31 zq+BQt)o?sL?GH#g>-O3Lh~X?ISTq|hRv=3AX6Pf=cMU@`4C5Dm;TM&}2w)u}&oQ$s zH6fykIa;A<5<&)`93PJeC^eJ`UX=M9@La3bdbLYwQl_bM`;A@4rIYyH5r$j*PLhl(a~!Lh_0-5!kSa&@jRM329q!fyXeW|gY}JNF5>h4K~U3v;-_EcEEg(i zcFcy~F@TM^{pzbf{u}@Ln^LW?k_b*F5i=ZRTa^LEAKp0Khoifh3$$Tq(&vEv?OYF zdr#fC?hx?xA3Z2(`1tgZg1e2T91c^H)#CWxK`pm^wzI!|W$y1HI_K`s}UF zX6t|X)n9${t;a=y-g)Eg#YS*8%z{R{6r5|GG6P_TFTMCB*Qo=V>>TWGUfA+%OQxge zZd_2Hg4!&eoWAwl4;?C+f!k=d`~9>m0Lvu?1Oa&E##2Rw4I2mkc#s?hORaG!Cewad zWwvJ=?Cv1|P19`F{Tnx~KuNN+EQ$hSx_{n({)HEoJBmS^H8>?!o-T2Rb?q~n` zzxf@N%9e9B%8?r`{pN3fjpfNRH=YjcM#Jl&_NHmrd+&bh>XKuD#D*EiLZ^@?^MCNS zfB9Gc=fCnl{NMhf?|Oz|=!ELJ{&5~Sn~Vv@omRKA(7XNqd&g&|*wo`R$;tvNvCwVX zmNhrcJEy0kys!*|S0%yFKtMN)jHL)N5twE^8Q*_&hajM7($txWS<5vHmo6?VukPuV z(r|I<=F13K6sMpX8*qw1!qq6tPLxQXs#uy}P+A7BI7wm_PqW!$Z}1AOytsnpOqq7u-n_K0aeq$ZyrTcbM+!| zr_G?bwA`%wUa#Ab1S!i;`Or7$lP5c77#IdH6uEq9VRh9LviRoLUN;=Ol4-&5>RNYp z8ZRy`bh_>H;eb=?!Ttf#e5C0JOQq88vIQ7$b-mWzYA-Cd>+t&3XMgy^(bnb7mtOs1 zk!9Foag$JUM_0 z*!2es3-yNQL9TuG^@nb2fqGHRhn{)X^|ktfrrG%3=}=L(=8E%^^Jcg>PUdgD^T6{P zyC;Vor>EKOJn4S+$3CMJv^CVIU)pNB5}!(Y>(_ttNA&<|Wcun$+kspB($8%`G5ec; z{il=2J=h!k+5h5`dwYG;WQK=MtMQ-z#hdr{PaMAxY;ygn-ori74!shk(Xg*6t={Va zLp|)Py8~`G&YhhDQ+M<#y0~$n+34+m^lb&X?daplxq%!)NUOP2K=NoeDq(OiOb5wn zoN%}8AR~F|#cj>jYpr@#=D|Y!|MIu~X}I|UfzHWEbUrO%p1ZoL*E$-7faQ*b9S;+N zYKzUY(@~se0IH(m(=^wp@uN3BeE9fen8Z-muWc;+<-hhfj;1sp92zi3WaahOU;hXH z{BK|A)a_bLu-xl36sX*a$7e_X@>l-PU-;bff9+5HOdL%Og%U0uc(%Q=vC&!FsC&o5 zbDk6m>o&@-`TX-wuYe-Yvl8(9;@0L%Pv08$CqH`qM}i>7v*&4AWck$#m%3gfbZY&< zv_GEoYPG5qi7F*ll+4TvV5#tg@VaE9*c?sK{3>P&#u>82)L;!gGvJnlbArx9RE-^ zr~*Pt?mB>i${_%nqLf+Avoeo41H)lz8V)H%%^hP}1;MSg8#h}^x6bx&gXDZPIDF~3 z?dfDSl9g348=5xm zH~lCpt%a?p47d9S-@UDr(nx;%g4!rG9GTVoiF^tmv$caIqOR)h}HcE|M*|b zqUh%3ZEA^guTB8;E zo@v{*VREHGh)2^5@w^-ik}rPYk5;-qO{pk2R-h;f%4Av=y``T(2FRnSlo?=qUYH=L zAOKmKW^tmV)CjRnU8j(hMH*+UP^?NZt_Z1o4=I@#23IPFOsEVZPzZIPVV6?u`z|K1 zsH9=iM#Hu=+^W@dq%~?yjH&KeAV*XKBwTEEf6;DlQS-+=JYw}Q~^b{6s!U=Vz8FI`w#?sl51Ywb$KwR_SHgU-Un>ZL`U^fuS7Q!<~= zXI_Bsz8CYn^DBSn*V9=2+CTf9f(M7k$<-^{8u5fMPx@JsnXteZGXz+-9tzpJ{qD(? zYfle{?BpcIbXFB8j)Hj6;*hu?kttN-e^FJIew@#cC)Hh=Gp52xzU_NAV!C!c)z`YawC@7{ZQdqsmHo2O3> z$4A5T&d$ML5?#IZa?XtlmoESIzx(yqU;q8h3)`kw-#fVDTgkI4Oe<{SkKCF=e#@To}Qi@G@8*UAR&~9GQk*ZUcffC2)40}1AYZYAR_}p zU?Y$ON=PG3j@{Ea?(Ut_+I!{rJ2fIQ1uS)r{6jXgSc-v^`S8^y}+eLYY@5+0Y5C2XBw^8 zh!fj&C-6ZC5QF*Hij<(=-up|@Z8^40NC-fSlS2wf7$S(|7%;Zu!5Evm6?h?t5Qk+J zs~isq6Jg?8PLD#H0-Hu&?8XY0;&`B?s6-|dMG3=@<2WUeAsh>EByl1GSWQY|#Jaxc zd5&*cl2ktS)E9<>TcWVk-g!5Sn*;}OErQp$BL1UWmz?C4@Hn8aY~cI z$R3+as2Wo$=N|v?FCxrtZQKVaxxUkfBn@E%A!J#8U<|?tq|&)YbNBL9D( z6Ep&kKlmVu>asu};{d{lni>R5K0Oauwx|1@VJ($KGZ!sHAh8#4q2eF)S!g1VkDO;0PDd{(i5J@eUtGLnoBgtP>4Q&(fV?5c!@5KmDl> ze*8y1T&r%+OpIQA_Z^Poe5CE~nD>@$9z2?`ypHcnK?n@bd;GbRm#$sE@YZz(vnNkY zBTUAdh%pwdUw)#n2Y3&Vs-M@ch?W0GfsVus`TVrE=v|KQp=<5Boy{?`qP5$jS-+1G~or@RV z1Z*S7%xPh6NyBGz}r{fBECoB(DO}0 z*OSR4!noaTSNC=yq!EgJ-=7_uX2e9b)wA5FQdUL^oR$EHu&CYIW|+8G(1eT%LxK<@ zUUdGc4|u_V3|(^=1{Mg-y1)Y*F*uH46wx@QF&hHM^@k8*3ZfWLfE^XoY|r)Eu5L;a z4Sf65sl%yEw7z!Fw#2{qvme&$j%$ReOq?rD#?;GZl2>lsskgiUbS_`Jn#wB|ZZ9L8 z5Y+sIcb1AdO%~*~?hj*}P9{`2GJ7Z@{K_Ws;>N-Ge7@lCuWcHZ8QP_%9)6T1)a|!$ z$m)*!Ez(%P;nU#vZp{FwyfgiEG&7=%gmfv)puH$Wv zm4qM&7w1xzA7iLywUpV6e*T|6bM?|)=$P{p<0+*KlEu>0`ODYW_G;C~o;-2)-qLu% z8=aV6A9xg%FTMG~NY0+0J@wK%yG%A%Tdo6^d+70lw^vHP^6S5Q{*ef~9a##VdSAx$ zT_s2IqvfUT^!%A=jdK;Y==*o=p#cToweg#;-PzZ}Tq?QW^6szI>OJU%91QEWxu46A zzVPBD$Mq8$`p}ai_STzShv|0ffE_DZ_sL%m4W87v9*n$iMvee~^;#bSnAD4?pzuv(HV9FQhXW z8q#vMC`e+p(W(tCOexP3K@&!^IY~|Ww&Qi(q$E9YYzCb?{U9N%z%W@R5N)g3wt)q^ zjqZ)B*GxmN?e*iBC>ep1B7`GCScCyaA}@}~Qi=y`?1RV)S-dVgrp5GoR*C6O$#&HCQ>L`qGfWLn?^P%39c857o5 zcU=7bhzUZ7A{^}Q?wO|9>)7Lk^4Guix4En;OMDc&o^Pg- z`D)dXl$@GO^@pCS6*x&49hvfdAEGFo$((=c!2@THU%TDFtn%Okr?Oc^ky(y&86lXP zdeC;a>#b&Q=ni$#uj^SgCT1XW$zH?l8}_Xm?{?dC@$~smec};@LxWH_FkAfX|M+u? zV2g79@}-;mn=Ar2k>cevQ(bzoEM``Bq@|^ILr-_~*2p-UQYp(vmMi@IcQ0PLu`_?* zD8qsfz^6a-_&@yP8{>;)X7S+iozH zdF#gfSjmW6h!K5XKX*EP>eQirU+`&iboAVjgEJ==`G52O{LI(B`rv{D<7NsR8xX5kOC@9M8#@jC?piaUWfu<} zSl?VJWYx{}_UhJ76bZv&ps0za3ial{~kV@R~|Lo=ry<ZlktEu%${| zyRj$m!N$fq&q>3+TdVi(-M+WAw8cX|aEzOGZr#4N#j}d4@COfydyNhwmNb>iWw_AO zMMf#+V-}kT=m`xt9?p-9)vMO9KfHJUP81nKy?y)Mt?z#0`SZesXGd`fva4+f43K1ib2I z#8}GlGfuXw7=8ZC+1!Z-Mj2`^%oL6tU5pt2&=`|Rk?LOOwbx%C86T4~xmL$ZYKcax zHPDTX&4z1%&7B6HNI_mi2(SXd7{5oaB!aS1t+N8t*Mq>bW~XKlj9?rv3=`tOi!9x2 z8X*Y7^#1P7sZ*yQOZxqNC5e*hNf&rA^_l$SpM2r-i$zHglPV%I9Hv#0&TAZofAqCi zU;E)(rHKi10ax!?TltR_+r*jDHVG~{_}=!S&YmgD<~VIhkkg@9qz7&0v707{2J5^@anu*7CD zN^z`AP~-)HA5zzYAVP=*h-VoR0w%Cxg8?)tvLrB(fDy}K%o2`)0mB9YE65barrp9& z?$&xi&w!pQ_%u;ClPVsjtS2kScK7c}xj{-hkP^r3fk^__s)wN$LqbB}=)IxYAIgFn z`$0m)DG3r7@C+>%6D*1u2uc|r%l*}x_l``=2EBGVMI^068M{21RnzLg8bl~sTe&<^ zNhkB!z-gX1H22w`{Uj?Ztwu$en=4LvFH35#4(xz@eE+SC$Tsl6Le%~Vx zojW`3!7z%Ts;Wx1;8WxUG>8U$qk8_qX%U7LcsmVI%cU|YuT!&K-LZ8cR{{b7Ug$jZ z;Bn4U`rRfkx&6NLr7!=v<>;8T77p0G;r8kCM{9NL0`ts+2VK89Xf{+{fMLmZ@#d~6 zik0iHd*DD)Q%FFerdGzMG9UZIM=oDk?)T{z zzx+yf;EaurGYn2BP-ED3r+0s)S03RGj`O0^_}BmH*}Z-L+4nvD`o-5A8sEC|6$YaZ zeeBo%!{^>O_Y9w(H~^XFmbQYO+KuNwskN72o;Al8^!rQE`u?&qR`JmA;_J5;PN@0G zL%Oajqb3JZYisoWW+3N_8=d;l9C%zhl5!MNiiBpzXzuj@Kw*Gd9^bJ8RaE-EZS(_0 z{f&uo>Ddo{I7Tq?d|6cf;wyhE zF;PUM#i_#Y{PK_gzyIxbo;x$Xx4otCNK%S_`ju~e@`E4f?k^_(H9gqTnYTs6VjZZDb43W|zJoXF)2!{InC@|^LR$xIeY zBFW9>l4b428@mZ!YSsFCwd%;oNDz3BJ@&}by*r8wSVnHO`%7=%=v(g0^!)1TiY#R3 z=BIDpS{fUjz}S@~NstpR@FpfJ%XhX@sgdQ?dZTtDl?l%rKeV&i@V$uA_|}azNm6*0 z87rz=tHXBBup&D%b^;?=P0-4WK01}SdHr5Gm(Qlgx2xMXn{Pp(qos1PsB-X&VTuX!lblxm38ga1IL+i zWezxVH!tpSYCT&Rzwn){{G9jMpZ>sq{m-u-d!+d4clWcyX#wv%`&eatIlW_F%aycN zAKkcZ6UGpDMQ=7d$IWP}6GV<5Nt_V64#t?q4n)XwktlFO%dB+xMmBnZOx?uHlm!=C+t4?gFGk#BSXf-!&yhp9|Cqa^NbEU&Jt zO-xK61TDi%rL|Ve9qJbG9X)_y;QD~zX5grZBMF{i(@|iCgfc*cjAq+@7$Sj7K~4!F zA4Fkb4;6-Wqrh_lR#2QUrid~O12OSY#9%g#pydS^5|%+xWHFetdL}?RFQ70$1Yw?^ zq=2{0nmw$Oh~dGQ?nH}+PMgXY)D+fi19@a%37p&(73>A#%Dr_M+AsP@k&}|JI107nJ+N;*X$dg5-uMb?) zviG*{m*bp5VeHnOjhU&ZOv9a;PTszGE1Q~{KbW~N9RAwB z{K&Vy_4fi-jAJM(u-mo0AmIjqmgH0E^bcQMiu~q@6Js>STh*n<9(iW3U5h}M;(p*J zgU~HjDi^Qr40?L6)kHLqlHBHQsHjf9ls`CA0kL&;>7Es6>zgLPL}1H21SiipLEN^& zm9jth;kDDJr<=`{nUPP_n!Wow8@Um`KNzjG zdt)=5Jr{oY^EdwJPd;+zGJI3~hsnjIZ~cRH;Mn>1J%zsaR;97i0SZW_rgeR9Xvc0u z7+!7YF+nJyC{gIKEZf+xja5_@1KYI$#bLy{Kv>;y0YyuViyW1iAo771_-JILyx&@? z_iQ)9KYa0ZUP;H1{bN7&>4D*E#mvs`-ixojb!2{&g=|VmOrJb-^TsRr?D+0Zt&l55 zGz5t8V(LW!FA%fa@*sf_UcPqOvpt4kEXz_PE)2sciZI4u6h$Fu)a-V%!Eu}@2nz@1 zUC;AuhvjiZL0=#8To6SR#;%r1bh~{5LW&~9jVmeJ_W@-wqd8IJSuQ7XFm&8-hq`VAJQPwt z;6u#1A;4T4)yR&+w)z%A|vbMHrnnuqV z*p6abL9&n{Y#JisxxGY!FBgy1>UGz3&zw1R>(;G&G3Odx5pqEsD9LPT%p0kU?$y?( zCn}!b`0OwI(%Y|JtktSxtPJc z-U`FQ$DTN}UtOtf)H3RHzuno|SUY-j)_2&*Lz!esQc_DR*BLwpk!U)tnT6TQ@7#wd z!SS)E(M zq4(ET-<&>??wB{e|AJb|rw%2>No9iQ4mLO%|!M^R&z_UyX#-6AuVHlN5C5C|l zk9^OLBP0rp<@sB?O$H;57{yYS#>n@Wh8eEEeIb#{-CJuyRz18hec{b_=cdwq%QxD3 z%(*YW{?^QRiU$(ralO$sXhV`x9%aN-hUa-0#SlS+2=r`-u$su+xOAo8?}njIKp6U7 zE|YL=>%hXSq9`24wOWn4cW`eG~Bn2cg-vfh_!Z@ z7S?;M9j@QC`-9%FX7oFzlkx*5Ba_$Ox~cno&-KSkspmd$yj#7h2__({SOBV;&lTp8 zEJuKh=S%DReLx9e9n*`>M%>ug zgkg8HxzgI*RoC_e9)&PKQi@}xmDMYs{Mg4|_~r%2ccQRSoJ;kqy;JATuC3NsPF6Y8 z=o-D<{_zJE)A+GhR_?^^1vOSj7qaV{JG~ATRDWVB({HNPz5BVeytlV~?DXRDy`8fU zE?&B@gk8YFsP4KCo_l6@XVB^2PNw9I^}TdL)&~wJI@`OrST6TFg5JOTz{5{E{*Afm z6OyQX{hODQSvoOU>Fy1C9ZwXPL}H~_JkcFEt${9Zd@iE}u2--38qK&|7S>jGe&iD; z2Ky_IKJPE^; zvAnsr^xOaLx4!V-zt*<*fWNNmnZ)=w-E2!^t(lpJTYI%*=Swet(>s37-q<45`ugc3 z#Y7_U)(@{UIG$Zhe)aD<&pncc9IOw$H(!>Lqg6=Yoi)frIHW-sxe%~gQt^W*rKA`f z@)C-}h=;H-)H#7Ih|y@Ff*r?*(RRmKm?|udGdizfA*)o{FT3d^R3rQ^9vG{edv23T(9jn_SP)jNT@Q3 zyc8i2Mv){b-%-@8i^PMh=BbJS!k!;c+rvI|m`DT+rZ5h}0Eb~<#8IrU{E-l+6H1Cf zOy3wni=-qzc_@XizQP{0F_x8N~9_kHU%dtf{;-l^}kItpinovki z&5qo-d3SPRtk&*ms>TV$et$5(INIG`;W8N?vu3vo;z&^x6vTJ$-+lC%XLeV1fEPf9 z9T+;L)EN4eN~PAUTDE_9ajd#qR}*3)t=T?ZU)q2qID6*YjT`Hbnvow7Q5-Af);D(^ zee{_t*Dqn{#t0Byox;eWUbmshWNISU8+1>ee%~uE{Q$&* z|D(yU(QoE+sZ$T6zW#TELWT7Q zdlOUVqNw)n>RVoQ^5=f$JS2O+{~xbSE|f-Q*+x~pba5A|F$+!Ih(ex(Nv-(<66Qx8@cOT)%m~~67mSkAsEV>c>Bh?*+g<^2glBws`XnemHe3AyMKSb-9*$Oh&g)XQ0RLs zV3)TyrpG2QPpIv&3E8$?Q4|G1um&c_F{&&?5O3}7Z13#YwhJMgnVA_5`~803wrz+) zk>!5vCmwkC>|t3-FZadY`)^+qrIa7JG>o=(cTS%^9Yqubba`tF#gR`i4SY`!g!hPu zf{?yxc(*%2%Vyj-_9E7aI5juMr{;=t=cMvUmzmfeG6pC`>`ZDV-~81?(EBn;*-rbu_;@b z%=rwRIWi*_b4X+og@jf}4^T+43yg5IvH-CRIZ0M>M-QHT>w9mg3e1$W!LU`R$dGs= z6UosDEt^vhADvoQm?a$R$59Xm@4oYLRuQvWW_5Ly5l}umnJs1z)VH@cr^YAVe&gjz zX@=u79LM##Yljbf@`M9AqYjD7dd%maz<5zk+Jc5tKaVqvbj_ow+4DQpU)?h^1|Xw ztG%O2D6M8BCOI)N5mUdX4|Pv%bsN)D#csDPh(r+4{rhXG#(n>Vzn_^a96B;iX=M79 z7?vg`k0lc$^=8LMiM>X%*Bc~NM$?$1$LGHEd%qh{!?2A~r8v7NNNihstRlQLkJ`r<8=qVUcGcMudc! zf@%f^AtHcMVDguEE1jc7# zoB9LaZik-H9{9Q`1W^(o$uix*g?@+wCq^NNUCcl+j<8{~NJu+Sb$pM4LF9(m^YUq) zf~v8%!MKXR2mnVk_5;TfIehqF>X|2s$Brc)ey}oCA}sA(zrFU~|KP9gtlF2CjPdD5 z1}>^@G+R}(-!oNJy>b0^uDF=W9ZZjx2gCND)z0wR-p;1u*rj}OdSTAR!Lc(3dxPHm z!a>6}2m?e>u2%QI_01pn9%ES8v#j&yPpSznq`t(HL33ky?Jnl%k%Kd5A3W>0*4S9y zFwFgy?Rqeb_P_J(|Gj+6^TT06p)iU8P;Ke4#+NVM z(g)FVA9>%qSFX!ye%K$d0#~dQ3q>K9O>XUOVVE`6}!(q8ttnM{RnN&oPq^O-<`|IEM>VcUF zQBbYzw=RC~Pk;5F{Mf_C4~QHZ_VodbY-cEmTpXYmUwy5wcPWi#$4jMbo&hKdL(lUF z%Rr1)H`YT!LPB5+IgaBv?lYhH3;}d>bTYz1x{%A{vOLe9IB^0Ygb>=?+;m+xkw{1^ z&x`nX{`dc8mYi?1k}fEfQ{pL{lk7AK~UoOvu#fN83iuU4Vkx9Va z*4v$i4P!SBDFxoOt9PgKLUr)EC{~Ka2VVcqKl)}2khHN%pFRacC;aN)zmQP+xiWkB z{4aUz_9uSqv0K;WR3UNd$tgFEuV2j1&X4@^Po0`QmZmNb8haC?*5YIoa*7>TS~k@k zYEh?V{Ez?ci(d{<$cn1WGlXEk#1e~HUKk9AxpIDIeFc8<>9dVNN0Ws(j5J9MU_?VF zib)t@3gf`H{lNeP^ys;xBc(B=Fp(=xNC{aGC<|CGj{CZY0j9CThy(+VBN6oa1Ui8~ zh`a_y1`XUWHaJEDkxICj04S%+j?WM*AVgI`MQk#lf@Snv*Cvq(<4xB~0CoUk!yXR3 zj_rCzq`TTE7Y4Mc2LkHa4nahUBJ8_N7&DF=T5dFSBFBWL>liMto7DFC5V{~Xe*b^| zPBQP-_isG>_>aEx>fc13B@3B!x-4L9Kp*jHLdkTt*HTIJz3;v7=tB>y$->6YzTNF> ziL{aw{m8s~uR1YWtha{wT*Wbhh;^nW#@>AWjbwrl90fjP*~Hjnd2MSsCyv)@d+8!h zrawr<#KD>Jh3|a*;Ns-fYb!HnPY!fkDIKD8e`n<#NmSgh+v)TtX5{bx!<~i2QnwKl z#+1F?P7L%?N$WOs$LUi*HaEA*rIA*%e)RAYJG+BUe@{^PWJ-n(YIpnNbBjA0x6YhC z{pve!Ln=cm5)rtjHNTME*cym@nk7Kj>jxIfhzU3L0ziG&_E4O+?b^{}BlUXQb-Id} zR5fK{ird~Y6gjuHycM`uRI`oxCSYKBWW3kd^+E6bkDa`E<9?}9eEZtonWK-~xbkYp zYkuna554sAbq@Qk!%ohQwgv-Po!r>G&+yEz{M5NGe&wy)Nb#lbznaTWr!wHF$3FAo zYhS?-gb1v!*rm+Ge*M*>N9Xp|M5vP9PIvYc=?x|>y!A#>VkgU$UU#@NfH@^aaW9ds z)b{sBi)92Y90!T4bmQK7x37t+pxe|py3arNd_ckG>h{{sPAMZNv^4SEZ+-D!H1`@d zjUzVwv;Xq**+PO7la}H75MIA_EsBg!{n*EsmRESd#0U{WeBUR8K!Om3kfI2I-f$4w zZmrpfz2M{nXCT14+uP%#V?BMyawH7>w3cjjd;7IUv(=$-m`WxNA3oUEhbkvP$OoYn zM3Cd8$kzi91YrmRFtoi|ZSU}*Ll{6r1Dwz1EK?64^&yRdFrYC&kYm^=^cjxD87(m~ zzO_}odimOZ_0F)nqWA6@M%}l&z#TGC&l>LAX2UghW2YXNd)9Ej-`TVVyN=Zh95ZxX z2CxLW2=>Cr^8wp1yiVWj4(y(8H#+@#W4GN{34$sOm*VJF6b38dx3;SXjvU?I-H~L~7}QkJ@b&)c?K>$!&KGl-0FFey*{E;rrnp=`yn%&@>i+rt z-PKAd6VY*?p*L<{eEg$NwOZ)V@h6h`qgXu1DKmmLY53fMW2bt$v3RHy1d*y?-DnYp ziV{yaaP9VmBgYP57A_n(lF5x|DQ&;GjTpaJES^4dWPR&~q)L7eSGQLXX0P8|^8C*6 zqf?mLo~0{ltR<+e_ZDaKkhUbTGtf6#%=X(3jG2TY0MHR7xVzioMdpe3yNV# z*tY71@!X+e*8=T!r@3F78Xx()uU%17W1X6L;P@dTbAm9vzWuTyYe$bg+U~}Z$OGZl zNapd~ZTkFA99J^#+#&AjRd;jy?l1rIPckBNZ+)4O6NC?95cj&l)@~D0w!XXXcwrP# zRTPe$Jd{Yw`9j$o>KG!3kn1}WGqaZ2OC&Xcuu>}VbN~D|vm@hS%=InC_kg4(ywDfa z)YU7uF!eQ6ymsx{+Uf>`2tteyVp%qdqQDI~h7UcT0Yu>0d?9u8(7}U?^Swd$>b1+6 zOsd!Gh_VC`iUD!x6MW%}wZafTXp5LIM8x6c5Gzs)&iLy-K6UHzd2!-1IUP4Q1>6D?< zC``3`%@~MXBaXw!36bSQK}g*I*fz4Mk9^KB!64LgvhwLqJ&-FK5b!5Y%$bpIHabS% zIe6^E?%oE^G7Kg=_0@bnW9l&hWTcV;*j!r<`i396eaj6vCM*^-C6V+TgJp2F-Z*@8 zW@mSIYIL@`*WKIe~;_Gkz--PJ5t8CA_@WAPX<$G0#qIRz-Fv+P|Wo@+)1`q;9QR8a0&GRWm%t6i{ z8B1;6Z?3FVS;SI?#XQvetxO_=X&fTQbja95?#iXRpZzC4xwg7=@9w*2&Y!`t{KAWO zN6MM8vHaG?^5aiG+8x+eu3fE^Dz4>*7Mm}G`!$EgZX%&Np*=cQY4qIvy;dfZbb4O7 zh_2t=IDPtHeXsuHvmbozJKx=J^mF-2M41z3a~EIRuGV*CMG+LSvAG|`{^*=0#?za_ zfxq88`Q+l#%D&rGjKS-pqYss4$2WKPBUewW+0hBSx#2Y1gL=&zE2nQQZx!=$I)V0k z5sR?p0o!x*mY+-}1U}KI?Pk&{2oLiAkO4--q5y#fMpTOvbJpp zzE24DER!J|5LiTE82AjrE~X+Yv>Q!Xl4y*1o`r};e!%i#t%i?3bWyjR^>-|lQ~SpEIMVB2U{>$^+FuoXnQ z9}h8dT({|&9t{~k1hyMCo5LuIEK`qxg?uM4dV$~bVjDO<299m-c_#LQ$siiH?1AC* z-LBnlJGx;r)K5xrOou)Uc;shtd{S{HCo_qZ8e_t6CS|&EArr!YAdCqs3WN}3T8`&= z07M*MlmzB@Hzx~WVE)mczVW$#zar<(Y-9bkuuUxoVnJ-H8z0T{bSge&wO_N7CBN0b+0xcA?+4f!=;+u$Jb-j1|7^?2>GHS53+ZePh8hg$C(4{`*a;mUUz1P+fnXiBS zHL7NPmK;8wXCzwAjzMSeBhQ^xW%P-UJZ0$q`6mifQ>fAF$$~gKQLL{FQmRh`45_)>mABAIpu90_cmGe*aX_5I=G ztcI}2b3Q8xANuIW5*%^#;k9e;VB~6|EFdD22OnBEe)aC=o$9`l z9UJOhhVYq8B@Rr}=pR0FnDCr13`A>f%67qvV;JRU9 zWXyvoaHDRwo08Qy2qBJb*FqRGMBoITl^BMn0?$fB^8IjiYnxI?sW71N`u!_YvlY{3 z0QQeg7R_OoVlT5G#Yq}AoJ_Ir0g?7JXh8AGN z5siQH`~xndeEpk$H#;+4ESLNkz<|bpCl(P!4v?f939@tlyd3ZyuWQIcV@fNPT= z?sof%h$1gJ-87z4)-7zU(LnqhgC9XXB@#f&6LEGN=9V1~9Ac>m&`{-+rB z{_M+tz?QZH&!3)}1c$3Ny*Iu1nA!gOx!L*sU5aA3xw1|uK6LEB=I%ym;vg%ln=9)v z8z}`YUC!wCV6Q&dukVc%=VO9vTRXPyMIpU;%jd+z&dyGE(95KYj54V8swW?u>Nf{S z+VlL-^P8^UH}!B~VIlCvT(*4e!e4GIZ3lYYFjh{TDlgw%>NM=h@k4w#BA}*asxSmlcWHvU}lF2;J^Mnwiv0WTR z3<=LY@JKP8zH+(w>KorJXAeufFznf$AFZtIZf$Kn{M1JTIhdY1dgorVI5K)~X`f|@ z%JOF)IB)sRm78~my(kE6-*>gFHB}j1UhWK<0~kai%)Ik@l_<2@-5Xy#Ht=eyrrp_I zouH5J$T zcD|T5I@aM6vOSpJ-yXDf7Jaq(z~fVUyIj9J*x6hu&m}!K$@Be8A-Qr(fBCg<)SEra z3IaeAX^EG_TC1rllt7qHCT$>cVMr9wVxrljMp&F0jKi*JBr+-L@D9`GMg08JzxYRg@)!U3g9`+@rZwbvczcpdvMh)Y1dO8PO(W^*8&*q0?;EYqbW32vKn4q?0TR!}zSp&oPf>NhcJBPt z&;R@f+MR2M7fwwq)1r`xdsw7>P)eR3oHj|Y$*E;7Pdf&TmzcahIu(`E{ncVpJ zyxy--Xa_-rDB|Vv^(*y+wB}hxMrA{mbzOgUeDTV)*A5*&+%cPXudIdcF*&T ziMi?Bjk@CpiwBQ18jWtV6_fDb?72HP?!?dm0EA&++fF*2ZZ?~FNsZZ-8%q=xXw;aU zt*mVBmU0!>iYLbB-@fohu`=bkPPT|W9fW}1xmO=4k7g6_(I=+@&$xDV5EEy8x5@MJ z$3FU@Z~Wc23%Rk$k<`0aE;9%_PGFfn&u1%>xqHhy`Fvhb_nLdLBGNd*EJqoWZTrCCld%=BEFp+tBe9JOJ zZ@ad~V&YhKI+e<#6kgIm;4NmD&;HDZ1%B?A|J{Ew+(?$Vd+Y0zM(Iov{^-L8hn_t( zK2fh%SuE~#ItfKmWClaV^$gz+AAj_@3-4a!RTK_wfq|!<{N!#Q9-37C;cI_w>FvOG z3b~2v%bPLA)DI6Im_B)MBA*sfO26vxU6K``zYN?1e1P@{-6*dP)#_rXB`ENg! zNp*|K)S%&Zjr*90h1@t{;K)e%-xx$Uh zH+?706=a@^)9F-%A%uv5cz@xdQ|_tKSxg#%YF-4=KWiNNl#C(e%T zY_)GKt$yVBXTSfC-z!&&%l9{DW*1z?Hw-J468HD_2ZKQ+UyM*^a^^H;ldHd8m{S*zq<;9qm6N9*zjOCiu_SX!_S)qe zs+6-$+GuSo%oSLUh=PVtytQ8s1~8K=-Mn>IN|1bRzP`O|d)1MVk)A&3dRuLaRA z0b#4zkJv<3j!Kh8bBf1CIt_ zLgKG)wvA5Dak_+!AYcMNCWJ&0^?jFRSqLFORN;hNu|QZR48ygxwS|RwLdnGFY_+!S zh2HJEw?s)%Wo6LsA33n-xGpP7!)Aj45f+3-r#m__HZ(1UaJo4h8Utj7CgIqXy>%}V zbzM(tnW65tI$pLsAqdIQ%FLB_FLhnlwCtF`fK`{5_jXtR;m`j34+p&_4CCMU*Z*eq z{<1@HWh9Ha;LiQkKl#gVzPoO1^syI{LC=Z}x2pp^D2d6FrOC5m=Aes520C3?uP)zT zZT7coyIW?h-`Y3$2(vUrSY>3Ym>v<;B9A#3P&=0|IXX~ykz<@FY@sm53`d#qz5&O| zvoK0qeM9DjgvzpnV^~5Mh;T>=B{EA1(0CP4YS~5{G7)4*MB!!ezWMr_^}XIeAC4Y6U2W(mXU=WjUxrc89}Z+Wl}VS~4wRA< z@j}P+P8>hEbZaY}5@lX1WXjZOIfm)jexX>Qlo=Trp{lQ?7KZHs@OQh-p}?x59qjj>ee&$t2ad(a z6!?THB|{sT=57!>qoreKpPm|D;AajeQwhy+Ofe}RKl5YNUU=`u{bW(Qv)b{)|^4)kUzlNAX3&ENRf zC(fVAWKzc_mD@M3$B+h=!zWT+;3He-)QawdbVkO6wLJgz3zwlE0T!vF2YT%W1UAQU z9HU~y7IMXON>gMG5{ek$M{XR0wrT9vTTF~)Mm#)skc1+LX|;AQp~yUnHB}8mpRhOr z(OzwjV>K8eDOJu?Dgbh`6I52wFGgf`Db1VN5qj*t{D zBp6m>S%f17M+k-?^};Z8f{5@Ugb;qur`hDq~|~jKnBfqEM+=y4Tq4<K5%f}c8p@NI5skl8PfNAqm{%5pMOx1{Fo;1 zukCE_Z%Z2Ur+@YrB0s;jw$X2sum1JR*Y00vbh?|Hw}6MV1U_=|*nYh_H+!DJNr^S? zUfx}~*S3xD;~)KSKFycPejGa<1)$k%`mUp; zHAg1#@XEFowx=m{L+rPQs}&2N7H!i%r3uI;vNu57N?%+<~6 zpl{ly6S#HixA<6xIHJTK_$`Q$Zo7$m3a}wXRutRsVTlDra$O1`q?AHRaP0S5TbACj z4b!!8tC4N>qH1%$+wF2Bpiv*ufJPnx6bJxt6pDx!DMG%?vx>?p0uLaJV&Yl>fw5;h zzU#r5W|UNd&;RHD{8xYZw>PJbO%H)%-L%R(>sV4U8wccRRt2J0R zhzz|wY}VR!54dl<_@+IyB~5+n!ZlS=@7=kBLfiF*$4-xXPO83-0za86CA=6OJ$*D+ zs>Ik!WR+w>-dw#`-`NcMZf~!?etYTJXC9}Kef;Rb(B19Yxhl0x9naN__l41jE=1>Z+oF@dFJ}6p3UVl zMYyzd-w(`Ar?t7<@*VNe;j?eQdHeRQ6@sO^H;w9Ugi$7uNdtn{c5m0}OXFh|K>b2x z&askf_p6JuGY1Zi{_y1&7LJsc?yhWZEGILAgs7f6zQE$-?b{bhMb36Dd$@Dz@Mvk& zKYo0Y04AiKZw37>$Fr&3-R=MR#V;XJ78K3%EnZ01>xN;uGMma4vQNMNv@F3#9zB;U z7yrxeect!Nb7ziUy?BX%aVc8>NCc3SDrIBX^BmhYT+g!i_xB;j5Q2WcO9;y|EFwfy zlmF`9{wKnvhaq!$c|VRIpb_A>TX%2E99dYLi$g#3{Bov2HY z5ruvjhM7#J*XwZ{rzjFa=)G#8NLV)VvziP-5O`j_)%3zZx6Gl@6J@2>w|@I`|EgB2 z0W2bb1x{2XHHax7F=jEOE~S(WhZWo~>zlemP1Ce(JBlLs%&A!z#gYiz7?egA%GvB2 zm);IxTv0WjIy3^lCrWdkA z>8M)Es&7P*6^5bfy1p5DeynAr(2I(Ne0xYkoH4xQpzY|LkY$OQ1gr>UA#S%C2o#+F zxqf5>E@ouIblaUihO*~^f$g!B#tgR1I0j4%BTNxu8PD@#5UH{<^lZZlc_gN$wm$Xz zlaHQX*xYO+Q)GW@BXI2U#Mt!o?9E$Ok^&dR0mi5@d8plN-M)A&lgSlF$8jRdz{nVM zY||JFb|p<@`T>{jUX07@wHQrWBPI*OoWxwO+YW(2XXI zNwHXNG#aPQEH*m*p5^*OZ)<0}GBGpk404%Vb!Yd;;rS~U?`U!s@!`^)?Xj7nWg8-& zbZpC~IF(3m-@EHYFjua)p_9&vrE(??LQO4iZ*OdD+%1hy#-4ZT@ZrDt`ilrlvY@y@ zqfjUW9#0WBXl);VU}|qiuhp6#`N)S?*X}f1)y57936si+CytNp?CjjTv-;s5`N;ap z?ntS)ym!5|AN<5mJ@hwUTl(;`2UhOgH#|PWdRU-KOO4sXNAKRML89BHU^$^r-JCqy z)oX*64sgiJnk*$7t&XB`q08A;S62Llm=9^ZyH`DM{A{h-U=a1a;ei7S%lE1*3j`ri zZ`B8lt;ZjJLXVsmet2tXtvxk0mCb01oZ8-M*kt2M^OYg z@%1;~#=dp#p;H2rj^o&O4T>3A6Pk_obY=R5*WL^Q10hr@mG)~jQ4|qEiY$a-6h$Gl4JfKs_n&_HnOj%iIDhVetM|6HH`d?dMHrSn zIaTC1kp&Qh!KWUZIKD8wucQC?$6pP26+*1*CITE!WE?X>F0CrS9@f_LSxH0y@nO4d zdNcx}luMFaHec&)zw*8B$<(6&aUjeU3$Z9so*PE+>aD9D0G1bFmMdhF!+yv}NWoZy zF|dZGPM;#C-+cYMFI`;P-gY;)n)O<*HRwkG+9rxZ7)kz+=?9VCtnK&h zo^DvW;h3fmu&AZ7otEvyLCg|FOorAnQah8;D-~N}8~y4^(C~))QS4FQGiV%h3|2VE z`ZjCZIawAkMgTA@P_zWgin7YHjL*bwLJ|RhF%t`jH*^;`h{ucNQ<5@2J0)`V>b<3uG@_1VYi13q`0ZD}+gRU>EL0d(uV24eUA-gAv@$D^$eOB5 zY_6<9Y&GpC-5o#M#{9yeXOzP&Mhj8 z6^ZuwCr_-eZ{E9iPa)aN&?LreZ3_@s!+r#ux?97L$Vybw(kx6D$~gv8 z1cI%-jhTa|Mqh) zdF|qxH;21{ENgGQx?HKaFa2<%-LsX1yHYm|FG-|yryKphaAsCgm!j!G3Fpe6354ck(ESJO^6~7WA76`^)DZM z=;^|E(Q$o>uw~mp-~k*>j#rwyySd8v&84-4$#ISoFvelzV+{LULu9xC;8{k>sgehk zoW%U|pZzJ{GAWEP1ccB4!~vyo41D+Vax% zcU~=*vkUWwR7AscLts28iL%5+y`cejZL`e~7DN`HBn)7L{DE$0nnokAxm-=AG(dca z-IONqc?N_cq}i>#t@fTRGL;C#00l9kAPk}aKuEpNqnM{KW-0CWyMPlIgk^#E5rhyD zNMunl2NuQH@>~WY9uZ8@)Wr1v_}y=EVdrzd^So}fQUps}frj4l#_QAZ+@XsZ_x<+7pxF%v9pyp6 zcIC=#Zx}HmqiCb;&UUr2zr3}%aPYvbo6G4`j%T&NwN)jJ2yL`m#d67UoTNIjw7g>J zeP`I7KYBcsO!S&HUXz#BuMX|b1LvQ(cK_=}5b%;nI1v*xEBWO6ADlmTZ2q+$zVxk^-u}Uxo43~Jz2$zp*A48dX*HrSaLmXtIAZ|K zAsQOc55-o;w`|u7{K0S-MG=860z-m=7&sKWvj-NEnZVVnzSW1G8QBBs1)=Xy;6dv9 zj!7d&VsR=VN3lmhgh7~8L;ylnmS7l*icDBWOC{4u4b#{e4g)W;^uY9_U;o$N`pd7} z-&iN>n;YA!doxoo245n&BkVWTMZwXeOouY&Bcq)6YKN zYHdRR>a9V(TzvKe5BZ@!99TS)%je1m<{la?%0KsWKRz`x&dID7G<0h(k-#ixa$MAD zZ4TP|cW>_O?=|uTE@r*HVXm$35}-VE{v=NnLw9l+)dhi+$+lZVNyt`rIzzLi+kG|3 zclv`Uincb^0kT#$b|p^hHM@^J{cye8zjymqG9xXXIJ&-h=aI)wzyA8`goL(j_xk-H z@-fRuA|FCZ2%Aw1D9b~`=Sb%%ydM4scg_l8lw zH;+090#OkKi3bSeld17arCcg%YO*`VkM zu(`1T0V4}y5L%QPK^zg5?Cv%MMK%H;Wycm54#o%%9mlhV0?&w|xVm(6YGi~&SW|e8 z1vv5|2>ZI@hkTt4t>uV_F5>iRY^vXGd+@=9+OLz!Vlki^Q8;7-?-6w^NM}_ zKEKzJn$5V{2pb)^*9cp6)NL{5Q0|+2&tTg_Xc}nPH{u|S!Y~4nZd;boJ8)nwH}3Oz z5W3yivtq|3DCSroV=7~yvN(zY&oos+IEEkqeAk88we1cILmGq}A|Q!obolY`1 zj$=DSF^a{__27@c_!>qzmi32ycWP>)Uf-Etn3|g(&lD>4))0Dt=lPf4{C=ymRj;ji zR2-Py@<^&$yA=o6@}k2JJkT@xqvH#NQgSO3osmbdzU_l=jn_KhFvwpY&Q zlXB_ywauUTiGTKir#~rkS>*evM5m8xBqC&GrQ6&(GC%L@MmeLpR!=Jxw|foG zGekf!id`H*>_*V)d3rvbSg1(mpe+NNBbF-iYy`Ufo`|9>4>%r1C=xgx`{sq^^=ljT zR6eaRYOAsSlOK9Kr-9w={TDCn{q@V8FMk(at7YnzI*1jW;@*&Lb;-ugV0Xi7*VtYQ z*7sn)7d5T8)$1}GqQFr_YTE7TxyeKk5yl{)1Az~M&^2w_=o1*Ch>(~{BCqmdWDhV6 z0SsBl0f1-}5&{8;DTtuwyOzZWvZf{_D6$U0h%crSGz9vvIg+u%60 zFsdCqny>Zu)47T2&amH)dObl-k$kG4$y}z8+-q1AGO-8rU^9`Jc=-HNM$qpxI@d3} zSnnI6S}B%Ny{0}G^m1up_xp{S?^)`ZM}P9K{`{r?_V51vWF{Z6_Q{7&a87c52E6d? zcUVetX>j6L%8reL$F)i^=?r6nJxTWTptD(RKlR)b{h(_E;mVC|+cq1`o)a)KFBrqt z{$@3wC{C5i6Jy0*FTf0+&Wr%3x7&--+Ce^53F9aVp#<2W9kLusa3SJ^@BiOF939J_ zKRSE%>?xl{UKH8B@1w}^y;Qll+uPl!wi$`54I6|-491e0hyw=z&}y|Pr3fJg!pL(m zjdRKD{8TxqCWV9oA+=r9_X30nCJmsWy-MKeWnQS%hnTGGV@ZQDS&Ec+L zcduN2JEN^lD22uq?d0rdHDw}`D@w6eP`pZUA-vR~bM=E^+l^Gdl+j?lx)g=ZnKLIJJbxygk*B6+>h-$gczQdO67bBUPwdvc zOlom`X+IXQAD02Zu4mr4wRz{}p1ZxuMf?zi#gTp{#mp>>x0XRZ+gVw@I+9#?|I^RU z92vD7H;f{V<2*N>nx4aw!1Mh2`sTviF@Q~D=*Pf5a^{KovAk~e>-#kZ6M^GQ+fo$8 zbzNCat*Dq=&8N^Eyr>L#k}`65kg@YQc4v` z4#O~vBSHwJR1`%4V?Y_t1A?Lum5rIm^|0zAdqM0` zYFSpJW4Ix9Y~AQv!$B|bK}JbbGAWkjh`?~15CClZ&K!-koi<-BPz(^3n#rd&^w8@7MRD(34iycUrw6;n*;u#Y#4!Cc&QX z8i4v?L@7cs3QWDn5Ge4(&@F^8#|lHk@;pyuh1j)yYZ!Y5&$9>t2q^%-aU6hAI-Ts9 zIs{~LWYX#g=1?BZ6#I=qV1QP`u5LAW%%u})0VhO~ogA4;2vT4-QEc!e=r(o>m6WQn zqvPqZ@snBxnRGC)zHS)z4jq3#qfWO6rdhLFHf`?j&8MY1@4RsKfr-GkIklK7jH_Cz zc7JR0-p-lhhu;6O4+Mna)zr$y706f{yX%A#m6SF$Gkx{y)vfi7m93rKoqZm&Yj;Fm<4_c9*EXw=6x~TXb9c--K z6mTdCR8>J*%YOJnPvx>ojO^W7b#CGDv5NB46AwDJdHv>%@4xWU^B;bHuh+?B(n08~ zF0C3?-So)78DPT0*n9Bl`GAVHUp@K2$kNg+)3801UE654t5;{{3L>MdFK?w)BP$g| zmNSRjU-`>Ftv7}T4jrbH?p1dz-K#e`?ZNQDhaM&jx3RU`YVTz;Wl>^QtJT12xS<{e zA&nwPffM+??JykMH+_L)_F5*9?EuN&`R3b-BE9#m3{e20SYWtZG7n=vuzds=1el`- zj|+nEURxl9@IBr{NP!=O{h{Flu(4Zf4~L?pL^K|n!~ajE?{W1ZgaAMYknaZ~FEE&d zevr$i6Pn^jKE@aT;CUV!#hCCALbKy$ltdIF445E-yqrJ~5Z4JD+mC6;h-l~-W7#xA zaUowhaO82@BM9?BOrw|$DXF7$kYpw;WA5~*Ihx3#v(u!7|}lBh8pk9g8` z9M=gE%SNH^MnQx)MM)-AF`1GFdcWNox|SC+lIydb_ONRh03byY0%zbwU^oPOwG9$z z3Q8uqv?kz%$;wzko2ldjgm3S+uP!&bQGT~4-Mr`DyyLDcgX%8mbojM3r`HKv?P0!9 zz&PMIVA;cnQa^}9nS&q_MLrHa0RtHL_OKtiw(o(S9sr72ma}d9y+j^_9>RgbL1+%B z?d6ggPq(}6_DCuD;t$>dtT=aKapXWYp=J_t7J8gxgaQ%wcWUpvesOPn2)ZTm;hJ{I(n_2xYq6<2mHLce0WOO(LVXHNhWCRmv-mXsK7t#2ur zlH8n?6sG`rrSYbuGZi0ZZ9Fq}v zS!5XyTK3-krR94|caI#LS)8xjxcP3c=ia!rJiT!6@Zs~vk3as>%NN4f@IyVO4&jHP zYZsH{2Om6wAZd1*7he8>W$I_nESRP_KX>@V$-})~-*xNfA2_$y6gStGso$bvvsxRr zYK`uoK07x(==ZN)yF~~)HZnqCJm|Mop1k?ih2C(GFDnG5{lGL0H>Y?HKllDrQid@l zgoK_C5R8J55MukGl+b?U=YG>d`ER^@r|m>6BZ{I3VFbd6@ElSE!yVM?Gz3^xggA7s zU47TGtSE}2C;|XL2qA<)5O|*F`+mLMzj=Fmd40d$Z23VD#=(21@A)Cd*!O)(q2~oG z!^c4gW6B`x*%m^8T5#M=S_1W(kfWkxxQ6FwP!668XdJmJVP{kQz9=(f0YiL1Ihfpd|2` zBE|@)=>%jP*K?pX^jtUQnE=JkU;u#VMSe(e5XFKJ(vZ4gj3MdkE?^bMHvP~OI7I!( z1T2wQt82J%h=f?iQf=6sE#@Gmt--}cXMZerB5;T#i=wFY8~1a0X=QDNh2!=00fm4G zWkC>zHA_yVDwEU3_A*P!#UK1>mPgHuH)kCKLvQNn!`CnUJ3djU2p_t7;h#nB+4%O# zJ#!^3Wr3%2Vt;yWW__){aN<4m!INg&8@Mfh=?V+Nz~n1VdSN5e{Ks zY5DBb+<1A?g2-mM+(9*U_1epUJ=9X^n-_2DJ!5g<(1FGLJJ&8HG?{QjNb0F<>V|C# zBD=r8SxA@i`SAd-#lz*s`t4_)K8yp+Y`%4B_DCBSc(IU8yZyUcW_tj0Ny*k1XJ-hT zYqnQMjx1ij`oagE|4`d-(t}cCV_2BC-hH!0ji5LYbZXjx@uI_nm)^NQs>xGxtfWk? z+zs=&!0;6{nI4-te(l{Gomwr9#5NriM?n@OTkS|z2e~n^qnoNIoH>5_+Vyu&oy~pW ze}DbiM^Ehc@3_G-VgB--$!tbwZ4|I3^SzmSu;-A;WPXhSa4fW_%~=wA!(sgl@i^o$Gcy zL7P-F^R+rFNqK8%lZf;OmXOqtRN&Q7hOel}6N84ja%HGgPF8#1@%2ZB1ixcIYx?YgK8rRH9(TggAPaPVVGOCFMsH}<{KXr*J`+*?^{)^@txwj{Gt zQ?u{9`%bsBY8iVt)&ox$C8gdso5sGF%VLOU%OiMn$8XTK{}m2d>okeHlIpDo<)e* z!z2W3q0C~!>q8Gjm}2JjH{S~VFjYK~m+|KIZV&{NLPCflYg!`XIzihU0)ZVZS7zr9 z96xyYKxx#pdK7pZM<}HsjXmE#cl20&Z=YdV>?2K0ablv?YL!YQO6mV+fKd<*27^Yc zxxTeot?hR@oo2IcnxLsF88XS{qYr=9gm-)?l=B(Og_0y8 zgakndAPGbN=&1!5`+?Ps0xOkCvjWd?9AbDzka$WXRn-jLVp%?REb0!n?_b~9y#MCg z?+R)%fHYUg3cT7I+BOBdwY{W5vS~S8NL;;gsn_d4!VHHN1ej%6h>11S6^B z>jj=4#*yW@hGU8{A4Z;*92uWI>N$jmjw&!b#4G{jLOG5(!il!$?=@=(GKAIK5Ra;9 z5i=>wqA2i;Aq(Pixh%;lz&s_0k&;591jJ<9rg}v7yI#*k5hym=skJH@8lu9B<0D|a zVzAsA&-4(qLr=%lk0U>Dydd;k-x}&2iNl^fBnU+jwA~mW-0uxVhH<07vpqtYc1vd& zNsyCY_;3HPwbZL`xu(ftVZx45YGP(@x7%x4z%FmD(_UY4XTukKX`=N>wp($YyemEc1bxYYAA8_pG)3ml1DMV$Dp9A6O{o#!sbk$L5cHiU#tG3M>MS;WatXDB7QbZA9zWR5sKl{`J!-30UjzAQ6R;Sf> zEHCup00G^#QB11?yK6+cjS;4fYf|75h!8@JZrdLW7iZ=eUTE#NXo#X1@B(|{jePcP8gfoqPECR@aIlO=+6v+I^#=$Q+Sbmqr^qyQzfAFpTR3x<24|!g6qD zd)ILs4x^MH7=u3Id6}?DNy#Qt=FqO~)_KYV6zw&;5HWrbF+8WLau{(pmR4*40)~w! zOsNT@UxQI7%fxYeS|$k*M_A7Ge1J&Lv*MVtlnq11jRBGbPG)(9V|$XcnKO{g!YMAR9q5$tVk)*21H+p<>gZ|i;A1rkn_ zrDU3sRLDyZFo%vUwi`R5`ap97FV2kF18;rl_0X08-zesvE=_$DBvz`Oo4lY5s*R+0 zIySs40mOSeZSa;a3s53J(Ih;I3UsHIY={hej#TP0Uq2%o&r=`FC%Wom-K?Jr(wQ z-+$#b>M@#{#1H`*NdlH79#CJPh{1$LF*8`Yd*uh7*ODYf6eI|79LF)GL(6nyKSCgk zXlK~r7+Bxk;(7L+civ(c{N4fp*tQ)5YB+WXVm}C6*9`(EjzdZ*rBM_Gl>YxiV=|d= zqoCH<->t4Aeowbw`WL_Ylb`(H`|F!4f$J~~^Iler0~bR4_dmSW7_vP_c7D(hRcastv&1v0?3LR>-T@~&3{mNQ56%nS1yXGn9dg%k!g0U zBWK?q`?h5py6Nbyt!N1b6DI})kP#WCWKv3}r%J^LI1!|A5ZN7*rCd6lADuefsBZ|A zhM^q;l9UrG4P&b@loZj8+{hnF*cnNxPds*95C>+b6HsAyZj`eyGtm#n2hxrqcO zC#w@GRHKnmRbwd%bR6`wphqGgVV(?q1Z^Z57H@}v6;g9(xR!?i00W(h34tMC(7a;o zufjpQ)3Do(ErDfX>ZCJ@G3@PbuJk%Xfz9%QN*Re|(z2B1<4kqu=8-e!aQ{_BYPk{B zGX7$IhQv&Qd!%izn>fun2g;?-5dW4|oa@-%%;bTVJ#T=ujg|f4WFlEP)@g6a4ASGx zVI=)zw+uEp;nLgb*NP zxL^~Eu?^1PaIrZ&$M`tlK=QGD2*#LXvOs`x4y|r=?yBkvyK3jO@>(;`IpTd$=lYKr z&v<^{^qH`DT)B2FxVv*_Hkt43^z%3=qS1rr&va@{iW|><_KR)HOy|)o=A)Bo78Q~w z>+SmP8@GrjmX{xNt?>MVmnAVG%A=2Opoz zW&x9I3RVq|Ul!F7!juDu4ass% zG{sC?{>s1oumAI>{==_4|IL@MM+3vms}dQ=5&|d*p-C&j1=K|{r&z_aJ^@9V&F>xV zm#okTFDPKl+p)UcFVDWxDr#6ko^*8rvl$!xm0x$)rT-ukL@>3sKxzxy4_ zP3L5P`}DSV?VI4spUZFFEkE`3{llbX)YeG2BHV>`n}@Y%afP>SS})LI9xS*<&ju}& z)M2XP+*JOJ8+&*6vXTc$HXNRQ^YG3ms%S7B4!R3#qv@n%GB4Hc&cWXPc9BO25Nsfg zbOi?so9~c(yni~%#>d6}O~d_ObmoB<-}sNO?fg!?+Fmja%ks11@_9fn4~H*5{J@Xo z@wL-~ufOH3KXT=PFfBF%XZGGlKO(IM8?-0)N8kN@-w@uP^chY2FhLR_4@k< ztwwV=8lCp{vMia-^YLux*U3{)pG{bP>)y?c^`&~apfv=Lf8aa6x3RME=v$vQNPT6XAj={o)4@nt*)(ZQp@cwEtgV0|J9>z zdwpS<9_%Jc*qzpDl`_u^fzqMx7={6m;nwSUnI(A@Rf3D^z0bU<-8~b}t9L#9j>%|L zvR0KkGf{l<+?|F>WMHzve3 zrn+RD^ZU2oeba(c+6nO&e&Zj9@=fo1_xnt0%!;DD(B0a)&59CZY}>Y!QgTj-NiCz& z$tW9Y8v?hkZ9l9v9Lr&XpBx_|o^xJu#*`)~uSy6ZMj(kMN)}j^5bH}1Ul9llhSUD> zwA%=aIMd1iPDaP^+T(l?%&4R<@xD=8dNBXecF*%lwVq9Z6|g zR%OLwUF+vK?5syKb$ncik_LgJMG=^Qh>}nRs$3LV&9cLKKrA>Ij7Bj7Mtz)#3fWmH zF@`~C+tkufNXXM7*AQ5iRdJ3X!k}6Rfq}tndie`sLTl;svG}`}se6lkcOtduG3q8vX5H(LuM|W;tSAtb(SyUCa zT!{VmiSgZ)^H)zIlqFkb-aFrU`Yq3VM-Bp+TrW;Nhg{yi(?8rkeeI>+ETj6) z(ZTl3tqJIxU<%C(fBE{K`JsXiR5hZ6nos7*d>e4J*t!Vp0NFXEqtG|2ylAvL7tXF- zIJbHB%%%QFF-~pUe&_q&{idhiG@B1Sr(;=GS*C|Kk8i(pn`-Ld=Gk+rlW6F9HfQ?n z?|PqM5a0La^XYIt&k6$~_QcbVzkcK8q8P>V!~W>@aQOPxo!{;cUOvC(-g@rWTg}dN zI$eD7gMRbsa-(;AaL={erPb9JUVYhe%_|RI=+uLFoKNR(AYfj+mAo>oj^3a z03DvDufBS^v9aMhF2*!VQev1~mO{!f3=ISu?bRrnc6*)|ztAdwV*a z-jBj`IxW+r0H`a=4}9dW{NvyJmCwBSO-}$J2|!XxAq2+w|F6DaBZ_|Kjrgy8?|Y%d zZ+-Jq#ByGI=@kuxmLSjb`=AJlk>`bn#|Ou!bIWQ-fzz@&n#6Rr_;&oz3+Wbl9ngKWEulS4JLRg@!>cbC8gi! zKuv+t3k|1CqA+M60BzGMtMsGa{yqa}1Ryn>({aj_87DH2vRRUyoQzq*Ta6A9u>&p7 zb`V9#FiFNXo_xq{HWwG#j-z{RXKkUKW<^m-mjD^}IS?fmk}Cv-VHhP-h~SFzk};yG z(u6_n)dSaZyR9%UD*%A!H~mf*8`^}xKu{JrkxXV$!v-!)AJ|xU=HW|^J+S%6BjMYg zS*_Li?d$s47uYAhmfty1`x8lmn&Vim%QiPH1C`TBHX4js1uHI!Qjh0JmX(s>dBhb| z12ZAe*~$3M3xED^mNq_g z)UVZhnal4Ym7GI+|L&<^x?oup#fDU_;UQfB=0F+0C?$XWPd-nm z)9jtUds>_xA2#cu7J?9BnkFHHF)>e44Upw}^+v<>9m6mj$H5pAid@e|7%W__2Z1~s zeu>VU$l1`eCZGPVfB04oTz;K*@FS;X=5)X>^?NyN*Fve|q%8Yh-vrfdT0Q6sc=5v{Ih*GK;FHs;(dk0mi1TXaXlVLNrbQjI zb>ZBjO{)%_I|B;7QUB|q=H~aWk`hW3FV|N@XZ5c+a zs#Fzxe=kmRlw{DbTHHXX(mVSnsw^v}Flr2j^Zow*wHwbK-P;;(@15@MOyW7778LDn z-9F3?w(fo7W6?BXs%TyO&du)9-u2&Y!0TK02L7~*=bu^)E*`$||G=Hq%MX2+0_HaQ zm#_Tf&6_VP@Q2-QH}uwbcK7;kylP@pudO4$=QL=e-f_`|!~NUwxO7aU^|s)%%MV@M zKiUVleCX*XuHSm4D2?gSw7-3L;e4Byk`*xya)^jd;`hGm>1OCXa_OPPVEOFQnVQvn z@=YH!j4-cQG)%K7e#@KRj-=aI>}}5u!;U|kY}J~M9B(RjVAbHmkA52nHx_zx$=n{NgYC_G0UNu1TbLRmO&4L5v_m+js9N1arY= z(VP`!=y--{QrAWRuP(27j{lYCb~&mG(YUv_&x900K@AAPPy>vBK`{jC<*z)4d!|oU zcoK)Me){Rh6(B+g1Oz5fN(lf8TyLSXG&t=GUbN)sr~cmeeAhc4dE3Pvvx&kIC~SHT znkM|_-F-k@-}j0#g-BH?bE&`DTjf$&7FbznI0jXmiL$7Bra?@oXwAem&nn8OEOXno zZQF!ETZVUZJd3l6DcD+AiqgEdv{Dc3)lRcwc_k2+-sfN5Rhrwx!PwLYx+HHcxY%Ju znn7q1*3L9zt(I{rm@hXEEz%w0t~cKm8FzQ zD!@6H0g#?#U#lgrLB5*V|8P2GWyilkG}8<8-XXS)^e_-uHsy zMmZ+PUOlI=vHrl-$@W)KF}Zl@{Py9$37Uh)FZ~#*755IFUG1&R2YY8Xn=n2!tFb84 zT>X*bZ(Tj}z>;sb&Gv=m^N@-t9zOoqRieOipZVP8^4e&0*zSa9&u#4Oo?h6z^x%~T zY~Z^#=+7o|R;8>Q%}!1Zb`STr(_%j8pNz-jr{44SDD6M=*rO2invKojNp|JJqd}_$ za3^T43-y(Uul^O^Yi;d(=KO`rf&pUeAZu%D=MQo7@atduNvH8@rSzF|XB-1#l1fqh z%uoM^8`rly&%1Q#e7(JTZ+rLRxpl>iW`o)eNGpjcf~qv#@cj9U@o4(~Z++Y_q2#Xb ztl-+~R^5RmKfAnU+Lnb_v+kB#{5z$4S$)C<==3g>&aytybW;vSPBl?+mkcyPfCN-tN)KN&kM~=zak0{(j8;27aZ#YDJ^ur%U zOx|x%jK^bC2`scBp=Fwu>)M0@YKSM_{O~N^l}dH%HqhDW;Zcz$XVx~BnziA0gcU`C z?`+?(9K&`@r8$72+h|p#Y&X}jWdI1DdFxwEH;B?oRt42n=$W=30$@G$-tPzk4?|L< z6@VfnUYb;mMxz$`0zxftE+v2*QIch&DmmpMO?cVQBWVgQDr#86Sw5L3T1$)xLWB@g zaELUJ3P5C*g{Vp(xe-MvH9**)?RMCxW8Wzk8Z`ib>3c%as=|%t+BBL&f?d~5<79bt z#SXlRF-oazn3i7$rssJsfkaEa&|X+y?b&wyb6>jgho9U!oz_aeNNBgH3@%AfbEs9i zUQy{RihbmsucT@=YNlbIoF1Nx4~`G-yz1_Am!#vJ-CJ9CcP_7Q9DenW zLjq81wX)hRWvzGIjg`wuT&`ZaINZHm99?HD86DkSus*Q9_*hxJPU)p5uYTZo?_-3= zlyW<$D+*BH3-!<_rWrl3Tc+1+mJm20t;+dya(r^IHy;l=jRu9O>PL7gZ>+*P9+`9P+kZCy|VZ~m5 z{n>7(asApi2FKge@96xo)s)E{9ICCL8$b` z+PXm~Ap`*M!V51Tt@d`eJS3uDYGwYPULpU;J0T1&>t`z{2=7(%_U z(8=@E_iLN$YxU52=eypnkpJbcJnK3Zgir&75P=W^sFs#`MVYy-n?=+4Y(S{>*wsr# z!Lv!M6|^mvGl>Y|op`jHy9ZgGxEz2mftf++0#PUMRGLtCH zQ^h&gfMJNV!b|`y9Vu{C@kXOYiHV>|Ai&Bnq+Jx1Z4sh# z<}RFUT`T)}5|;p0$Y#(Wn@@ja@7lAD@dbdcQPh0?tH0IX{h}SMd6o-EEeAG${Oo4i zkUODS3T0PSmWz|!{%ymyT-%vV#?QRt(e7fH7871%>+72c8K;B0K&8`B|D7Lrf0D?{ zS1v5NwP!!~wO3!;N`<_1_3FXy&OA9B4eqk|ROVxf)Ac3$wHLm8dgu1x^&K$)H@@*X zF&;nprtit)s+wlWetGZpJCkF6?WJ$D8s5&WTN^dqS@XW~jlVQdVHj%T{Mp@m*DqgO z@*P|j=|c}*oKK^b*5Yh37*D4_%;v*laGJ;>KG?rIiRbH!o^4i1o~m4@1~tSE{`qY*Wr?cj~g#f^ z*J~ncnzQfvp1(i%zyE`0X(2$^?D`Pos^mGNP(Sg=x!vRAMy-Lh6R{i#mB`!xI@1+F zx*60gA{7Fo(YU!->vn@)v-$kBxaPT0EF5GNWoj7C_Wsbb>{%uq8hARPrJ7fu(O9+( zdhN~)CbZ((aRZ1{#s!5qPV<^?M2i4Zl%oRRDoUl}=pu6({%D@KKDUE~ zR4S;OX*Ok*P&!v8Nk@l{?|CK=f>jl|yR%<+LDAQtsq@5$J-T!9TC)c%<9a^JoP{xy z*5TK`{@{ZTz4Fb^FLc}Gd~5a5&G~3F>u;|=@_ta9K+n2&^V+#b|H&)2Ke^npdY$Ic z^fiw>R%x=@mB*vQYhSW%L;;E>bqQ&RVo1} z%VO5?t={JP?b}5pH8NT5JcXf5N7goPe1m|;AGx? z+p3ts>)-se-2upHom@XWzrJ*Ol;_iaG&^pDi^GG{^AC4U_LyhYk1~vTp+tPInz*z_ z5kDQh!MOgyXTIzQr%kt$q%#&}fBC5|wih;%tcc^-cC09#wYrO0C4|JbA7tZ58(_`E z-~XP?^P3<2jjw$AxtC9y%bVFGHZ3Pk6Vo&yP^Rl09-NdVvpf}2V|Qw{SMC1caam+R zXm)xVEKU8uSzcT#Sz$mrIUU@+v-QYBS6yT=rEGu|C_ewSFWlQY*}r=W09C8k4-O7I z&jUa~E;f5Byqfb8FlAbHpb>G^h&=As&D9v1ea+O+cMjRfnQVil-i5 zKAC~hpr3P)OpCK?-O0Qtv3WX-DAc*)NyZ4aJ>SL{RzMgC<~cKvvP^7X6}lFMX!r2w z@|h)Cl}xFy7D!d>50bP%?OKC#(djGrWMGPl!{rV&AEhB z0WGm0N<$=sz}jgvvn(O9xOA3;^@XBjQghC^;4qHUI1!S01@jsX7cfEqBZMd_Op8#c zT+610flRD5#5n1+eBXf>3WM_ZzxSPcr@Mwh7T2%tZyf+sEUqt57v0^tckapDW#N?aTGQ-{gdtq+tsfrkO~%tXFWjKJcXzjnkzp97h2{}<9kRO8J?Wol z%q+K7rvC2HR(lmU8^OtzghqOJQdSXv_|4z)%Inwm4^DOu4|~nl>e{+tJ3D(Pc{*P2 zEjhl2XE%QL=YNV5Co1&c_!qzAHlNV51QT#-mDY9UK*`k^nHy^E^igX{{L(N&x_nX;Ra)%2J3@vPzdsxnXTS zOF2XusH(~^#!8`Dt%eZN5R3;y+p?ro1R*Q50D?tUMnzRfo-#&E10t+|61pT9hkgy2 zpp=zW4>BqC5026_H7x>&37}sI%CUv)nxZvDJ%|j`K?RFSg`#+zq!Xv=+u!-NXWsI}WE4@g zaPQtu-C0U!r3sfl|3}Y%?03Dnh=wvWBI6B%p?(d#V!w>RuC(6rz+-GOMtOSa z{Hm03k)ii~@O?+aVN^+^?T>u-2PW~btd69byJmAXDdQ?NNX>L=9?tIFy{&kJML{V| zv)r;Q#o5wgC(9YvN>=hiAAWa#Jnqkr4{mJ}Vju_*fn|}k+wJ>}p?MO=S^C5sI3lh+%HD*OTiL0%x10hSs zO5gVZ0Qc<*EyeQk@^Cc4HcyhsCCyq8C}EmTJ!sauy#>Lz=Xn4C(=^e+{!|F%SX2d0`N1p3O2*y0#{jJUN~9<6J1H zkl6@)g9N=^x85`a27^iJ+HT+&fnyk0{=W?rb(KX6&0d3O8l?CJiW06UU*I1y@mI-XOpaa<>pp&7_W8DizwUw+Vi8GS1zoq zC#Q49SRqB{;l~}!LvbQ`Rh2of%Fe3mw}M)&!IL>Q%!TE1{wCKsV~% zXxK;E=r|M|QB+JEkJFC5-YcWxA~ zy>KIslJRhm=eYvLXc`SCV?aP_p?5NhUVZ&ul));`J4+2@w8Glid-ujk0{8FqWo~ZW zyvw;7mnAnEfBCsL6m26-co`+pp$Yx|;b3?&mMrVG!jt3IR~A-+Ah>n?+QoAh)0p?? z(_i`3|JAIwC+WP|3e!=frqc`OIwdOtHSv4hGnX#Rs}v~K4Ba@dqBJuNiV#o=*nu~i zkALU)zpTulVK3LuJY?FnESi^+HJkOdwKdDKD5a)t0SGO}E@bjMfB5^+d|C^AR^%AL zC!cz(D*4$nmu%|VmQ|!lXtKv2zjEfnna<*h+vptk=fmOnt6%zZG&ylxLu<)7hY%Wu zA%t)&^RdSsD}^RR`Ixc88@`(PCdq^>I5J_%P;3-TwAJ1cqYB{q308hO6Z3)N0!di1< z(RA@QUw+PKB5 z03>nAheFE3(*Xkj$DC?dB&t7Ll-xKT?jK%z{@zZ1G7j$DJp90e z)HjzG7F&z!OKUzxS1z1kdAiWJprC8o(zdL6&CTaAOfA#(Dwf+0oy{gn1B_{}*WTUP z_rhS3Tfg#0FZRX8-};GPJUl$Kk!6>V4`%s%YTJIP6oLk0B1w|C((icBI|=28lAzJN zv%R~wd*WD#*y`e=ms@TB;VYLK%|Mi8n&jL2!|61B@#W{Yx3^t~xQ=n3ybAyT2zpLP zfw+A3{H>e28Y9W%o6h-v8T^GMkP@1458(oA8;3H%~^<6PM2% zMg1}_G&eXvj49IsX_2qAni6S5peU)7nQzi0VuXTbgCdPjrsX7xbA>6T3-!>m&B^fO z@rNGyg`fZV|MUO&>1ei3Yjdebc5`+4z%3RArP*Q>qyCu{eEGt9bQ%w)nOE}^a{|vV6z3X12rI@e!hz8`+>^MBr`B@bTR z2pjEgjXkg#2r)&DoXw6OdhpTxy}f#;rz&fCZE2e9FDy5r>AXnke1elXASMZGaymUY zcXn|+8AOS6+IFManT_CVrcdWp#Y@IWx8W3XB&!DSt86Y*W$h0N3z}(pkZ`jq@^-UX z@^U^cMxz-tOxwg1t3t}*(Oe@MHn31!ND3sN)U5f`cdW83g%IQU*uuWl3R4)Ev=-QL zRK4RFZ)u5yj&u3yQ}^!dNaW;set5L!7YPHVMaN4f5<^?FSGzjg{WEv&v?B(^N z?cL$zmeV>XQzt7<6;fs8M-h&)XdIQpQH(LZa{eO3E@Sy=|KRNAI@EY?=dNdv9Ba?3 z`F@x43Y&%jm1za9U3)o=v-_Pu2|=w^>-YP%Nozq^TTwhi2vCz~sj9LJ>jA5TQUF3Ixma6mEiIm%&(f@l>aO{J{k;#5kM_-=_l;L~ zKJn5G!NI}7K^TU^VSj09iBbyR`N+m-mUQa=sLHj_VGwZ6%Z%F=h$_~s)mh2?Fqoc> zL(jOh*52($idEeOLI55{)hNpPvkF7#hF-&~6>$cDKD&9AV7|0q9&R0&mP>5o-u|gX z#J8*=_gg{7K_ded3r+iUG@A=^e}BB#Z4yGxuebBzAQSp{Qt?W9b%&UM=dw}v@^osO z4g{*ml4-7v=e*nPy0!_lRGbN|C{|ht3ZY>boO4*@D_t47UYuvZuFW&vv}zEbc~zc{ z#!E{}hT=7w`b?$6^_h<(j?bE@e+gu(D2K%R{m}N_A-IbNaGd%~0K{NDhqBQYL z;+M;+X->yS-DZ6_E^Ev6xDe5V5Th$d2qDO3Y?gJu z_lJM#5C8C&zxxOO!LR+yKboH&98O{>&9IFJaltBCR?x=O4}yci@MJP`O`}_L4Y#U0 zwS&`#-MW>RBi9L)#1JD{ia15R#T9Ckn>X)`^DJmMF5=()1Ap}w|I2Tl z?#~$ZFjOw}@9dwLt~Z&2C{6P$d-UN)@>m!qx_14=hab9{7Q7O`0$92tK!GDFu9FLqlPyFCJKm0B4{^mFL)*im{cYgBc z#`DPc{V0lD*EI}7DMg5lFE5_i-9N&n6{o3f^O6gQplw^Z((^Q1sCD`WJ0E!W({H@| zLKc~mXx?-JsSvB?YpXru2r7X*+Z)W&3KXts?GKNtEzxOPX(TNUMuj}xOAs`$Q5?h( zSHmTeF0VF9E;S@s0plX^J=3Bp_Ty!|_75sUenCgy^$(JBqwl)ydMORAzY9h$|3 zWd{Tz+qSE!^4xlqmagL*O{Pnoc35vja)$GE5MI4@?Tw9%4aPMuiloTSTsRYzz0;%P zdf3kLx#N(mh$Ix#(HK$UhW2THe7;xz)&KI#kQ1{`4M7FR$NeM-I@2VoIFLXKDMypB z2EaA^=ECZ*A9uJznYuLVKC7&EEC z%1Q`?PWjaL&LyU(=}b62u!8w>j>B$FzuW7PD;kxb$qssv#RQD^kx=$d26^g z7G8G+AdO{ZV^b6?%8D#53a%isN@5opvN8ckg`q2etdQW8)We2h7-h+z0F28qFQfVB zD4n%h9h(II^q>99TyTMfmYSGEXePO;1odKm0qryq=q17k>~AN=U_NC9p4}p&r-v2GfXqHKA4WESp*$TEmNBmIs{7O0JP%j zhGTrk2cP`zkG%7@{_rcCk6--K7hn3NU;O>Q`1C8U?-~+zX4BCuJry#`Q}fO!ytNCr zx3jw^?sNzzyJ&b49PJAw8`BtStFv=B6{53$^Ty}@@)wfys6QF)j?+rG855I;W(*5W zJkKeRF;g%{telKi+JWs?qj9yfGc5}{o=YhJ=gM}%ZoAQ|xmm%}dBK=yhn_CySw6R2 zv|!47K4*-a2><(k^q22_{bYRb@_e53PxmcL6T{%SAdVFa4u!F_8n!~?@Cp!DI@xMS4A}!9>kZ4u*g5TmSj>H*P`$=0$wv{N*t8H53rZ`|Vo;E%!SV@Rip47Z+#*FZfyICySaI1E}8u+D-9@?tud2E~NntYjLK{&<3k zZCH*5Sjub?%^geuMHZ$UgD9#pCTgv&*3xwhmKPNhb)zy+%;XlUh)-YhK`@ zWI9?}2-Bo2%hC&LMVc6{oo1yJni`Te7qdmWtmq9TegFVvMeL5VApqT9xW|) zF%+ETNmXCJ(0%qB*O$7%;lc5ZK^YoUi2Qxu@(!7S&wlm`0dg(Kd68ba>a=~=@ckUP zN&|>ZC`r(6pY#XuEG-1A`C$}A!?@IlN(Ay!7D?d;mXwlnE)`B$8PBx2p@$L7%cABQ7S@+HS2)WH%@HsF079@)f3mRD+C3O0Vg#)P%8{>jfA&B8 z;2Uqev2vy}8S?Jp**9Lgvwm^&t8!9^?$D{&+HvrqMjn8e$9yA@@hq?uTGF=UQuwv4%iPm1mjf`}c>{ z1k@Jsl&s2f+HTfA@{w=<+LvwQ zvr=GWa;c?;Nt$Db(gIN8)=aOAb4QUX&KN5csI=Au;k+u-ynOk_URLNhi|f9dB{?zt zjiqHGvw(uKN})|0#}z^W0K7(HI-T*HRh2L;>f2_Xmt|Q_r*o|km!MQ2Elb82)X0Q5 zA5Q~hZ66J}863^B=`7815C+ZhqyU&$u2WPBSyq;1wrMdba$eQyp^|`LcWab9{lV{? z6v{R|*YFHLAc0XD0RTF!TD#p&vb^qQ`}cO;h4onyKl0$EvkQ%8({Fb=uReF}l~-?a zZ9sEuH>>qW7w3hO%WyDNIhS!MbFR};4yN%W&ZDAAOVOWZUabjx7;Hpg$a8ifW;47?>EC5=5vjt5iVDIkz;}>^5g< z?1eSgUZ#M|(&2Gd`~DyK?}rER{{Enn^49Gw->digbERRyHNXwiTS}^O<=k?{;&yFK zOSQ9qb8X?mbY8swUEdj6;f2lf)HE!^%Cj;`BSHwJw5qE8<0B5WVHov#GfET3pkd)+Qc6`-MF>F%IgWFmC!v%g zgaD<55J4CUAsAznQV4KSMnzGq_g22+!{77lH{Z~hrR4~gbiG#m(09FKHoPezkXqLk z+J3haCs`%r{f;6+h$1Q&n@`6G@Vpdye;7*z7%!%Ivd~`0S>$(X(5m0tJ#=i{w53aB z2&96DD+ZAQ2;}j+tQb=u%@M%fcoGF(=o`lV&YmcFuiFc4E193#YJ9$BcJXM%PyhIT z{F~)^TY(%?#3iSOks(w{Jy$A&GzS1WCN+p*l6oVg1VpnDXL*umhGnK{YC85TOJH4GuakECWe}^*({2pIM2!|uLOW*7+9`p88*vh5-}wxC7uwf z;<^$VVp>()wp4*OFFTHs8@PztpMWwcue(qO(S7>8A7*<8eL`4nQ zsyhqIO|RiF-sQY4svt3fy-AiAFweN++P1<|SQ13Sz?Y5CzKE2&ja3UD|Ya}BQD zx^{c_*7nxn+S-}-ec;0i5vLJ;@#`<1^e4A(-G!Av)QZ!@#JDWW`-$p!I?J-mr)I;d zIlkAQ3~hp_V6|0V{96R=XoWo?(0m9F+=mFPp_U|fB*YG`r50vMF|Wn z8h*=^2S4~dAKF}Sf9_xY1Hm}eP(vtk=2BZ|U>HWD(IAABd4&KJtb!kS?BNj3=R(bS zAsM&bFfdB0{aWDfk6B8@W9wn97qFv2I*e}?KvreD*C{6P*~`nL*;MD;a=oIE(JQxq#&AO~g*D|Nm5h569 zeDy+DmT_7brBaqgSyeW}+5_j82S+<)VKTvOOezbPdFeXHu`MJuMcyQzm{x5rbq!>p zW8J)WbpFD6|Kt?fM#Tum5Em7q-bn)2df_;kAYl|)mGTUm=6oJIo;{mP-Fi2$ywUMd z-65TZ#x{l$D5Ew^4JffqW82V{WyvgN2;$JkWo|)pJ}#h1bH#v3RfQbepG^CZxb2R$ zQrA~D&W}!y`iI*a7au(xj)#LWR3N9YJO`t96EtlX*4()AZV6_%oI=JBCJ~o8(FD2E6ObGH0l^c z0MvvonZ{W<0}^$cD`_^*^0?mhO%SBBL{;VDMw?Zj-9ncZo2SD8CQ8J*DoS0eiTIRL zXBZS`Me*zuWP=2flR$n>#J>kfc>lWS6j zxL}LxtIuEGEh{0@VmMP~*h!MCEExn?aha8cA}AWqd%a#5hC<0SO(&Dd#m&uH7(}y4 zp2v#%YI6L+zw-81Ug8zcwUO`K+gVy%n@0WR#r1faFRw3LzkZ!_&Z}H2q!pD!YOM{! zfDA$c3tmtl2$hy)_3Di)k34gFRE+0$ut``25r9mU|L{95oxl36!`D9%TzJnv{h6O5 zj*|<4WQC};y?@kb2DV{q2`HdPqeGi0)b7Am8?P+UwVHdT)2^fYytN|9ViV8f=|Ra; zFcqhhvg$7Uoqzb_omR6-=ax@L!#)xyDaz3#PRj~%i6!jRYJdEvU#>UTiyW6(c{+$I zRIl(duok9;e)!z?$EQX>e2SH2Imd$`i07eUce)!{qM~UzoD{?%TuF@$202GnZK1}J za;djydVwgRXBe6@p|sS>wk+TGg%AcI5Mn81#RR5?lv*k=n8d7LNHb98oJ)Yrtf=;n z`;GSEJRb+mAm^20oEQ{Y*4D}4u*#+)mr~B7Y4w;0!(g+U?_ch5d!S)8RN`JIAwT zFElN7^Uiao)gZV%%OgzD#q;OSZk)Yz_IzMlon}h`%>e>bmwaJ7??PZ;4OkIp`5X~6 zoy196nr;|nB{nF85E0BO!5Gt0-rrf6#gW8yNR?xm7QzNLxYpYTJJ>Qz*G-cmj^jqN z=6T+6f1s6GUS6(*gcu3aPJvjuyqc#&lM;)XlgVTe*8Yu? z+q!d0QY|hn9vvM)2u;&;UANV09UPw|X+}*uY&0#?OpEgB#+$C5Z-q97$^@G1?d>(Z znh=6I#@^muydg=V~1>MZU}XG`riwds}3%g;Uk>Pen$wEZ+0V3YWkr7L(c zpE*uTXi>Kag-}2|d7hpBWvRbY7p|?Hq}`5kf9Y zsc9@Muk=U9!?_A;PF9tn+n9{vjL)cPP?J9+nttoI+^U8mO*35v4~+9 zl2u6@`F@b*F{M}v0k||xUGohK76Rxh*Te`C4lE0J77bm?^~mwb(06<&g%$eKB+*6g zhcziQ<9St@MXn*OLeFZ3zGvyGst`iM$s9tMBr+{!7&?VdylFQYYpu)r47b2Op5MIeX^uBdHdqbQ21svddp%I({?b0FK{ngPqfOfWqqLalcFx{E1RG1E$c zC@09lWh}}pvrJPH66Z<9^M!6(CzDC0-JtFOIgG`u!nRWj9a8`)iX3Sz0T4hz3@}1G zDO)XKN(P~xvCN<@(nKM`MCsbDmZ~hvW~c5_YCtX&2bxj>=TT}oh6a)gwOph>@6%qi5Tr3zHf)B^A>efr^(^nKv8l zM$HqlQWDSOlvRoAc#}!gTUy~|v-8-~Z+*w@TVFRF)bH;;eE!^@ee#RT zYiCdP`}M_*@szb{=1bo^0)~iYjABy&6~~F|`UNYb5C)+|$*QUzFS`Bd>2zV;;_=cZX1iljwsxq*w^<#s!?U0-vYl_ti^ zoyC*ENhPpjQ;gzi9BDjSZwC$>Rnc*lC5X~)!!tA$T!{iR2{o1)D#k>&7OXbwq>MXF zujk0g^tdACq*Mw(!$iO!oO1vn(^5(SArwsJaWRf2wk%kg!$MtEH$W^Vw-X9%V5J&06SDL?zH+y>obSDkLn5B5){%x#K}QMA+dN3P8PC z392$rW-}MV8deFPkFpF02E-H_!{sJcWtvsR#?$L-4_@*b&DcxEsgbE{KA)SW$tzJx zkW_^m_|o!Z#X`?D41y^V2nZsiW)Q-KW;@QN$D;@++r`917Dh%{vUamE8=RWN778|f z({Y^X{L~5D(dd+7Yq7Cb!CJkyym@)uFhLTZY@XW`@x0;tzVAW;nGp_@w|m;Z6rz(d zzj0?&^-rGv_^*BBqks42YlBX0p$O1BzV%)Awr@T8uD71fX2YZS`fC#|N>P%~bSj+`7t4cr(Fvgto`wE!QN+~4} z07_Ye8k8({nzn6I+m(PI!?G+>Y2i4Ql+rV-n&ny!%<_aW29nYPq#R5+mq6npXRN9$ z!(3WfE{e=|gV8)~w7RZowE{0s%B_3j0#t^Hlww7ZF~+zQh+v=*(Da!n-#Qr`AgE4< z2TF3KVc6*8+1#tU7370a-f1jF{m6pmW^egV{?~6&f6;dr=7R|W*ucuQwd;m^yT^uM zr&)r59*stx*GQA9C?yvaR8k9xg>s4M5JN(cmCDAhWvTX3SBK46;Sr-V8%H^tKJ>(u zt)tPUhaO{$Sw?j0)w`Xpt1U;3ZV=eBJR z27@fik|Y5TkNOAy@JE06-gS-0d6L@;jfL@W@N2*NuYT#@{o~E0u8<5O>5|~;>eHd` za*c*1aNJ(W+|X<5cp4pSdrB%qH2^~LTdzF3xv}ud^WV5|=_0^L2*H)i^W3&9->Vlz zkr(me;xc1`u~JK(#gS!Mwh}AdCSVn^F^1H7=56nWT!yZlWTSTIusoTZ_Bkb$-M)S> zTs(J$AoBQQj|d@>Bso4fM2uCX-r4Kl-aQ=;M;gJv38|s-YK}}Yo6iwKl_)7jX)VS^@lmKTL%d-JUL z^p~E668dhyxw!8X9~~V@t@5%^0F)hFmX`DnQcYVigc!|t}&U#(zkt<8f9b>UsvH113 zJpERnJkEep5Gq+Dv%sy@!$y%*^)Q&vra%fpEU68IeZS@zCNZgnG#~(KE(jr|5JJMu zjio40uN@6*?Tb;w3}V`C-OuO#s`1?GpTTr;=eg*se@2&M)#XR}ChZV_yoWc%)RyImiQM;sH56z9AW+%gOVpy#++s{NA_ zNa($LJI{aP8=ISFKl_C*?VjwJ2Ko4(ew82mNU@$~?EJh;cB^i}Cd6=;&Yn>VFb;$U&@uM~RomQJTh_^QsgSs(CyMyzu^j zl4V(hko!cj`($ZC2moq29Yd&rMxGl;p_SH{kl|=FPs(m1_`~1(?Kj@o`SJhn-~FYZ z`&&FJh0?FRdZSyj{`rspgYk4ID`caB7S})g)J7XlR~Bm1WT>Gmquqb|Q{Vry|LnUT z+Au9QP)bEP3q0?5I7xDOczOT`LHE8ArwKurX~~sjj8#>Yrs<_i7nOt%VoGT=87WaZ z2I<(Ai=bl}h$&4hbo=_t8i7K}d;9%T@SxT*>|lI4?vJOYNrxwQ>mJ$JzrDOrpU!56 z-<(%s_xL0!^S|}|f6b+&tYn%_y9@Q8UZ0P%GwT}`)>&1>QPFF6G>_i@U|qsA^u33# zoUNITV_Id&hNG!%g}sF}j4W&#l+x*R3ZP2TOlVYbrU4)ZWm274U+IK4DiYQMVs}7}#d0c>sBQhwZHh|Q#GO0M12pMTo0f3NU07CEW>|!jh zT<$J)!XoDaZ#aa^mfN2wf)$I39GdGhf;_}F5*wYh%f&d$VYwgok{X*eM` zO}JwC^Pjv1vJEI|9)V>RYXCgoxlg%85Eza*1 z@@+y*LJR~n!?Lm+U{k>hIe7$(p#jN|F? zQxC4K`=*V7#?YW#l!-2piR(eb%@_tCP#Mfv#yKy!3!qyh<+gBBKw z2cDlMb4qX&%?@u4NMZVfu_{AYYO@SXyPOqOnUAM)DOjNtt)VQRZcqElUVZM>ub1^f zl>s%Mec#{vg;uSeGV3!R|GXKN$LWqV%%rG@0_WD&(r5@2#J27D)+Cy_VO;_fQlsa& zX&hHtB5RHSv@PE@V8bzvkA{UnjB5iC1DcEpVw+ae@g1L-;{D(DeXG5duYLZraXi0v zXMgW>es_DPoR5h%Z*EU&fzzrPah@Yd*4NH)f#10C(pbZjNs(lNrGg8Ht#Wt!^iTii zf8@!_-}5*BPNUm2Af=Wmv~1LDN};BPK}^G-CZQPP(PZp6j%zz*mfuH7LkJNhtP-5b zEK8*1^Z67)c%O7FfjJl_Uwh%2p6kE;gCG6ES3bS6c0P%-*YEAz+`0REAAer!Ov3Wy z;6K{Uo-FrK>Z5HvE%e06h%c3SmuJ6ptq-NLzQIrck9ZppR?%drMSyHz! zE0adErU+3`6)YP~1}CRC8w=*8$1Xke(4%M1U)emfez1RP3AEi#;Cvy?BcJowBLkJdxqvfbOa z?-D{ZV6w{VzT;XH38jQ)nm3xhQYxC4HV%Q8&c^75=QvB5O0 zkU5MqEre$|t$MAjA})%m6eZ(HmH`CYp+ij}KxVm`iSo|9JJBcs$iRjddVZD*2@GN) zinXc1cz@towqNt1K{htd_?9m#CM%|d)D;(bxzK3rZ*84fT8$=C8)MHht1Mk#U92@1 zatR=Gg#=L?zx?V;O0#)1N)a<$i+B^$B%}Eh*%qsE2~Y~@a5`-@!cl)0N?5PgkwLoM zt`I^gwY1rrv!noyP-arXC|C2W1ct#lU>wZmSvr}`hU0>Zke|K#Tt7KXzU|4M_{fKT z)bLh+>EHj0E9aNfS(JdAVWWAl4G1A|9N(WBz?dS0(lo8rYE@NLSyc1vGRfyf3J^Lv zop9z^E>T)&fV|MREpT|crwPcoWK!H;(bN)nj=Q|P`WGMjgVFc|o8auZOW*pf-?Dq} zmSfrd@g$1lQU7injk`;WmVrax^9%zZ{J`a_ORa^4PQx$^+jCF$lOO(ppK3IhPlltN zgTwcJ%eQ%r<$7nC03{ffLfei}RRuM$QsREY;D7hc?*~~Zr4T|XWw+bC4}>U+!m_OU zoa%13Ya->E)HPrs_hfxB@HUp0mmyG+I)C=Lombzu)oGqHOwTg?2QEL{U0qw;yfB@` z2pZJ#zWmDd)%8mn*S2;?_ZLJvoldjUCY0K?Ex1OAhCw@vVx+;j_079??=r@;)~4&9 z^head7zj$VZNo;r4iI62BaQL$Lf1B(MrWb!kcGgTk0ya%D=Qw+lS1xoH&-}}O z_X~gW$!GIYq!l{uA2jN%u(c4)$Btz`_VD9z96Q8h4=q^I3+CjbHhqK9el;q>8oX!gch|nNOvi+ljc|3XjPT%mB)3S); zIBbLk&m9xRX`V=>O?NyV_d0b|amzLo6r6M4_dU;3hM_Fua8w)(r&%c~0ZYjOV2)W2A)8j7i2yA$XQ0zVAZ_tEvja z@IHCA)9L(g^omkSDOIc0AcXVz{JvWVA#s`VMzBir4MXlbE2ymt?gb80(M5KoidZ8@CXz&un=Ll?_3*Y z;CLA2X%c5MDJ6mSJSnK%~~@sP0qw=|Jbt3L{x@rYpDpvwq;2nlu}wMtN|B= zRJo}cuUH{L#YM(oR*_O*DG-FXDp}K~CdjBRD5br+?+6}e`8*gn|tYhIM| zEGwvi+?r!JZYAKYTiaJ2c&OIsbX(26y_3@cpu_=rVzSsoqGoGZPRdFF63jD&2tC-| zrCKw_?)y=DJEQqr*deV4HUwPBBJ@qy0mPt8a4tp4$|TDvG5T!$!fUUsb}z+~_^ltf zSYND1C7-GRWcFaR=5%KYGP z3L$p9Kq`>Om8C43SdE&0czS{e#sF##Ij;bu+A>=Uy>5E}=5rt-1wfJ{J9{UGM@Nny zC~O_>9DoA!M{#F;v$Yg-yZ)Q5oE;wT&f|9O%lZD@JHPhJzY*sO zQ42ziF-Fj=*XvRW#!K6_%d$+<)N!2q@Xa6y%CbZVA%v7tLI~gY4Z}#1gb;F{XJk6A z!sc_&J@@u^{uRX7fBLmw-x_4I*^E##Y?AG*Lj{$Pxis8g{j)FqgP;7l|M<(lJ)TV^ z;AK8{z2^V@dw=mS|NVcvyS-CyH4(l%n@JQ`2>)1(;8?3fc#=%ew3WyL(t&A`BbxmGoe$_Cb5*M8U6nk|0oGPDiULxnhU;23)e>Xq+fFMq zS;@y!U6fhPqm2M^EO?o1E_F|bk#CzUA0dL31X5vzVIeDwabWvW!NJj~l0s-eT$-g( zS{CVSWJ=|SM0qexi+X#3xaQ5>Tb5;g$^5gCR0o zjhrAs5pml9+PqRqx@R^YO$)G>m(PEGYd#$ew}$g6u9c2&kp^u)bpAr%V8b$J(@0Cf z1fP|8QRRYb*QN@n;b;aG_HBz8xYMmGSxwVa1Cr%TW5e>orB;Zm+|aVOw2(!yY1)EI zC=g4kPkiR9rPVw-oa}DB=6ZH5Y!CW}Wpcucsh9I&mRf$@>~_P2;P%m8!4cE))6YM9 z=KR%WYhgI-=Oq#f_d087QY5xPo-5VRd1sw(tc4bXTJZJ&&5&AL)Li5Zu?eK1iKydT?rEDIbo(?ihTt2+= z)arZN?JvH%ds0}J&$V88?YiSMO4O_r(n7Y|KIeG%q#ydW>(s~N@idMA0KIkx0Dy_e zvMipbX~iHUi=B>?k^)4a(hwwBifr4Ez;T>B7Z}SyfB(#S@4=^@Dx=YAjT)wXb9*O= zrc&}=uUpOH<0vIoZEI)0-fXzO*Q|Nlx30bUt#8|U9wMzcB8+O+K`i9OTN2o_Z-Iky}q%?hBU?}bHC<#w4k#*=XpxKEcLtbmUmRZ2{XfE^M?>l_*S- zN;`%b#j$PMgb+lk%ma+ihX$d|-qLtFX}0`noY!2-gR)GDvdDADc5M8@YtO4n zGf@e}h2j~@i;{oqw|J zo7f^~UgcU~imgJas;Y*=@njsg+wIk*)|utT@&3MH!bZo}#W3P3SieMVTS(xA9SljF z7t`^~vT>~yjz{y8)1%GRW+wo{$#^iCkK@#ELObXb$|8o76&zxdXL+X)7F8*@BV=Jy z!$bsQ9L+|qO%VmR@7=z$bI)*X->@FM@W|H7+YFkIUp@EGTJ4)J-p1HDd-iNl>s3rd zS;-|&^3-?iG)=220~(s9$vHnfJhUv5=Xq5z-!&`@lv2@b9@wtr0@_xo6s9Ir+H>4S ztzOEi5c!Y&gTMVFfBkRwTFbM7r^38>XRFP69UP`fZW{KP&2#N`Gs@H1>~w$s*12j=?-n70^nl>2!Q@dJxuwwhvn_=+>+}&F6&}Bq@LrLg+e<;2dZTAKz?UIlFw+ zA8=**b-d7UcTY|i8(o`-GAUA2&w0jJ{?4nb5|%{aNonltj3iQsT0Iwx=EI1Yjrzjw z;bGSgFF*Fk!QE?Z&)hj4_qrXTt5|aZ&BOf(rPL06PN0>GTC00}v|INb!}@=-F9voB z$&u0}t2lBNdkeEs-wn)coDtXVt*(Mx?Co!fTpYxJ3vnN#K^;?P=|am{Xt^St5zDe1 zn+t1wqdOUmk|?$f^W{4O&vPinT$GfWuImz5A%yZ&5kkX8EsAD_Yg5i@b^G{aWH^3W z@}evkTP?*ZO>v&*5MoLI5aiBIKj?MSq$sOoCG-NvlfzxvwlNFU+SrYz58w;;=s%Pv-4TjSw=JOslG5c@c!x z($d0Y91AG&>6}X?G(`Z^f)HYaA*#~cYX-0quIm=ODzX%6@Rql~saxU`W*<>Hx@!~Q7A^@a6wdwY9PoIz}w zn#-!Pe6t+{vn)$;Dh=wGmSfpKS8-HO4Epm_OIdUMS&|@15Wz`WBA`tJYY6kKG)-Fr zRF%19Qh-TW=4Dw*t`I^?OG~C{I-zYSofRcS2ICs2^1_7+ue|a~l+5qvX&7Vo9S*JF zeTmgFDTG)Gy?$mT%Q9Xuj43dP3GqCM0YF7oCP`EjnddvUp&njt)IyVKXFM-T&0O1| zl1WG`YTj3pP=(dK-J$0N0)c6f91RkiHYK2TW2wEc5l=_)WY)2XXCOIGO+rlp%IH)J zS>(1b@FS0%UulJ>)0tsdhsodz-*^#V;1gvSG!yyCxw8n`6ySwcjX|Eq*=&%U6#2co zHyx|YUObKBeO)IT61uULTo1wSpdW3+GT1xQYY)}sylv0&P1w#<>FmO>h zIoHAGTb7B4)o_Att?4zPX4`v*(>Us!+dQ7kR7nhkJ)6}#P7cyIV;GXfX2Z9E)G$dB z!LfokMfxO3ZO6-Ft_obM)yA_^$5N&eo&%b7?AH-AN=+EcIRR?5?qZB9seK6Dx;?IP zN}!V13tV0f8;$mOJWOZ(Dw;af-P=8a$@I!DU(BkVpdU zd5%MAXJG{)6nJM=muj7jm3&stGuTh87%k4z z*}>is)(QdK>2$nC#}6BZYfD3}Z?1jId!O+tt^k&*;-w@A7Dd%;`2d3`E8Ln}Ri!8@ zixR;(W74*5!z5f38dR2J3JogG4a;%7z%qR$(dp@6|L}liysR`t1OVkaL^8g&y@NCe z{F?0t8*7{Ei_8D|Xa3!T4?NLZSU7w3?1f9`T_57Q*IIP@CxfaeAq1RBP^l~}lmhWQ znob6cYsWSN177I)%S$zZd<_jiv0ZN>f`K9ira~x!umK=^`b`(|QFQUr=6G<*3<=nM z1U8>X(+4-#lO$@?eFDT=9yx#S-d3DO0Jmm&PO&_g2N|DSS?`Uf<^E{4=)q>kPg9U^ zw&LJnoP}Y?@{HJ_LO6<|X%T0MT4;9ws&v7+J@;@pTWWVPQt@mSrQ8piMI|w+TAhXI zEb6s`&~{&X<#w$RrmS4)Esn;g5V^11+LeHGT8?Mh4${rg&>~B6)m>_2d0EWMX_ke} zR#m1}5NM|M1}7G7Ku8;b8l{neTqJp5LJ1%xZXp=Lu7sTN!Y~m*G);0sNZYpy!Kl_1 zFKW$}9|oQT!@=Z@dnd59lt)QGwI7&`z%-CL8BU!>gCJ!AY?v5>>UdH<`Pc+Mlm3_M#*T8==w=L9-HaYj?LpPcTwvF@#VD@TPADxrvISs)C?AWS!I1Om6)8jUf*co>=P z(L$03k^l=dP(Y z2J4fP>H7Nm{o$jsDB{3Bp0G~XYxjF6!%HRW=-bR5U`eGM$75-e5O zG^S~%s~ke?y0+vj@?A=eqUM3=oGwR(;!i$#kwCIc#t|vJP9Fy?Lz@o{hk`R?sfYVV zYpwW+GaGNdcaOLpqyRONT4qxe=*OsG*}HrRigy( z9P_dIOvt(_xsni2KoAv-15AVvoO1=_>f^g^u zwrhEw$2mtzV~oqPB$kOS!$PD;Gl*%$8qYOr#t^{-J483NrWh*LR9Z_4sn&Wnn=vkc zlD=z00<1Qku%?R1Okxrz;l|af}gtDxwx~v%rgj~c7oAx zPE48=IWbHk_-@BIckbMcTQ|S|(;t0wd|NgdG&i8_w9N8%{_N&Sil|gkD^82F?c1%k z@48+CDCTT!5YJC121aQmr^&3{X)WuFl^QA8jtt)qR{7E}Jzh4LXx5ZmVaIZjMg?bO z&4kjvZ4O$Udk=?RY-=Igt=1%&?4PU_X}P`LJ9l=MG%Q_aX)#;h=*b4MCNI+Z(v>R@ zjz{w>qewSG7G)g}DjQ}IoJ&*)fe=|6^yZ^O!*oN}s)cMgZgtv4nPCha1(nRhz-^QS zro-}v!-D0Ec!3XTRu;GRPrUYak!9Q6s2kBLo!N$lMn|gJuniMi8mM3Sm0$Tke&@Gd z`P!F4122-*#zuQFlode_j)&Kud3JK|X6QSmRQ=xC=+U95S>rg%#p#pJJktvOn>TJK zf+q9Qw(Ts-G%^icbt8)diXhzC+P-_|Q7!1^)~;-(MVSI=_(5=RddP|rV;F`$)mVkL zL8$LudH?mVp4;x`k_eUc+FrwCoiVPYA4N*>EKi`>KR%vqY-}Z2S=Y5~+n(n=23%%E z%{gziS^#OwM5O9YbAUK-NEn<46?fI70x+n?@V2)M88B6CIgSFXw|GU5IlONoA zh+PY?vMej#4<)SQR%d&yQ;d#|W^UfpI|EXtt9ix^>MWM?t)0!|Nm|t?uZ2sAV_*X` zamNObx_)#zNs#T`zdr#8P)Z3QF4i>{c~L0@DK^Ti0GPP0o98((un^+0NUbbt%W~?5 zK?os)o?{~n@;oa_?z>(xUudWhLbh$UqPAsORaH5jo7a_v$YYBwA%qfSJ6@KTjHiG3 z2fyn-e&rQ{-8|2v;@NESINZs#T+={MH|>645#*V~rY=`pHTYzHdTw`D)CE^&t%QIY z*NyF%MOBc{c6YYdC-1z!duHvy!Q}i_>%9-|hTZ-&S(?O`%6k3%{e#Ks;{IfFvzsk+ z%2200NKbBm=;FY-pDZP{eFIAhkU5{_q7j@a;E+qd)aux+dwwQC^7m?tf4#xjq=&zjYJ_-rH}y+;%;) zZXO*R*@k_xT!c}}A|wu4#|Psv85`7EWGmA&xz?8HawR3l3fpB`^}5}cMRYKJOX|5Vfby3ZVWR8jiHZ`;& z$%R2EQV?lCOjMK&hQ#;%qGpfn)c@xR9w==Z24{_F7>^g5Wg#%p7#W5EA18K7Eg*z> zp08E~rPOg81t1_;N-ZEv^DGMdrU3vlgy?EyC(Wfa-B0IlDlHHK6hYUuF~%l_2Gwi* zE+NE#P_nu~OQ*A>7BrGkC?_UWpzUCUqZM%ON^Up~9h zvg=mt6WFkNk(36Y?$;ko@6P$dN!g;o$3OaoB^Lt^TQA@oZ#_%K}iQOw6nDhfi;;f0itDH2`+cG&p>L+M!<$w zJSj!PnyRczQK;t{Wp&*?kCc(`rT}>YmCjW_uvUCh&Vvx*F-+J@6 zbTJLsjh!|BpZ(ga$}tTG7pZlyKLje@+1M|#=9w$` zaQUNu;R~~LR2S&_m8Tc;+zcI!;CM7knRGq*qd)M;$^3p0?PPV_2(F6Ip?hG{+Xuq~~|2hEVIOVF0M2$dnQo zBM7h*SV2Tg15#MA(zZ>l1Oz|>s++Ro3QABM#8DJUDH&skKm}x)ua1r$&Bv#Y4(<<+ zPoSvBlf`&^dNMqGbo@w?lVAFkpUWq&T1do!;~O+GZI`k^3wJ!Yv)-YULJTS)G;t~^ z+i@Ec17Vf29AlW5V+#?}F$vHRVoMe9!yel$HfT`;w|y=xc-mJ*4kkg_Z*%XW3$Z0w%@>bG8b>cWgQb5wOIy^Wr4CBg`E8CsUyYIYp zx?G*v-ayp*kstjFzx5mcc0CG))9EV9vjz~$1+qj!8AOHco{}{(5G!R_aX~cC^Ywl= zPZtLED&9~7#C~)6AdD5z_PnlVZ*#!8EH2lNxnd`i4zRoEB-|I$R?yiVr>6|KNjzZX9n%!EiCP zUEgfEKl0~);NSh*Ul+o7bi99lBeo4!Be}cTUfURSdKaJ}+Ux!O!{wt}vp@Qi-@0=C za#q#%KY0J@6W8{SkEj8P3>IbSdp5ESt##Z2a3~rc_ zKnPI_V+~oOH9!~v2o#0D@+~5{jWqS_b`Y5u8Ya++R-!f?yJ9tj5GaTgHYg$xi$+$W zidw#A1vF{4Y9NFZaAFVw3V3C!?*`VH&9Et0RxqE!cF#VU%zB-jMOGVZ8g`-wqpIcd zt^V2RGJooFr&jT5aROE4xBSylCV4Z@v&9N+^a7hSNy-F8gI+6{H(b|QT6>-aL>^FO zQ}V)RzWB#~_I06^gKdk7@iLcO5@HOxZdS2IW3z&qfUM3P$I%j0c~#4@-|Jz+Xe1p@ zP7EMi+ifIve2H_`Afgc`k`|4qgA{KWvdklp)j(Ts({u=gV+wn z^Ko~rhoD-eOgEFum-|m&d-~Sg6R78+1b(I zQy+fP@p^7-WqI^Vzw%2aQ3PlTX#;=(d77r}ZU;fRSkxs05a0%BAw=D*0*vb8r!xXD?%}`9*aY{6t3$&mZ~*1Ll6R_F`{AAd*S1M?mOT4&T71marIbV z4iMH_=Xq{Y@>u`CMXi)FArwMT2nL;27)HoMu7QxOCD10JKyic+#lfOpVnRw*6Jqe1 z`%dV%X1>ZqBO9O;lA(!-MJY9vB)ncaRQrL2Bm@wXqNq66oqp^_AZ2pn+|{+MGpSJ8 z=;f)re)VF%eeL$q(Mh`eov+-ye{?ch@Nd35{=;wVzxLkfWFn4EWVg3I882PWwy6Tp z__3e-zkS~i{}fQBhl!(vYsj5-zqQu+*0;Y3L}mjxUo6Mz6kx;{b1eIB{LDXUl*x+n zc$J(^rU>Bie74NeMY?LYJKO7hDfIC0LD!~K8AVoVjZEmQZ>}Stx~`{15k}5nt&L4` z=HmI$d@{)vpZe6tva0&TOCPfc5qXB2!m+7gaY`vQpy4@t{a$1`7*njd;4EXh)9Hm_ zEF_;yCX>ZsUD!E?TxqIjY5C9T#iSl6aiK(Q^q#Wo=qFHzP=85vtp&j@cMJlL0*^;t#<}z z&z=>$3N0h>T_I4KHc6IhsWjt4ke1)3cvfqpvwOxcg2`-|mzm(Tfi%^P1ZY(w(_dTe zzqPk!O_M2wpw9Cj`rJo70}u%A-MXn(C+F4&6yw2WARxz-TwDu^$*~gB0LZcIT1zII zw63TXFIizYre_$_Ns1sR3& z{cC&S`Q12hlx}hhA%p;;NQeev3u039I-r&bkfjj@(6Q{gEXRvk#TyBMVOR*0ykJcu zYF1QLU2(&-f~3&!*Z$@I^y|O&%X!TV-vABSiOH?g0yre{K*x?eKU@_v*Km%8nP)V6 zJDvCL9mj)98*Te|vUun9ch8@1dqF=-Y2GYgU36pEYMGBtlh*b}<@Cq~Nh0ND*WKQ7 zY}0NO+@B^r+CCbOvE$BGIWL&jvfJI`hI4rTZUt*t^M0>YraWl5by2#uV>xD4RvMFR zJnFULg4X~5N+~s!W-M?63~`dLG{73d_SU7vaKBXLdaIu#Nm-T~XU@isdpem+4o?Ip z9N|`I%(Cp+r>;hB@SShIS&>XKu-<9A7MYi9IG?VyJq=7qV7g>1C5A;b(#R}XElq0a zs@PuZV#7Tej{yMdEkEpq$#nJb^r+_#wzfOV)l@c8Kx#U%RthwAx7|weB_J>>atooB z?@s2afT7^^`nr=YinPc#`)yXHrfG`0$pHXdnx?rZxrTt6mMBUfo2yS=dF#y&0Ja#y zC`k;D6nrHb>U*tKnQZqQ3&KWt#D#%J^pb6!-8h^s4#%YlQ-wT!mO$5 zY`W|YLfecKM0qw`U)%fE8>f&^hz+J1q>;j^?V3U8Ke+#})fp_8X{|)y`J$+}W|n6Y zL6xjbqKWS%Q`n8Nk6*v`{_S@aG2Ou$%BL4DUwd>slj-c@#mjHr|KNp>e&LO;ed*aJ zFAT>O;Kk>@_w!h&cE5M?!O@*}-jub-XR`)s9Q^zL@NXe8 zPp8X@0h8d+^=cs{&`SC>Pbo%FHS6O%VC0)zX8e!5vU~(k^g0j&XF~*U^(6bS0Oo}+u8rv59@TIk@=Qp|5w;l|) z`=(K<{k%ekxwkdQvJ5Jdi+Xo=cbUxBV{!1C?pa0eW@N1r7_*!o7dIa;^ zcV^>dV>_Pj;AK%ACHnGOC{el9g^-mmJpb(J@v;&{p`k6!1*=v$A5YW3G;-Di?X}q= z15(tgZd(o^jbl1YgQ`(Xf_~^rjg#fdGXPSu6pTRbI3A(07j+b-^VJe#EH&U#Pp8w^ z4_G6TWPaw%nZ;t^0yFk3ri)6$>0|*df3?UfAzFb=AhApuM4QR{5i-m)E0UD=+a18l z8Wv|Zw~h}-h7Va*Kq=OD&iire&Vz?N-$~OHP`l!)-ELD^p}NG>5QNk&&31-+{n%Cv#YXs`HfdGGQ!vp9CX6A1Ld+>0RTeV%^AnGIbO^t zGAPipZZL!r8x4Z4p_Gu+Hf?Gz(|NnUvH$3_$m@Q`?X^8A^=vYaJDzCd&;P4m{pr8; z*OFC&9M^NKCNDGq8cWXUV!C|cL(eRyqek*DXf37-FNPca!DLpyd}C-Bfy9(*i4781 zxN4Z|`s35F=Xr}FtMVfD14NPSTCjnJWv;T!Mo!V3e)lJzzWwOlU@)jF-HqDw9)Lv zNg%A6m%eR0_Tv-JG!el$6IgM85wVIYw=EAqu1X0I6hNDpXeb&58!3?DuHiI-3jn#+ z2s>6@W?I#RptEQ9DvkQgKEs8Q@QbkeFiv73WeQ#%blaU0{MAJo@K}59Hq3heWO%40O z;X?x{LZJzvV^hbXk#E_i>G_Tq+OuU+6xmZxUe^#vts21*fSlEmi$$7Elf|m4{4g>c z_w4x#{cfBrQ=3{_TU(>i$oKuf_}6|a5smgx=&x;Woo{N|01S|+fzcWS8s|u5TI)qx z)e?I?@NHq3YIAeD$Q$3bEb2aS<*9DC!K?cI-P?{!%eugpB}5G|SS;rXs1;HREsp2Y z{#rN93dfBX%M>GUdU!BAIZe~lvAww6q6E8+Wm!&M^ZR%2ZEp^!1vMtU!P;`UTCTDe zUi_%zg{!=pP7=fM-KcF@R^)kSb~gLHcDLKjs>(LISoy0dJ3dZ$P3Pkz4qcN1$R#!X z;dtKi%`<~uCkX!WKmQlgd4B1_rS*O*na`6fv#mIe+uPfly-p{~vb0Kruv4?rb&UP} z^7tsNvh>QuK{7uUqNzn?5j>ns33XY`J;!b83TfrJc2Q>rCZgg^Q#VbcAyP$p{ZhZo z7M6rv+f_+={NTRjS{h=cti$8_!yoznIPA22Gh3xXf?gabUKm*Mx_a-u8;|ZjR1%bB zX?Tw5I#pH8lf0>(Rw)6@1C3{$!ce<9r#Ayf=BxgnbzCu1B