From fc5e65a22dcd263301522f289a3d1b852f001888 Mon Sep 17 00:00:00 2001 From: Yufeng Li Date: Fri, 26 Jun 2020 09:29:29 -0700 Subject: [PATCH] Add quantization support for GPT2 past state and use model to generate outputs in OpTester (#4340) * Make quantization support GPT2 past state * Make OpTester to be able to generate reference outputs with a model. With it, there is no need to compute outputs manually, which are impossible for some cases. --- onnxruntime/contrib_ops/cpu/bert/attention.cc | 84 ++----- onnxruntime/contrib_ops/cpu/bert/attention.h | 23 +- .../contrib_ops/cpu/bert/attention_base.h | 33 +++ .../contrib_ops/cpu/bert/attention_cpu_base.h | 218 ++++++++++++++++++ .../contrib_ops/cpu/bert/attention_helper.h | 137 +---------- .../cpu/quantization/attention_quant.cc | 71 ++---- .../cpu/quantization/attention_quant.h | 4 +- onnxruntime/contrib_ops/cuda/bert/attention.h | 2 +- .../quantization/attention_quantization.cc | 40 ++-- .../quantization/attention_quantization.h | 3 +- .../core/graph/contrib_ops/contrib_defs.cc | 16 +- .../python/tools/quantization/quantize.py | 3 +- .../test/common/tensor_op_test_utils.h | 43 ++++ .../test/contrib_ops/attention_op_test.cc | 29 +++ .../dynamic_quantize_matmul_test.cc | 156 ++----------- .../test/contrib_ops/layer_norm_op_test.cc | 215 ++--------------- .../contrib_ops/quantize_attention_op_test.cc | 65 ++++++ .../test/providers/provider_test_utils.cc | 64 ++++- .../test/providers/provider_test_utils.h | 10 +- .../test/testdata/attention_past_state.onnx | Bin 0 -> 1056367 bytes .../testdata/attention_past_state.u8s8.onnx | Bin 0 -> 1059004 bytes .../testdata/attention_past_state.u8u8.onnx | Bin 0 -> 1059004 bytes 22 files changed, 591 insertions(+), 625 deletions(-) create mode 100644 onnxruntime/contrib_ops/cpu/bert/attention_base.h create mode 100644 onnxruntime/contrib_ops/cpu/bert/attention_cpu_base.h create mode 100644 onnxruntime/test/testdata/attention_past_state.onnx create mode 100644 onnxruntime/test/testdata/attention_past_state.u8s8.onnx create mode 100644 onnxruntime/test/testdata/attention_past_state.u8u8.onnx diff --git a/onnxruntime/contrib_ops/cpu/bert/attention.cc b/onnxruntime/contrib_ops/cpu/bert/attention.cc index 7698832874..760e86a1a9 100644 --- a/onnxruntime/contrib_ops/cpu/bert/attention.cc +++ b/onnxruntime/contrib_ops/cpu/bert/attention.cc @@ -44,7 +44,7 @@ Status AttentionBase::CheckInputs(const Tensor* input, const auto& dims = input->Shape().GetDims(); if (dims.size() != 3) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 0 is expected to have 3 dimensions, got ", + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 'input' is expected to have 3 dimensions, got ", dims.size()); } int batch_size = static_cast(dims[0]); @@ -56,7 +56,7 @@ Status AttentionBase::CheckInputs(const Tensor* input, const auto& weights_dims = weights->Shape().GetDims(); if (weights_dims.size() != 2) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 1 is expected to have 2 dimensions, got ", + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 'weights' is expected to have 2 dimensions, got ", weights_dims.size()); } if (weights_dims[0] != dims[2]) { @@ -64,12 +64,12 @@ Status AttentionBase::CheckInputs(const Tensor* input, "Input 1 dimension 0 should have same length as dimension 2 of input 0"); } if (weights_dims[1] != 3 * weights_dims[0]) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 1 dimension 1 should be 3 times of dimension 0"); + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 'weights' dimension 1 should be 3 times of dimension 0"); } const auto& bias_dims = bias->Shape().GetDims(); if (bias_dims.size() != 1) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 2 is expected to have 1 dimension, got ", + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 'bias' is expected to have 1 dimension, got ", bias_dims.size()); } if (bias_dims[0] != weights_dims[1]) { @@ -80,40 +80,40 @@ Status AttentionBase::CheckInputs(const Tensor* input, if (mask_index != nullptr) { // mask_index is optional // unidirectional (like GPT2) does not need mask input. Here we do not allowed the input for unidirectional. if (is_unidirectional_) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 3 (mask_index) is not allowed for unidirectional"); + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 'mask_index' is not allowed for unidirectional"); } const auto& mask_dims = mask_index->Shape().GetDims(); if (mask_dims.size() != 1) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 3 is expected to have 1 dimension, got ", + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 'mask_index' is expected to have 1 dimension, got ", mask_dims.size()); } if (static_cast(mask_dims[0]) != batch_size) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Inputs 3 and 0 shall have same length at dimension 0"); + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Inputs 'mask_index' and 'input' shall have same length at dimension 0"); } } if (past != nullptr) { // past is optional if (!is_unidirectional_) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 4 (past) is only allowed for unidirectional"); + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 'past' is only allowed for unidirectional"); } const auto& past_dims = past->Shape().GetDims(); if (past_dims.size() != 5) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 4 is expected to have 5 dimension, got ", + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Input 'past' is expected to have 5 dimension, got ", past_dims.size()); } if (static_cast(past_dims[0]) != 2) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Inputs 4 dimension 0 shall have length of 2"); + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Inputs 'past' dimension 0 shall have length of 2"); } if (static_cast(past_dims[1]) != batch_size) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Inputs 4 dimension 1 shall have same length as dimension 0 of input 0"); + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Inputs 'past' dimension 1 shall have same length as dimension 0 of input 0"); } if (static_cast(past_dims[2]) != num_heads_) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Inputs 4 dimension 2 shall have length of num_heads", num_heads_); + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Inputs 'past' dimension 2 shall have length of num_heads", num_heads_); } if (static_cast(past_dims[4]) != hidden_size / num_heads_) { - return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Inputs 4 dimension 2 shall have length of ", hidden_size / num_heads_); + return ORT_MAKE_STATUS(ONNXRUNTIME, INVALID_ARGUMENT, "Inputs 'past' dimension 2 shall have length of ", hidden_size / num_heads_); } } @@ -147,7 +147,7 @@ Tensor* AttentionBase::GetPresent(OpKernelContext* context, } template -Attention::Attention(const OpKernelInfo& info) : OpKernel(info), AttentionBase(info) { +Attention::Attention(const OpKernelInfo& info) : OpKernel(info), AttentionCPUBase(info) { } template @@ -168,19 +168,15 @@ Status Attention::Compute(OpKernelContext* context) const { Tensor* output = context->Output(0, shape); - int past_sequence_length = 0; - Tensor* present = GetPresent(context, past, batch_size, head_size, sequence_length, past_sequence_length); - - // Total sequence length including that of past state: S* = S' + S - const int all_sequence_length = past_sequence_length + sequence_length; - constexpr size_t element_size = sizeof(T); AllocatorPtr allocator; ORT_RETURN_IF_ERROR(context->GetTempSpaceAllocator(&allocator)); auto* tp = context->GetOperatorThreadPool(); - // STEP.1: gemm_data(BS, 3NH) = input(BS, NH) x weights(NH, 3NH) + bias(3NH) + + // Compute Q, K, V + // gemm_data(BS, 3NH) = input(BS, NH) x weights(NH, 3NH) + bias(3NH) auto gemm_data = allocator->Alloc(SafeInt(batch_size) * sequence_length * 3 * hidden_size * element_size); BufferUniquePtr gemm_buffer(gemm_data, BufferDeleter(allocator)); auto Q = reinterpret_cast(gemm_data); @@ -235,51 +231,15 @@ Status Attention::Compute(OpKernelContext* context) const { qkv_dest + qkv_offset, // C head_size, // ldc nullptr // use single-thread - ); + ); } }); } - // STEP.2: compute the attention score. It does 2 things: - // I. attention_probs(B, N, S, S*) = 1/sqrt(H) x Q(B, N, S, H) x K'(B, N, S*, H -> B, N, H, S*) + - // 1 x mask_data(B, N, S, S*) - // II.attention_probs(B, N, S, S*) = Softmax(attention_probs) - size_t attention_probs_bytes = SafeInt(batch_size) * num_heads_ * sequence_length * all_sequence_length * element_size; - auto attention_probs = allocator->Alloc(attention_probs_bytes); - BufferUniquePtr scratch_buffer(attention_probs, BufferDeleter(allocator)); - - size_t mask_data_bytes = 0; - if (mask_index != nullptr) { - mask_data_bytes = SafeInt(batch_size) * sequence_length * all_sequence_length * element_size; - } else if (is_unidirectional_) { - mask_data_bytes = SafeInt(sequence_length) * all_sequence_length * element_size; - } - - void* mask_data = nullptr; - if (mask_data_bytes > 0) { - mask_data = allocator->Alloc(mask_data_bytes); - memset(mask_data, 0, mask_data_bytes); - } - BufferUniquePtr mask_data_buffer(mask_data, BufferDeleter(allocator)); - - const int32_t* mask_index_data = mask_index != nullptr ? mask_index->template Data() : nullptr; - const T* past_data = past != nullptr ? past->template Data() : nullptr; - T* present_data = present != nullptr ? present->template MutableData() : nullptr; - - ComputeAttentionProbs(static_cast(attention_probs), Q, K, mask_index_data, static_cast(mask_data), - batch_size, sequence_length, past_sequence_length, head_size, num_heads_, is_unidirectional_, - past_data, present_data, tp); - - // STEP.3: compute the attentionScore * Value. It does: out_tmp(B, N, S, H) = attention_probs(B, N, S, S*) x V(B, N, S*, H) - auto out_tmp_data = - allocator->Alloc(SafeInt(batch_size) * num_heads_ * sequence_length * head_size * element_size); - BufferUniquePtr out_tmp_buffer(out_tmp_data, BufferDeleter(allocator)); - - ComputeVxAttentionScore(output->template MutableData(), static_cast(out_tmp_data), static_cast(attention_probs), V, - batch_size, sequence_length, past_sequence_length, head_size, num_heads_, hidden_size, - past_data, present_data, tp); - - return Status::OK(); + // Compute the attention score and apply the score to V + return ApplyAttention(Q, K, V, mask_index, past, output, + batch_size, sequence_length, + head_size, hidden_size, context); } } // namespace contrib diff --git a/onnxruntime/contrib_ops/cpu/bert/attention.h b/onnxruntime/contrib_ops/cpu/bert/attention.h index fa686284ac..bd82f07f81 100644 --- a/onnxruntime/contrib_ops/cpu/bert/attention.h +++ b/onnxruntime/contrib_ops/cpu/bert/attention.h @@ -3,34 +3,15 @@ #pragma once +#include "attention_cpu_base.h" #include "core/common/common.h" #include "core/framework/op_kernel.h" namespace onnxruntime { namespace contrib { -class AttentionBase { - protected: - AttentionBase(const OpKernelInfo& info); - Status CheckInputs(const Tensor* input, - const Tensor* weights, - const Tensor* bias, - const Tensor* mask_index, - const Tensor* past) const; - - Tensor* GetPresent(OpKernelContext* context, - const Tensor* past, - int batch_size, - int head_size, - int sequence_length, - int& past_sequence_length) const; - - int num_heads_; // number of attention heads - bool is_unidirectional_; // whether every token can only attend to previous tokens. -}; - template -class Attention : public OpKernel, public AttentionBase { +class Attention : public OpKernel, public AttentionCPUBase { public: explicit Attention(const OpKernelInfo& info); Status Compute(OpKernelContext* context) const override; diff --git a/onnxruntime/contrib_ops/cpu/bert/attention_base.h b/onnxruntime/contrib_ops/cpu/bert/attention_base.h new file mode 100644 index 0000000000..b9c1934236 --- /dev/null +++ b/onnxruntime/contrib_ops/cpu/bert/attention_base.h @@ -0,0 +1,33 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/framework/op_kernel.h" + +namespace onnxruntime { +namespace contrib { + +class AttentionBase { + protected: + AttentionBase(const OpKernelInfo& info); + Status CheckInputs(const Tensor* input, + const Tensor* weights, + const Tensor* bias, + const Tensor* mask_index, + const Tensor* past) const; + + Tensor* GetPresent(OpKernelContext* context, + const Tensor* past, + int batch_size, + int head_size, + int sequence_length, + int& past_sequence_length) const; + + int num_heads_; // number of attention heads + bool is_unidirectional_; // whether every token can only attend to previous tokens. +}; + +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cpu/bert/attention_cpu_base.h b/onnxruntime/contrib_ops/cpu/bert/attention_cpu_base.h new file mode 100644 index 0000000000..1f01504ded --- /dev/null +++ b/onnxruntime/contrib_ops/cpu/bert/attention_cpu_base.h @@ -0,0 +1,218 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "attention_base.h" +#include "attention_helper.h" + +#include "core/common/common.h" +#include "core/common/safeint.h" +#include "core/framework/op_kernel.h" + +namespace onnxruntime { +namespace contrib { + +class AttentionCPUBase : public AttentionBase { + protected: + AttentionCPUBase(const OpKernelInfo& info) : AttentionBase(info) {} + + template + Status ApplyAttention(const T* Q, // Q data. Its size is BxNxSxH + const T* K, // K data. Its size is BxNxSxH + const T* V, // V value with size BxNxSxH + const Tensor* mask_index, // mask index. nullptr if no mask or its size is B + const Tensor* past, // past state + Tensor* output, // output tensor + int batch_size, // batch size + int sequence_length, // sequence length + int head_size, // head size + int hidden_size, // hidden size + OpKernelContext* context) const { + AllocatorPtr allocator; + ORT_RETURN_IF_ERROR(context->GetTempSpaceAllocator(&allocator)); + + auto* tp = context->GetOperatorThreadPool(); + + int past_sequence_length = 0; + Tensor* present = GetPresent(context, past, batch_size, head_size, sequence_length, past_sequence_length); + + // Total sequence length including that of past state: S* = S' + S + const int all_sequence_length = past_sequence_length + sequence_length; + + // Compute the attention score. It does 2 things: + // I. attention_probs(B, N, S, S*) = 1/sqrt(H) x Q(B, N, S, H) x K'(B, N, S*, H -> B, N, H, S*) + + // 1 x mask_data(B, N, S, S*) + // II.attention_probs(B, N, S, S*) = Softmax(attention_probs) + size_t attention_probs_bytes = SafeInt(batch_size) * num_heads_ * sequence_length * all_sequence_length * sizeof(T); + auto attention_probs = allocator->Alloc(attention_probs_bytes); + BufferUniquePtr scratch_buffer(attention_probs, BufferDeleter(allocator)); + + size_t mask_data_bytes = 0; + if (mask_index != nullptr) { + mask_data_bytes = SafeInt(batch_size) * sequence_length * all_sequence_length * sizeof(T); + } else if (is_unidirectional_) { + mask_data_bytes = SafeInt(sequence_length) * all_sequence_length * sizeof(T); + } + + void* mask_data = nullptr; + if (mask_data_bytes > 0) { + mask_data = allocator->Alloc(mask_data_bytes); + memset(mask_data, 0, mask_data_bytes); + } + BufferUniquePtr mask_data_buffer(mask_data, BufferDeleter(allocator)); + + const int32_t* mask_index_data = mask_index != nullptr ? mask_index->template Data() : nullptr; + const T* past_data = past != nullptr ? past->template Data() : nullptr; + T* present_data = present != nullptr ? present->template MutableData() : nullptr; + + ComputeAttentionProbs(static_cast(attention_probs), Q, K, + mask_index_data, static_cast(mask_data), + batch_size, sequence_length, past_sequence_length, head_size, + past_data, present_data, tp); + + // Compute the attentionScore * Value. It does: out_tmp(B, N, S, H) = attention_probs(B, N, S, S*) x V(B, N, S*, H) + auto out_tmp_data = + allocator->Alloc(SafeInt(batch_size) * num_heads_ * sequence_length * head_size * sizeof(T)); + BufferUniquePtr out_tmp_buffer(out_tmp_data, BufferDeleter(allocator)); + + ComputeVxAttentionScore(output->template MutableData(), static_cast(out_tmp_data), static_cast(attention_probs), V, + batch_size, sequence_length, past_sequence_length, head_size, hidden_size, + past_data, present_data, tp); + + return Status::OK(); + } + + private: + // Helper function to compute the attention probs. It does 2 things: + // I. attention_probs(B, N, S, S*) = 1/sqrt(H) x Q(B, N, S, H) x K'(B, N, S*, H -> B, N, H, S*) + + // 1 x mask_data(B, N, S, S*) + // II.attention_probs(B, N, S, S*) = Softmax(attention_probs) + template + void ComputeAttentionProbs(T* attention_probs, // output buffer for the attention probs. Its size is BxNxSxS + const T* Q, // Q data. Its size is BxNxSxH + const T* K, // k data. Its size is BxNxSxH + const int32_t* mask_index, // mask index. nullptr if no mask or its size is B + T* mask_data, // buffer for mask data. Its size is: SxS* if is_unidirectional_; BxSxS* if mask_index; null otherwise + int batch_size, // batch size of self-attention + int sequence_length, // sequence length of self-attention + int past_sequence_length, // sequence length of past state + int head_size, // head size of self-attention + const T* past, // past state + T* present, // present state + ThreadPool* tp) const { + const int all_sequence_length = past_sequence_length + sequence_length; // S* = S' + S + const size_t past_chunk_length = static_cast(past_sequence_length * head_size); // S' x H + const size_t input_chunk_length = static_cast(sequence_length * head_size); // S x H + const size_t present_chunk_length = past_chunk_length + input_chunk_length; // S* x H + + { + if (mask_data != nullptr) { + PrepareMask(mask_index, mask_data, is_unidirectional_, batch_size, sequence_length, past_sequence_length); + } else { // no any mask + memset(attention_probs, 0, batch_size * num_heads_ * sequence_length * all_sequence_length * sizeof(T)); + } + + const int loop_len = batch_size * num_heads_; + const float alpha = 1.0f / sqrt(static_cast(head_size)); + + // The cost of Gemm + const double cost = static_cast(head_size * sequence_length * all_sequence_length); + + ThreadPool::TryParallelFor(tp, loop_len, cost, [&](std::ptrdiff_t begin, std::ptrdiff_t end) { + for (std::ptrdiff_t i = begin; i != end; ++i) { + const std::ptrdiff_t batch_index = i / num_heads_; + + // broadcast mask data: SxS* or (Bx)SxS* -> (BxNx)SxS* + if (mask_data != nullptr) { + const T* broadcast_data_src = is_unidirectional_ ? reinterpret_cast(mask_data) : reinterpret_cast(mask_data) + batch_index * sequence_length * all_sequence_length; + T* broadcast_data_dest = reinterpret_cast(attention_probs) + sequence_length * all_sequence_length * i; + memcpy(broadcast_data_dest, broadcast_data_src, sequence_length * all_sequence_length * sizeof(T)); + } + + const T* k = K + input_chunk_length * i; + if (nullptr != present) { + // concatenate past_K and K : (BxNx)S'xH, (BxNx)SxH -> (BxNx)S*xH + k = ConcatStateChunk(past, k, present, past_chunk_length, present_chunk_length, i); + } + + // gemm + // original transposed each iteration + // A: Q (B x N x) S x H (B x N x) S x H S x H + // B: K' (B x N x) S* x H (B x N x) H x S* H x S* + // C: attention_probs (B x N x) S x S* (B x N x) S x S* S x S* + math::Gemm(CblasNoTrans, CblasTrans, sequence_length, all_sequence_length, head_size, alpha, + Q + input_chunk_length * i, k, 1.0, + reinterpret_cast(attention_probs) + sequence_length * all_sequence_length * i, nullptr); + } + }); + } + + // attention_probs(B, N, S, S*) = Softmax(attention_probs) + { + const int N = batch_size * num_heads_ * sequence_length; + const int D = all_sequence_length; + ComputeAttentionSoftmaxInplace(attention_probs, N, D, tp); + } + } + + template + void ComputeVxAttentionScore(T* output, // buffer for the result with size BxSxNxH + T* tmp_buffer, // buffer for temp use with size is BxNxSxH + const T* attention_probs, // Attention probs with size BxNxSxS* + const T* V, // V value with size BxNxSxH + int batch_size, // batch size + int sequence_length, // sequence length + int past_sequence_length, // sequence length in past state + int head_size, // head size + int hidden_size, // hidden size + const T* past, // past state + T* present, // present state + ThreadPool* tp) const { + const int all_sequence_length = past_sequence_length + sequence_length; // S* = S' + S + const size_t past_chunk_length = static_cast(past_sequence_length * head_size); // S' x H + const size_t input_chunk_length = static_cast(sequence_length * head_size); // S x H + const size_t present_chunk_length = past_chunk_length + input_chunk_length; // S* x H + + // Move the pointer of past and present to start of v values. + if (nullptr != past) { + past += batch_size * num_heads_ * past_sequence_length * head_size; + } + if (nullptr != present) { + present += batch_size * num_heads_ * all_sequence_length * head_size; + } + + const double cost = + static_cast(sequence_length) * static_cast(head_size) * static_cast(sequence_length); + + ThreadPool::TryParallelFor(tp, batch_size * num_heads_, cost, [&](std::ptrdiff_t begin, std::ptrdiff_t end) { + for (std::ptrdiff_t i = begin; i != end; ++i) { + const T* v = V + input_chunk_length * i; + if (nullptr != present) { + // concatenate past_V and V: (BxNx)S'xH, (BxNx)SxH -> (BxNx)S*xH + v = ConcatStateChunk(past, v, present, past_chunk_length, present_chunk_length, i); + } + + T* current_tmp_data = reinterpret_cast(tmp_buffer) + input_chunk_length * i; + math::MatMul(sequence_length, head_size, all_sequence_length, + attention_probs + sequence_length * all_sequence_length * i, + v, current_tmp_data, nullptr); + + // transpose: out(B, S, N, H) = transpose out_tmp(B, N, S, H) + const int batch_index = static_cast(i / num_heads_); + const int head_index = static_cast(i % num_heads_); + T* src = current_tmp_data; + T* dest = output + (batch_index * sequence_length * num_heads_ + head_index) * head_size; + const auto bytes_to_copy = SafeInt(head_size) * sizeof(T); + for (int j = 0; j < sequence_length; j++) { + memcpy(dest, src, bytes_to_copy); + src += head_size; + dest += hidden_size; + } + } + }); + } +}; + +} // namespace contrib +} // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cpu/bert/attention_helper.h b/onnxruntime/contrib_ops/cpu/bert/attention_helper.h index 4a94d0b45b..7507cb2ab2 100644 --- a/onnxruntime/contrib_ops/cpu/bert/attention_helper.h +++ b/onnxruntime/contrib_ops/cpu/bert/attention_helper.h @@ -68,7 +68,7 @@ void PrepareMask(const int32_t* mask_index, int past_sequence_length) { const int all_sequence_length = past_sequence_length + sequence_length; T* p_mask = mask_data; - if (is_unidirectional) { + if (is_unidirectional) { // unidirectional mask has shape SxS* for (int s_i = 0; s_i < sequence_length - 1; s_i++) { for (int m_i = past_sequence_length + s_i + 1; m_i < all_sequence_length; m_i++) { @@ -97,7 +97,7 @@ void PrepareMask(const int32_t* mask_index, // Concatenate a past state chunk S'xH with input state chunk SxH into present state chunk S*xH // Returns a pointer to the start of present state chunk. -template +template T* ConcatStateChunk(const T* past, const T* chunk, T* present, size_t past_chunk_length, size_t present_chunk_length, std::ptrdiff_t i) { T* start = present + i * present_chunk_length; @@ -112,138 +112,5 @@ T* ConcatStateChunk(const T* past, const T* chunk, T* present, size_t past_chunk return start; } -// Helper function to compute the attention probs. It does 2 things: -// I. attention_probs(B, N, S, S*) = 1/sqrt(H) x Q(B, N, S, H) x K'(B, N, S*, H -> B, N, H, S*) + -// 1 x mask_data(B, N, S, S*) -// II.attention_probs(B, N, S, S*) = Softmax(attention_probs) -template -void ComputeAttentionProbs(T* attention_probs, // output buffer for the attention probs. Its size is BxNxSxS - const T* Q, // Q data. Its size is BxNxSxH - const T* K, // k data. Its size is BxNxSxH - const int32_t* mask_index, // mask index. nullptr if no mask or its size is B - T* mask_data, // buffer for mask data. Its size is: SxS* if is_unidirectional; BxSxS* if mask_index; null otherwise - int batch_size, // batch size of self-attention - int sequence_length, // sequence length of self-attention - int past_sequence_length, // sequence length of past state - int head_size, // head size of self-attention - int num_heads, // number of heads of self-attention - bool is_unidirectional, // indicate if it is unidrectional. - const T* past, // past state - T* present, // present state - ThreadPool* tp) { - const int all_sequence_length = past_sequence_length + sequence_length; // S* = S' + S - const size_t past_chunk_length = static_cast(past_sequence_length * head_size); // S' x H - const size_t input_chunk_length = static_cast(sequence_length * head_size); // S x H - const size_t present_chunk_length = past_chunk_length + input_chunk_length; // S* x H - - { - if (mask_data != nullptr) { - PrepareMask(mask_index, mask_data, is_unidirectional, batch_size, sequence_length, past_sequence_length); - } else { // no any mask - memset(attention_probs, 0, batch_size * num_heads * sequence_length * all_sequence_length * sizeof(T)); - } - - const int loop_len = batch_size * num_heads; - const float alpha = 1.0f / sqrt(static_cast(head_size)); - - // The cost of Gemm - const double cost = static_cast(head_size * sequence_length * all_sequence_length); - - ThreadPool::TryParallelFor(tp, loop_len, cost, [&](std::ptrdiff_t begin, std::ptrdiff_t end) { - for (std::ptrdiff_t i = begin; i != end; ++i) { - const std::ptrdiff_t batch_index = i / num_heads; - - // broadcast mask data: SxS* or (Bx)SxS* -> (BxNx)SxS* - if (mask_data != nullptr) { - const T* broadcast_data_src = is_unidirectional ? reinterpret_cast(mask_data) : reinterpret_cast(mask_data) + batch_index * sequence_length * all_sequence_length; - T* broadcast_data_dest = reinterpret_cast(attention_probs) + sequence_length * all_sequence_length * i; - memcpy(broadcast_data_dest, broadcast_data_src, sequence_length * all_sequence_length * sizeof(T)); - } - - const T* k = K + input_chunk_length * i; - if (nullptr != present) { - // concatenate past_K and K : (BxNx)S'xH, (BxNx)SxH -> (BxNx)S*xH - k = ConcatStateChunk(past, k, present, past_chunk_length, present_chunk_length, i); - } - - // gemm - // original transposed each iteration - // A: Q (B x N x) S x H (B x N x) S x H S x H - // B: K' (B x N x) S* x H (B x N x) H x S* H x S* - // C: attention_probs (B x N x) S x S* (B x N x) S x S* S x S* - math::Gemm(CblasNoTrans, CblasTrans, sequence_length, all_sequence_length, head_size, alpha, - Q + input_chunk_length * i, k, 1.0, - reinterpret_cast(attention_probs) + sequence_length * all_sequence_length * i, nullptr); - } - }); - } - - // attention_probs(B, N, S, S*) = Softmax(attention_probs) - { - const int N = batch_size * num_heads * sequence_length; - const int D = all_sequence_length; - ComputeAttentionSoftmaxInplace(attention_probs, N, D, tp); - } -} - -template -void ComputeVxAttentionScore(T* output, // buffer for the result with size BxSxNxH - T* tmp_buffer, // buffer for temp use with size is BxNxSxH - const T* attention_probs, // Attention probs with size BxNxSxS* - const T* V, // V value with size BxNxSxH - int batch_size, // batch size - int sequence_length, // sequence length - int past_sequence_length, // sequence length in past state - int head_size, // head size - int num_heads, // number of heads - int hidden_size, // hidden size - const T* past, // past state - T* present, // present state - ThreadPool* tp) { - const int all_sequence_length = past_sequence_length + sequence_length; // S* = S' + S - const size_t past_chunk_length = static_cast(past_sequence_length * head_size); // S' x H - const size_t input_chunk_length = static_cast(sequence_length * head_size); // S x H - const size_t present_chunk_length = past_chunk_length + input_chunk_length; // S* x H - - // Move the pointer of past and present to start of v values. - if (nullptr != past) { - past += batch_size * num_heads * past_sequence_length * head_size; - } - if (nullptr != present) { - present += batch_size * num_heads * all_sequence_length * head_size; - } - - const double cost = - static_cast(sequence_length) * static_cast(head_size) * static_cast(sequence_length); - - ThreadPool::TryParallelFor(tp, batch_size * num_heads, cost, [&](std::ptrdiff_t begin, std::ptrdiff_t end) { - for (std::ptrdiff_t i = begin; i != end; ++i) { - - const T* v = V + input_chunk_length * i; - if (nullptr != present) { - // concatenate past_V and V: (BxNx)S'xH, (BxNx)SxH -> (BxNx)S*xH - v = ConcatStateChunk(past, v, present, past_chunk_length, present_chunk_length, i); - } - - T* current_tmp_data = reinterpret_cast(tmp_buffer) + input_chunk_length * i; - math::MatMul(sequence_length, head_size, all_sequence_length, - attention_probs + sequence_length * all_sequence_length * i, - v, current_tmp_data, nullptr); - - // transpose: out(B, S, N, H) = transpose out_tmp(B, N, S, H) - const int batch_index = static_cast(i / num_heads); - const int head_index = static_cast(i % num_heads); - T* src = current_tmp_data; - T* dest = output + (batch_index * sequence_length * num_heads + head_index) * head_size; - const auto bytes_to_copy = SafeInt(head_size) * sizeof(T); - for (int j = 0; j < sequence_length; j++) { - memcpy(dest, src, bytes_to_copy); - src += head_size; - dest += hidden_size; - } - } - }); -} - } // namespace contrib } // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cpu/quantization/attention_quant.cc b/onnxruntime/contrib_ops/cpu/quantization/attention_quant.cc index ef344d8db4..b3b6c5775d 100644 --- a/onnxruntime/contrib_ops/cpu/quantization/attention_quant.cc +++ b/onnxruntime/contrib_ops/cpu/quantization/attention_quant.cc @@ -34,21 +34,23 @@ REGISTER_KERNEL_TYPED(float, uint8_t, int8_t) REGISTER_KERNEL_TYPED(float, uint8_t, uint8_t) template -QAttention::QAttention(const OpKernelInfo& info) : OpKernel(info), AttentionBase(info) { +QAttention::QAttention(const OpKernelInfo& info) : OpKernel(info), AttentionCPUBase(info) { } template Status QAttention::Compute(OpKernelContext* context) const { // Input and output shapes: - // Input 0 - input : (batch_size, sequence_length, hidden_size) - // Input 1 - weights : (hidden_size, 3 * hidden_size) - // Input 2 - bias : (3 * hidden_size) - // Input 3 - input_scale : scalar - // Input 4 - weight_scale : scalar - // Input 5 - mask_index : (batch_size) - // Input 6 - input_zero_point : scalar - // Input 7 - weight_zero_point : scalar - // Output : (batch_size, sequence_length, hidden_size) + // Input 0 - input : (batch_size, sequence_length, hidden_size) + // Input 1 - weights : (hidden_size, 3 * hidden_size) + // Input 2 - bias : (3 * hidden_size) + // Input 3 - input_scale : scalar + // Input 4 - weight_scale : scalar + // Input 5 - mask_index : (batch_size) + // Input 6 - input_zero_point : scalar + // Input 7 - weight_zero_point : scalar + // Input 8 - past : (2, batch_size, num_heads, past_sequence_length, head_size) + // Output 0 : (batch_size, sequence_length, hidden_size) + // Output 1 - present : (2, batch_size, num_heads, past_sequence_length + sequence_length, head_size) // ORT_RETURN_IF_ERROR(CheckInputs(context)); const Tensor* input = context->Input(0); const Tensor* weights = context->Input(1); @@ -58,8 +60,9 @@ Status QAttention::Compute(OpKernelContext* context) const { const Tensor* mask_index = context->Input(5); const Tensor* i_zp_tensor = context->Input(6); const Tensor* w_zp_tensor = context->Input(7); + const Tensor* past_tensor = context->Input(8); - ORT_RETURN_IF_ERROR(AttentionBase::CheckInputs(input, weights, bias, mask_index, nullptr)); + ORT_RETURN_IF_ERROR(AttentionBase::CheckInputs(input, weights, bias, mask_index, past_tensor)); ORT_RETURN_IF_NOT(IsScalarOr1ElementVector(input_scale_tensor), "input scale must be a scalar or 1D tensor of size 1"); @@ -146,51 +149,15 @@ Status QAttention::Compute(OpKernelContext* context) const { &dequant_scale, // output scale bias_data + weights_offset, // bias nullptr // use single-thread - ); + ); } }); } - // STEP.2: compute the attention score. It does 2 things: - // I. attention_probs(B, N, S, S) = 1/sqrt(H) x Q(B, N, S, H) x K'(B, N, S, H -> B, N, H, S) + - // 1 x mask_data(B, N, S, S) - // II.attention_probs(B, N, S, S) = Softmax(attention_probs) - size_t attention_probs_bytes = SafeInt(batch_size) * num_heads_ * sequence_length * sequence_length * element_size; - auto attention_probs = allocator->Alloc(attention_probs_bytes); - BufferUniquePtr scratch_buffer(attention_probs, BufferDeleter(allocator)); - - size_t mask_data_bytes = 0; - if (mask_index != nullptr) { - mask_data_bytes = SafeInt(batch_size) * sequence_length * sequence_length * element_size; - } else if (is_unidirectional_) { - mask_data_bytes = SafeInt(sequence_length) * sequence_length * element_size; - } - - void* mask_data = nullptr; - if (mask_data_bytes > 0) { - mask_data = allocator->Alloc(mask_data_bytes); - memset(mask_data, 0, mask_data_bytes); - } - BufferUniquePtr mask_data_buffer(mask_data, BufferDeleter(allocator)); - - const int32_t* mask_index_data = mask_index != nullptr ? mask_index->template Data() : nullptr; - - int past_sequence_length = 0; - const T* past_data = nullptr; - T* present_data = nullptr; - ComputeAttentionProbs(static_cast(attention_probs), Q, K, mask_index_data, static_cast(mask_data), - batch_size, sequence_length, past_sequence_length, head_size, num_heads_, is_unidirectional_, - past_data, present_data, tp); - - // STEP.3: compute the attentionScore * Value. It does: out_tmp(B, N, S, H) = attention_probs(B, N, S, S) x V(B, N, S, H) - auto out_tmp_data = - allocator->Alloc(SafeInt(batch_size) * num_heads_ * sequence_length * head_size * element_size); - BufferUniquePtr out_tmp_buffer(out_tmp_data, BufferDeleter(allocator)); - - ComputeVxAttentionScore(output->template MutableData(), static_cast(out_tmp_data), static_cast(attention_probs), V, - batch_size, sequence_length, past_sequence_length, head_size, num_heads_, hidden_size, past_data, present_data, tp); - - return Status::OK(); + // Compute the attention score and apply the score to V + return ApplyAttention(Q, K, V, mask_index, past_tensor, output, + batch_size, sequence_length, + head_size, hidden_size, context); } } // namespace contrib diff --git a/onnxruntime/contrib_ops/cpu/quantization/attention_quant.h b/onnxruntime/contrib_ops/cpu/quantization/attention_quant.h index af5858aca5..961347bf09 100644 --- a/onnxruntime/contrib_ops/cpu/quantization/attention_quant.h +++ b/onnxruntime/contrib_ops/cpu/quantization/attention_quant.h @@ -2,12 +2,12 @@ #include "core/common/common.h" #include "core/framework/op_kernel.h" -#include "contrib_ops/cpu/bert/attention.h" +#include "contrib_ops/cpu/bert/attention_cpu_base.h" namespace onnxruntime { namespace contrib { template -class QAttention : public OpKernel, public AttentionBase { +class QAttention : public OpKernel, public AttentionCPUBase { public: QAttention(const OpKernelInfo& info); diff --git a/onnxruntime/contrib_ops/cuda/bert/attention.h b/onnxruntime/contrib_ops/cuda/bert/attention.h index b32ca9ee35..cd9bfe7d10 100644 --- a/onnxruntime/contrib_ops/cuda/bert/attention.h +++ b/onnxruntime/contrib_ops/cuda/bert/attention.h @@ -6,7 +6,7 @@ #include "core/common/common.h" #include "core/framework/op_kernel.h" #include "core/providers/cuda/cuda_common.h" -#include "contrib_ops/cpu/bert/attention.h" +#include "contrib_ops/cpu/bert/attention_base.h" namespace onnxruntime { namespace contrib { diff --git a/onnxruntime/contrib_ops/cuda/quantization/attention_quantization.cc b/onnxruntime/contrib_ops/cuda/quantization/attention_quantization.cc index c1a5a251eb..974fdecf2f 100644 --- a/onnxruntime/contrib_ops/cuda/quantization/attention_quantization.cc +++ b/onnxruntime/contrib_ops/cuda/quantization/attention_quantization.cc @@ -47,7 +47,8 @@ Status QAttention::CheckInputs(const Tensor* input, const Tensor* weight_scale_tensor, const Tensor* mask_index, const Tensor* i_zp_tensor, - const Tensor* w_zp_tensor) const { + const Tensor* w_zp_tensor, + const Tensor* past_tensor) const { // Input and output shapes: // Input 0 - input : (batch_size, sequence_length, hidden_size) // Input 1 - weights : (hidden_size, 3 * hidden_size) @@ -59,7 +60,7 @@ Status QAttention::CheckInputs(const Tensor* input, // Input 7 - weight_zero_point : scalar // Output : (batch_size, sequence_length, hidden_size) - ORT_RETURN_IF_ERROR(AttentionBase::CheckInputs(input, weights, bias, mask_index, nullptr)); + ORT_RETURN_IF_ERROR(AttentionBase::CheckInputs(input, weights, bias, mask_index, past_tensor)); ORT_RETURN_IF_NOT(IsScalarOr1ElementVector(input_scale_tensor), "input scale must be a scalar or 1D tensor of size 1"); @@ -88,15 +89,17 @@ Status QAttention::CheckInputs(const Tensor* input, template Status QAttention::ComputeInternal(OpKernelContext* context) const { // Input and output shapes: - // Input 0 - input : (batch_size, sequence_length, hidden_size) - // Input 1 - weights : (hidden_size, 3 * hidden_size) - // Input 2 - bias : (3 * hidden_size) - // Input 3 - input_scale : scalar - // Input 4 - weight_scale : scalar - // Input 5 - mask_index : (batch_size) - // Input 6 - input_zero_point : scalar - // Input 7 - weight_zero_point : scalar - // Output : (batch_size, sequence_length, hidden_size) + // Input 0 - input : (batch_size, sequence_length, hidden_size) + // Input 1 - weights : (hidden_size, 3 * hidden_size) + // Input 2 - bias : (3 * hidden_size) + // Input 3 - input_scale : scalar + // Input 4 - weight_scale : scalar + // Input 5 - mask_index : (batch_size) + // Input 6 - input_zero_point : scalar + // Input 7 - weight_zero_point : scalar + // Input 8 - past : (2, batch_size, num_heads, past_sequence_length, head_size) + // Output 0 - output : (batch_size, sequence_length, hidden_size) + // Output 1 - present : (2, batch_size, num_heads, past_sequence_length + sequence_length, head_size) // ORT_RETURN_IF_ERROR(CheckInputs(context)); const Tensor* input = context->Input(0); const Tensor* weights = context->Input(1); @@ -106,6 +109,7 @@ Status QAttention::ComputeInternal(OpKernelContext* context) const { const Tensor* mask_index = context->Input(5); const Tensor* i_zp_tensor = context->Input(6); const Tensor* w_zp_tensor = context->Input(7); + const Tensor* past_tensor = context->Input(8); ORT_RETURN_IF_ERROR(CheckInputs(input, weights, @@ -114,7 +118,8 @@ Status QAttention::ComputeInternal(OpKernelContext* context) const { weight_scale_tensor, mask_index, i_zp_tensor, - w_zp_tensor)); + w_zp_tensor, + past_tensor)); const auto& shape = input->Shape(); int batch_size = static_cast(shape[0]); @@ -160,9 +165,9 @@ Status QAttention::ComputeInternal(OpKernelContext* context) const { m, n); - const int past_sequence_length = 0; - const T* past_data = nullptr; - T* present_data = nullptr; + int past_sequence_length = 0; + Tensor* present_tensor = GetPresent(context, past_tensor, batch_size, head_size, sequence_length, past_sequence_length); + size_t workSpaceSize = GetAttentionWorkspaceSize(element_size, batch_size, num_heads_, head_size, sequence_length, past_sequence_length); auto temp_buffer = GetScratchBuffer(workSpaceSize); if (!LaunchAttentionKernel( @@ -178,9 +183,8 @@ Status QAttention::ComputeInternal(OpKernelContext* context) const { element_size, is_unidirectional_, past_sequence_length, - past_data, - present_data - )) { + nullptr == past_tensor ? nullptr : past_tensor->template Data(), + nullptr == present_tensor ? nullptr : present_tensor->template MutableData())) { // Get last error to reset it to cudaSuccess. CUDA_CALL(cudaGetLastError()); return Status(common::ONNXRUNTIME, common::FAIL); diff --git a/onnxruntime/contrib_ops/cuda/quantization/attention_quantization.h b/onnxruntime/contrib_ops/cuda/quantization/attention_quantization.h index ba636d5635..e019c0c1b2 100644 --- a/onnxruntime/contrib_ops/cuda/quantization/attention_quantization.h +++ b/onnxruntime/contrib_ops/cuda/quantization/attention_quantization.h @@ -36,7 +36,8 @@ class QAttention final : public CudaKernel, public AttentionBase { const Tensor* weight_scale_tensor, const Tensor* mask_index, const Tensor* i_zp_tensor, - const Tensor* w_zp_tensor) const; + const Tensor* w_zp_tensor, + const Tensor* past_tensor) const; }; } // namespace cuda diff --git a/onnxruntime/core/graph/contrib_ops/contrib_defs.cc b/onnxruntime/core/graph/contrib_ops/contrib_defs.cc index 7b65660efd..eb67c26e52 100644 --- a/onnxruntime/core/graph/contrib_ops/contrib_defs.cc +++ b/onnxruntime/core/graph/contrib_ops/contrib_defs.cc @@ -406,11 +406,23 @@ mask_index shall not be provided.)DOC"; "zero point of quantized weight tensor. It's a scalar, which means a per-tensor/layer quantization.", "T2", OpSchema::Optional) + .Input( + 8, + "past", + "past state for key and value with shape (2, batch_size, num_heads, past_sequence_length, head_size).", + "T3", + OpSchema::Optional) .Output( 0, "output", "3D output tensor with shape (batch_size, sequence_length, hidden_size)", "T3") + .Output( + 1, + "present", + "present state for key and value with shape (2, batch_size, num_heads, past_sequence_length + sequence_length, head_size)", + "T3", + OpSchema::Optional) .TypeConstraint("T1", {"tensor(int8)", "tensor(uint8)"}, "Constrain input and output types to int8 tensors.") .TypeConstraint("T2", {"tensor(int8)", "tensor(uint8)"}, "Constrain input and output types to int8 tensors.") .TypeConstraint("T3", {"tensor(float)", "tensor(float16)"}, "Constrain input and output types to float tensors.") @@ -2616,7 +2628,9 @@ Example 4: .Attr("axis", "The first normalization dimension: normalization will be performed along dimensions axis : rank(inputs).", AttributeProto::INT, static_cast(-1)) - .Attr("epsilon", "The epsilon value to use to avoid division by zero.", AttributeProto::FLOAT, 1e-5f) + .Attr("epsilon", + "The epsilon value to use to avoid division by zero.", + AttributeProto::FLOAT, 1e-5f) .AllowUncheckedAttributes() .Input(0, "X", "Input data tensor from the previous layer.", "T") .Input(1, "scale", "Scale tensor.", "T") diff --git a/onnxruntime/python/tools/quantization/quantize.py b/onnxruntime/python/tools/quantization/quantize.py index ab8710dc55..1458421a2c 100644 --- a/onnxruntime/python/tools/quantization/quantize.py +++ b/onnxruntime/python/tools/quantization/quantize.py @@ -1361,8 +1361,9 @@ class ONNXQuantizer: inputs.extend(quantized_input_names) inputs.extend([node.input[2]]) inputs.extend(scale_names) - inputs.extend([node.input[3]]) + inputs.extend([node.input[3] if len(node.input) > 3 else ""]) inputs.extend(zero_point_names) + inputs.extend([node.input[4] if len(node.input) > 4 else ""]) kwargs = {} for attribute in node.attribute: diff --git a/onnxruntime/test/common/tensor_op_test_utils.h b/onnxruntime/test/common/tensor_op_test_utils.h index bb35f89082..ca824e1bea 100644 --- a/onnxruntime/test/common/tensor_op_test_utils.h +++ b/onnxruntime/test/common/tensor_op_test_utils.h @@ -57,6 +57,49 @@ class RandomValueGenerator { return val; } + // Gaussian distribution for float + template + typename std::enable_if< + std::is_floating_point::value, + std::vector>::type + Gaussian(const std::vector& dims, TFloat mean, TFloat stddev) { + std::vector val(detail::SizeFromDims(dims)); + std::normal_distribution distribution(mean, stddev); + for (size_t i = 0; i < val.size(); ++i) { + val[i] = distribution(generator_); + } + return val; + } + + // Gaussian distribution for Integer + template + typename std::enable_if< + std::is_integral::value, + std::vector>::type + Gaussian(const std::vector& dims, TInt mean, TInt stddev) { + std::vector val(detail::SizeFromDims(dims)); + std::normal_distribution distribution(static_cast(mean), static_cast(stddev)); + for (size_t i = 0; i < val.size(); ++i) { + val[i] = static_cast(std::round(distribution(generator_))); + } + return val; + } + + // Gaussian distribution for Integer and Clamp to [min, max] + template + typename std::enable_if< + std::is_integral::value, + std::vector>::type + Gaussian(const std::vector& dims, TInt mean, TInt stddev, TInt min, TInt max) { + std::vector val(detail::SizeFromDims(dims)); + std::normal_distribution distribution(static_cast(mean), static_cast(stddev)); + for (size_t i = 0; i < val.size(); ++i) { + int64_t round_val = static_cast(std::round(distribution(generator_))); + val[i] = static_cast(std::min(std::max(round_val, min), max)); + } + return val; + } + template inline std::vector OneHot(const std::vector& dims, int64_t stride) { std::vector val(detail::SizeFromDims(dims), T(0)); diff --git a/onnxruntime/test/contrib_ops/attention_op_test.cc b/onnxruntime/test/contrib_ops/attention_op_test.cc index 6f7a15ab64..897401c197 100644 --- a/onnxruntime/test/contrib_ops/attention_op_test.cc +++ b/onnxruntime/test/contrib_ops/attention_op_test.cc @@ -659,5 +659,34 @@ TEST(AttentionTest, AttentionPastStateBatch2) { use_past_state, past_sequence_length, head_size, &past_data, &present_data); } +TEST(AttentionTest, AttentionPastState_dynamic) { + // create rand inputs + RandomValueGenerator random{}; + + std::vector input_dims{2, 5, 768}; + std::vector input_data = random.Gaussian(input_dims, 0.0f, 0.3f); + + std::vector weight_dims{768, 2304}; + std::vector weight_data = random.Gaussian(weight_dims, 0.0f, 0.3f); + + std::vector bias_dims{2304}; + std::vector bias_data = random.Gaussian(bias_dims, 0.0f, 0.3f); + + std::vector past_dims{2, 2, 12, 15, 64}; + std::vector past_data = random.Gaussian(past_dims, 0.0f, 0.3f); + + OpTester test("Attention", 1, onnxruntime::kMSDomain); + test.AddAttribute("num_heads", 12); + test.AddAttribute("unidirectional", 1); + test.AddInput("input", input_dims, input_data); + test.AddInput("weight", weight_dims, weight_data); + test.AddInput("bias", bias_dims, bias_data); + test.AddMissingOptionalInput(); + test.AddInput("past", past_dims, past_data); + + test.AddReferenceOutputs("testdata/attention_past_state.onnx"); + test.Run(); +} + } // namespace test } // namespace onnxruntime diff --git a/onnxruntime/test/contrib_ops/dynamic_quantize_matmul_test.cc b/onnxruntime/test/contrib_ops/dynamic_quantize_matmul_test.cc index 839278b8a7..991cbf87bd 100644 --- a/onnxruntime/test/contrib_ops/dynamic_quantize_matmul_test.cc +++ b/onnxruntime/test/contrib_ops/dynamic_quantize_matmul_test.cc @@ -21,142 +21,40 @@ namespace onnxruntime { namespace test { template -class DynamicQuantizeMatMulOpTester : public OpTester { - public: - DynamicQuantizeMatMulOpTester(const char* op, - const std::vector& A_dims, - const std::vector& B_dims, - const std::vector& Y_dims, - string&& model_file, - int opset_version = 1, - const char* domain = onnxruntime::kMSDomain) : OpTester(op, opset_version, domain), - A_dims_(A_dims), - B_dims_(B_dims), - Y_dims_(Y_dims), - model_file_(model_file) { - Init(); - } - void Init() { - // create rand inputs - RandomValueGenerator random{}; - A_data_ = random.Uniform(A_dims_, -1.0f, 1.0f); +void TestDynamicQuantizeMatMul(const std::vector& A_dims, + std::vector B_dims, + const std::string& reference_model) { + // create rand inputs + RandomValueGenerator random{}; - std::vector tmp_B_data = random.Uniform(B_dims_, std::numeric_limits::min(), std::numeric_limits::max()); - std::transform(tmp_B_data.begin(), tmp_B_data.end(), std::back_inserter(B_data_), [](int32_t v) -> T { - return static_cast(v); - }); + std::vector A_data = random.Uniform(A_dims, -1.0f, 1.0f); - B_zero_point_ = {static_cast(random.Uniform({1}, std::numeric_limits::min(), std::numeric_limits::max())[0])}; - B_scale_ = random.Uniform({1}, -0.1f, 0.1f); + std::vector B_data; + std::vector tmp_B_data = random.Uniform(B_dims, std::numeric_limits::min(), std::numeric_limits::max()); + std::transform(tmp_B_data.begin(), tmp_B_data.end(), std::back_inserter(B_data), [](int32_t v) -> T { + return static_cast(v); + }); - const int64_t Y_size = std::accumulate(Y_dims_.cbegin(), Y_dims_.cend(), static_cast(1), std::multiplies{}); - Y_data_.resize(Y_size); + std::vector B_scale = random.Uniform({1}, -0.1f, 0.1f); + std::vector B_zero_point = {static_cast(random.Uniform({1}, std::numeric_limits::min(), std::numeric_limits::max())[0])}; - AddInput("A", A_dims_, A_data_); - AddInput("B", B_dims_, B_data_); - AddInput("b_scale", {1}, B_scale_); - AddInput("b_zero_point", {1}, B_zero_point_); + OpTester test("DynamicQuantizeMatMul", 1, onnxruntime::kMSDomain); + test.AddInput("A", A_dims, A_data); + test.AddInput("B", B_dims, B_data); + test.AddInput("b_scale", {1}, B_scale); + test.AddInput("b_zero_point", {1}, B_zero_point); - AddOutput("Y", Y_dims_, Y_data_); - } - void Run() { -#ifndef NDEBUG - // run_called_ to true to avoid a complaining in the destructor of OpTester - run_called_ = true; -#endif - std::vector cpu_fetches; - std::vector subgraph_fetches; - Compute(cpu_fetches); - ComputeOriginalSubgraph(subgraph_fetches); - - // Compare CPU with original subgraph on the output(0) - Y - ASSERT_TRUE(cpu_fetches.size() >= subgraph_fetches.size()); - for (size_t i = 0; i < subgraph_fetches.size(); i++) { - if (cpu_fetches[i].IsTensor() && subgraph_fetches[i].IsTensor()) { - VLOGS_DEFAULT(1) << "Checking tensor " << i; - CheckTensor(subgraph_fetches[i].Get(), cpu_fetches[i].Get(), 1e-3, 1e-3); - } - } - } - - private: - void Compute(std::vector& cpu_fetches) { - auto p_model = BuildGraph(); - auto& graph = p_model->MainGraph(); - - Status status = graph.Resolve(); - ASSERT_TRUE(status.IsOK()) << status; - - // Hookup the inputs and outputs - NameMLValMap feeds; - std::vector output_names; - FillFeedsAndOutputNames(feeds, output_names); - - SessionOptions so; - so.session_logid = op_; - so.session_log_verbosity_level = 1; - - RunOptions run_options; - run_options.run_tag = op_; - run_options.run_log_verbosity_level = 1; - - // run with DynamicQuantizeMatMul - InferenceSession session_object{so, GetEnvironment()}; - std::string s1; - p_model->ToProto().SerializeToString(&s1); - std::istringstream str(s1); - ASSERT_TRUE((status = session_object.Load(str)).IsOK()) << status; - ASSERT_TRUE((status = session_object.Initialize()).IsOK()) << status; - ASSERT_TRUE((status = session_object.Run(run_options, feeds, output_names, &cpu_fetches)).IsOK()); - } - - void ComputeOriginalSubgraph(std::vector& subgraph_fetches) { - NameMLValMap feeds; - OrtValue ml_value; - std::vector output_names; - FillFeedsAndOutputNames(feeds, output_names); - - SessionOptions so; - so.session_logid = op_; - so.session_log_verbosity_level = 1; - so.graph_optimization_level = TransformerLevel::Level1; - - RunOptions run_options; - run_options.run_tag = op_; - run_options.run_log_verbosity_level = 1; - - Status status; - InferenceSession subgraph_session_object{so, GetEnvironment()}; - ASSERT_TRUE((status = subgraph_session_object.Load(model_file_)).IsOK()) << status; - ASSERT_TRUE((status = subgraph_session_object.Initialize()).IsOK()) << status; - ASSERT_TRUE((status = subgraph_session_object.Run(run_options, feeds, output_names, &subgraph_fetches)).IsOK()) << status; - } - - private: - std::vector A_dims_; - std::vector B_dims_; - std::vector Y_dims_; - - std::vector A_data_; - std::vector B_data_; - std::vector B_scale_; - std::vector B_zero_point_; - std::vector Y_data_; - - std::string model_file_; -}; + test.AddReferenceOutputs(reference_model); + test.Run(); +} TEST(DynamicQuantizeMatMul, Int8_test) { #ifdef MLAS_SUPPORTS_GEMM_U8X8 std::vector A_dims{4, 128}; std::vector B_dims{128, 128}; std::vector Y_dims{4, 128}; - DynamicQuantizeMatMulOpTester test("DynamicQuantizeMatMul", - A_dims, - B_dims, - Y_dims, - "testdata/dynamic_quantize_matmul_int8.onnx"); - test.Run(); + + TestDynamicQuantizeMatMul(A_dims, B_dims, "testdata/dynamic_quantize_matmul_int8.onnx"); #endif } @@ -164,12 +62,8 @@ TEST(DynamicQuantizeMatMul, UInt8_test) { std::vector A_dims{4, 128}; std::vector B_dims{128, 128}; std::vector Y_dims{4, 128}; - DynamicQuantizeMatMulOpTester test("DynamicQuantizeMatMul", - A_dims, - B_dims, - Y_dims, - "testdata/dynamic_quantize_matmul_uint8.onnx"); - test.Run(); + + TestDynamicQuantizeMatMul(A_dims, B_dims, "testdata/dynamic_quantize_matmul_uint8.onnx"); } } // namespace test diff --git a/onnxruntime/test/contrib_ops/layer_norm_op_test.cc b/onnxruntime/test/contrib_ops/layer_norm_op_test.cc index 63c4499e2a..c8c62434d2 100644 --- a/onnxruntime/test/contrib_ops/layer_norm_op_test.cc +++ b/onnxruntime/test/contrib_ops/layer_norm_op_test.cc @@ -18,206 +18,29 @@ using namespace std; namespace onnxruntime { namespace test { -class LayerNormOpTester : public OpTester { - public: - LayerNormOpTester(const char* op, - const std::vector& X_dims, - const std::vector& scale_dims, - const std::vector& B_dims, - const std::vector& Y_dims, - float epsilon, - int64_t axis = -1, - int64_t keep_dims = 1, - int opset_version = 1, - const char* domain = onnxruntime::kOnnxDomain) : OpTester(op, opset_version, domain), - X_dims_(X_dims), - scale_dims_(scale_dims), - B_dims_(B_dims), - Y_dims_(Y_dims), - epsilon_(epsilon), - axis_(axis), - keep_dims_(keep_dims) { - Init(); - } - void Init() { - AddAttribute("axis", axis_); - AddAttribute("keep_dims", keep_dims_); - AddAttribute("epsilon", epsilon_); - - // create rand inputs - RandomValueGenerator random{}; - X_data_ = random.Uniform(X_dims_, 0.0f, 1.0f); - scale_data_ = random.Uniform(scale_dims_, 0.0f, 1.0f); - B_data_ = random.Uniform(B_dims_, 0.0f, 1.0f); - - const int64_t Y_size = std::accumulate(Y_dims_.cbegin(), Y_dims_.cend(), static_cast(1), std::multiplies{}); - Y_data_.resize(Y_size); - - AddInput("X", X_dims_, X_data_); - AddInput("scale", scale_dims_, scale_data_, true); - AddInput("B", B_dims_, B_data_, true); - - AddOutput("output", Y_dims_, Y_data_); - AddOutput("mean", mean_); - AddOutput("inv_std_var", inv_std_var_); - } - void Run() { -#ifndef NDEBUG - run_called_ = true; -#endif - std::vector cpu_fetches; - std::vector cuda_fetches; - std::vector subgraph_fetches; - ComputeWithCPU(cpu_fetches); - ComputeWithCUDA(cuda_fetches); - ComputeOriSubgraphWithCPU(subgraph_fetches); - - // Compare CPU with original subgraph on the output(0) - Y - ASSERT_TRUE(cpu_fetches.size() >= subgraph_fetches.size()); - for (size_t i = 0; i < subgraph_fetches.size(); i++) { - if (cpu_fetches[i].IsTensor() && subgraph_fetches[i].IsTensor()) { - VLOGS_DEFAULT(1) << "Checking tensor " << i; - CheckTensor(subgraph_fetches[i].Get(), cpu_fetches[i].Get(), 1e-3, 1e-3); - } - } - - // Compare GPU with original subgraph on the output(0) - Y - if (DefaultCudaExecutionProvider()) { - ASSERT_TRUE(cuda_fetches.size() >= subgraph_fetches.size()); - for (size_t i = 0; i < subgraph_fetches.size(); i++) { - if (cuda_fetches[i].IsTensor() && subgraph_fetches[i].IsTensor()) { - VLOGS_DEFAULT(1) << "Checking tensor " << i; - CheckTensor(subgraph_fetches[i].Get(), cuda_fetches[i].Get(), 1e-3, 1e-3); - } - } - } - } - - private: - void ComputeWithCPU(std::vector& cpu_fetches); - void ComputeWithCUDA(std::vector& cuda_fetches); - void ComputeOriSubgraphWithCPU(std::vector& subgraph_fetches); - - private: - std::vector X_dims_; - std::vector scale_dims_; - std::vector B_dims_; - std::vector Y_dims_; - - std::vector X_data_; - std::vector scale_data_; - std::vector B_data_; - std::vector Y_data_; - float mean_; - float inv_std_var_; - - float epsilon_; - int64_t axis_; - int64_t keep_dims_; -}; - -void LayerNormOpTester::ComputeWithCPU(std::vector& cpu_fetches) { - auto p_model = BuildGraph(); - auto& graph = p_model->MainGraph(); - - Status status = graph.Resolve(); - ASSERT_TRUE(status.IsOK()) << status; - - // Hookup the inputs and outputs - std::unordered_map feeds; - std::vector output_names; - FillFeedsAndOutputNames(feeds, output_names); - - SessionOptions so; - so.session_logid = op_; - so.session_log_verbosity_level = 1; - - RunOptions run_options; - run_options.run_tag = op_; - run_options.run_log_verbosity_level = 1; - - // run with LayerNormalization - InferenceSession layernorm_session_object{so, GetEnvironment()}; - std::string s1; - p_model->ToProto().SerializeToString(&s1); - std::istringstream str(s1); - ASSERT_TRUE((status = layernorm_session_object.Load(str)).IsOK()) << status; - ASSERT_TRUE((status = layernorm_session_object.Initialize()).IsOK()) << status; - ASSERT_TRUE((status = layernorm_session_object.Run(run_options, feeds, output_names, &cpu_fetches)).IsOK()); -} - -void LayerNormOpTester::ComputeWithCUDA(std::vector& cuda_fetches) { - if (DefaultCudaExecutionProvider() == nullptr) { - return; - } - - auto p_model = BuildGraph(); - auto& graph = p_model->MainGraph(); - - Status status = graph.Resolve(); - ASSERT_TRUE(status.IsOK()) << status; - - // Hookup the inputs and outputs - std::unordered_map feeds; - std::vector output_names; - FillFeedsAndOutputNames(feeds, output_names); - - SessionOptions so; - so.session_logid = op_; - so.session_log_verbosity_level = 1; - - RunOptions run_options; - run_options.run_tag = op_; - run_options.run_log_verbosity_level = 1; - - auto cuda_execution_provider = DefaultCudaExecutionProvider(); - InferenceSession cuda_session_object{so, GetEnvironment()}; - EXPECT_TRUE(cuda_session_object.RegisterExecutionProvider(std::move(cuda_execution_provider)).IsOK()); - - std::string s; - p_model->ToProto().SerializeToString(&s); - std::istringstream str2(s); - - EXPECT_TRUE((status = cuda_session_object.Load(str2)).IsOK()) << status; - EXPECT_TRUE((status = cuda_session_object.Initialize()).IsOK()) << status; - EXPECT_TRUE((status = cuda_session_object.Run(run_options, feeds, output_names, &cuda_fetches)).IsOK()) << status; -} - -void LayerNormOpTester::ComputeOriSubgraphWithCPU(std::vector& subgraph_fetches) { - NameMLValMap feeds; - OrtValue ml_value; - std::vector output_names{"Y"}; - - CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), X_dims_, X_data_, &ml_value); - feeds.insert(std::make_pair("X", ml_value)); - CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), scale_dims_, scale_data_, &ml_value); - feeds.insert(std::make_pair("Scale", ml_value)); - CreateMLValue(TestCPUExecutionProvider()->GetAllocator(0, OrtMemTypeDefault), B_dims_, B_data_, &ml_value); - feeds.insert(std::make_pair("B", ml_value)); - - SessionOptions so; - so.session_logid = op_; - so.session_log_verbosity_level = 1; - - RunOptions run_options; - run_options.run_tag = op_; - run_options.run_log_verbosity_level = 1; - - Status status; - InferenceSession subgraph_session_object{so, GetEnvironment()}; - ASSERT_TRUE((status = subgraph_session_object.Load("testdata/layernorm.onnx")).IsOK()) << status; - ASSERT_TRUE((status = subgraph_session_object.Initialize()).IsOK()) << status; - ASSERT_TRUE((status = subgraph_session_object.Run(run_options, feeds, output_names, &subgraph_fetches)).IsOK()) << status; -} - TEST(LayerNormTest, BERTLayerNorm) { - float epsilon = 1e-12f; + OpTester tester("LayerNormalization", 1 /*opset_version*/); + tester.AddAttribute("axis", -1); + tester.AddAttribute("epsilon", 1e-12f); + + // create rand inputs + RandomValueGenerator random{}; + std::vector X_dims{4, 128}; + std::vector X_data = random.Uniform(X_dims, 0.0f, 1.0f); + tester.AddInput("X", X_dims, X_data); + std::vector scale_dims{128}; + std::vector scale_data = random.Uniform(scale_dims, 0.0f, 1.0f); + tester.AddInput("Scale", scale_dims, scale_data); + std::vector B_dims{128}; - std::vector Y_dims{4, 128}; - LayerNormOpTester test("LayerNormalization", X_dims, scale_dims, B_dims, Y_dims, epsilon, -1, 1, 9); - test.Run(); + std::vector B_data = random.Uniform(B_dims, 0.0f, 1.0f); + tester.AddInput("B", B_dims, B_data); + + tester.AddReferenceOutputs("testdata/layernorm.onnx"); + + tester.Run(); } } // namespace test diff --git a/onnxruntime/test/contrib_ops/quantize_attention_op_test.cc b/onnxruntime/test/contrib_ops/quantize_attention_op_test.cc index a77fa49ba2..4b4dfb2b79 100644 --- a/onnxruntime/test/contrib_ops/quantize_attention_op_test.cc +++ b/onnxruntime/test/contrib_ops/quantize_attention_op_test.cc @@ -705,5 +705,70 @@ TEST(QAttentionTest, QAttentionUnidirectional_CUDA) { true /*is_unidirectional*/); } +template +void TestQuantizedAttentionPastState(int64_t batch, + int64_t seq_len, + int64_t past_seq_len, + int64_t hidden_size, + int64_t head_number, + int64_t head_size, + const std::string& reference_model) { + // create rand inputs + RandomValueGenerator random{}; + + constexpr InputT input_min = std::numeric_limits::min(); + constexpr InputT input_max = std::numeric_limits::max(); + constexpr int32_t input_range = input_max - input_min; + + InputT input_mean = (input_min + input_max) / 2 + 1; + std::vector input_zero_point{input_mean}; + + std::vector input_dims{batch, seq_len, hidden_size}; + std::vector input_data = random.Gaussian(input_dims, input_mean, static_cast(input_range / 6), input_min, input_max); + + constexpr WeightT weight_min = std::numeric_limits::min(); + constexpr WeightT weight_max = std::numeric_limits::max(); + constexpr int32_t weight_range = weight_max - weight_min; + + WeightT weight_mean = (weight_min + weight_max) / 2 + 1; + std::vector weight_zero_point{weight_mean}; + + std::vector weight_dims{hidden_size, 3 * hidden_size}; + std::vector weight_data = random.Gaussian(weight_dims, weight_mean, static_cast(weight_range / 6), weight_min, weight_max); + + std::vector bias_dims{3 * hidden_size}; + std::vector bias_data = random.Gaussian(bias_dims, 0.0f, 0.3f); + + std::vector input_scale{0.005f}; + std::vector weight_scale{0.005f}; + + std::vector past_dims{2, batch, head_number, past_seq_len, head_size}; + std::vector past_data = random.Gaussian(past_dims, 0.0f, 0.3f); + + OpTester test("QAttention", 1, onnxruntime::kMSDomain); + test.AddAttribute("num_heads", head_number); + test.AddAttribute("unidirectional", 1); + test.AddInput("input", input_dims, input_data); + test.AddInput("weight", weight_dims, weight_data); + test.AddInput("bias", bias_dims, bias_data); + test.AddInput("input_scale", {1}, input_scale); + test.AddInput("weight_scale", {1}, weight_scale); + test.AddMissingOptionalInput(); + test.AddInput("input_zero_point", {1}, input_zero_point); + test.AddInput("weight_zero_point", {1}, weight_zero_point); + test.AddInput("past", past_dims, past_data); + + test.AddReferenceOutputs(reference_model); + test.Run(); +} + +TEST(QAttentionTest, QAttentionPastState_u8u8) { + TestQuantizedAttentionPastState(2, 5, 15, 768, 12, 64, "testdata/attention_past_state.u8u8.onnx"); +} + +TEST(QAttentionTest, QAttentionPastState_u8s8) { + TestQuantizedAttentionPastState(2, 5, 15, 768, 12, 64, "testdata/attention_past_state.u8s8.onnx"); +} + } // namespace test } // namespace onnxruntime diff --git a/onnxruntime/test/providers/provider_test_utils.cc b/onnxruntime/test/providers/provider_test_utils.cc index ab826ca0f5..889571ec27 100644 --- a/onnxruntime/test/providers/provider_test_utils.cc +++ b/onnxruntime/test/providers/provider_test_utils.cc @@ -393,6 +393,16 @@ void OpTester::FillFeedsAndOutputNames( } } +void OpTester::FillFeeds(std::unordered_map& feeds) { + for (size_t i = 0; i < input_data_.size(); ++i) { + if (std::find(initializer_index_.begin(), initializer_index_.end(), i) == + initializer_index_.end() && + input_data_[i].def_.Exists()) { + feeds[input_data_[i].def_.Name()] = input_data_[i].data_; + } + } +} + void OpTester::SetOutputAbsErr(const char* name, float v) { auto it = std::find_if(output_data_.begin(), output_data_.end(), @@ -524,7 +534,7 @@ std::vector OpTester::ExecuteModel( // Disable expected_failure_string checks for OpenVINO EP if (provider_type != kOpenVINOExecutionProvider) { EXPECT_THAT(status.ErrorMessage(), - testing::HasSubstr(expected_failure_string)); + testing::HasSubstr(expected_failure_string)); } } else { LOGS_DEFAULT(ERROR) << "Initialize failed with status: " @@ -804,7 +814,7 @@ void OpTester::Run( valid = false; for (auto& custom_session_registry : custom_session_registries_) { if (KernelRegistry::HasImplementationOf(*custom_session_registry->GetKernelRegistry(), - node, execution_provider->Type())) { + node, execution_provider->Type())) { valid = true; break; } @@ -843,6 +853,56 @@ void OpTester::Run( } } +void OpTester::AddReferenceOutputs(const std::string& model_path) { + SessionOptions so; + so.session_logid = op_; + so.session_log_verbosity_level = 1; + + RunOptions run_options; + run_options.run_tag = op_; + run_options.run_log_verbosity_level = 1; + + Status status; + InferenceSession subgraph_session_object{so, GetEnvironment()}; + ASSERT_TRUE((status = subgraph_session_object.Load(model_path)).IsOK()) << status; + ASSERT_TRUE((status = subgraph_session_object.Initialize()).IsOK()) << status; + + // Retrieve output names + auto model_outputs = subgraph_session_object.GetModelOutputs(); + ASSERT_TRUE(model_outputs.first.IsOK()); + std::vector output_names; + std::transform(model_outputs.second->begin(), + model_outputs.second->end(), + std::back_inserter(output_names), + [](const onnxruntime::NodeArg* node_arg) -> std::string { return node_arg->Name(); }); + + NameMLValMap feeds; + FillFeeds(feeds); + + std::vector subgraph_fetches; + ASSERT_TRUE((status = subgraph_session_object.Run(run_options, feeds, output_names, &subgraph_fetches)).IsOK()) << status; + + for (size_t out_idx = 0; out_idx < subgraph_fetches.size(); out_idx++) { + // Retrieve TypeProto + ASSERT_TRUE(subgraph_fetches[out_idx].Type()->IsTensorType()) << status; + const Tensor& t = subgraph_fetches[out_idx].Get(); + const TensorTypeBase* tensor_type = DataTypeImpl::TensorTypeFromONNXEnum(t.GetElementType()); + + // Construct a temp TypeProto with shape information + ONNX_NAMESPACE::TypeProto tmp_type_proto(*(tensor_type->GetTypeProto())); + auto mutable_shape = tmp_type_proto.mutable_tensor_type()->mutable_shape(); + for (auto i : t.Shape().GetDims()) { + auto* mutable_dim = mutable_shape->add_dim(); + mutable_dim->set_dim_value(i); + } + + output_data_.push_back(Data(NodeArg(output_names[out_idx], &tmp_type_proto), + std::move(subgraph_fetches[out_idx]), + optional(), + optional())); + } +} + #ifdef ENABLE_TRAINING template std::vector OpTester::ExecuteModel( Model& model, training::TrainingSession& session_object, diff --git a/onnxruntime/test/providers/provider_test_utils.h b/onnxruntime/test/providers/provider_test_utils.h index 162556586f..6a8c7830e9 100644 --- a/onnxruntime/test/providers/provider_test_utils.h +++ b/onnxruntime/test/providers/provider_test_utils.h @@ -210,7 +210,8 @@ const SequenceTensorTypeProto SequenceTensorType::s_sequence // 1. Create one with the op name // 2. Call AddAttribute with any attributes // 3. Call AddInput for all the inputs -// 4. Call AddOutput with all expected outputs +// 4. Call AddOutput with all expected outputs, +// Or call AddReferenceOutputs to compute reference outputs with the model // 5. Call Run // Not all tensor types and output types are added, if a new input type is used, add it to the TypeToDataType list // above for new output types, add a new specialization for Check<> See current usage for an example, should be self @@ -382,6 +383,9 @@ class OpTester { optional(), optional())); } + // Generate the reference outputs with the model file + void AddReferenceOutputs(const std::string& model_path); + void AddCustomOpRegistry(std::shared_ptr registry) { custom_schema_registries_.push_back(registry->GetOpschemaRegistry()); custom_session_registries_.push_back(registry); @@ -483,6 +487,8 @@ class OpTester { void FillFeedsAndOutputNames(std::unordered_map& feeds, std::vector& output_names); + void FillFeeds(std::unordered_map& feeds); + template std::vector ExecuteModel(Model& model, SessionType& session_object, @@ -545,7 +551,7 @@ class OpTester { const static std::unordered_set reserved_symbolic{"batch", "seq"}; for (size_t i = 0; i < dim_params_data.size(); ++i) { - if (reserved_symbolic.find(dim_params_data[i])!= reserved_symbolic.end()) { + if (reserved_symbolic.find(dim_params_data[i]) != reserved_symbolic.end()) { new_shape.add_dim()->set_dim_param(dim_params_data[i]); } else { ASSERT_TRUE(std::stoi(dim_params_data[i]) == dims[i]); diff --git a/onnxruntime/test/testdata/attention_past_state.onnx b/onnxruntime/test/testdata/attention_past_state.onnx new file mode 100644 index 0000000000000000000000000000000000000000..4bcb19d3b2491d7c4da0ec0cc98b6e344576c86c GIT binary patch literal 1056367 zcmeI2?UGyBb)JXhkllkBDdU!s$X!yLFt#Hzo=G>pq5KFfyW*;;a+S+UDpmT^)ic!* zX^%McuxZQrlev@VLh?Jekh_VpH@X2FbmO28fOFPfACD}NXyBZ^)_T_aF04NE@w1bo zxBqng&9`5@I(>5f`RHf=_kX|a{kZq^^_#cvuTLIN&QFhj@#^yJ)x~i1{OSBvZhLh*dZeG6lxcA}ZZ(hIq(FebM^thVBaGAoin!>1>!mKuhM|V;f zrWA4o<7En?vjYPR#^2rkCj(43 zPS2`04j;bpf94ziD3v@(|C{yF0qO5)Iw<$`>(^ghU5s*H^Q+#A^znK6sDIk~#oxcb zy80jK!}0T@+t)YkxqA09wdbh!dAc?=aymNw(_tQdomcM9Ht5V zG@(y=`ReleVl;jJ?B?Zq_y6!#fa!*Dx?ytK`Kz_3K|>jL)8bm|wj3XT7JFU%!2I`OiQ2;Jg3y+eeRHd_P@# zb-nyK7c&{0Jf59Ziy5ES7L%_}Y0b{puJ6}gpHx3f+-K(*LV`a_^1lTz9&85iZ&GQ~ zbW2WaxT=0UTvh+q>Eq$aQDXn?KYf0F^7QYn@?S@D_kEt*`}XVC*B9g5-u!9__;Iq= ziy!o!zRT~v{O;M!e{)m+Rd0FPZ`KBqc>CTF#IRNem{dilKkM?asinn9EECYdb9(4@b} zWSY;Z&Y{URFIR`*{LXR3%##LY{#`cn{LV@B&QYcMyMyO;e9Xtdi(L9J|1r*gO!6P2 zTtRO2d=^Y{tLL-ePxFWKP_xxEm`rM>LGH+Tbu>+;m7La<@kwPK|2RjPr9aNk@(Jgs z&;H`|KU_>^&!42fdw-ZeIL{ySPmlim^4;~tG+8>oc=1v1@%1-9I?7)Z^Q&@0&j*XE z&hKA!mLtuBtWB(Gy=mun4Dz|-s?Dc^TASxP&gXYb*X|hB-Z3wIly6HN8I0yXX6Wa$ z<$#!uHV=rOE?tsu&(`Xnt>(lu&F#gM%+2oS()(#%{g-bq-@JSK&AY3MX&TncpI-c4 z@58rO-~RgL2QMGLeDv~3o(4bZ-Hw!WXMZsarpvG3-1GjRqQhzZcW_nt*@gXn{v)I6 z50~>`R`WGn&11C4W48Prp65|@(yaPLyso{w)1aT(_va@}R(Yh6ylE~~^H>hY*|0hb z`qk+;+vdwEk=*e?)jhN7LRI7FCicxV z_@n$u9#@0$>G6CTq&a@y=<~~RM+a$VyS;31|FU_(^Ek89?=5$h6s_K@^ILLt!?X2U zMz#LVw+!aD^w)10*WU88d`lh#!_=6;ICp25*$)%@+nphe?#(md-`zSk-;>QX9G-r> z*qhQcUyX!3`iHst;pp^x%OQ}epViHTCodo6neeZBi~S?WEfN z^YhyFS6wi#G|#Hx{D|pk@Afx@H2yamay^+eG8f4Vs>zJfR=3jPdG%|yemKmN$z2%b z!8tmwB$MXxrs=Sj%&5OgX1P6OdyNKb$!v3LmCgLtHBX&azhG;OJ|7YDThn)51wnOA z*W5av5&u5dpC|PwFAbwSD@R#YqulrTj7Z-riy@H~{PaQoH0>t0gW)25Q#wyP^co)YO7iS$Pvo1@w7l;{suQ(}~B9-l4j-yhz$f1VQK+}d&4FK?zqe{}z{ z?B8LIGajt@w_oqye9I^=E#u+(E%m<}Q7@xyWq3~IiYvlu2BUwSC9mSh^`o03fH&CB#V&{6t*I^DBo$U*fh zyM8)UlFWXXq+ie$vuZHjJRb5anPe{w^H6#>mXSI@?-;y<$9&S{> zQHS-vN9Up@c?p@!(nF|kuijm~xn5N_Ok@8>%gbL~=TCEdxb$R_) z@4voCUwY4<-MsGQzN7~kmH9HPPW~S*p3fYA`sj-flK}rL|7w+I%QT;y9&MyQ((e=V z4J1AJn*TnMe_NT(4^B(#?RQOj_I3MIEH@`V4p~jK^k{4KTq0>K!%xy*bxATkGqn+Sj|37j9UfN>5&@QLA>ZxpnzX zn4aL|(D{!%cxQRyq(?K0Z^HDXtb!^%m`R_eCj%=~b-z1j(arLA>vUe*`1I%(@4viA zUvJN!q!+y`$62;jdLUD^ReBCq_pEJxOYUTPSW~?vJrJwAC57&%M=cdL>G4;+c&ooh z{jB%BSD&AKetCWU=JWgnF6Wy)JxgvZx)M8`)BLleCes! z)=bwHhxAOWS*EFhDUF;}YG4YPUnTwb^K*{=S>7LS^IJX08>Abpx1ZrrX3>1kN? z5yVG%LjQ5P>^#e3nhqX}6Z(F7Wbx5|OV78jF49-*^QZHdy=0>FLH;!T-u#Pi{`>W> zFMpFhPmB5O56eN9egS{nOM@<5nT|~l8`G=wEwdPO=~>uf&?VcZkMig1t9*JUw(;9< z`RjeMXu2ogxLV`WgRonRrrF;=Uz;oGsn-hg`gdb>mSxEKX-2v`A5mH3)3dRSoAWA> z)e4`UhE)w9`URy*Hs#M94T7V`)5N&IW5X4QJ9+dO&~ReBauL6shWHLEvM zP7gz>w>Ef=vY7Av{4}JWpN6DEGK~3gtD%)1msL>Z$07anIAk^7(}S_isM35-Q1h`F zetI%;G~ZRy)3PV&MV{};CHa%&-^zoC^fat)1kTgTGc`R4S-Yiv)5^Ex$145wRAlXz z`sbfnFN66X+0FSWMt_jpe6x{Vq%XnE+u2WZJ@eb9C(pilf1UmnvGU7BdR(@6E_w3l z{2v`Z``g#opI!at?Kj_EzxYA^$FiU2e<=Fmc=64>dUKloIq{<#t*qXheQ~__?Qr?# z_-xag=bPT_e{uZyR(`A72djU#T)jE`;`qtp%I|&pC}lYwrOol>kN?k$|MlI=<45WL zzB^9;{q*tA*S`%Gk z#A!E`LST(R_CEt)4GBL$APj-I{}}*b`c0t_s1msGKLeo3gx3&=Lg3c_41g#Nr%nh| z2rT^10H{FWB?N*HSo)s<5TxUj34s-XmH!z4D=NH#KnwyE|1$t$w45p-uq06RKLcQi zg%=PAL14}Q41f?lr$`7a2(0^`0kGi0-w=pE;Ew+p01=u_jS#pcaM%9~0HgtdI0Wwb zp8*i3`BWO7z-x{}})=1VELc3Doe+EEgWl(zr0$csh0Ei#~Y79(ZyZ;#gfu%v|0SG+ce+EDR4NziS z0uTA00T5RkRPLX^gZ^g#_!j{ch9&T@{}}*b#X;fj2{iCO1HiosC@?C4CjMstL{$fM zdneGy{|o@{GN8Vo1e*Du0T5IklAc$NaS1thSW{}})Q zB|^!L3GC>927qHNP+B|!yZWC25Kkji?3cjK{$~LA6$6!pBe1*w835r#Lcwkc?BRa~ zfLk?CSTq9r_@4m~O(oRpmB3#9X8?GW19b%>u%G`K0KsHJxlRe}>3;@*Q$0{tECT!b zp8*g{CsgZ`z~2660QeLHRfQt3zyBEkp@c%QE(sjqe+Ga{MNm{E0*CmY0T4+k)asGI zLH=g|c$5S+1tM^m{}}**q(Z3<2^{Eu27p6NP*NNMhx(rZ5JxLi>W{#|{$~LA69pB8 zA#k|=8319#LZR*mwBUaRfIC%CP!s}f_@4m~MJ?3njX*2@X8?GU1@#0W(2oBZ072wJ zna&8bp8pvDAp}E_t_ZZ~e+Gan zWl&560&V)A0T4kk)aZ#otNv#Ic#;OS1R&6^{}})QBtwag2(;{f27n`NP>O#7ZTp`A z;9oOT=!Zb-{$~LA5eJpHC(yqC8367@LxFAxbl`smfE#sCh<5^A_@4pbT{YC_g+M3% zX8?GS2X#0n(2f5Y0M2DYc}@s)71?U|uB@ z;FW-c{}}*Y)k9s@35fWg0bpGwPD^U|K08?~#C{{}})t6+}&z35fci0bp4w zV>p@5h&$<27oUeQH@Ok#r)3zuqha_c156^{}}+TghVkW2^91{1HhzWNZJ#DlKy7^ zcv2FzSR_!?{|o?&k|AeD1j_oK0pLhVlwy!TVgEA#3~Gjy{SYYae+GabEm4U*0>%B$ z0I(+-GIm3ty#E;hZp1_(<_IY8KLfy=YDm}%0S*3V0C-Uobyy>y!v72aYqB9R0F3E|bbS!e;(rE!4?R(ZEdpx%&j7F`9I|ymK#%_!04@YY5vB+z z@;?K>lyXSc0|8C`X8?Fm6g5~PpvwOY087##R|f=i`JVycKvI-oh=4NxGXM-}hg9to z(B^*zfPGC-fgJ+s{LcWeBOWp}Pe7ml835)*MFD0ADD*!Az>IoG)H(r;{$~JKR~7kN zA)wO#3;-+gAy4B3bo!qGU|d$DZ-jtS|1$uL=!Z0I6VU2^27ql{k-ZH9YW>duupuC_ zG)+LS{}}+Lg+=lv2q^YH1HgoWNYXL^&HiTqSXLIfTOgp?{|o>N5+X;#1a$kK0bp2K zq;7zKa{n^`3}}cH?Gn)Le+GbEZIQWt0_y$G0MIWYGBis-zyBEkX2nI~>IoR|KLbF$ zib&8Z0So?T09aKQd21(N!v72a?J^=iqXcaDp8;T0UZky@fD!*Q0F>*9^lTEa;(rE! zO?{EIZUScf&j8RZB(gI}z>fbJ044=S(y9p<@;?JWwUS8AA^}VOX8>4K7&&VuV9Ngt z0L@Y&H-iLh`JVw`P-3L4n1C_=GXNB8iPY>7u;zaTfIW?ov0eh^{LcW;D<(2CN5G!{ z835))M#5?d81z2_K&_fc%o+iU{$~JKQyKYcC1BG33;?ZiA}?bEZ2F%8U`%GDtCWCI z|1$uT>WQ>$5wPli27oP{k*!VwX8q3q&?zXgGDX0y{}}+LghsL|2^jW213;ysNXilc z%l>BoSW+6fY9wIV{|o?)k|HNV1Z?}C0bodKq^gjBasM*_6l#i;>=3Z-e+GaZt&yod z0_OeC0MI8YGBQKJzW*5jX2eFK>IgXSKLbFWsz}HR0T2FX09a8Qd1@oz!v72aZL%UC zBLsZ-p8;S*ZltM#9bZijt;(rE!4ZV@2E&^`+&j8RREV3~{z>ohK044-S zlBx(e@;?JWm9j|20s&9{X8>4G964$t;L86D08P>&7Xt)*`JVw`Kysw0h=4QyGXNB6 zi&XRz@aBI8fPT%9p&kP6{LcW;BQ7#gPr#r5835`h*g3;-?iA`j&ReEOdOpj>vOr-Xo0|1$uT=!-OT6Y%PP27qqek(~|#ZvD>y z&>=9gP))$E{}}+Pg-3EK2srjX13-nsNJ29K&;DlsXjUG%X&~U*{|o>P5+eu21bq9S z0ialVq^5v?bN@2{6ljbT^b+vye+GbF?U7k|0`C3K04OgqGEhsvzyBEkYQ;xl#R&xP zKLen+%1A&ffe8L*0BBVod6gy*!v73_(lR5yQUWpj&j3&=Khi2pAc+4N0EKl%c%1~I z_@4owQ-5StmOvQ)GXTm8jp!-~#PL4^K&1dlswja#{$~IbRT{xH5{Tq~27pEdl2b_n zq5RJPC@D2!DqDgw{tOn*SL9`ZP#JqZA^DUd5Yqn)fKqZJt}+5K{m%eUCPUIG zL?EdD832X!Mp#`0qWYf!pi75jQ-(lT|1$u}2#%<#2*mY213;A!Nu~&a!2V|d6j2;O zH4%vHe+GaiC6Y@C0-^oS04O0jVk#mK+y4vzMN%Y{0tABlp8-%nbA;4GAiDn<0D81Y zCh`Qr`=0?IFFGQsA%OoG0BXcYBH{$_|04!~xatU~g#i9f1hh1RJRl%U0RLwK!q5o^ z0VM?R|D1r5Zjc59WC`H^8v?S_i3R~31n~bY0Uhli3kZl3!2b&ZqSy%r0Tl!m>gR7B z0;&)sNr)1__bURT;0Xo+1q4>==M@`(0zs03C;?nwB_N8PU=S!ypsIdevjG$rBngNT z!1L<_MBx(*0)+{ztDm3P0169|@S+58{9OW~_z4Dqq6F@$pP$(PiVBk8q6F~!eFCBl zAQ%J+61cB^-eChMC`dw!62R@72#B_TU=S!qV3YcJj}4%hAPFo=0IzQ*Ald|iL7)(U z&Fbe}Hh@BcB&;X_oL)yjv<(DgRnnfFgn0tXOiU;UiO25^8NxmA<^Zta4AXbTAj zfxQWIp?=O}1K3-T+$c%_uXaN~w21_Rz@7xUQ9q}$0qiMAZWASdQ@bJ{+D3vwU@rn) zsh@M%0QM3jH;EF!r`-_{Z6v`Uum^$e)X&Lm0DB0MTSN)q(k=;zwvu2F*qK0=>gQ}W zfSm=&4Wb0_XtxAJn@KPT>`0(n^>aEKz>b3Ca!~>}v}*#Q?Iaikb|TQV`Z=EsU?)Lx zu_yuj**yW#h7t?{I}qqz{X}2`*g=q7DoOx%N)QljDZwDnn1F=(iNOZYSdd&ON&s)l z5D;xD!649(fQZ7jhc z(13uP`iaB_&_Iw}B1!;PN)ixlEx{o0AOT7B6N?StK|ykXC;>bvOF*=_1cSf>1Z34u zG&X<-1j)yu1aPD@0nzpn3<6sTNUNWCYyevY$%mo@@S{8d(FPL?0$T{ktDlH$09ypf zN1_C9V+jJHEhZQQY6+B3KQY+=Y6Zy$q6F|_83LkBCKv>22$WGjQP}`$1j*Z?1aM+0 z0-|ju7z8#FD5ZYlvH@%qByWlmz=!1sh&GyF5ZFMVocf8(2CzYpyd_Ei7nUR-+G>J9 z;2wdJ>L)fEz&%0ohA06%SeAfjvk3-)I|RzApXh7=cLd3gq6BbYX#%3{CKv?P2$WVo z@!0^@1j!Gg1n^&Z0-_Bk7z8Q=%B!CY*Z?YmL&*_fTbXL zAxZ%6=^!B5bb>+PmVgfRlLZ?9mMlR4=V>8OqWi>*PC$$L$%73bx))2-kMHylDA9jn z#wMUg{ba%h5Zj9-s>gMj2$WC&F(VVuq<(T?1BmR!61C$wT?9&KfS7R!=u$t~umQyN zVu{LeoHha_R6xwA1hlE2eAob@da*>^_)Q;y5;`DeOal7UPeyD2F}+x#YTTxgKnW!f zGa>L({QfQVi!Q8QlCNuY!lh#8N7PW6)&8$diSmZ%t~X(doX4aAH_K&$%6iwz)} z7faNO&-4-~p$B5dBA{3OWX1*%%Znwd#bufalu!gQBN5Q7esW_2h~&i*wc;_|1WIUv zm~jZ`RzKOX0mSiQiAr&pb^;|-LCh!ww5y-|*Z`tpoBJv>7RfN^^+wVfPXKRs1RpaAy7gc z#Pm+Uiu%cu4Zyn>OVo$2><}oS4`TWzU`PFA$_C)uizTYVRh9^pPzW(S6R@Oya%BVX z?8Oqb;VD}LN@#?behJu8KiRSY`1N9m%5ao50wq*JOs@p2sh@n=0K9s!L|ypF9)S`% zA*N3P_S8?tYydvJSfVQ2WRXA#r4Z920gLJXoi@62-sFX*|P!o@nVTeaFBHZB~(L9F9fWspZwVX zym+xh9r(vSffBkQrVj%4)lUU%06x4}q6*yOfj|l65Yqzz59+4|HUJM^EKvjA@j;-3 zc8F=8fDiRk1si~UFP5kP=XfDdLOsN^PQZ)$se=u`x))32k8k`CD4`!>+9u#f{ZzsR zVB3o&vd1-^2$WC|F)b7Dq<(5)1F-DH61n3UUj#~Mh?sT>_)__{ATA5;`KLO#=SZPep71HoaIPYuw_IKnW!g(;@+n>Zc|) z0E=EMkuzTLNuY$5h-r_2PxVt38-P78mdF^VcqLFmO~kZDz^nSHiw(e<7fa-ePy7-n zp(kS6BH&m3RK^Bi%Znwl#U-8zlu#5gEfMgnerjU_u;j%Ox#AJu1WIU%n05&GRzKCT z0od_kiA-^bcLF6;MNBIMysMx3*Z{0}u|%Hu!#{x%x+10x0{+!cg=_#eyjUVj+!295 z31tz}0)YtXr$#ma3tlXdBi@KXpoF%Fsh>a$^;0DqfPOEQ$Pi~lAy7hH#MDk8iu$RO z4M4jWOXPsh@h;0JM6sL|*tI9)S`%Bc@IQ@zhVn zYydjFSRyOj5RpI$r4dsjfr#p-W;OtgUM!IlUWiGcgw}|uk3dZIQ#BiaJ};KY2q#1( zP(p3Q)J7nx`l*`@K${m!847yu=#ry?fmhd?5rLW?9JPXP5pAQO;R9uYMVK>ZMy z6VM<=au6qg`XO*bKwNqRR6qdrL*SNx0yUC?Gy&8Pfdv6+?Gdj$fkp568v;B8Ebm9@ z3{X7;Rs^KgN4(MmR=wvH1lRyd`%yYRnukD@fVBRISC&Av_q>Jx8$ekIq1zk$BPs>U+;85MTq4_M>z> zG!B7n1f=yyJXr$Ude3JNU;~i#qjWSB4uS0iq!md#Ndnt@&!-Sz1CaEibS(4@frki4 zYm#_!1Rm-=pF@BRK+ccSkx(}T9ws2IO5#Zoc)0gm0Rc7uDL+cbLE8{$LO@!V#FHV= zr1x9{0X6^`KT1bI*$`+(Kw6o^lOWKn_gn=5HUJ4fO2`Fjdqr~fuz^=XLdI+!qbmvFulTkAS zb|)aMQsQ+*VE5j02L#vvy7Hs+xo8;z`w)=UDe<}?uut!~2LfyW-S|=ZRFn*X{Rl`a zm3Unc*su581pzjIF8nBcCOU?|z67MTO1$<7?Av?pg8&;q`+k%@5fwvVe*)5KC0^SE z_U}D+LVyjRZ9ht%hlU|=2mxum60cnXhxDF%A;1REt{7_)+>4lna4&2uLfJcx@19r}vx!0XBd({3v|} zx`jYn1f;b~yu%5!)qBo?02{#Jew01|)k2^>0@CUw-k}8A>pdqyfDPbKKT6+^W+Bif z0crgb?=S*w_MWpKzy@%bAEoa`u@GpNfV6^%cL;%Yd(UYQU;{YBkJ9&|R|vFCKw87Z z+n+$&z2`g#umSAvN9j9JD+JmnAgyBJ?MtBj-g6=Z*Z}tRqx5}f6#`ulkk&Er_9M_m z?>Q3!YykWDQTi^F3W072NGq9m`w-}+_nZm=Hh_KnD18q)g+NyXq_s@E-3fHnd(MRb z8^G>0g1K7on($}I-2y{(ATGPa9PN3`Fb3O#v0Gj(z`byLZ zf$j-NtD1OC33T6kB0zu*ps63FuS1&4(|e*ofDNFDAEmEBmk^L5AgyiUJxoBV_r!q!8^FVUl)eI0LO_myw7QA+5COT~ z6A1!r01x?5`g1f10Z9VV`X=6X0+PKa76jM;w);`~QxpjSSpw1uC*C#!vb`r71lRz! z`BC~a^aufI0@4~MUOfTn-V+Z3YykCsl>P)YLO`B?w91KBM?k*!M1%kvK%F0@-$#oO zC_zA4=fvAgphWM92>~{M&3=@A7bQZV3;}7S6K@lNGQB4%1lRyJ`BC~kbO?b`1f;c2 zy!!-7^`5v8U<0`CN9lJ^Aq2`1kXAeK?h+{1dm=-C4dAXHrGG|)5GYANTJOYLCs4BY z#D)MHz`7r$e?ox}C`&+E@x-eVDBF9YLx2sS>PP9<=nn#=2}o<6cq;;>dry1_umP<6 zDE$icL7+SVY1I>NL7;r^$p8Xu01H1#|Bdz_pn-t2?umCpKtu1z0Rn6QWhN5JgMbbK zW%M6Cs*c{11q9dtO3Wg32LUYvO6Wg&Of9`94+yXU$WJ3w2LU|<x zq~{TugMcOi()y1cPgC#71p;gUvJ(l#K|mJ)S^YFYfiL4XZFYA&HQ2xufArT^%0H1?jHAixG7Gnr5t z1auOR(SP(PI(tu65MTq4m`&&m0$K@3=s$W4t-U8N2(STkKb=q+1oRT-mJnAfTB**ZPlszUJPO8wA(@x}8uc3}0(J^)gSfDNFH*@ccEV3j}{`j38w)!vgg1lRx$ zpI)d40(J=;uK(yK*zG-;Lx2t7(D{XiAYhrmq56-0zvbSOI|SGO4x3;o2m-bV9H#&1 zciZkg*+YO0;E)-Hejs3-z#;mNey{c3lRpI50QR3^s0RY}3GA=`=y%%hJrzKJ4Pf6n zhISy}fxy1{kA9zr-cthv*Z}sMWGDv$J_zim|LAx5=si_HfDK@uS%z*P;Dx|G`j38( zm)=tc1lRy}pJu2A0)7bWuK(zF_~|{BK!6Qk*Lj9!AmE9>uKJIDy{Fz&3k28zcAID@ z1_Hha?56+dSNrNc)j)s^V3(POULfF&z%Kfaeyz9OQx62#0GdxV)B*v21e)tV`j!5A zPel-5186$e&}$Zm0wTehECR|L9lv?LC!2fDPcG`G!Uy;F-Wf`j7s(=iXBr z1lRz!PdF3;0pA3+>p%LZzI#t~5MTq?HsjC-1iTa2rvK=ldG9^-L4XaQe#)T^2>2&Z zum9+u`0qUxLVyjRZqA_%2t*)Ir~l~hN9a8@LVyim^Q1!=5Qsrwv;L#M8>9DB2>~{M zO|uSNKp+Z%P5O`iUXu9~FUI?%Otfn4n zfIvI~EB!}*6|eVH3;{NP#oR*+5Qs=%q5tTAN9;W{Lx2t7X7Zr~2*e~1{hJ^E$J9Ml z-x6Q|Bm$@b0#OMt0PufQ?NfImzyRR;A0q@NzyJu`d`gEv_yK?fAP|{A^Z^jL_tXx7 z=+6L900d$ah`j;C?mg8*Aod18{~-{aK;#V|dhe+p0+BZWx(@*e#N7bsJp>>ScLSjJ z5P(43|5spo4*>`y0zrQTp!X1fKqe4#0ML5~KwwTF&kdkS+2>9Fp=qm&u(2Rh`4WL=&Tm=CLc-#Q!Dg+?Vlz_ht zplRh?2LTB9+W_b(1R&6yfVT~xdF5OQ0SI{80O%+LAg~JoUmL(Km2)iwAmD2Qpq~(c zz-|OQZ2-Gf&eafrfTs}S~PQc3suzTg)0Raej z*#PJy1R$^v0UsN{K9zG11R&sJ1E7x(fWUqPJZu2_RnA=yfPjY$fG$D+0{arMzX9x9 zIrl*T0`@lmdI$js>`%b@2C#qS+z9~)SlSa00MS50D1=j2pmqp>IQIl<(vQk2w2?! z=o|zf&;|jU8$cVCa|Q$;U~>bYZxDb$I|M9l0PR%HDG-2w#SMV2K>z}65wN!bv{gCh zKmY>vHUN4C0SL55z}g1TUgeww0SH*z0O%M5AkZcOTN^-|m2(yZAYf|)pkEMxK)VDi zZ2;|7&S?;UfTaz9Zb1M7Z4~+VK$n$sHUuD`djp_95P(3p1T=2| z-B!-&5P*Q@4S?=I00LbT(7OS2T{-7N00Me90D1!f2y{juz$Z!RZe6GK%n#nKnEZIfszEuZU7}KCpH8iP<8`=KLj99mO#l3plszthX4dhZUE4S z00c@CD7OKWuAKM~fIzto0QL}oKzRbCHh}V#lK})EP-+8!JOm)1fk2rJKttu^009V; z*#O`U0SM?IP+|knQ8`&a00Jd80H{L%0$K>jZva{*& zfdB-gHvou300No_$Zi0dDkm2RKtOf_fHwpnpo@Uy2B52QvVi~uBsTzPLjVHW2*_;! z+A1d>2tYt?1AsLIAfS(c)CQohax#Jd1f(_qNJ9Vu8VSg302(VNCkQ}5W&?mT1R$W3 zfW!u%vvRV600bm904PHM0$K@lzX52ioV*|af$lc|7()O8dI@yB0qCuq%pd@Pt~US( zLjVGr33R&wXs(>xAOL}GHvsrT00O!Rbh!cOuAJ;30D&$y0O&#h0@?|5w*hFcoctgF zf$lZ{*g^mT`U!Nk0qC!s3?Tr4t~LP3LI4652z0XnSg4#FApn7HHUPLn00K4$bg=>0 zsGKYz0D&$x0H{I$0#*pLzX4dOoID`_f%Z25m_h&ob_le+0obXWOd$Y)wl@HXLI47m z2(-HaSgM>{Apn7PHvo7-00OoMw7CJ;s+?>g0D(3)0BAx00@et$w*gqIoO~ewf%Y~4 zSV8~-_6W4K0obdYj3EGlwl)ArLI4653AD2TSgf3!Apn7PHUKz600K4%w6Ou$temVN z0D(3(04PEL0#*qez5!URoV+0bfx|Zd7(xI7b_pE10obja%pm}QLpJ~jLI47m2^_Wo zSgxGhApn8HHURiR00OoN9I^q}uAJ;40D(g`0O&yg0@exazX4dUoctjGf&DiC*g*gS z_6h8}0obpc3LpT1eK!EeK>z|C2<*23c&MBjAOL~=HUPLm00KS;?6U#*sGKSw0D*lr z0H{F#0$vF0z5#ftoH`%?f!#L%m_YynehBQk0r;t$N+1A%T{i%TK>z}t2<)~2c&eOQ zAOL~gHUM}*00O=U?6Lv)s+?*d0D)aL0BAt~0^SHT-vGQ-PCXESK=Ta%RuF)IKLSlR z0DqNJ5d2w>WCQS7IaNUb0!=mmC_w-M zUI{$B0eG#Px*!07hc^HiK>z}N2|Tm`_^q7EAOL}fHUJ1g00N!~Y~KJpS59pZfWY<* z06q|afNuiZHUQt1Qyl~#ux$f?4g?_Jok0Bt;JtF{g8&5THvrf`00RCA)NKI%E2lyT zK%i~|fD8m65P`tv4Io10)Cd6xY~BFi0s#obAh2lzh*3FJLI47rHUOwV00L16+}{AA zR8E}`fWZ9?045NCKpX;hH-I>mQz-->aCZZM2m~MyiNN{>5UFx%g#ZNBHvo7*00OZH zR5yTFl~XMQAW+=^paB60L?f`;0HRe+y%2!FY6E};1RxNPz+wZ4S2-0!00N5*01^;@ zKtuvJ8$iU$sTl$gxY+>U009WZB#<|Nn3YpC1R#(%04P8J0#OOz|EQg(ZU`g-eE%Z@ z2m~hZ%b)h1zJBxe{q@No^nP;u=;Zf%M<-9eyuAMERqvydqjy(-fARIzo8FV-@1Fhg zwD;^EuU`M^)%D4zz3-<>d-J#RA1A;3LGQyaUthjE`MCE{`sC@&oBy%*;oHl1*C&JC zd3y7+-s9uPtJ5DJf71IGCqK*=T%<7Xuikuhm1F(t`c=B{<(Gfbd-l!y>n4eP`DeYO jx8Gj9yLxkdGVTpisI&W_e*ffO-d?jQ+Rq<<`tkn<9OPtV literal 0 HcmV?d00001 diff --git a/onnxruntime/test/testdata/attention_past_state.u8s8.onnx b/onnxruntime/test/testdata/attention_past_state.u8s8.onnx new file mode 100644 index 0000000000000000000000000000000000000000..b6fe39ff3c042920a674c4ac1f1541b865cece39 GIT binary patch literal 1059004 zcmeI2U6UKxk)GB3>LzAJLA4ZXsY6Co`XX-$ZtCPad6q z^!nASZ=V0@{pG9cmw&!GdwBl5`~3XZ|Lgz0==`|z@a3yF@2^kqjk;$izyIR$&DBMJ z@bqE%Q)dv~F3vvqyVtMYU59%w`opIu)r)68>^%I(%dg&FJ^S#<@h2zWnf>RVj~;Y> z)d{DC{}ti(-r3{dU0#22_4cAadirSkPU<)IHr)7^ z*?%IyaO3#Aeq;ac8~;bV@#mrBQTX4a6AlP}kHbN+uV20V?CN3=`x<}hJPRLp!$-wg z=l6ele|7ce@Zs?3@$Bbmd#>KS2<K=xM6hG`HxrcN*N9&PmgCm$0SC*(|eQi zdJ@C#+9cxj;kZe6>H1>r^+BzZz-!Wt$p>hYAlq5u!`|w||0ZNS4!6X#`ircG{YBRQ zA$;6FJr2y?{`cqI(}(}(2bNdy{b`)#0#`YG6L8D{eTd31_`wKMsk;S|+FWyXWU~y9`6mRyp>ki!8$R z^SLpcRA~(3IrYIYI$xVd-L-$)iRZ;Xig?0ec8t2)WHKKgqoT@W9M7rGkI^O%)`wVm z=dfnN(H18BOEh75=csKL)Xa*y_^5quA=w!@r6j zmZ2sK7mr43TpT;nt&fz^xR%qhGCr)$(I3Yslki9PJf6@!d-U&L{^P}H^7MZAyYsX7 zK{tL-oE`t&<-6;P@%hsS@rP%RJNK?%|L`~t`0}dQ&~C4~s(bsY^BAcNvNUPN>rGqU z(TnE}>oyF;@*p+7x&{d z_(^9rQo@}@H4Mh{ug=)>qF2-5c>Q;0UHQ?4MG^mqsEYo49!%DJeb)0BRC!G1-=Ae3 zbtg^gU!cp{yEzSt$i65~7%lP$BYD+as^>8uj+1_U78LdAIN9XOI+57%Ufn&D`XaQ( z(eqi*>(+WOsZZoJcg|-)F%H%DqH23_Di%>hz1Zn;7EEHN=d+*}9oFkDX2B$Uf3EyS zAH@s%XCGFZ+(q!+R>R;YF_B*UqaTkLoE^tu5GM9?8vHzd6310}wgD}T$8@;?N zcC;6Et=VP0+n1FEmvJU%AI*1-5N*9#%Ufb~{qyBp25bEtZ|Rk{6w9{^*WU8$cuO1v z{m_`+Fm|UO+4lqc+0GC~_v)GOZ)T2-_e68``)40i`%akVi;)mVe?L~=ADn$O9|EEJ z$-0?v|HV<93IDQF?HBRpuzyW=`Y^vM8h14#*S5dzg0j*$tNP^;yoFs^)J{pMlVN1d29I2t0Ab*={2{OGveRI`s1V?#HC>nXXPNuY7qNg z&WP~6QVof)n1>JIr(rjl4Tg*Gm9t?=48qssBCTTm*siY(aY}?=0Ky+}Yz`)~DN*zm zQ(_Qn9-deBFZ#FbAE(4Hwssiy%juLT2DdMZ{_V#&!`_mAi}n7Ew+!OaGVCwkvi^6P zc+0T7Ww?CHc;l29##uQGt?3QR6C(U!_zkC;62)Zol=$V`!}0!Twqf}Fb3P|}=gV_q zSk@n=WNl9L!tX}&m#cP%cFdqZGHQ&oo45NrI zJWN+rl`-r?Tjx8Ca3|5x2x-ddt51CEx_ zteNw3G*M8alFxWalKu0wk}PkD8VnC5>ffmS^}k2QqDFBE8BI=)-@LtgclGLeQC&Zb z{i&Aczq*!BbpBPip(vxp%EAMP&OcmU|HJ#QF2a}I(?`>vJFzd}0Yq)S^y`!V2i0?x zlP5=?JO~2(P5jj=&X#dJIXsdGe}vyB$_*quu_}KbiNCFk%Y(zxI{U5(&#Y!ovto1N zV~oW_3y-80&$Dh0+Pdh=Tf=V|wOcoh_o~M+Dm>V#p^6Vr7WEFc{k=KX zJ6Y=Mpx)P;lq)w>sKS$%depkz*W5b)CJaw-V(9os9K4e_al)gS>YFe;;i{nu4`#xr z;mN=P)w+bTT2TK9~syd`!r zJglkTGF)%o5V{DDT54><&{1CJU@Sad42ur`KK>0-^F~Rrzc@<^@f30 zI5?gjSEA|CFpNSdf3v7EEzXy}`NC7Nb;XsLE-en>nb>xjh6aW-VpgGnA!Ph1=)Z{1 zIg0bRKhE--KgkShLiiwj8Xj)U)`ajhto{h%ah%ZqI$YL`@)(DMd&7Xf2#+it|CjJw z`syNl#Xfyl{@4j73LnHz!|%<%fBj#tzrOq?d>$6_*^Bw03%`Kh>x4lUt_;V9hmGN< z@GVmfy6`Nl8g#+7;iLHZ@+u#miLLzhoBw(rEE?{KH!jxr@E~kv(J=dq?$TTdPrVkH zm%kh9vn)c6Pcy>Z@rc?QAD)e^+?*GQELQmNG_00Lc(PGb=BV!4T$>XKk2|WN6&`slhE@@tchnQvG^FcHqJqMMj{2>ehV+g3UKC5;D&ZN} zq7Q3^RyE&?_@E_(j(>y^6Y`IlEY|mN?04zJD9!jFq$uj^dw4vycD_%=Q^YTe@_~8~ z{}QT6z25OAkKRNTo`uvZ!;YrBSE$cU}cuRb&QiP`>OSi0l{u%YsEB}aYj!!X)UU2htBfAJ+f~&W)U&MOK z+s3DlUcbK%|N2+^x<#}#q{8d>fancKl?%T$=8?HUx!_={N(9J(}DT-udc8Dc=fjP zrnB~d)1Orb+&uc?v+(=KEa2HsHeR*#%}_4jpLaf(*7feQ%df6FAI?fDU!DFpP&8oj!UK*`kzOgt{8+Uge z#El^Q!&~?TvpVPU^Ut5&|IO#0pFX*M{r0mjKK}CM^~YD=ym|fh`q}s6KXU#y{sZ49 zC)MV*cyk>78Tj#ZRxI9}d~#Af9hkp4JYV%@ch#H4Cnxu2<7aVuZ}IQVi#Pk9oZPRj z{OHM1$Z|Lczd5}4@&9@DzrTHPauojW+mrC$C-;84ybpcy?YA$g@1<(e@izn@;D^8w z1HezSsT2Y?2plm0ZV=%m1pE*P{$~U5({3t-z!HJze+Ix35?(;S4S~}C3;;L%rcemf z2~7RZ0H`zJX9##9F!MhHz)Qoa69P2?mH!z4H7NWD0Vf3J{$~I<={RLVU_oHve+IyU z3O_->2Z5UZ82~<7PL&Xt6R7*20WinH4-jxcV9Eas02e){NC;E}mi^BFsJQSq1UwM9 z;eQ5zho(~_1ZD(o`kw)SG$7!Iz%BnX0Q@wcO5GENfRp}Hrf&l4{LcXJtqZF6 zL13-_82~;6K$WfutoJ_yz_l3;@r{pmq-gHu|3d;6Vb^ z=$OD}|1$s_OM}uK5V*tt3;+ijphUj}?(#naz^^u_+&+Oj{m%ffF9IrbOW4TndDuO%vF`{|o@rN}xE81a|R1 z1Hhv~sM#`so&3)Luq*{?b4Xw}|1$s_N`#UP6WG!J3;@GgpfrC3cJ)64z@J8_*e-#c z{m%ffD+VfaM__mVGXUI)go4cy*u(z}0JCbKFmD9*@jnB=n@XtHDuKQH&j7G02kLT0 zU_bve0G!E$a*Yz$)Bg+rqk5n$Uj+8`KLfy*PN>!#84)s3+z>ij_)Ev$o~ugW4fRm9|W56KLfyrUZ~O*fyVsL z0I($ts&PS}IsY>NTnL6DO%Z6&{|o?A%Agnz1e)|e1Hgk~sL>LEM*YtKup|v?aX_G1 z|1$s_NQM#(5op-|3;;vgpcMNAn)W{fz`kav&<=sd{m%ffBMvGtPoR1KGXTtsh62qH zXuV3ZV$g1lsgJ1HiI! zsLcX_R{hTaupkj?FifCb|1$s#ONY`75NO%|3;+Wfp#-}G+V(#Kz^-Am)DtK;A+jdy@p@{LcU|DIkiEERG$NI=&A3;=@?qNI!o2>YJ_kg--s-5vpH z|1$vWX^4vQB_Qs920*@IA#-yC-TB4Ht2qgDE10X-qkg*v8>HW_DFe4@k$&P>m|1$ux zQw<4QA)vwk3;-)?qK@1MsPI1nAUD~NuMq+|{LcU|A}7kojDQmVGXOHv4e8n-pvC_T z02_Luio6J@@jn9~FX51_2?Bcj&j2tXD2m96fFl1h0J2gJ$yy+w$^Q%h3yPwKoCv7$ zKLa2q>5!`d0=oRq05BjaO2~+SGXFCGGSUvI=1)MI{}}-JYl;f;A)wCx41j#ZL#Eji z(C2>!K=z`dfNTgT^gjb28}*QA?gTXYp8=4&s>nYV0xJE_0LVo?wgA7*1{tBEC?v}KLa2O z1(9UV1T_1f0g$t@$UO%Fs{PLZ$U#Enm@xs}{$~JWEG<&cfPixUGXOHs5Gm$MK)e4L z0QqW*%=Hsc?|%k>ei4yjwgmM1p8=4qxJX<*feif508ps4#NYOdu=&GXOM8iQFN7k#abe@{0QXce+EE)8Y5%91hVr#13<5s$SgYo`T3s#ke$d#SS^7J{m%eU zt0of5jX;k6X8`1;GV;|*AWQ!<0JO@9yfPz@r~erMnaPZFl@iF*{|o@7dLpg72;}O2 z20&gqBU_yWvh_a$K&PO{Dk}o{`kw)imC#65C4r3n&j3)VD3Z#FK+gVW0OX`Ja@9y6 zYyUFD3;>0iBBgu?gB1Z?=90g#E@ zNK+XBBmQRqDAN_`ZFz9~h5s12@jnAVr~b$) zEde+FX8@!X8qrk}@Z*05fJy<9R8j(t{LcVLsx*RYB;d*a3;>M^B&U=FT=|~?kWy;I zR!G2?{}})ZB}hsM2{`jV10bQ+2(6ERH~%vL^l6Zc(h+dye+EE0u@PAv0e}8y0H_lo z2_+-o(Ekj8WNIU@HUb{~&j8S-Lh?yPz@`5g0IB3gTxA4&`kw)yOopVBh=5c7GXN6l zjj*~1c=bO6K$i~5CJh0%{$~KB5gbuf5%BAO27oFdl1vf;j{VO7NTN7`Y9ip-{|o?4 zN+g#Q1YG-{0gys+#8gDUxBnRcilj&?2?#j%KLa3v<_M{WfOr2h0Q6{)Oymi;_df$b zUUWoMLjeCX0Mv+)M8pZ;|04!~xatU~g#i8!1hh1RJRl%U0RKk%27%-R>gwlbHh|=UBmq$Zcz&6HD13rJ zATfbu_4A4iAh93`FG>K%-y|T4pI{J3O5mpYdCdlpRFDK0C4k>=6A*0x!61;3z-{&O z4jVv1K@wV&0B&DJK(qw}gFrF@tJKeXYyimwNnlX|czrbi(IyZK0*MH$RzL5u0VEP6 zVMPhx^mPP8+dwb~Bq6X){k+cxkVKFK6(xYr*AozJ1i>JXfWUh7^9dV30znc|lmIT@ zL_o9^1cQJ$flcb?Gd2KmK@w1u03P2=K(rYIgMcuB&FbeAu=4!?_lXgdf7 z0Z{^Xsh`i;07L~zFi`^d`)&fF4Ivl=1PR=&ey(5x5ELY#LRkwkwuWF3Xq~_=>gQTEfYt>`08s*X zdN%^1%^?^BS|+fY`nj47pk+aFyeI)2y(>uY zN&xpZK|r)+1cN|h1e#DkXRrY@CP)qvC4hIEAt2f`fph@*}78}69g5&^E0(iDr0;0_$7z7R^(5(77jSb*HL2|n&0UX;j0nzpm3<3uc zXj=W8#|ChaAh}tT0Df(rfM^2=27v6QuKGEh4PZw>a=9o09NIPk(RLCH0y`0CTm78R2C$PLxmc6{{%oIs zXhR7GfgK36uYMx10qh`1E)^w!J0%E+wv=EH*qVTZ`ia2?u(cq$P?P}Plp!G6RDwZZ zO9C?LCkh+DmV)FmQ35zqihyWa2?l|!2uP`)IBWo036hIM3E)dP0-}v27zDN;Ag6vJ zu>ou$NG=g2fGZ^lh_;qs5V(_or22`)25_e!xj>Wvo|GjZ+FXJ`;0^+^>L(f-z#W3* zV^IP)QksBhdkF@CjRd6CPdqk&je_JuQ3Ci;o`7hB2?l`;1mx9EL^gm8g5)Dn0=O{+ z0nru{3<7Hjq)WL6@ug~Q3ALyB>~Y^6AS{k2&7a$ zvDpA_36eKN3E;uB1Vo!nFbLcrkXHRfX9Kt)NM4E(z=5d=h_;(x5LhCRTK&Xl16UFy zFGLC8zw`t|8%{6?)CiOD#vl! z2&7N}F})Jdrhf8a1Mup_6m{b_eFReIfS5iB=utt>Vv4G9n??dDlt4_61T?Cj zoY(+7dND=KcugmP6j~srKLR?{PgZOI{=Ar?Vw|RxKngVw(;ES;>L)KY0B>GQQ7=Bz zOCW_Fi0O-fUiFh18-On_rl=N|X(o_D5ybRFK(qSEjSaw)7gN-V$8-}&p$TI8A)s6R zWXA^J$BQW{#bMeBq)-Jhy%5l@e)3}j@Z!Z3b>c7m1XAdNm_7*TS3eoD0r>D@iYjqe z4g^vtgP0x&%&;~K>6UamTWXT3#--{_K#96rzNTCj5S|^Z; z`pJ_Gz`7Sx)Q7L~A&^2J#I#KyAN7+d8-Q&url<~AS2h64UQAIN zp2~|r3XKrcE`hw%Pqu6TcD4$0P^=@iVAR!6#^;LL(JR>SW!Q9umR-m#T5DD8#@G2=!ckj6R@LxDq#c2 z+lwi($2FD+q)-qsb0%O({nWw+kh2$4=MB4&OBY^tBC*Z}hLVv3A$id6zB)I`kO2v}7=b+G~D=EW5G;uE_B zQs{}8c@eOyekx-F$jgf@kdS^d<;29T2%Q{;+AY!gVKDPrbBz_$9S zjtw9mFQ&*8hgc_&LRG}fg@ASSQy&{ZE?!KLC;qTcAcd}onFj&;>Zd|BfIPgIB1_!i zfj|mn5iSY7a>ctd!;Rk;NQs|7BItlnw zKNYh9==5TWtZ;)z0x6V6OpOFQs-K$K05p0rMNW9ZCxH}NBc?tAKGjdvYykSam?9&b z;FUlMwGmSr0k7((ZZ-gIUQCe>KJZH*h2Ds%i-2GCQ#l)eE-$9Y1{ZiHkV0|9)I`9u z`l+1_K$90!ZM41JL2c6j`8t2plm0Qdmz#OwW4riAg??kY9N66 zAy5*~AVzW!CxH4PFeM-^Jpw8qfchaYBcMQyq##WI^+TW{Agw*(r6*AJp1&c$L%{TY zl+FOvLtsHbT7ASzO<>V`eu4lSKx#iq$4B!Js1uOZAMw%>sP~?qA;1QZ){oNBQ9J~e z2}moDcqs`i_nub}U;{|$N9ow;9RfEANNbRI=?L8HJ+C3a29VB=(veX+1a1?MRw40H z5xCuZ-hlucKq@~<$3^Q9SVcfuhr~-mU{&vV4+3leY5XW16{SO9H34ZQ5-$aT)xGCk z2(SU9@S}80bPj=a1f;b{Jb41^de8e1U;~i%qjW@64uSOqq}50~X#(qe&nFOI1CaKk zbUZW;flUOY^+-Hf0-JizXAocmkoBW<+sNjymcn|sfv5MTq4^rLhv^bLW# z2uN#^cya{p>OG%BfDJ&-kJ6D)Hw5k`AgxN`NfEfa_gn!1HUKF0N@!BD7_)+>)lnjCW z2uLfHcx@2aulL*q0XBd({3v}UI)=c$1f;b}yygk)+k5VV02@H_ew02D6+>Wu0@7+F zUeg5j?>%=yfDNE&KT4m6h9PhW0cpJwuUP_z^qzYmzy{E)AEi%2!4No%fV5(X*Cc_% zde7YuU;}8PekYyi#pQTh~=3xQ?`NGq3kO%Q0N_nZO&Hh?Dl zD18RHg+NmTq_s=D!wEFid(MFX8^Gayls*B~LZCSU(&{DNp#+-iJtskc4d75eO5cxW zAS=R63o0qpNb={r#?1ezxxtzzQsOQ8AQb0P%T0QU8x^nGX*0&NhG)-mz+ zBhW_gITHeG0Q>n-`Yx0Tfp!Q;E17uv5NN0OoC*OpfPMTZeGfW?KwAW)wM@L-3AELF z&V>LQ!0vvOz5|s)pgjW8Y9`*U1lsF8CqsY@U{^m%Uynv1&?W(CJri#?0&Vu5vmw9+ zu$v#HuSTH|XqSMrqKUT)fp&Y(=@4K8*u{_1*P>4dv`s)-)5P1JK-<0Nd(zy`35AEmEBmk^L5AgyiU-AzEM_r!q!8^GOul)eI0LO_myw7Q9R z7Xi866A1!r0C)LO`g1f10Z9VV`X=6H0+PKa76jM;Hv3WfQxpjSSpw1uC*CFkvb`r7 z1lRyJ`BC~a^aufI0@4~M-g*Mky(b<7*Z|i1QTh|q2myHl(kdt3Is)>&Cn5yc0M_|Y z`hBzrffNL!bxyq11XA>#m=ItCSnWsYcTpk)(h!hVI`LK!NYi_wLVyiml^>(${-3f&d$U)LcSq5YR|KO8?&DXzV>XL4XZF zW-_5P2T6<4k5MTpne>$Nu2MCqD?V0kk)zP!|OB6KGHW-cQ!wdoqLo8$erg3T;6k2Lf&B-}|{a z^qw3czy{FHq(WH`$b&#T`uBcn9=#_^2(SUPF{{uO1acwJhW@>unM?1<69Q}i%}*;- z1%Z4BG_QZ}C+5?8GKByeK-2RIO+g?h0!{1R`*}I_o?Ico2GH!pLQxRNi$Jsb_kLPl zy(e1;umLnVv(OU+awE{B{=J`-Tkpvi0&D=yO)b;}f&2(Gr+@D!<=1;Mh5#EtQ*#R~ zK_Ev0P3hnJIXU*8oFTvl(9GmQNf5}BKr{OHeoCIbCu<0>0W>kY&=CZ3CD4Tay`Pb5 z@5vhiYygK(FH{79dfihQIrpC2A;1Q3*aSmC z5XhUrVfy!eciz1xdkC-r95Tbu4+L^2aESiB-^9L*35_V1nd&HTmRm#u-khog8&=AUGoi%K)^DA zyY%nL9=duxZAj4+vN%uu1>kKeOI@>Vp6q z!1^hNIv`-5zMC<6jM2&~q> z_ji5to+=^02C!<@p$iCjA+SpS-rw`md+LM$8^G;phbkc8hrn(9dw<7I@2M05Yydas z9h!iECjvM1@BOu>-cu_C*Z`I%9*Tf~F9OT@_x{RP@2M67YykC{haMo{jX+)h-hcMi zd+LP%8^B`fp#})}Be2lF_n-Xro{AyB22jmCv;YB*1SOe*>VZoWCIe0e=qx={W=-upr=V16WkfPY{5Bw+(=fLjVGG z0=_nYdgc5K0SNfo0O&UaAh1lp(+03yIjsl*Z@{n&bttR zfQJo$E<*qU>j>E20M=E``w)PD{SAN~LjVHn30U6%)>qCa5P*R74S)_q00Nr`*xmp( zRnBJ+fPn1{fc`=N0-Fg~-T*dN&ZiK7faMKw*hQhIoCk|0`@ildI|vuY)`=22C#kQTnPaPSla;TCh0O%hCAaEFgoHu~OD(7wpKp^K0fbKy60*4aFcLO-Ia_)x!1oGVg=p6(ga5#Zn zH-N({=L85qAlD6m&OrbIO%TX)18Aah&VT>}^4tLE8w4QG41pXsfMzP^6bL{d#|?n4 zK>z|x5y)=?XsU9~fdB;Z+W_bp1R&5Hf!sEL<|^kT2tXjW4S=vU1LX z00i>d0O%J4AkZv|LfNntm0!z}65zxN@v{gChLI48#HvoDB0SL55K>G&JUgew&0SIW{0O$||AkZcO-5Wrgm2)-( zAfS5#pg$0RK)VDqZvgF9&gl?zHUN480SL$u(6#}{RZb)bKtS6DKt~_|0Z9V7HUP=Wi3I@&=-L42 z2LvD>OF+{GAX_=nAOHbP8vxya00g87=-B|ID<>WVAfRUhpcfE;fII;$8-RS}M1%kY zv}^!$0s;_7K|sd_kfL&8LI46fHURnn0SKfapkV_@Q#nx~009jf09}9p1X2-5zX7DG zoVXBxK>7`U9zXyB=?J9W0Mb=XWC%bY^#(u(AOL}s1k!E*DJv&71R#)h1AspSAdr?o z$_*fG>+_z-|Vx(xvK5P(2>0;x8D^p%qV1R#)V1AsgP zAfSOjnhiih<>UYX2&CBn;0^%@=pc|{1JF@9SwH{+DK-G8LjVF=2*_^$S}G?G2tYu7 z1AsXMAfShU^ah}(ax#Gc1f(|rh(iDZnh3~l0GcW%7YIN=b_0Mn1R$V`faC_Ct8%h| z00bmA0BA!10@?`3Z2;OTCm#qvKyCwoH3T4_kATz$ps#W=f&c`hHULON00J5b$ZP-_ zD<>xiKtN^#fHMRjpp$^a2B5QYvVs5vBsKsjLjVF=3ADcfXsw*QAOL~(Hvkwz00Mdm zw7mi7t(?pt0D-nQ00=_>0-6c5y8&pfoZKJ)fp#|l_(A{zx(T$o0qCxr>>vPvHa7t1 zLI48V3ADEXXs?|7AOL~(HUQW{00R06w6y`~ubd1a0D-nP0LVfB0yz+9X9LKga&m+K z1lrjE;0gf<q3xVc0fLtmkPY6Ju`3(T35P(2F1e)Fe z@~NCmApn7z|)2<*NASgD*kAOL~gHvpJH00MRh?79Kishmn6 z0D)aM0Ej^V0+tBuwgFhGoLV3Nf!#I$ctHRHwg~L90obaXY9IiCT{Zw{K>z~Q2yDLr zSgV|RAOL~wHvm{c00Q<1Y`X#2tDK4;0D)~c07yXq0u~8uw*gqJoSGm2f$cT`I6(jc zHVJIA0obgZsvrP?Z8iWXK>z|)3EaH_SgoA8AOL~8Hvkwx00MRi+_eGNt(?jr0D-$U z00=<<0+tDE-T*9DPHhl?z~&7AJ`jL_Z33G%0Na&Q9Rwh-X#;=`1R!9Y!1@iqdgasy z0SK($0AK?F2-qjEZUeAiITbJ0!c5P*OW0;@Iv zAC*%j1R$_#1AqzyAmD|-?G3<7<^o+_tS2tZ(Y1AqqvAmEEYeFN}SIn_b{0`(058W4bhHv)?dz+2_i3jqi$HUL;a z00RC9R2zW5%BdIv5U4f)NI(Dr9tli00FRYZGXx+o-2mVK0SNdc5H|pyl~XkYAP_eI zC_n%LUJ2lTubroE2m}Ir|04qkI41CiUvwV6eD&u2_32MLKRLN~`lHVA>BCPiuRr^u z^Z4}m-PNBizPftVxqtHQqd%N=9{tnR%Rhc`efp&H-IM#Lo$~GY$LSxw-+A!q%gcAC zA9Nm{9GyO#zWJlh2j$Th?>@Wy>MC4+5hX$;Jrmv_f9_S{MG3X;sqC> zy!ThHKD&xF{_*;YaN&zjf7yBT`u%kUI&0oc<`?szh%s-F!4(G4p)uW}W z<2&QpJ4a>w$<24h=eORe=Xeye8ZUATr;Z0V@0i>=W3qfkr*m)8D{DUfGX6ZC{T$Ao zOy)Iw(7E3|Kc9V6zUa;iT1Q`&r&MW` zFDA9v-Sed+yH#HC!fsK^xLePvJE(s$tiPy7>Q<5BX+?EfEV)>yq5Q0>vV2h;US7Bm zLiyRK7OSXY#j1Qg^%grm_TJByvvRCC*zNo5Ot?8NCtHS9(`F}wXCif zZcyb_zNo4!V~>}T98`J53y1ZhhxM$6^`eLMrVs0phV@9JMU}&Fc$hPP+xa+5sb6kA z48JVzD^}kZhUL0J`eiJ1v`}h!On>z;8)k9oz|mAZQ#*#~R(^j>KbaOg`()be*^4RE zqcV@#q&f=IG%_An@M7m~%&qdR+jC)jdoGOY+8Wn;7zSTV@wimRjhPqEpA@$w73N_4 zZk&DtOe>gFS;mwnb^lE2{hm+F^0F$85T-c4<=5i8y0iRz`FkMTUxbMkGk+X072WFM z(8{8_WUQiFE54YIs(4|skX$^Zs7bh(kFgj&jInyjO-yQ)m&GEp*C%73|1d$MX#PyeWVq=DyP5deD`-(Uw?fuJinN2 d7gHUFZ@!t7j(+<==lJsF^VhFlee>jl{|^~w0&)NV literal 0 HcmV?d00001 diff --git a/onnxruntime/test/testdata/attention_past_state.u8u8.onnx b/onnxruntime/test/testdata/attention_past_state.u8u8.onnx new file mode 100644 index 0000000000000000000000000000000000000000..5ce855c0d8b6bda2d807973baa9f1eab97d338f0 GIT binary patch literal 1059004 zcmeI2U6UKxk)GB3>LzAJLA4ZXsY6Co`XX-$ZtCPad6q z^!nASZ=V0@{pG9cmw&!GdwBl5`~3XZ|Lgz0==`|z@a3yF@2^kqjk;$izyIR$&DBMJ z@bqE%Q)dv~F3vvqyVtMYU59%w`opIu)r)68>^%I(%dg&FJ^S#<@h2zWnf>RVj~;Y> z)d{DC{}ti(-r3{dU0#22_4cAadirSkPU<)IHr)7^ z*?%IyaO3#Aeq;ac8~;bV@#mrBQTX4a6AlP}kHbN+uV20V?CN3=`x<}hJPRLp!$-wg z=l6ele|7ce@Zs?3@$Bbmd#>KS2<K=xM6hG`HxrcN*N9&PmgCm$0SC*(|eQi zdJ@C#+9cxj;kZe6>H1>r^+BzZz-!Wt$p>hYAlq5u!`|w||0ZNS4!6X#`ircG{YBRQ zA$;6FJr2y?{`cqI(}(}(2bNdy{b`)#0#`YG6L8D{eTd31_`wKMsk;S|+FWyXWU~y9`6mRyp>ki!8$R z^SLpcRA~(3IrYIYI$xVd-L-$)iRZ;Xig?0ec8t2)WHKKgqoT@W9M7rGkI^O%)`wVm z=dfnN(H18BOEh75=csKL)Xa*y_^5quA=w!@r6j zmZ2sK7mr43TpT;nt&fz^xR%qhGCr)$(I3Yslki9PJf6@!d-U&L{^P}H^7MZAyYsX7 zK{tL-oE`t&<-6;P@%hsS@rP%RJNK?%|L`~t`0}dQ&~C4~s(bsY^BAcNvNUPN>rGqU z(TnE}>oyF;@*p+7x&{d z_(^9rQo@}@H4Mh{ug=)>qF2-5c>Q;0UHQ?4MG^mqsEYo49!%DJeb)0BRC!G1-=Ae3 zbtg^gU!cp{yEzSt$i65~7%lP$BYD+as^>8uj+1_U78LdAIN9XOI+57%Ufn&D`XaQ( z(eqi*>(+WOsZZoJcg|-)F%H%DqH23_Di%>hz1Zn;7EEHN=d+*}9oFkDX2B$Uf3EyS zAH@s%XCGFZ+(q!+R>R;YF_B*UqaTkLoE^tu5GM9?8vHzd6310}wgD}T$8@;?N zcC;6Et=VP0+n1FEmvJU%AI*1-5N*9#%Ufb~{qyBp25bEtZ|Rk{6w9{^*WU8$cuO1v z{m_`+Fm|UO+4lqc+0GC~_v)GOZ)T2-_e68``)40i`%akVi;)mVe?L~=ADn$O9|EEJ z$-0?v|HV<93IDQF?HBRpuzyW=`Y^vM8h14#*S5dzg0j*$tNP^;yoFs^)J{pMlVN1d29I2t0Ab*={2{OGveRI`s1V?#HC>nXXPNuY7qNg z&WP~6QVof)n1>JIr(rjl4Tg*Gm9t?=48qssBCTTm*siY(aY}?=0Ky+}Yz`)~DN*zm zQ(_Qn9-deBFZ#FbAE(4Hwssiy%juLT2DdMZ{_V#&!`_mAi}n7Ew+!OaGVCwkvi^6P zc+0T7Ww?CHc;l29##uQGt?3QR6C(U!_zkC;62)Zol=$V`!}0!Twqf}Fb3P|}=gV_q zSk@n=WNl9L!tX}&m#cP%cFdqZGHQ&oo45NrI zJWN+rl`-r?Tjx8Ca3|5x2x-ddt51CEx_ zteNw3G*M8alFxWalKu0wk}PkD8VnC5>ffmS^}k2QqDFBE8BI=)-@LtgclGLeQC&Zb z{i&Aczq*!BbpBPip(vxp%EAMP&OcmU|HJ#QF2a}I(?`>vJFzd}0Yq)S^y`!V2i0?x zlP5=?JO~2(P5jj=&X#dJIXsdGe}vyB$_*quu_}KbiNCFk%Y(zxI{U5(&#Y!ovto1N zV~oW_3y-80&$Dh0+Pdh=Tf=V|wOcoh_o~M+Dm>V#p^6Vr7WEFc{k=KX zJ6Y=Mpx)P;lq)w>sKS$%depkz*W5b)CJaw-V(9os9K4e_al)gS>YFe;;i{nu4`#xr z;mN=P)w+bTT2TK9~syd`!r zJglkTGF)%o5V{DDT54><&{1CJU@Sad42ur`KK>0-^F~Rrzc@<^@f30 zI5?gjSEA|CFpNSdf3v7EEzXy}`NC7Nb;XsLE-en>nb>xjh6aW-VpgGnA!Ph1=)Z{1 zIg0bRKhE--KgkShLiiwj8Xj)U)`ajhto{h%ah%ZqI$YL`@)(DMd&7Xf2#+it|CjJw z`syNl#Xfyl{@4j73LnHz!|%<%fBj#tzrOq?d>$6_*^Bw03%`Kh>x4lUt_;V9hmGN< z@GVmfy6`Nl8g#+7;iLHZ@+u#miLLzhoBw(rEE?{KH!jxr@E~kv(J=dq?$TTdPrVkH zm%kh9vn)c6Pcy>Z@rc?QAD)e^+?*GQELQmNG_00Lc(PGb=BV!4T$>XKk2|WN6&`slhE@@tchnQvG^FcHqJqMMj{2>ehV+g3UKC5;D&ZN} zq7Q3^RyE&?_@E_(j(>y^6Y`IlEY|mN?04zJD9!jFq$uj^dw4vycD_%=Q^YTe@_~8~ z{}QT6z25OAkKRNTo`uvZ!;YrBSE$cU}cuRb&QiP`>OSi0l{u%YsEB}aYj!!X)UU2htBfAJ+f~&W)U&MOK z+s3DlUcbK%|N2+^x<#}#q{8d>fancKl?%T$=8?HUx!_={N(9J(}DT-udc8Dc=fjP zrnB~d)1Orb+&uc?v+(=KEa2HsHeR*#%}_4jpLaf(*7feQ%df6FAI?fDU!DFpP&8oj!UK*`kzOgt{8+Uge z#El^Q!&~?TvpVPU^Ut5&|IO#0pFX*M{r0mjKK}CM^~YD=ym|fh`q}s6KXU#y{sZ49 zC)MV*cyk>78Tj#ZRxI9}d~#Af9hkp4JYV%@ch#H4Cnxu2<7aVuZ}IQVi#Pk9oZPRj z{OHM1$Z|Lczd5}4@&9@DzrTHPauojW+mrC$C-;84ybpcy?YA$g@1<(e@izn@;D^8w z1HezSsT2Y?2plm0ZV=%m1pE*P{$~U5({3t-z!HJze+Ix35?(;S4S~}C3;;L%rcemf z2~7RZ0H`zJX9##9F!MhHz)Qoa69P2?mH!z4H7NWD0Vf3J{$~I<={RLVU_oHve+IyU z3O_->2Z5UZ82~<7PL&Xt6R7*20WinH4-jxcV9Eas02e){NC;E}mi^BFsJQSq1UwM9 z;eQ5zho(~_1ZD(o`kw)SG$7!Iz%BnX0Q@wcO5GENfRp}Hrf&l4{LcXJtqZF6 zL13-_82~;6K$WfutoJ_yz_l3;@r{pmq-gHu|3d;6Vb^ z=$OD}|1$s_OM}uK5V*tt3;+ijphUj}?(#naz^^u_+&+Oj{m%ffF9IrbOW4TndDuO%vF`{|o@rN}xE81a|R1 z1Hhv~sM#`so&3)Luq*{?b4Xw}|1$s_N`#UP6WG!J3;@GgpfrC3cJ)64z@J8_*e-#c z{m%ffD+VfaM__mVGXUI)go4cy*u(z}0JCbKFmD9*@jnB=n@XtHDuKQH&j7G02kLT0 zU_bve0G!E$a*Yz$)Bg+rqk5n$Uj+8`KLfy*PN>!#84)s3+z>ij_)Ev$o~ugW4fRm9|W56KLfyrUZ~O*fyVsL z0I($ts&PS}IsY>NTnL6DO%Z6&{|o?A%Agnz1e)|e1Hgk~sL>LEM*YtKup|v?aX_G1 z|1$s_NQM#(5op-|3;;vgpcMNAn)W{fz`kav&<=sd{m%ffBMvGtPoR1KGXTtsh62qH zXuV3ZV$g1lsgJ1HiI! zsLcX_R{hTaupkj?FifCb|1$s#ONY`75NO%|3;+Wfp#-}G+V(#Kz^-Am)DtK;A+jdy@p@{LcU|DIkiEERG$NI=&A3;=@?qNI!o2>YJ_kg--s-5vpH z|1$vWX^4vQB_Qs920*@IA#-yC-TB4Ht2qgDE10X-qkg*v8>HW_DFe4@k$&P>m|1$ux zQw<4QA)vwk3;-)?qK@1MsPI1nAUD~NuMq+|{LcU|A}7kojDQmVGXOHv4e8n-pvC_T z02_Luio6J@@jn9~FX51_2?Bcj&j2tXD2m96fFl1h0J2gJ$yy+w$^Q%h3yPwKoCv7$ zKLa2q>5!`d0=oRq05BjaO2~+SGXFCGGSUvI=1)MI{}}-JYl;f;A)wCx41j#ZL#Eji z(C2>!K=z`dfNTgT^gjb28}*QA?gTXYp8=4&s>nYV0xJE_0LVo?wgA7*1{tBEC?v}KLa2O z1(9UV1T_1f0g$t@$UO%Fs{PLZ$U#Enm@xs}{$~JWEG<&cfPixUGXOHs5Gm$MK)e4L z0QqW*%=Hsc?|%k>ei4yjwgmM1p8=4qxJX<*feif508ps4#NYOdu=&GXOM8iQFN7k#abe@{0QXce+EE)8Y5%91hVr#13<5s$SgYo`T3s#ke$d#SS^7J{m%eU zt0of5jX;k6X8`1;GV;|*AWQ!<0JO@9yfPz@r~erMnaPZFl@iF*{|o@7dLpg72;}O2 z20&gqBU_yWvh_a$K&PO{Dk}o{`kw)imC#65C4r3n&j3)VD3Z#FK+gVW0OX`Ja@9y6 zYyUFD3;>0iBBgu?gB1Z?=90g#E@ zNK+XBBmQRqDAN_`ZFz9~h5s12@jnAVr~b$) zEde+FX8@!X8qrk}@Z*05fJy<9R8j(t{LcVLsx*RYB;d*a3;>M^B&U=FT=|~?kWy;I zR!G2?{}})ZB}hsM2{`jV10bQ+2(6ERH~%vL^l6Zc(h+dye+EE0u@PAv0e}8y0H_lo z2_+-o(Ekj8WNIU@HUb{~&j8S-Lh?yPz@`5g0IB3gTxA4&`kw)yOopVBh=5c7GXN6l zjj*~1c=bO6K$i~5CJh0%{$~KB5gbuf5%BAO27oFdl1vf;j{VO7NTN7`Y9ip-{|o?4 zN+g#Q1YG-{0gys+#8gDUxBnRcilj&?2?#j%KLa3v<_M{WfOr2h0Q6{)Oymi;_df$b zUUWoMLjeCX0Mv+)M8pZ;|04!~xatU~g#i8!1hh1RJRl%U0RKk%27%-R>gwlbHh|=UBmq$Zcz&6HD13rJ zATfbu_4A4iAh93`FG>K%-y|T4pI{J3O5mpYdCdlpRFDK0C4k>=6A*0x!61;3z-{&O z4jVv1K@wV&0B&DJK(qw}gFrF@tJKeXYyimwNnlX|czrbi(IyZK0*MH$RzL5u0VEP6 zVMPhx^mPP8+dwb~Bq6X){k+cxkVKFK6(xYr*AozJ1i>JXfWUh7^9dV30znc|lmIT@ zL_o9^1cQJ$flcb?Gd2KmK@w1u03P2=K(rYIgMcuB&FbeAu=4!?_lXgdf7 z0Z{^Xsh`i;07L~zFi`^d`)&fF4Ivl=1PR=&ey(5x5ELY#LRkwkwuWF3Xq~_=>gQTEfYt>`08s*X zdN%^1%^?^BS|+fY`nj47pk+aFyeI)2y(>uY zN&xpZK|r)+1cN|h1e#DkXRrY@CP)qvC4hIEAt2f`fph@*}78}69g5&^E0(iDr0;0_$7z7R^(5(77jSb*HL2|n&0UX;j0nzpm3<3uc zXj=W8#|ChaAh}tT0Df(rfM^2=27v6QuKGEh4PZw>a=9o09NIPk(RLCH0y`0CTm78R2C$PLxmc6{{%oIs zXhR7GfgK36uYMx10qh`1E)^w!J0%E+wv=EH*qVTZ`ia2?u(cq$P?P}Plp!G6RDwZZ zO9C?LCkh+DmV)FmQ35zqihyWa2?l|!2uP`)IBWo036hIM3E)dP0-}v27zDN;Ag6vJ zu>ou$NG=g2fGZ^lh_;qs5V(_or22`)25_e!xj>Wvo|GjZ+FXJ`;0^+^>L(f-z#W3* zV^IP)QksBhdkF@CjRd6CPdqk&je_JuQ3Ci;o`7hB2?l`;1mx9EL^gm8g5)Dn0=O{+ z0nru{3<7Hjq)WL6@ug~Q3ALyB>~Y^6AS{k2&7a$ zvDpA_36eKN3E;uB1Vo!nFbLcrkXHRfX9Kt)NM4E(z=5d=h_;(x5LhCRTK&Xl16UFy zFGLC8zw`t|8%{6?)CiOD#vl! z2&7N}F})Jdrhf8a1Mup_6m{b_eFReIfS5iB=utt>Vv4G9n??dDlt4_61T?Cj zoY(+7dND=KcugmP6j~srKLR?{PgZOI{=Ar?Vw|RxKngVw(;ES;>L)KY0B>GQQ7=Bz zOCW_Fi0O-fUiFh18-On_rl=N|X(o_D5ybRFK(qSEjSaw)7gN-V$8-}&p$TI8A)s6R zWXA^J$BQW{#bMeBq)-Jhy%5l@e)3}j@Z!Z3b>c7m1XAdNm_7*TS3eoD0r>D@iYjqe z4g^vtgP0x&%&;~K>6UamTWXT3#--{_K#96rzNTCj5S|^Z; z`pJ_Gz`7Sx)Q7L~A&^2J#I#KyAN7+d8-Q&url<~AS2h64UQAIN zp2~|r3XKrcE`hw%Pqu6TcD4$0P^=@iVAR!6#^;LL(JR>SW!Q9umR-m#T5DD8#@G2=!ckj6R@LxDq#c2 z+lwi($2FD+q)-qsb0%O({nWw+kh2$4=MB4&OBY^tBC*Z}hLVv3A$id6zB)I`kO2v}7=b+G~D=EW5G;uE_B zQs{}8c@eOyekx-F$jgf@kdS^d<;29T2%Q{;+AY!gVKDPrbBz_$9S zjtw9mFQ&*8hgc_&LRG}fg@ASSQy&{ZE?!KLC;qTcAcd}onFj&;>Zd|BfIPgIB1_!i zfj|mn5iSY7a>ctd!;Rk;NQs|7BItlnw zKNYh9==5TWtZ;)z0x6V6OpOFQs-K$K05p0rMNW9ZCxH}NBc?tAKGjdvYykSam?9&b z;FUlMwGmSr0k7((ZZ-gIUQCe>KJZH*h2Ds%i-2GCQ#l)eE-$9Y1{ZiHkV0|9)I`9u z`l+1_K$90!ZM41JL2c6j`8t2plm0Qdmz#OwW4riAg??kY9N66 zAy5*~AVzW!CxH4PFeM-^Jpw8qfchaYBcMQyq##WI^+TW{Agw*(r6*AJp1&c$L%{TY zl+FOvLtsHbT7ASzO<>V`eu4lSKx#iq$4B!Js1uOZAMw%>sP~?qA;1QZ){oNBQ9J~e z2}moDcqs`i_nub}U;{|$N9ow;9RfEANNbRI=?L8HJ+C3a29VB=(veX+1a1?MRw40H z5xCuZ-hlucKq@~<$3^Q9SVcfuhr~-mU{&vV4+3leY5XW16{SO9H34ZQ5-$aT)xGCk z2(SU9@S}80bPj=a1f;b{Jb41^de8e1U;~i%qjW@64uSOqq}50~X#(qe&nFOI1CaKk zbUZW;flUOY^+-Hf0-JizXAocmkoBW<+sNjymcn|sfv5MTq4^rLhv^bLW# z2uN#^cya{p>OG%BfDJ&-kJ6D)Hw5k`AgxN`NfEfa_gn!1HUKF0N@!BD7_)+>)lnjCW z2uLfHcx@2aulL*q0XBd({3v}UI)=c$1f;b}yygk)+k5VV02@H_ew02D6+>Wu0@7+F zUeg5j?>%=yfDNE&KT4m6h9PhW0cpJwuUP_z^qzYmzy{E)AEi%2!4No%fV5(X*Cc_% zde7YuU;}8PekYyi#pQTh~=3xQ?`NGq3kO%Q0N_nZO&Hh?Dl zD18RHg+NmTq_s=D!wEFid(MFX8^Gayls*B~LZCSU(&{DNp#+-iJtskc4d75eO5cxW zAS=R63o0qpNb={r#?1ezxxtzzQsOQ8AQb0P%T0QU8x^nGX*0&NhG)-mz+ zBhW_gITHeG0Q>n-`Yx0Tfp!Q;E17uv5NN0OoC*OpfPMTZeGfW?KwAW)wM@L-3AELF z&V>LQ!0vvOz5|s)pgjW8Y9`*U1lsF8CqsY@U{^m%Uynv1&?W(CJri#?0&Vu5vmw9+ zu$v#HuSTH|XqSMrqKUT)fp&Y(=@4K8*u{_1*P>4dv`s)-)5P1JK-<0Nd(zy`35AEmEBmk^L5AgyiU-AzEM_r!q!8^GOul)eI0LO_myw7Q9R z7Xi866A1!r0C)LO`g1f10Z9VV`X=6H0+PKa76jM;Hv3WfQxpjSSpw1uC*CFkvb`r7 z1lRyJ`BC~a^aufI0@4~M-g*Mky(b<7*Z|i1QTh|q2myHl(kdt3Is)>&Cn5yc0M_|Y z`hBzrffNL!bxyq11XA>#m=ItCSnWsYcTpk)(h!hVI`LK!NYi_wLVyiml^>(${-3f&d$U)LcSq5YR|KO8?&DXzV>XL4XZF zW-_5P2T6<4k5MTpne>$Nu2MCqD?V0kk)zP!|OB6KGHW-cQ!wdoqLo8$erg3T;6k2Lf&B-}|{a z^qw3czy{FHq(WH`$b&#T`uBcn9=#_^2(SUPF{{uO1acwJhW@>unM?1<69Q}i%}*;- z1%Z4BG_QZ}C+5?8GKByeK-2RIO+g?h0!{1R`*}I_o?Ico2GH!pLQxRNi$Jsb_kLPl zy(e1;umLnVv(OU+awE{B{=J`-Tkpvi0&D=yO)b;}f&2(Gr+@D!<=1;Mh5#EtQ*#R~ zK_Ev0P3hnJIXU*8oFTvl(9GmQNf5}BKr{OHeoCIbCu<0>0W>kY&=CZ3CD4Tay`Pb5 z@5vhiYygK(FH{79dfihQIrpC2A;1Q3*aSmC z5XhUrVfy!eciz1xdkC-r95Tbu4+L^2aESiB-^9L*35_V1nd&HTmRm#u-khog8&=AUGoi%K)^DA zyY%nL9=duxZAj4+vN%uu1>kKeOI@>Vp6q z!1^hNIv`-5zMC<6jM2&~q> z_ji5to+=^02C!<@p$iCjA+SpS-rw`md+LM$8^G;phbkc8hrn(9dw<7I@2M05Yydas z9h!iECjvM1@BOu>-cu_C*Z`I%9*Tf~F9OT@_x{RP@2M67YykC{haMo{jX+)h-hcMi zd+LP%8^B`fp#})}Be2lF_n-Xro{AyB22jmCv;YB*1SOe*>VZoWCIe0e=qx={W=-upr=V16WkfPY{5Bw+(=fLjVGG z0=_nYdgc5K0SNfo0O&UaAh1lp(+03yIjsl*Z@{n&bttR zfQJo$E<*qU>j>E20M=E``w)PD{SAN~LjVHn30U6%)>qCa5P*R74S)_q00Nr`*xmp( zRnBJ+fPn1{fc`=N0-Fg~-T*dN&ZiK7faMKw*hQhIoCk|0`@ildI|vuY)`=22C#kQTnPaPSla;TCh0O%hCAaEFgoHu~OD(7wpKp^K0fbKy60*4aFcLO-Ia_)x!1oGVg=p6(ga5#Zn zH-N({=L85qAlD6m&OrbIO%TX)18Aah&VT>}^4tLE8w4QG41pXsfMzP^6bL{d#|?n4 zK>z|x5y)=?XsU9~fdB;Z+W_bp1R&5Hf!sEL<|^kT2tXjW4S=vU1LX z00i>d0O%J4AkZv|LfNntm0!z}65zxN@v{gChLI48#HvoDB0SL55K>G&JUgew&0SIW{0O$||AkZcO-5Wrgm2)-( zAfS5#pg$0RK)VDqZvgF9&gl?zHUN480SL$u(6#}{RZb)bKtS6DKt~_|0Z9V7HUP=Wi3I@&=-L42 z2LvD>OF+{GAX_=nAOHbP8vxya00g87=-B|ID<>WVAfRUhpcfE;fII;$8-RS}M1%kY zv}^!$0s;_7K|sd_kfL&8LI46fHURnn0SKfapkV_@Q#nx~009jf09}9p1X2-5zX7DG zoVXBxK>7`U9zXyB=?J9W0Mb=XWC%bY^#(u(AOL}s1k!E*DJv&71R#)h1AspSAdr?o z$_*fG>+_z-|Vx(xvK5P(2>0;x8D^p%qV1R#)V1AsgP zAfSOjnhiih<>UYX2&CBn;0^%@=pc|{1JF@9SwH{+DK-G8LjVF=2*_^$S}G?G2tYu7 z1AsXMAfShU^ah}(ax#Gc1f(|rh(iDZnh3~l0GcW%7YIN=b_0Mn1R$V`faC_Ct8%h| z00bmA0BA!10@?`3Z2;OTCm#qvKyCwoH3T4_kATz$ps#W=f&c`hHULON00J5b$ZP-_ zD<>xiKtN^#fHMRjpp$^a2B5QYvVs5vBsKsjLjVF=3ADcfXsw*QAOL~(Hvkwz00Mdm zw7mi7t(?pt0D-nQ00=_>0-6c5y8&pfoZKJ)fp#|l_(A{zx(T$o0qCxr>>vPvHa7t1 zLI48V3ADEXXs?|7AOL~(HUQW{00R06w6y`~ubd1a0D-nP0LVfB0yz+9X9LKga&m+K z1lrjE;0gf<q3xVc0fLtmkPY6Ju`3(T35P(2F1e)Fe z@~NCmApn7z|)2<*NASgD*kAOL~gHvpJH00MRh?79Kishmn6 z0D)aM0Ej^V0+tBuwgFhGoLV3Nf!#I$ctHRHwg~L90obaXY9IiCT{Zw{K>z~Q2yDLr zSgV|RAOL~wHvm{c00Q<1Y`X#2tDK4;0D)~c07yXq0u~8uw*gqJoSGm2f$cT`I6(jc zHVJIA0obgZsvrP?Z8iWXK>z|)3EaH_SgoA8AOL~8Hvkwx00MRi+_eGNt(?jr0D-$U z00=<<0+tDE-T*9DPHhl?z~&7AJ`jL_Z33G%0Na&Q9Rwh-X#;=`1R!9Y!1@iqdgasy z0SK($0AK?F2-qjEZUeAiITbJ0!c5P*OW0;@Iv zAC*%j1R$_#1AqzyAmD|-?G3<7<^o+_tS2tZ(Y1AqqvAmEEYeFN}SIn_b{0`(058W4bhHv)?dz+2_i3jqi$HUL;a z00RC9R2zW5%BdIv5U4f)NI(Dr9tli00FRYZGXx+o-2mVK0SNdc5H|pyl~XkYAP_eI zC_n%LUJ2lTubroE2m}Ir|04qkI41CiUvwV6eD&u2_32MLKRLN~`lHVA>BCPiuRr^u z^Z4}m-PNBizPftVxqtHQqd%N=9{tnR%Rhc`efp&H-Ee8Ad^`Se`iJj#9(?-p^4;kN zoyR9frw^xZ{;2aodGy7*&n~~Z3fG^6&mMH{og9U;KAfJl;o6(_W_=0)89<-dj|4@8rYIU!DFSUT_i0dw=!n zv#VI+AFsa%7ryxPmz_ti-(Ocijy3@K^f#U3H*c@rU4{IHoqh;)emm5UPXER1npM%h zeA2l$>Xy|Vm(|_>^26{>ar&t9;NUePTj>)YvCd+4ZI`<~MvgYG2=w0*yY;NPgZd}K`ipv`ZWSq>R#d0Ol8c2J%Fn7Q%NNz*<%J6&l%I`i zv5G2Ith!k4e|)RA+`RIwk^PyVqZ0)2s3-FRT~ct7p}#7u~Nnyk^p_IN4DL6uj$a9A&TSkG!$FM3#S`mi2pSdTPXR5=WXhdJ}NosYwm`sLQc@XPYP zV)cDtSgsqSU&ca53#FFF^j9CVVHTGT98JYDwPToW<@d+*lWDQDPo~YDy_iBhD)X34 zs-rMXBja%eFLv(6+$!I?Jr~Be=fb$It#Q4FVerKik4t6Tn0fL1NpVY3VGhRc#_2b} zw1P>MWlVWe_s^u>@A=d$FRRiBVT$uxel5nV<7^|0DzIDr3MRChmMN#Edo?pqmd{Jdp9(HS} z6}N|4(Yp<)KGcd{&2>c>YSYHXVD*t!^y)d)M_SRVa{8;zcYk;F_171}^NZPbG1YPS Y=9@|B=(itqjxS$6fBov!H%~tJ{|7z-Z~y=R literal 0 HcmV?d00001