saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-29 20:14:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Move attention test data to file (#17158)

Browse source 
(1) Move attention test data from code to file to avoid prefast crash
(which blocks python packaging pipeline)
(2) Enable some test cases that previously disabled in Windows
(3) Fix an assertion error in
`MultiHeadAttentionTest.CrossAttention_WithPastPassedInDirectly_NoMask`
This test case is for Whisper cross attention. When Memory efficient
attention was enabled, format is converted to BNSH, which trigger
assertion error since memory efficient attention asserts BSNH format.
Temporarily disable memory efficient attention for this case. I also
disabled the test since Whisper does not use it anymore, and ROCm fails
in the test.
This commit is contained in:
Tianlei Wu 2023-08-15 21:31:57 -07:00 committed by GitHub
parent 33ecde9af1
commit 6b29837ed2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 5741 additions and 6474 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

32
onnxruntime/contrib_ops/cuda/bert/attention_impl.cu
View file

@ -336,6 +336,7 @@ Status PrepareQkv(contrib::AttentionParameters& parameters,
}
// attention with past/present state
else if (data.past_key != nullptr || data.present_key != nullptr) {
// Below logic does not support memory efficient attention with past (like pass_past_in_kv) but without bias
if (data.bias == nullptr) {
// cross attention with past state
if (data.past_key != nullptr && data.present_key == nullptr) {
@ -344,7 +345,7 @@ Status PrepareQkv(contrib::AttentionParameters& parameters,
assert(data.key == nullptr);
assert(data.value == nullptr);
ORT_RETURN_IF_ERROR(LaunchTransQkv(stream, 1, sequence_length, batch_size, qk_head_size, num_heads,
max_threads_per_block, false, data.query, q));
max_threads_per_block, false, data.query, q));
}
// cross attention with present state or self attention with present state
else if (data.past_key == nullptr && data.present_key != nullptr) {
@ -356,13 +357,13 @@ Status PrepareQkv(contrib::AttentionParameters& parameters,
// TODO: supporting packed qkv for self attention may benefit performance
ORT_RETURN_IF_ERROR(LaunchTransQkv(stream, 1, sequence_length, batch_size, qk_head_size, num_heads,
max_threads_per_block, false, data.query, q));
max_threads_per_block, false, data.query, q));
// TODO: supporting packed kv for cross attention may benefit performance
ORT_RETURN_IF_ERROR(LaunchTransQkv(stream, 1, kv_sequence_length, batch_size, qk_head_size, num_heads,
max_threads_per_block, false, data.key, data.present_key));
max_threads_per_block, false, data.key, data.present_key));
ORT_RETURN_IF_ERROR(LaunchTransQkv(stream, 1, kv_sequence_length, batch_size, v_head_size, num_heads,
max_threads_per_block, false, data.value, data.present_value));
max_threads_per_block, false, data.value, data.present_value));
}
// self attention with past and present state
else {
@ -375,11 +376,11 @@ Status PrepareQkv(contrib::AttentionParameters& parameters,
assert(data.value != nullptr);
// TODO: supporting packed qkv for self attention may benefit performance
ORT_RETURN_IF_ERROR(LaunchTransQkv(stream, 1, sequence_length, batch_size, qk_head_size, num_heads,
max_threads_per_block, false, data.query, q));
max_threads_per_block, false, data.query, q));
ORT_RETURN_IF_ERROR(LaunchTransQkv(stream, 1, kv_sequence_length, batch_size, qk_head_size, num_heads,
max_threads_per_block, false, data.key, k));
max_threads_per_block, false, data.key, k));
ORT_RETURN_IF_ERROR(LaunchTransQkv(stream, 1, kv_sequence_length, batch_size, v_head_size, num_heads,
max_threads_per_block, false, data.value, v));
max_threads_per_block, false, data.value, v));
}
qkv_format = AttentionQkvFormat::Q_K_V_BNSH;
}
@ -397,9 +398,9 @@ Status PrepareQkv(contrib::AttentionParameters& parameters,
// query => q, temp_k_workspace => k, temp_v_workspace => v
LaunchAddBias(stream, max_threads_per_block,
batch_size, sequence_length, kv_sequence_length,
num_heads, qk_head_size, v_head_size,
data.bias, data.query, data.temp_k_workspace, data.temp_v_workspace, q, k, v);
batch_size, sequence_length, kv_sequence_length,
num_heads, qk_head_size, v_head_size,
data.bias, data.query, data.temp_k_workspace, data.temp_v_workspace, q, k, v);
DUMP_TENSOR_D("q(BSNH)", q, batch_size * sequence_length, num_heads, qk_head_size);
DUMP_TENSOR_D("k(BSNH)", k, batch_size * kv_sequence_length, num_heads, qk_head_size);
@ -419,11 +420,11 @@ Status PrepareQkv(contrib::AttentionParameters& parameters,
// temp_k_workspace (BxSxNxH) => present_k (BxNxSxH)
ORT_RETURN_IF_ERROR(LaunchTransQkv(stream, 1, kv_sequence_length, batch_size, qk_head_size, num_heads,
max_threads_per_block, false, data.temp_k_workspace, data.present_key));
max_threads_per_block, false, data.temp_k_workspace, data.present_key));
// temp_v_workspace (BxSxNxH_v) => present_v (BxNxSxH_v)
ORT_RETURN_IF_ERROR(LaunchTransQkv(stream, 1, kv_sequence_length, batch_size, v_head_size, num_heads,
max_threads_per_block, false, data.temp_v_workspace, data.present_value));
max_threads_per_block, false, data.temp_v_workspace, data.present_value));
DUMP_TENSOR_D("q(BSNH)", q, batch_size * sequence_length, num_heads, qk_head_size);
DUMP_TENSOR_D("k(BSNH)", data.temp_k_workspace, batch_size * kv_sequence_length, num_heads, qk_head_size);
@ -688,8 +689,7 @@ Status QkvToContext(
if (qkv_format == AttentionQkvFormat::Q_K_V_BNSH) {
k = data.present_key;
v = data.present_value;
}
else {
} else {
assert(qkv_format == AttentionQkvFormat::Q_K_V_BSNH);
k = data.temp_k_workspace;
v = data.temp_v_workspace;
@ -1111,12 +1111,12 @@ Status DecoderQkvToContext(
constexpr int max_sequence_length = 0;
ORT_RETURN_IF_ERROR(ComputeSoftmaxWithRawMask<T>(ort_stream, kv_sequence_length, sequence_length, batch_size,
num_heads, nullptr, key_padding_mask, add_before_softmax,
false/*broadcast rpb*/, scratch1, scratch2, is_unidirectional,
false /*broadcast rpb*/, scratch1, scratch2, is_unidirectional,
1.0f, mask_dimension, max_sequence_length, false, nullptr,
mask_filter_value));
} else {
ORT_RETURN_IF_ERROR(ComputeSoftmax<T>(stream, kv_sequence_length, sequence_length, batch_size, num_heads,
add_before_softmax, false/*broadcast rpb*/, scratch1, scratch2,
add_before_softmax, false /*broadcast rpb*/, scratch1, scratch2,
is_unidirectional));
}

7
onnxruntime/contrib_ops/cuda/bert/multihead_attention.cc
View file

@ -166,17 +166,23 @@ Status MultiHeadAttention<T>::ComputeInternal(OpKernelContext* context) const {
}
}
const bool pass_key_value_as_past = (parameters.pass_past_in_kv && nullptr != key && nullptr != value);
#if USE_FLASH_ATTENTION
bool is_long_sequence = sizeof(T) == 2 || // sequence length threshold is 0 for FP16
parameters.sequence_length >= attention::kMinSequenceLengthForMemoryEfficientAttentionFp32 ||
parameters.kv_sequence_length >= attention::kMinSequenceLengthForMemoryEfficientAttentionFp32;
// Exclude this case since PrepareQkv will convert the format to BNSH.
bool past_no_bias = (pass_key_value_as_past || past_key != nullptr || present_key != nullptr) && bias == nullptr;
bool is_good_for_rpb = relative_position_bias != nullptr && parameters.sequence_length % (4 * sizeof(T)) == 0;
bool use_memory_efficient_attention = fused_runner == nullptr &&
fused_cross_attention_kernel == nullptr &&
!disable_memory_efficient_attention_ &&
is_long_sequence &&
!past_no_bias &&
(relative_position_bias == nullptr || is_good_for_rpb) &&
(nullptr == key_padding_mask || is_mask_1d_key_seq_len_start) &&
has_memory_efficient_attention(sm, sizeof(T) == 2);
@ -226,7 +232,6 @@ Status MultiHeadAttention<T>::ComputeInternal(OpKernelContext* context) const {
data.mask_index = (nullptr == key_padding_mask) ? nullptr : key_padding_mask->Data<int>();
data.mask_index_dims = (nullptr == key_padding_mask) ? gsl::span<const int64_t>() : key_padding_mask->Shape().GetDims();
data.past = nullptr;
const bool pass_key_value_as_past = (parameters.pass_past_in_kv && nullptr != key && nullptr != value);
data.past_key = pass_key_value_as_past ? reinterpret_cast<const CudaT*>(key->Data<T>())
: (nullptr == past_key) ? nullptr
: reinterpret_cast<const CudaT*>(past_key->Data<T>());

88
onnxruntime/test/contrib_ops/attention_op_test.cc
View file

@ -846,30 +846,12 @@ void RawAttentionEmptyPastState(bool past_present_share_buffer) {
}
}
// Disable Causal_EmptyPastState temporarily in Windows build since prefast crashes in python package pipelines
// TODO(tianleiwu): change the test to load test data from file.
#ifndef _MSC_VER
TEST(AttentionTest, Causal_EmptyPastState) {
int batch_size = 1;
int sequence_length = 2;
int hidden_size = 64;
int number_of_heads = 2;
std::vector<float> input_data = {
0.00838f, 0.007523f, -0.00872f, 0.002882f, -0.003567f, 0.000859f, -0.002821f, 0.000563f, 0.007675f, -0.002758f,
0.000947f, 0.001149f, -0.001534f, 0.0006075f, 0.002853f, 0.004517f, 0.00825f, 0.003687f, -0.002161f, 0.001167f,
0.005913f, 0.00394f, -0.002136f, 0.00946f, 0.000461f, -0.003593f, -0.002377f, -0.001609f, -0.006363f, 0.0013485f,
-0.006706f, -0.005188f, 0.002165f, 0.006073f, 0.007717f, -0.007675f, 0.000827f, 0.004253f, 0.00697f, -0.0035f,
-0.00301f, 0.006565f, -0.0002068f, -0.004593f, 0.00198f, 0.00107f, -0.003082f, 0.002243f, 0.00983f, 0.00608f,
0.001682f, 0.001701f, -0.006935f, 0.004765f, -0.002333f, 0.003805f, -0.00905f, 0.00599f, 0.00998f, -0.001602f,
0.00744f, -0.008514f, 0.005424f, -0.002413f, 0.00862f, 0.00459f, -0.002516f, 0.00283f, -0.00272f, -0.005207f,
-0.00738f, -0.005386f, -0.00951f, 0.008415f, 0.002865f, -0.00726f, 0.00494f, 0.002226f, 0.0000424f, -0.007507f,
0.002193f, -0.004482f, 0.002386f, 0.005997f, -0.001786f, 0.009f, 0.006435f, -0.0067f, -0.001984f, 0.001514f,
-0.004917f, 0.003468f, -0.0013685f, -0.007122f, 0.00788f, 0.000825f, 0.00621f, -0.00437f, 0.005653f, 0.009674f,
0.003576f, 0.00956f, 0.0064f, 0.00283f, -0.00797f, 0.00867f, 0.004536f, -0.00985f, 0.004856f, -0.006878f,
0.006012f, -0.0042f, -0.00328f, -0.00885f, -0.0079f, 0.004917f, -0.00594f, 0.003452f, -0.006355f, -0.003536f,
0.0022f, 0.003494f, -0.008865f, 0.00461f, -0.00485f, 0.00889f, -0.002272f, 0.00596f};
std::vector<float> weight_data;
std::vector<float> bias_data;
GetAttentionWeight(weight_data);
@ -878,74 +860,13 @@ TEST(AttentionTest, Causal_EmptyPastState) {
// No mask_index
std::vector<int32_t> mask_index_data = {};
std::vector<float> output_data = {
0.0027942657f, 0.0067901611f, 0.0070953369f, -0.0020713806f, 0.0055351257f, 0.0030479431f, -0.0060462952f,
-0.0087127686f, 0.0030956268f, -0.00036644936f, 0.0014686584f, -0.0038146973f, 0.0072097778f, -0.0052490234f,
0.0056114197f, 0.0050926208f, 0.0080947876f, 0.0074501038f, 0.0079498291f, 0.0098876953f, -0.0066146851f,
0.0064735413f, 0.0093307495f, -0.00051593781f, -0.0047683716f, -0.0069198608f, 0.0094604492f, 0.0066146851f,
-0.0040054321f, 0.0017976761f, -0.0058059692f, -0.0087051392f, 0.0054740906f, 0.0022010803f, 0.0075340271f,
0.0047035217f, 0.00340271f, 0.0096969604f, -0.0016756058f, 0.0020771027f, -0.0063018799f, 0.0073280334f,
-0.0056381226f, 0.004032135f, -0.0082473755f, 0.0045280457f, 0.0045814514f, -0.0026607513f, -0.0031585693f,
-0.003660202f, -0.0053253174f, -0.0089187622f, -0.0073509216f, 0.0048408508f, 0.0058364868f, 0.0069313049f,
-0.0071868896f, 0.008392334f, -0.0018663406f, -0.0092163086f, -0.00048780441f, -0.0054283142f, -0.0061683655f,
0.0078048706f, 0.0025291443f, 0.0065917969f, 0.0072250366f, -0.0018520355f, 0.005531311f, 0.003118515f,
-0.0061264038f, -0.0090484619f, 0.003276825f, -0.00047063828f, 0.0015802383f, -0.0037345886f, 0.0069732666f,
-0.0054092407f, 0.0052947998f, 0.004940033f, 0.0085220337f, 0.007194519f, 0.0078659058f, 0.0095214844f,
-0.0065574646f, 0.0064315796f, 0.0093383789f, -0.00058555603f, -0.0046386719f, -0.0067710876f, 0.0096130371f,
0.0064315796f, -0.0040740967f, 0.0017337799f, -0.0057067871f, -0.008682251f, 0.0054855347f, 0.0019645691f,
0.0075149536f, 0.0047187805f, 0.0036354065f, 0.0096282959f, -0.0019168854f, 0.0021934509f, -0.0063018799f,
0.0072937012f, -0.006187439f, 0.0039825439f, -0.0081253052f, 0.0046577454f, 0.0045700073f, -0.0028266907f,
-0.0028438568f, -0.0035438538f, -0.0053100586f, -0.0090332031f, -0.0071105957f, 0.004699707f, 0.0058021545f,
0.0071411133f, -0.0071678162f, 0.0085449219f, -0.0018749237f, -0.0095825195f, -0.00049686432f, -0.0053634644f,
-0.0057945251f, 0.0078277588f};
std::vector<float> input_data;
std::vector<float> output_data;
std::vector<float> present_data;
GetCausal_EmptyPastState(input_data, output_data, present_data);
std::vector<float> past_data = {};
std::vector<float> present_data = {
0.0070152283f, -0.0049858093f, -0.0029277802f, 0.0078277588f, -0.001991272f, -0.0010290146f, -0.0084457397f,
-0.0028400421f, 0.0048294067f, 0.0012731552f, 0.0047149658f, 0.0069084167f, 0.0027809143f, 0.0014457703f,
-0.0010128021f, -0.0011024475f, 8.4400177e-05f, -0.0049972534f, -0.0040206909f, 0.002073288f, -0.0034713745f,
-0.0087203979f, -0.0047302246f, -0.0023326874f, -0.0063209534f, -0.0031681061f, -0.006942749f, 0.0064888f,
0.0014505386f, -0.0037765503f, 0.0067138672f, -0.0018196106f,
0.0064506531f, -0.0049514771f, -0.0036487579f, 0.0081558228f, -0.0024414062f, -0.0014820099f, -0.0086212158f,
-0.0025672913f, 0.0047111511f, 0.0011997223f, 0.0042953491f, 0.0067138672f, 0.0028495789f, 0.0015869141f,
-0.00037360191f, -0.0012044907f, 0.00029373169f, -0.005065918f, -0.0038700104f, 0.0014038086f, -0.0030422211f,
-0.0084838867f, -0.004863739f, -0.0028686523f, -0.0063362122f, -0.0034809113f, -0.0075874329f, 0.0066947937f,
0.0019130707f, -0.0036792755f, 0.0070266724f, -0.0016460419f,
-0.003238678f, -0.0066452026f, 0.0043983459f, -0.0016002655f, 0.0045623779f, 0.0065002441f, -0.0072174072f,
-0.0050315857f, 0.0087356567f, 0.0061645508f, 0.0069580078f, -0.003320694f, -0.0087814331f, 0.0062255859f,
0.0035037994f, 0.00064849854f, -0.0018444061f, 0.0043945312f, 0.01008606f, -0.0089874268f, -0.0087585449f,
0.0020160675f, 0.00207901f, -0.0097732544f, -0.0042991638f, 0.0070266724f, -0.0028743744f, 0.0087051392f,
0.0099868774f, 0.0076217651f, -0.0027103424f, -0.006439209f,
-0.0033836365f, -0.0063171387f, 0.0043144226f, -0.001707077f, 0.0044555664f, 0.0069885254f, -0.0072593689f,
-0.0050468445f, 0.008895874f, 0.0050582886f, 0.0064926147f, -0.0030384064f, -0.0083618164f, 0.0065307617f,
0.0038928986f, 0.0005645752f, -0.0024528503f, 0.0043983459f, 0.0099029541f, -0.0088043213f, -0.0081558228f,
0.0021705627f, 0.0018062592f, -0.0094985962f, -0.0045890808f, 0.0068702698f, -0.002532959f, 0.0081863403f,
0.009765625f, 0.0077362061f, -0.0026664734f, -0.0060920715f,
0.0027942657f, 0.0067901611f, 0.0070953369f, -0.0020713806f, 0.0055351257f, 0.0030479431f, -0.0060462952f,
-0.0087127686f, 0.0030956268f, -0.00036644936f, 0.0014686584f, -0.0038146973f, 0.0072097778f, -0.0052490234f,
0.0056114197f, 0.0050926208f, 0.0080947876f, 0.0074501038f, 0.0079498291f, 0.0098876953f, -0.0066146851f,
0.0064735413f, 0.0093307495f, -0.00051593781f, -0.0047683716f, -0.0069198608f, 0.0094604492f, 0.0066146851f,
-0.0040054321f, 0.0017976761f, -0.0058059692f, -0.0087051392f,
0.0022659302f, 0.0063896179f, 0.0073509216f, -0.0016336441f, 0.0055236816f, 0.0031890869f, -0.0062026978f,
-0.0093917847f, 0.0034580231f, -0.00057506561f, 0.0016918182f, -0.0036563873f, 0.0067405701f, -0.005569458f,
0.0049743652f, 0.0047874451f, 0.0089492798f, 0.0069389343f, 0.0077819824f, 0.0091552734f, -0.0065002441f,
0.0063934326f, 0.0093460083f, -0.00065517426f, -0.0045127869f, -0.0066223145f, 0.009765625f, 0.0062484741f,
-0.0041465759f, 0.0016708374f, -0.0056037903f, -0.0086669922f,
0.0054740906f, 0.0022010803f, 0.0075340271f, 0.0047035217f, 0.00340271f, 0.0096969604f, -0.0016756058f,
0.0020771027f, -0.0063018799f, 0.0073280334f, -0.0056381226f, 0.004032135f, -0.0082473755f, 0.0045280457f,
0.0045814514f, -0.0026607513f, -0.0031585693f, -0.003660202f, -0.0053253174f, -0.0089187622f, -0.0073509216f,
0.0048408508f, 0.0058364868f, 0.0069313049f, -0.0071868896f, 0.008392334f, -0.0018663406f, -0.0092163086f,
-0.00048780441f, -0.0054283142f, -0.0061683655f, 0.0078048706f,
0.0054931641f, 0.0017261505f, 0.0074958801f, 0.0047340393f, 0.003868103f, 0.0095596313f, -0.0021572113f,
0.0023078918f, -0.0063018799f, 0.0072631836f, -0.0067367554f, 0.0039329529f, -0.0080032349f, 0.0047874451f,
0.0045623779f, -0.0029945374f, -0.0025291443f, -0.0034275055f, -0.0052986145f, -0.0091400146f, -0.0068702698f,
0.0045623779f, 0.0057678223f, 0.0073547363f, -0.0071487427f, 0.0087051392f, -0.0018835068f, -0.0099411011f,
-0.00050640106f, -0.0052947998f, -0.0054206848f, 0.0078430176f};
bool is_unidirectional = true;
bool use_past_state = true;
int past_sequence_length = 0;
@ -987,7 +908,6 @@ TEST(AttentionTest, Causal_EmptyPastState) {
use_past_state, past_sequence_length, &past_data, &present_data);
}
}
#endif
TEST(AttentionTest, AttentionEmptyPastState) {
RawAttentionEmptyPastState(false);

6586
onnxruntime/test/contrib_ops/attention_op_test_helper.cc
View file

File diff suppressed because it is too large Load diff

12
onnxruntime/test/contrib_ops/attention_op_test_helper.h
View file

@ -53,8 +53,6 @@ struct PackedAttentionTestData : public BaseAttentionTestData {
std::vector<int32_t> cumulative_sequence_length;
};
// Disable some tests in Windows since prefast build might crash with large test data.
#if !defined(_MSC_VER)
// Return packed weights and bias for input projection.
void GetAttentionWeight(std::vector<float>& weight_data, int elements = 64 * 3 * 64, int offset = 0, int step = 1);
void GetAttentionBias(std::vector<float>& bias_data, int elements = 3 * 64, int offset = 0, int step = 1);
@ -68,8 +66,6 @@ void GetCrossAttentionData_Batch1_HeadSize32_LeftSidePadding_NoBias(AttentionTes
void GetCrossAttentionData_Batch2_HeadSize32_NoBias_NoMask_PackedKV(AttentionTestData& data);
void GetSelfAttentionData_Batch2_HeadSize32_NoBias_NoMask_PackedQKV(AttentionTestData& data);
void GetPackedMultiHeadAttentionData_Batch2_HeadSize32_NoRelPosBias(PackedAttentionTestData& data);
#endif
void GetCrossAttentionData_HeadSize16_8(AttentionTestData& data);
void GetCrossAttentionData_HeadSize16_8_NoBias(AttentionTestData& data);
@ -87,10 +83,16 @@ void GetSelfAttentionData_WithPastAndPresent_HeadSize8_NoMask_NoRelPosBias(Atten
void GetSelfAttentionData_WithPastAndPresent_HeadSize8_NoMask_NoRelPosBias_NoBias(AttentionTestData& data);
void GetCrossAttentionData_WithPastPassedInDirectly_NoMask(AttentionTestData& data);
void GetCausal_EmptyPastState(std::vector<float>& input, std::vector<float>& output, std::vector<float>& present);
void GetAttentionDataCutlassRelPosBias(AttentionTestData& data);
void GetAttentionDataWithNeoXRotaryEmbedding(std::vector<float>& input, std::vector<float>& weights, std::vector<float>& bias,
void GetAttentionDataWithNeoXRotaryEmbedding(std::vector<float>& input,
std::vector<float>& weights,
std::vector<float>& bias,
std::vector<float>& output);
void GetPackedMultiHeadAttentionData_Batch2_HeadSize32_NoRelPosBias(PackedAttentionTestData& data);
void GetPackedMultiHeadAttentionData_Batch2_HeadSize8_RelPosBias(PackedAttentionTestData& data);
void GetPackedMultiHeadAttentionData_Batch2_HeadSize8_BroadcastRelPosBias(PackedAttentionTestData& data);

6
onnxruntime/test/contrib_ops/multihead_attention_op_test.cc
View file

@ -456,8 +456,6 @@ static void RunMultiHeadAttentionTests(AttentionTestData& data, bool disable_cpu
}
}
// Disable some tests in Windows since prefast build might crash with large test data.
#if !defined(_MSC_VER)
// Test fused cross attention kernel
// It requires head_size > 32 and head_size <= 64 for T4 GPU; hidden_size == v_hidden_size.
TEST(MultiHeadAttentionTest, CrossAttention_Batch2_HeadSize40) {
@ -509,7 +507,6 @@ TEST(MultiHeadAttentionTest, SelfAttention_Batch2_HeadSize32_NoBias_NoMask_Packe
GetSelfAttentionData_Batch2_HeadSize32_NoBias_NoMask_PackedQKV(data);
RunMultiHeadAttentionTests(data);
}
#endif
// This tests qk_head_size != v_head_size
TEST(MultiHeadAttentionTest, CrossAttention_Batch2_HeadSize16_8) {
@ -577,7 +574,8 @@ TEST(MultiHeadAttentionTest, SelfAttention_WithPastAndPresent_NoMask_NoRelPosBia
RunMultiHeadAttentionTests(data, /*disable_cpu=*/false, /*disable_cuda=*/true);
}
TEST(MultiHeadAttentionTest, CrossAttention_WithPastPassedInDirectly_NoMask) {
// This test is disabled since it is not used in Whisper anymore, and it fails in ROCm.
TEST(MultiHeadAttentionTest, DISABLED_CrossAttention_WithPastPassedInDirectly_NoMask) {
// Whisper decoder cross attention with past_kv in place of current KV and no present_kv
AttentionTestData data;
GetCrossAttentionData_WithPastPassedInDirectly_NoMask(data);

2
onnxruntime/test/contrib_ops/packed_multihead_attention_op_test.cc
View file

@ -199,7 +199,6 @@ static void RunPackedMultiHeadAttentionTest(
}
}
#if !defined(_MSC_VER)
TEST(PackedMultiHeadAttentionTest, PackedQKV_NoPadding_NoBias_trt) {
AttentionTestData data;
GetSelfAttentionData_Batch2_HeadSize32_NoBias_NoMask_PackedQKV(data);
@ -411,7 +410,6 @@ TEST(PackedMultiHeadAttentionTest, PackedQKV_Padding_NoBias_unfused) {
data.token_count,
AttentionKernelType::AttentionKernel_Unfused);
}
#endif
TEST(PackedMultiHeadAttentionTest, PackedQKV_Padding_NoBias_RelPosBias) {
PackedAttentionTestData data;

4996
onnxruntime/test/testdata/attention/attention_test_data.txt vendored Normal file
View file

File diff suppressed because it is too large Load diff

362
onnxruntime/test/testdata/attention/packed_multihead_attention_test_data.txt vendored Normal file
View file

@ -0,0 +1,362 @@
name:PackedMultiHeadAttentionData_Batch2_HeadSize32_NoRelPosBias.query_data
-0.35420692,1.31206024,-2.80201197,2.42258096,-0.86031514,-1.44535458,-0.10832444,-2.00132895,
1.62475216,0.10978927,1.84596729,0.48908550,1.44369888,0.87542874,-1.16434252,0.52133209,
1.54848897,-2.21174526,-0.28574878,0.70815033,1.18327498,3.14097571,-0.25795099,1.89341247,
-0.11603792,0.38110194,0.40873206,-1.14149106,0.79770875,-0.98069525,-1.53588808,0.50821728,
3.50846076,-2.50027657,-2.59866142,1.58495271,2.21110034,-2.74877763,-1.00267041,0.62646407,
2.50227380,-0.27291518,-0.33037442,0.75840306,0.45437157,-0.79876304,0.83509272,2.53716302,
0.01348384,-2.16307616,2.01661849,2.10746121,-1.70485222,1.35548759,1.39401650,-0.99451691,
-4.13484812,0.56262714,-0.92725742,-0.16389316,-1.31260049,2.32357836,-3.05251694,-1.12570131,
1.87849474,-1.80381167,0.52235699,2.38887334,-1.58878529,0.69571090,1.65044296,-0.27024290,
3.59580970,-1.97888982,1.17034674,0.26716161,-1.16770899,0.74609619,0.78886843,0.15717520,
-0.93303132,-0.84753871,-4.32799959,-1.94716609,-1.16980326,1.62631667,2.41053247,3.78186774,
0.26432252,-0.40396988,2.04414082,0.65150046,0.47777444,-2.57569051,0.99004912,2.47947693,
===
name:PackedMultiHeadAttentionData_Batch2_HeadSize32_NoRelPosBias.key_data
-0.04407793,1.29459429,1.05810797,1.92067695,-0.65047157,0.99029726,-1.69796586,1.15320420,
-1.66444266,1.78305888,1.20582056,1.69975281,0.34572244,-0.60833001,2.59864879,-1.05330181,
-1.16554165,-0.03781542,-1.13475525,0.71595150,-0.91169560,1.26686060,1.60492957,-0.53510487,
-1.40180850,1.83253956,2.70238972,-1.48750985,0.47105616,-0.79477602,-1.93152475,1.04042351,
-2.65206385,1.26134932,-1.01682174,0.64366758,0.95474619,2.06720352,0.51750720,-0.07041813,
0.53124994,-3.26612782,1.37013340,0.13939659,-0.57418114,0.80680281,-3.40751696,-0.15847699,
0.97837782,-0.09121911,1.18452120,0.52711177,-1.86135840,-0.11258313,0.85863215,-2.60261130,
0.72695309,1.44092011,0.43785980,-1.63415265,-1.05772328,0.12997569,0.07356137,-0.62493324,
-0.43267637,-1.80009198,0.92961007,2.05127883,-2.85521173,-0.21652693,-0.89153922,0.15524670,
-2.16850328,1.46751809,2.51663852,-0.49499366,0.19886012,0.77093124,-1.14819765,1.47111738,
2.42824388,1.56369960,1.69934130,-0.42460468,-2.25951004,-1.18074155,3.51091242,-0.30183151,
-1.83517075,-0.56233191,2.35561657,-3.63751698,-3.20001125,-1.66120780,3.23455381,-1.86251283,
===
name:PackedMultiHeadAttentionData_Batch2_HeadSize32_NoRelPosBias.value_data
-0.89167893,0.02633595,-0.84866279,1.43489110,-2.91941142,-0.20650116,1.85965109,0.45669034,
0.07678832,0.04492294,0.67326981,0.97103029,1.53470886,-1.10242307,0.86584085,-0.34770033,
-1.24311507,-1.80293822,-1.01317739,-0.71518499,0.77814674,-0.59236068,-2.00310278,3.13277125,
-1.20754123,2.01506066,0.82650810,2.06084490,-0.46267471,1.56365979,4.31514502,-1.03099275,
-2.11639142,-1.50897706,1.63863683,2.32786226,1.32746494,0.75751448,0.57184196,0.86446053,
-0.62406683,0.78861046,0.01044065,3.51772785,-1.33701336,0.27977663,-0.35464612,0.74973166,
0.03352100,1.55007398,0.69849420,-2.47725606,-1.89363778,-1.79874682,-0.56210291,-1.75556040,
1.07565808,-0.18023658,1.63777173,1.28198206,2.19431949,0.67998970,-0.52531999,-1.89906740,
1.35158050,-2.21481490,-0.11812399,-1.74263430,-0.57895988,-0.04181165,0.78120053,-2.22377038,
-0.53264999,-2.03721714,0.21023634,2.55751204,-1.04522800,0.85386503,0.41594937,-2.98181081,
1.14034331,-1.41539204,0.13379651,3.47018123,1.53924727,1.50004411,2.87318921,1.62624204,
0.64942807,-4.54302311,-1.50294220,-1.75212634,0.27900690,-3.05124855,3.30960631,-0.07991691,
===
name:PackedMultiHeadAttentionData_Batch2_HeadSize32_NoRelPosBias.qkv_data
-0.35420692,1.31206024,-2.80201197,2.42258096,-0.86031514,-1.44535458,-0.10832444,-2.00132895,
1.62475216,0.10978927,1.84596729,0.48908550,1.44369888,0.87542874,-1.16434252,0.52133209,
1.54848897,-2.21174526,-0.28574878,0.70815033,1.18327498,3.14097571,-0.25795099,1.89341247,
-0.11603792,0.38110194,0.40873206,-1.14149106,0.79770875,-0.98069525,-1.53588808,0.50821728,
-0.04407793,1.29459429,1.05810797,1.92067695,-0.65047157,0.99029726,-1.69796586,1.15320420,
-1.66444266,1.78305888,1.20582056,1.69975281,0.34572244,-0.60833001,2.59864879,-1.05330181,
-1.16554165,-0.03781542,-1.13475525,0.71595150,-0.91169560,1.26686060,1.60492957,-0.53510487,
-1.40180850,1.83253956,2.70238972,-1.48750985,0.47105616,-0.79477602,-1.93152475,1.04042351,
-0.89167893,0.02633595,-0.84866279,1.43489110,-2.91941142,-0.20650116,1.85965109,0.45669034,
0.07678832,0.04492294,0.67326981,0.97103029,1.53470886,-1.10242307,0.86584085,-0.34770033,
-1.24311507,-1.80293822,-1.01317739,-0.71518499,0.77814674,-0.59236068,-2.00310278,3.13277125,
-1.20754123,2.01506066,0.82650810,2.06084490,-0.46267471,1.56365979,4.31514502,-1.03099275,
3.50846076,-2.50027657,-2.59866142,1.58495271,2.21110034,-2.74877763,-1.00267041,0.62646407,
2.50227380,-0.27291518,-0.33037442,0.75840306,0.45437157,-0.79876304,0.83509272,2.53716302,
0.01348384,-2.16307616,2.01661849,2.10746121,-1.70485222,1.35548759,1.39401650,-0.99451691,
-4.13484812,0.56262714,-0.92725742,-0.16389316,-1.31260049,2.32357836,-3.05251694,-1.12570131,
-2.65206385,1.26134932,-1.01682174,0.64366758,0.95474619,2.06720352,0.51750720,-0.07041813,
0.53124994,-3.26612782,1.37013340,0.13939659,-0.57418114,0.80680281,-3.40751696,-0.15847699,
0.97837782,-0.09121911,1.18452120,0.52711177,-1.86135840,-0.11258313,0.85863215,-2.60261130,
0.72695309,1.44092011,0.43785980,-1.63415265,-1.05772328,0.12997569,0.07356137,-0.62493324,
-2.11639142,-1.50897706,1.63863683,2.32786226,1.32746494,0.75751448,0.57184196,0.86446053,
-0.62406683,0.78861046,0.01044065,3.51772785,-1.33701336,0.27977663,-0.35464612,0.74973166,
0.03352100,1.55007398,0.69849420,-2.47725606,-1.89363778,-1.79874682,-0.56210291,-1.75556040,
1.07565808,-0.18023658,1.63777173,1.28198206,2.19431949,0.67998970,-0.52531999,-1.89906740,
1.87849474,-1.80381167,0.52235699,2.38887334,-1.58878529,0.69571090,1.65044296,-0.27024290,
3.59580970,-1.97888982,1.17034674,0.26716161,-1.16770899,0.74609619,0.78886843,0.15717520,
-0.93303132,-0.84753871,-4.32799959,-1.94716609,-1.16980326,1.62631667,2.41053247,3.78186774,
0.26432252,-0.40396988,2.04414082,0.65150046,0.47777444,-2.57569051,0.99004912,2.47947693,
-0.43267637,-1.80009198,0.92961007,2.05127883,-2.85521173,-0.21652693,-0.89153922,0.15524670,
-2.16850328,1.46751809,2.51663852,-0.49499366,0.19886012,0.77093124,-1.14819765,1.47111738,
2.42824388,1.56369960,1.69934130,-0.42460468,-2.25951004,-1.18074155,3.51091242,-0.30183151,
-1.83517075,-0.56233191,2.35561657,-3.63751698,-3.20001125,-1.66120780,3.23455381,-1.86251283,
1.35158050,-2.21481490,-0.11812399,-1.74263430,-0.57895988,-0.04181165,0.78120053,-2.22377038,
-0.53264999,-2.03721714,0.21023634,2.55751204,-1.04522800,0.85386503,0.41594937,-2.98181081,
1.14034331,-1.41539204,0.13379651,3.47018123,1.53924727,1.50004411,2.87318921,1.62624204,
0.64942807,-4.54302311,-1.50294220,-1.75212634,0.27900690,-3.05124855,3.30960631,-0.07991691,
===
name:PackedMultiHeadAttentionData_Batch2_HeadSize32_NoRelPosBias.fp16_output_data
-0.89160156,0.02633667,-0.84863281,1.4345703,-2.9199219,-0.20654297,1.859375,0.45678711,
0.076782227,0.044921875,0.67333984,0.97119141,1.5351562,-1.1025391,0.86572266,-0.34765625,
-1.2431641,-1.8027344,-1.0126953,-0.71533203,0.77832031,-0.59228516,-2.0039062,3.1328125,
-1.2080078,2.015625,0.82666016,2.0605469,-0.46264648,1.5634766,4.3164062,-1.03125,
0.21158576,-1.98279130,0.45935997,-0.40457720,0.04772174,0.22094353,0.71238005,-1.20860445,
-0.56270063,-1.10830855,0.14455934,2.87315512,-1.14114404,0.66515017,0.16263856,-1.75517499,
0.77650774,-0.44058144,0.31942442,1.51513433,0.41078627,0.41566271,1.74393702,0.51457298,
0.78953874,-3.10888410,-0.47052401,-0.75475156,0.90861011,-1.82471263,2.04898596,-0.67790967,
1.17194295,-2.17825341,-0.02712549,-1.53178656,-0.48020893,-0.00040733,0.77035600,-2.06380320,
-0.53738528,-1.89084208,0.19988713,2.60725045,-1.06034219,0.82412785,0.37603328,-2.78852081,
1.08301103,-1.26178384,0.16304730,3.16210985,1.36142719,1.32916999,2.69524455,1.45106804,
0.67150640,-4.31703520,-1.34025633,-1.59496248,0.37821823,-2.85797405,3.11096096,-0.17414713f
===
name:PackedMultiHeadAttentionData_Batch2_HeadSize8_RelPosBias.query_data
-1.83615911,0.08698978,0.05601556,-1.14510250,-2.30377889,-0.39893439,0.73342341,-0.09851928,
-0.45148617,-0.16055907,-1.48271382,-0.07961921,-0.65701288,-0.25778309,-0.72851723,0.86755788,
-0.69794613,1.27221310,0.01081287,0.61428916,-0.00753688,0.33604777,-0.65822184,-0.83206612,
0.14741525,1.17016482,-0.68234873,-1.34005868,-0.22676668,-0.29179415,-0.15037467,0.15722550,
-0.20033565,-1.51847255,0.95205748,0.54009491,1.19315910,0.81655478,0.87503016,0.09732430,
-0.53218621,-0.11167067,0.67364228,-0.59705222,-0.24946509,0.20462716,-0.56092483,-0.65660709,
===
name:PackedMultiHeadAttentionData_Batch2_HeadSize8_RelPosBias.key_data
0.86949563,-0.10868365,-0.37917313,-1.23103046,0.25640076,-1.50652349,0.71594471,0.49057019,
-1.41292810,-0.19686662,1.25451696,-1.59823179,-1.16262913,0.84965342,0.61178929,-1.26162946,
0.56394553,-0.14227687,1.25733399,-0.11350060,0.02068172,-0.21710102,1.00936651,0.04366954,
0.72092402,-1.80008531,1.11335325,-0.58614230,0.15335107,-0.43153843,-0.81098813,1.15529966,
0.47295785,0.65468878,-1.44158995,-0.05122741,-0.34755200,0.66963655,0.72664660,1.59155345,
-1.13806772,0.70947856,-0.65793264,-0.50718778,-1.20698619,0.32613355,0.61786091,-0.34040576,
===
name:PackedMultiHeadAttentionData_Batch2_HeadSize8_RelPosBias.value_data
-1.19203627,0.38844836,0.68121153,0.21624038,-1.77549291,0.18574584,0.90408206,-0.22868094,
-0.95558548,1.38712502,0.81038797,0.14359820,0.15352470,0.00469783,0.03943123,0.53865469,
-1.60517013,-0.88120216,1.91317511,-0.54577649,0.62921578,0.57512373,-0.31156173,0.57847321,
0.04376111,-0.09540533,-0.38106504,1.00102639,-0.56325150,0.90788037,-0.33356044,0.74308902,
-0.15860432,-0.24945745,0.67483073,0.18782829,-0.56960964,1.16764832,-0.72244978,0.55027384,
-0.37327161,1.19222152,-0.23447749,0.06147140,0.32951999,1.06427121,2.26385999,0.23828916,
===
name:PackedMultiHeadAttentionData_Batch2_HeadSize8_RelPosBias.qkv_data
-1.83615911,0.08698978,0.05601556,-1.14510250,-2.30377889,-0.39893439,0.73342341,-0.09851928,
0.86949563,-0.10868365,-0.37917313,-1.23103046,0.25640076,-1.50652349,0.71594471,0.49057019,
-1.19203627,0.38844836,0.68121153,0.21624038,-1.77549291,0.18574584,0.90408206,-0.22868094,
-0.45148617,-0.16055907,-1.48271382,-0.07961921,-0.65701288,-0.25778309,-0.72851723,0.86755788,
-1.41292810,-0.19686662,1.25451696,-1.59823179,-1.16262913,0.84965342,0.61178929,-1.26162946,
-0.95558548,1.38712502,0.81038797,0.14359820,0.15352470,0.00469783,0.03943123,0.53865469,
-0.69794613,1.27221310,0.01081287,0.61428916,-0.00753688,0.33604777,-0.65822184,-0.83206612,
0.56394553,-0.14227687,1.25733399,-0.11350060,0.02068172,-0.21710102,1.00936651,0.04366954,
-1.60517013,-0.88120216,1.91317511,-0.54577649,0.62921578,0.57512373,-0.31156173,0.57847321,
0.14741525,1.17016482,-0.68234873,-1.34005868,-0.22676668,-0.29179415,-0.15037467,0.15722550,
0.72092402,-1.80008531,1.11335325,-0.58614230,0.15335107,-0.43153843,-0.81098813,1.15529966,
0.04376111,-0.09540533,-0.38106504,1.00102639,-0.56325150,0.90788037,-0.33356044,0.74308902,
-0.20033565,-1.51847255,0.95205748,0.54009491,1.19315910,0.81655478,0.87503016,0.09732430,
0.47295785,0.65468878,-1.44158995,-0.05122741,-0.34755200,0.66963655,0.72664660,1.59155345,
-0.15860432,-0.24945745,0.67483073,0.18782829,-0.56960964,1.16764832,-0.72244978,0.55027384,
-0.53218621,-0.11167067,0.67364228,-0.59705222,-0.24946509,0.20462716,-0.56092483,-0.65660709,
-1.13806772,0.70947856,-0.65793264,-0.50718778,-1.20698619,0.32613355,0.61786091,-0.34040576,
-0.37327161,1.19222152,-0.23447749,0.06147140,0.32951999,1.06427121,2.26385999,0.23828916,
===
name:PackedMultiHeadAttentionData_Batch2_HeadSize8_RelPosBias.rel_pos_bias_data
0.4781123,0.82420444,0.654424,0.3995186,
0.5482078,0.55570245,0.4216576,0.46001542,
0.4781123,0.82420444,0.654424,0.3995186,
0.5482078,0.55570245,0.4216576,0.46001542,
===
name:PackedMultiHeadAttentionData_Batch2_HeadSize8_RelPosBias.fp16_output_data
-1.1923828,0.38842773,0.68115234,0.21618652,-1.7753906,0.18579102,0.90429688,-0.2286377,
-0.95556641,1.3867188,0.81054688,0.14355469,0.15356445,0.004699707,0.039428711,0.53857422,
-0.72265625,-0.49584961,1.1572266,-0.098266602,-0.10211182,0.93652344,-0.56201172,0.56103516,
-0.27758789,0.89697266,-0.26831055,0.27734375,0.12451172,1.0283203,1.6679688,0.35424805,
-1.3427734,-0.76660156,1.6884766,-0.41259766,0.41162109,0.68261719,-0.38598633,0.57324219,
-0.17407227,0.57763672,-0.3046875,0.51025391,-0.097045898,0.98974609,1.0234375,0.47949219,
===
name:PackedMultiHeadAttentionData_Batch2_HeadSize8_BroadcastRelPosBias.query_data
-1.83615911,0.08698978,0.05601556,-1.14510250,-2.30377889,-0.39893439,0.73342341,-0.09851928,
-0.45148617,-0.16055907,-1.48271382,-0.07961921,-0.65701288,-0.25778309,-0.72851723,0.86755788,
-1.61145306,1.15819526,0.92267495,0.05697182,-1.15679109,0.30815905,-0.16966841,1.59468949,
1.06674862,-0.94525659,-1.26431036,0.54500824,2.33555818,-1.55515289,-1.15762365,0.94301772,
-0.11946517,-0.52010381,1.17592037,1.18040121,1.71579909,0.74054289,-0.28932455,0.43039253,
-0.59887785,-1.49919224,1.02610373,0.28144905,0.54606986,0.89283150,-1.55426455,0.81868863,
0.52380687,-0.98392582,-1.63426089,-0.48980984,0.39993629,-0.29305443,0.89878768,-0.56480503,
0.36021376,0.23860840,-0.31765643,-0.26952648,-1.24825561,1.21374047,0.23826340,0.63743669,
-0.24714193,1.01905107,-0.93621284,-0.84905517,-1.20235336,-2.04996300,-0.01251672,-1.23131406,
0.75130051,1.48657084,-0.17923722,-0.58394587,0.90255487,-0.60496974,-0.31978402,0.04127264,
-0.74666536,1.14945197,0.85595274,0.87929201,0.52238762,-0.38138887,-0.09198012,0.10379392,
0.03910975,0.37068886,0.38653150,-0.84676158,0.88043237,-0.70293593,-1.86805880,0.50196815,
1.79820418,0.49160564,-1.04994130,-0.65047348,0.03673789,-1.43390560,0.33562672,0.16967399,
-0.71324098,-0.24858820,-0.42040613,-0.47096899,-2.18410730,1.29030848,0.95546865,-0.01637810,
2.10404348,-0.25014257,2.06826448,0.68731880,-0.72305721,-3.06288505,1.35467649,-2.27782655,
-2.04115343,0.01709199,1.14326739,-2.85616302,-1.63115931,1.21209049,-2.88173485,0.29401016,
0.96896434,1.49846828,-1.48152626,-0.40123838,0.66036439,-2.08949637,0.93013203,1.78407586,
-0.16418101,0.30182290,0.76461935,0.89762378,-0.70261180,1.31333566,0.86440170,-0.55341989,
===
name:PackedMultiHeadAttentionData_Batch2_HeadSize8_BroadcastRelPosBias.key_data
0.86949563,-0.10868365,-0.37917313,-1.23103046,0.25640076,-1.50652349,0.71594471,0.49057019,
-1.41292810,-0.19686662,1.25451696,-1.59823179,-1.16262913,0.84965342,0.61178929,-1.26162946,
-0.39776036,-0.32152289,0.83949518,-0.68910766,2.10605407,-1.13187802,-0.02778089,-0.46233135,
-2.59193254,-0.02682346,0.07309747,-1.76444602,-1.02990091,-0.92442840,0.80414194,-1.34978604,
0.18744308,0.95530021,1.00306034,-0.06650767,0.14434582,-0.07039906,-0.42095628,1.47149456,
-0.32872301,0.28885615,-0.12510297,-0.35719556,0.36152089,-0.06192274,-0.28507957,0.75713438,
-0.59390724,-0.20499256,0.45794842,0.49178654,-0.25790799,-0.32643789,0.38682714,-0.26962680,
0.32060453,0.06873609,1.35627246,-0.55090475,0.75872916,-0.72157520,-0.75826746,-0.43329650,
-1.35081804,-0.40014201,0.26041570,0.23221464,0.90103126,-0.98436964,0.78283572,-1.78756905,
0.56045622,0.52480292,-0.67152452,-1.51145959,0.35886180,-0.41121563,0.61315238,-1.03636253,
0.32577509,-0.24800496,0.61258811,-0.14612751,0.82892537,-0.14666887,0.26352236,0.27397016,
-0.54198223,-0.38484800,-0.05891366,-0.79304564,-0.58239877,0.26545495,-0.38297540,0.79097033,
1.70193112,-0.31023103,-0.33280775,0.70939517,-1.27211368,-0.58704966,-1.81665146,0.03955793,
-0.40429014,0.48408982,1.63168097,0.71464306,0.93015838,-0.16759235,-2.20110703,0.78718776,
1.27983427,1.74217772,-1.50595713,1.52578330,0.56935966,-0.94778556,-1.28090227,-0.02918145,
0.99273205,2.60591888,1.39687920,-0.34032899,0.35641205,0.91440523,-0.69939649,2.64614010,
0.65847164,-1.24047744,-0.11745319,0.80385536,-0.55450106,-0.64750642,-0.84041762,0.10586573,
-1.74471772,0.38858974,0.77225429,-0.47355813,0.59074765,-0.50501788,-1.72981727,-1.25862873,
===
name:PackedMultiHeadAttentionData_Batch2_HeadSize8_BroadcastRelPosBias.value_data
-1.19203627,0.38844836,0.68121153,0.21624038,-1.77549291,0.18574584,0.90408206,-0.22868094,
-0.95558548,1.38712502,0.81038797,0.14359820,0.15352470,0.00469783,0.03943123,0.53865469,
0.50737989,1.88879848,-0.66337180,-1.29574990,-1.86634719,-1.01549423,-0.28819263,-0.00019580,
-2.00984406,-0.95374167,0.84587288,-1.78314638,-0.26245379,-1.30501473,-1.76663613,0.81748939,
0.80457681,-0.33367968,-0.10504961,0.27371234,-0.07084391,-0.40500754,-0.04883647,-0.94577336,
1.02707481,-0.13490768,-0.18182218,-0.93900180,1.18116772,1.98105478,-0.36385471,1.03757906,
0.14796616,-0.83527970,0.16718683,1.49241018,0.43081248,-1.43449056,0.39671475,-0.70072436,
0.17739578,0.32338822,-0.67136252,0.01797877,-0.59793222,0.78523242,0.25322497,1.50049233,
-0.67545807,0.50556731,0.18512395,-1.26351786,-1.42564654,-1.06456351,-1.15261269,-0.49967349,
-1.68690670,-1.10258508,0.07205302,1.72984779,-0.86800003,-0.91829145,-0.97764057,-0.55795985,
-0.85447711,0.16153991,1.38688803,-1.06463683,-1.42907071,0.43985835,-0.49964419,0.67612225,
-0.17095472,-0.31460449,0.78414583,0.25754923,0.49601099,1.01128983,-0.85820115,1.31935477,
1.41204929,-1.20291567,0.22243243,2.67244053,0.75709999,-0.44346970,0.94684786,-0.61597395,
-0.37711632,-0.30456182,-0.73898333,0.39148647,-0.04324996,-1.18526721,-1.38802266,0.82284755,
0.33568430,-3.36413050,1.40020049,1.05561769,-1.90628219,1.38226020,0.99804544,1.02540088,
-1.05158651,-3.61572313,-2.07312417,0.55376863,-0.07229304,0.02950740,-1.17457461,0.59849799,
0.56384206,0.21395147,1.26861954,2.02004290,-0.83739442,-0.11827546,-0.42885846,-1.55389440,
-1.04708695,1.04990900,0.61408597,0.48327276,0.61544299,-0.57864964,-0.80768973,0.39645281,
===
name:PackedMultiHeadAttentionData_Batch2_HeadSize8_BroadcastRelPosBias.qkv_data
-1.83615911,0.08698978,0.05601556,-1.14510250,-2.30377889,-0.39893439,0.73342341,-0.09851928,
0.86949563,-0.10868365,-0.37917313,-1.23103046,0.25640076,-1.50652349,0.71594471,0.49057019,
-1.19203627,0.38844836,0.68121153,0.21624038,-1.77549291,0.18574584,0.90408206,-0.22868094,
-0.45148617,-0.16055907,-1.48271382,-0.07961921,-0.65701288,-0.25778309,-0.72851723,0.86755788,
-1.41292810,-0.19686662,1.25451696,-1.59823179,-1.16262913,0.84965342,0.61178929,-1.26162946,
-0.95558548,1.38712502,0.81038797,0.14359820,0.15352470,0.00469783,0.03943123,0.53865469,
-1.61145306,1.15819526,0.92267495,0.05697182,-1.15679109,0.30815905,-0.16966841,1.59468949,
-0.39776036,-0.32152289,0.83949518,-0.68910766,2.10605407,-1.13187802,-0.02778089,-0.46233135,
0.50737989,1.88879848,-0.66337180,-1.29574990,-1.86634719,-1.01549423,-0.28819263,-0.00019580,
1.06674862,-0.94525659,-1.26431036,0.54500824,2.33555818,-1.55515289,-1.15762365,0.94301772,
-2.59193254,-0.02682346,0.07309747,-1.76444602,-1.02990091,-0.92442840,0.80414194,-1.34978604,
-2.00984406,-0.95374167,0.84587288,-1.78314638,-0.26245379,-1.30501473,-1.76663613,0.81748939,
-0.11946517,-0.52010381,1.17592037,1.18040121,1.71579909,0.74054289,-0.28932455,0.43039253,
0.18744308,0.95530021,1.00306034,-0.06650767,0.14434582,-0.07039906,-0.42095628,1.47149456,
0.80457681,-0.33367968,-0.10504961,0.27371234,-0.07084391,-0.40500754,-0.04883647,-0.94577336,
-0.59887785,-1.49919224,1.02610373,0.28144905,0.54606986,0.89283150,-1.55426455,0.81868863,
-0.32872301,0.28885615,-0.12510297,-0.35719556,0.36152089,-0.06192274,-0.28507957,0.75713438,
1.02707481,-0.13490768,-0.18182218,-0.93900180,1.18116772,1.98105478,-0.36385471,1.03757906,
0.52380687,-0.98392582,-1.63426089,-0.48980984,0.39993629,-0.29305443,0.89878768,-0.56480503,
-0.59390724,-0.20499256,0.45794842,0.49178654,-0.25790799,-0.32643789,0.38682714,-0.26962680,
0.14796616,-0.83527970,0.16718683,1.49241018,0.43081248,-1.43449056,0.39671475,-0.70072436,
0.36021376,0.23860840,-0.31765643,-0.26952648,-1.24825561,1.21374047,0.23826340,0.63743669,
0.32060453,0.06873609,1.35627246,-0.55090475,0.75872916,-0.72157520,-0.75826746,-0.43329650,
0.17739578,0.32338822,-0.67136252,0.01797877,-0.59793222,0.78523242,0.25322497,1.50049233,
-0.24714193,1.01905107,-0.93621284,-0.84905517,-1.20235336,-2.04996300,-0.01251672,-1.23131406,
-1.35081804,-0.40014201,0.26041570,0.23221464,0.90103126,-0.98436964,0.78283572,-1.78756905,
-0.67545807,0.50556731,0.18512395,-1.26351786,-1.42564654,-1.06456351,-1.15261269,-0.49967349,
0.75130051,1.48657084,-0.17923722,-0.58394587,0.90255487,-0.60496974,-0.31978402,0.04127264,
0.56045622,0.52480292,-0.67152452,-1.51145959,0.35886180,-0.41121563,0.61315238,-1.03636253,
-1.68690670,-1.10258508,0.07205302,1.72984779,-0.86800003,-0.91829145,-0.97764057,-0.55795985,
-0.74666536,1.14945197,0.85595274,0.87929201,0.52238762,-0.38138887,-0.09198012,0.10379392,
0.32577509,-0.24800496,0.61258811,-0.14612751,0.82892537,-0.14666887,0.26352236,0.27397016,
-0.85447711,0.16153991,1.38688803,-1.06463683,-1.42907071,0.43985835,-0.49964419,0.67612225,
0.03910975,0.37068886,0.38653150,-0.84676158,0.88043237,-0.70293593,-1.86805880,0.50196815,
-0.54198223,-0.38484800,-0.05891366,-0.79304564,-0.58239877,0.26545495,-0.38297540,0.79097033,
-0.17095472,-0.31460449,0.78414583,0.25754923,0.49601099,1.01128983,-0.85820115,1.31935477,
1.79820418,0.49160564,-1.04994130,-0.65047348,0.03673789,-1.43390560,0.33562672,0.16967399,
1.70193112,-0.31023103,-0.33280775,0.70939517,-1.27211368,-0.58704966,-1.81665146,0.03955793,
1.41204929,-1.20291567,0.22243243,2.67244053,0.75709999,-0.44346970,0.94684786,-0.61597395,
-0.71324098,-0.24858820,-0.42040613,-0.47096899,-2.18410730,1.29030848,0.95546865,-0.01637810,
-0.40429014,0.48408982,1.63168097,0.71464306,0.93015838,-0.16759235,-2.20110703,0.78718776,
-0.37711632,-0.30456182,-0.73898333,0.39148647,-0.04324996,-1.18526721,-1.38802266,0.82284755,
2.10404348,-0.25014257,2.06826448,0.68731880,-0.72305721,-3.06288505,1.35467649,-2.27782655,
1.27983427,1.74217772,-1.50595713,1.52578330,0.56935966,-0.94778556,-1.28090227,-0.02918145,
0.33568430,-3.36413050,1.40020049,1.05561769,-1.90628219,1.38226020,0.99804544,1.02540088,
-2.04115343,0.01709199,1.14326739,-2.85616302,-1.63115931,1.21209049,-2.88173485,0.29401016,
0.99273205,2.60591888,1.39687920,-0.34032899,0.35641205,0.91440523,-0.69939649,2.64614010,
-1.05158651,-3.61572313,-2.07312417,0.55376863,-0.07229304,0.02950740,-1.17457461,0.59849799,
0.96896434,1.49846828,-1.48152626,-0.40123838,0.66036439,-2.08949637,0.93013203,1.78407586,
0.65847164,-1.24047744,-0.11745319,0.80385536,-0.55450106,-0.64750642,-0.84041762,0.10586573,
0.56384206,0.21395147,1.26861954,2.02004290,-0.83739442,-0.11827546,-0.42885846,-1.55389440,
-0.16418101,0.30182290,0.76461935,0.89762378,-0.70261180,1.31333566,0.86440170,-0.55341989,
-1.74471772,0.38858974,0.77225429,-0.47355813,0.59074765,-0.50501788,-1.72981727,-1.25862873,
-1.04708695,1.04990900,0.61408597,0.48327276,0.61544299,-0.57864964,-0.80768973,0.39645281,
===
name:PackedMultiHeadAttentionData_Batch2_HeadSize8_BroadcastRelPosBias.rel_pos_bias_data
0.09734076,-0.01747033,0.008497253,-0.03361112,-0.028750911,-0.017142132,-0.11563814,0.10432467,
0.057628587,0.030893803,-0.096876964,0.11924802,-0.009177148,0.05799888,-0.030559167,0.034150958,
0.07427484,0.028848544,-0.031371966,0.07186346,-0.093020484,-0.066411436,0.06858949,0.07350862,
0.008785307,-0.07727124,0.09184219,0.081774,0.008513406,-0.04256621,-0.078802094,-0.09632437,
-0.0073524266,0.035430342,-0.01940021,0.05564849,-0.03556633,-0.11830491,-0.05839123,-0.05537042,
0.06146671,0.0026960075,-0.07718696,-0.06268193,0.039188996,0.07048777,-0.0009800941,-0.04509197,
0.04516627,-0.09005861,0.06470743,0.12054373,0.03366275,0.009849519,0.015899211,0.11534238,
-0.002090156,-0.0851126,-0.05993013,0.030745044,-0.02415888,0.06568785,-0.012787953,0.021764219,
-0.07153495,0.072637364,0.09073421,0.041491434,-0.11252555,-0.059031114,0.11179088,0.0655573,
-0.11293891,0.05239816,-0.110543296,-0.017368436,-0.050893307,0.04980643,-0.12137364,-0.08896659,
0.04159136,-0.021769747,0.018932432,0.087893024,-0.014453113,0.012075469,0.0026388168,-0.07094294,
0.035509557,0.065090835,0.115079954,-0.11314371,0.121840075,0.030929685,-0.051646978,-0.10540052,
-0.08004643,-0.089356855,-0.112021506,0.07381505,-0.06653316,-0.11657183,0.08264731,0.10898076,
-0.006200552,-0.0960899,-0.015346348,-0.047637567,0.051793993,-0.09059112,-0.034093216,0.108242184,
-0.049242377,-0.12165685,-0.09124553,0.05479528,-0.048966378,-0.027208641,0.10195236,0.047513425,
0.013226762,-0.07403794,0.06855075,-0.06551643,-0.084110215,0.11237715,0.07026932,-0.014076158,
===
name:PackedMultiHeadAttentionData_Batch2_HeadSize8_BroadcastRelPosBias.fp16_output_data
-1.1923828,0.38842773,0.68115234,0.21618652,-1.7753906,0.18579102,0.90429688,-0.2286377,
-0.95556641,1.3867188,0.81054688,0.14355469,0.15356445,0.004699707,0.039428711,0.53857422,
0.46826172,-0.38012695,0.23950195,0.50976562,-0.35839844,-0.4597168,0.0065727234,-0.62060547,
-0.28198242,-0.4128418,-0.44287109,0.22363281,0.062408447,0.058746338,-0.75292969,0.82275391,
0.18103027,0.17089844,0.31738281,-0.098022461,-1.0439453,-0.43823242,-0.19689941,-0.25537109,
-0.38745117,-0.32739258,-0.40332031,0.23425293,0.13305664,-0.3503418,-1.0107422,0.859375,
0.14135742,-0.1895752,0.50634766,0.25634766,-1.0400391,-0.37524414,-0.16821289,-0.30371094,
-0.68164062,-1.4511719,-0.52636719,0.19763184,0.082946777,0.1776123,-0.97021484,0.73828125,
0.23144531,-0.75341797,0.52148438,0.52148438,-0.92041016,-0.27001953,0.054473877,-0.19238281,
-0.68212891,-1.3339844,-0.75,0.37451172,-0.060058594,0.0034179688,-0.8671875,0.65429688,
0.20629883,-0.37646484,0.37646484,0.13183594,-0.94873047,-0.34375,-0.059875488,-0.22937012,
-0.58105469,-0.63671875,-0.50732422,0.30419922,0.086791992,-0.23132324,-0.91210938,0.74902344,
0.49804688,-1.3681641,0.75146484,1.0007825,-0.93847656,0.23278809,0.44604492,0.047668457,
-1.1943359,-0.75732422,0.48681641,-0.67480469,-0.0028400421,-0.37817383,-1.2705078,0.81201172,
0.13586426,0.0039863586,0.36181641,0.22192383,-0.89648438,-0.609375,-0.26049805,-0.43920898,
-1.046875,-0.19360352,0.30297852,-0.20007324,0.20483398,-0.50097656,-1.1142578,0.71923828,
0.37207031,-1.8447266,0.88818359,0.71191406,-1.3359375,0.56396484,0.52001953,0.36254883,
-0.73681641,-0.90722656,-0.29516602,0.021835327,0.030609131,-0.098083496,-0.96777344,0.75927734
===

17
onnxruntime/test/util/include/tensors_from_file.h Normal file
View file

@ -0,0 +1,17 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#pragma once
#include <string>
#include <vector>
#include <unordered_map>
namespace onnxruntime {
namespace test {
// Read a dictionary of name to float tensors mapping from a text file.
void LoadTensorsFromFile(const std::string& path, std::unordered_map<std::string, std::vector<float>>& tensors);
} // namespace test
} // namespace onnxruntime

107
onnxruntime/test/util/tensors_from_file.cc Normal file
View file

@ -0,0 +1,107 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include <algorithm>
#include <cctype>
#include <fstream>
#include <sstream>
#include "test/util/include/tensors_from_file.h"
#include "core/common/common.h"
namespace onnxruntime {
namespace test {
static inline void TrimLeft(std::string& s) {
s.erase(s.begin(), std::find_if(s.begin(), s.end(), [](unsigned char ch) {
return !std::isspace(ch);
}));
}
static inline void TrimRight(std::string& s) {
s.erase(std::find_if(s.rbegin(), s.rend(), [](unsigned char ch) {
return !std::isspace(ch);
}).base(),
s.end());
}
static inline void Trim(std::string& s) {
TrimRight(s);
TrimLeft(s);
}
static inline bool StartsWith(const std::string& s, const std::string& prefix) {
return s.compare(0, prefix.size(), prefix) == 0;
}
void LoadTensorsFromFile(const std::string& path, std::unordered_map<std::string, std::vector<float>>& tensors) {
std::ifstream infile(path);
if (!infile.good()) {
ORT_THROW("Cannot open file:", path);
}
// File format is like the following:
// name:TestCaseName1.TensorName1
// 1.1, 2.3
// ===
// name:TestCaseName2.TensorName2
// 3.3, 4.5,
// 5.6, 6.7
// ===
// Note that "name:" and "===" shall be at the beginning of a line without leading space!
const std::string name_prefix = "name:";
const std::string end_tensor_prefix = "===";
std::string line;
while (std::getline(infile, line)) {
if (line.empty()) {
continue;
}
std::string name;
if (StartsWith(line, name_prefix)) {
name = line.substr(name_prefix.length());
Trim(name);
}
if (name.empty()) {
ORT_THROW("Failed to find name in line:", line);
}
std::vector<float> values;
while (std::getline(infile, line)) {
if (line.empty()) {
continue;
}
if (StartsWith(line, end_tensor_prefix)) {
break;
}
std::istringstream ss(line);
for (std::string token; std::getline(ss, token, ',');) {
Trim(token);
if (token.empty()) {
continue;
}
ORT_TRY {
float value = std::stof(token);
values.push_back(value);
}
ORT_CATCH(const std::exception&) {
ORT_THROW("Failed to parse float from name='", name, ", token='", token);
}
}
}
if (values.size() == 0) {
ORT_THROW("No values for name=", name);
}
auto result = tensors.insert({name, values});
if (result.second == false) {
ORT_THROW("Failed to insert: duplicated name=", name);
}
}
}
} // namespace test
} // namespace onnxruntime