mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-05-23 22:13:38 +00:00
fd8ee4894d
### Description Implement JSEP GroupQueryAttention ### Motivation and Context Required to enable certain LLM models to run using WebGPU.
1314 lines
38 KiB
Text
1314 lines
38 KiB
Text
[
|
|
{
|
|
"name": "GroupQueryAttention 0",
|
|
"operator": "GroupQueryAttention",
|
|
"opset": { "domain": "com.microsoft", "version": 1 },
|
|
"attributes": [
|
|
{ "name": "num_heads", "data": 1, "type": "int" },
|
|
{ "name": "kv_num_heads", "data": 1, "type": "int" }
|
|
],
|
|
"cases": [
|
|
{
|
|
"name": "T[0]",
|
|
"inputs": [
|
|
{
|
|
"data": [0, 1, 2, 3, 4, 5, 6, 7],
|
|
"dims": [1, 1, 8],
|
|
"type": "float32"
|
|
},
|
|
// key, BS*
|
|
{
|
|
"data": [16, 17, 18, 19, 20, 21, 22, 23],
|
|
"dims": [1, 1, 8],
|
|
"type": "float32"
|
|
},
|
|
// value, BS*
|
|
{
|
|
"data": [32, 33, 34, 35, 36, 37, 38, 39],
|
|
"dims": [1, 1, 8],
|
|
"type": "float32"
|
|
},
|
|
// pask key, BNSH
|
|
{
|
|
"data": [],
|
|
"dims": [1, 1, 0, 8],
|
|
"type": "float32"
|
|
},
|
|
// pask value, BNSH
|
|
{
|
|
"data": [],
|
|
"dims": [1, 1, 0, 8],
|
|
"type": "float32"
|
|
},
|
|
// seqlens_k
|
|
{
|
|
"data": [1],
|
|
"dims": [1],
|
|
"type": "int32"
|
|
},
|
|
// total_sequence_length
|
|
{
|
|
"data": [1],
|
|
"dims": [1],
|
|
"type": "int32"
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"data": [32, 33, 34, 35, 36, 37, 38, 39],
|
|
"dims": [1, 1, 8],
|
|
"type": "float32"
|
|
},
|
|
{
|
|
// present key, BNSH
|
|
"data": [16, 17, 18, 19, 20, 21, 22, 23],
|
|
"dims": [1, 1, 1, 8],
|
|
"type": "float32"
|
|
},
|
|
{
|
|
// present value, BNSH
|
|
"data": [32, 33, 34, 35, 36, 37, 38, 39],
|
|
"dims": [1, 1, 1, 8],
|
|
"type": "float32"
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "GroupQueryAttention 1",
|
|
"operator": "GroupQueryAttention",
|
|
"opset": { "domain": "com.microsoft", "version": 1 },
|
|
"attributes": [
|
|
{ "name": "num_heads", "data": 1, "type": "int" },
|
|
{ "name": "kv_num_heads", "data": 1, "type": "int" }
|
|
],
|
|
"cases": [
|
|
{
|
|
"name": "T[0]",
|
|
"inputs": [
|
|
{
|
|
"data": [0, 1, 2, 3, 4, 5, 6, 7],
|
|
"dims": [1, 1, 8],
|
|
"type": "float32"
|
|
},
|
|
// key, BS*
|
|
{
|
|
"data": [16, 17, 18, 19, 20, 21, 22, 23],
|
|
"dims": [1, 1, 8],
|
|
"type": "float32"
|
|
},
|
|
// value, BS*
|
|
{
|
|
"data": [32, 33, 34, 35, 36, 37, 38, 39],
|
|
"dims": [1, 1, 8],
|
|
"type": "float32"
|
|
},
|
|
// past key, BS*
|
|
{
|
|
"data": [40, 41, 42, 43, 44, 45, 46, 47],
|
|
"dims": [1, 1, 1, 8],
|
|
"type": "float32"
|
|
},
|
|
// past value, BS*
|
|
{
|
|
"data": [48, 49, 50, 51, 52, 53, 54, 55],
|
|
"dims": [1, 1, 1, 8],
|
|
"type": "float32"
|
|
},
|
|
// seqlens_k, unimplemented
|
|
{
|
|
"data": [1],
|
|
"dims": [1],
|
|
"type": "int32"
|
|
},
|
|
// total_sequence_length, unimplemented
|
|
{
|
|
"data": [2],
|
|
"dims": [1],
|
|
"type": "int32"
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"data": [48, 49, 50, 51, 52, 53, 54, 55],
|
|
"dims": [1, 1, 8],
|
|
"type": "float32"
|
|
},
|
|
{
|
|
// present key, BNSH
|
|
"data": [40, 41, 42, 43, 44, 45, 46, 47, 16, 17, 18, 19, 20, 21, 22, 23],
|
|
"dims": [1, 1, 2, 8],
|
|
"type": "float32"
|
|
},
|
|
{
|
|
// present value, BNSH
|
|
"data": [48, 49, 50, 51, 52, 53, 54, 55, 32, 33, 34, 35, 36, 37, 38, 39],
|
|
"dims": [1, 1, 2, 8],
|
|
"type": "float32"
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "GroupQueryAttention 2",
|
|
"operator": "GroupQueryAttention",
|
|
"opset": { "domain": "com.microsoft", "version": 1 },
|
|
"attributes": [
|
|
{ "name": "num_heads", "data": 2, "type": "int" },
|
|
{ "name": "kv_num_heads", "data": 1, "type": "int" }
|
|
],
|
|
"cases": [
|
|
{
|
|
"name": "T[0]",
|
|
"inputs": [
|
|
{
|
|
"data": [
|
|
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28,
|
|
29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47
|
|
],
|
|
"dims": [1, 3, 16],
|
|
"type": "float32"
|
|
},
|
|
// key, BS*
|
|
{
|
|
"data": [48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71],
|
|
"dims": [1, 3, 8],
|
|
"type": "float32"
|
|
},
|
|
// value, BS*
|
|
{
|
|
"data": [72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95],
|
|
"dims": [1, 3, 8],
|
|
"type": "float32"
|
|
},
|
|
// pask key, BNSH
|
|
{
|
|
"data": [],
|
|
"dims": [1, 1, 0, 8],
|
|
"type": "float32"
|
|
},
|
|
// pask value, BNSH
|
|
{
|
|
"data": [],
|
|
"dims": [1, 1, 0, 8],
|
|
"type": "float32"
|
|
},
|
|
// seqlens_k
|
|
{
|
|
"data": [3],
|
|
"dims": [1],
|
|
"type": "int32"
|
|
},
|
|
// total_sequence_length
|
|
{
|
|
"data": [3],
|
|
"dims": [1],
|
|
"type": "int32"
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"data": [
|
|
72, 73, 74, 75, 76, 77, 78, 79, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 80, 81,
|
|
82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 88, 89, 90, 91, 92, 93, 94, 95
|
|
],
|
|
"dims": [1, 3, 16],
|
|
"type": "float32"
|
|
},
|
|
{
|
|
// present key, BNSH
|
|
"data": [48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71],
|
|
"dims": [1, 1, 3, 8],
|
|
"type": "float32"
|
|
},
|
|
{
|
|
// present value, BNSH
|
|
"data": [72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95],
|
|
"dims": [1, 1, 3, 8],
|
|
"type": "float32"
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "GroupQueryAttention 3",
|
|
"operator": "GroupQueryAttention",
|
|
"opset": { "domain": "com.microsoft", "version": 1 },
|
|
"attributes": [
|
|
{ "name": "num_heads", "data": 1, "type": "int" },
|
|
{ "name": "kv_num_heads", "data": 1, "type": "int" }
|
|
],
|
|
"cases": [
|
|
{
|
|
"name": "T[0]",
|
|
"inputs": [
|
|
{
|
|
"data": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23],
|
|
"dims": [1, 3, 8],
|
|
"type": "float32"
|
|
},
|
|
// key, BS*
|
|
{
|
|
"data": [24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47],
|
|
"dims": [1, 3, 8],
|
|
"type": "float32"
|
|
},
|
|
// value, BS*
|
|
{
|
|
"data": [48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71],
|
|
"dims": [1, 3, 8],
|
|
"type": "float32"
|
|
},
|
|
// pask key, BNSH
|
|
{
|
|
"data": [],
|
|
"dims": [1, 1, 0, 8],
|
|
"type": "float32"
|
|
},
|
|
// pask value, BNSH
|
|
{
|
|
"data": [],
|
|
"dims": [1, 1, 0, 8],
|
|
"type": "float32"
|
|
},
|
|
// seqlens_k
|
|
{
|
|
"data": [3],
|
|
"dims": [1],
|
|
"type": "int32"
|
|
},
|
|
// total_sequence_length
|
|
{
|
|
"data": [3],
|
|
"dims": [1],
|
|
"type": "int32"
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"data": [48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71],
|
|
"dims": [1, 3, 8],
|
|
"type": "float32"
|
|
},
|
|
{
|
|
// present key, BNSH
|
|
"data": [24, 25, 26, 27, 28, 29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47],
|
|
"dims": [1, 1, 3, 8],
|
|
"type": "float32"
|
|
},
|
|
{
|
|
// present value, BNSH
|
|
"data": [48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71],
|
|
"dims": [1, 1, 3, 8],
|
|
"type": "float32"
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "GroupQueryAttention 4",
|
|
"operator": "GroupQueryAttention",
|
|
"opset": { "domain": "com.microsoft", "version": 1 },
|
|
"attributes": [
|
|
{ "name": "num_heads", "data": 4, "type": "int" },
|
|
{ "name": "kv_num_heads", "data": 2, "type": "int" }
|
|
],
|
|
"cases": [
|
|
{
|
|
"name": "T[0]",
|
|
"inputs": [
|
|
{
|
|
"data": [
|
|
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28,
|
|
29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54,
|
|
55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80,
|
|
81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95
|
|
],
|
|
"dims": [1, 3, 32],
|
|
"type": "float32"
|
|
},
|
|
// key, BS*
|
|
{
|
|
"data": [
|
|
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28,
|
|
29, 30, 31, 32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47
|
|
],
|
|
"dims": [1, 3, 16],
|
|
"type": "float32"
|
|
},
|
|
// value, BS*
|
|
{
|
|
"data": [
|
|
48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73,
|
|
74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95
|
|
],
|
|
"dims": [1, 3, 16],
|
|
"type": "float32"
|
|
},
|
|
// past key, BNSH
|
|
{
|
|
"data": [],
|
|
"dims": [1, 2, 0, 8],
|
|
"type": "float32"
|
|
},
|
|
// past value, BNSH
|
|
{
|
|
"data": [],
|
|
"dims": [1, 2, 0, 8],
|
|
"type": "float32"
|
|
},
|
|
// seqlens_k
|
|
{
|
|
"data": [3],
|
|
"dims": [1],
|
|
"type": "int32"
|
|
},
|
|
// total_sequence_length
|
|
{
|
|
"data": [3],
|
|
"dims": [1],
|
|
"type": "int32"
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"data": [
|
|
48, 49, 50, 51, 52, 53, 54, 55, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 56, 57,
|
|
58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71, 64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75,
|
|
76, 77, 78, 79, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 80, 81, 82, 83, 84, 85,
|
|
86, 87, 88, 89, 90, 91, 92, 93, 94, 95, 88, 89, 90, 91, 92, 93, 94, 95
|
|
],
|
|
"dims": [1, 3, 32],
|
|
"type": "float32"
|
|
},
|
|
{
|
|
// present key, BNSH
|
|
"data": [
|
|
0, 1, 2, 3, 4, 5, 6, 7, 16, 17, 18, 19, 20, 21, 22, 23, 32, 33, 34, 35, 36, 37, 38, 39, 8, 9, 10, 11, 12,
|
|
13, 14, 15, 24, 25, 26, 27, 28, 29, 30, 31, 40, 41, 42, 43, 44, 45, 46, 47
|
|
],
|
|
"dims": [1, 2, 3, 8],
|
|
"type": "float32"
|
|
},
|
|
{
|
|
// present value, BNSH
|
|
"data": [
|
|
48, 49, 50, 51, 52, 53, 54, 55, 64, 65, 66, 67, 68, 69, 70, 71, 80, 81, 82, 83, 84, 85, 86, 87, 56, 57,
|
|
58, 59, 60, 61, 62, 63, 72, 73, 74, 75, 76, 77, 78, 79, 88, 89, 90, 91, 92, 93, 94, 95
|
|
],
|
|
"dims": [1, 2, 3, 8],
|
|
"type": "float32"
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "GroupQueryAttention 5",
|
|
"operator": "GroupQueryAttention",
|
|
"opset": { "domain": "com.microsoft", "version": 1 },
|
|
"attributes": [
|
|
{ "name": "num_heads", "data": 2, "type": "int" },
|
|
{ "name": "kv_num_heads", "data": 1, "type": "int" }
|
|
],
|
|
"cases": [
|
|
{
|
|
"name": "T[0]",
|
|
"inputs": [
|
|
{
|
|
"data": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
|
|
"dims": [1, 1, 16],
|
|
"type": "float32"
|
|
},
|
|
// key, BS*
|
|
{
|
|
"data": [16, 17, 18, 19, 20, 21, 22, 23],
|
|
"dims": [1, 1, 8],
|
|
"type": "float32"
|
|
},
|
|
// value, BS*
|
|
{
|
|
"data": [24, 25, 26, 27, 28, 29, 30, 31],
|
|
"dims": [1, 1, 8],
|
|
"type": "float32"
|
|
},
|
|
// pask key, BNSH
|
|
{
|
|
"data": [],
|
|
"dims": [1, 1, 0, 8],
|
|
"type": "float32"
|
|
},
|
|
// pask value, BNSH
|
|
{
|
|
"data": [],
|
|
"dims": [1, 1, 0, 8],
|
|
"type": "float32"
|
|
},
|
|
// seqlens_k
|
|
{
|
|
"data": [1],
|
|
"dims": [1],
|
|
"type": "int32"
|
|
},
|
|
// total_sequence_length
|
|
{
|
|
"data": [1],
|
|
"dims": [1],
|
|
"type": "int32"
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"data": [24, 25, 26, 27, 28, 29, 30, 31, 24, 25, 26, 27, 28, 29, 30, 31],
|
|
"dims": [1, 1, 16],
|
|
"type": "float32"
|
|
},
|
|
{
|
|
// present key, BNSH
|
|
"data": [16, 17, 18, 19, 20, 21, 22, 23],
|
|
"dims": [1, 1, 1, 8],
|
|
"type": "float32"
|
|
},
|
|
{
|
|
// present value, BNSH
|
|
"data": [24, 25, 26, 27, 28, 29, 30, 31],
|
|
"dims": [1, 1, 1, 8],
|
|
"type": "float32"
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "GroupQueryAttention 6",
|
|
"operator": "GroupQueryAttention",
|
|
"opset": { "domain": "com.microsoft", "version": 1 },
|
|
"attributes": [
|
|
{ "name": "num_heads", "data": 1, "type": "int" },
|
|
{ "name": "kv_num_heads", "data": 1, "type": "int" }
|
|
],
|
|
"cases": [
|
|
{
|
|
"name": "T[0]",
|
|
"inputs": [
|
|
{
|
|
"data": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23],
|
|
"dims": [1, 3, 8],
|
|
"type": "float32"
|
|
},
|
|
// key, BS*
|
|
{
|
|
"data": [48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71],
|
|
"dims": [1, 3, 8],
|
|
"type": "float32"
|
|
},
|
|
// value, BS*
|
|
{
|
|
"data": [72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95],
|
|
"dims": [1, 3, 8],
|
|
"type": "float32"
|
|
},
|
|
// pask key, BNSH
|
|
{
|
|
"data": [],
|
|
"dims": [1, 1, 0, 8],
|
|
"type": "float32"
|
|
},
|
|
// pask value, BNSH
|
|
{
|
|
"data": [],
|
|
"dims": [1, 1, 0, 8],
|
|
"type": "float32"
|
|
},
|
|
// seqlens_k
|
|
{
|
|
"data": [3],
|
|
"dims": [1],
|
|
"type": "int32"
|
|
},
|
|
// total_sequence_length
|
|
{
|
|
"data": [3],
|
|
"dims": [1],
|
|
"type": "int32"
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"data": [72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95],
|
|
"dims": [1, 3, 8],
|
|
"type": "float32"
|
|
},
|
|
{
|
|
// present key, BNSH
|
|
"data": [48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71],
|
|
"dims": [1, 1, 3, 8],
|
|
"type": "float32"
|
|
},
|
|
{
|
|
// present value, BNSH
|
|
"data": [72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95],
|
|
"dims": [1, 1, 3, 8],
|
|
"type": "float32"
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "GroupQueryAttention 7",
|
|
"operator": "GroupQueryAttention",
|
|
"opset": { "domain": "com.microsoft", "version": 1 },
|
|
"attributes": [
|
|
{ "name": "num_heads", "data": 1, "type": "int" },
|
|
{ "name": "kv_num_heads", "data": 1, "type": "int" }
|
|
],
|
|
"cases": [
|
|
{
|
|
"name": "T[0]",
|
|
"inputs": [
|
|
{
|
|
"data": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23],
|
|
"dims": [1, 3, 8],
|
|
"type": "float32"
|
|
},
|
|
// key, BS*
|
|
{
|
|
"data": [48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64, 65, 66, 67, 68, 69, 70, 71],
|
|
"dims": [1, 3, 8],
|
|
"type": "float32"
|
|
},
|
|
// value, BS*
|
|
{
|
|
"data": [72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89, 90, 91, 92, 93, 94, 95],
|
|
"dims": [1, 3, 8],
|
|
"type": "float32"
|
|
},
|
|
// past key, BS*
|
|
{
|
|
"data": [96, 97, 98, 99, 100, 101, 102, 103],
|
|
"dims": [1, 1, 1, 8],
|
|
"type": "float32"
|
|
},
|
|
// past value, BS*
|
|
{
|
|
"data": [104, 105, 106, 107, 108, 109, 110, 111],
|
|
"dims": [1, 1, 1, 8],
|
|
"type": "float32"
|
|
},
|
|
// seqlens_k, unimplemented
|
|
{
|
|
"data": [3],
|
|
"dims": [1],
|
|
"type": "int32"
|
|
},
|
|
// total_sequence_length, unimplemented
|
|
{
|
|
"data": [4],
|
|
"dims": [1],
|
|
"type": "int32"
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"data": [
|
|
104, 105, 106, 107, 108, 109, 110, 111, 104, 105, 106, 107, 108, 109, 110, 111, 104, 105, 106, 107, 108,
|
|
109, 110, 111
|
|
],
|
|
"dims": [1, 3, 8],
|
|
"type": "float32"
|
|
},
|
|
{
|
|
// present key, BNSH
|
|
"data": [
|
|
96, 97, 98, 99, 100, 101, 102, 103, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 64,
|
|
65, 66, 67, 68, 69, 70, 71
|
|
],
|
|
"dims": [1, 1, 4, 8],
|
|
"type": "float32"
|
|
},
|
|
{
|
|
// present value, BNSH
|
|
"data": [
|
|
104, 105, 106, 107, 108, 109, 110, 111, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87,
|
|
88, 89, 90, 91, 92, 93, 94, 95
|
|
],
|
|
"dims": [1, 1, 4, 8],
|
|
"type": "float32"
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": " GroupQueryAttention 8",
|
|
"operator": "GroupQueryAttention",
|
|
"opset": { "domain": "com.microsoft", "version": 1 },
|
|
"attributes": [
|
|
{ "name": "num_heads", "data": 4, "type": "int" },
|
|
{ "name": "kv_num_heads", "data": 2, "type": "int" }
|
|
],
|
|
"cases": [
|
|
{
|
|
"name": "T[0]",
|
|
"inputs": [
|
|
{
|
|
"data": [
|
|
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28,
|
|
29, 30, 31
|
|
],
|
|
"dims": [1, 1, 32],
|
|
"type": "float32"
|
|
},
|
|
// key, BS*
|
|
{
|
|
"data": [32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47],
|
|
"dims": [1, 1, 16],
|
|
"type": "float32"
|
|
},
|
|
// value, BS*
|
|
{
|
|
"data": [48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63],
|
|
"dims": [1, 1, 16],
|
|
"type": "float32"
|
|
},
|
|
// pask key, BNSH
|
|
{
|
|
"data": [],
|
|
"dims": [1, 2, 0, 8],
|
|
"type": "float32"
|
|
},
|
|
// pask value, BNSH
|
|
{
|
|
"data": [],
|
|
"dims": [1, 2, 0, 8],
|
|
"type": "float32"
|
|
},
|
|
// seqlens_k
|
|
{
|
|
"data": [1],
|
|
"dims": [1],
|
|
"type": "int32"
|
|
},
|
|
// total_sequence_length
|
|
{
|
|
"data": [1],
|
|
"dims": [1],
|
|
"type": "int32"
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"data": [
|
|
48, 49, 50, 51, 52, 53, 54, 55, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63, 56, 57,
|
|
58, 59, 60, 61, 62, 63
|
|
],
|
|
"dims": [1, 1, 32],
|
|
"type": "float32"
|
|
},
|
|
{
|
|
// present key, BNSH
|
|
"data": [32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47],
|
|
"dims": [1, 2, 1, 8],
|
|
"type": "float32"
|
|
},
|
|
{
|
|
// present value, BNSH
|
|
"data": [48, 49, 50, 51, 52, 53, 54, 55, 56, 57, 58, 59, 60, 61, 62, 63],
|
|
"dims": [1, 2, 1, 8],
|
|
"type": "float32"
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "GroupQueryAttention 9",
|
|
"operator": "GroupQueryAttention",
|
|
"opset": { "domain": "com.microsoft", "version": 1 },
|
|
"attributes": [
|
|
{ "name": "num_heads", "data": 2, "type": "int" },
|
|
{ "name": "kv_num_heads", "data": 2, "type": "int" }
|
|
],
|
|
"cases": [
|
|
{
|
|
"name": "T[0]",
|
|
"inputs": [
|
|
{
|
|
"data": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
|
|
"dims": [1, 1, 16],
|
|
"type": "float32"
|
|
},
|
|
// key, BS*
|
|
{
|
|
"data": [16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31],
|
|
"dims": [1, 1, 16],
|
|
"type": "float32"
|
|
},
|
|
// value, BS*
|
|
{
|
|
"data": [32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47],
|
|
"dims": [1, 1, 16],
|
|
"type": "float32"
|
|
},
|
|
// pask key, BNSH
|
|
{
|
|
"data": [],
|
|
"dims": [1, 2, 0, 8],
|
|
"type": "float32"
|
|
},
|
|
// pask value, BNSH
|
|
{
|
|
"data": [],
|
|
"dims": [1, 2, 0, 8],
|
|
"type": "float32"
|
|
},
|
|
// seqlens_k
|
|
{
|
|
"data": [1],
|
|
"dims": [1],
|
|
"type": "int32"
|
|
},
|
|
// total_sequence_length
|
|
{
|
|
"data": [1],
|
|
"dims": [1],
|
|
"type": "int32"
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"data": [32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47],
|
|
"dims": [1, 1, 16],
|
|
"type": "float32"
|
|
},
|
|
{
|
|
// present key, BNSH
|
|
"data": [16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31],
|
|
"dims": [1, 2, 1, 8],
|
|
"type": "float32"
|
|
},
|
|
{
|
|
// present value, BNSH
|
|
"data": [32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47],
|
|
"dims": [1, 2, 1, 8],
|
|
"type": "float32"
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "GroupQueryAttention 10",
|
|
"operator": "GroupQueryAttention",
|
|
"opset": { "domain": "com.microsoft", "version": 1 },
|
|
"attributes": [
|
|
{ "name": "num_heads", "data": 1, "type": "int" },
|
|
{ "name": "kv_num_heads", "data": 1, "type": "int" }
|
|
],
|
|
"cases": [
|
|
{
|
|
"name": "T[0]",
|
|
"inputs": [
|
|
{
|
|
"data": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
|
|
"dims": [1, 1, 16],
|
|
"type": "float32"
|
|
},
|
|
// key, BS*
|
|
{
|
|
"data": [16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31],
|
|
"dims": [1, 1, 16],
|
|
"type": "float32"
|
|
},
|
|
// value, BS*
|
|
{
|
|
"data": [32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47],
|
|
"dims": [1, 1, 16],
|
|
"type": "float32"
|
|
},
|
|
// pask key, BNSH
|
|
{
|
|
"data": [],
|
|
"dims": [1, 1, 0, 16],
|
|
"type": "float32"
|
|
},
|
|
// pask value, BNSH
|
|
{
|
|
"data": [],
|
|
"dims": [1, 1, 0, 16],
|
|
"type": "float32"
|
|
},
|
|
// seqlens_k
|
|
{
|
|
"data": [1],
|
|
"dims": [1],
|
|
"type": "int32"
|
|
},
|
|
// total_sequence_length
|
|
{
|
|
"data": [1],
|
|
"dims": [1],
|
|
"type": "int32"
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"data": [32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47],
|
|
"dims": [1, 1, 16],
|
|
"type": "float32"
|
|
},
|
|
{
|
|
// present key, BNSH
|
|
"data": [16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31],
|
|
"dims": [1, 1, 1, 16],
|
|
"type": "float32"
|
|
},
|
|
{
|
|
// present value, BNSH
|
|
"data": [32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47],
|
|
"dims": [1, 1, 1, 16],
|
|
"type": "float32"
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "GroupQueryAttention 11",
|
|
"operator": "GroupQueryAttention",
|
|
"opset": { "domain": "com.microsoft", "version": 1 },
|
|
"attributes": [
|
|
{ "name": "num_heads", "data": 1, "type": "int" },
|
|
{ "name": "kv_num_heads", "data": 1, "type": "int" }
|
|
],
|
|
"cases": [
|
|
{
|
|
"name": "T[0]",
|
|
"inputs": [
|
|
{
|
|
"data": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15],
|
|
"dims": [1, 2, 8],
|
|
"type": "float32"
|
|
},
|
|
// key, BS*
|
|
{
|
|
"data": [16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31],
|
|
"dims": [1, 2, 8],
|
|
"type": "float32"
|
|
},
|
|
// value, BS*
|
|
{
|
|
"data": [32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47],
|
|
"dims": [1, 2, 8],
|
|
"type": "float32"
|
|
},
|
|
// pask key, BNSH
|
|
{
|
|
"data": [],
|
|
"dims": [1, 1, 0, 8],
|
|
"type": "float32"
|
|
},
|
|
// pask value, BNSH
|
|
{
|
|
"data": [],
|
|
"dims": [1, 1, 0, 8],
|
|
"type": "float32"
|
|
},
|
|
// seqlens_k
|
|
{
|
|
"data": [2],
|
|
"dims": [1],
|
|
"type": "int32"
|
|
},
|
|
// total_sequence_length
|
|
{
|
|
"data": [2],
|
|
"dims": [1],
|
|
"type": "int32"
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"data": [32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47],
|
|
"dims": [1, 2, 8],
|
|
"type": "float32"
|
|
},
|
|
{
|
|
// present key, BNSH
|
|
"data": [16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28, 29, 30, 31],
|
|
"dims": [1, 1, 2, 8],
|
|
"type": "float32"
|
|
},
|
|
{
|
|
// present value, BNSH
|
|
"data": [32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47],
|
|
"dims": [1, 1, 2, 8],
|
|
"type": "float32"
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "GroupQueryAttention 12",
|
|
"operator": "GroupQueryAttention",
|
|
"opset": { "domain": "com.microsoft", "version": 1 },
|
|
"attributes": [
|
|
{ "name": "num_heads", "data": 1, "type": "int" },
|
|
{ "name": "kv_num_heads", "data": 1, "type": "int" }
|
|
],
|
|
"cases": [
|
|
{
|
|
"name": "T[0]",
|
|
"inputs": [
|
|
{
|
|
"data": [
|
|
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28,
|
|
29, 30, 31
|
|
],
|
|
"dims": [1, 1, 32],
|
|
"type": "float32"
|
|
},
|
|
// key, BS*
|
|
{
|
|
"data": [
|
|
32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57,
|
|
58, 59, 60, 61, 62, 63
|
|
],
|
|
"dims": [1, 1, 32],
|
|
"type": "float32"
|
|
},
|
|
// value, BS*
|
|
{
|
|
"data": [
|
|
64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89,
|
|
90, 91, 92, 93, 94, 95
|
|
],
|
|
"dims": [1, 1, 32],
|
|
"type": "float32"
|
|
},
|
|
// pask key, BNSH
|
|
{
|
|
"data": [],
|
|
"dims": [1, 1, 0, 32],
|
|
"type": "float32"
|
|
},
|
|
// pask value, BNSH
|
|
{
|
|
"data": [],
|
|
"dims": [1, 1, 0, 32],
|
|
"type": "float32"
|
|
},
|
|
// seqlens_k
|
|
{
|
|
"data": [1],
|
|
"dims": [1],
|
|
"type": "int32"
|
|
},
|
|
// total_sequence_length
|
|
{
|
|
"data": [1],
|
|
"dims": [1],
|
|
"type": "int32"
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"data": [
|
|
64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89,
|
|
90, 91, 92, 93, 94, 95
|
|
],
|
|
"dims": [1, 1, 32],
|
|
"type": "float32"
|
|
},
|
|
{
|
|
// present key, BNSH
|
|
"data": [
|
|
32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57,
|
|
58, 59, 60, 61, 62, 63
|
|
],
|
|
"dims": [1, 1, 1, 32],
|
|
"type": "float32"
|
|
},
|
|
{
|
|
// present value, BNSH
|
|
"data": [
|
|
64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89,
|
|
90, 91, 92, 93, 94, 95
|
|
],
|
|
"dims": [1, 1, 1, 32],
|
|
"type": "float32"
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "GroupQueryAttention 13",
|
|
"operator": "GroupQueryAttention",
|
|
"opset": { "domain": "com.microsoft", "version": 1 },
|
|
"attributes": [
|
|
{ "name": "num_heads", "data": 1, "type": "int" },
|
|
{ "name": "kv_num_heads", "data": 1, "type": "int" }
|
|
],
|
|
"cases": [
|
|
{
|
|
"name": "T[0]",
|
|
"inputs": [
|
|
{
|
|
"data": [
|
|
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28,
|
|
29, 30, 31
|
|
],
|
|
"dims": [1, 4, 8],
|
|
"type": "float32"
|
|
},
|
|
// key, BS*
|
|
{
|
|
"data": [
|
|
32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57,
|
|
58, 59, 60, 61, 62, 63
|
|
],
|
|
"dims": [1, 4, 8],
|
|
"type": "float32"
|
|
},
|
|
// value, BS*
|
|
{
|
|
"data": [
|
|
64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89,
|
|
90, 91, 92, 93, 94, 95
|
|
],
|
|
"dims": [1, 4, 8],
|
|
"type": "float32"
|
|
},
|
|
// pask key, BNSH
|
|
{
|
|
"data": [],
|
|
"dims": [1, 1, 0, 8],
|
|
"type": "float32"
|
|
},
|
|
// pask value, BNSH
|
|
{
|
|
"data": [],
|
|
"dims": [1, 1, 0, 8],
|
|
"type": "float32"
|
|
},
|
|
// seqlens_k
|
|
{
|
|
"data": [4],
|
|
"dims": [1],
|
|
"type": "int32"
|
|
},
|
|
// total_sequence_length
|
|
{
|
|
"data": [4],
|
|
"dims": [1],
|
|
"type": "int32"
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"data": [
|
|
64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89,
|
|
90, 91, 92, 93, 94, 95
|
|
],
|
|
"dims": [1, 4, 8],
|
|
"type": "float32"
|
|
},
|
|
{
|
|
// present key, BNSH
|
|
"data": [
|
|
32, 33, 34, 35, 36, 37, 38, 39, 40, 41, 42, 43, 44, 45, 46, 47, 48, 49, 50, 51, 52, 53, 54, 55, 56, 57,
|
|
58, 59, 60, 61, 62, 63
|
|
],
|
|
"dims": [1, 1, 4, 8],
|
|
"type": "float32"
|
|
},
|
|
{
|
|
// present value, BNSH
|
|
"data": [
|
|
64, 65, 66, 67, 68, 69, 70, 71, 72, 73, 74, 75, 76, 77, 78, 79, 80, 81, 82, 83, 84, 85, 86, 87, 88, 89,
|
|
90, 91, 92, 93, 94, 95
|
|
],
|
|
"dims": [1, 1, 4, 8],
|
|
"type": "float32"
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "GroupQueryAttention PackedQKV 14",
|
|
"operator": "GroupQueryAttention",
|
|
"opset": { "domain": "com.microsoft", "version": 1 },
|
|
"attributes": [
|
|
{ "name": "num_heads", "data": 2, "type": "int" },
|
|
{ "name": "kv_num_heads", "data": 1, "type": "int" }
|
|
],
|
|
"cases": [
|
|
{
|
|
"name": "T[0]",
|
|
"inputs": [
|
|
{
|
|
"data": [
|
|
0, 1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 12, 13, 14, 15, 16, 17, 18, 19, 20, 21, 22, 23, 24, 25, 26, 27, 28,
|
|
29, 30, 31
|
|
],
|
|
"dims": [1, 1, 32],
|
|
"type": "float32"
|
|
},
|
|
// key, BS*
|
|
{
|
|
"data": null,
|
|
"type": "float32"
|
|
},
|
|
// value, BS*
|
|
{
|
|
"data": null,
|
|
"type": "float32"
|
|
},
|
|
// pask key, BNSH
|
|
{
|
|
"data": [],
|
|
"dims": [1, 1, 0, 8],
|
|
"type": "float32"
|
|
},
|
|
// pask value, BNSH
|
|
{
|
|
"data": [],
|
|
"dims": [1, 1, 0, 8],
|
|
"type": "float32"
|
|
},
|
|
// seqlens_k
|
|
{
|
|
"data": [1],
|
|
"dims": [1],
|
|
"type": "int32"
|
|
},
|
|
// total_sequence_length
|
|
{
|
|
"data": [1],
|
|
"dims": [1],
|
|
"type": "int32"
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"data": [24, 25, 26, 27, 28, 29, 30, 31, 24, 25, 26, 27, 28, 29, 30, 31],
|
|
"dims": [1, 1, 16],
|
|
"type": "float32"
|
|
},
|
|
{
|
|
// present key, BNSH
|
|
"data": [16, 17, 18, 19, 20, 21, 22, 23],
|
|
"dims": [1, 1, 1, 8],
|
|
"type": "float32"
|
|
},
|
|
{
|
|
// present value, BNSH
|
|
"data": [24, 25, 26, 27, 28, 29, 30, 31],
|
|
"dims": [1, 1, 1, 8],
|
|
"type": "float32"
|
|
}
|
|
]
|
|
}
|
|
]
|
|
},
|
|
{
|
|
"name": "GroupQueryAttention PackedQKV 15",
|
|
"operator": "GroupQueryAttention",
|
|
"opset": { "domain": "com.microsoft", "version": 1 },
|
|
"attributes": [
|
|
{ "name": "num_heads", "data": 4, "type": "int" },
|
|
{ "name": "kv_num_heads", "data": 2, "type": "int" }
|
|
],
|
|
"cases": [
|
|
{
|
|
"name": "T[0]",
|
|
"inputs": [
|
|
{
|
|
"data": [
|
|
1, 1, 2, 3, 4, 5, 6, 7, 8, 11, 12, 13, 14, 15, 16, 17, 8, 12, 233, 4, 5, 6, 7, 8, 5, 6, 7, 8, 1, 1, 3, 4,
|
|
8, 12, 233, 4, 5, 6, 7, 8, 5, 6, 7, 8, 1, 1, 3, 4, 1, 9, 1, 1, 2, 2, 2, 2, 1, 12, 21, 131, 22, 21, 2, 2,
|
|
131, 22, 21, 2, 2, 131, 22, 21, 1, 1, 1, 1, 2, 2, 2, 2, 1, 1, 1, 1, 2, 2, 2, 2, 131, 22, 21, 2, 2, 131,
|
|
22, 21, 1, 1, 2, 3, 4, 5, 6, 7, 8, 11, 12, 13, 14, 15, 16, 17, 8, 12, 233, 4, 5, 6, 7, 8, 5, 6, 7, 8, 1,
|
|
1, 3, 4, 8, 12, 233, 4, 5, 6, 7, 8, 5, 6, 7, 8, 1, 1, 3, 4, 1, 9, 1, 1, 2, 2, 2, 2, 1, 12, 21, 131, 22,
|
|
21, 2, 2, 131, 22, 21, 2, 2, 131, 22, 21, 1, 1, 1, 1, 2, 2, 2, 2, 1, 1, 1, 1, 2, 2, 2, 2, 131, 22, 21, 2,
|
|
2, 131, 22, 21
|
|
],
|
|
"dims": [1, 3, 64],
|
|
"type": "float32"
|
|
},
|
|
// key
|
|
{
|
|
"data": null,
|
|
"type": "float32"
|
|
},
|
|
// value
|
|
{
|
|
"data": null,
|
|
"type": "float32"
|
|
},
|
|
// pask key, BNSH
|
|
{
|
|
"data": [],
|
|
"dims": [1, 2, 0, 8],
|
|
"type": "float32"
|
|
},
|
|
// pask value, BNSH
|
|
{
|
|
"data": [],
|
|
"dims": [1, 2, 0, 8],
|
|
"type": "float32"
|
|
},
|
|
// seqlens_k
|
|
{
|
|
"data": [3],
|
|
"dims": [1],
|
|
"type": "int32"
|
|
},
|
|
// total_sequence_length
|
|
{
|
|
"data": [3],
|
|
"dims": [1],
|
|
"type": "int32"
|
|
}
|
|
],
|
|
"outputs": [
|
|
{
|
|
"data": [
|
|
1, 9, 1, 1, 2, 2, 2, 2, 1, 9, 1, 1, 2, 2, 2, 2, 1, 12, 21, 131, 22, 21, 2, 2, 1, 12, 21, 131, 22, 21, 2,
|
|
2, 8, 12, 233, 4, 5, 6, 7, 8, 8, 12, 233, 4, 5, 6, 7, 8, 5, 6, 7, 8, 1, 1, 3, 4, 5, 6, 7, 8, 1, 1, 3, 4,
|
|
1, 1, 1, 1, 2, 2, 2, 2, 1, 1, 1, 1, 2, 2, 2, 2, 5, 6, 7, 8, 1, 1, 3, 4, 5, 6, 7, 8, 1, 1, 3, 4
|
|
],
|
|
"dims": [1, 3, 32],
|
|
"type": "float32"
|
|
},
|
|
{
|
|
// present key, BNSH
|
|
"data": [
|
|
8, 12, 233, 4, 5, 6, 7, 8, 1, 1, 2, 3, 4, 5, 6, 7, 131, 22, 21, 2, 2, 131, 22, 21, 5, 6, 7, 8, 1, 1, 3, 4,
|
|
8, 11, 12, 13, 14, 15, 16, 17, 1, 1, 1, 1, 2, 2, 2, 2
|
|
],
|
|
"dims": [1, 2, 3, 8],
|
|
"type": "float32"
|
|
},
|
|
{
|
|
// present value, BNSH
|
|
"data": [
|
|
1, 9, 1, 1, 2, 2, 2, 2, 8, 12, 233, 4, 5, 6, 7, 8, 1, 1, 1, 1, 2, 2, 2, 2, 1, 12, 21, 131, 22, 21, 2, 2,
|
|
5, 6, 7, 8, 1, 1, 3, 4, 131, 22, 21, 2, 2, 131, 22, 21
|
|
],
|
|
"dims": [1, 2, 3, 8],
|
|
"type": "float32"
|
|
}
|
|
]
|
|
}
|
|
]
|
|
}
|
|
]
|