mirror of
https://github.com/saymrwulf/onnxruntime.git
synced 2026-07-24 19:43:35 +00:00
Implement missing pieces for ARM QLinearConv support (#5894)
This commit is contained in:
Tracy Sharpe
GitHub
parent
d46dbeafd3
commit
57c92066c2
13 changed files with 699 additions and 369 deletions
|
|
@ -134,7 +134,7 @@ Status RegisterQuantizationKernels(KernelRegistry& kernel_registry) {
|
|||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, DynamicQuantizeMatMul)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, MatMulIntegerToFloat)>,
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, DynamicQuantizeLSTM)>,
|
||||
#if defined(MLAS_TARGET_AMD64_IX86)
|
||||
#ifdef MLAS_SUPPORTS_GEMM_U8X8
|
||||
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, QLinearConv)>,
|
||||
#endif
|
||||
};
|
||||
|
|
|
|||
|
|
@ -65,11 +65,6 @@ Abstract:
|
|||
#define MLAS_SUPPORTS_GEMM_U8X8
|
||||
#endif
|
||||
|
||||
#if defined(MLAS_TARGET_AMD64_IX86)
|
||||
#define MLAS_SUPPORTS_GEMM_U8X8_AND_REQUANTIZE_OUTPUT
|
||||
#define MLAS_SUPPORTS_TRANSPOSE
|
||||
#endif
|
||||
|
||||
#if defined(MLAS_TARGET_AMD64) || defined(MLAS_TARGET_ARM64) || (defined(MLAS_TARGET_ARM) && !defined(_MSC_VER))
|
||||
#define MLAS_SUPPORTS_PACKED_GEMM_U8X8
|
||||
#endif
|
||||
|
|
@ -668,36 +663,13 @@ MlasQuantizeLinear(
|
|||
void
|
||||
MLASCALL
|
||||
MlasRequantizeOutput(
|
||||
const int32_t* Input,
|
||||
uint8_t* Output,
|
||||
const int32_t* Bias,
|
||||
size_t M,
|
||||
size_t N,
|
||||
float Scale,
|
||||
uint8_t ZeroPoint
|
||||
);
|
||||
|
||||
void
|
||||
MLASCALL
|
||||
MlasRequantizeOutputColumn(
|
||||
const int32_t* Input,
|
||||
uint8_t* Output,
|
||||
const int32_t* Bias,
|
||||
size_t M,
|
||||
size_t N,
|
||||
const float Scale,
|
||||
uint8_t ZeroPoint
|
||||
);
|
||||
|
||||
void
|
||||
MLASCALL
|
||||
MlasRequantizeOutputColumn(
|
||||
const int32_t* Input,
|
||||
uint8_t* Output,
|
||||
const int32_t* Bias,
|
||||
size_t M,
|
||||
size_t N,
|
||||
const float* Scale,
|
||||
bool PerColumnScale,
|
||||
uint8_t ZeroPoint
|
||||
);
|
||||
|
||||
|
|
|
|||
|
|
@ -47,7 +47,7 @@ MLAS_INTERNAL_DATA const struct {
|
|||
88.7762626647950f,
|
||||
-88.3762626647949f,
|
||||
88.3762626647949f,
|
||||
12582912.f,
|
||||
MLAS_ROUNDING_BIAS_MAGIC,
|
||||
1.44269504088896341f,
|
||||
-6.93145752e-1f,
|
||||
-1.42860677e-6f,
|
||||
|
|
|
|||
|
|
@ -782,6 +782,47 @@ MlasPartitionWork(
|
|||
}
|
||||
}
|
||||
|
||||
//
|
||||
// Define the minimum floating point value (and its bit value equivalent) that
|
||||
// has no fractional bits. This number can be used for fast rounding of floating
|
||||
// point numbers to integers.
|
||||
//
|
||||
|
||||
#define MLAS_ROUNDING_BIAS_MAGIC 12582912.f
|
||||
#define MLAS_ROUNDING_BIAS_MAGIC_BITS 0x4B400000
|
||||
|
||||
//
|
||||
// Helpers to cast a floating point type to and from an integer bit format.
|
||||
//
|
||||
|
||||
MLAS_FORCEINLINE
|
||||
uint32_t
|
||||
MlasBitsOfFp32(
|
||||
float FloatValue
|
||||
)
|
||||
{
|
||||
union {
|
||||
uint32_t IntegerValue;
|
||||
float FloatValue;
|
||||
} u;
|
||||
u.FloatValue = FloatValue;
|
||||
return u.IntegerValue;
|
||||
}
|
||||
|
||||
MLAS_FORCEINLINE
|
||||
float
|
||||
MlasFp32FromBits(
|
||||
uint32_t IntegerValue
|
||||
)
|
||||
{
|
||||
union {
|
||||
uint32_t IntegerValue;
|
||||
float FloatValue;
|
||||
} u;
|
||||
u.IntegerValue = IntegerValue;
|
||||
return u.FloatValue;
|
||||
}
|
||||
|
||||
//
|
||||
// Define the missing ARM64 NEON intrinsic macros from arm64_neon.h that enable
|
||||
// cross-compiler support.
|
||||
|
|
|
|||
|
|
@ -19,35 +19,6 @@ Abstract:
|
|||
|
||||
#include "mlasi.h"
|
||||
|
||||
union MLAS_FLOAT32BITS
|
||||
{
|
||||
uint32_t u32;
|
||||
float fp32;
|
||||
};
|
||||
|
||||
MLAS_FORCEINLINE
|
||||
static
|
||||
uint32_t
|
||||
MlasBitsOfFp32(
|
||||
float f
|
||||
)
|
||||
{
|
||||
MLAS_FLOAT32BITS uf;
|
||||
uf.fp32 = f;
|
||||
return uf.u32;
|
||||
}
|
||||
|
||||
MLAS_FORCEINLINE
|
||||
static
|
||||
float
|
||||
MlasFp32FromBits(
|
||||
uint32_t u
|
||||
)
|
||||
{
|
||||
MLAS_FLOAT32BITS uf = { u };
|
||||
return uf.fp32;
|
||||
}
|
||||
|
||||
MLAS_FORCEINLINE
|
||||
static
|
||||
void
|
||||
|
|
|
|||
|
|
@ -289,45 +289,239 @@ MlasQuantizeLinear<uint8_t>(
|
|||
|
||||
#if defined(MLAS_SSE2_INTRINSICS)
|
||||
|
||||
MLAS_FORCEINLINE
|
||||
MLAS_INT32X4
|
||||
MlasRequantizeOutputVector(
|
||||
MLAS_INT32X4 IntegerVector,
|
||||
MLAS_INT32X4 BiasVector,
|
||||
MLAS_FLOAT32X4 ScaleVector,
|
||||
MLAS_FLOAT32X4 MinimumValueVector,
|
||||
MLAS_FLOAT32X4 MaximumValueVector,
|
||||
MLAS_INT32X4 ZeroPointVector
|
||||
void
|
||||
MLASCALL
|
||||
MlasRequantizeOutput(
|
||||
const int32_t* Input,
|
||||
uint8_t* Output,
|
||||
const int32_t* Bias,
|
||||
size_t M,
|
||||
size_t N,
|
||||
const float* Scale,
|
||||
bool PerColumnScale,
|
||||
uint8_t ZeroPoint
|
||||
)
|
||||
/*++
|
||||
|
||||
Routine Description:
|
||||
|
||||
This routine requantizes the intermediate buffer to the output buffer
|
||||
optionally adding the supplied bias.
|
||||
|
||||
Arguments:
|
||||
|
||||
Input - Supplies the input matrix.
|
||||
|
||||
Output - Supplies the output matrix.
|
||||
|
||||
Bias - Supplies the optional bias vector to be added to the input buffer
|
||||
before requantization.
|
||||
|
||||
Buffer - Supplies the output matrix.
|
||||
|
||||
M - Supplies the number of elements of the bias vector and the number of
|
||||
rows in the output matrix.
|
||||
|
||||
N - Supplies the number of columns of the output matrix.
|
||||
|
||||
Scale - Supplies the quantization scale.
|
||||
|
||||
PerColumnScale - Supplies true if the quantization scale has per-column
|
||||
values, else false if a single quantization scale applies to the
|
||||
entire matrix.
|
||||
|
||||
ZeroPoint - Supplies the quantization zero point value.
|
||||
|
||||
Return Value:
|
||||
|
||||
None.
|
||||
|
||||
--*/
|
||||
{
|
||||
IntegerVector = _mm_add_epi32(IntegerVector, BiasVector);
|
||||
MLAS_FLOAT32X4 FloatVector = _mm_cvtepi32_ps(IntegerVector);
|
||||
const __m128 PerMatrixScaleVector = PerColumnScale ? _mm_setzero_ps() : _mm_load1_ps(Scale);
|
||||
const __m128 MinimumValueVector = _mm_set1_ps(float(0 - ZeroPoint));
|
||||
const __m128 MaximumValueVector = _mm_set1_ps(float(255 - ZeroPoint));
|
||||
const __m128i ZeroPointVector = _mm_set1_epi32(ZeroPoint);
|
||||
|
||||
//
|
||||
// Scale the input vector and clamp the values to the minimum and maximum
|
||||
// range (adjusted by the zero point value).
|
||||
// Step through each row of the output matrix.
|
||||
//
|
||||
|
||||
FloatVector = MlasMultiplyFloat32x4(FloatVector, ScaleVector);
|
||||
while (M-- > 0) {
|
||||
|
||||
// N.B. MINPS and MAXPS returns the value from the second vector if the
|
||||
// value from the first vector is a NaN.
|
||||
FloatVector = _mm_max_ps(FloatVector, MinimumValueVector);
|
||||
FloatVector = _mm_min_ps(FloatVector, MaximumValueVector);
|
||||
const int32_t* bias = Bias;
|
||||
const float* scale = PerColumnScale ? Scale : nullptr;
|
||||
size_t n = N;
|
||||
|
||||
//
|
||||
// Convert the float values to integer using "round to nearest even" and
|
||||
// then shift the output range using the zero point value.
|
||||
//
|
||||
//
|
||||
// Process 16 columns of the matrices at a time.
|
||||
//
|
||||
|
||||
// N.B. Assumes MXCSR has been configured with the default rounding mode of
|
||||
// "round to nearest even".
|
||||
IntegerVector = _mm_cvtps_epi32(FloatVector);
|
||||
IntegerVector = _mm_add_epi32(IntegerVector, ZeroPointVector);
|
||||
while (n >= 16) {
|
||||
|
||||
return IntegerVector;
|
||||
//
|
||||
// Load the input data and optionally add the per-column bias.
|
||||
//
|
||||
|
||||
__m128i IntegerVector0 = _mm_loadu_si128((const __m128i *)&Input[0]);
|
||||
__m128i IntegerVector1 = _mm_loadu_si128((const __m128i *)&Input[4]);
|
||||
__m128i IntegerVector2 = _mm_loadu_si128((const __m128i *)&Input[8]);
|
||||
__m128i IntegerVector3 = _mm_loadu_si128((const __m128i *)&Input[12]);
|
||||
Input += 16;
|
||||
|
||||
if (bias != nullptr) {
|
||||
IntegerVector0 = _mm_add_epi32(IntegerVector0, _mm_loadu_si128((const __m128i *)&bias[0]));
|
||||
IntegerVector1 = _mm_add_epi32(IntegerVector1, _mm_loadu_si128((const __m128i *)&bias[4]));
|
||||
IntegerVector2 = _mm_add_epi32(IntegerVector2, _mm_loadu_si128((const __m128i *)&bias[8]));
|
||||
IntegerVector3 = _mm_add_epi32(IntegerVector3, _mm_loadu_si128((const __m128i *)&bias[12]));
|
||||
bias += 16;
|
||||
}
|
||||
|
||||
//
|
||||
// Convert to integer values to float and apply the per-tensor or
|
||||
// per-column scaling.
|
||||
//
|
||||
|
||||
__m128 FloatVector0 = _mm_cvtepi32_ps(IntegerVector0);
|
||||
__m128 FloatVector1 = _mm_cvtepi32_ps(IntegerVector1);
|
||||
__m128 FloatVector2 = _mm_cvtepi32_ps(IntegerVector2);
|
||||
__m128 FloatVector3 = _mm_cvtepi32_ps(IntegerVector3);
|
||||
|
||||
if (scale != nullptr) {
|
||||
|
||||
FloatVector0 = _mm_mul_ps(FloatVector0, _mm_loadu_ps(&scale[0]));
|
||||
FloatVector1 = _mm_mul_ps(FloatVector1, _mm_loadu_ps(&scale[4]));
|
||||
FloatVector2 = _mm_mul_ps(FloatVector2, _mm_loadu_ps(&scale[8]));
|
||||
FloatVector3 = _mm_mul_ps(FloatVector3, _mm_loadu_ps(&scale[12]));
|
||||
scale += 16;
|
||||
|
||||
} else {
|
||||
|
||||
FloatVector0 = _mm_mul_ps(FloatVector0, PerMatrixScaleVector);
|
||||
FloatVector1 = _mm_mul_ps(FloatVector1, PerMatrixScaleVector);
|
||||
FloatVector2 = _mm_mul_ps(FloatVector2, PerMatrixScaleVector);
|
||||
FloatVector3 = _mm_mul_ps(FloatVector3, PerMatrixScaleVector);
|
||||
}
|
||||
|
||||
FloatVector0 = _mm_max_ps(FloatVector0, MinimumValueVector);
|
||||
FloatVector1 = _mm_max_ps(FloatVector1, MinimumValueVector);
|
||||
FloatVector2 = _mm_max_ps(FloatVector2, MinimumValueVector);
|
||||
FloatVector3 = _mm_max_ps(FloatVector3, MinimumValueVector);
|
||||
|
||||
FloatVector0 = _mm_min_ps(FloatVector0, MaximumValueVector);
|
||||
FloatVector1 = _mm_min_ps(FloatVector1, MaximumValueVector);
|
||||
FloatVector2 = _mm_min_ps(FloatVector2, MaximumValueVector);
|
||||
FloatVector3 = _mm_min_ps(FloatVector3, MaximumValueVector);
|
||||
|
||||
IntegerVector0 = _mm_cvtps_epi32(FloatVector0);
|
||||
IntegerVector1 = _mm_cvtps_epi32(FloatVector1);
|
||||
IntegerVector2 = _mm_cvtps_epi32(FloatVector2);
|
||||
IntegerVector3 = _mm_cvtps_epi32(FloatVector3);
|
||||
|
||||
IntegerVector0 = _mm_add_epi32(IntegerVector0, ZeroPointVector);
|
||||
IntegerVector1 = _mm_add_epi32(IntegerVector1, ZeroPointVector);
|
||||
IntegerVector2 = _mm_add_epi32(IntegerVector2, ZeroPointVector);
|
||||
IntegerVector3 = _mm_add_epi32(IntegerVector3, ZeroPointVector);
|
||||
|
||||
__m128i WordVector0 = _mm_packus_epi16(IntegerVector0, IntegerVector1);
|
||||
__m128i WordVector1 = _mm_packus_epi16(IntegerVector2, IntegerVector3);
|
||||
|
||||
__m128i ByteVector = _mm_packus_epi16(WordVector0, WordVector1);
|
||||
|
||||
_mm_storeu_si128((__m128i*)Output, ByteVector);
|
||||
Output += 16;
|
||||
|
||||
n -= 16;
|
||||
}
|
||||
|
||||
//
|
||||
// Process the remaining columns of the matrices.
|
||||
//
|
||||
|
||||
while (n > 0) {
|
||||
|
||||
//
|
||||
// Load the input data and optionally add the per-column bias.
|
||||
//
|
||||
|
||||
__m128i IntegerVector;
|
||||
|
||||
if (n >= 4) {
|
||||
|
||||
IntegerVector = _mm_loadu_si128((const __m128i*)&Input[0]);
|
||||
Input += 4;
|
||||
|
||||
if (bias != nullptr) {
|
||||
IntegerVector = _mm_add_epi32(IntegerVector, _mm_loadu_si128((const __m128i*)&bias[0]));
|
||||
bias += 4;
|
||||
}
|
||||
|
||||
} else {
|
||||
|
||||
int32_t IntegerValue = *Input++;
|
||||
|
||||
if (bias != nullptr) {
|
||||
IntegerValue += *bias++;
|
||||
}
|
||||
|
||||
IntegerVector = _mm_cvtsi32_si128(IntegerValue);
|
||||
}
|
||||
|
||||
//
|
||||
// Convert to integer values to float and apply the per-tensor or
|
||||
// per-column scaling.
|
||||
//
|
||||
|
||||
__m128 FloatVector = _mm_cvtepi32_ps(IntegerVector);
|
||||
__m128 ScaleVector;
|
||||
|
||||
if (scale != nullptr) {
|
||||
|
||||
if (n >= 4) {
|
||||
ScaleVector = _mm_loadu_ps(scale);
|
||||
scale += 4;
|
||||
} else {
|
||||
ScaleVector = _mm_load_ss(scale);
|
||||
scale += 1;
|
||||
}
|
||||
|
||||
} else {
|
||||
ScaleVector = PerMatrixScaleVector;
|
||||
}
|
||||
|
||||
FloatVector = _mm_mul_ps(FloatVector, ScaleVector);
|
||||
|
||||
FloatVector = _mm_max_ps(FloatVector, MinimumValueVector);
|
||||
FloatVector = _mm_min_ps(FloatVector, MaximumValueVector);
|
||||
|
||||
IntegerVector = _mm_cvtps_epi32(FloatVector);
|
||||
IntegerVector = _mm_add_epi32(IntegerVector, ZeroPointVector);
|
||||
|
||||
IntegerVector = _mm_packus_epi16(IntegerVector, IntegerVector);
|
||||
IntegerVector = _mm_packus_epi16(IntegerVector, IntegerVector);
|
||||
|
||||
uint32_t OutputValue = uint32_t(_mm_cvtsi128_si32(IntegerVector));
|
||||
|
||||
if (n >= 4) {
|
||||
|
||||
*reinterpret_cast<uint32_t*>(Output) = OutputValue;
|
||||
Output += 4;
|
||||
|
||||
n -= 4;
|
||||
|
||||
} else {
|
||||
|
||||
*Output = uint8_t(OutputValue);
|
||||
Output += 1;
|
||||
|
||||
n -= 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#elif defined(MLAS_NEON64_INTRINSICS)
|
||||
|
||||
void
|
||||
MLASCALL
|
||||
MlasRequantizeOutput(
|
||||
|
|
@ -336,7 +530,8 @@ MlasRequantizeOutput(
|
|||
const int32_t* Bias,
|
||||
size_t M,
|
||||
size_t N,
|
||||
float Scale,
|
||||
const float* Scale,
|
||||
bool PerColumnScale,
|
||||
uint8_t ZeroPoint
|
||||
)
|
||||
/*++
|
||||
|
|
@ -364,98 +559,9 @@ Arguments:
|
|||
|
||||
Scale - Supplies the quantization scale.
|
||||
|
||||
ZeroPoint - Supplies the quantization zero point value.
|
||||
|
||||
Return Value:
|
||||
|
||||
None.
|
||||
|
||||
--*/
|
||||
{
|
||||
MLAS_FLOAT32X4 ScaleVector = MlasBroadcastFloat32x4(Scale);
|
||||
MLAS_FLOAT32X4 MinimumValueVector = MlasBroadcastFloat32x4(float(0 - ZeroPoint));
|
||||
MLAS_FLOAT32X4 MaximumValueVector = MlasBroadcastFloat32x4(float(255 - ZeroPoint));
|
||||
MLAS_INT32X4 ZeroPointVector = MlasBroadcastInt32x4(ZeroPoint);
|
||||
MLAS_INT32X4 BiasVector = _mm_setzero_si128();
|
||||
|
||||
//
|
||||
// Step through each row of the output matrix.
|
||||
//
|
||||
|
||||
while (M-- > 0) {
|
||||
|
||||
if (Bias != nullptr) {
|
||||
BiasVector = MlasBroadcastInt32x4(*Bias++);
|
||||
}
|
||||
|
||||
size_t n = N;
|
||||
|
||||
while (n >= 4) {
|
||||
|
||||
MLAS_INT32X4 IntegerVector = _mm_loadu_si128((const __m128i *)Input);
|
||||
IntegerVector = MlasRequantizeOutputVector(IntegerVector, BiasVector,
|
||||
ScaleVector, MinimumValueVector, MaximumValueVector, ZeroPointVector);
|
||||
|
||||
IntegerVector = _mm_packus_epi16(IntegerVector, IntegerVector);
|
||||
IntegerVector = _mm_packus_epi16(IntegerVector, IntegerVector);
|
||||
|
||||
*((int32_t*)Output) = _mm_cvtsi128_si32(IntegerVector);
|
||||
|
||||
Input += 4;
|
||||
Output += 4;
|
||||
n -= 4;
|
||||
}
|
||||
|
||||
while (n > 0) {
|
||||
|
||||
MLAS_INT32X4 IntegerVector = _mm_cvtsi32_si128(*Input);
|
||||
IntegerVector = MlasRequantizeOutputVector(IntegerVector, BiasVector,
|
||||
ScaleVector, MinimumValueVector, MaximumValueVector, ZeroPointVector);
|
||||
|
||||
*Output = (uint8_t)_mm_cvtsi128_si32(IntegerVector);
|
||||
|
||||
Input += 1;
|
||||
Output += 1;
|
||||
n -= 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
void
|
||||
MLASCALL
|
||||
MlasRequantizeOutputColumn(
|
||||
const int32_t* Input,
|
||||
uint8_t* Output,
|
||||
const int32_t* Bias,
|
||||
size_t M,
|
||||
size_t N,
|
||||
float Scale,
|
||||
uint8_t ZeroPoint
|
||||
)
|
||||
/*++
|
||||
|
||||
Routine Description:
|
||||
|
||||
This routine requantizes the intermediate buffer to the output buffer
|
||||
optionally adding the supplied bias.
|
||||
|
||||
Arguments:
|
||||
|
||||
Input - Supplies the input matrix.
|
||||
|
||||
Output - Supplies the output matrix.
|
||||
|
||||
Bias - Supplies the optional bias vector to be added to the input buffer
|
||||
before requantization.
|
||||
|
||||
Buffer - Supplies the output matrix.
|
||||
|
||||
M - Supplies the number of elements of the bias vector and the number of
|
||||
rows in the output matrix.
|
||||
|
||||
N - Supplies the number of columns of the output matrix.
|
||||
|
||||
Scale - Supplies the quantization scale.
|
||||
PerColumnScale - Supplies true if the quantization scale has per-column
|
||||
values, else false if a single quantization scale applies to the
|
||||
entire matrix.
|
||||
|
||||
ZeroPoint - Supplies the quantization zero point value.
|
||||
|
||||
|
|
@ -465,11 +571,8 @@ Return Value:
|
|||
|
||||
--*/
|
||||
{
|
||||
MLAS_FLOAT32X4 ScaleVector = MlasBroadcastFloat32x4(Scale);
|
||||
MLAS_FLOAT32X4 MinimumValueVector = MlasBroadcastFloat32x4(float(0 - ZeroPoint));
|
||||
MLAS_FLOAT32X4 MaximumValueVector = MlasBroadcastFloat32x4(float(255 - ZeroPoint));
|
||||
MLAS_INT32X4 ZeroPointVector = MlasBroadcastInt32x4(ZeroPoint);
|
||||
MLAS_INT32X4 BiasVector = _mm_setzero_si128();
|
||||
const float32x4_t PerMatrixScaleVector = PerColumnScale ? vdupq_n_f32(0) : vld1q_dup_f32(Scale);
|
||||
const int16x8_t ZeroPointVector = vdupq_n_s16(ZeroPoint);
|
||||
|
||||
//
|
||||
// Step through each row of the output matrix.
|
||||
|
|
@ -478,61 +581,203 @@ Return Value:
|
|||
while (M-- > 0) {
|
||||
|
||||
const int32_t* bias = Bias;
|
||||
|
||||
const float* scale = PerColumnScale ? Scale : nullptr;
|
||||
size_t n = N;
|
||||
|
||||
while (n >= 4) {
|
||||
//
|
||||
// Process 16 columns of the matrices at a time.
|
||||
//
|
||||
|
||||
MLAS_INT32X4 IntegerVector = _mm_loadu_si128((const __m128i *)Input);
|
||||
while (n >= 16) {
|
||||
|
||||
//
|
||||
// Load the input data and optionally add the per-column bias.
|
||||
//
|
||||
|
||||
int32x4x4_t IntegerVector;
|
||||
|
||||
IntegerVector.val[0] = vld1q_s32(&Input[0]);
|
||||
IntegerVector.val[1] = vld1q_s32(&Input[4]);
|
||||
IntegerVector.val[2] = vld1q_s32(&Input[8]);
|
||||
IntegerVector.val[3] = vld1q_s32(&Input[12]);
|
||||
Input += 16;
|
||||
|
||||
if (bias != nullptr) {
|
||||
BiasVector = _mm_loadu_si128((const __m128i*)bias);
|
||||
bias += 4;
|
||||
IntegerVector.val[0] = vaddq_s32(IntegerVector.val[0], vld1q_s32(&bias[0]));
|
||||
IntegerVector.val[1] = vaddq_s32(IntegerVector.val[1], vld1q_s32(&bias[4]));
|
||||
IntegerVector.val[2] = vaddq_s32(IntegerVector.val[2], vld1q_s32(&bias[8]));
|
||||
IntegerVector.val[3] = vaddq_s32(IntegerVector.val[3], vld1q_s32(&bias[12]));
|
||||
bias += 16;
|
||||
}
|
||||
|
||||
IntegerVector = MlasRequantizeOutputVector(IntegerVector, BiasVector,
|
||||
ScaleVector, MinimumValueVector, MaximumValueVector, ZeroPointVector);
|
||||
//
|
||||
// Convert to integer values to float and apply the per-tensor or
|
||||
// per-column scaling.
|
||||
//
|
||||
|
||||
IntegerVector = _mm_packus_epi16(IntegerVector, IntegerVector);
|
||||
IntegerVector = _mm_packus_epi16(IntegerVector, IntegerVector);
|
||||
float32x4x4_t FloatVector;
|
||||
|
||||
*((int32_t*)Output) = _mm_cvtsi128_si32(IntegerVector);
|
||||
FloatVector.val[0] = vcvtq_f32_s32(IntegerVector.val[0]);
|
||||
FloatVector.val[1] = vcvtq_f32_s32(IntegerVector.val[1]);
|
||||
FloatVector.val[2] = vcvtq_f32_s32(IntegerVector.val[2]);
|
||||
FloatVector.val[3] = vcvtq_f32_s32(IntegerVector.val[3]);
|
||||
|
||||
Input += 4;
|
||||
Output += 4;
|
||||
n -= 4;
|
||||
if (scale != nullptr) {
|
||||
|
||||
float32x4x4_t PerColumnScaleVector;
|
||||
|
||||
PerColumnScaleVector.val[0] = vld1q_f32(&scale[0]);
|
||||
PerColumnScaleVector.val[1] = vld1q_f32(&scale[4]);
|
||||
PerColumnScaleVector.val[2] = vld1q_f32(&scale[8]);
|
||||
PerColumnScaleVector.val[3] = vld1q_f32(&scale[12]);
|
||||
scale += 16;
|
||||
|
||||
FloatVector.val[0] = vmulq_f32(FloatVector.val[0], PerColumnScaleVector.val[0]);
|
||||
FloatVector.val[1] = vmulq_f32(FloatVector.val[1], PerColumnScaleVector.val[1]);
|
||||
FloatVector.val[2] = vmulq_f32(FloatVector.val[2], PerColumnScaleVector.val[2]);
|
||||
FloatVector.val[3] = vmulq_f32(FloatVector.val[3], PerColumnScaleVector.val[3]);
|
||||
|
||||
} else {
|
||||
|
||||
FloatVector.val[0] = vmulq_f32(FloatVector.val[0], PerMatrixScaleVector);
|
||||
FloatVector.val[1] = vmulq_f32(FloatVector.val[1], PerMatrixScaleVector);
|
||||
FloatVector.val[2] = vmulq_f32(FloatVector.val[2], PerMatrixScaleVector);
|
||||
FloatVector.val[3] = vmulq_f32(FloatVector.val[3], PerMatrixScaleVector);
|
||||
}
|
||||
|
||||
//
|
||||
// Convert the float values to integer using "round to nearest even".
|
||||
// Results are saturated to the range of int32_t.
|
||||
//
|
||||
|
||||
IntegerVector.val[0] = vcvtnq_s32_f32(FloatVector.val[0]);
|
||||
IntegerVector.val[1] = vcvtnq_s32_f32(FloatVector.val[1]);
|
||||
IntegerVector.val[2] = vcvtnq_s32_f32(FloatVector.val[2]);
|
||||
IntegerVector.val[3] = vcvtnq_s32_f32(FloatVector.val[3]);
|
||||
|
||||
//
|
||||
// Pack the integers with saturation to 16-bit values and shift by
|
||||
// the zero point, then pack the integers again to unsigned bytes.
|
||||
//
|
||||
|
||||
int16x8x2_t WordVector;
|
||||
|
||||
WordVector.val[0] = vqmovn_high_s32(vqmovn_s32(IntegerVector.val[0]), IntegerVector.val[1]);
|
||||
WordVector.val[1] = vqmovn_high_s32(vqmovn_s32(IntegerVector.val[2]), IntegerVector.val[3]);
|
||||
|
||||
WordVector.val[0] = vqaddq_s16(WordVector.val[0], ZeroPointVector);
|
||||
WordVector.val[1] = vqaddq_s16(WordVector.val[1], ZeroPointVector);
|
||||
|
||||
vst1q_u8(Output, vqmovun_high_s16(vqmovun_s16(WordVector.val[0]), WordVector.val[1]));
|
||||
Output += 16;
|
||||
|
||||
n -= 16;
|
||||
}
|
||||
|
||||
//
|
||||
// Process the remaining columns of the matrices.
|
||||
//
|
||||
|
||||
while (n > 0) {
|
||||
|
||||
MLAS_INT32X4 IntegerVector = _mm_cvtsi32_si128(*Input);
|
||||
//
|
||||
// Load the input data and optionally add the per-column bias.
|
||||
//
|
||||
|
||||
if (bias != nullptr) {
|
||||
BiasVector = _mm_cvtsi32_si128(*bias);
|
||||
bias += 1;
|
||||
int32x4_t IntegerVector;
|
||||
|
||||
if (n >= 4) {
|
||||
|
||||
IntegerVector = vld1q_s32(&Input[0]);
|
||||
Input += 4;
|
||||
|
||||
if (bias != nullptr) {
|
||||
IntegerVector = vaddq_s32(IntegerVector, vld1q_s32(&bias[0]));
|
||||
bias += 4;
|
||||
}
|
||||
|
||||
} else {
|
||||
|
||||
IntegerVector = vld1q_dup_s32(Input);
|
||||
Input += 1;
|
||||
|
||||
if (bias != nullptr) {
|
||||
IntegerVector = vaddq_s32(IntegerVector, vld1q_dup_s32(bias));
|
||||
bias += 1;
|
||||
}
|
||||
}
|
||||
|
||||
IntegerVector = MlasRequantizeOutputVector(IntegerVector, BiasVector,
|
||||
ScaleVector, MinimumValueVector, MaximumValueVector, ZeroPointVector);
|
||||
//
|
||||
// Convert to integer values to float and apply the per-tensor or
|
||||
// per-column scaling.
|
||||
//
|
||||
|
||||
*Output = (uint8_t)_mm_cvtsi128_si32(IntegerVector);
|
||||
float32x4_t FloatVector = vcvtq_f32_s32(IntegerVector);
|
||||
float32x4_t ScaleVector;
|
||||
|
||||
Input += 1;
|
||||
Output += 1;
|
||||
n -= 1;
|
||||
if (scale != nullptr) {
|
||||
|
||||
if (n >= 4) {
|
||||
ScaleVector = vld1q_f32(scale);
|
||||
scale += 4;
|
||||
} else {
|
||||
ScaleVector = vld1q_dup_f32(scale);
|
||||
scale += 1;
|
||||
}
|
||||
|
||||
} else {
|
||||
ScaleVector = PerMatrixScaleVector;
|
||||
}
|
||||
|
||||
FloatVector = vmulq_f32(FloatVector, ScaleVector);
|
||||
|
||||
//
|
||||
// Convert the float values to integer using "round to nearest even".
|
||||
// Results are saturated to the range of int32_t.
|
||||
//
|
||||
|
||||
IntegerVector = vcvtnq_s32_f32(FloatVector);
|
||||
|
||||
//
|
||||
// Pack the integers with saturation to 16-bit values and shift by
|
||||
// the zero point, then pack the integers again to unsigned bytes.
|
||||
//
|
||||
|
||||
int16x8_t WordVector = vcombine_s16(vqmovn_s32(IntegerVector), vdup_n_s16(0));
|
||||
WordVector = vqaddq_s16(WordVector, ZeroPointVector);
|
||||
|
||||
uint8x16_t ByteVector = vcombine_u8(vqmovun_s16(WordVector), vdup_n_u8(0));
|
||||
|
||||
if (n >= 4) {
|
||||
|
||||
vst1q_lane_u32(reinterpret_cast<uint32_t*>(Output), vreinterpretq_u32_u8(ByteVector), 0);
|
||||
Output += 4;
|
||||
|
||||
n -= 4;
|
||||
|
||||
} else {
|
||||
|
||||
vst1q_lane_u8(Output, ByteVector, 0);
|
||||
Output += 1;
|
||||
|
||||
n -= 1;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
#else
|
||||
|
||||
void
|
||||
MLASCALL
|
||||
MlasRequantizeOutputColumn(
|
||||
MlasRequantizeOutput(
|
||||
const int32_t* Input,
|
||||
uint8_t* Output,
|
||||
const int32_t* Bias,
|
||||
size_t M,
|
||||
size_t N,
|
||||
const float* Scale,
|
||||
bool PerColumnScale,
|
||||
uint8_t ZeroPoint
|
||||
)
|
||||
/*++
|
||||
|
|
@ -558,7 +803,11 @@ Arguments:
|
|||
|
||||
N - Supplies the number of columns of the output matrix.
|
||||
|
||||
Scale - Supplies the quantization scale vector.
|
||||
Scale - Supplies the quantization scale.
|
||||
|
||||
PerColumnScale - Supplies true if the quantization scale has per-column
|
||||
values, else false if a single quantization scale applies to the
|
||||
entire matrix.
|
||||
|
||||
ZeroPoint - Supplies the quantization zero point value.
|
||||
|
||||
|
|
@ -568,10 +817,9 @@ Return Value:
|
|||
|
||||
--*/
|
||||
{
|
||||
MLAS_FLOAT32X4 MinimumValueVector = MlasBroadcastFloat32x4(float(0 - ZeroPoint));
|
||||
MLAS_FLOAT32X4 MaximumValueVector = MlasBroadcastFloat32x4(float(255 - ZeroPoint));
|
||||
MLAS_INT32X4 ZeroPointVector = MlasBroadcastInt32x4(ZeroPoint);
|
||||
MLAS_INT32X4 BiasVector = _mm_setzero_si128();
|
||||
const float PerMatrixScaleValue = PerColumnScale ? 0.0f : *Scale;
|
||||
const float MinimumValue = float(0 - ZeroPoint);
|
||||
const float MaximumValue = float(255 - ZeroPoint);
|
||||
|
||||
//
|
||||
// Step through each row of the output matrix.
|
||||
|
|
@ -581,53 +829,36 @@ Return Value:
|
|||
|
||||
const int32_t* bias = Bias;
|
||||
const float* scale = Scale;
|
||||
|
||||
size_t n = N;
|
||||
|
||||
while (n >= 4) {
|
||||
|
||||
MLAS_INT32X4 IntegerVector = _mm_loadu_si128((const __m128i *)Input);
|
||||
|
||||
if (bias != nullptr) {
|
||||
BiasVector = _mm_loadu_si128((const __m128i*)bias);
|
||||
bias += 4;
|
||||
}
|
||||
|
||||
MLAS_FLOAT32X4 ScaleVector = MlasLoadFloat32x4(scale);
|
||||
scale += 4;
|
||||
|
||||
IntegerVector = MlasRequantizeOutputVector(IntegerVector, BiasVector,
|
||||
ScaleVector, MinimumValueVector, MaximumValueVector, ZeroPointVector);
|
||||
|
||||
IntegerVector = _mm_packus_epi16(IntegerVector, IntegerVector);
|
||||
IntegerVector = _mm_packus_epi16(IntegerVector, IntegerVector);
|
||||
|
||||
*((int32_t*)Output) = _mm_cvtsi128_si32(IntegerVector);
|
||||
|
||||
Input += 4;
|
||||
Output += 4;
|
||||
n -= 4;
|
||||
}
|
||||
|
||||
while (n > 0) {
|
||||
|
||||
MLAS_INT32X4 IntegerVector = _mm_cvtsi32_si128(*Input);
|
||||
int32_t IntegerValue = *Input++;
|
||||
|
||||
if (bias != nullptr) {
|
||||
BiasVector = _mm_cvtsi32_si128(*bias);
|
||||
bias += 1;
|
||||
IntegerValue += *bias++;
|
||||
}
|
||||
|
||||
MLAS_FLOAT32X4 ScaleVector = _mm_load_ss(scale);
|
||||
scale += 1;
|
||||
float FloatValue = float(IntegerValue);
|
||||
float ScaleValue = PerColumnScale ? *scale++ : PerMatrixScaleValue;
|
||||
|
||||
IntegerVector = MlasRequantizeOutputVector(IntegerVector, BiasVector,
|
||||
ScaleVector, MinimumValueVector, MaximumValueVector, ZeroPointVector);
|
||||
FloatValue *= ScaleValue;
|
||||
FloatValue = std::max(FloatValue, MinimumValue);
|
||||
FloatValue = std::min(FloatValue, MaximumValue);
|
||||
|
||||
*Output = (uint8_t)_mm_cvtsi128_si32(IntegerVector);
|
||||
//
|
||||
// Use the fast rounding trick adapted from XNNPACK: bias the floating
|
||||
// point value by the first floating point value that has no
|
||||
// fractional bits. The add operation performs the "round to nearest
|
||||
// even". Extract the mantissa bits from this floating point value to
|
||||
// obtain the rounded integer value.
|
||||
//
|
||||
|
||||
IntegerValue = int32_t(MlasBitsOfFp32(FloatValue + MLAS_ROUNDING_BIAS_MAGIC)) -
|
||||
MLAS_ROUNDING_BIAS_MAGIC_BITS;
|
||||
|
||||
*Output++ = uint8_t(IntegerValue + ZeroPoint);
|
||||
|
||||
Input += 1;
|
||||
Output += 1;
|
||||
n -= 1;
|
||||
}
|
||||
}
|
||||
|
|
|
|||
|
|
@ -16,7 +16,177 @@ Abstract:
|
|||
|
||||
#include "mlasi.h"
|
||||
|
||||
#ifdef MLAS_TARGET_AMD64_IX86
|
||||
#if defined(MLAS_SSE2_INTRINSICS) || defined(MLAS_NEON_INTRINSICS)
|
||||
|
||||
MLAS_FORCEINLINE
|
||||
void
|
||||
MlasTranspose8x8Block(
|
||||
const uint8_t* Input,
|
||||
size_t InputStride,
|
||||
uint8_t* Output,
|
||||
size_t OutputStride
|
||||
)
|
||||
/*++
|
||||
|
||||
Routine Description:
|
||||
|
||||
This routine transposes an eight by eight element block from the input
|
||||
matrix to the output matrix.
|
||||
|
||||
Arguments:
|
||||
|
||||
Input - Supplies the input buffer.
|
||||
|
||||
InputStride - Supplies the number of elements between rows of the input
|
||||
matrix.
|
||||
|
||||
Output - Supplies the output buffer.
|
||||
|
||||
OutputStride - Supplies the number of elements between rows of the output
|
||||
matrix.
|
||||
|
||||
Return Value:
|
||||
|
||||
None.
|
||||
|
||||
--*/
|
||||
{
|
||||
|
||||
#if defined(MLAS_SSE2_INTRINSICS)
|
||||
|
||||
__m128i a0 = _mm_loadl_epi64((const __m128i*)&Input[InputStride * 0]);
|
||||
__m128i a1 = _mm_loadl_epi64((const __m128i*)&Input[InputStride * 1]);
|
||||
__m128i b0 = _mm_unpacklo_epi8(a0, a1);
|
||||
|
||||
__m128i a2 = _mm_loadl_epi64((const __m128i*)&Input[InputStride * 2]);
|
||||
__m128i a3 = _mm_loadl_epi64((const __m128i*)&Input[InputStride * 3]);
|
||||
__m128i b1 = _mm_unpacklo_epi8(a2, a3);
|
||||
|
||||
__m128i a4 = _mm_loadl_epi64((const __m128i*)&Input[InputStride * 4]);
|
||||
__m128i a5 = _mm_loadl_epi64((const __m128i*)&Input[InputStride * 5]);
|
||||
__m128i b2 = _mm_unpacklo_epi8(a4, a5);
|
||||
|
||||
__m128i a6 = _mm_loadl_epi64((const __m128i*)&Input[InputStride * 6]);
|
||||
__m128i a7 = _mm_loadl_epi64((const __m128i*)&Input[InputStride * 7]);
|
||||
__m128i b3 = _mm_unpacklo_epi8(a6, a7);
|
||||
|
||||
__m128i c0 = _mm_unpacklo_epi16(b0, b1);
|
||||
__m128i c1 = _mm_unpackhi_epi16(b0, b1);
|
||||
__m128i c2 = _mm_unpacklo_epi16(b2, b3);
|
||||
__m128i c3 = _mm_unpackhi_epi16(b2, b3);
|
||||
|
||||
__m128 d0 = _mm_castsi128_ps(_mm_unpacklo_epi32(c0, c2));
|
||||
_mm_storel_pi((__m64*)&Output[OutputStride * 0], d0);
|
||||
_mm_storeh_pi((__m64*)&Output[OutputStride * 1], d0);
|
||||
|
||||
__m128 d1 = _mm_castsi128_ps(_mm_unpackhi_epi32(c0, c2));
|
||||
_mm_storel_pi((__m64*)&Output[OutputStride * 2], d1);
|
||||
_mm_storeh_pi((__m64*)&Output[OutputStride * 3], d1);
|
||||
|
||||
__m128 d2 = _mm_castsi128_ps(_mm_unpacklo_epi32(c1, c3));
|
||||
_mm_storel_pi((__m64*)&Output[OutputStride * 4], d2);
|
||||
_mm_storeh_pi((__m64*)&Output[OutputStride * 5], d2);
|
||||
|
||||
__m128 d3 = _mm_castsi128_ps(_mm_unpackhi_epi32(c1, c3));
|
||||
_mm_storel_pi((__m64*)&Output[OutputStride * 6], d3);
|
||||
_mm_storeh_pi((__m64*)&Output[OutputStride * 7], d3);
|
||||
|
||||
#elif defined(MLAS_NEON_INTRINSICS)
|
||||
|
||||
uint8x8_t a0 = vld1_u8(&Input[InputStride * 0]);
|
||||
uint8x8_t a1 = vld1_u8(&Input[InputStride * 1]);
|
||||
uint8x8x2_t b0 = vzip_u8(a0, a1);
|
||||
|
||||
uint8x8_t a2 = vld1_u8(&Input[InputStride * 2]);
|
||||
uint8x8_t a3 = vld1_u8(&Input[InputStride * 3]);
|
||||
uint8x8x2_t b1 = vzip_u8(a2, a3);
|
||||
|
||||
uint8x8_t a4 = vld1_u8(&Input[InputStride * 4]);
|
||||
uint8x8_t a5 = vld1_u8(&Input[InputStride * 5]);
|
||||
uint8x8x2_t b2 = vzip_u8(a4, a5);
|
||||
|
||||
uint8x8_t a6 = vld1_u8(&Input[InputStride * 6]);
|
||||
uint8x8_t a7 = vld1_u8(&Input[InputStride * 7]);
|
||||
uint8x8x2_t b3 = vzip_u8(a6, a7);
|
||||
|
||||
uint16x4x2_t c0 = vzip_u16(vreinterpret_u16_u8(b0.val[0]), vreinterpret_u16_u8(b1.val[0]));
|
||||
uint16x4x2_t c1 = vzip_u16(vreinterpret_u16_u8(b0.val[1]), vreinterpret_u16_u8(b1.val[1]));
|
||||
uint16x4x2_t c2 = vzip_u16(vreinterpret_u16_u8(b2.val[0]), vreinterpret_u16_u8(b3.val[0]));
|
||||
uint16x4x2_t c3 = vzip_u16(vreinterpret_u16_u8(b2.val[1]), vreinterpret_u16_u8(b3.val[1]));
|
||||
|
||||
uint32x2x2_t d0 = vzip_u32(vreinterpret_u32_u16(c0.val[0]), vreinterpret_u32_u16(c2.val[0]));
|
||||
uint32x2x2_t d1 = vzip_u32(vreinterpret_u32_u16(c0.val[1]), vreinterpret_u32_u16(c2.val[1]));
|
||||
uint32x2x2_t d2 = vzip_u32(vreinterpret_u32_u16(c1.val[0]), vreinterpret_u32_u16(c3.val[0]));
|
||||
uint32x2x2_t d3 = vzip_u32(vreinterpret_u32_u16(c1.val[1]), vreinterpret_u32_u16(c3.val[1]));
|
||||
|
||||
vst1_u8(&Output[OutputStride * 0], vreinterpret_u8_u32(d0.val[0]));
|
||||
vst1_u8(&Output[OutputStride * 1], vreinterpret_u8_u32(d0.val[1]));
|
||||
vst1_u8(&Output[OutputStride * 2], vreinterpret_u8_u32(d1.val[0]));
|
||||
vst1_u8(&Output[OutputStride * 3], vreinterpret_u8_u32(d1.val[1]));
|
||||
vst1_u8(&Output[OutputStride * 4], vreinterpret_u8_u32(d2.val[0]));
|
||||
vst1_u8(&Output[OutputStride * 5], vreinterpret_u8_u32(d2.val[1]));
|
||||
vst1_u8(&Output[OutputStride * 6], vreinterpret_u8_u32(d3.val[0]));
|
||||
vst1_u8(&Output[OutputStride * 7], vreinterpret_u8_u32(d3.val[1]));
|
||||
|
||||
#endif
|
||||
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
MLAS_FORCEINLINE
|
||||
void
|
||||
MlasTranspose8xNVector(
|
||||
const uint8_t* Input,
|
||||
size_t InputStride,
|
||||
uint8_t* Output,
|
||||
size_t OutputStride
|
||||
)
|
||||
/*++
|
||||
|
||||
Routine Description:
|
||||
|
||||
This routine transposes an eight element vector from the input matrix
|
||||
to the output matrix.
|
||||
|
||||
Arguments:
|
||||
|
||||
Input - Supplies the input buffer.
|
||||
|
||||
InputStride - Supplies the number of elements between rows of the input
|
||||
matrix.
|
||||
|
||||
Output - Supplies the output buffer.
|
||||
|
||||
OutputStride - Supplies the number of elements between rows of the output
|
||||
matrix.
|
||||
|
||||
Return Value:
|
||||
|
||||
None.
|
||||
|
||||
--*/
|
||||
{
|
||||
uint8_t a0 = Input[InputStride * 0];
|
||||
uint8_t a1 = Input[InputStride * 1];
|
||||
uint8_t a2 = Input[InputStride * 2];
|
||||
uint8_t a3 = Input[InputStride * 3];
|
||||
|
||||
Output[OutputStride * 0] = a0;
|
||||
Output[OutputStride * 1] = a1;
|
||||
Output[OutputStride * 2] = a2;
|
||||
Output[OutputStride * 3] = a3;
|
||||
|
||||
uint8_t a4 = Input[InputStride * 4];
|
||||
uint8_t a5 = Input[InputStride * 5];
|
||||
uint8_t a6 = Input[InputStride * 6];
|
||||
uint8_t a7 = Input[InputStride * 7];
|
||||
|
||||
Output[OutputStride * 4] = a4;
|
||||
Output[OutputStride * 5] = a5;
|
||||
Output[OutputStride * 6] = a6;
|
||||
Output[OutputStride * 7] = a7;
|
||||
}
|
||||
|
||||
void
|
||||
MLASCALL
|
||||
|
|
@ -64,60 +234,22 @@ Return Value:
|
|||
uint8_t* d = Output;
|
||||
size_t m = M;
|
||||
|
||||
#if defined(MLAS_SSE2_INTRINSICS) || defined(MLAS_NEON_INTRINSICS)
|
||||
|
||||
while (m >= 8) {
|
||||
|
||||
__m128i a0 = _mm_loadl_epi64((const __m128i*)&s[N * 0]);
|
||||
__m128i a1 = _mm_loadl_epi64((const __m128i*)&s[N * 1]);
|
||||
__m128i b0 = _mm_unpacklo_epi8(a0, a1);
|
||||
|
||||
__m128i a2 = _mm_loadl_epi64((const __m128i*)&s[N * 2]);
|
||||
__m128i a3 = _mm_loadl_epi64((const __m128i*)&s[N * 3]);
|
||||
__m128i b1 = _mm_unpacklo_epi8(a2, a3);
|
||||
|
||||
__m128i a4 = _mm_loadl_epi64((const __m128i*)&s[N * 4]);
|
||||
__m128i a5 = _mm_loadl_epi64((const __m128i*)&s[N * 5]);
|
||||
__m128i b2 = _mm_unpacklo_epi8(a4, a5);
|
||||
|
||||
__m128i a6 = _mm_loadl_epi64((const __m128i*)&s[N * 6]);
|
||||
__m128i a7 = _mm_loadl_epi64((const __m128i*)&s[N * 7]);
|
||||
__m128i b3 = _mm_unpacklo_epi8(a6, a7);
|
||||
|
||||
__m128i c0 = _mm_unpacklo_epi16(b0, b1);
|
||||
__m128i c1 = _mm_unpackhi_epi16(b0, b1);
|
||||
__m128i c2 = _mm_unpacklo_epi16(b2, b3);
|
||||
__m128i c3 = _mm_unpackhi_epi16(b2, b3);
|
||||
|
||||
__m128 d0 = _mm_castsi128_ps(_mm_unpacklo_epi32(c0, c2));
|
||||
_mm_storel_pi((__m64*)&d[M * 0], d0);
|
||||
_mm_storeh_pi((__m64*)&d[M * 1], d0);
|
||||
|
||||
__m128 d1 = _mm_castsi128_ps(_mm_unpackhi_epi32(c0, c2));
|
||||
_mm_storel_pi((__m64*)&d[M * 2], d1);
|
||||
_mm_storeh_pi((__m64*)&d[M * 3], d1);
|
||||
|
||||
__m128 d2 = _mm_castsi128_ps(_mm_unpacklo_epi32(c1, c3));
|
||||
_mm_storel_pi((__m64*)&d[M * 4], d2);
|
||||
_mm_storeh_pi((__m64*)&d[M * 5], d2);
|
||||
|
||||
__m128 d3 = _mm_castsi128_ps(_mm_unpackhi_epi32(c1, c3));
|
||||
_mm_storel_pi((__m64*)&d[M * 6], d3);
|
||||
_mm_storeh_pi((__m64*)&d[M * 7], d3);
|
||||
MlasTranspose8x8Block(s, N, d, M);
|
||||
|
||||
s += N * 8;
|
||||
d += 8;
|
||||
m -= 8;
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
while (m > 0) {
|
||||
|
||||
d[M * 0] = s[0];
|
||||
d[M * 1] = s[1];
|
||||
d[M * 2] = s[2];
|
||||
d[M * 3] = s[3];
|
||||
d[M * 4] = s[4];
|
||||
d[M * 5] = s[5];
|
||||
d[M * 6] = s[6];
|
||||
d[M * 7] = s[7];
|
||||
MlasTranspose8xNVector(s, 1, d, M);
|
||||
|
||||
s += N;
|
||||
d += 1;
|
||||
|
|
@ -142,14 +274,7 @@ Return Value:
|
|||
|
||||
while (m >= 8) {
|
||||
|
||||
d[0] = s[N * 0];
|
||||
d[1] = s[N * 1];
|
||||
d[2] = s[N * 2];
|
||||
d[3] = s[N * 3];
|
||||
d[4] = s[N * 4];
|
||||
d[5] = s[N * 5];
|
||||
d[6] = s[N * 6];
|
||||
d[7] = s[N * 7];
|
||||
MlasTranspose8xNVector(s, N, d, 1);
|
||||
|
||||
s += N * 8;
|
||||
d += 8;
|
||||
|
|
@ -170,5 +295,3 @@ Return Value:
|
|||
n -= 1;
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -168,7 +168,7 @@ std::vector<std::unique_ptr<GraphTransformer>> GenerateTransformers(TransformerL
|
|||
transformers.emplace_back(onnxruntime::make_unique<NchwcTransformer>());
|
||||
}
|
||||
|
||||
#if defined(MLAS_TARGET_AMD64_IX86)
|
||||
#ifdef MLAS_SUPPORTS_GEMM_U8X8
|
||||
transformers.emplace_back(onnxruntime::make_unique<NhwcTransformer>());
|
||||
#endif
|
||||
#endif
|
||||
|
|
|
|||
|
|
@ -70,7 +70,7 @@ Status QLinearMatMul::Compute(OpKernelContext* ctx) const {
|
|||
|
||||
const float real_multiplier = (a_scale_data * b_scale_data) / y_scale_data;
|
||||
|
||||
#ifdef MLAS_SUPPORTS_GEMM_U8X8_AND_REQUANTIZE_OUTPUT
|
||||
#ifdef MLAS_SUPPORTS_GEMM_U8X8
|
||||
AllocatorPtr alloc;
|
||||
ORT_RETURN_IF_ERROR(ctx->GetTempSpaceAllocator(&alloc));
|
||||
auto gemm_output_data = alloc->Alloc(SafeInt<size_t>(sizeof(int32_t)) *
|
||||
|
|
@ -85,7 +85,7 @@ Status QLinearMatMul::Compute(OpKernelContext* ctx) const {
|
|||
#endif
|
||||
|
||||
for (size_t i = 0; i < helper.OutputOffsets().size(); i++) {
|
||||
#ifdef MLAS_SUPPORTS_GEMM_U8X8_AND_REQUANTIZE_OUTPUT
|
||||
#ifdef MLAS_SUPPORTS_GEMM_U8X8
|
||||
QGemm(static_cast<int>(helper.M()),
|
||||
static_cast<int>(helper.N()),
|
||||
static_cast<int>(helper.K()),
|
||||
|
|
@ -105,7 +105,8 @@ Status QLinearMatMul::Compute(OpKernelContext* ctx) const {
|
|||
nullptr,
|
||||
static_cast<size_t>(helper.M()),
|
||||
static_cast<size_t>(helper.N()),
|
||||
real_multiplier,
|
||||
&real_multiplier,
|
||||
false,
|
||||
*y_offset->template Data<uint8_t>());
|
||||
#else
|
||||
GemmlowpMultiplyu8u8_u8(a->template Data<uint8_t>() + helper.LeftOffsets()[i],
|
||||
|
|
|
|||
|
|
@ -13,7 +13,7 @@
|
|||
|
||||
namespace onnxruntime {
|
||||
|
||||
#if defined(MLAS_TARGET_AMD64_IX86)
|
||||
#ifdef MLAS_SUPPORTS_GEMM_U8X8
|
||||
|
||||
class QLinearConv : public OpKernel {
|
||||
public:
|
||||
|
|
@ -69,6 +69,8 @@ ONNX_CPU_OPERATOR_KERNEL(
|
|||
|
||||
namespace contrib {
|
||||
|
||||
// Register an alternate version of this kernel that supports the channels_last
|
||||
// attribute in order to consume and produce NHWC tensors.
|
||||
ONNX_OPERATOR_KERNEL_EX(
|
||||
QLinearConv,
|
||||
kMSDomain,
|
||||
|
|
@ -525,23 +527,14 @@ Status QLinearConv::Compute(OpKernelContext* context) const {
|
|||
}
|
||||
}
|
||||
|
||||
if (output_scales.size() == 1) {
|
||||
MlasRequantizeOutputColumn(worker_gemm_output,
|
||||
worker_requantize_output,
|
||||
Bdata,
|
||||
static_cast<size_t>(output_count),
|
||||
static_cast<size_t>(M),
|
||||
output_scales[0],
|
||||
Y_zero_point_value);
|
||||
} else {
|
||||
MlasRequantizeOutputColumn(worker_gemm_output,
|
||||
worker_requantize_output,
|
||||
Bdata,
|
||||
static_cast<size_t>(output_count),
|
||||
static_cast<size_t>(M),
|
||||
output_scales.data(),
|
||||
Y_zero_point_value);
|
||||
}
|
||||
MlasRequantizeOutput(worker_gemm_output,
|
||||
worker_requantize_output,
|
||||
Bdata,
|
||||
static_cast<size_t>(output_count),
|
||||
static_cast<size_t>(M),
|
||||
output_scales.data(),
|
||||
output_scales.size() > 1,
|
||||
Y_zero_point_value);
|
||||
};
|
||||
|
||||
concurrency::ThreadPool::TrySimpleParallelFor(thread_pool, thread_count, conv_worker);
|
||||
|
|
@ -681,18 +674,10 @@ Status QLinearConv::Compute(OpKernelContext* context) const {
|
|||
|
||||
const float real_multiplier = (X_scale_value * W_scale_value) / Y_scale_value;
|
||||
|
||||
#ifdef MLAS_SUPPORTS_GEMM_U8X8_AND_REQUANTIZE_OUTPUT
|
||||
// Use an intermediate int32_t buffer for the GEMM computation before
|
||||
// requantizing to the output type.
|
||||
auto* gemm_output_data = alloc->Alloc(SafeInt<size_t>(sizeof(int32_t)) * Y_offset);
|
||||
BufferUniquePtr gemm_output_buffer(gemm_output_data, BufferDeleter(alloc));
|
||||
auto* gemm_output = static_cast<int32_t*>(gemm_output_buffer.get());
|
||||
#else
|
||||
// Compute the fixed point multiplier and shift for requantizing with GEMMLOWP.
|
||||
int32_t integer_multiplier;
|
||||
int right_shift;
|
||||
QuantizeMultiplier(real_multiplier, &integer_multiplier, &right_shift);
|
||||
#endif
|
||||
|
||||
const auto* Xdata = X->template Data<uint8_t>();
|
||||
const auto* Wdata = W->template Data<uint8_t>();
|
||||
|
|
@ -737,29 +722,6 @@ Status QLinearConv::Compute(OpKernelContext* context) const {
|
|||
}
|
||||
}
|
||||
|
||||
#ifdef MLAS_SUPPORTS_GEMM_U8X8_AND_REQUANTIZE_OUTPUT
|
||||
QGemm(static_cast<int>(group_output_channels),
|
||||
static_cast<int>(output_image_size),
|
||||
static_cast<int>(kernel_dim),
|
||||
Wdata + group_id * W_offset,
|
||||
static_cast<int>(kernel_dim),
|
||||
W_zero_point_value,
|
||||
col_buffer_data == nullptr ? Xdata : col_buffer_data,
|
||||
static_cast<int>(output_image_size),
|
||||
X_zero_point_value,
|
||||
false,
|
||||
gemm_output,
|
||||
static_cast<int>(output_image_size),
|
||||
context->GetOperatorThreadPool());
|
||||
|
||||
MlasRequantizeOutput(gemm_output,
|
||||
Ydata,
|
||||
Bdata != nullptr ? Bdata + group_id * group_output_channels : nullptr,
|
||||
static_cast<size_t>(group_output_channels),
|
||||
static_cast<size_t>(output_image_size),
|
||||
real_multiplier,
|
||||
Y_zero_point_value);
|
||||
#else
|
||||
GemmlowpMultiplyu8u8_u8(Wdata + group_id * W_offset,
|
||||
col_buffer_data == nullptr ? Xdata : col_buffer_data,
|
||||
Ydata,
|
||||
|
|
@ -772,7 +734,6 @@ Status QLinearConv::Compute(OpKernelContext* context) const {
|
|||
integer_multiplier,
|
||||
right_shift,
|
||||
Bdata != nullptr ? Bdata + group_id * group_output_channels : nullptr);
|
||||
#endif
|
||||
|
||||
Xdata += X_offset;
|
||||
Ydata += Y_offset;
|
||||
|
|
|
|||
|
|
@ -301,8 +301,6 @@ static void SimpleTransposeSingleAxisOutwards(const T* input_data, T* output_dat
|
|||
}
|
||||
}
|
||||
|
||||
#ifdef MLAS_SUPPORTS_TRANSPOSE
|
||||
|
||||
static void SimpleTransposeSingleAxisOutwards(const uint8_t* input_data, uint8_t* output_data,
|
||||
int64_t num_loops, int64_t num_writers,
|
||||
int64_t writes_per_loop, int64_t writes_per_writer_per_loop) {
|
||||
|
|
@ -316,8 +314,6 @@ static void SimpleTransposeSingleAxisOutwards(const uint8_t* input_data, uint8_t
|
|||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
// `input_shape_override` overrides the shape of `input` for compute purposes.
|
||||
static void TransposeSingleAxisOutwards(const std::vector<size_t>& permutations, const Tensor& input, Tensor& output,
|
||||
int64_t from, int64_t to, const TensorShape* input_shape_override = nullptr) {
|
||||
|
|
@ -411,8 +407,6 @@ static void SimpleTransposeSingleAxisInwards(const T* input_data, T* output_data
|
|||
}
|
||||
}
|
||||
|
||||
#ifdef MLAS_SUPPORTS_TRANSPOSE
|
||||
|
||||
static void SimpleTransposeSingleAxisInwards(const uint8_t* input_data, uint8_t* output_data,
|
||||
int64_t num_loops, int64_t num_readers,
|
||||
int64_t reads_per_loop, int64_t reads_per_reader_per_loop) {
|
||||
|
|
@ -426,8 +420,6 @@ static void SimpleTransposeSingleAxisInwards(const uint8_t* input_data, uint8_t*
|
|||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
// moving a single axis inwards where the read/write size is a power of 2 and between 8 and 64 bits.
|
||||
// `input_shape_override` overrides the shape of `input` for compute purposes.
|
||||
static void TransposeSingleAxisInwards(const std::vector<size_t>& permutations, const Tensor& input, Tensor& output,
|
||||
|
|
|
|||
|
|
@ -297,7 +297,7 @@ void NhwcTransformerTester(const std::function<void(NhwcTestHelper& helper)>& bu
|
|||
|
||||
#ifndef DISABLE_CONTRIB_OPS
|
||||
|
||||
#if defined(MLAS_TARGET_AMD64_IX86)
|
||||
#ifdef MLAS_SUPPORTS_GEMM_U8X8
|
||||
|
||||
TEST(NhwcTransformerTests, Conv) {
|
||||
auto test_case = [&](const std::vector<int64_t>& input_shape, const std::vector<int64_t>& weights_shape) {
|
||||
|
|
|
|||
|
|
@ -260,7 +260,7 @@ TEST(QLinearConvTest, WithGroup_2D) {
|
|||
{});
|
||||
}
|
||||
|
||||
#if defined(MLAS_TARGET_AMD64_IX86)
|
||||
#ifdef MLAS_SUPPORTS_GEMM_U8X8
|
||||
|
||||
template <typename T1, typename T2>
|
||||
class QLinearConvOpTester {
|
||||
|
|
@ -764,6 +764,44 @@ TEST(QLinearConvTest, Conv3D_U8S8_Depthwise) {
|
|||
test.Run();
|
||||
}
|
||||
|
||||
TEST(QLinearConvTest, Conv2D_U8S8_Requantize_NoBias) {
|
||||
for (int64_t channels = 1; channels <= 32; channels++) {
|
||||
QLinearConvOpTester<uint8_t, int8_t> test;
|
||||
test.GenerateRandomInput({1, 8, 5, 5}, .05f, 4);
|
||||
test.GenerateRandomWeights({channels, 8, 3, 3}, .125f, 0);
|
||||
test.SetPads({1, 1, 1, 1});
|
||||
test.SetOutputScaleAndZeroPoint(.55f, 56);
|
||||
test.Run();
|
||||
}
|
||||
}
|
||||
|
||||
TEST(QLinearConvTest, Conv2D_U8S8_Requantize_Bias) {
|
||||
for (int64_t channels = 1; channels <= 32; channels++) {
|
||||
QLinearConvOpTester<uint8_t, int8_t> test;
|
||||
test.GenerateRandomInput({1, 8, 5, 5}, .05f, 4);
|
||||
test.GenerateRandomWeights({channels, 8, 3, 3}, .125f, 0);
|
||||
test.GenerateRandomBias();
|
||||
test.SetPads({1, 1, 1, 1});
|
||||
test.SetOutputScaleAndZeroPoint(.55f, 56);
|
||||
test.Run();
|
||||
}
|
||||
}
|
||||
|
||||
TEST(QLinearConvTest, Conv2D_U8S8_Requantize_Bias_PerChannel) {
|
||||
std::vector<float> weight_scales;
|
||||
for (int64_t channels = 1; channels <= 32; channels++) {
|
||||
QLinearConvOpTester<uint8_t, int8_t> test;
|
||||
test.GenerateRandomInput({1, 8, 5, 5}, .05f, 4);
|
||||
test.GenerateRandomWeights({channels, 8, 3, 3}, .125f, 0);
|
||||
weight_scales.push_back(.120f + .002f * static_cast<float>(channels));
|
||||
test.SetWeightScales(weight_scales);
|
||||
test.GenerateRandomBias();
|
||||
test.SetPads({1, 1, 1, 1});
|
||||
test.SetOutputScaleAndZeroPoint(.55f, 56);
|
||||
test.Run();
|
||||
}
|
||||
}
|
||||
|
||||
#endif
|
||||
|
||||
} // namespace
|
||||
|
|
|
|||
Loading…
Reference in a new issue