saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-29 20:14:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Implement QLinearMul in mlas (#4593)

Browse source 
* Implement QLinearMul
This commit is contained in:
Zhang Lei 2020-09-04 15:02:19 -07:00 committed by GitHub
parent b5c2932ae8
commit ec88f14a7a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
11 changed files with 1020 additions and 339 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

1
cmake/onnxruntime_mlas.cmake
View file

@ -18,6 +18,7 @@ set(mlas_common_srcs
${ONNXRUNTIME_ROOT}/core/mlas/lib/compute.cpp
${ONNXRUNTIME_ROOT}/core/mlas/lib/quantize.cpp
${ONNXRUNTIME_ROOT}/core/mlas/lib/qladd.cpp
${ONNXRUNTIME_ROOT}/core/mlas/lib/qlmul.cpp
)
if(MSVC)

4
onnxruntime/contrib_ops/cpu/cpu_contrib_kernels.cc
View file

@ -45,6 +45,8 @@ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1,
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, int8_t, QLinearSigmoid);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, QLinearAdd);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, int8_t, QLinearAdd);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, QLinearMul);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, int8_t, QLinearMul);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, QAttention);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, DynamicQuantizeMatMul);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, MatMulIntegerToFloat);
@ -121,6 +123,8 @@ Status RegisterQuantizationKernels(KernelRegistry& kernel_registry) {
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, int8_t, QLinearSigmoid)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, QLinearAdd)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, int8_t, QLinearAdd)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, QLinearMul)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, int8_t, QLinearMul)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, QAttention)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, DynamicQuantizeMatMul)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, MatMulIntegerToFloat)>,

27
onnxruntime/contrib_ops/cpu/qlinear_binary_op.cc
View file

@ -137,6 +137,31 @@ Status QLinearAdd<T>::Compute(OpKernelContext* context) const {
1.0);
}
template <typename T>
Status QLinearMul<T>::Compute(OpKernelContext* context) const {
return QLinearBroadcastTwo<T>(
*context,
[](gsl::span<T> output, const T& input0, gsl::span<const T> input1,
float A_scale, float B_scale, float C_scale, T A_zero_point, T B_zero_point, T C_zero_point) {
MlasQLinearMul(input1.data(), B_scale, B_zero_point,
&input0, A_scale, A_zero_point,
C_scale, C_zero_point, output.data(), output.size(), true);
},
[](gsl::span<T> output, gsl::span<const T> input0, const T& input1,
float A_scale, float B_scale, float C_scale, T A_zero_point, T B_zero_point, T C_zero_point) {
MlasQLinearMul(input0.data(), A_scale, A_zero_point,
&input1, B_scale, B_zero_point,
C_scale, C_zero_point, output.data(), output.size(), true);
},
[](gsl::span<T> output, gsl::span<const T> input0, gsl::span<const T> input1,
float A_scale, float B_scale, float C_scale, T A_zero_point, T B_zero_point, T C_zero_point) {
MlasQLinearMul(input0.data(), A_scale, A_zero_point,
input1.data(), B_scale, B_zero_point,
C_scale, C_zero_point, output.data(), output.size(), false);
},
1.0);
}
#define REG_QLINEAR_ELEMENTWISE_TYPED_KERNEL(op_name, version, data_type, KERNEL_CLASS) \
ONNX_CPU_OPERATOR_TYPED_MS_KERNEL( \
op_name, version, data_type, \
@ -146,6 +171,8 @@ Status QLinearAdd<T>::Compute(OpKernelContext* context) const {
REG_QLINEAR_ELEMENTWISE_TYPED_KERNEL(QLinearAdd, 1, int8_t, QLinearAdd);
REG_QLINEAR_ELEMENTWISE_TYPED_KERNEL(QLinearAdd, 1, uint8_t, QLinearAdd);
REG_QLINEAR_ELEMENTWISE_TYPED_KERNEL(QLinearMul, 1, int8_t, QLinearMul);
REG_QLINEAR_ELEMENTWISE_TYPED_KERNEL(QLinearMul, 1, uint8_t, QLinearMul);
} // namespace contrib
} // namespace onnxruntime

8
onnxruntime/contrib_ops/cpu/qlinear_binary_op.h
View file

@ -18,5 +18,13 @@ class QLinearAdd final : public OpKernel {
Status Compute(OpKernelContext* context) const override;
};
template <typename T>
class QLinearMul final : public OpKernel {
public:
QLinearMul(const OpKernelInfo& info) : OpKernel(info) {}
Status Compute(OpKernelContext* context) const override;
};
} // namespace contrib
} // namespace onnxruntime

17
onnxruntime/core/mlas/inc/mlas.h
View file

@ -554,3 +554,20 @@ MlasQLinearAdd(
size_t N,
bool IsScalarB
);
template<typename DataType>
void
MLASCALL
MlasQLinearMul(
const DataType* InputA,
float ScaleA,
int32_t ZeroPointA,
const DataType* InputB,
float ScaleB,
int32_t ZeroPointB,
float ScaleC,
int32_t ZeroPointC,
DataType* OutputC,
size_t N,
bool IsScalarB
);

5
onnxruntime/core/mlas/lib/intrinsics/avx2/qladd_avx2.cpp
View file

@ -18,7 +18,6 @@ Abstract:
--*/
#include "../../mlasi.h"
#include "../../qladd.h"
template <typename DataType>
@ -30,6 +29,7 @@ MlasShiftRight24Epi32(
);
template <>
MLAS_FORCEINLINE
__m256i
MlasShiftRight24Epi32<int8_t>(
__m256i v
@ -39,6 +39,7 @@ MlasShiftRight24Epi32<int8_t>(
}
template <>
MLAS_FORCEINLINE
__m256i
MlasShiftRight24Epi32<uint8_t>(
__m256i v
@ -57,6 +58,7 @@ MlasPackS16_256(
);
template <>
MLAS_FORCEINLINE
__m256i
MlasPackS16_256<uint8_t>(
__m256i a,
@ -67,6 +69,7 @@ MlasPackS16_256<uint8_t>(
}
template <>
MLAS_FORCEINLINE
__m256i
MlasPackS16_256<int8_t>(
__m256i a,

301
onnxruntime/core/mlas/lib/qladd.cpp
View file

@ -88,232 +88,6 @@ MlasQLinearAddKernelRawHelper(
#if defined(MLAS_NEON_INTRINSICS)
#if ! defined(_MSC_VER)
#define vld1q_s8_ex(pD, align) vld1q_s8((int8_t*)__builtin_assume_aligned(pD, ((align)/8)))
#define vst1_s8_ex(pD, D, align) vst1_s8((int8_t*)__builtin_assume_aligned(pD, ((align)/8)), D)
#define vst1q_s8_ex(pD, D, align) vst1q_s8((int8_t*)__builtin_assume_aligned(pD, ((align)/8)), D)
#define vld1q_u8_ex(pD, align) vld1q_u8((uint8_t*)__builtin_assume_aligned(pD, ((align)/8)))
#define vst1_u8_ex(pD, D, align) vst1_u8((uint8_t*)__builtin_assume_aligned(pD, ((align)/8)), D)
#define vst1q_u8_ex(pD, D, align) vst1q_u8((uint8_t*)__builtin_assume_aligned(pD, ((align)/8)), D)
#define vst1_lane_u32_ex(pD, D, lane, align) vst1_lane_u32((uint32_t*)__builtin_assume_aligned(pD, ((align)/8)), D, lane)
#define vst1_lane_u16_ex(pD, D, lane, align) vst1_lane_u16((uint16_t*)__builtin_assume_aligned(pD, ((align)/8)), D, lane)
#endif
template <typename DataType>
class MLAS_SignedUnsignedIntOps;
template <>
class MLAS_SignedUnsignedIntOps<uint8_t>
{
public:
typedef uint8_t T;
typedef uint8x8_t i8x8_t;
typedef uint8x16_t i8x16_t;
typedef uint16x8_t i16x8_t;
static MLAS_FORCEINLINE i8x8_t vmov_n_i8(T value)
{
return vmov_n_u8(value);
}
static MLAS_FORCEINLINE i8x8_t vget_low_i8(i8x16_t a)
{
return vget_low_u8(a);
}
static MLAS_FORCEINLINE i8x8_t vget_high_i8(i8x16_t a)
{
return vget_high_u8(a);
}
static MLAS_FORCEINLINE i16x8_t vsubl_i8(i8x8_t a, i8x8_t b)
{
return vsubl_u8(a, b);
}
static MLAS_FORCEINLINE int16x8_t vreinterpretq_s16_i16(i16x8_t a)
{
return vreinterpretq_s16_u16(a);
}
static MLAS_FORCEINLINE uint32x4_t vreinterpretq_u32_i8(i8x16_t a)
{
return vreinterpretq_u32_u8(a);
}
static MLAS_FORCEINLINE uint16x8_t vreinterpretq_u16_i8(i8x16_t a)
{
return vreinterpretq_u16_u8(a);
}
static MLAS_FORCEINLINE uint32x2_t vreinterpret_u32_i8(i8x8_t a)
{
return vreinterpret_u32_u8(a);
}
static MLAS_FORCEINLINE uint16x4_t vreinterpret_u16_i8(i8x8_t a)
{
return vreinterpret_u16_u8(a);
}
static MLAS_FORCEINLINE i8x16_t vld1q_i8(T const * ptr)
{
return vld1q_u8_ex(ptr, 8);
}
static MLAS_FORCEINLINE void vst1_i8(T* ptr, i8x8_t a)
{
vst1_u8_ex(ptr, a, 8);
}
static MLAS_FORCEINLINE void vst1q_i8(T* ptr, i8x16_t a)
{
vst1q_u8_ex(ptr, a, 8);
}
template <int n>
static MLAS_FORCEINLINE void vst1_lane_i8(T* ptr, i8x8_t a)
{
vst1_lane_u8(ptr, a, n);
}
template <int n>
static MLAS_FORCEINLINE i8x16_t vextq_i8(i8x16_t lo, i8x16_t hi)
{
return vextq_u8(lo, hi, n);
}
template <int n>
static MLAS_FORCEINLINE i8x8_t vext_i8(i8x8_t lo, i8x8_t hi)
{
return vext_u8(lo, hi, n);
}
static MLAS_FORCEINLINE i8x16_t combine_i8_s16(int16x8_t v0, int16x8_t v1)
{
#if defined(MLAS_NEON64_INTRINSICS)
return vqmovun_high_s16(vqmovun_s16(v0), v1);
#else
return vcombine_u8(vqmovun_s16(v0), vqmovun_s16(v1));
#endif
}
};
template <>
class MLAS_SignedUnsignedIntOps<int8_t>
{
public:
typedef int8_t T;
typedef int8x8_t i8x8_t;
typedef int8x16_t i8x16_t;
typedef int16x8_t i16x8_t;
static MLAS_FORCEINLINE i8x8_t vmov_n_i8(T value)
{
return vmov_n_s8(value);
}
static MLAS_FORCEINLINE i8x8_t vget_low_i8(i8x16_t a)
{
return vget_low_s8(a);
}
static MLAS_FORCEINLINE i8x8_t vget_high_i8(i8x16_t a)
{
return vget_high_s8(a);
}
static MLAS_FORCEINLINE i16x8_t vsubl_i8(i8x8_t a, i8x8_t b)
{
return vsubl_s8(a, b);
}
static MLAS_FORCEINLINE int16x8_t vreinterpretq_s16_i16(i16x8_t a)
{
return a;
}
static MLAS_FORCEINLINE uint32x4_t vreinterpretq_u32_i8(i8x16_t a)
{
return vreinterpretq_u32_s8(a);
}
static MLAS_FORCEINLINE uint16x8_t vreinterpretq_u16_i8(i8x16_t a)
{
return vreinterpretq_u16_s8(a);
}
static MLAS_FORCEINLINE uint32x2_t vreinterpret_u32_i8(i8x8_t a)
{
return vreinterpret_u32_s8(a);
}
static MLAS_FORCEINLINE uint16x4_t vreinterpret_u16_i8(i8x8_t a)
{
return vreinterpret_u16_s8(a);
}
static MLAS_FORCEINLINE i8x16_t vld1q_i8(T const * ptr)
{
return vld1q_s8_ex(ptr, 8);
}
static MLAS_FORCEINLINE void vst1_i8(T* ptr, i8x8_t a)
{
vst1_s8_ex(ptr, a, 8);
}
static MLAS_FORCEINLINE void vst1q_i8(T* ptr, i8x16_t a)
{
vst1q_s8_ex(ptr, a, 8);
}
template <int n>
static MLAS_FORCEINLINE void vst1_lane_i8(T* ptr, i8x8_t a)
{
vst1_lane_s8(ptr, a, n);
}
template <int n>
static MLAS_FORCEINLINE i8x16_t vextq_i8(i8x16_t lo, i8x16_t hi)
{
return vextq_s8(lo, hi, n);
}
template <int n>
static MLAS_FORCEINLINE i8x8_t vext_i8(i8x8_t lo, i8x8_t hi)
{
return vext_s8(lo, hi, n);
}
static MLAS_FORCEINLINE i8x16_t combine_i8_s16(int16x8_t v0, int16x8_t v1)
{
#if defined(MLAS_NEON64_INTRINSICS)
return vqmovn_high_s16(vqmovn_s16(v0), v1);
#else
return vcombine_s8(vqmovn_s16(v0), vqmovn_s16(v1));
#endif
}
};
#if defined(MLAS_NEON64_INTRINSICS)
#define MlasMoveHighS16S32(s16x8) vmovl_high_s16(s16x8)
#define MlasCombineS16S32(lo, hi) vqmovn_high_s32(vqmovn_s32(lo), hi)
#else
#define MlasMoveHighS16S32(s16x8) vmovl_s16(vget_high_s16(s16x8))
#define MlasCombineS16S32(lo, hi) vcombine_s16(vqmovn_s32(lo), vqmovn_s32(hi))
#endif
template<typename DataType, bool IsScalarB>
static
void
@ -512,7 +286,6 @@ MlasQLinearAddKernelHelper(
int32x4_t vacc0_lo, vacc1_lo, vacc0_hi, vacc1_hi;
if (IsScalarB) {
const typename SUI::i8x16_t VectorA0 = SUI::vld1q_i8(TailDataA);
InputA += 16;
const int16x8_t va0_s16x8 = SUI::vreinterpretq_s16_i16(SUI::vsubl_i8(SUI::vget_low_i8(VectorA0), VectorZeroPointA));
const int16x8_t va1_s16x8 = SUI::vreinterpretq_s16_i16(SUI::vsubl_i8(SUI::vget_high_i8(VectorA0), VectorZeroPointA));
@ -523,8 +296,6 @@ MlasQLinearAddKernelHelper(
} else {
const typename SUI::i8x16_t VectorA0 = SUI::vld1q_i8(TailDataA);
const typename SUI::i8x16_t VectorB0 = SUI::vld1q_i8(TailDataB);
InputA += 16;
InputB += 16;
const int16x8_t va0_s16x8 = SUI::vreinterpretq_s16_i16(SUI::vsubl_i8(SUI::vget_low_i8(VectorA0), VectorZeroPointA));
const int16x8_t vb0_s16x8 = SUI::vreinterpretq_s16_i16(SUI::vsubl_i8(SUI::vget_low_i8(VectorB0), VectorZeroPointB));
const int16x8_t va1_s16x8 = SUI::vreinterpretq_s16_i16(SUI::vsubl_i8(SUI::vget_high_i8(VectorA0), VectorZeroPointA));
@ -580,64 +351,6 @@ MlasQLinearAddKernelHelper(
#elif defined(MLAS_SSE2_INTRINSICS)
template <typename DataType>
MLAS_FORCEINLINE
static
MLAS_INT32X4
MlasShiftRightInt32(
MLAS_INT32X4 v,
int imm
);
template<>
MLAS_INT32X4
MlasShiftRightInt32<int8_t>(
MLAS_INT32X4 v,
int imm
)
{
return _mm_srai_epi32(v, imm);
}
template<>
MLAS_INT32X4
MlasShiftRightInt32<uint8_t>(
MLAS_INT32X4 v,
int imm
)
{
return _mm_srli_epi32(v, imm);
}
template <typename DataType>
MLAS_FORCEINLINE
static
MLAS_INT32X4
MlasPackS16_128(
MLAS_INT32X4 a,
MLAS_INT32X4 b
);
template <>
MLAS_INT32X4
MlasPackS16_128<uint8_t>(
MLAS_INT32X4 a,
MLAS_INT32X4 b
)
{
return _mm_packus_epi16(a, b);
}
template <>
MLAS_INT32X4
MlasPackS16_128<int8_t>(
MLAS_INT32X4 a,
MLAS_INT32X4 b
)
{
return _mm_packs_epi16(a, b);
}
template<typename DataType, bool IsScalarB>
static
void
@ -700,20 +413,16 @@ MlasQLinearAddKernelHelper(
if (N > 0) {
uint8_t TailData[8] = { 0 };
{
MlasCopyTailBytes(TailData, (const uint8_t*)InputA, N);
const auto va_low_half = _mm_loadl_epi64((const MLAS_INT32X4*)TailData);
const auto va_i16x8 = _mm_unpacklo_epi8(va_low_half, va_low_half);
InputA += 8;
va_lo = _mm_cvtepi32_ps(MlasShiftRightInt32<DataType>(_mm_unpacklo_epi16(va_i16x8, va_i16x8), 24));
va_hi = _mm_cvtepi32_ps(MlasShiftRightInt32<DataType>(_mm_unpackhi_epi16(va_i16x8, va_i16x8), 24));
}
MlasCopyTailBytes(TailData, (const uint8_t*)InputA, N);
const auto va_low_half = _mm_loadl_epi64((const MLAS_INT32X4*)TailData);
const auto va_i16x8 = _mm_unpacklo_epi8(va_low_half, va_low_half);
va_lo = _mm_cvtepi32_ps(MlasShiftRightInt32<DataType>(_mm_unpacklo_epi16(va_i16x8, va_i16x8), 24));
va_hi = _mm_cvtepi32_ps(MlasShiftRightInt32<DataType>(_mm_unpackhi_epi16(va_i16x8, va_i16x8), 24));
if (!IsScalarB) {
MlasCopyTailBytes(TailData, (const uint8_t*)InputB, N);
const auto vb_low_half = _mm_loadl_epi64((const MLAS_INT32X4*)TailData);
const auto vb_i16x8 = _mm_unpacklo_epi8(vb_low_half, vb_low_half);
InputB += 8;
vb_lo = _mm_cvtepi32_ps(MlasShiftRightInt32<DataType>(_mm_unpacklo_epi16(vb_i16x8, vb_i16x8), 24));
vb_hi = _mm_cvtepi32_ps(MlasShiftRightInt32<DataType>(_mm_unpackhi_epi16(vb_i16x8, vb_i16x8), 24));
}

322
onnxruntime/core/mlas/lib/qladd.h
View file

@ -76,3 +76,325 @@ MlasCalcQLinearAddParameters(
int32_t& MultiplierA,
int32_t& MultiplierB
);
#if defined(MLAS_NEON_INTRINSICS)
#if ! defined(_MSC_VER)
#define vld1q_s8_ex(pD, align) vld1q_s8((int8_t*)__builtin_assume_aligned(pD, ((align)/8)))
#define vst1_s8_ex(pD, D, align) vst1_s8((int8_t*)__builtin_assume_aligned(pD, ((align)/8)), D)
#define vst1q_s8_ex(pD, D, align) vst1q_s8((int8_t*)__builtin_assume_aligned(pD, ((align)/8)), D)
#define vld1q_u8_ex(pD, align) vld1q_u8((uint8_t*)__builtin_assume_aligned(pD, ((align)/8)))
#define vst1_u8_ex(pD, D, align) vst1_u8((uint8_t*)__builtin_assume_aligned(pD, ((align)/8)), D)
#define vst1q_u8_ex(pD, D, align) vst1q_u8((uint8_t*)__builtin_assume_aligned(pD, ((align)/8)), D)
#define vst1_lane_u32_ex(pD, D, lane, align) vst1_lane_u32((uint32_t*)__builtin_assume_aligned(pD, ((align)/8)), D, lane)
#define vst1_lane_u16_ex(pD, D, lane, align) vst1_lane_u16((uint16_t*)__builtin_assume_aligned(pD, ((align)/8)), D, lane)
#endif
template <typename DataType>
class MLAS_SignedUnsignedIntOps;
template <>
class MLAS_SignedUnsignedIntOps<uint8_t>
{
public:
typedef uint8_t T;
typedef uint8x8_t i8x8_t;
typedef uint8x16_t i8x16_t;
typedef uint16x8_t i16x8_t;
static MLAS_FORCEINLINE i8x8_t vmov_n_i8(T value)
{
return vmov_n_u8(value);
}
static MLAS_FORCEINLINE i8x8_t vget_low_i8(i8x16_t a)
{
return vget_low_u8(a);
}
static MLAS_FORCEINLINE i8x8_t vget_high_i8(i8x16_t a)
{
return vget_high_u8(a);
}
static MLAS_FORCEINLINE i16x8_t vsubl_i8(i8x8_t a, i8x8_t b)
{
return vsubl_u8(a, b);
}
static MLAS_FORCEINLINE int16x8_t vreinterpretq_s16_i16(i16x8_t a)
{
return vreinterpretq_s16_u16(a);
}
static MLAS_FORCEINLINE uint32x4_t vreinterpretq_u32_i8(i8x16_t a)
{
return vreinterpretq_u32_u8(a);
}
static MLAS_FORCEINLINE uint16x8_t vreinterpretq_u16_i8(i8x16_t a)
{
return vreinterpretq_u16_u8(a);
}
static MLAS_FORCEINLINE uint32x2_t vreinterpret_u32_i8(i8x8_t a)
{
return vreinterpret_u32_u8(a);
}
static MLAS_FORCEINLINE uint16x4_t vreinterpret_u16_i8(i8x8_t a)
{
return vreinterpret_u16_u8(a);
}
static MLAS_FORCEINLINE i8x16_t vld1q_i8(T const * ptr)
{
return vld1q_u8_ex(ptr, 8);
}
static MLAS_FORCEINLINE void vst1_i8(T* ptr, i8x8_t a)
{
vst1_u8_ex(ptr, a, 8);
}
static MLAS_FORCEINLINE void vst1q_i8(T* ptr, i8x16_t a)
{
vst1q_u8_ex(ptr, a, 8);
}
template <int n>
static MLAS_FORCEINLINE void vst1_lane_i8(T* ptr, i8x8_t a)
{
vst1_lane_u8(ptr, a, n);
}
template <int n>
static MLAS_FORCEINLINE i8x16_t vextq_i8(i8x16_t lo, i8x16_t hi)
{
return vextq_u8(lo, hi, n);
}
template <int n>
static MLAS_FORCEINLINE i8x8_t vext_i8(i8x8_t lo, i8x8_t hi)
{
return vext_u8(lo, hi, n);
}
static MLAS_FORCEINLINE i8x16_t combine_i8_s16(int16x8_t v0, int16x8_t v1)
{
#if defined(MLAS_NEON64_INTRINSICS)
return vqmovun_high_s16(vqmovun_s16(v0), v1);
#else
return vcombine_u8(vqmovun_s16(v0), vqmovun_s16(v1));
#endif
}
};
template <>
class MLAS_SignedUnsignedIntOps<int8_t>
{
public:
typedef int8_t T;
typedef int8x8_t i8x8_t;
typedef int8x16_t i8x16_t;
typedef int16x8_t i16x8_t;
static MLAS_FORCEINLINE i8x8_t vmov_n_i8(T value)
{
return vmov_n_s8(value);
}
static MLAS_FORCEINLINE i8x8_t vget_low_i8(i8x16_t a)
{
return vget_low_s8(a);
}
static MLAS_FORCEINLINE i8x8_t vget_high_i8(i8x16_t a)
{
return vget_high_s8(a);
}
static MLAS_FORCEINLINE i16x8_t vsubl_i8(i8x8_t a, i8x8_t b)
{
return vsubl_s8(a, b);
}
static MLAS_FORCEINLINE int16x8_t vreinterpretq_s16_i16(i16x8_t a)
{
return a;
}
static MLAS_FORCEINLINE uint32x4_t vreinterpretq_u32_i8(i8x16_t a)
{
return vreinterpretq_u32_s8(a);
}
static MLAS_FORCEINLINE uint16x8_t vreinterpretq_u16_i8(i8x16_t a)
{
return vreinterpretq_u16_s8(a);
}
static MLAS_FORCEINLINE uint32x2_t vreinterpret_u32_i8(i8x8_t a)
{
return vreinterpret_u32_s8(a);
}
static MLAS_FORCEINLINE uint16x4_t vreinterpret_u16_i8(i8x8_t a)
{
return vreinterpret_u16_s8(a);
}
static MLAS_FORCEINLINE i8x16_t vld1q_i8(T const * ptr)
{
return vld1q_s8_ex(ptr, 8);
}
static MLAS_FORCEINLINE void vst1_i8(T* ptr, i8x8_t a)
{
vst1_s8_ex(ptr, a, 8);
}
static MLAS_FORCEINLINE void vst1q_i8(T* ptr, i8x16_t a)
{
vst1q_s8_ex(ptr, a, 8);
}
template <int n>
static MLAS_FORCEINLINE void vst1_lane_i8(T* ptr, i8x8_t a)
{
vst1_lane_s8(ptr, a, n);
}
template <int n>
static MLAS_FORCEINLINE i8x16_t vextq_i8(i8x16_t lo, i8x16_t hi)
{
return vextq_s8(lo, hi, n);
}
template <int n>
static MLAS_FORCEINLINE i8x8_t vext_i8(i8x8_t lo, i8x8_t hi)
{
return vext_s8(lo, hi, n);
}
static MLAS_FORCEINLINE i8x16_t combine_i8_s16(int16x8_t v0, int16x8_t v1)
{
#if defined(MLAS_NEON64_INTRINSICS)
return vqmovn_high_s16(vqmovn_s16(v0), v1);
#else
return vcombine_s8(vqmovn_s16(v0), vqmovn_s16(v1));
#endif
}
};
#if defined(MLAS_NEON64_INTRINSICS)
#define MlasMoveHighS16S32(s16x8) vmovl_high_s16(s16x8)
#define MlasCombineS16S32(lo, hi) vqmovn_high_s32(vqmovn_s32(lo), hi)
#else
#define MlasMoveHighS16S32(s16x8) vmovl_s16(vget_high_s16(s16x8))
#define MlasCombineS16S32(lo, hi) vcombine_s16(vqmovn_s32(lo), vqmovn_s32(hi))
#endif
#elif defined(MLAS_SSE2_INTRINSICS)
template <typename DataType>
MLAS_FORCEINLINE
MLAS_INT32X4
MlasShiftRightInt32(
MLAS_INT32X4 v,
int imm
);
template<>
MLAS_FORCEINLINE
MLAS_INT32X4
MlasShiftRightInt32<int8_t>(
MLAS_INT32X4 v,
int imm
)
{
return _mm_srai_epi32(v, imm);
}
template<>
MLAS_FORCEINLINE
MLAS_INT32X4
MlasShiftRightInt32<uint8_t>(
MLAS_INT32X4 v,
int imm
)
{
return _mm_srli_epi32(v, imm);
}
template <typename DataType>
MLAS_FORCEINLINE
MLAS_INT32X4
MlasShiftRightInt16(
MLAS_INT32X4 v,
int imm
);
template<>
MLAS_FORCEINLINE
MLAS_INT32X4
MlasShiftRightInt16<int8_t>(
MLAS_INT32X4 v,
int imm
)
{
return _mm_srai_epi16(v, imm);
}
template<>
MLAS_FORCEINLINE
MLAS_INT32X4
MlasShiftRightInt16<uint8_t>(
MLAS_INT32X4 v,
int imm
)
{
return _mm_srli_epi16(v, imm);
}
template <typename DataType>
MLAS_FORCEINLINE
MLAS_INT32X4
MlasPackS16_128(
MLAS_INT32X4 a,
MLAS_INT32X4 b
);
template <>
MLAS_FORCEINLINE
MLAS_INT32X4
MlasPackS16_128<uint8_t>(
MLAS_INT32X4 a,
MLAS_INT32X4 b
)
{
return _mm_packus_epi16(a, b);
}
template <>
MLAS_FORCEINLINE
MLAS_INT32X4
MlasPackS16_128<int8_t>(
MLAS_INT32X4 a,
MLAS_INT32X4 b
)
{
return _mm_packs_epi16(a, b);
}
#endif

385
onnxruntime/core/mlas/lib/qlmul.cpp Normal file
View file

@ -0,0 +1,385 @@
/*++
Copyright (c) Microsoft Corporation. All rights reserved.
Licensed under the MIT License.
Module Name:
qlmul.cpp
Abstract:
This module implements routines to quantize linear mul.
For quantization formula as specified in the ONNX operator documentation is:
Output = Saturate(RoundToEven(Input / Scale) + ZeroPoint)
--*/
#include "qladd.h"
#if defined(MLAS_NEON64_INTRINSICS)
template <typename SUI, bool IsLow>
MLAS_FORCEINLINE
static
int16x8_t
MlasExtendToS16Debias(
typename SUI::i8x16_t Int8Vector,
typename SUI::i8x8_t VectorBias
)
{
auto HalfVector = IsLow ? SUI::vget_low_i8(Int8Vector) : SUI::vget_high_i8(Int8Vector);
return SUI::vreinterpretq_s16_i16(SUI::vsubl_i8(HalfVector, VectorBias));
}
MLAS_FORCEINLINE
static
int16x8_t
MlasQLinearMulVectorS16(
int16x8_t va_s16x8,
int16x8_t vb_s16x8,
float32x4_t VectorScaleRatio,
float32x4_t VectorZeroPointC
)
{
int32x4_t vacc0_lo = vmull_s16(vget_low_s16(va_s16x8), vget_low_s16(vb_s16x8));
int32x4_t vacc0_hi = vmull_s16(vget_high_s16(va_s16x8), vget_high_s16(vb_s16x8));
auto vacc0_lo_f32 = vaddq_f32(VectorZeroPointC, vmulq_f32(VectorScaleRatio, vcvtq_f32_s32(vacc0_lo)));
auto vacc0_hi_f32 = vaddq_f32(VectorZeroPointC, vmulq_f32(VectorScaleRatio, vcvtq_f32_s32(vacc0_hi)));
// using rounding to nearst, ties to even
vacc0_lo = vcvtnq_s32_f32(vacc0_lo_f32);
vacc0_hi = vcvtnq_s32_f32(vacc0_hi_f32);
// Pack and saturate.
return vcombine_s16(vqmovn_s32(vacc0_lo), vqmovn_s32(vacc0_hi));
}
template<typename DataType, bool IsScalarB>
static
void
MlasQLinearMulKernel(
const DataType* InputA,
float ScaleA,
int32_t ZeroPointA,
const DataType* InputB,
float ScaleB,
int32_t ZeroPointB,
float ScaleC,
int32_t ZeroPointC,
DataType* OutputC,
size_t N
)
{
typedef MLAS_SignedUnsignedIntOps<DataType> SUI;
const float32x4_t VectorScaleRatio = vmovq_n_f32(ScaleA * ScaleB / ScaleC);
const typename SUI::i8x8_t VectorZeroPointA = SUI::vmov_n_i8((DataType)ZeroPointA);
const typename SUI::i8x8_t VectorZeroPointB = SUI::vmov_n_i8((DataType)ZeroPointB);
const float32x4_t VectorZeroPointC = vmovq_n_f32((float)ZeroPointC);
typename SUI::T TailDataA[16] = { 0 };
typename SUI::T TailDataB[16] = { 0 };
int16x8_t vb0_s16x8, vb1_s16x8;
if (IsScalarB) {
const typename SUI::i8x8_t VectorB0 = SUI::vmov_n_i8(*InputB);
vb0_s16x8 = SUI::vreinterpretq_s16_i16(SUI::vsubl_i8(VectorB0, VectorZeroPointB));
vb1_s16x8 = vb0_s16x8;
}
while (N > 0) {
if (N < 16) {
MlasCopyTailBytes((uint8_t*)TailDataA, (const uint8_t*)InputA, N);
InputA = (const DataType*)TailDataA;
if (!IsScalarB) {
MlasCopyTailBytes((uint8_t*)TailDataB, (const uint8_t*)InputB, N);
InputB = (const DataType*)TailDataB;
}
}
const typename SUI::i8x16_t VectorA0 = SUI::vld1q_i8(InputA);
InputA += 16;
const int16x8_t va0_s16x8 = MlasExtendToS16Debias<SUI, /* IsLow = */ true>(VectorA0, VectorZeroPointA);
const int16x8_t va1_s16x8 = MlasExtendToS16Debias<SUI, /* IsLow = */ false>(VectorA0, VectorZeroPointA);;
if (!IsScalarB) {
const typename SUI::i8x16_t VectorB0 = SUI::vld1q_i8(InputB);
InputB += 16;
vb0_s16x8 = MlasExtendToS16Debias<SUI, /* IsLow = */ true>(VectorB0, VectorZeroPointB);
vb1_s16x8 = MlasExtendToS16Debias<SUI, /* IsLow = */ false>(VectorB0, VectorZeroPointB);
}
const int16x8_t vacc0 = MlasQLinearMulVectorS16(va0_s16x8, vb0_s16x8, VectorScaleRatio, VectorZeroPointC);
const int16x8_t vacc1 = MlasQLinearMulVectorS16(va1_s16x8, vb1_s16x8, VectorScaleRatio, VectorZeroPointC);
typename SUI::i8x16_t vc = SUI::combine_i8_s16(vacc0, vacc1);
if (N >= 16) {
N -= 16;
SUI::vst1q_i8(OutputC, vc);
OutputC += 16;
} else {
SUI::vst1q_i8(TailDataA, vc);
MlasCopyTailBytes((uint8_t*)OutputC, (const uint8_t*)TailDataA, N);
N = 0;
}
}
}
#elif defined(MLAS_SSE2_INTRINSICS)
template <class DataType, bool IsLow>
MLAS_FORCEINLINE
static
__m128i
MlasExtendToS16(
__m128i Int8Vector,
__m128i ZeroVector
);
template <>
MLAS_FORCEINLINE
__m128i
MlasExtendToS16<uint8_t, /* bool IsLow = */ true>(
__m128i Int8Vector,
__m128i ZeroVector
)
{
return _mm_unpacklo_epi8(Int8Vector, ZeroVector);
}
template <>
MLAS_FORCEINLINE
__m128i
MlasExtendToS16<uint8_t, /* bool IsLow = */ false>(
__m128i Int8Vector,
__m128i ZeroVector
)
{
return _mm_unpackhi_epi8(Int8Vector, ZeroVector);
}
template <>
MLAS_FORCEINLINE
__m128i
MlasExtendToS16<int8_t, /* bool IsLow = */ true>(
__m128i Int8Vector,
__m128i ZeroVector
)
{
MLAS_UNREFERENCED_PARAMETER(ZeroVector);
return _mm_srai_epi16(_mm_unpacklo_epi8(Int8Vector, Int8Vector), 8);
}
template <>
MLAS_FORCEINLINE
__m128i
MlasExtendToS16<int8_t, /* bool IsLow = */ false>(
__m128i Int8Vector,
__m128i ZeroVector
)
{
MLAS_UNREFERENCED_PARAMETER(ZeroVector);
return _mm_srai_epi16(_mm_unpackhi_epi8(Int8Vector, Int8Vector), 8);
}
template <class DataType, bool IsLow>
MLAS_FORCEINLINE
static
__m128i
MlasExtendToS16Debias(
__m128i Int8Vector,
__m128i ZeroVector,
__m128i VectorBias
)
{
return _mm_sub_epi16(MlasExtendToS16<DataType, IsLow>(Int8Vector, ZeroVector), VectorBias);
}
MLAS_FORCEINLINE
static
__m128i
MlasQLinearMulVectorS16(
__m128i va_s16x8,
__m128i vb_s16x8,
__m128 VectorScaleRatio,
__m128 VectorZeroPointC
)
{
const auto ab_lo = _mm_mullo_epi16(va_s16x8, vb_s16x8);
const auto ab_hi = _mm_mulhi_epi16(va_s16x8, vb_s16x8);
auto r_lo = _mm_unpacklo_epi16(ab_lo, ab_hi);
auto r_hi = _mm_unpackhi_epi16(ab_lo, ab_hi);
r_lo = _mm_cvtps_epi32(_mm_add_ps(_mm_mul_ps(_mm_cvtepi32_ps(r_lo), VectorScaleRatio), VectorZeroPointC));
r_hi = _mm_cvtps_epi32(_mm_add_ps(_mm_mul_ps(_mm_cvtepi32_ps(r_hi), VectorScaleRatio), VectorZeroPointC));
return _mm_packs_epi32(r_lo, r_hi);
}
template<typename DataType, bool IsScalarB>
static
void
MlasQLinearMulKernel(
const DataType* InputA,
float ScaleA,
int32_t ZeroPointA,
const DataType* InputB,
float ScaleB,
int32_t ZeroPointB,
float ScaleC,
int32_t ZeroPointC,
DataType* OutputC,
size_t N
)
{
const auto VectorZeroPointA = _mm_set1_epi16((int16_t)ZeroPointA);
const auto VectorZeroPointB = _mm_set1_epi16((int16_t)ZeroPointB);
const auto VectorZeroPointC = MlasBroadcastFloat32x4((float)ZeroPointC);
const auto VectorScaleRatio = MlasBroadcastFloat32x4(ScaleA * ScaleB / ScaleC);
const auto ZeroVector = _mm_setzero_si128();
uint8_t TailDataA[16] = { 0 };
uint8_t TailDataB[16] = { 0 };
__m128i vb_lo_s16x8, vb_hi_s16x8;
if (IsScalarB) {
vb_lo_s16x8 = _mm_sub_epi16(_mm_set1_epi16((int16_t)*InputB), VectorZeroPointB);
vb_hi_s16x8 = vb_lo_s16x8;
}
while (N > 0) {
if (N < 16) {
MlasCopyTailBytes(TailDataA, (const uint8_t*)InputA, N);
InputA = (const DataType*)TailDataA;
if (!IsScalarB) {
MlasCopyTailBytes(TailDataB, (const uint8_t*)InputB, N);
InputB = (const DataType*)TailDataB;
}
}
const auto va_i8x16 = _mm_loadu_si128((const MLAS_INT32X4*)InputA);
InputA += 16;
const auto va_lo_s16x8 = MlasExtendToS16Debias<DataType, true>(va_i8x16, ZeroVector, VectorZeroPointA);
const auto va_hi_s16x8 = MlasExtendToS16Debias<DataType, false>(va_i8x16, ZeroVector, VectorZeroPointA);
if (!IsScalarB) {
const auto vb_i8x16 = _mm_loadu_si128((const MLAS_INT32X4*)InputB);
InputB += 16;
vb_lo_s16x8 = MlasExtendToS16Debias<DataType, true>(vb_i8x16, ZeroVector, VectorZeroPointB);
vb_hi_s16x8 = MlasExtendToS16Debias<DataType, false>(vb_i8x16, ZeroVector, VectorZeroPointB);
}
const auto vc_lo_s16x8 = MlasQLinearMulVectorS16(va_lo_s16x8, vb_lo_s16x8, VectorScaleRatio, VectorZeroPointC);
const auto vc_hi_s16x8 = MlasQLinearMulVectorS16(va_hi_s16x8, vb_hi_s16x8, VectorScaleRatio, VectorZeroPointC);
auto vc = MlasPackS16_128<DataType>(vc_lo_s16x8, vc_hi_s16x8);
if (N >= 16) {
_mm_storeu_si128((__m128i*)OutputC, vc);
OutputC += 16;
N -= 16;
} else {
_mm_storeu_si128((__m128i*)TailDataA, vc);
MlasCopyTailBytes((uint8_t*)OutputC, TailDataA, N);
N = 0;
}
}
}
#else
// Pure C++ implementation.
template<typename DataType, bool IsScalarB>
static
void
MlasQLinearMulKernel(
const DataType* InputA,
float ScaleA,
int32_t ZeroPointA,
const DataType* InputB,
float ScaleB,
int32_t ZeroPointB,
float ScaleC,
int32_t ZeroPointC,
DataType* OutputC,
size_t N
)
{
const float MinimumValue = (float)((int)std::numeric_limits<DataType>::min() - ZeroPointC);
const float MaximumValue = (float)((int)std::numeric_limits<DataType>::max() - ZeroPointC);
float ValueB;
if (IsScalarB) {
ValueB = ScaleB * (int32_t(InputB[0]) - ZeroPointB);
}
for (size_t n = 0; n < N; n++) {
float ValueA = ScaleA * (int32_t(InputA[n]) - ZeroPointA);
if (!IsScalarB) {
ValueB = ScaleB * (int32_t(InputB[n]) - ZeroPointB);
}
float ValueC = (ValueA * ValueB) / ScaleC;
ValueC = std::min(std::max(ValueC, MinimumValue), MaximumValue);
OutputC[n] = (DataType)(int32_t)std::nearbyintf(ValueC + ZeroPointC);
}
}
#endif
template <typename DataType>
void
MLASCALL
MlasQLinearMul(
const DataType* InputA,
float ScaleA,
int32_t ZeroPointA,
const DataType* InputB,
float ScaleB,
int32_t ZeroPointB,
float ScaleC,
int32_t ZeroPointC,
DataType* OutputC,
size_t N,
bool IsScalarB
)
{
if (IsScalarB) {
MlasQLinearMulKernel<DataType, true>(
InputA, ScaleA, ZeroPointA, InputB, ScaleB, ZeroPointB, ScaleC, ZeroPointC, OutputC, N);
} else {
MlasQLinearMulKernel<DataType, false>(
InputA, ScaleA, ZeroPointA, InputB, ScaleB, ZeroPointB, ScaleC, ZeroPointC, OutputC, N);
}
}
// Explicit instantiation
template
void
MlasQLinearMul<uint8_t>(
const uint8_t* InputA,
float ScaleA,
int32_t ZeroPointA,
const uint8_t* InputB,
float ScaleB,
int32_t ZeroPointB,
float ScaleC,
int32_t ZeroPointC,
uint8_t* OutputC,
size_t N,
bool IsScalarB
);
template
void
MlasQLinearMul<int8_t>(
const int8_t* InputA,
float ScaleA,
int32_t ZeroPointA,
const int8_t* InputB,
float ScaleB,
int32_t ZeroPointB,
float ScaleC,
int32_t ZeroPointC,
int8_t* OutputC,
size_t N,
bool IsScalarB
);

183
onnxruntime/test/contrib_ops/qlinear_binary_op_test.cc
View file

@ -128,6 +128,10 @@ static auto add_function = [](float a_dequantized, float b_dequantized) {
return a_dequantized + b_dequantized;
};
static auto mul_function = [](float a_dequantized, float b_dequantized) {
return a_dequantized * b_dequantized;
};
TEST(QLinearBinaryOpTest, AddU8VectorVectorFull) {
const std::vector<float>& A(A4Add);
float A_scale = 8.0f / 256.0f;
@ -368,5 +372,184 @@ TEST(QLinearBinaryOpTest, AddS8VectorScalarBroadcast) {
C_scale, C_zero_point);
}
//
// Tests for QLinearMul
//
TEST(QLinearBinaryOpTest, MulU8VectorVectorFull) {
const std::vector<float>& A(A4Add);
float A_scale = 8.0f / 256.0f;
uint8_t A_zero_point = 128;
const std::vector<float>& B(B4Add);
float B_scale = 8.0f / 256.0f;
uint8_t B_zero_point = 128;
float C_scale = 64.0f / 256.0f;
uint8_t C_zero_point = 128;
RunQLinearMathTestFromFloat("QLinearMul", mul_function,
A, {63}, A_scale, A_zero_point, B, {63}, B_scale, B_zero_point, C_scale, C_zero_point);
}
TEST(QLinearBinaryOpTest, MulU8VectorVectorBroadcast) {
const std::vector<float>& A(A4Add);
float A_scale = 8.0f / 256.0f;
uint8_t A_zero_point = 128;
std::vector<float> B = {
4.00f, 0.25f, 0.00f, -0.25f, 0.50f, -0.25f, -0.00f, 0.25f,
-1.50f, -2.25f, 2.50f, 3.75f, -3.75f, -4.00f, 5.00f, 5.50f,
-0.50f, -1.25f, 0.75f, 0.00f, 2.25f
};
float B_scale = 8.0f / 256.0f;
uint8_t B_zero_point = 128;
float C_scale = 64.0f / 256.0f;
uint8_t C_zero_point = 128;
RunQLinearMathTestFromFloat("QLinearMul", mul_function,
A, {3, 3, 7}, A_scale, A_zero_point, B, {3, 1, 7}, B_scale, B_zero_point, C_scale, C_zero_point);
}
TEST(QLinearBinaryOpTest, MulU8ScalarVectorFull) {
const std::vector<float>& A(A4Add);
float A_scale = 8.0f / 256.0f;
uint8_t A_zero_point = 128;
std::vector<float> B = { 0.25f };
float B_scale = 8.0f / 256.0f;
uint8_t B_zero_point = 96;
float C_scale = 8.0f / 256.0f;
uint8_t C_zero_point = 100;
RunQLinearMathTestFromFloat("QLinearMul", mul_function,
B, {1}, B_scale, B_zero_point, A, {63}, A_scale, A_zero_point, C_scale, C_zero_point);
}
TEST(QLinearBinaryOpTest, MulU8ScalarVectorBroadcast) {
const std::vector<float>& A(A4Add);
float A_scale = 8.0f / 256.0f;
uint8_t A_zero_point = 128;
std::vector<float> B = { 0.25f, -0.25f, -0.00f };
float B_scale = 8.0f / 256.0f;
uint8_t B_zero_point = 96;
float C_scale = 8.0f / 256.0f;
uint8_t C_zero_point = 100;
RunQLinearMathTestFromFloat("QLinearMul", mul_function,
B, {3, 1, 1}, B_scale, B_zero_point, A, {3, 7, 3}, A_scale, A_zero_point, C_scale, C_zero_point);
}
TEST(QLinearBinaryOpTest, MulU8VectorScalarFull) {
const std::vector<float>& A(A4Add);
float A_scale = 8.0f / 256.0f;
uint8_t A_zero_point = 128;
std::vector<float> B = { 0.25f };
float B_scale = 8.0f / 256.0f;
uint8_t B_zero_point = 96;
float C_scale = 16.0f / 256.0f;
uint8_t C_zero_point = 128;
RunQLinearMathTestFromFloat("QLinearMul", mul_function,
A, {63}, A_scale, A_zero_point, B, {1}, B_scale, B_zero_point, C_scale, C_zero_point);
}
TEST(QLinearBinaryOpTest, MulU8VectorScalarBroadcast) {
const std::vector<float>& A(A4Add);
float A_scale = 8.0f / 256.0f;
uint8_t A_zero_point = 128;
std::vector<float> B = { 0.25f, -0.25f, -0.00f };
float B_scale = 8.0f / 256.0f;
uint8_t B_zero_point = 96;
float C_scale = 16.0f / 256.0f;
uint8_t C_zero_point = 128;
RunQLinearMathTestFromFloat("QLinearMul", mul_function,
A, {3, 7, 3}, A_scale, A_zero_point, B, {1, 1, 3}, B_scale, B_zero_point, C_scale, C_zero_point);
}
TEST(QLinearBinaryOpTest, MulS8VectorVectorFull) {
const std::vector<float>& A(A4Add);
float A_scale = 8.0f / 256.0f;
int8_t A_zero_point = 0;
const std::vector<float>& B(B4Add);
float B_scale = 8.0f / 256.0f;
int8_t B_zero_point = 0;
float C_scale = 64.0f / 256.0f;
int8_t C_zero_point = -16;
RunQLinearMathTestFromFloat("QLinearMul", mul_function,
A, {63}, A_scale, A_zero_point, B, {63}, B_scale, B_zero_point, C_scale, C_zero_point);
}
TEST(QLinearBinaryOpTest, MulS8VectorVectorBroadcast) {
const std::vector<float>& A(A4Add);
float A_scale = 8.0f / 256.0f;
int8_t A_zero_point = 0;
std::vector<float> B = {
4.00f, 0.25f, 0.00f, -0.25f, 0.50f, -0.25f, -0.00f, 0.25f,
-1.50f, -2.25f, 2.50f, 3.75f, -3.75f, -4.00f, 5.00f, 5.50f,
-0.50f, -1.25f, 0.75f, 1.25f, 2.25f
};
float B_scale = 8.0f / 256.0f;
int8_t B_zero_point = 0;
float C_scale = 16.0f / 256.0f;
int8_t C_zero_point = -16;
RunQLinearMathTestFromFloat("QLinearMul", mul_function,
A, {3, 3, 7}, A_scale, A_zero_point, B, {3, 1, 7}, B_scale, B_zero_point, C_scale, C_zero_point);
}
TEST(QLinearBinaryOpTest, MulS8ScalarVectorFull) {
const std::vector<float>& A(A4Add);
float A_scale = 8.0f / 256.0f;
int8_t A_zero_point = 0;
std::vector<float> B = { 0.25f };
float B_scale = 2.0f / 256.0f;
int8_t B_zero_point = 16;
float C_scale = 8.0f / 256.0f;
int8_t C_zero_point = 10;
RunQLinearMathTestFromFloat("QLinearMul", mul_function,
B, {1}, B_scale, B_zero_point, A, {63}, A_scale, A_zero_point, C_scale, C_zero_point);
}
TEST(QLinearBinaryOpTest, MulS8ScalarVectorBroadcast) {
const std::vector<float>& A(A4Add);
float A_scale = 8.0f / 256.0f;
int8_t A_zero_point = 0;
std::vector<float> B = { 0.25f, -0.25f, -0.00f };
float B_scale = 2.0f / 256.0f;
int8_t B_zero_point = 16;
float C_scale = 8.0f / 256.0f;
int8_t C_zero_point = 10;
RunQLinearMathTestFromFloat("QLinearMul", mul_function,
B, {3, 1, 1}, B_scale, B_zero_point, A, {3, 7, 3}, A_scale, A_zero_point, C_scale, C_zero_point);
}
TEST(QLinearBinaryOpTest, MulS8VectorScalarFull) {
const std::vector<float>& A(A4Add);
float A_scale = 8.0f / 256.0f;
int8_t A_zero_point = 0;
std::vector<float> B = { 0.25f };
float B_scale = 2.0f / 256.0f;
int8_t B_zero_point = 16;
float C_scale = 8.0f / 256.0f;
int8_t C_zero_point = 10;
RunQLinearMathTestFromFloat("QLinearMul", mul_function,
A, {63}, A_scale, A_zero_point, B, {1}, B_scale, B_zero_point, C_scale, C_zero_point);
}
TEST(QLinearBinaryOpTest, MulS8VectorScalarBroadcast) {
const std::vector<float>& A(A4Add);
float A_scale = 8.0f / 256.0f;
int8_t A_zero_point = 0;
std::vector<float> B = { 0.25f, -0.25f, -0.00f };
float B_scale = 2.0f / 256.0f;
int8_t B_zero_point = 16;
float C_scale = 8.0f / 256.0f;
int8_t C_zero_point = 10;
RunQLinearMathTestFromFloat("QLinearMul", mul_function,
A, {3, 7, 3}, A_scale, A_zero_point, B, {1, 1, 3}, B_scale, B_zero_point, C_scale, C_zero_point);
}
} // namespace test
} // namespace onnxruntime

106
onnxruntime/test/mlas/unittest.cpp
View file

@ -2495,9 +2495,25 @@ public:
}
};
class MlasQLinearAddTest : public MlasTestBase
class MlasQLinearBinaryOpTest : public MlasTestBase
{
public:
typedef void (MLASCALL *QLinearBinaryOpS8)(
const int8_t* InputA, float ScaleA, int32_t ZeroPointA,
const int8_t* InputB, float ScaleB, int32_t ZeroPointB,
float ScaleC, int32_t ZeroPointC, int8_t* OutputC,
size_t N, bool IsScalarB);
typedef void (MLASCALL *QLinearBinaryOpU8)(
const uint8_t* InputA, float ScaleA, int32_t ZeroPointA,
const uint8_t* InputB, float ScaleB, int32_t ZeroPointB,
float ScaleC, int32_t ZeroPointC, uint8_t* OutputC,
size_t N, bool IsScalarB);
private:
std::function<float(float, float)> ScalarOp;
std::string ScalarOpName;
QLinearBinaryOpS8 QLinearS8Op;
QLinearBinaryOpU8 QLinearU8Op;
MatrixGuardBuffer<uint8_t> BufferInputA;
MatrixGuardBuffer<uint8_t> BufferInputB;
MatrixGuardBuffer<uint8_t> BufferOutput;
@ -2505,7 +2521,7 @@ private:
template <typename T>
T
QLinearAddScalar(
QLinearBinaryScalar(
T a,
float ScaleA,
int32_t ZeroPointA,
@ -2516,23 +2532,28 @@ private:
int32_t ZeroPointC
)
{
constexpr int qmax = std::numeric_limits<T>::max();
constexpr int qmin = std::numeric_limits<T>::min();
float ValueA = ScaleA * (static_cast<int>(a) - ZeroPointA);
float ValueB = ScaleB * (static_cast<int>(b) - ZeroPointB);
float ValueC = std::nearbyintf((ValueA + ValueB) / ScaleC) + ZeroPointC;
float ValueC = std::nearbyintf(ScalarOp(ValueA, ValueB) / ScaleC) + ZeroPointC;
int qc = static_cast<int>(ValueC);
qc = std::min(qc, qmax);
qc = std::max(qc, qmin);
return static_cast<T>(qc);
}
template <typename T, bool IsScalarB>
template <typename T>
void
Test(
void (MLASCALL *QLinearBinaryOp)(
const T* InputA, float ScaleA, int32_t ZeroPointA,
const T* InputB, float ScaleB, int32_t ZeroPointB,
float ScaleC, int32_t ZeroPointC, T* OutputC,
size_t N, bool IsScalarB),
size_t N,
bool IsScalarB,
float ScaleA,
int32_t ZeroPointA,
float ScaleB,
@ -2559,17 +2580,18 @@ private:
if (!IsScalarB) {
InputB[n] = static_cast<T>(distribution(generator));
}
OutputReference[n] = QLinearAddScalar(InputA[n], ScaleA, ZeroPointA, InputB[IsScalarB ? 0 : n], ScaleB, ZeroPointB, ScaleC, ZeroPointC);
OutputReference[n] = QLinearBinaryScalar(InputA[n], ScaleA, ZeroPointA, InputB[IsScalarB ? 0 : n], ScaleB, ZeroPointB, ScaleC, ZeroPointC);
}
MlasQLinearAdd(InputA, ScaleA, ZeroPointA, InputB, ScaleB, ZeroPointB, ScaleC, ZeroPointC, OutputC, N, IsScalarB);
QLinearBinaryOp(InputA, ScaleA, ZeroPointA, InputB, ScaleB, ZeroPointB, ScaleC, ZeroPointC, OutputC, N, IsScalarB);
for (size_t n = 0; n < N; n++) {
int diff = (int)OutputC[n] - (int)OutputReference[n];
if (diff < -1 || diff > 1) {
printf("Test IsScalarB=%d difference @%u of %u, %d(%f,%d) + %d(%f,%d) => %d(%f,%d) (expecting %d)\n",
printf("Test IsScalarB=%d difference @%u of %u, %d(%f,%d) %s %d(%f,%d) => %d(%f,%d) (expecting %d)\n",
int(IsScalarB), static_cast<unsigned>(n), static_cast<unsigned>(N),
static_cast<int>(InputA[n]), ScaleA, ZeroPointA,
ScalarOpName.c_str(),
static_cast<int>(InputB[IsScalarB ? 0 : n]), ScaleB, ZeroPointB,
static_cast<int>(OutputC[n]), ScaleC, ZeroPointC,
static_cast<int>(OutputReference[n]));
@ -2578,54 +2600,49 @@ private:
}
public:
explicit MlasQLinearBinaryOpTest(
std::function<float(float, float)> P_ScalarOp,
const std::string& P_ScalarOpName,
QLinearBinaryOpS8 P_QLinearS8Op,
QLinearBinaryOpU8 P_QLinearU8Op
)
: ScalarOp(P_ScalarOp),
ScalarOpName(P_ScalarOpName),
QLinearS8Op(P_QLinearS8Op),
QLinearU8Op(P_QLinearU8Op)
{
}
void
ExecuteShort(
void
) override
{
// uint8_t test
static const uint8_t zero_points[] = { 0, 18, 75, 128, 157, 231, 255 };
static const float c_scales[] = { 18.0f, 90.0f };
const int8_t* s_zero_points = (const int8_t*)(&zero_points[0]);
for (size_t a = 0; a < _countof(zero_points); a++) {
uint8_t offa = zero_points[a];
for (size_t b = 0; b < _countof(zero_points); b++) {
uint8_t offb = zero_points[b];
for (size_t c = 0; c < _countof(zero_points); c++) {
uint8_t offc = zero_points[c];
for (size_t s = 0; s < _countof(c_scales); s++) {
for (size_t n = 1; n < 128; n++) {
// u8, vector + vector
Test<uint8_t>(QLinearU8Op, n, false, 10.f, zero_points[a], 10.f, zero_points[b], c_scales[s], zero_points[c]);
for (size_t n = 1; n < 128; n++) {
// vector + vector
Test<uint8_t, false>(n, 10.f, offa, 10.f, offb, 20.f, offc);
// u8, vector + scalar
Test<uint8_t>(QLinearU8Op, n, true, 10.f, zero_points[a], 10.f, zero_points[b], c_scales[s], zero_points[c]);
// vector + scalar
Test<uint8_t, true>(n, 10.f, offa, 10.f, offb, 20.f, offc);
// s8, vector + vector
Test<int8_t>(QLinearS8Op, n, false, 10.f, s_zero_points[a], 10.f, s_zero_points[b], c_scales[s], s_zero_points[c]);
// s8, vector + scalar
Test<int8_t>(QLinearS8Op, n, true, 10.f, s_zero_points[a], 10.f, s_zero_points[b], c_scales[s], s_zero_points[c]);
}
}
}
}
}
static const int8_t szero_points[] = { -128, -110, -53, 0, 29, 103, 127 };
for (size_t a = 0; a < _countof(zero_points); a++) {
int8_t offa = szero_points[a];
for (size_t b = 0; b < _countof(zero_points); b++) {
int8_t offb = szero_points[b];
for (size_t c = 0; c < _countof(zero_points); c++) {
int8_t offc = szero_points[c];
for (size_t n = 1; n < 128; n++) {
// vector + vector
Test<int8_t, false>(n, 10.f, offa, 10.f, offb, 20.f, offc);
// vector + scalar
Test<int8_t, true>(n, 10.f, offa, 10.f, offb, 20.f, offc);
}
}
}
}
}
};
@ -2787,7 +2804,12 @@ main(
}
printf("QLinearAdd tests.\n");
onnxruntime::make_unique<MlasQLinearAddTest>()->ExecuteShort();
onnxruntime::make_unique<MlasQLinearBinaryOpTest>(
[](float a, float b) { return a + b; }, "+", MlasQLinearAdd<int8_t>, MlasQLinearAdd<uint8_t>)->ExecuteShort();
printf("QLinearMul tests.\n");
onnxruntime::make_unique<MlasQLinearBinaryOpTest>(
[] (float a, float b) { return a * b; }, "*", MlasQLinearMul<int8_t>, MlasQLinearMul<uint8_t>)->ExecuteShort();
printf("Done.\n");