From ec88f14a7a4d5a058135ed63ab6d7e9df0d34e9c Mon Sep 17 00:00:00 2001 From: Zhang Lei Date: Fri, 4 Sep 2020 15:02:19 -0700 Subject: [PATCH] Implement QLinearMul in mlas (#4593) * Implement QLinearMul --- cmake/onnxruntime_mlas.cmake | 1 + .../contrib_ops/cpu/cpu_contrib_kernels.cc | 4 + .../contrib_ops/cpu/qlinear_binary_op.cc | 27 ++ .../contrib_ops/cpu/qlinear_binary_op.h | 8 + onnxruntime/core/mlas/inc/mlas.h | 17 + .../mlas/lib/intrinsics/avx2/qladd_avx2.cpp | 5 +- onnxruntime/core/mlas/lib/qladd.cpp | 301 +------------- onnxruntime/core/mlas/lib/qladd.h | 322 +++++++++++++++ onnxruntime/core/mlas/lib/qlmul.cpp | 385 ++++++++++++++++++ .../contrib_ops/qlinear_binary_op_test.cc | 183 +++++++++ onnxruntime/test/mlas/unittest.cpp | 106 +++-- 11 files changed, 1020 insertions(+), 339 deletions(-) create mode 100644 onnxruntime/core/mlas/lib/qlmul.cpp diff --git a/cmake/onnxruntime_mlas.cmake b/cmake/onnxruntime_mlas.cmake index b6cbdae02a..ef5d4b8bc7 100644 --- a/cmake/onnxruntime_mlas.cmake +++ b/cmake/onnxruntime_mlas.cmake @@ -18,6 +18,7 @@ set(mlas_common_srcs ${ONNXRUNTIME_ROOT}/core/mlas/lib/compute.cpp ${ONNXRUNTIME_ROOT}/core/mlas/lib/quantize.cpp ${ONNXRUNTIME_ROOT}/core/mlas/lib/qladd.cpp + ${ONNXRUNTIME_ROOT}/core/mlas/lib/qlmul.cpp ) if(MSVC) diff --git a/onnxruntime/contrib_ops/cpu/cpu_contrib_kernels.cc b/onnxruntime/contrib_ops/cpu/cpu_contrib_kernels.cc index 49ba8f163f..3df764b20a 100644 --- a/onnxruntime/contrib_ops/cpu/cpu_contrib_kernels.cc +++ b/onnxruntime/contrib_ops/cpu/cpu_contrib_kernels.cc @@ -45,6 +45,8 @@ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, int8_t, QLinearSigmoid); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, QLinearAdd); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, int8_t, QLinearAdd); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, QLinearMul); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, int8_t, QLinearMul); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, QAttention); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, DynamicQuantizeMatMul); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, MatMulIntegerToFloat); @@ -121,6 +123,8 @@ Status RegisterQuantizationKernels(KernelRegistry& kernel_registry) { BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, diff --git a/onnxruntime/contrib_ops/cpu/qlinear_binary_op.cc b/onnxruntime/contrib_ops/cpu/qlinear_binary_op.cc index c6fe78dac0..b07cd8625b 100644 --- a/onnxruntime/contrib_ops/cpu/qlinear_binary_op.cc +++ b/onnxruntime/contrib_ops/cpu/qlinear_binary_op.cc @@ -137,6 +137,31 @@ Status QLinearAdd::Compute(OpKernelContext* context) const { 1.0); } +template +Status QLinearMul::Compute(OpKernelContext* context) const { + return QLinearBroadcastTwo( + *context, + [](gsl::span output, const T& input0, gsl::span input1, + float A_scale, float B_scale, float C_scale, T A_zero_point, T B_zero_point, T C_zero_point) { + MlasQLinearMul(input1.data(), B_scale, B_zero_point, + &input0, A_scale, A_zero_point, + C_scale, C_zero_point, output.data(), output.size(), true); + }, + [](gsl::span output, gsl::span input0, const T& input1, + float A_scale, float B_scale, float C_scale, T A_zero_point, T B_zero_point, T C_zero_point) { + MlasQLinearMul(input0.data(), A_scale, A_zero_point, + &input1, B_scale, B_zero_point, + C_scale, C_zero_point, output.data(), output.size(), true); + }, + [](gsl::span output, gsl::span input0, gsl::span input1, + float A_scale, float B_scale, float C_scale, T A_zero_point, T B_zero_point, T C_zero_point) { + MlasQLinearMul(input0.data(), A_scale, A_zero_point, + input1.data(), B_scale, B_zero_point, + C_scale, C_zero_point, output.data(), output.size(), false); + }, + 1.0); +} + #define REG_QLINEAR_ELEMENTWISE_TYPED_KERNEL(op_name, version, data_type, KERNEL_CLASS) \ ONNX_CPU_OPERATOR_TYPED_MS_KERNEL( \ op_name, version, data_type, \ @@ -146,6 +171,8 @@ Status QLinearAdd::Compute(OpKernelContext* context) const { REG_QLINEAR_ELEMENTWISE_TYPED_KERNEL(QLinearAdd, 1, int8_t, QLinearAdd); REG_QLINEAR_ELEMENTWISE_TYPED_KERNEL(QLinearAdd, 1, uint8_t, QLinearAdd); +REG_QLINEAR_ELEMENTWISE_TYPED_KERNEL(QLinearMul, 1, int8_t, QLinearMul); +REG_QLINEAR_ELEMENTWISE_TYPED_KERNEL(QLinearMul, 1, uint8_t, QLinearMul); } // namespace contrib } // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cpu/qlinear_binary_op.h b/onnxruntime/contrib_ops/cpu/qlinear_binary_op.h index b3df924741..cbadccd207 100644 --- a/onnxruntime/contrib_ops/cpu/qlinear_binary_op.h +++ b/onnxruntime/contrib_ops/cpu/qlinear_binary_op.h @@ -18,5 +18,13 @@ class QLinearAdd final : public OpKernel { Status Compute(OpKernelContext* context) const override; }; +template +class QLinearMul final : public OpKernel { + public: + QLinearMul(const OpKernelInfo& info) : OpKernel(info) {} + + Status Compute(OpKernelContext* context) const override; +}; + } // namespace contrib } // namespace onnxruntime diff --git a/onnxruntime/core/mlas/inc/mlas.h b/onnxruntime/core/mlas/inc/mlas.h index 77a6decc06..1fb137d135 100644 --- a/onnxruntime/core/mlas/inc/mlas.h +++ b/onnxruntime/core/mlas/inc/mlas.h @@ -554,3 +554,20 @@ MlasQLinearAdd( size_t N, bool IsScalarB ); + +template +void +MLASCALL +MlasQLinearMul( + const DataType* InputA, + float ScaleA, + int32_t ZeroPointA, + const DataType* InputB, + float ScaleB, + int32_t ZeroPointB, + float ScaleC, + int32_t ZeroPointC, + DataType* OutputC, + size_t N, + bool IsScalarB + ); diff --git a/onnxruntime/core/mlas/lib/intrinsics/avx2/qladd_avx2.cpp b/onnxruntime/core/mlas/lib/intrinsics/avx2/qladd_avx2.cpp index 266b289046..30e09e7a4a 100644 --- a/onnxruntime/core/mlas/lib/intrinsics/avx2/qladd_avx2.cpp +++ b/onnxruntime/core/mlas/lib/intrinsics/avx2/qladd_avx2.cpp @@ -18,7 +18,6 @@ Abstract: --*/ -#include "../../mlasi.h" #include "../../qladd.h" template @@ -30,6 +29,7 @@ MlasShiftRight24Epi32( ); template <> +MLAS_FORCEINLINE __m256i MlasShiftRight24Epi32( __m256i v @@ -39,6 +39,7 @@ MlasShiftRight24Epi32( } template <> +MLAS_FORCEINLINE __m256i MlasShiftRight24Epi32( __m256i v @@ -57,6 +58,7 @@ MlasPackS16_256( ); template <> +MLAS_FORCEINLINE __m256i MlasPackS16_256( __m256i a, @@ -67,6 +69,7 @@ MlasPackS16_256( } template <> +MLAS_FORCEINLINE __m256i MlasPackS16_256( __m256i a, diff --git a/onnxruntime/core/mlas/lib/qladd.cpp b/onnxruntime/core/mlas/lib/qladd.cpp index 8e8818b196..62d2433759 100644 --- a/onnxruntime/core/mlas/lib/qladd.cpp +++ b/onnxruntime/core/mlas/lib/qladd.cpp @@ -88,232 +88,6 @@ MlasQLinearAddKernelRawHelper( #if defined(MLAS_NEON_INTRINSICS) -#if ! defined(_MSC_VER) - -#define vld1q_s8_ex(pD, align) vld1q_s8((int8_t*)__builtin_assume_aligned(pD, ((align)/8))) -#define vst1_s8_ex(pD, D, align) vst1_s8((int8_t*)__builtin_assume_aligned(pD, ((align)/8)), D) -#define vst1q_s8_ex(pD, D, align) vst1q_s8((int8_t*)__builtin_assume_aligned(pD, ((align)/8)), D) -#define vld1q_u8_ex(pD, align) vld1q_u8((uint8_t*)__builtin_assume_aligned(pD, ((align)/8))) -#define vst1_u8_ex(pD, D, align) vst1_u8((uint8_t*)__builtin_assume_aligned(pD, ((align)/8)), D) -#define vst1q_u8_ex(pD, D, align) vst1q_u8((uint8_t*)__builtin_assume_aligned(pD, ((align)/8)), D) -#define vst1_lane_u32_ex(pD, D, lane, align) vst1_lane_u32((uint32_t*)__builtin_assume_aligned(pD, ((align)/8)), D, lane) -#define vst1_lane_u16_ex(pD, D, lane, align) vst1_lane_u16((uint16_t*)__builtin_assume_aligned(pD, ((align)/8)), D, lane) - -#endif - -template -class MLAS_SignedUnsignedIntOps; - -template <> -class MLAS_SignedUnsignedIntOps -{ -public: - typedef uint8_t T; - typedef uint8x8_t i8x8_t; - typedef uint8x16_t i8x16_t; - typedef uint16x8_t i16x8_t; - - static MLAS_FORCEINLINE i8x8_t vmov_n_i8(T value) - { - return vmov_n_u8(value); - } - - static MLAS_FORCEINLINE i8x8_t vget_low_i8(i8x16_t a) - { - return vget_low_u8(a); - } - - static MLAS_FORCEINLINE i8x8_t vget_high_i8(i8x16_t a) - { - return vget_high_u8(a); - } - - static MLAS_FORCEINLINE i16x8_t vsubl_i8(i8x8_t a, i8x8_t b) - { - return vsubl_u8(a, b); - } - - static MLAS_FORCEINLINE int16x8_t vreinterpretq_s16_i16(i16x8_t a) - { - return vreinterpretq_s16_u16(a); - } - - static MLAS_FORCEINLINE uint32x4_t vreinterpretq_u32_i8(i8x16_t a) - { - return vreinterpretq_u32_u8(a); - } - - static MLAS_FORCEINLINE uint16x8_t vreinterpretq_u16_i8(i8x16_t a) - { - return vreinterpretq_u16_u8(a); - } - - static MLAS_FORCEINLINE uint32x2_t vreinterpret_u32_i8(i8x8_t a) - { - return vreinterpret_u32_u8(a); - } - - static MLAS_FORCEINLINE uint16x4_t vreinterpret_u16_i8(i8x8_t a) - { - return vreinterpret_u16_u8(a); - } - - static MLAS_FORCEINLINE i8x16_t vld1q_i8(T const * ptr) - { - return vld1q_u8_ex(ptr, 8); - } - - static MLAS_FORCEINLINE void vst1_i8(T* ptr, i8x8_t a) - { - vst1_u8_ex(ptr, a, 8); - } - - static MLAS_FORCEINLINE void vst1q_i8(T* ptr, i8x16_t a) - { - vst1q_u8_ex(ptr, a, 8); - } - - template - static MLAS_FORCEINLINE void vst1_lane_i8(T* ptr, i8x8_t a) - { - vst1_lane_u8(ptr, a, n); - } - - template - static MLAS_FORCEINLINE i8x16_t vextq_i8(i8x16_t lo, i8x16_t hi) - { - return vextq_u8(lo, hi, n); - } - - template - static MLAS_FORCEINLINE i8x8_t vext_i8(i8x8_t lo, i8x8_t hi) - { - return vext_u8(lo, hi, n); - } - - static MLAS_FORCEINLINE i8x16_t combine_i8_s16(int16x8_t v0, int16x8_t v1) - { - -#if defined(MLAS_NEON64_INTRINSICS) - return vqmovun_high_s16(vqmovun_s16(v0), v1); -#else - return vcombine_u8(vqmovun_s16(v0), vqmovun_s16(v1)); -#endif - - } -}; - -template <> -class MLAS_SignedUnsignedIntOps -{ -public: - typedef int8_t T; - typedef int8x8_t i8x8_t; - typedef int8x16_t i8x16_t; - typedef int16x8_t i16x8_t; - - static MLAS_FORCEINLINE i8x8_t vmov_n_i8(T value) - { - return vmov_n_s8(value); - } - - static MLAS_FORCEINLINE i8x8_t vget_low_i8(i8x16_t a) - { - return vget_low_s8(a); - } - - static MLAS_FORCEINLINE i8x8_t vget_high_i8(i8x16_t a) - { - return vget_high_s8(a); - } - - static MLAS_FORCEINLINE i16x8_t vsubl_i8(i8x8_t a, i8x8_t b) - { - return vsubl_s8(a, b); - } - - static MLAS_FORCEINLINE int16x8_t vreinterpretq_s16_i16(i16x8_t a) - { - return a; - } - - static MLAS_FORCEINLINE uint32x4_t vreinterpretq_u32_i8(i8x16_t a) - { - return vreinterpretq_u32_s8(a); - } - - static MLAS_FORCEINLINE uint16x8_t vreinterpretq_u16_i8(i8x16_t a) - { - return vreinterpretq_u16_s8(a); - } - - static MLAS_FORCEINLINE uint32x2_t vreinterpret_u32_i8(i8x8_t a) - { - return vreinterpret_u32_s8(a); - } - - static MLAS_FORCEINLINE uint16x4_t vreinterpret_u16_i8(i8x8_t a) - { - return vreinterpret_u16_s8(a); - } - - static MLAS_FORCEINLINE i8x16_t vld1q_i8(T const * ptr) - { - return vld1q_s8_ex(ptr, 8); - } - - static MLAS_FORCEINLINE void vst1_i8(T* ptr, i8x8_t a) - { - vst1_s8_ex(ptr, a, 8); - } - - static MLAS_FORCEINLINE void vst1q_i8(T* ptr, i8x16_t a) - { - vst1q_s8_ex(ptr, a, 8); - } - - template - static MLAS_FORCEINLINE void vst1_lane_i8(T* ptr, i8x8_t a) - { - vst1_lane_s8(ptr, a, n); - } - - template - static MLAS_FORCEINLINE i8x16_t vextq_i8(i8x16_t lo, i8x16_t hi) - { - return vextq_s8(lo, hi, n); - } - - template - static MLAS_FORCEINLINE i8x8_t vext_i8(i8x8_t lo, i8x8_t hi) - { - return vext_s8(lo, hi, n); - } - - static MLAS_FORCEINLINE i8x16_t combine_i8_s16(int16x8_t v0, int16x8_t v1) - { - -#if defined(MLAS_NEON64_INTRINSICS) - return vqmovn_high_s16(vqmovn_s16(v0), v1); -#else - return vcombine_s8(vqmovn_s16(v0), vqmovn_s16(v1)); -#endif - - } -}; - -#if defined(MLAS_NEON64_INTRINSICS) - -#define MlasMoveHighS16S32(s16x8) vmovl_high_s16(s16x8) -#define MlasCombineS16S32(lo, hi) vqmovn_high_s32(vqmovn_s32(lo), hi) - -#else - -#define MlasMoveHighS16S32(s16x8) vmovl_s16(vget_high_s16(s16x8)) -#define MlasCombineS16S32(lo, hi) vcombine_s16(vqmovn_s32(lo), vqmovn_s32(hi)) - -#endif - template static void @@ -512,7 +286,6 @@ MlasQLinearAddKernelHelper( int32x4_t vacc0_lo, vacc1_lo, vacc0_hi, vacc1_hi; if (IsScalarB) { const typename SUI::i8x16_t VectorA0 = SUI::vld1q_i8(TailDataA); - InputA += 16; const int16x8_t va0_s16x8 = SUI::vreinterpretq_s16_i16(SUI::vsubl_i8(SUI::vget_low_i8(VectorA0), VectorZeroPointA)); const int16x8_t va1_s16x8 = SUI::vreinterpretq_s16_i16(SUI::vsubl_i8(SUI::vget_high_i8(VectorA0), VectorZeroPointA)); @@ -523,8 +296,6 @@ MlasQLinearAddKernelHelper( } else { const typename SUI::i8x16_t VectorA0 = SUI::vld1q_i8(TailDataA); const typename SUI::i8x16_t VectorB0 = SUI::vld1q_i8(TailDataB); - InputA += 16; - InputB += 16; const int16x8_t va0_s16x8 = SUI::vreinterpretq_s16_i16(SUI::vsubl_i8(SUI::vget_low_i8(VectorA0), VectorZeroPointA)); const int16x8_t vb0_s16x8 = SUI::vreinterpretq_s16_i16(SUI::vsubl_i8(SUI::vget_low_i8(VectorB0), VectorZeroPointB)); const int16x8_t va1_s16x8 = SUI::vreinterpretq_s16_i16(SUI::vsubl_i8(SUI::vget_high_i8(VectorA0), VectorZeroPointA)); @@ -580,64 +351,6 @@ MlasQLinearAddKernelHelper( #elif defined(MLAS_SSE2_INTRINSICS) -template -MLAS_FORCEINLINE -static -MLAS_INT32X4 -MlasShiftRightInt32( - MLAS_INT32X4 v, - int imm - ); - -template<> -MLAS_INT32X4 -MlasShiftRightInt32( - MLAS_INT32X4 v, - int imm - ) -{ - return _mm_srai_epi32(v, imm); -} - -template<> -MLAS_INT32X4 -MlasShiftRightInt32( - MLAS_INT32X4 v, - int imm - ) -{ - return _mm_srli_epi32(v, imm); -} - -template -MLAS_FORCEINLINE -static -MLAS_INT32X4 -MlasPackS16_128( - MLAS_INT32X4 a, - MLAS_INT32X4 b - ); - -template <> -MLAS_INT32X4 -MlasPackS16_128( - MLAS_INT32X4 a, - MLAS_INT32X4 b - ) -{ - return _mm_packus_epi16(a, b); -} - -template <> -MLAS_INT32X4 -MlasPackS16_128( - MLAS_INT32X4 a, - MLAS_INT32X4 b - ) -{ - return _mm_packs_epi16(a, b); -} - template static void @@ -700,20 +413,16 @@ MlasQLinearAddKernelHelper( if (N > 0) { uint8_t TailData[8] = { 0 }; - { - MlasCopyTailBytes(TailData, (const uint8_t*)InputA, N); - const auto va_low_half = _mm_loadl_epi64((const MLAS_INT32X4*)TailData); - const auto va_i16x8 = _mm_unpacklo_epi8(va_low_half, va_low_half); - InputA += 8; - va_lo = _mm_cvtepi32_ps(MlasShiftRightInt32(_mm_unpacklo_epi16(va_i16x8, va_i16x8), 24)); - va_hi = _mm_cvtepi32_ps(MlasShiftRightInt32(_mm_unpackhi_epi16(va_i16x8, va_i16x8), 24)); - } + MlasCopyTailBytes(TailData, (const uint8_t*)InputA, N); + const auto va_low_half = _mm_loadl_epi64((const MLAS_INT32X4*)TailData); + const auto va_i16x8 = _mm_unpacklo_epi8(va_low_half, va_low_half); + va_lo = _mm_cvtepi32_ps(MlasShiftRightInt32(_mm_unpacklo_epi16(va_i16x8, va_i16x8), 24)); + va_hi = _mm_cvtepi32_ps(MlasShiftRightInt32(_mm_unpackhi_epi16(va_i16x8, va_i16x8), 24)); if (!IsScalarB) { MlasCopyTailBytes(TailData, (const uint8_t*)InputB, N); const auto vb_low_half = _mm_loadl_epi64((const MLAS_INT32X4*)TailData); const auto vb_i16x8 = _mm_unpacklo_epi8(vb_low_half, vb_low_half); - InputB += 8; vb_lo = _mm_cvtepi32_ps(MlasShiftRightInt32(_mm_unpacklo_epi16(vb_i16x8, vb_i16x8), 24)); vb_hi = _mm_cvtepi32_ps(MlasShiftRightInt32(_mm_unpackhi_epi16(vb_i16x8, vb_i16x8), 24)); } diff --git a/onnxruntime/core/mlas/lib/qladd.h b/onnxruntime/core/mlas/lib/qladd.h index 5ce00907c0..8a2e148c49 100644 --- a/onnxruntime/core/mlas/lib/qladd.h +++ b/onnxruntime/core/mlas/lib/qladd.h @@ -76,3 +76,325 @@ MlasCalcQLinearAddParameters( int32_t& MultiplierA, int32_t& MultiplierB ); + +#if defined(MLAS_NEON_INTRINSICS) + +#if ! defined(_MSC_VER) + +#define vld1q_s8_ex(pD, align) vld1q_s8((int8_t*)__builtin_assume_aligned(pD, ((align)/8))) +#define vst1_s8_ex(pD, D, align) vst1_s8((int8_t*)__builtin_assume_aligned(pD, ((align)/8)), D) +#define vst1q_s8_ex(pD, D, align) vst1q_s8((int8_t*)__builtin_assume_aligned(pD, ((align)/8)), D) +#define vld1q_u8_ex(pD, align) vld1q_u8((uint8_t*)__builtin_assume_aligned(pD, ((align)/8))) +#define vst1_u8_ex(pD, D, align) vst1_u8((uint8_t*)__builtin_assume_aligned(pD, ((align)/8)), D) +#define vst1q_u8_ex(pD, D, align) vst1q_u8((uint8_t*)__builtin_assume_aligned(pD, ((align)/8)), D) +#define vst1_lane_u32_ex(pD, D, lane, align) vst1_lane_u32((uint32_t*)__builtin_assume_aligned(pD, ((align)/8)), D, lane) +#define vst1_lane_u16_ex(pD, D, lane, align) vst1_lane_u16((uint16_t*)__builtin_assume_aligned(pD, ((align)/8)), D, lane) + +#endif + +template +class MLAS_SignedUnsignedIntOps; + +template <> +class MLAS_SignedUnsignedIntOps +{ +public: + typedef uint8_t T; + typedef uint8x8_t i8x8_t; + typedef uint8x16_t i8x16_t; + typedef uint16x8_t i16x8_t; + + static MLAS_FORCEINLINE i8x8_t vmov_n_i8(T value) + { + return vmov_n_u8(value); + } + + static MLAS_FORCEINLINE i8x8_t vget_low_i8(i8x16_t a) + { + return vget_low_u8(a); + } + + static MLAS_FORCEINLINE i8x8_t vget_high_i8(i8x16_t a) + { + return vget_high_u8(a); + } + + static MLAS_FORCEINLINE i16x8_t vsubl_i8(i8x8_t a, i8x8_t b) + { + return vsubl_u8(a, b); + } + + static MLAS_FORCEINLINE int16x8_t vreinterpretq_s16_i16(i16x8_t a) + { + return vreinterpretq_s16_u16(a); + } + + static MLAS_FORCEINLINE uint32x4_t vreinterpretq_u32_i8(i8x16_t a) + { + return vreinterpretq_u32_u8(a); + } + + static MLAS_FORCEINLINE uint16x8_t vreinterpretq_u16_i8(i8x16_t a) + { + return vreinterpretq_u16_u8(a); + } + + static MLAS_FORCEINLINE uint32x2_t vreinterpret_u32_i8(i8x8_t a) + { + return vreinterpret_u32_u8(a); + } + + static MLAS_FORCEINLINE uint16x4_t vreinterpret_u16_i8(i8x8_t a) + { + return vreinterpret_u16_u8(a); + } + + static MLAS_FORCEINLINE i8x16_t vld1q_i8(T const * ptr) + { + return vld1q_u8_ex(ptr, 8); + } + + static MLAS_FORCEINLINE void vst1_i8(T* ptr, i8x8_t a) + { + vst1_u8_ex(ptr, a, 8); + } + + static MLAS_FORCEINLINE void vst1q_i8(T* ptr, i8x16_t a) + { + vst1q_u8_ex(ptr, a, 8); + } + + template + static MLAS_FORCEINLINE void vst1_lane_i8(T* ptr, i8x8_t a) + { + vst1_lane_u8(ptr, a, n); + } + + template + static MLAS_FORCEINLINE i8x16_t vextq_i8(i8x16_t lo, i8x16_t hi) + { + return vextq_u8(lo, hi, n); + } + + template + static MLAS_FORCEINLINE i8x8_t vext_i8(i8x8_t lo, i8x8_t hi) + { + return vext_u8(lo, hi, n); + } + + static MLAS_FORCEINLINE i8x16_t combine_i8_s16(int16x8_t v0, int16x8_t v1) + { + +#if defined(MLAS_NEON64_INTRINSICS) + return vqmovun_high_s16(vqmovun_s16(v0), v1); +#else + return vcombine_u8(vqmovun_s16(v0), vqmovun_s16(v1)); +#endif + + } +}; + +template <> +class MLAS_SignedUnsignedIntOps +{ +public: + typedef int8_t T; + typedef int8x8_t i8x8_t; + typedef int8x16_t i8x16_t; + typedef int16x8_t i16x8_t; + + static MLAS_FORCEINLINE i8x8_t vmov_n_i8(T value) + { + return vmov_n_s8(value); + } + + static MLAS_FORCEINLINE i8x8_t vget_low_i8(i8x16_t a) + { + return vget_low_s8(a); + } + + static MLAS_FORCEINLINE i8x8_t vget_high_i8(i8x16_t a) + { + return vget_high_s8(a); + } + + static MLAS_FORCEINLINE i16x8_t vsubl_i8(i8x8_t a, i8x8_t b) + { + return vsubl_s8(a, b); + } + + static MLAS_FORCEINLINE int16x8_t vreinterpretq_s16_i16(i16x8_t a) + { + return a; + } + + static MLAS_FORCEINLINE uint32x4_t vreinterpretq_u32_i8(i8x16_t a) + { + return vreinterpretq_u32_s8(a); + } + + static MLAS_FORCEINLINE uint16x8_t vreinterpretq_u16_i8(i8x16_t a) + { + return vreinterpretq_u16_s8(a); + } + + static MLAS_FORCEINLINE uint32x2_t vreinterpret_u32_i8(i8x8_t a) + { + return vreinterpret_u32_s8(a); + } + + static MLAS_FORCEINLINE uint16x4_t vreinterpret_u16_i8(i8x8_t a) + { + return vreinterpret_u16_s8(a); + } + + static MLAS_FORCEINLINE i8x16_t vld1q_i8(T const * ptr) + { + return vld1q_s8_ex(ptr, 8); + } + + static MLAS_FORCEINLINE void vst1_i8(T* ptr, i8x8_t a) + { + vst1_s8_ex(ptr, a, 8); + } + + static MLAS_FORCEINLINE void vst1q_i8(T* ptr, i8x16_t a) + { + vst1q_s8_ex(ptr, a, 8); + } + + template + static MLAS_FORCEINLINE void vst1_lane_i8(T* ptr, i8x8_t a) + { + vst1_lane_s8(ptr, a, n); + } + + template + static MLAS_FORCEINLINE i8x16_t vextq_i8(i8x16_t lo, i8x16_t hi) + { + return vextq_s8(lo, hi, n); + } + + template + static MLAS_FORCEINLINE i8x8_t vext_i8(i8x8_t lo, i8x8_t hi) + { + return vext_s8(lo, hi, n); + } + + static MLAS_FORCEINLINE i8x16_t combine_i8_s16(int16x8_t v0, int16x8_t v1) + { + +#if defined(MLAS_NEON64_INTRINSICS) + return vqmovn_high_s16(vqmovn_s16(v0), v1); +#else + return vcombine_s8(vqmovn_s16(v0), vqmovn_s16(v1)); +#endif + + } +}; + +#if defined(MLAS_NEON64_INTRINSICS) + +#define MlasMoveHighS16S32(s16x8) vmovl_high_s16(s16x8) +#define MlasCombineS16S32(lo, hi) vqmovn_high_s32(vqmovn_s32(lo), hi) + +#else + +#define MlasMoveHighS16S32(s16x8) vmovl_s16(vget_high_s16(s16x8)) +#define MlasCombineS16S32(lo, hi) vcombine_s16(vqmovn_s32(lo), vqmovn_s32(hi)) + +#endif + +#elif defined(MLAS_SSE2_INTRINSICS) + +template +MLAS_FORCEINLINE +MLAS_INT32X4 +MlasShiftRightInt32( + MLAS_INT32X4 v, + int imm + ); + +template<> +MLAS_FORCEINLINE +MLAS_INT32X4 +MlasShiftRightInt32( + MLAS_INT32X4 v, + int imm + ) +{ + return _mm_srai_epi32(v, imm); +} + +template<> +MLAS_FORCEINLINE +MLAS_INT32X4 +MlasShiftRightInt32( + MLAS_INT32X4 v, + int imm + ) +{ + return _mm_srli_epi32(v, imm); +} + +template +MLAS_FORCEINLINE +MLAS_INT32X4 +MlasShiftRightInt16( + MLAS_INT32X4 v, + int imm + ); + +template<> +MLAS_FORCEINLINE +MLAS_INT32X4 +MlasShiftRightInt16( + MLAS_INT32X4 v, + int imm + ) +{ + return _mm_srai_epi16(v, imm); +} + +template<> +MLAS_FORCEINLINE +MLAS_INT32X4 +MlasShiftRightInt16( + MLAS_INT32X4 v, + int imm + ) +{ + return _mm_srli_epi16(v, imm); +} + +template +MLAS_FORCEINLINE +MLAS_INT32X4 +MlasPackS16_128( + MLAS_INT32X4 a, + MLAS_INT32X4 b + ); + +template <> +MLAS_FORCEINLINE +MLAS_INT32X4 +MlasPackS16_128( + MLAS_INT32X4 a, + MLAS_INT32X4 b + ) +{ + return _mm_packus_epi16(a, b); +} + +template <> +MLAS_FORCEINLINE +MLAS_INT32X4 +MlasPackS16_128( + MLAS_INT32X4 a, + MLAS_INT32X4 b + ) +{ + return _mm_packs_epi16(a, b); +} + +#endif diff --git a/onnxruntime/core/mlas/lib/qlmul.cpp b/onnxruntime/core/mlas/lib/qlmul.cpp new file mode 100644 index 0000000000..a1d881db34 --- /dev/null +++ b/onnxruntime/core/mlas/lib/qlmul.cpp @@ -0,0 +1,385 @@ +/*++ + +Copyright (c) Microsoft Corporation. All rights reserved. + +Licensed under the MIT License. + +Module Name: + + qlmul.cpp + +Abstract: + + This module implements routines to quantize linear mul. + + For quantization formula as specified in the ONNX operator documentation is: + + Output = Saturate(RoundToEven(Input / Scale) + ZeroPoint) + +--*/ + +#include "qladd.h" + +#if defined(MLAS_NEON64_INTRINSICS) + +template +MLAS_FORCEINLINE +static +int16x8_t +MlasExtendToS16Debias( + typename SUI::i8x16_t Int8Vector, + typename SUI::i8x8_t VectorBias + ) +{ + auto HalfVector = IsLow ? SUI::vget_low_i8(Int8Vector) : SUI::vget_high_i8(Int8Vector); + return SUI::vreinterpretq_s16_i16(SUI::vsubl_i8(HalfVector, VectorBias)); +} + +MLAS_FORCEINLINE +static +int16x8_t +MlasQLinearMulVectorS16( + int16x8_t va_s16x8, + int16x8_t vb_s16x8, + float32x4_t VectorScaleRatio, + float32x4_t VectorZeroPointC + ) +{ + int32x4_t vacc0_lo = vmull_s16(vget_low_s16(va_s16x8), vget_low_s16(vb_s16x8)); + int32x4_t vacc0_hi = vmull_s16(vget_high_s16(va_s16x8), vget_high_s16(vb_s16x8)); + auto vacc0_lo_f32 = vaddq_f32(VectorZeroPointC, vmulq_f32(VectorScaleRatio, vcvtq_f32_s32(vacc0_lo))); + auto vacc0_hi_f32 = vaddq_f32(VectorZeroPointC, vmulq_f32(VectorScaleRatio, vcvtq_f32_s32(vacc0_hi))); + // using rounding to nearst, ties to even + vacc0_lo = vcvtnq_s32_f32(vacc0_lo_f32); + vacc0_hi = vcvtnq_s32_f32(vacc0_hi_f32); + // Pack and saturate. + return vcombine_s16(vqmovn_s32(vacc0_lo), vqmovn_s32(vacc0_hi)); +} + +template +static +void +MlasQLinearMulKernel( + const DataType* InputA, + float ScaleA, + int32_t ZeroPointA, + const DataType* InputB, + float ScaleB, + int32_t ZeroPointB, + float ScaleC, + int32_t ZeroPointC, + DataType* OutputC, + size_t N + ) +{ + typedef MLAS_SignedUnsignedIntOps SUI; + + const float32x4_t VectorScaleRatio = vmovq_n_f32(ScaleA * ScaleB / ScaleC); + const typename SUI::i8x8_t VectorZeroPointA = SUI::vmov_n_i8((DataType)ZeroPointA); + const typename SUI::i8x8_t VectorZeroPointB = SUI::vmov_n_i8((DataType)ZeroPointB); + const float32x4_t VectorZeroPointC = vmovq_n_f32((float)ZeroPointC); + + typename SUI::T TailDataA[16] = { 0 }; + typename SUI::T TailDataB[16] = { 0 }; + int16x8_t vb0_s16x8, vb1_s16x8; + if (IsScalarB) { + const typename SUI::i8x8_t VectorB0 = SUI::vmov_n_i8(*InputB); + vb0_s16x8 = SUI::vreinterpretq_s16_i16(SUI::vsubl_i8(VectorB0, VectorZeroPointB)); + vb1_s16x8 = vb0_s16x8; + } + + while (N > 0) { + if (N < 16) { + MlasCopyTailBytes((uint8_t*)TailDataA, (const uint8_t*)InputA, N); + InputA = (const DataType*)TailDataA; + if (!IsScalarB) { + MlasCopyTailBytes((uint8_t*)TailDataB, (const uint8_t*)InputB, N); + InputB = (const DataType*)TailDataB; + } + } + + const typename SUI::i8x16_t VectorA0 = SUI::vld1q_i8(InputA); + InputA += 16; + const int16x8_t va0_s16x8 = MlasExtendToS16Debias(VectorA0, VectorZeroPointA); + const int16x8_t va1_s16x8 = MlasExtendToS16Debias(VectorA0, VectorZeroPointA);; + + if (!IsScalarB) { + const typename SUI::i8x16_t VectorB0 = SUI::vld1q_i8(InputB); + InputB += 16; + vb0_s16x8 = MlasExtendToS16Debias(VectorB0, VectorZeroPointB); + vb1_s16x8 = MlasExtendToS16Debias(VectorB0, VectorZeroPointB); + } + + const int16x8_t vacc0 = MlasQLinearMulVectorS16(va0_s16x8, vb0_s16x8, VectorScaleRatio, VectorZeroPointC); + const int16x8_t vacc1 = MlasQLinearMulVectorS16(va1_s16x8, vb1_s16x8, VectorScaleRatio, VectorZeroPointC); + typename SUI::i8x16_t vc = SUI::combine_i8_s16(vacc0, vacc1); + + if (N >= 16) { + N -= 16; + SUI::vst1q_i8(OutputC, vc); + OutputC += 16; + } else { + SUI::vst1q_i8(TailDataA, vc); + MlasCopyTailBytes((uint8_t*)OutputC, (const uint8_t*)TailDataA, N); + N = 0; + } + } +} + +#elif defined(MLAS_SSE2_INTRINSICS) + +template +MLAS_FORCEINLINE +static +__m128i +MlasExtendToS16( + __m128i Int8Vector, + __m128i ZeroVector + ); + +template <> +MLAS_FORCEINLINE +__m128i +MlasExtendToS16( + __m128i Int8Vector, + __m128i ZeroVector + ) +{ + return _mm_unpacklo_epi8(Int8Vector, ZeroVector); +} + +template <> +MLAS_FORCEINLINE +__m128i +MlasExtendToS16( + __m128i Int8Vector, + __m128i ZeroVector + ) +{ + return _mm_unpackhi_epi8(Int8Vector, ZeroVector); +} + +template <> +MLAS_FORCEINLINE +__m128i +MlasExtendToS16( + __m128i Int8Vector, + __m128i ZeroVector + ) +{ + MLAS_UNREFERENCED_PARAMETER(ZeroVector); + return _mm_srai_epi16(_mm_unpacklo_epi8(Int8Vector, Int8Vector), 8); +} + +template <> +MLAS_FORCEINLINE +__m128i +MlasExtendToS16( + __m128i Int8Vector, + __m128i ZeroVector + ) +{ + MLAS_UNREFERENCED_PARAMETER(ZeroVector); + return _mm_srai_epi16(_mm_unpackhi_epi8(Int8Vector, Int8Vector), 8); +} + +template +MLAS_FORCEINLINE +static +__m128i +MlasExtendToS16Debias( + __m128i Int8Vector, + __m128i ZeroVector, + __m128i VectorBias + ) +{ + return _mm_sub_epi16(MlasExtendToS16(Int8Vector, ZeroVector), VectorBias); +} + +MLAS_FORCEINLINE +static +__m128i +MlasQLinearMulVectorS16( + __m128i va_s16x8, + __m128i vb_s16x8, + __m128 VectorScaleRatio, + __m128 VectorZeroPointC + ) +{ + const auto ab_lo = _mm_mullo_epi16(va_s16x8, vb_s16x8); + const auto ab_hi = _mm_mulhi_epi16(va_s16x8, vb_s16x8); + auto r_lo = _mm_unpacklo_epi16(ab_lo, ab_hi); + auto r_hi = _mm_unpackhi_epi16(ab_lo, ab_hi); + r_lo = _mm_cvtps_epi32(_mm_add_ps(_mm_mul_ps(_mm_cvtepi32_ps(r_lo), VectorScaleRatio), VectorZeroPointC)); + r_hi = _mm_cvtps_epi32(_mm_add_ps(_mm_mul_ps(_mm_cvtepi32_ps(r_hi), VectorScaleRatio), VectorZeroPointC)); + return _mm_packs_epi32(r_lo, r_hi); +} + +template +static +void +MlasQLinearMulKernel( + const DataType* InputA, + float ScaleA, + int32_t ZeroPointA, + const DataType* InputB, + float ScaleB, + int32_t ZeroPointB, + float ScaleC, + int32_t ZeroPointC, + DataType* OutputC, + size_t N + ) +{ + const auto VectorZeroPointA = _mm_set1_epi16((int16_t)ZeroPointA); + const auto VectorZeroPointB = _mm_set1_epi16((int16_t)ZeroPointB); + const auto VectorZeroPointC = MlasBroadcastFloat32x4((float)ZeroPointC); + const auto VectorScaleRatio = MlasBroadcastFloat32x4(ScaleA * ScaleB / ScaleC); + const auto ZeroVector = _mm_setzero_si128(); + + uint8_t TailDataA[16] = { 0 }; + uint8_t TailDataB[16] = { 0 }; + __m128i vb_lo_s16x8, vb_hi_s16x8; + + if (IsScalarB) { + vb_lo_s16x8 = _mm_sub_epi16(_mm_set1_epi16((int16_t)*InputB), VectorZeroPointB); + vb_hi_s16x8 = vb_lo_s16x8; + } + + while (N > 0) { + if (N < 16) { + MlasCopyTailBytes(TailDataA, (const uint8_t*)InputA, N); + InputA = (const DataType*)TailDataA; + if (!IsScalarB) { + MlasCopyTailBytes(TailDataB, (const uint8_t*)InputB, N); + InputB = (const DataType*)TailDataB; + } + } + + const auto va_i8x16 = _mm_loadu_si128((const MLAS_INT32X4*)InputA); + InputA += 16; + const auto va_lo_s16x8 = MlasExtendToS16Debias(va_i8x16, ZeroVector, VectorZeroPointA); + const auto va_hi_s16x8 = MlasExtendToS16Debias(va_i8x16, ZeroVector, VectorZeroPointA); + + if (!IsScalarB) { + const auto vb_i8x16 = _mm_loadu_si128((const MLAS_INT32X4*)InputB); + InputB += 16; + vb_lo_s16x8 = MlasExtendToS16Debias(vb_i8x16, ZeroVector, VectorZeroPointB); + vb_hi_s16x8 = MlasExtendToS16Debias(vb_i8x16, ZeroVector, VectorZeroPointB); + } + + const auto vc_lo_s16x8 = MlasQLinearMulVectorS16(va_lo_s16x8, vb_lo_s16x8, VectorScaleRatio, VectorZeroPointC); + const auto vc_hi_s16x8 = MlasQLinearMulVectorS16(va_hi_s16x8, vb_hi_s16x8, VectorScaleRatio, VectorZeroPointC); + auto vc = MlasPackS16_128(vc_lo_s16x8, vc_hi_s16x8); + + if (N >= 16) { + _mm_storeu_si128((__m128i*)OutputC, vc); + OutputC += 16; + N -= 16; + } else { + _mm_storeu_si128((__m128i*)TailDataA, vc); + MlasCopyTailBytes((uint8_t*)OutputC, TailDataA, N); + N = 0; + } + } +} + +#else + +// Pure C++ implementation. +template +static +void +MlasQLinearMulKernel( + const DataType* InputA, + float ScaleA, + int32_t ZeroPointA, + const DataType* InputB, + float ScaleB, + int32_t ZeroPointB, + float ScaleC, + int32_t ZeroPointC, + DataType* OutputC, + size_t N + ) +{ + const float MinimumValue = (float)((int)std::numeric_limits::min() - ZeroPointC); + const float MaximumValue = (float)((int)std::numeric_limits::max() - ZeroPointC); + + float ValueB; + + if (IsScalarB) { + ValueB = ScaleB * (int32_t(InputB[0]) - ZeroPointB); + } + + for (size_t n = 0; n < N; n++) { + float ValueA = ScaleA * (int32_t(InputA[n]) - ZeroPointA); + if (!IsScalarB) { + ValueB = ScaleB * (int32_t(InputB[n]) - ZeroPointB); + } + float ValueC = (ValueA * ValueB) / ScaleC; + ValueC = std::min(std::max(ValueC, MinimumValue), MaximumValue); + OutputC[n] = (DataType)(int32_t)std::nearbyintf(ValueC + ZeroPointC); + } +} + +#endif + +template +void +MLASCALL +MlasQLinearMul( + const DataType* InputA, + float ScaleA, + int32_t ZeroPointA, + const DataType* InputB, + float ScaleB, + int32_t ZeroPointB, + float ScaleC, + int32_t ZeroPointC, + DataType* OutputC, + size_t N, + bool IsScalarB + ) +{ + if (IsScalarB) { + MlasQLinearMulKernel( + InputA, ScaleA, ZeroPointA, InputB, ScaleB, ZeroPointB, ScaleC, ZeroPointC, OutputC, N); + } else { + MlasQLinearMulKernel( + InputA, ScaleA, ZeroPointA, InputB, ScaleB, ZeroPointB, ScaleC, ZeroPointC, OutputC, N); + } +} + +// Explicit instantiation +template +void +MlasQLinearMul( + const uint8_t* InputA, + float ScaleA, + int32_t ZeroPointA, + const uint8_t* InputB, + float ScaleB, + int32_t ZeroPointB, + float ScaleC, + int32_t ZeroPointC, + uint8_t* OutputC, + size_t N, + bool IsScalarB + ); + +template +void +MlasQLinearMul( + const int8_t* InputA, + float ScaleA, + int32_t ZeroPointA, + const int8_t* InputB, + float ScaleB, + int32_t ZeroPointB, + float ScaleC, + int32_t ZeroPointC, + int8_t* OutputC, + size_t N, + bool IsScalarB + ); diff --git a/onnxruntime/test/contrib_ops/qlinear_binary_op_test.cc b/onnxruntime/test/contrib_ops/qlinear_binary_op_test.cc index dcdeeac94f..805abcb433 100644 --- a/onnxruntime/test/contrib_ops/qlinear_binary_op_test.cc +++ b/onnxruntime/test/contrib_ops/qlinear_binary_op_test.cc @@ -128,6 +128,10 @@ static auto add_function = [](float a_dequantized, float b_dequantized) { return a_dequantized + b_dequantized; }; +static auto mul_function = [](float a_dequantized, float b_dequantized) { + return a_dequantized * b_dequantized; +}; + TEST(QLinearBinaryOpTest, AddU8VectorVectorFull) { const std::vector& A(A4Add); float A_scale = 8.0f / 256.0f; @@ -368,5 +372,184 @@ TEST(QLinearBinaryOpTest, AddS8VectorScalarBroadcast) { C_scale, C_zero_point); } +// +// Tests for QLinearMul +// +TEST(QLinearBinaryOpTest, MulU8VectorVectorFull) { + const std::vector& A(A4Add); + float A_scale = 8.0f / 256.0f; + uint8_t A_zero_point = 128; + const std::vector& B(B4Add); + float B_scale = 8.0f / 256.0f; + uint8_t B_zero_point = 128; + float C_scale = 64.0f / 256.0f; + uint8_t C_zero_point = 128; + + RunQLinearMathTestFromFloat("QLinearMul", mul_function, + A, {63}, A_scale, A_zero_point, B, {63}, B_scale, B_zero_point, C_scale, C_zero_point); +} + +TEST(QLinearBinaryOpTest, MulU8VectorVectorBroadcast) { + const std::vector& A(A4Add); + float A_scale = 8.0f / 256.0f; + uint8_t A_zero_point = 128; + std::vector B = { + 4.00f, 0.25f, 0.00f, -0.25f, 0.50f, -0.25f, -0.00f, 0.25f, + -1.50f, -2.25f, 2.50f, 3.75f, -3.75f, -4.00f, 5.00f, 5.50f, + -0.50f, -1.25f, 0.75f, 0.00f, 2.25f + }; + float B_scale = 8.0f / 256.0f; + uint8_t B_zero_point = 128; + float C_scale = 64.0f / 256.0f; + uint8_t C_zero_point = 128; + + RunQLinearMathTestFromFloat("QLinearMul", mul_function, + A, {3, 3, 7}, A_scale, A_zero_point, B, {3, 1, 7}, B_scale, B_zero_point, C_scale, C_zero_point); +} + +TEST(QLinearBinaryOpTest, MulU8ScalarVectorFull) { + const std::vector& A(A4Add); + float A_scale = 8.0f / 256.0f; + uint8_t A_zero_point = 128; + std::vector B = { 0.25f }; + float B_scale = 8.0f / 256.0f; + uint8_t B_zero_point = 96; + float C_scale = 8.0f / 256.0f; + uint8_t C_zero_point = 100; + + RunQLinearMathTestFromFloat("QLinearMul", mul_function, + B, {1}, B_scale, B_zero_point, A, {63}, A_scale, A_zero_point, C_scale, C_zero_point); +} + +TEST(QLinearBinaryOpTest, MulU8ScalarVectorBroadcast) { + const std::vector& A(A4Add); + float A_scale = 8.0f / 256.0f; + uint8_t A_zero_point = 128; + std::vector B = { 0.25f, -0.25f, -0.00f }; + float B_scale = 8.0f / 256.0f; + uint8_t B_zero_point = 96; + float C_scale = 8.0f / 256.0f; + uint8_t C_zero_point = 100; + + RunQLinearMathTestFromFloat("QLinearMul", mul_function, + B, {3, 1, 1}, B_scale, B_zero_point, A, {3, 7, 3}, A_scale, A_zero_point, C_scale, C_zero_point); +} + +TEST(QLinearBinaryOpTest, MulU8VectorScalarFull) { + const std::vector& A(A4Add); + float A_scale = 8.0f / 256.0f; + uint8_t A_zero_point = 128; + std::vector B = { 0.25f }; + float B_scale = 8.0f / 256.0f; + uint8_t B_zero_point = 96; + float C_scale = 16.0f / 256.0f; + uint8_t C_zero_point = 128; + + RunQLinearMathTestFromFloat("QLinearMul", mul_function, + A, {63}, A_scale, A_zero_point, B, {1}, B_scale, B_zero_point, C_scale, C_zero_point); +} + +TEST(QLinearBinaryOpTest, MulU8VectorScalarBroadcast) { + const std::vector& A(A4Add); + float A_scale = 8.0f / 256.0f; + uint8_t A_zero_point = 128; + std::vector B = { 0.25f, -0.25f, -0.00f }; + float B_scale = 8.0f / 256.0f; + uint8_t B_zero_point = 96; + float C_scale = 16.0f / 256.0f; + uint8_t C_zero_point = 128; + + RunQLinearMathTestFromFloat("QLinearMul", mul_function, + A, {3, 7, 3}, A_scale, A_zero_point, B, {1, 1, 3}, B_scale, B_zero_point, C_scale, C_zero_point); +} + +TEST(QLinearBinaryOpTest, MulS8VectorVectorFull) { + const std::vector& A(A4Add); + float A_scale = 8.0f / 256.0f; + int8_t A_zero_point = 0; + const std::vector& B(B4Add); + float B_scale = 8.0f / 256.0f; + int8_t B_zero_point = 0; + float C_scale = 64.0f / 256.0f; + int8_t C_zero_point = -16; + + RunQLinearMathTestFromFloat("QLinearMul", mul_function, + A, {63}, A_scale, A_zero_point, B, {63}, B_scale, B_zero_point, C_scale, C_zero_point); +} + +TEST(QLinearBinaryOpTest, MulS8VectorVectorBroadcast) { + const std::vector& A(A4Add); + float A_scale = 8.0f / 256.0f; + int8_t A_zero_point = 0; + std::vector B = { + 4.00f, 0.25f, 0.00f, -0.25f, 0.50f, -0.25f, -0.00f, 0.25f, + -1.50f, -2.25f, 2.50f, 3.75f, -3.75f, -4.00f, 5.00f, 5.50f, + -0.50f, -1.25f, 0.75f, 1.25f, 2.25f + }; + float B_scale = 8.0f / 256.0f; + int8_t B_zero_point = 0; + float C_scale = 16.0f / 256.0f; + int8_t C_zero_point = -16; + + RunQLinearMathTestFromFloat("QLinearMul", mul_function, + A, {3, 3, 7}, A_scale, A_zero_point, B, {3, 1, 7}, B_scale, B_zero_point, C_scale, C_zero_point); +} + +TEST(QLinearBinaryOpTest, MulS8ScalarVectorFull) { + const std::vector& A(A4Add); + float A_scale = 8.0f / 256.0f; + int8_t A_zero_point = 0; + std::vector B = { 0.25f }; + float B_scale = 2.0f / 256.0f; + int8_t B_zero_point = 16; + float C_scale = 8.0f / 256.0f; + int8_t C_zero_point = 10; + + RunQLinearMathTestFromFloat("QLinearMul", mul_function, + B, {1}, B_scale, B_zero_point, A, {63}, A_scale, A_zero_point, C_scale, C_zero_point); +} + +TEST(QLinearBinaryOpTest, MulS8ScalarVectorBroadcast) { + const std::vector& A(A4Add); + float A_scale = 8.0f / 256.0f; + int8_t A_zero_point = 0; + std::vector B = { 0.25f, -0.25f, -0.00f }; + float B_scale = 2.0f / 256.0f; + int8_t B_zero_point = 16; + float C_scale = 8.0f / 256.0f; + int8_t C_zero_point = 10; + + RunQLinearMathTestFromFloat("QLinearMul", mul_function, + B, {3, 1, 1}, B_scale, B_zero_point, A, {3, 7, 3}, A_scale, A_zero_point, C_scale, C_zero_point); +} + +TEST(QLinearBinaryOpTest, MulS8VectorScalarFull) { + const std::vector& A(A4Add); + float A_scale = 8.0f / 256.0f; + int8_t A_zero_point = 0; + std::vector B = { 0.25f }; + float B_scale = 2.0f / 256.0f; + int8_t B_zero_point = 16; + float C_scale = 8.0f / 256.0f; + int8_t C_zero_point = 10; + + RunQLinearMathTestFromFloat("QLinearMul", mul_function, + A, {63}, A_scale, A_zero_point, B, {1}, B_scale, B_zero_point, C_scale, C_zero_point); +} + +TEST(QLinearBinaryOpTest, MulS8VectorScalarBroadcast) { + const std::vector& A(A4Add); + float A_scale = 8.0f / 256.0f; + int8_t A_zero_point = 0; + std::vector B = { 0.25f, -0.25f, -0.00f }; + float B_scale = 2.0f / 256.0f; + int8_t B_zero_point = 16; + float C_scale = 8.0f / 256.0f; + int8_t C_zero_point = 10; + + RunQLinearMathTestFromFloat("QLinearMul", mul_function, + A, {3, 7, 3}, A_scale, A_zero_point, B, {1, 1, 3}, B_scale, B_zero_point, C_scale, C_zero_point); +} + } // namespace test } // namespace onnxruntime diff --git a/onnxruntime/test/mlas/unittest.cpp b/onnxruntime/test/mlas/unittest.cpp index 6eceb909e7..3a4d8cc25d 100644 --- a/onnxruntime/test/mlas/unittest.cpp +++ b/onnxruntime/test/mlas/unittest.cpp @@ -2495,9 +2495,25 @@ public: } }; -class MlasQLinearAddTest : public MlasTestBase +class MlasQLinearBinaryOpTest : public MlasTestBase { +public: + typedef void (MLASCALL *QLinearBinaryOpS8)( + const int8_t* InputA, float ScaleA, int32_t ZeroPointA, + const int8_t* InputB, float ScaleB, int32_t ZeroPointB, + float ScaleC, int32_t ZeroPointC, int8_t* OutputC, + size_t N, bool IsScalarB); + typedef void (MLASCALL *QLinearBinaryOpU8)( + const uint8_t* InputA, float ScaleA, int32_t ZeroPointA, + const uint8_t* InputB, float ScaleB, int32_t ZeroPointB, + float ScaleC, int32_t ZeroPointC, uint8_t* OutputC, + size_t N, bool IsScalarB); + private: + std::function ScalarOp; + std::string ScalarOpName; + QLinearBinaryOpS8 QLinearS8Op; + QLinearBinaryOpU8 QLinearU8Op; MatrixGuardBuffer BufferInputA; MatrixGuardBuffer BufferInputB; MatrixGuardBuffer BufferOutput; @@ -2505,7 +2521,7 @@ private: template T - QLinearAddScalar( + QLinearBinaryScalar( T a, float ScaleA, int32_t ZeroPointA, @@ -2516,23 +2532,28 @@ private: int32_t ZeroPointC ) { - constexpr int qmax = std::numeric_limits::max(); constexpr int qmin = std::numeric_limits::min(); float ValueA = ScaleA * (static_cast(a) - ZeroPointA); float ValueB = ScaleB * (static_cast(b) - ZeroPointB); - float ValueC = std::nearbyintf((ValueA + ValueB) / ScaleC) + ZeroPointC; + float ValueC = std::nearbyintf(ScalarOp(ValueA, ValueB) / ScaleC) + ZeroPointC; int qc = static_cast(ValueC); qc = std::min(qc, qmax); qc = std::max(qc, qmin); return static_cast(qc); } - template + template void Test( + void (MLASCALL *QLinearBinaryOp)( + const T* InputA, float ScaleA, int32_t ZeroPointA, + const T* InputB, float ScaleB, int32_t ZeroPointB, + float ScaleC, int32_t ZeroPointC, T* OutputC, + size_t N, bool IsScalarB), size_t N, + bool IsScalarB, float ScaleA, int32_t ZeroPointA, float ScaleB, @@ -2559,17 +2580,18 @@ private: if (!IsScalarB) { InputB[n] = static_cast(distribution(generator)); } - OutputReference[n] = QLinearAddScalar(InputA[n], ScaleA, ZeroPointA, InputB[IsScalarB ? 0 : n], ScaleB, ZeroPointB, ScaleC, ZeroPointC); + OutputReference[n] = QLinearBinaryScalar(InputA[n], ScaleA, ZeroPointA, InputB[IsScalarB ? 0 : n], ScaleB, ZeroPointB, ScaleC, ZeroPointC); } - MlasQLinearAdd(InputA, ScaleA, ZeroPointA, InputB, ScaleB, ZeroPointB, ScaleC, ZeroPointC, OutputC, N, IsScalarB); + QLinearBinaryOp(InputA, ScaleA, ZeroPointA, InputB, ScaleB, ZeroPointB, ScaleC, ZeroPointC, OutputC, N, IsScalarB); for (size_t n = 0; n < N; n++) { int diff = (int)OutputC[n] - (int)OutputReference[n]; if (diff < -1 || diff > 1) { - printf("Test IsScalarB=%d difference @%u of %u, %d(%f,%d) + %d(%f,%d) => %d(%f,%d) (expecting %d)\n", + printf("Test IsScalarB=%d difference @%u of %u, %d(%f,%d) %s %d(%f,%d) => %d(%f,%d) (expecting %d)\n", int(IsScalarB), static_cast(n), static_cast(N), static_cast(InputA[n]), ScaleA, ZeroPointA, + ScalarOpName.c_str(), static_cast(InputB[IsScalarB ? 0 : n]), ScaleB, ZeroPointB, static_cast(OutputC[n]), ScaleC, ZeroPointC, static_cast(OutputReference[n])); @@ -2578,54 +2600,49 @@ private: } public: + explicit MlasQLinearBinaryOpTest( + std::function P_ScalarOp, + const std::string& P_ScalarOpName, + QLinearBinaryOpS8 P_QLinearS8Op, + QLinearBinaryOpU8 P_QLinearU8Op + ) + : ScalarOp(P_ScalarOp), + ScalarOpName(P_ScalarOpName), + QLinearS8Op(P_QLinearS8Op), + QLinearU8Op(P_QLinearU8Op) + { + } + void ExecuteShort( void ) override { - // uint8_t test static const uint8_t zero_points[] = { 0, 18, 75, 128, 157, 231, 255 }; + static const float c_scales[] = { 18.0f, 90.0f }; + + const int8_t* s_zero_points = (const int8_t*)(&zero_points[0]); for (size_t a = 0; a < _countof(zero_points); a++) { - uint8_t offa = zero_points[a]; - for (size_t b = 0; b < _countof(zero_points); b++) { - uint8_t offb = zero_points[b]; - for (size_t c = 0; c < _countof(zero_points); c++) { - uint8_t offc = zero_points[c]; + for (size_t s = 0; s < _countof(c_scales); s++) { + for (size_t n = 1; n < 128; n++) { + // u8, vector + vector + Test(QLinearU8Op, n, false, 10.f, zero_points[a], 10.f, zero_points[b], c_scales[s], zero_points[c]); - for (size_t n = 1; n < 128; n++) { - // vector + vector - Test(n, 10.f, offa, 10.f, offb, 20.f, offc); + // u8, vector + scalar + Test(QLinearU8Op, n, true, 10.f, zero_points[a], 10.f, zero_points[b], c_scales[s], zero_points[c]); - // vector + scalar - Test(n, 10.f, offa, 10.f, offb, 20.f, offc); + // s8, vector + vector + Test(QLinearS8Op, n, false, 10.f, s_zero_points[a], 10.f, s_zero_points[b], c_scales[s], s_zero_points[c]); + + // s8, vector + scalar + Test(QLinearS8Op, n, true, 10.f, s_zero_points[a], 10.f, s_zero_points[b], c_scales[s], s_zero_points[c]); + } } } } } - - static const int8_t szero_points[] = { -128, -110, -53, 0, 29, 103, 127 }; - for (size_t a = 0; a < _countof(zero_points); a++) { - int8_t offa = szero_points[a]; - - for (size_t b = 0; b < _countof(zero_points); b++) { - int8_t offb = szero_points[b]; - - for (size_t c = 0; c < _countof(zero_points); c++) { - int8_t offc = szero_points[c]; - - for (size_t n = 1; n < 128; n++) { - // vector + vector - Test(n, 10.f, offa, 10.f, offb, 20.f, offc); - - // vector + scalar - Test(n, 10.f, offa, 10.f, offb, 20.f, offc); - } - } - } - } - } }; @@ -2787,7 +2804,12 @@ main( } printf("QLinearAdd tests.\n"); - onnxruntime::make_unique()->ExecuteShort(); + onnxruntime::make_unique( + [](float a, float b) { return a + b; }, "+", MlasQLinearAdd, MlasQLinearAdd)->ExecuteShort(); + + printf("QLinearMul tests.\n"); + onnxruntime::make_unique( + [] (float a, float b) { return a * b; }, "*", MlasQLinearMul, MlasQLinearMul)->ExecuteShort(); printf("Done.\n");