From 57c92066c2cbd384deecbe74e0950ab3e726718a Mon Sep 17 00:00:00 2001 From: Tracy Sharpe <42477615+tracysh@users.noreply.github.com> Date: Sun, 22 Nov 2020 23:19:27 -0800 Subject: [PATCH] Implement missing pieces for ARM QLinearConv support (#5894) --- .../contrib_ops/cpu/cpu_contrib_kernels.cc | 2 +- onnxruntime/core/mlas/inc/mlas.h | 30 +- onnxruntime/core/mlas/lib/compute.cpp | 2 +- onnxruntime/core/mlas/lib/mlasi.h | 41 ++ onnxruntime/core/mlas/lib/qladd.h | 29 - onnxruntime/core/mlas/lib/quantize.cpp | 611 ++++++++++++------ onnxruntime/core/mlas/lib/transpose.cpp | 233 +++++-- .../core/optimizer/graph_transformer_utils.cc | 2 +- .../cpu/math/quantize_linear_matmul.cc | 7 +- .../core/providers/cpu/nn/qlinearconv.cc | 61 +- .../core/providers/cpu/tensor/transpose.cc | 8 - .../test/optimizer/nhwc_transformer_test.cc | 2 +- .../providers/cpu/nn/qlinearconv_op_test.cc | 40 +- 13 files changed, 699 insertions(+), 369 deletions(-) diff --git a/onnxruntime/contrib_ops/cpu/cpu_contrib_kernels.cc b/onnxruntime/contrib_ops/cpu/cpu_contrib_kernels.cc index 131ebc62fa..71576f11a2 100644 --- a/onnxruntime/contrib_ops/cpu/cpu_contrib_kernels.cc +++ b/onnxruntime/contrib_ops/cpu/cpu_contrib_kernels.cc @@ -134,7 +134,7 @@ Status RegisterQuantizationKernels(KernelRegistry& kernel_registry) { BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, -#if defined(MLAS_TARGET_AMD64_IX86) +#ifdef MLAS_SUPPORTS_GEMM_U8X8 BuildKernelCreateInfo, #endif }; diff --git a/onnxruntime/core/mlas/inc/mlas.h b/onnxruntime/core/mlas/inc/mlas.h index 1cb788ded0..0c3f282e3b 100644 --- a/onnxruntime/core/mlas/inc/mlas.h +++ b/onnxruntime/core/mlas/inc/mlas.h @@ -65,11 +65,6 @@ Abstract: #define MLAS_SUPPORTS_GEMM_U8X8 #endif -#if defined(MLAS_TARGET_AMD64_IX86) -#define MLAS_SUPPORTS_GEMM_U8X8_AND_REQUANTIZE_OUTPUT -#define MLAS_SUPPORTS_TRANSPOSE -#endif - #if defined(MLAS_TARGET_AMD64) || defined(MLAS_TARGET_ARM64) || (defined(MLAS_TARGET_ARM) && !defined(_MSC_VER)) #define MLAS_SUPPORTS_PACKED_GEMM_U8X8 #endif @@ -668,36 +663,13 @@ MlasQuantizeLinear( void MLASCALL MlasRequantizeOutput( - const int32_t* Input, - uint8_t* Output, - const int32_t* Bias, - size_t M, - size_t N, - float Scale, - uint8_t ZeroPoint - ); - -void -MLASCALL -MlasRequantizeOutputColumn( - const int32_t* Input, - uint8_t* Output, - const int32_t* Bias, - size_t M, - size_t N, - const float Scale, - uint8_t ZeroPoint - ); - -void -MLASCALL -MlasRequantizeOutputColumn( const int32_t* Input, uint8_t* Output, const int32_t* Bias, size_t M, size_t N, const float* Scale, + bool PerColumnScale, uint8_t ZeroPoint ); diff --git a/onnxruntime/core/mlas/lib/compute.cpp b/onnxruntime/core/mlas/lib/compute.cpp index 86a86ec1ba..04b9cc3a1b 100644 --- a/onnxruntime/core/mlas/lib/compute.cpp +++ b/onnxruntime/core/mlas/lib/compute.cpp @@ -47,7 +47,7 @@ MLAS_INTERNAL_DATA const struct { 88.7762626647950f, -88.3762626647949f, 88.3762626647949f, - 12582912.f, + MLAS_ROUNDING_BIAS_MAGIC, 1.44269504088896341f, -6.93145752e-1f, -1.42860677e-6f, diff --git a/onnxruntime/core/mlas/lib/mlasi.h b/onnxruntime/core/mlas/lib/mlasi.h index d5e4aef0ca..95164f7352 100644 --- a/onnxruntime/core/mlas/lib/mlasi.h +++ b/onnxruntime/core/mlas/lib/mlasi.h @@ -782,6 +782,47 @@ MlasPartitionWork( } } +// +// Define the minimum floating point value (and its bit value equivalent) that +// has no fractional bits. This number can be used for fast rounding of floating +// point numbers to integers. +// + +#define MLAS_ROUNDING_BIAS_MAGIC 12582912.f +#define MLAS_ROUNDING_BIAS_MAGIC_BITS 0x4B400000 + +// +// Helpers to cast a floating point type to and from an integer bit format. +// + +MLAS_FORCEINLINE +uint32_t +MlasBitsOfFp32( + float FloatValue + ) +{ + union { + uint32_t IntegerValue; + float FloatValue; + } u; + u.FloatValue = FloatValue; + return u.IntegerValue; +} + +MLAS_FORCEINLINE +float +MlasFp32FromBits( + uint32_t IntegerValue + ) +{ + union { + uint32_t IntegerValue; + float FloatValue; + } u; + u.IntegerValue = IntegerValue; + return u.FloatValue; +} + // // Define the missing ARM64 NEON intrinsic macros from arm64_neon.h that enable // cross-compiler support. diff --git a/onnxruntime/core/mlas/lib/qladd.h b/onnxruntime/core/mlas/lib/qladd.h index 8a2e148c49..91bf05023c 100644 --- a/onnxruntime/core/mlas/lib/qladd.h +++ b/onnxruntime/core/mlas/lib/qladd.h @@ -19,35 +19,6 @@ Abstract: #include "mlasi.h" -union MLAS_FLOAT32BITS -{ - uint32_t u32; - float fp32; -}; - -MLAS_FORCEINLINE -static -uint32_t -MlasBitsOfFp32( - float f - ) -{ - MLAS_FLOAT32BITS uf; - uf.fp32 = f; - return uf.u32; -} - -MLAS_FORCEINLINE -static -float -MlasFp32FromBits( - uint32_t u - ) -{ - MLAS_FLOAT32BITS uf = { u }; - return uf.fp32; -} - MLAS_FORCEINLINE static void diff --git a/onnxruntime/core/mlas/lib/quantize.cpp b/onnxruntime/core/mlas/lib/quantize.cpp index 522a25bad2..b24ffe0d69 100644 --- a/onnxruntime/core/mlas/lib/quantize.cpp +++ b/onnxruntime/core/mlas/lib/quantize.cpp @@ -289,45 +289,239 @@ MlasQuantizeLinear( #if defined(MLAS_SSE2_INTRINSICS) -MLAS_FORCEINLINE -MLAS_INT32X4 -MlasRequantizeOutputVector( - MLAS_INT32X4 IntegerVector, - MLAS_INT32X4 BiasVector, - MLAS_FLOAT32X4 ScaleVector, - MLAS_FLOAT32X4 MinimumValueVector, - MLAS_FLOAT32X4 MaximumValueVector, - MLAS_INT32X4 ZeroPointVector +void +MLASCALL +MlasRequantizeOutput( + const int32_t* Input, + uint8_t* Output, + const int32_t* Bias, + size_t M, + size_t N, + const float* Scale, + bool PerColumnScale, + uint8_t ZeroPoint ) +/*++ + +Routine Description: + + This routine requantizes the intermediate buffer to the output buffer + optionally adding the supplied bias. + +Arguments: + + Input - Supplies the input matrix. + + Output - Supplies the output matrix. + + Bias - Supplies the optional bias vector to be added to the input buffer + before requantization. + + Buffer - Supplies the output matrix. + + M - Supplies the number of elements of the bias vector and the number of + rows in the output matrix. + + N - Supplies the number of columns of the output matrix. + + Scale - Supplies the quantization scale. + + PerColumnScale - Supplies true if the quantization scale has per-column + values, else false if a single quantization scale applies to the + entire matrix. + + ZeroPoint - Supplies the quantization zero point value. + +Return Value: + + None. + +--*/ { - IntegerVector = _mm_add_epi32(IntegerVector, BiasVector); - MLAS_FLOAT32X4 FloatVector = _mm_cvtepi32_ps(IntegerVector); + const __m128 PerMatrixScaleVector = PerColumnScale ? _mm_setzero_ps() : _mm_load1_ps(Scale); + const __m128 MinimumValueVector = _mm_set1_ps(float(0 - ZeroPoint)); + const __m128 MaximumValueVector = _mm_set1_ps(float(255 - ZeroPoint)); + const __m128i ZeroPointVector = _mm_set1_epi32(ZeroPoint); // - // Scale the input vector and clamp the values to the minimum and maximum - // range (adjusted by the zero point value). + // Step through each row of the output matrix. // - FloatVector = MlasMultiplyFloat32x4(FloatVector, ScaleVector); + while (M-- > 0) { - // N.B. MINPS and MAXPS returns the value from the second vector if the - // value from the first vector is a NaN. - FloatVector = _mm_max_ps(FloatVector, MinimumValueVector); - FloatVector = _mm_min_ps(FloatVector, MaximumValueVector); + const int32_t* bias = Bias; + const float* scale = PerColumnScale ? Scale : nullptr; + size_t n = N; - // - // Convert the float values to integer using "round to nearest even" and - // then shift the output range using the zero point value. - // + // + // Process 16 columns of the matrices at a time. + // - // N.B. Assumes MXCSR has been configured with the default rounding mode of - // "round to nearest even". - IntegerVector = _mm_cvtps_epi32(FloatVector); - IntegerVector = _mm_add_epi32(IntegerVector, ZeroPointVector); + while (n >= 16) { - return IntegerVector; + // + // Load the input data and optionally add the per-column bias. + // + + __m128i IntegerVector0 = _mm_loadu_si128((const __m128i *)&Input[0]); + __m128i IntegerVector1 = _mm_loadu_si128((const __m128i *)&Input[4]); + __m128i IntegerVector2 = _mm_loadu_si128((const __m128i *)&Input[8]); + __m128i IntegerVector3 = _mm_loadu_si128((const __m128i *)&Input[12]); + Input += 16; + + if (bias != nullptr) { + IntegerVector0 = _mm_add_epi32(IntegerVector0, _mm_loadu_si128((const __m128i *)&bias[0])); + IntegerVector1 = _mm_add_epi32(IntegerVector1, _mm_loadu_si128((const __m128i *)&bias[4])); + IntegerVector2 = _mm_add_epi32(IntegerVector2, _mm_loadu_si128((const __m128i *)&bias[8])); + IntegerVector3 = _mm_add_epi32(IntegerVector3, _mm_loadu_si128((const __m128i *)&bias[12])); + bias += 16; + } + + // + // Convert to integer values to float and apply the per-tensor or + // per-column scaling. + // + + __m128 FloatVector0 = _mm_cvtepi32_ps(IntegerVector0); + __m128 FloatVector1 = _mm_cvtepi32_ps(IntegerVector1); + __m128 FloatVector2 = _mm_cvtepi32_ps(IntegerVector2); + __m128 FloatVector3 = _mm_cvtepi32_ps(IntegerVector3); + + if (scale != nullptr) { + + FloatVector0 = _mm_mul_ps(FloatVector0, _mm_loadu_ps(&scale[0])); + FloatVector1 = _mm_mul_ps(FloatVector1, _mm_loadu_ps(&scale[4])); + FloatVector2 = _mm_mul_ps(FloatVector2, _mm_loadu_ps(&scale[8])); + FloatVector3 = _mm_mul_ps(FloatVector3, _mm_loadu_ps(&scale[12])); + scale += 16; + + } else { + + FloatVector0 = _mm_mul_ps(FloatVector0, PerMatrixScaleVector); + FloatVector1 = _mm_mul_ps(FloatVector1, PerMatrixScaleVector); + FloatVector2 = _mm_mul_ps(FloatVector2, PerMatrixScaleVector); + FloatVector3 = _mm_mul_ps(FloatVector3, PerMatrixScaleVector); + } + + FloatVector0 = _mm_max_ps(FloatVector0, MinimumValueVector); + FloatVector1 = _mm_max_ps(FloatVector1, MinimumValueVector); + FloatVector2 = _mm_max_ps(FloatVector2, MinimumValueVector); + FloatVector3 = _mm_max_ps(FloatVector3, MinimumValueVector); + + FloatVector0 = _mm_min_ps(FloatVector0, MaximumValueVector); + FloatVector1 = _mm_min_ps(FloatVector1, MaximumValueVector); + FloatVector2 = _mm_min_ps(FloatVector2, MaximumValueVector); + FloatVector3 = _mm_min_ps(FloatVector3, MaximumValueVector); + + IntegerVector0 = _mm_cvtps_epi32(FloatVector0); + IntegerVector1 = _mm_cvtps_epi32(FloatVector1); + IntegerVector2 = _mm_cvtps_epi32(FloatVector2); + IntegerVector3 = _mm_cvtps_epi32(FloatVector3); + + IntegerVector0 = _mm_add_epi32(IntegerVector0, ZeroPointVector); + IntegerVector1 = _mm_add_epi32(IntegerVector1, ZeroPointVector); + IntegerVector2 = _mm_add_epi32(IntegerVector2, ZeroPointVector); + IntegerVector3 = _mm_add_epi32(IntegerVector3, ZeroPointVector); + + __m128i WordVector0 = _mm_packus_epi16(IntegerVector0, IntegerVector1); + __m128i WordVector1 = _mm_packus_epi16(IntegerVector2, IntegerVector3); + + __m128i ByteVector = _mm_packus_epi16(WordVector0, WordVector1); + + _mm_storeu_si128((__m128i*)Output, ByteVector); + Output += 16; + + n -= 16; + } + + // + // Process the remaining columns of the matrices. + // + + while (n > 0) { + + // + // Load the input data and optionally add the per-column bias. + // + + __m128i IntegerVector; + + if (n >= 4) { + + IntegerVector = _mm_loadu_si128((const __m128i*)&Input[0]); + Input += 4; + + if (bias != nullptr) { + IntegerVector = _mm_add_epi32(IntegerVector, _mm_loadu_si128((const __m128i*)&bias[0])); + bias += 4; + } + + } else { + + int32_t IntegerValue = *Input++; + + if (bias != nullptr) { + IntegerValue += *bias++; + } + + IntegerVector = _mm_cvtsi32_si128(IntegerValue); + } + + // + // Convert to integer values to float and apply the per-tensor or + // per-column scaling. + // + + __m128 FloatVector = _mm_cvtepi32_ps(IntegerVector); + __m128 ScaleVector; + + if (scale != nullptr) { + + if (n >= 4) { + ScaleVector = _mm_loadu_ps(scale); + scale += 4; + } else { + ScaleVector = _mm_load_ss(scale); + scale += 1; + } + + } else { + ScaleVector = PerMatrixScaleVector; + } + + FloatVector = _mm_mul_ps(FloatVector, ScaleVector); + + FloatVector = _mm_max_ps(FloatVector, MinimumValueVector); + FloatVector = _mm_min_ps(FloatVector, MaximumValueVector); + + IntegerVector = _mm_cvtps_epi32(FloatVector); + IntegerVector = _mm_add_epi32(IntegerVector, ZeroPointVector); + + IntegerVector = _mm_packus_epi16(IntegerVector, IntegerVector); + IntegerVector = _mm_packus_epi16(IntegerVector, IntegerVector); + + uint32_t OutputValue = uint32_t(_mm_cvtsi128_si32(IntegerVector)); + + if (n >= 4) { + + *reinterpret_cast(Output) = OutputValue; + Output += 4; + + n -= 4; + + } else { + + *Output = uint8_t(OutputValue); + Output += 1; + + n -= 1; + } + } + } } +#elif defined(MLAS_NEON64_INTRINSICS) + void MLASCALL MlasRequantizeOutput( @@ -336,7 +530,8 @@ MlasRequantizeOutput( const int32_t* Bias, size_t M, size_t N, - float Scale, + const float* Scale, + bool PerColumnScale, uint8_t ZeroPoint ) /*++ @@ -364,98 +559,9 @@ Arguments: Scale - Supplies the quantization scale. - ZeroPoint - Supplies the quantization zero point value. - -Return Value: - - None. - ---*/ -{ - MLAS_FLOAT32X4 ScaleVector = MlasBroadcastFloat32x4(Scale); - MLAS_FLOAT32X4 MinimumValueVector = MlasBroadcastFloat32x4(float(0 - ZeroPoint)); - MLAS_FLOAT32X4 MaximumValueVector = MlasBroadcastFloat32x4(float(255 - ZeroPoint)); - MLAS_INT32X4 ZeroPointVector = MlasBroadcastInt32x4(ZeroPoint); - MLAS_INT32X4 BiasVector = _mm_setzero_si128(); - - // - // Step through each row of the output matrix. - // - - while (M-- > 0) { - - if (Bias != nullptr) { - BiasVector = MlasBroadcastInt32x4(*Bias++); - } - - size_t n = N; - - while (n >= 4) { - - MLAS_INT32X4 IntegerVector = _mm_loadu_si128((const __m128i *)Input); - IntegerVector = MlasRequantizeOutputVector(IntegerVector, BiasVector, - ScaleVector, MinimumValueVector, MaximumValueVector, ZeroPointVector); - - IntegerVector = _mm_packus_epi16(IntegerVector, IntegerVector); - IntegerVector = _mm_packus_epi16(IntegerVector, IntegerVector); - - *((int32_t*)Output) = _mm_cvtsi128_si32(IntegerVector); - - Input += 4; - Output += 4; - n -= 4; - } - - while (n > 0) { - - MLAS_INT32X4 IntegerVector = _mm_cvtsi32_si128(*Input); - IntegerVector = MlasRequantizeOutputVector(IntegerVector, BiasVector, - ScaleVector, MinimumValueVector, MaximumValueVector, ZeroPointVector); - - *Output = (uint8_t)_mm_cvtsi128_si32(IntegerVector); - - Input += 1; - Output += 1; - n -= 1; - } - } -} - -void -MLASCALL -MlasRequantizeOutputColumn( - const int32_t* Input, - uint8_t* Output, - const int32_t* Bias, - size_t M, - size_t N, - float Scale, - uint8_t ZeroPoint - ) -/*++ - -Routine Description: - - This routine requantizes the intermediate buffer to the output buffer - optionally adding the supplied bias. - -Arguments: - - Input - Supplies the input matrix. - - Output - Supplies the output matrix. - - Bias - Supplies the optional bias vector to be added to the input buffer - before requantization. - - Buffer - Supplies the output matrix. - - M - Supplies the number of elements of the bias vector and the number of - rows in the output matrix. - - N - Supplies the number of columns of the output matrix. - - Scale - Supplies the quantization scale. + PerColumnScale - Supplies true if the quantization scale has per-column + values, else false if a single quantization scale applies to the + entire matrix. ZeroPoint - Supplies the quantization zero point value. @@ -465,11 +571,8 @@ Return Value: --*/ { - MLAS_FLOAT32X4 ScaleVector = MlasBroadcastFloat32x4(Scale); - MLAS_FLOAT32X4 MinimumValueVector = MlasBroadcastFloat32x4(float(0 - ZeroPoint)); - MLAS_FLOAT32X4 MaximumValueVector = MlasBroadcastFloat32x4(float(255 - ZeroPoint)); - MLAS_INT32X4 ZeroPointVector = MlasBroadcastInt32x4(ZeroPoint); - MLAS_INT32X4 BiasVector = _mm_setzero_si128(); + const float32x4_t PerMatrixScaleVector = PerColumnScale ? vdupq_n_f32(0) : vld1q_dup_f32(Scale); + const int16x8_t ZeroPointVector = vdupq_n_s16(ZeroPoint); // // Step through each row of the output matrix. @@ -478,61 +581,203 @@ Return Value: while (M-- > 0) { const int32_t* bias = Bias; - + const float* scale = PerColumnScale ? Scale : nullptr; size_t n = N; - while (n >= 4) { + // + // Process 16 columns of the matrices at a time. + // - MLAS_INT32X4 IntegerVector = _mm_loadu_si128((const __m128i *)Input); + while (n >= 16) { + + // + // Load the input data and optionally add the per-column bias. + // + + int32x4x4_t IntegerVector; + + IntegerVector.val[0] = vld1q_s32(&Input[0]); + IntegerVector.val[1] = vld1q_s32(&Input[4]); + IntegerVector.val[2] = vld1q_s32(&Input[8]); + IntegerVector.val[3] = vld1q_s32(&Input[12]); + Input += 16; if (bias != nullptr) { - BiasVector = _mm_loadu_si128((const __m128i*)bias); - bias += 4; + IntegerVector.val[0] = vaddq_s32(IntegerVector.val[0], vld1q_s32(&bias[0])); + IntegerVector.val[1] = vaddq_s32(IntegerVector.val[1], vld1q_s32(&bias[4])); + IntegerVector.val[2] = vaddq_s32(IntegerVector.val[2], vld1q_s32(&bias[8])); + IntegerVector.val[3] = vaddq_s32(IntegerVector.val[3], vld1q_s32(&bias[12])); + bias += 16; } - IntegerVector = MlasRequantizeOutputVector(IntegerVector, BiasVector, - ScaleVector, MinimumValueVector, MaximumValueVector, ZeroPointVector); + // + // Convert to integer values to float and apply the per-tensor or + // per-column scaling. + // - IntegerVector = _mm_packus_epi16(IntegerVector, IntegerVector); - IntegerVector = _mm_packus_epi16(IntegerVector, IntegerVector); + float32x4x4_t FloatVector; - *((int32_t*)Output) = _mm_cvtsi128_si32(IntegerVector); + FloatVector.val[0] = vcvtq_f32_s32(IntegerVector.val[0]); + FloatVector.val[1] = vcvtq_f32_s32(IntegerVector.val[1]); + FloatVector.val[2] = vcvtq_f32_s32(IntegerVector.val[2]); + FloatVector.val[3] = vcvtq_f32_s32(IntegerVector.val[3]); - Input += 4; - Output += 4; - n -= 4; + if (scale != nullptr) { + + float32x4x4_t PerColumnScaleVector; + + PerColumnScaleVector.val[0] = vld1q_f32(&scale[0]); + PerColumnScaleVector.val[1] = vld1q_f32(&scale[4]); + PerColumnScaleVector.val[2] = vld1q_f32(&scale[8]); + PerColumnScaleVector.val[3] = vld1q_f32(&scale[12]); + scale += 16; + + FloatVector.val[0] = vmulq_f32(FloatVector.val[0], PerColumnScaleVector.val[0]); + FloatVector.val[1] = vmulq_f32(FloatVector.val[1], PerColumnScaleVector.val[1]); + FloatVector.val[2] = vmulq_f32(FloatVector.val[2], PerColumnScaleVector.val[2]); + FloatVector.val[3] = vmulq_f32(FloatVector.val[3], PerColumnScaleVector.val[3]); + + } else { + + FloatVector.val[0] = vmulq_f32(FloatVector.val[0], PerMatrixScaleVector); + FloatVector.val[1] = vmulq_f32(FloatVector.val[1], PerMatrixScaleVector); + FloatVector.val[2] = vmulq_f32(FloatVector.val[2], PerMatrixScaleVector); + FloatVector.val[3] = vmulq_f32(FloatVector.val[3], PerMatrixScaleVector); + } + + // + // Convert the float values to integer using "round to nearest even". + // Results are saturated to the range of int32_t. + // + + IntegerVector.val[0] = vcvtnq_s32_f32(FloatVector.val[0]); + IntegerVector.val[1] = vcvtnq_s32_f32(FloatVector.val[1]); + IntegerVector.val[2] = vcvtnq_s32_f32(FloatVector.val[2]); + IntegerVector.val[3] = vcvtnq_s32_f32(FloatVector.val[3]); + + // + // Pack the integers with saturation to 16-bit values and shift by + // the zero point, then pack the integers again to unsigned bytes. + // + + int16x8x2_t WordVector; + + WordVector.val[0] = vqmovn_high_s32(vqmovn_s32(IntegerVector.val[0]), IntegerVector.val[1]); + WordVector.val[1] = vqmovn_high_s32(vqmovn_s32(IntegerVector.val[2]), IntegerVector.val[3]); + + WordVector.val[0] = vqaddq_s16(WordVector.val[0], ZeroPointVector); + WordVector.val[1] = vqaddq_s16(WordVector.val[1], ZeroPointVector); + + vst1q_u8(Output, vqmovun_high_s16(vqmovun_s16(WordVector.val[0]), WordVector.val[1])); + Output += 16; + + n -= 16; } + // + // Process the remaining columns of the matrices. + // + while (n > 0) { - MLAS_INT32X4 IntegerVector = _mm_cvtsi32_si128(*Input); + // + // Load the input data and optionally add the per-column bias. + // - if (bias != nullptr) { - BiasVector = _mm_cvtsi32_si128(*bias); - bias += 1; + int32x4_t IntegerVector; + + if (n >= 4) { + + IntegerVector = vld1q_s32(&Input[0]); + Input += 4; + + if (bias != nullptr) { + IntegerVector = vaddq_s32(IntegerVector, vld1q_s32(&bias[0])); + bias += 4; + } + + } else { + + IntegerVector = vld1q_dup_s32(Input); + Input += 1; + + if (bias != nullptr) { + IntegerVector = vaddq_s32(IntegerVector, vld1q_dup_s32(bias)); + bias += 1; + } } - IntegerVector = MlasRequantizeOutputVector(IntegerVector, BiasVector, - ScaleVector, MinimumValueVector, MaximumValueVector, ZeroPointVector); + // + // Convert to integer values to float and apply the per-tensor or + // per-column scaling. + // - *Output = (uint8_t)_mm_cvtsi128_si32(IntegerVector); + float32x4_t FloatVector = vcvtq_f32_s32(IntegerVector); + float32x4_t ScaleVector; - Input += 1; - Output += 1; - n -= 1; + if (scale != nullptr) { + + if (n >= 4) { + ScaleVector = vld1q_f32(scale); + scale += 4; + } else { + ScaleVector = vld1q_dup_f32(scale); + scale += 1; + } + + } else { + ScaleVector = PerMatrixScaleVector; + } + + FloatVector = vmulq_f32(FloatVector, ScaleVector); + + // + // Convert the float values to integer using "round to nearest even". + // Results are saturated to the range of int32_t. + // + + IntegerVector = vcvtnq_s32_f32(FloatVector); + + // + // Pack the integers with saturation to 16-bit values and shift by + // the zero point, then pack the integers again to unsigned bytes. + // + + int16x8_t WordVector = vcombine_s16(vqmovn_s32(IntegerVector), vdup_n_s16(0)); + WordVector = vqaddq_s16(WordVector, ZeroPointVector); + + uint8x16_t ByteVector = vcombine_u8(vqmovun_s16(WordVector), vdup_n_u8(0)); + + if (n >= 4) { + + vst1q_lane_u32(reinterpret_cast(Output), vreinterpretq_u32_u8(ByteVector), 0); + Output += 4; + + n -= 4; + + } else { + + vst1q_lane_u8(Output, ByteVector, 0); + Output += 1; + + n -= 1; + } } } } +#else + void MLASCALL -MlasRequantizeOutputColumn( +MlasRequantizeOutput( const int32_t* Input, uint8_t* Output, const int32_t* Bias, size_t M, size_t N, const float* Scale, + bool PerColumnScale, uint8_t ZeroPoint ) /*++ @@ -558,7 +803,11 @@ Arguments: N - Supplies the number of columns of the output matrix. - Scale - Supplies the quantization scale vector. + Scale - Supplies the quantization scale. + + PerColumnScale - Supplies true if the quantization scale has per-column + values, else false if a single quantization scale applies to the + entire matrix. ZeroPoint - Supplies the quantization zero point value. @@ -568,10 +817,9 @@ Return Value: --*/ { - MLAS_FLOAT32X4 MinimumValueVector = MlasBroadcastFloat32x4(float(0 - ZeroPoint)); - MLAS_FLOAT32X4 MaximumValueVector = MlasBroadcastFloat32x4(float(255 - ZeroPoint)); - MLAS_INT32X4 ZeroPointVector = MlasBroadcastInt32x4(ZeroPoint); - MLAS_INT32X4 BiasVector = _mm_setzero_si128(); + const float PerMatrixScaleValue = PerColumnScale ? 0.0f : *Scale; + const float MinimumValue = float(0 - ZeroPoint); + const float MaximumValue = float(255 - ZeroPoint); // // Step through each row of the output matrix. @@ -581,53 +829,36 @@ Return Value: const int32_t* bias = Bias; const float* scale = Scale; - size_t n = N; - while (n >= 4) { - - MLAS_INT32X4 IntegerVector = _mm_loadu_si128((const __m128i *)Input); - - if (bias != nullptr) { - BiasVector = _mm_loadu_si128((const __m128i*)bias); - bias += 4; - } - - MLAS_FLOAT32X4 ScaleVector = MlasLoadFloat32x4(scale); - scale += 4; - - IntegerVector = MlasRequantizeOutputVector(IntegerVector, BiasVector, - ScaleVector, MinimumValueVector, MaximumValueVector, ZeroPointVector); - - IntegerVector = _mm_packus_epi16(IntegerVector, IntegerVector); - IntegerVector = _mm_packus_epi16(IntegerVector, IntegerVector); - - *((int32_t*)Output) = _mm_cvtsi128_si32(IntegerVector); - - Input += 4; - Output += 4; - n -= 4; - } - while (n > 0) { - MLAS_INT32X4 IntegerVector = _mm_cvtsi32_si128(*Input); + int32_t IntegerValue = *Input++; if (bias != nullptr) { - BiasVector = _mm_cvtsi32_si128(*bias); - bias += 1; + IntegerValue += *bias++; } - MLAS_FLOAT32X4 ScaleVector = _mm_load_ss(scale); - scale += 1; + float FloatValue = float(IntegerValue); + float ScaleValue = PerColumnScale ? *scale++ : PerMatrixScaleValue; - IntegerVector = MlasRequantizeOutputVector(IntegerVector, BiasVector, - ScaleVector, MinimumValueVector, MaximumValueVector, ZeroPointVector); + FloatValue *= ScaleValue; + FloatValue = std::max(FloatValue, MinimumValue); + FloatValue = std::min(FloatValue, MaximumValue); - *Output = (uint8_t)_mm_cvtsi128_si32(IntegerVector); + // + // Use the fast rounding trick adapted from XNNPACK: bias the floating + // point value by the first floating point value that has no + // fractional bits. The add operation performs the "round to nearest + // even". Extract the mantissa bits from this floating point value to + // obtain the rounded integer value. + // + + IntegerValue = int32_t(MlasBitsOfFp32(FloatValue + MLAS_ROUNDING_BIAS_MAGIC)) - + MLAS_ROUNDING_BIAS_MAGIC_BITS; + + *Output++ = uint8_t(IntegerValue + ZeroPoint); - Input += 1; - Output += 1; n -= 1; } } diff --git a/onnxruntime/core/mlas/lib/transpose.cpp b/onnxruntime/core/mlas/lib/transpose.cpp index f52b514410..9a2c315c8c 100644 --- a/onnxruntime/core/mlas/lib/transpose.cpp +++ b/onnxruntime/core/mlas/lib/transpose.cpp @@ -16,7 +16,177 @@ Abstract: #include "mlasi.h" -#ifdef MLAS_TARGET_AMD64_IX86 +#if defined(MLAS_SSE2_INTRINSICS) || defined(MLAS_NEON_INTRINSICS) + +MLAS_FORCEINLINE +void +MlasTranspose8x8Block( + const uint8_t* Input, + size_t InputStride, + uint8_t* Output, + size_t OutputStride + ) +/*++ + +Routine Description: + + This routine transposes an eight by eight element block from the input + matrix to the output matrix. + +Arguments: + + Input - Supplies the input buffer. + + InputStride - Supplies the number of elements between rows of the input + matrix. + + Output - Supplies the output buffer. + + OutputStride - Supplies the number of elements between rows of the output + matrix. + +Return Value: + + None. + +--*/ +{ + +#if defined(MLAS_SSE2_INTRINSICS) + + __m128i a0 = _mm_loadl_epi64((const __m128i*)&Input[InputStride * 0]); + __m128i a1 = _mm_loadl_epi64((const __m128i*)&Input[InputStride * 1]); + __m128i b0 = _mm_unpacklo_epi8(a0, a1); + + __m128i a2 = _mm_loadl_epi64((const __m128i*)&Input[InputStride * 2]); + __m128i a3 = _mm_loadl_epi64((const __m128i*)&Input[InputStride * 3]); + __m128i b1 = _mm_unpacklo_epi8(a2, a3); + + __m128i a4 = _mm_loadl_epi64((const __m128i*)&Input[InputStride * 4]); + __m128i a5 = _mm_loadl_epi64((const __m128i*)&Input[InputStride * 5]); + __m128i b2 = _mm_unpacklo_epi8(a4, a5); + + __m128i a6 = _mm_loadl_epi64((const __m128i*)&Input[InputStride * 6]); + __m128i a7 = _mm_loadl_epi64((const __m128i*)&Input[InputStride * 7]); + __m128i b3 = _mm_unpacklo_epi8(a6, a7); + + __m128i c0 = _mm_unpacklo_epi16(b0, b1); + __m128i c1 = _mm_unpackhi_epi16(b0, b1); + __m128i c2 = _mm_unpacklo_epi16(b2, b3); + __m128i c3 = _mm_unpackhi_epi16(b2, b3); + + __m128 d0 = _mm_castsi128_ps(_mm_unpacklo_epi32(c0, c2)); + _mm_storel_pi((__m64*)&Output[OutputStride * 0], d0); + _mm_storeh_pi((__m64*)&Output[OutputStride * 1], d0); + + __m128 d1 = _mm_castsi128_ps(_mm_unpackhi_epi32(c0, c2)); + _mm_storel_pi((__m64*)&Output[OutputStride * 2], d1); + _mm_storeh_pi((__m64*)&Output[OutputStride * 3], d1); + + __m128 d2 = _mm_castsi128_ps(_mm_unpacklo_epi32(c1, c3)); + _mm_storel_pi((__m64*)&Output[OutputStride * 4], d2); + _mm_storeh_pi((__m64*)&Output[OutputStride * 5], d2); + + __m128 d3 = _mm_castsi128_ps(_mm_unpackhi_epi32(c1, c3)); + _mm_storel_pi((__m64*)&Output[OutputStride * 6], d3); + _mm_storeh_pi((__m64*)&Output[OutputStride * 7], d3); + +#elif defined(MLAS_NEON_INTRINSICS) + + uint8x8_t a0 = vld1_u8(&Input[InputStride * 0]); + uint8x8_t a1 = vld1_u8(&Input[InputStride * 1]); + uint8x8x2_t b0 = vzip_u8(a0, a1); + + uint8x8_t a2 = vld1_u8(&Input[InputStride * 2]); + uint8x8_t a3 = vld1_u8(&Input[InputStride * 3]); + uint8x8x2_t b1 = vzip_u8(a2, a3); + + uint8x8_t a4 = vld1_u8(&Input[InputStride * 4]); + uint8x8_t a5 = vld1_u8(&Input[InputStride * 5]); + uint8x8x2_t b2 = vzip_u8(a4, a5); + + uint8x8_t a6 = vld1_u8(&Input[InputStride * 6]); + uint8x8_t a7 = vld1_u8(&Input[InputStride * 7]); + uint8x8x2_t b3 = vzip_u8(a6, a7); + + uint16x4x2_t c0 = vzip_u16(vreinterpret_u16_u8(b0.val[0]), vreinterpret_u16_u8(b1.val[0])); + uint16x4x2_t c1 = vzip_u16(vreinterpret_u16_u8(b0.val[1]), vreinterpret_u16_u8(b1.val[1])); + uint16x4x2_t c2 = vzip_u16(vreinterpret_u16_u8(b2.val[0]), vreinterpret_u16_u8(b3.val[0])); + uint16x4x2_t c3 = vzip_u16(vreinterpret_u16_u8(b2.val[1]), vreinterpret_u16_u8(b3.val[1])); + + uint32x2x2_t d0 = vzip_u32(vreinterpret_u32_u16(c0.val[0]), vreinterpret_u32_u16(c2.val[0])); + uint32x2x2_t d1 = vzip_u32(vreinterpret_u32_u16(c0.val[1]), vreinterpret_u32_u16(c2.val[1])); + uint32x2x2_t d2 = vzip_u32(vreinterpret_u32_u16(c1.val[0]), vreinterpret_u32_u16(c3.val[0])); + uint32x2x2_t d3 = vzip_u32(vreinterpret_u32_u16(c1.val[1]), vreinterpret_u32_u16(c3.val[1])); + + vst1_u8(&Output[OutputStride * 0], vreinterpret_u8_u32(d0.val[0])); + vst1_u8(&Output[OutputStride * 1], vreinterpret_u8_u32(d0.val[1])); + vst1_u8(&Output[OutputStride * 2], vreinterpret_u8_u32(d1.val[0])); + vst1_u8(&Output[OutputStride * 3], vreinterpret_u8_u32(d1.val[1])); + vst1_u8(&Output[OutputStride * 4], vreinterpret_u8_u32(d2.val[0])); + vst1_u8(&Output[OutputStride * 5], vreinterpret_u8_u32(d2.val[1])); + vst1_u8(&Output[OutputStride * 6], vreinterpret_u8_u32(d3.val[0])); + vst1_u8(&Output[OutputStride * 7], vreinterpret_u8_u32(d3.val[1])); + +#endif + +} + +#endif + +MLAS_FORCEINLINE +void +MlasTranspose8xNVector( + const uint8_t* Input, + size_t InputStride, + uint8_t* Output, + size_t OutputStride + ) +/*++ + +Routine Description: + + This routine transposes an eight element vector from the input matrix + to the output matrix. + +Arguments: + + Input - Supplies the input buffer. + + InputStride - Supplies the number of elements between rows of the input + matrix. + + Output - Supplies the output buffer. + + OutputStride - Supplies the number of elements between rows of the output + matrix. + +Return Value: + + None. + +--*/ +{ + uint8_t a0 = Input[InputStride * 0]; + uint8_t a1 = Input[InputStride * 1]; + uint8_t a2 = Input[InputStride * 2]; + uint8_t a3 = Input[InputStride * 3]; + + Output[OutputStride * 0] = a0; + Output[OutputStride * 1] = a1; + Output[OutputStride * 2] = a2; + Output[OutputStride * 3] = a3; + + uint8_t a4 = Input[InputStride * 4]; + uint8_t a5 = Input[InputStride * 5]; + uint8_t a6 = Input[InputStride * 6]; + uint8_t a7 = Input[InputStride * 7]; + + Output[OutputStride * 4] = a4; + Output[OutputStride * 5] = a5; + Output[OutputStride * 6] = a6; + Output[OutputStride * 7] = a7; +} void MLASCALL @@ -64,60 +234,22 @@ Return Value: uint8_t* d = Output; size_t m = M; +#if defined(MLAS_SSE2_INTRINSICS) || defined(MLAS_NEON_INTRINSICS) + while (m >= 8) { - __m128i a0 = _mm_loadl_epi64((const __m128i*)&s[N * 0]); - __m128i a1 = _mm_loadl_epi64((const __m128i*)&s[N * 1]); - __m128i b0 = _mm_unpacklo_epi8(a0, a1); - - __m128i a2 = _mm_loadl_epi64((const __m128i*)&s[N * 2]); - __m128i a3 = _mm_loadl_epi64((const __m128i*)&s[N * 3]); - __m128i b1 = _mm_unpacklo_epi8(a2, a3); - - __m128i a4 = _mm_loadl_epi64((const __m128i*)&s[N * 4]); - __m128i a5 = _mm_loadl_epi64((const __m128i*)&s[N * 5]); - __m128i b2 = _mm_unpacklo_epi8(a4, a5); - - __m128i a6 = _mm_loadl_epi64((const __m128i*)&s[N * 6]); - __m128i a7 = _mm_loadl_epi64((const __m128i*)&s[N * 7]); - __m128i b3 = _mm_unpacklo_epi8(a6, a7); - - __m128i c0 = _mm_unpacklo_epi16(b0, b1); - __m128i c1 = _mm_unpackhi_epi16(b0, b1); - __m128i c2 = _mm_unpacklo_epi16(b2, b3); - __m128i c3 = _mm_unpackhi_epi16(b2, b3); - - __m128 d0 = _mm_castsi128_ps(_mm_unpacklo_epi32(c0, c2)); - _mm_storel_pi((__m64*)&d[M * 0], d0); - _mm_storeh_pi((__m64*)&d[M * 1], d0); - - __m128 d1 = _mm_castsi128_ps(_mm_unpackhi_epi32(c0, c2)); - _mm_storel_pi((__m64*)&d[M * 2], d1); - _mm_storeh_pi((__m64*)&d[M * 3], d1); - - __m128 d2 = _mm_castsi128_ps(_mm_unpacklo_epi32(c1, c3)); - _mm_storel_pi((__m64*)&d[M * 4], d2); - _mm_storeh_pi((__m64*)&d[M * 5], d2); - - __m128 d3 = _mm_castsi128_ps(_mm_unpackhi_epi32(c1, c3)); - _mm_storel_pi((__m64*)&d[M * 6], d3); - _mm_storeh_pi((__m64*)&d[M * 7], d3); + MlasTranspose8x8Block(s, N, d, M); s += N * 8; d += 8; m -= 8; } +#endif + while (m > 0) { - d[M * 0] = s[0]; - d[M * 1] = s[1]; - d[M * 2] = s[2]; - d[M * 3] = s[3]; - d[M * 4] = s[4]; - d[M * 5] = s[5]; - d[M * 6] = s[6]; - d[M * 7] = s[7]; + MlasTranspose8xNVector(s, 1, d, M); s += N; d += 1; @@ -142,14 +274,7 @@ Return Value: while (m >= 8) { - d[0] = s[N * 0]; - d[1] = s[N * 1]; - d[2] = s[N * 2]; - d[3] = s[N * 3]; - d[4] = s[N * 4]; - d[5] = s[N * 5]; - d[6] = s[N * 6]; - d[7] = s[N * 7]; + MlasTranspose8xNVector(s, N, d, 1); s += N * 8; d += 8; @@ -170,5 +295,3 @@ Return Value: n -= 1; } } - -#endif diff --git a/onnxruntime/core/optimizer/graph_transformer_utils.cc b/onnxruntime/core/optimizer/graph_transformer_utils.cc index 4c48d69088..72c6f384b1 100644 --- a/onnxruntime/core/optimizer/graph_transformer_utils.cc +++ b/onnxruntime/core/optimizer/graph_transformer_utils.cc @@ -168,7 +168,7 @@ std::vector> GenerateTransformers(TransformerL transformers.emplace_back(onnxruntime::make_unique()); } -#if defined(MLAS_TARGET_AMD64_IX86) +#ifdef MLAS_SUPPORTS_GEMM_U8X8 transformers.emplace_back(onnxruntime::make_unique()); #endif #endif diff --git a/onnxruntime/core/providers/cpu/math/quantize_linear_matmul.cc b/onnxruntime/core/providers/cpu/math/quantize_linear_matmul.cc index ca7285ca0f..bd0add9ae9 100644 --- a/onnxruntime/core/providers/cpu/math/quantize_linear_matmul.cc +++ b/onnxruntime/core/providers/cpu/math/quantize_linear_matmul.cc @@ -70,7 +70,7 @@ Status QLinearMatMul::Compute(OpKernelContext* ctx) const { const float real_multiplier = (a_scale_data * b_scale_data) / y_scale_data; -#ifdef MLAS_SUPPORTS_GEMM_U8X8_AND_REQUANTIZE_OUTPUT +#ifdef MLAS_SUPPORTS_GEMM_U8X8 AllocatorPtr alloc; ORT_RETURN_IF_ERROR(ctx->GetTempSpaceAllocator(&alloc)); auto gemm_output_data = alloc->Alloc(SafeInt(sizeof(int32_t)) * @@ -85,7 +85,7 @@ Status QLinearMatMul::Compute(OpKernelContext* ctx) const { #endif for (size_t i = 0; i < helper.OutputOffsets().size(); i++) { -#ifdef MLAS_SUPPORTS_GEMM_U8X8_AND_REQUANTIZE_OUTPUT +#ifdef MLAS_SUPPORTS_GEMM_U8X8 QGemm(static_cast(helper.M()), static_cast(helper.N()), static_cast(helper.K()), @@ -105,7 +105,8 @@ Status QLinearMatMul::Compute(OpKernelContext* ctx) const { nullptr, static_cast(helper.M()), static_cast(helper.N()), - real_multiplier, + &real_multiplier, + false, *y_offset->template Data()); #else GemmlowpMultiplyu8u8_u8(a->template Data() + helper.LeftOffsets()[i], diff --git a/onnxruntime/core/providers/cpu/nn/qlinearconv.cc b/onnxruntime/core/providers/cpu/nn/qlinearconv.cc index 11f7b382f4..002d8ffdeb 100644 --- a/onnxruntime/core/providers/cpu/nn/qlinearconv.cc +++ b/onnxruntime/core/providers/cpu/nn/qlinearconv.cc @@ -13,7 +13,7 @@ namespace onnxruntime { -#if defined(MLAS_TARGET_AMD64_IX86) +#ifdef MLAS_SUPPORTS_GEMM_U8X8 class QLinearConv : public OpKernel { public: @@ -69,6 +69,8 @@ ONNX_CPU_OPERATOR_KERNEL( namespace contrib { +// Register an alternate version of this kernel that supports the channels_last +// attribute in order to consume and produce NHWC tensors. ONNX_OPERATOR_KERNEL_EX( QLinearConv, kMSDomain, @@ -525,23 +527,14 @@ Status QLinearConv::Compute(OpKernelContext* context) const { } } - if (output_scales.size() == 1) { - MlasRequantizeOutputColumn(worker_gemm_output, - worker_requantize_output, - Bdata, - static_cast(output_count), - static_cast(M), - output_scales[0], - Y_zero_point_value); - } else { - MlasRequantizeOutputColumn(worker_gemm_output, - worker_requantize_output, - Bdata, - static_cast(output_count), - static_cast(M), - output_scales.data(), - Y_zero_point_value); - } + MlasRequantizeOutput(worker_gemm_output, + worker_requantize_output, + Bdata, + static_cast(output_count), + static_cast(M), + output_scales.data(), + output_scales.size() > 1, + Y_zero_point_value); }; concurrency::ThreadPool::TrySimpleParallelFor(thread_pool, thread_count, conv_worker); @@ -681,18 +674,10 @@ Status QLinearConv::Compute(OpKernelContext* context) const { const float real_multiplier = (X_scale_value * W_scale_value) / Y_scale_value; -#ifdef MLAS_SUPPORTS_GEMM_U8X8_AND_REQUANTIZE_OUTPUT - // Use an intermediate int32_t buffer for the GEMM computation before - // requantizing to the output type. - auto* gemm_output_data = alloc->Alloc(SafeInt(sizeof(int32_t)) * Y_offset); - BufferUniquePtr gemm_output_buffer(gemm_output_data, BufferDeleter(alloc)); - auto* gemm_output = static_cast(gemm_output_buffer.get()); -#else // Compute the fixed point multiplier and shift for requantizing with GEMMLOWP. int32_t integer_multiplier; int right_shift; QuantizeMultiplier(real_multiplier, &integer_multiplier, &right_shift); -#endif const auto* Xdata = X->template Data(); const auto* Wdata = W->template Data(); @@ -737,29 +722,6 @@ Status QLinearConv::Compute(OpKernelContext* context) const { } } -#ifdef MLAS_SUPPORTS_GEMM_U8X8_AND_REQUANTIZE_OUTPUT - QGemm(static_cast(group_output_channels), - static_cast(output_image_size), - static_cast(kernel_dim), - Wdata + group_id * W_offset, - static_cast(kernel_dim), - W_zero_point_value, - col_buffer_data == nullptr ? Xdata : col_buffer_data, - static_cast(output_image_size), - X_zero_point_value, - false, - gemm_output, - static_cast(output_image_size), - context->GetOperatorThreadPool()); - - MlasRequantizeOutput(gemm_output, - Ydata, - Bdata != nullptr ? Bdata + group_id * group_output_channels : nullptr, - static_cast(group_output_channels), - static_cast(output_image_size), - real_multiplier, - Y_zero_point_value); -#else GemmlowpMultiplyu8u8_u8(Wdata + group_id * W_offset, col_buffer_data == nullptr ? Xdata : col_buffer_data, Ydata, @@ -772,7 +734,6 @@ Status QLinearConv::Compute(OpKernelContext* context) const { integer_multiplier, right_shift, Bdata != nullptr ? Bdata + group_id * group_output_channels : nullptr); -#endif Xdata += X_offset; Ydata += Y_offset; diff --git a/onnxruntime/core/providers/cpu/tensor/transpose.cc b/onnxruntime/core/providers/cpu/tensor/transpose.cc index 2346ec5fc3..d99bbb70ac 100644 --- a/onnxruntime/core/providers/cpu/tensor/transpose.cc +++ b/onnxruntime/core/providers/cpu/tensor/transpose.cc @@ -301,8 +301,6 @@ static void SimpleTransposeSingleAxisOutwards(const T* input_data, T* output_dat } } -#ifdef MLAS_SUPPORTS_TRANSPOSE - static void SimpleTransposeSingleAxisOutwards(const uint8_t* input_data, uint8_t* output_data, int64_t num_loops, int64_t num_writers, int64_t writes_per_loop, int64_t writes_per_writer_per_loop) { @@ -316,8 +314,6 @@ static void SimpleTransposeSingleAxisOutwards(const uint8_t* input_data, uint8_t } } -#endif - // `input_shape_override` overrides the shape of `input` for compute purposes. static void TransposeSingleAxisOutwards(const std::vector& permutations, const Tensor& input, Tensor& output, int64_t from, int64_t to, const TensorShape* input_shape_override = nullptr) { @@ -411,8 +407,6 @@ static void SimpleTransposeSingleAxisInwards(const T* input_data, T* output_data } } -#ifdef MLAS_SUPPORTS_TRANSPOSE - static void SimpleTransposeSingleAxisInwards(const uint8_t* input_data, uint8_t* output_data, int64_t num_loops, int64_t num_readers, int64_t reads_per_loop, int64_t reads_per_reader_per_loop) { @@ -426,8 +420,6 @@ static void SimpleTransposeSingleAxisInwards(const uint8_t* input_data, uint8_t* } } -#endif - // moving a single axis inwards where the read/write size is a power of 2 and between 8 and 64 bits. // `input_shape_override` overrides the shape of `input` for compute purposes. static void TransposeSingleAxisInwards(const std::vector& permutations, const Tensor& input, Tensor& output, diff --git a/onnxruntime/test/optimizer/nhwc_transformer_test.cc b/onnxruntime/test/optimizer/nhwc_transformer_test.cc index 6363388b76..1e04c149bd 100644 --- a/onnxruntime/test/optimizer/nhwc_transformer_test.cc +++ b/onnxruntime/test/optimizer/nhwc_transformer_test.cc @@ -297,7 +297,7 @@ void NhwcTransformerTester(const std::function& bu #ifndef DISABLE_CONTRIB_OPS -#if defined(MLAS_TARGET_AMD64_IX86) +#ifdef MLAS_SUPPORTS_GEMM_U8X8 TEST(NhwcTransformerTests, Conv) { auto test_case = [&](const std::vector& input_shape, const std::vector& weights_shape) { diff --git a/onnxruntime/test/providers/cpu/nn/qlinearconv_op_test.cc b/onnxruntime/test/providers/cpu/nn/qlinearconv_op_test.cc index 80b3c56871..d34a777453 100644 --- a/onnxruntime/test/providers/cpu/nn/qlinearconv_op_test.cc +++ b/onnxruntime/test/providers/cpu/nn/qlinearconv_op_test.cc @@ -260,7 +260,7 @@ TEST(QLinearConvTest, WithGroup_2D) { {}); } -#if defined(MLAS_TARGET_AMD64_IX86) +#ifdef MLAS_SUPPORTS_GEMM_U8X8 template class QLinearConvOpTester { @@ -764,6 +764,44 @@ TEST(QLinearConvTest, Conv3D_U8S8_Depthwise) { test.Run(); } +TEST(QLinearConvTest, Conv2D_U8S8_Requantize_NoBias) { + for (int64_t channels = 1; channels <= 32; channels++) { + QLinearConvOpTester test; + test.GenerateRandomInput({1, 8, 5, 5}, .05f, 4); + test.GenerateRandomWeights({channels, 8, 3, 3}, .125f, 0); + test.SetPads({1, 1, 1, 1}); + test.SetOutputScaleAndZeroPoint(.55f, 56); + test.Run(); + } +} + +TEST(QLinearConvTest, Conv2D_U8S8_Requantize_Bias) { + for (int64_t channels = 1; channels <= 32; channels++) { + QLinearConvOpTester test; + test.GenerateRandomInput({1, 8, 5, 5}, .05f, 4); + test.GenerateRandomWeights({channels, 8, 3, 3}, .125f, 0); + test.GenerateRandomBias(); + test.SetPads({1, 1, 1, 1}); + test.SetOutputScaleAndZeroPoint(.55f, 56); + test.Run(); + } +} + +TEST(QLinearConvTest, Conv2D_U8S8_Requantize_Bias_PerChannel) { + std::vector weight_scales; + for (int64_t channels = 1; channels <= 32; channels++) { + QLinearConvOpTester test; + test.GenerateRandomInput({1, 8, 5, 5}, .05f, 4); + test.GenerateRandomWeights({channels, 8, 3, 3}, .125f, 0); + weight_scales.push_back(.120f + .002f * static_cast(channels)); + test.SetWeightScales(weight_scales); + test.GenerateRandomBias(); + test.SetPads({1, 1, 1, 1}); + test.SetOutputScaleAndZeroPoint(.55f, 56); + test.Run(); + } +} + #endif } // namespace