diff --git a/cmake/onnxruntime_mlas.cmake b/cmake/onnxruntime_mlas.cmake index 46b429f21c..c6204268b2 100644 --- a/cmake/onnxruntime_mlas.cmake +++ b/cmake/onnxruntime_mlas.cmake @@ -6,6 +6,7 @@ set(mlas_common_srcs ${ONNXRUNTIME_ROOT}/core/mlas/lib/threading.cpp ${ONNXRUNTIME_ROOT}/core/mlas/lib/sgemm.cpp ${ONNXRUNTIME_ROOT}/core/mlas/lib/qgemm.cpp + ${ONNXRUNTIME_ROOT}/core/mlas/lib/qdwconv.cpp ${ONNXRUNTIME_ROOT}/core/mlas/lib/convolve.cpp ${ONNXRUNTIME_ROOT}/core/mlas/lib/pooling.cpp ${ONNXRUNTIME_ROOT}/core/mlas/lib/transpose.cpp @@ -59,13 +60,13 @@ if(MSVC) elseif(onnxruntime_target_platform STREQUAL "x64") enable_language(ASM_MASM) - set(mlas_platform_srcs_avx - ${ONNXRUNTIME_ROOT}/core/mlas/lib/intrinsics/avx/min_max_elements.cpp + file(GLOB_RECURSE mlas_platform_srcs_avx CONFIGURE_DEPENDS + "${ONNXRUNTIME_ROOT}/core/mlas/lib/intrinsics/avx/*.cpp" ) set_source_files_properties(${mlas_platform_srcs_avx} PROPERTIES COMPILE_FLAGS "/arch:AVX") - set(mlas_platform_srcs_avx2 - ${ONNXRUNTIME_ROOT}/core/mlas/lib/intrinsics/avx2/qladd_avx2.cpp + file(GLOB_RECURSE mlas_platform_srcs_avx2 CONFIGURE_DEPENDS + "${ONNXRUNTIME_ROOT}/core/mlas/lib/intrinsics/avx2/*.cpp" ) set_source_files_properties(${mlas_platform_srcs_avx2} PROPERTIES COMPILE_FLAGS "/arch:AVX2") @@ -259,6 +260,7 @@ else() ${ONNXRUNTIME_ROOT}/core/mlas/lib/x86_64/TanhKernelFma3.S ${ONNXRUNTIME_ROOT}/core/mlas/lib/x86_64/ErfKernelFma3.S ${ONNXRUNTIME_ROOT}/core/mlas/lib/intrinsics/avx2/qladd_avx2.cpp + ${ONNXRUNTIME_ROOT}/core/mlas/lib/intrinsics/avx2/qdwconv_avx2.cpp ) set_source_files_properties(${mlas_platform_srcs_avx2} PROPERTIES COMPILE_FLAGS "-mavx2 -mfma") @@ -352,7 +354,7 @@ else() endif() add_library(onnxruntime_mlas STATIC ${mlas_common_srcs} ${mlas_platform_srcs}) -target_include_directories(onnxruntime_mlas PRIVATE ${ONNXRUNTIME_ROOT}/core/mlas/inc ${ONNXRUNTIME_ROOT}/core/mlas/lib ${ONNXRUNTIME_ROOT}/core/mlas/lib/amd64) +target_include_directories(onnxruntime_mlas PRIVATE ${ONNXRUNTIME_ROOT}/core/mlas/inc ${ONNXRUNTIME_ROOT}/core/mlas/lib) set_target_properties(onnxruntime_mlas PROPERTIES FOLDER "ONNXRuntime") if (WIN32) target_compile_options(onnxruntime_mlas PRIVATE "/wd6385") diff --git a/onnxruntime/core/mlas/inc/mlas.h b/onnxruntime/core/mlas/inc/mlas.h index 3405c6767e..9dfadf22ec 100644 --- a/onnxruntime/core/mlas/inc/mlas.h +++ b/onnxruntime/core/mlas/inc/mlas.h @@ -422,14 +422,14 @@ MlasConv( MLAS_THREADPOOL* ThreadPool ); -template void MLASCALL MlasConvDepthwise( - const uint8_t* Input, + const uint8_t* const* Input, uint8_t InputZeroPoint, - const FilterType* Filter, - FilterType FilterZeroPoint, + const uint8_t* Filter, + uint8_t FilterZeroPoint, + bool FilterIsSigned, int32_t* Output, size_t Channels, size_t OutputCount, diff --git a/onnxruntime/core/mlas/lib/convolve.cpp b/onnxruntime/core/mlas/lib/convolve.cpp index 9de09e9fe4..72141607cf 100644 --- a/onnxruntime/core/mlas/lib/convolve.cpp +++ b/onnxruntime/core/mlas/lib/convolve.cpp @@ -1257,210 +1257,3 @@ Return Value: *WorkingBufferSize = TargetThreadCount * MLAS_CONV_WORKING_BUFFER_SIZE_PER_THREAD; } } - -template -void -MLASCALL -MlasConvDepthwise( - const uint8_t* Input, - uint8_t InputZeroPoint, - const FilterType* Filter, - FilterType FilterZeroPoint, - int32_t* Output, - size_t Channels, - size_t OutputCount, - size_t KernelSize - ) -/*++ - -Routine Description: - - This routine implements the depthwise convolution operation. - - The input tensor is organized in channels last format (NHWC) after applying - the Im2col transform, so the length of each row of the input tensor is - Channels times KernelSize. The number of columns of the input tensor is - OutputCount. - - The filter tensor is organized in HW1O format, so the length of each row of - the filter tensor is Channels. The number of columns of the filter tensor - is KernelSize. - -Arguments: - - Input - Supplies the input tensor in channels last format. - - InputZeroPoint - Supplies the zero point offset of the input tensor. - - Filter - Supplies the filter tensor. - - FilterZeroPoint - Supplies the zero point offset of the filter tensor. - - Output - Supplies the output tensor in channels last format. - - Channels - Supplies the number of channels. - - OutputCount - Supplies the number of channel sized output elements to - produce. - - KernelSize - Supplies the total number of channel sized kernel elements to - consume. - -Return Value: - - None. - ---*/ -{ -#if defined(MLAS_SSE2_INTRINSICS) - const __m128i ZeroVector = _mm_setzero_si128(); - const __m128i InputZeroPointVector = _mm_set1_epi16(InputZeroPoint); - const __m128i FilterZeroPointVector = _mm_set1_epi16(FilterZeroPoint); -#elif defined(MLAS_NEON_INTRINSICS) - const uint8x8_t InputZeroPointVector = vdup_n_u8(InputZeroPoint); - const uint8x8_t FilterZeroPointVector = vdup_n_u8(uint8_t(FilterZeroPoint)); -#endif - - while (OutputCount > 0) { - - size_t ChannelOffset = 0; - size_t c = Channels; - -#if defined(MLAS_SSE2_INTRINSICS) - - while (c >= 8) { - - __m128i Accumulator0 = _mm_setzero_si128(); - __m128i Accumulator1 = _mm_setzero_si128(); - size_t ChannelKernelOffset = ChannelOffset; - - for (size_t k = 0; k < KernelSize; k++) { - - __m128i InputVector = _mm_loadl_epi64((const __m128i*)&Input[ChannelKernelOffset]); - __m128i FilterVector = _mm_loadl_epi64((const __m128i*)&Filter[ChannelKernelOffset]); - - InputVector = _mm_unpacklo_epi8(InputVector, ZeroVector); - - if (std::is_signed::value) { - FilterVector = _mm_srai_epi16(_mm_unpacklo_epi8(ZeroVector, FilterVector), 8); - } else { - FilterVector = _mm_unpacklo_epi8(FilterVector, ZeroVector); - } - - InputVector = _mm_sub_epi16(InputVector, InputZeroPointVector); - FilterVector = _mm_sub_epi16(FilterVector, FilterZeroPointVector); - - // N.B. Emulate PMULLD functionality on SSE2 by computing the low - // and high parts of the result and interleaving the results. - __m128i MultiplyLowWords = _mm_mullo_epi16(InputVector, FilterVector); - __m128i MultiplyHighWords = _mm_mulhi_epi16(InputVector, FilterVector); - __m128i Multiply0 = _mm_unpacklo_epi16(MultiplyLowWords, MultiplyHighWords); - __m128i Multiply1 = _mm_unpackhi_epi16(MultiplyLowWords, MultiplyHighWords); - - Accumulator0 = _mm_add_epi32(Accumulator0, Multiply0); - Accumulator1 = _mm_add_epi32(Accumulator1, Multiply1); - ChannelKernelOffset += Channels; - } - - _mm_storeu_si128((__m128i*)&Output[0], Accumulator0); - _mm_storeu_si128((__m128i*)&Output[4], Accumulator1); - Output += 8; - - ChannelOffset += 8; - c -= 8; - } - -#elif defined(MLAS_NEON_INTRINSICS) - - while (c >= 8) { - - int32x4_t Accumulator0 = vdupq_n_s32(0); - int32x4_t Accumulator1 = vdupq_n_s32(0); - size_t ChannelKernelOffset = ChannelOffset; - - for (size_t k = 0; k < KernelSize; k++) { - - uint8x8_t InputVector = vld1_u8(&Input[ChannelKernelOffset]); - uint8x8_t FilterVector = vld1_u8(reinterpret_cast(&Filter[ChannelKernelOffset])); - - int16x8_t InputVector16 = vreinterpretq_s16_u16(vsubl_u8(InputVector, InputZeroPointVector)); - int16x8_t FilterVector16; - - if (std::is_signed::value) { - FilterVector16 = vsubl_s8(vreinterpret_s8_u8(FilterVector), vreinterpret_s8_u8(FilterZeroPointVector)); - } else { - FilterVector16 = vreinterpretq_s16_u16(vsubl_u8(FilterVector, FilterZeroPointVector)); - } - - Accumulator0 = vmlal_s16(Accumulator0, vget_low_s16(InputVector16), vget_low_s16(FilterVector16)); -#if defined(MLAS_NEON64_INTRINSICS) - Accumulator1 = vmlal_high_s16(Accumulator1, InputVector16, FilterVector16); -#else - Accumulator1 = vmlal_s16(Accumulator1, vget_high_s16(InputVector16), vget_high_s16(FilterVector16)); -#endif - - ChannelKernelOffset += Channels; - } - - vst1q_s32(&Output[0], Accumulator0); - vst1q_s32(&Output[4], Accumulator1); - Output += 8; - - ChannelOffset += 8; - c -= 8; - } - -#endif - - while (c > 0) { - - int32_t Accumulator = 0; - size_t ChannelKernelOffset = ChannelOffset; - - for (size_t k = 0; k < KernelSize; k++) { - - int32_t InputValue = int32_t(Input[ChannelKernelOffset]) - InputZeroPoint; - int32_t FilterValue = int32_t(Filter[ChannelKernelOffset]) - FilterZeroPoint; - - Accumulator += InputValue * FilterValue; - ChannelKernelOffset += Channels; - } - - *Output++ = Accumulator; - - ChannelOffset += 1; - c -= 1; - } - - Input += Channels * KernelSize; - OutputCount -= 1; - } -} - -template -void -MLASCALL -MlasConvDepthwise( - const uint8_t* Input, - uint8_t InputZeroPoint, - const int8_t* Filter, - int8_t FilterZeroPoint, - int32_t* Output, - size_t Channels, - size_t OutputCount, - size_t KernelSize - ); - -template -void -MLASCALL -MlasConvDepthwise( - const uint8_t* Input, - uint8_t InputZeroPoint, - const uint8_t* Filter, - uint8_t FilterZeroPoint, - int32_t* Output, - size_t Channels, - size_t OutputCount, - size_t KernelSize - ); diff --git a/onnxruntime/core/mlas/lib/intrinsics/avx/min_max_elements.cpp b/onnxruntime/core/mlas/lib/intrinsics/avx/min_max_elements.cpp index a3dd6cc1e0..0339f65eac 100644 --- a/onnxruntime/core/mlas/lib/intrinsics/avx/min_max_elements.cpp +++ b/onnxruntime/core/mlas/lib/intrinsics/avx/min_max_elements.cpp @@ -14,7 +14,7 @@ Abstract: --*/ -#include "../mlasi.h" +#include "mlasi.h" void MLASCALL @@ -103,4 +103,4 @@ MlasReduceMinimumMaximumF32KernelAvx( *Min = tmp_min; *Max = tmp_max; -} \ No newline at end of file +} diff --git a/onnxruntime/core/mlas/lib/intrinsics/avx2/qdwconv_avx2.cpp b/onnxruntime/core/mlas/lib/intrinsics/avx2/qdwconv_avx2.cpp new file mode 100644 index 0000000000..2fd46fbbcd --- /dev/null +++ b/onnxruntime/core/mlas/lib/intrinsics/avx2/qdwconv_avx2.cpp @@ -0,0 +1,183 @@ +/*++ + +Copyright (c) Microsoft Corporation. All rights reserved. + +Licensed under the MIT License. + +Module Name: + + qdwconv_avx2.cpp + +Abstract: + + This module implements the quantized integer depthwise convolution kernels. + + This implementation uses AVX2 instructions. + +--*/ + +#include "mlasi.h" + +template +void +MLASCALL +MlasConvDepthwiseKernelAvx2( + const uint8_t* const* Input, + uint8_t InputZeroPoint, + const FilterType* Filter, + FilterType FilterZeroPoint, + int32_t* Output, + size_t Channels, + size_t OutputCount, + size_t KernelSize + ) +{ + const __m256i InputZeroPointVector = _mm256_set1_epi16(InputZeroPoint); + const __m256i FilterZeroPointVector = _mm256_set1_epi16(FilterZeroPoint); + + while (OutputCount > 0) { + + size_t ChannelOffset = 0; + size_t c = Channels; + + while (c >= 16) { + + __m256i Accumulator0 = _mm256_setzero_si256(); + __m256i Accumulator1 = _mm256_setzero_si256(); + size_t ChannelKernelOffset = ChannelOffset; + + for (size_t k = 0; k < KernelSize; k++) { + + __m256i InputVector = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)&Input[k][ChannelOffset])); + __m256i FilterVector; + + if (std::is_signed::value) { + FilterVector = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*)&Filter[ChannelKernelOffset])); + } else { + FilterVector = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)&Filter[ChannelKernelOffset])); + } + + InputVector = _mm256_sub_epi16(InputVector, InputZeroPointVector); + FilterVector = _mm256_sub_epi16(FilterVector, FilterZeroPointVector); + + // N.B. The original SSE2 implementation used PMULLW/PMULHW in + // order to emulate the SSE 4.1 PMULLD instruction, however this + // implementation ends up being faster for some CPUs than + // extending to 32-bits and using PMULLD. + __m256i MultiplyLowWords = _mm256_mullo_epi16(InputVector, FilterVector); + __m256i MultiplyHighWords = _mm256_mulhi_epi16(InputVector, FilterVector); + __m256i Multiply0 = _mm256_unpacklo_epi16(MultiplyLowWords, MultiplyHighWords); + __m256i Multiply1 = _mm256_unpackhi_epi16(MultiplyLowWords, MultiplyHighWords); + + Accumulator0 = _mm256_add_epi32(Accumulator0, Multiply0); + Accumulator1 = _mm256_add_epi32(Accumulator1, Multiply1); + ChannelKernelOffset += Channels; + } + + // N.B. The above interleaving of the intermediate results leaves + // the accumulators in a swizzled layout, because the interleaving + // is per 128-bit half of the __m256i register. Reorder the results + // now to get the expected sequential order. + __m256i Reorder0 = _mm256_permute2x128_si256(Accumulator0, Accumulator1, 0x20); + __m256i Reorder1 = _mm256_permute2x128_si256(Accumulator0, Accumulator1, 0x31); + + _mm256_storeu_si256((__m256i*)&Output[0], Reorder0); + _mm256_storeu_si256((__m256i*)&Output[8], Reorder1); + Output += 16; + + ChannelOffset += 16; + c -= 16; + } + + if (c >= 8) { + + __m128i Accumulator0 = _mm_setzero_si128(); + __m128i Accumulator1 = _mm_setzero_si128(); + size_t ChannelKernelOffset = ChannelOffset; + + for (size_t k = 0; k < KernelSize; k++) { + + __m128i InputVector = _mm_loadl_epi64((const __m128i*)&Input[k][ChannelOffset]); + __m128i FilterVector = _mm_loadl_epi64((const __m128i*)&Filter[ChannelKernelOffset]); + + InputVector = _mm_cvtepu8_epi16(InputVector); + + if (std::is_signed::value) { + FilterVector = _mm_cvtepi8_epi16(FilterVector); + } else { + FilterVector = _mm_cvtepu8_epi16(FilterVector); + } + + InputVector = _mm_sub_epi16(InputVector, _mm256_castsi256_si128(InputZeroPointVector)); + FilterVector = _mm_sub_epi16(FilterVector, _mm256_castsi256_si128(FilterZeroPointVector)); + + __m128i MultiplyLowWords = _mm_mullo_epi16(InputVector, FilterVector); + __m128i MultiplyHighWords = _mm_mulhi_epi16(InputVector, FilterVector); + __m128i Multiply0 = _mm_unpacklo_epi16(MultiplyLowWords, MultiplyHighWords); + __m128i Multiply1 = _mm_unpackhi_epi16(MultiplyLowWords, MultiplyHighWords); + + Accumulator0 = _mm_add_epi32(Accumulator0, Multiply0); + Accumulator1 = _mm_add_epi32(Accumulator1, Multiply1); + ChannelKernelOffset += Channels; + } + + _mm_storeu_si128((__m128i*)&Output[0], Accumulator0); + _mm_storeu_si128((__m128i*)&Output[4], Accumulator1); + Output += 8; + + ChannelOffset += 8; + c -= 8; + } + + while (c > 0) { + + int32_t Accumulator = 0; + size_t ChannelKernelOffset = ChannelOffset; + + for (size_t k = 0; k < KernelSize; k++) { + + int32_t InputValue = int32_t(Input[k][ChannelOffset]) - InputZeroPoint; + int32_t FilterValue = int32_t(Filter[ChannelKernelOffset]) - FilterZeroPoint; + + Accumulator += InputValue * FilterValue; + ChannelKernelOffset += Channels; + } + + *Output++ = Accumulator; + + ChannelOffset += 1; + c -= 1; + } + + Input += KernelSize; + OutputCount -= 1; + } +} + +template +void +MLASCALL +MlasConvDepthwiseKernelAvx2( + const uint8_t* const* Input, + uint8_t InputZeroPoint, + const int8_t* Filter, + int8_t FilterZeroPoint, + int32_t* Output, + size_t Channels, + size_t OutputCount, + size_t KernelSize + ); + +template +void +MLASCALL +MlasConvDepthwiseKernelAvx2( + const uint8_t* const* Input, + uint8_t InputZeroPoint, + const uint8_t* Filter, + uint8_t FilterZeroPoint, + int32_t* Output, + size_t Channels, + size_t OutputCount, + size_t KernelSize + ); diff --git a/onnxruntime/core/mlas/lib/mlasi.h b/onnxruntime/core/mlas/lib/mlasi.h index da03f355b1..45003dfc32 100644 --- a/onnxruntime/core/mlas/lib/mlasi.h +++ b/onnxruntime/core/mlas/lib/mlasi.h @@ -513,6 +513,23 @@ void typedef MLAS_QUANTIZE_LINEAR_S8_KERNEL* PMLAS_QUANTIZE_LINEAR_S8_KERNEL; +template +struct MLAS_U8X8_KERNEL +{ + typedef + void + (MLASCALL DepthwiseKernel)( + const uint8_t* const* Input, + uint8_t InputZeroPoint, + const FilterType* Filter, + FilterType FilterZeroPoint, + int32_t* Output, + size_t Channels, + size_t OutputCount, + size_t KernelSize + ); +}; + extern "C" { #if defined(MLAS_TARGET_AMD64_IX86) @@ -701,6 +718,38 @@ MlasGemmU8X8PackedOperation( const MLAS_GEMM_U8X8_WORK_BLOCK* WorkBlock ); +// +// Quantized depthwise convolution kernels. +// + +template +void +MLASCALL +MlasConvDepthwiseKernel( + const uint8_t* const* Input, + uint8_t InputZeroPoint, + const FilterType* Filter, + FilterType FilterZeroPoint, + int32_t* Output, + size_t Channels, + size_t OutputCount, + size_t KernelSize + ); + +template +void +MLASCALL +MlasConvDepthwiseKernelAvx2( + const uint8_t* const* Input, + uint8_t InputZeroPoint, + const FilterType* Filter, + FilterType FilterZeroPoint, + int32_t* Output, + size_t Channels, + size_t OutputCount, + size_t KernelSize + ); + // // Environment information class. // @@ -733,6 +782,8 @@ struct MLAS_PLATFORM { PMLAS_COMPUTE_UNARY_FLOAT_KERNEL ErfKernelRoutine; PMLAS_QLINEAR_BINARY_OP_S8_KERNEL QLinearAddS8Kernel; PMLAS_QLINEAR_BINARY_OP_U8_KERNEL QLinearAddU8Kernel; + MLAS_U8X8_KERNEL::DepthwiseKernel* ConvDepthwiseU8S8Kernel; + MLAS_U8X8_KERNEL::DepthwiseKernel* ConvDepthwiseU8U8Kernel; PMLAS_COMPUTE_UNARY_FLOAT_KERNEL ComputeExpF32Kernel; PMLAS_COMPUTE_UNARY_FLOAT_KERNEL LogisticKernelRoutine; PMLAS_COMPUTE_UNARY_FLOAT_KERNEL TanhKernelRoutine; diff --git a/onnxruntime/core/mlas/lib/platform.cpp b/onnxruntime/core/mlas/lib/platform.cpp index 64f7134eac..f13cd272cd 100644 --- a/onnxruntime/core/mlas/lib/platform.cpp +++ b/onnxruntime/core/mlas/lib/platform.cpp @@ -142,6 +142,8 @@ Return Value: this->QLinearAddU8Kernel = MlasQLinearAddU8Kernel; this->QuantizeLinearS8Kernel = MlasQuantizeLinearS8Kernel; this->QuantizeLinearU8Kernel = MlasQuantizeLinearU8Kernel; + this->ConvDepthwiseU8S8Kernel = MlasConvDepthwiseKernel; + this->ConvDepthwiseU8U8Kernel = MlasConvDepthwiseKernel; this->NchwcBlockSize = 8; this->PreferredBufferAlignment = MLAS_DEFAULT_PREFERRED_BUFFER_ALIGNMENT; @@ -222,8 +224,10 @@ Return Value: this->ErfKernelRoutine = MlasErfKernelFma3; this->QLinearAddS8Kernel = MlasQLinearAddS8KernelAvx2; this->QLinearAddU8Kernel = MlasQLinearAddU8KernelAvx2; + this->ConvDepthwiseU8S8Kernel = MlasConvDepthwiseKernelAvx2; + this->ConvDepthwiseU8U8Kernel = MlasConvDepthwiseKernelAvx2; this->ComputeSumExpF32Kernel = MlasComputeSumExpF32KernelFma3; - + // // Check if the processor supports AVXVNNI features. // @@ -231,7 +235,7 @@ Return Value: unsigned Cpuid7_1[4]; #if defined(_WIN32) __cpuidex((int*)Cpuid7_1, 7, 1); -#else +#else __cpuid_count(7, 1, Cpuid7_1[0], Cpuid7_1[1], Cpuid7_1[2], Cpuid7_1[3]); #endif diff --git a/onnxruntime/core/mlas/lib/qdwconv.cpp b/onnxruntime/core/mlas/lib/qdwconv.cpp new file mode 100644 index 0000000000..7f698bd7d7 --- /dev/null +++ b/onnxruntime/core/mlas/lib/qdwconv.cpp @@ -0,0 +1,277 @@ +/*++ + +Copyright (c) Microsoft Corporation. All rights reserved. + +Licensed under the MIT License. + +Module Name: + + qdwconv.cpp + +Abstract: + + This module implements the quantized integer depthwise convolution routines. + +--*/ + +#include "mlasi.h" + +template +void +MLASCALL +MlasConvDepthwiseKernel( + const uint8_t* const* Input, + uint8_t InputZeroPoint, + const FilterType* Filter, + FilterType FilterZeroPoint, + int32_t* Output, + size_t Channels, + size_t OutputCount, + size_t KernelSize + ) +{ +#if defined(MLAS_SSE2_INTRINSICS) + const __m128i ZeroVector = _mm_setzero_si128(); + const __m128i InputZeroPointVector = _mm_set1_epi16(InputZeroPoint); + const __m128i FilterZeroPointVector = _mm_set1_epi16(FilterZeroPoint); +#elif defined(MLAS_NEON_INTRINSICS) + const uint8x8_t InputZeroPointVector = vdup_n_u8(InputZeroPoint); + const uint8x8_t FilterZeroPointVector = vdup_n_u8(uint8_t(FilterZeroPoint)); +#endif + + while (OutputCount > 0) { + + size_t ChannelOffset = 0; + size_t c = Channels; + +#if defined(MLAS_SSE2_INTRINSICS) + + while (c >= 8) { + + __m128i Accumulator0 = _mm_setzero_si128(); + __m128i Accumulator1 = _mm_setzero_si128(); + size_t ChannelKernelOffset = ChannelOffset; + + for (size_t k = 0; k < KernelSize; k++) { + + __m128i InputVector = _mm_loadl_epi64((const __m128i*)&Input[k][ChannelOffset]); + __m128i FilterVector = _mm_loadl_epi64((const __m128i*)&Filter[ChannelKernelOffset]); + + InputVector = _mm_unpacklo_epi8(InputVector, ZeroVector); + + if (std::is_signed::value) { + FilterVector = _mm_srai_epi16(_mm_unpacklo_epi8(ZeroVector, FilterVector), 8); + } else { + FilterVector = _mm_unpacklo_epi8(FilterVector, ZeroVector); + } + + InputVector = _mm_sub_epi16(InputVector, InputZeroPointVector); + FilterVector = _mm_sub_epi16(FilterVector, FilterZeroPointVector); + + // N.B. Emulate PMULLD functionality on SSE2 by computing the low + // and high parts of the result and interleaving the results. + __m128i MultiplyLowWords = _mm_mullo_epi16(InputVector, FilterVector); + __m128i MultiplyHighWords = _mm_mulhi_epi16(InputVector, FilterVector); + __m128i Multiply0 = _mm_unpacklo_epi16(MultiplyLowWords, MultiplyHighWords); + __m128i Multiply1 = _mm_unpackhi_epi16(MultiplyLowWords, MultiplyHighWords); + + Accumulator0 = _mm_add_epi32(Accumulator0, Multiply0); + Accumulator1 = _mm_add_epi32(Accumulator1, Multiply1); + ChannelKernelOffset += Channels; + } + + _mm_storeu_si128((__m128i*)&Output[0], Accumulator0); + _mm_storeu_si128((__m128i*)&Output[4], Accumulator1); + Output += 8; + + ChannelOffset += 8; + c -= 8; + } + +#elif defined(MLAS_NEON_INTRINSICS) + + while (c >= 8) { + + int32x4_t Accumulator0 = vdupq_n_s32(0); + int32x4_t Accumulator1 = vdupq_n_s32(0); + size_t ChannelKernelOffset = ChannelOffset; + + for (size_t k = 0; k < KernelSize; k++) { + + uint8x8_t InputVector = vld1_u8(&Input[k][ChannelOffset]); + uint8x8_t FilterVector = vld1_u8(reinterpret_cast(&Filter[ChannelKernelOffset])); + + int16x8_t InputVector16 = vreinterpretq_s16_u16(vsubl_u8(InputVector, InputZeroPointVector)); + int16x8_t FilterVector16; + + if (std::is_signed::value) { + FilterVector16 = vsubl_s8(vreinterpret_s8_u8(FilterVector), vreinterpret_s8_u8(FilterZeroPointVector)); + } else { + FilterVector16 = vreinterpretq_s16_u16(vsubl_u8(FilterVector, FilterZeroPointVector)); + } + + Accumulator0 = vmlal_s16(Accumulator0, vget_low_s16(InputVector16), vget_low_s16(FilterVector16)); +#if defined(MLAS_NEON64_INTRINSICS) + Accumulator1 = vmlal_high_s16(Accumulator1, InputVector16, FilterVector16); +#else + Accumulator1 = vmlal_s16(Accumulator1, vget_high_s16(InputVector16), vget_high_s16(FilterVector16)); +#endif + + ChannelKernelOffset += Channels; + } + + vst1q_s32(&Output[0], Accumulator0); + vst1q_s32(&Output[4], Accumulator1); + Output += 8; + + ChannelOffset += 8; + c -= 8; + } + +#endif + + while (c > 0) { + + int32_t Accumulator = 0; + size_t ChannelKernelOffset = ChannelOffset; + + for (size_t k = 0; k < KernelSize; k++) { + + int32_t InputValue = int32_t(Input[k][ChannelOffset]) - InputZeroPoint; + int32_t FilterValue = int32_t(Filter[ChannelKernelOffset]) - FilterZeroPoint; + + Accumulator += InputValue * FilterValue; + ChannelKernelOffset += Channels; + } + + *Output++ = Accumulator; + + ChannelOffset += 1; + c -= 1; + } + + Input += KernelSize; + OutputCount -= 1; + } +} + +template +void +MLASCALL +MlasConvDepthwiseKernel( + const uint8_t* const* Input, + uint8_t InputZeroPoint, + const int8_t* Filter, + int8_t FilterZeroPoint, + int32_t* Output, + size_t Channels, + size_t OutputCount, + size_t KernelSize + ); + +template +void +MLASCALL +MlasConvDepthwiseKernel( + const uint8_t* const* Input, + uint8_t InputZeroPoint, + const uint8_t* Filter, + uint8_t FilterZeroPoint, + int32_t* Output, + size_t Channels, + size_t OutputCount, + size_t KernelSize + ); + +void +MLASCALL +MlasConvDepthwise( + const uint8_t* const* Input, + uint8_t InputZeroPoint, + const uint8_t* Filter, + uint8_t FilterZeroPoint, + bool FilterIsSigned, + int32_t* Output, + size_t Channels, + size_t OutputCount, + size_t KernelSize + ) +/*++ + +Routine Description: + + This routine implements the depthwise convolution operation. + + The input tensor is organized in channels last format (NHWC) after applying + the Im2col transform, so the length of each row of the input tensor is + Channels times KernelSize. The number of columns of the input tensor is + OutputCount. + + The filter tensor is organized in HW1O format, so the length of each row of + the filter tensor is Channels. The number of columns of the filter tensor + is KernelSize. + +Arguments: + + Input - Supplies an indirection buffer to the elements of the input tensor. + + InputZeroPoint - Supplies the zero point offset of the input tensor. + + Filter - Supplies the filter tensor. + + FilterZeroPoint - Supplies the zero point offset of the filter tensor. + + FilterIsSigned - Supplies true if the filter tensor is signed data, else + false if the filter tensor is unsigned data. + + Output - Supplies the output tensor in channels last format. + + Channels - Supplies the number of channels. + + OutputCount - Supplies the number of channel sized output elements to + produce. + + KernelSize - Supplies the total number of channel sized kernel elements to + consume. + +Return Value: + + None. + +--*/ +{ + if (FilterIsSigned) { + +#if defined(MLAS_TARGET_AMD64) + MlasPlatform.ConvDepthwiseU8S8Kernel( +#else + MlasConvDepthwiseKernel( +#endif + Input, + InputZeroPoint, + reinterpret_cast(Filter), + static_cast(FilterZeroPoint), + Output, + Channels, + OutputCount, + KernelSize + ); + + } else { + +#if defined(MLAS_TARGET_AMD64) + MlasPlatform.ConvDepthwiseU8U8Kernel( +#else + MlasConvDepthwiseKernel( +#endif + Input, + InputZeroPoint, + Filter, + FilterZeroPoint, + Output, + Channels, + OutputCount, + KernelSize + ); + } +} diff --git a/onnxruntime/core/providers/cpu/nn/qlinearconv.cc b/onnxruntime/core/providers/cpu/nn/qlinearconv.cc index e6ce0404af..38c757a897 100644 --- a/onnxruntime/core/providers/cpu/nn/qlinearconv.cc +++ b/onnxruntime/core/providers/cpu/nn/qlinearconv.cc @@ -292,11 +292,12 @@ Status QLinearConv::Compute(OpKernelContext* context) const { // Weight tensor was not constant or prepacking is disabled. reordered_W = static_cast(alloc->Alloc(SafeInt(sizeof(uint8_t)) * W_shape.Size())); reordered_W_buffer = BufferUniquePtr(reordered_W, BufferDeleter(alloc)); - ReorderFilter(static_cast(W->DataRaw()), - reordered_W, - static_cast(M), - static_cast(W_shape[1]), - static_cast(kernel_size)); + ReorderFilter( + static_cast(W->DataRaw()), + reordered_W, + static_cast(M), + static_cast(W_shape[1]), + static_cast(kernel_size)); } } @@ -341,10 +342,17 @@ Status QLinearConv::Compute(OpKernelContext* context) const { } BufferUniquePtr col_buffer; + std::vector padding_data; - // Pointwise convolutions can use the original input tensor in place, - // otherwise a temporary buffer is required for the im2col transform. - if (kernel_size != 1 || !conv_attrs_.HasStridesOneAndNoPadding()) { + if (is_depthwise_conv) { + // Allocate indirection buffer pointers and prepare a padding vector for + // the im2col transform. + auto* col_data = alloc->Alloc(SafeInt(sizeof(const uint8_t*)) * kernel_size * output_image_size); + col_buffer = BufferUniquePtr(col_data, BufferDeleter(alloc)); + padding_data.resize(static_cast(C), X_zero_point_value); + } else if (kernel_size != 1 || !conv_attrs_.HasStridesOneAndNoPadding()) { + // Pointwise convolutions can use the original input tensor in place, + // otherwise a temporary buffer is required for the im2col transform. int64_t group_col_buffer_size = (kernel_rank > 2) ? group_count * col_buffer_size : col_buffer_size; auto* col_data = alloc->Alloc(SafeInt(sizeof(uint8_t)) * group_col_buffer_size); col_buffer = BufferUniquePtr(col_data, BufferDeleter(alloc)); @@ -377,33 +385,32 @@ Status QLinearConv::Compute(OpKernelContext* context) const { if (!channels_last_) { // Transpose the input from channels first (NCHW) to channels last (NHWC). - MlasTranspose(Xdata, - static_cast(transpose_input_buffer.get()), - static_cast(C), - static_cast(input_image_size)); + MlasTranspose( + Xdata, + static_cast(transpose_input_buffer.get()), + static_cast(C), + static_cast(input_image_size)); input_data = static_cast(transpose_input_buffer.get()); output_data = static_cast(transpose_output_buffer.get()); } - if (col_buffer) { - if (kernel_rank > 2) { - // Threaded implementation of ND convolution is not yet supported, so - // prepare all im2col transformations here. - for (int64_t group_id = 0; group_id < group_count; ++group_id) { - math::Im2col()( - input_data + group_id * group_input_channels, - group_input_channels, - C, - input_shape.GetDims().data(), - output_shape.GetDims().data(), - kernel_shape.data(), - strides.data(), - dilations.data(), - pads.data(), - static_cast(kernel_rank), - static_cast(col_buffer.get()) + group_id * col_buffer_size, - X_zero_point_value); - } + // Threaded implementation of ND convolution is not yet supported, so + // prepare all im2col transformations here. + if (!is_depthwise_conv && col_buffer && kernel_rank > 2) { + for (int64_t group_id = 0; group_id < group_count; ++group_id) { + math::Im2col()( + input_data + group_id * group_input_channels, + group_input_channels, + C, + input_shape.GetDims().data(), + output_shape.GetDims().data(), + kernel_shape.data(), + strides.data(), + dilations.data(), + pads.data(), + static_cast(kernel_rank), + static_cast(col_buffer.get()) + group_id * col_buffer_size, + X_zero_point_value); } } @@ -415,134 +422,144 @@ Status QLinearConv::Compute(OpKernelContext* context) const { auto* worker_gemm_output = gemm_output + output_start * M; auto* worker_requantize_output = output_data + output_start * M; - for (int64_t group_id = 0; group_id < group_count; ++group_id) { - // Prepare the im2col transformation or use the input buffer directly for - // pointwise convolutions. - const uint8_t* worker_gemm_input; - if (col_buffer) { - auto* worker_col_buffer = static_cast(col_buffer.get()) + output_start * kernel_dim; - if (kernel_rank == 2) { - math::Im2col()( - input_data + group_id * group_input_channels, - group_input_channels, - C, - input_shape[0], - input_shape[1], - kernel_shape[0], - kernel_shape[1], - dilations[0], - dilations[1], - pads[0], - pads[1], - strides[0], - strides[1], - output_shape[1], - output_start, - output_count, - worker_col_buffer, - X_zero_point_value); - } else if (kernel_rank == 1) { - math::Im2col()( - input_data + group_id * group_input_channels, - group_input_channels, - C, - 1, - input_shape[0], - 1, - kernel_shape[0], - 1, - dilations[0], - 0, - pads[0], - 1, - strides[0], - output_shape[0], - output_start, - output_count, - worker_col_buffer, - X_zero_point_value); + if (is_depthwise_conv) { + auto* worker_col_buffer = static_cast(col_buffer.get()) + output_start * kernel_size; + math::Im2col()( + input_data, + C, + input_shape.GetDims().data(), + output_shape.GetDims().data(), + kernel_shape.data(), + strides.data(), + dilations.data(), + pads.data(), + static_cast(kernel_rank), + output_start, + output_count, + worker_col_buffer, + padding_data.data()); + MlasConvDepthwise( + worker_col_buffer, + X_zero_point_value, + reordered_W, + W_zero_point_value, + is_W_signed, + worker_gemm_output, + static_cast(M), + static_cast(output_count), + static_cast(kernel_size)); + } else { + for (int64_t group_id = 0; group_id < group_count; ++group_id) { + // Prepare the im2col transformation or use the input buffer directly for + // pointwise convolutions. + const uint8_t* worker_gemm_input; + if (col_buffer) { + auto* worker_col_buffer = static_cast(col_buffer.get()) + output_start * kernel_dim; + if (kernel_rank == 2) { + math::Im2col()( + input_data + group_id * group_input_channels, + group_input_channels, + C, + input_shape[0], + input_shape[1], + kernel_shape[0], + kernel_shape[1], + dilations[0], + dilations[1], + pads[0], + pads[1], + strides[0], + strides[1], + output_shape[1], + output_start, + output_count, + worker_col_buffer, + X_zero_point_value); + } else if (kernel_rank == 1) { + math::Im2col()( + input_data + group_id * group_input_channels, + group_input_channels, + C, + 1, + input_shape[0], + 1, + kernel_shape[0], + 1, + dilations[0], + 0, + pads[0], + 1, + strides[0], + output_shape[0], + output_start, + output_count, + worker_col_buffer, + X_zero_point_value); + } else { + // Use the im2col buffer prepared outside the thread, indexed by group. + worker_col_buffer += group_id * col_buffer_size; + } + worker_gemm_input = worker_col_buffer; } else { - // Use the im2col buffer prepared outside the thread, indexed by group. - worker_col_buffer += group_id * col_buffer_size; + worker_gemm_input = input_data + output_start * kernel_dim; } - worker_gemm_input = worker_col_buffer; - } else { - worker_gemm_input = input_data + output_start * kernel_dim; - } - if (is_depthwise_conv) { - if (is_W_signed) { - MlasConvDepthwise(worker_gemm_input, - X_zero_point_value, - reinterpret_cast(reordered_W), - static_cast(W_zero_point_value), - worker_gemm_output, - static_cast(M), - static_cast(output_count), - static_cast(kernel_size)); - } else { - MlasConvDepthwise(worker_gemm_input, - X_zero_point_value, - reordered_W, - W_zero_point_value, - worker_gemm_output, - static_cast(M), - static_cast(output_count), - static_cast(kernel_size)); - } - } else { #ifdef MLAS_SUPPORTS_PACKED_GEMM_U8X8 if (packed_W_buffer_) { - MlasGemm(static_cast(output_count), - static_cast(group_output_channels), - static_cast(kernel_dim), - worker_gemm_input, - static_cast(kernel_dim), - X_zero_point_value, - static_cast(packed_W_buffer_.get()) + group_id * packed_W_size_, - W_zero_point_value, - is_W_signed, - worker_gemm_output + group_id * group_output_channels, - static_cast(M), - nullptr); + MlasGemm( + static_cast(output_count), + static_cast(group_output_channels), + static_cast(kernel_dim), + worker_gemm_input, + static_cast(kernel_dim), + X_zero_point_value, + static_cast(packed_W_buffer_.get()) + group_id * packed_W_size_, + W_zero_point_value, + is_W_signed, + worker_gemm_output + group_id * group_output_channels, + static_cast(M), + nullptr); } else #endif { - MlasGemm(static_cast(output_count), - static_cast(group_output_channels), - static_cast(kernel_dim), - worker_gemm_input, - static_cast(kernel_dim), - X_zero_point_value, - reordered_W + group_id * group_output_channels, - static_cast(M), - W_zero_point_value, - is_W_signed, - worker_gemm_output + group_id * group_output_channels, - static_cast(M), - nullptr); + MlasGemm( + static_cast(output_count), + static_cast(group_output_channels), + static_cast(kernel_dim), + worker_gemm_input, + static_cast(kernel_dim), + X_zero_point_value, + reordered_W + group_id * group_output_channels, + static_cast(M), + W_zero_point_value, + is_W_signed, + worker_gemm_output + group_id * group_output_channels, + static_cast(M), + nullptr); } } } - MlasRequantizeOutput(worker_gemm_output, - worker_requantize_output, - Bdata, - static_cast(output_count), - static_cast(M), - output_scales.data(), - output_scales.size() > 1, - Y_zero_point_value); + MlasRequantizeOutput( + worker_gemm_output, + worker_requantize_output, + Bdata, + static_cast(output_count), + static_cast(M), + output_scales.data(), + output_scales.size() > 1, + Y_zero_point_value); }; concurrency::ThreadPool::TrySimpleParallelFor(thread_pool, thread_count, conv_worker); if (!channels_last_) { // Transpose the output from channels last (NHWC) to channels first (NCHW). - MlasTranspose(output_data, - Ydata, - static_cast(output_image_size), - static_cast(M)); + MlasTranspose( + output_data, + Ydata, + static_cast(output_image_size), + static_cast(M)); } Xdata += X_offset; diff --git a/onnxruntime/core/util/math.h b/onnxruntime/core/util/math.h index 46f899b71d..54bc6c29d5 100644 --- a/onnxruntime/core/util/math.h +++ b/onnxruntime/core/util/math.h @@ -20,7 +20,6 @@ // still keep it simple, so all platforms would be able to support it fairly // easily. - // We include the cblas header here so that we can obtain the macros from cblas. extern "C" { #include "core/framework/cblas.h" @@ -49,7 +48,6 @@ void Log(int N, const T* x, T* y, Provider* provider); template void Sqr(int N, const T* x, T* y, Provider* provider); - #define DECLARE_BINARY_OP(name) \ template \ void name(int N, const T* a, const T* b, T* y, Provider* provider); \ @@ -240,6 +238,20 @@ struct Im2col { ptrdiff_t rank, T* data_col, T padding_value = 0); + void operator()( + const T* data_im, + int64_t input_channels, + const int64_t* input_shape, + const int64_t* output_shape, + const int64_t* kernel_shape, + const int64_t* stride, + const int64_t* dilation, + const int64_t* pad, + ptrdiff_t rank, + int64_t output_start, + int64_t output_count, + T const** data_indirection, + const T* padding_ptr); }; template diff --git a/onnxruntime/core/util/math_cpu.cc b/onnxruntime/core/util/math_cpu.cc index 82f6a820ee..fbed516ea7 100644 --- a/onnxruntime/core/util/math_cpu.cc +++ b/onnxruntime/core/util/math_cpu.cc @@ -210,7 +210,6 @@ template void Gemv(const CBLAS_TRANSPOSE TransA, int M, int SPECIALIZED_AXPY(float) #undef SPECIALIZED_AXPY - #define DELEGATE_SIMPLE_UNARY_FUNCTION(T, Funcname, expr) \ template <> \ void Funcname(int N, const T* x, T* y, CPUMathUtil*) { \ @@ -535,6 +534,115 @@ void Im2col::operator()( } while (NextPosition(rank, output_shape, d_output.data())); } +template +void Im2col::operator()( + const T* data_im, + int64_t input_channels, + const int64_t* input_shape, + const int64_t* output_shape, + const int64_t* kernel_shape, + const int64_t* stride, + const int64_t* dilation, + const int64_t* pad, + ptrdiff_t rank, + int64_t output_start, + int64_t output_count, + T const** data_indirection, + const T* padding_ptr) { + if (rank == 1) { + int64_t stride_w = stride[0]; + int64_t kernel_w = kernel_shape[0]; + int64_t dilation_w = dilation[0]; + int64_t pad_l = pad[0]; + int64_t input_w = input_shape[0]; + + int64_t ow = output_start * stride_w; + + while (output_count--) { + int64_t iw = ow - pad_l; + for (int64_t kw = 0; kw < kernel_w; kw++) { + const T* data_ptr = data_im + iw * input_channels; + data_indirection[kw] = is_a_ge_zero_and_a_lt_b(iw, input_w) ? data_ptr : padding_ptr; + iw += dilation_w; + } + data_indirection += kernel_w; + ow += stride_w; + } + + } else if (rank == 2) { + int64_t stride_h = stride[0]; + int64_t stride_w = stride[1]; + int64_t kernel_h = kernel_shape[0]; + int64_t kernel_w = kernel_shape[1]; + int64_t dilation_h = dilation[0]; + int64_t dilation_w = dilation[1]; + int64_t pad_t = pad[0]; + int64_t pad_l = pad[1]; + int64_t input_h = input_shape[0]; + int64_t input_w = input_shape[1]; + int64_t output_w = output_shape[1]; + + int64_t oh = (output_start / output_w) * stride_h; + int64_t ow = (output_start % output_w) * stride_w; + int64_t ow_end = output_w * stride_w; + + while (output_count--) { + for (int64_t kh = 0; kh < kernel_h; kh++) { + int64_t ih = kh * dilation_h + oh - pad_t; + if (is_a_ge_zero_and_a_lt_b(ih, input_h)) { + int64_t ihw = ih * input_w; + int64_t iw = ow - pad_l; + for (int64_t kw = 0; kw < kernel_w; kw++) { + const T* data_ptr = data_im + (ihw + iw) * input_channels; + data_indirection[kw] = is_a_ge_zero_and_a_lt_b(iw, input_w) ? data_ptr : padding_ptr; + iw += dilation_w; + } + } else { + std::fill_n(data_indirection, kernel_w, padding_ptr); + } + data_indirection += kernel_w; + } + ow += stride_w; + if (ow == ow_end) { + oh += stride_h; + ow = 0; + } + } + + } else { + // iterate dimensions on output image shape (without Batch and Channel) + std::vector d_output(rank, 0); + // inner iterate dimensions on kernel shape (without output channel and input channel) + std::vector d_kernel(rank, 0); + + // Skip ahead to the starting output index. + for (ptrdiff_t d_i = rank - 1; d_i >= 0; --d_i) { + d_output[d_i] = output_start % output_shape[d_i]; + output_start /= output_shape[d_i]; + } + + while (output_count--) { + // Loop over spatial axes in reverse order to choose an index on kernel dimensions + do { + // Loop over spatial axes in forward order to compute the indices in the image + // and the inner col, and whether the index lies in the padding. + int64_t index_im = 0; + bool is_padding = false; + for (ptrdiff_t d_i = 0; d_i < rank; ++d_i) { + int64_t d_input = d_output[d_i] * stride[d_i] - pad[d_i] + d_kernel[d_i] * dilation[d_i]; + is_padding |= !is_a_ge_zero_and_a_lt_b(d_input, input_shape[d_i]); + index_im *= input_shape[d_i]; + index_im += d_input; + } + const T* data_ptr = data_im + index_im * input_channels; + *data_indirection++ = is_padding ? padding_ptr : data_ptr; + } while (NextPosition(rank, kernel_shape, d_kernel.data())); + // Loop over spatial axes along the output image shape + NextPosition(rank, output_shape, d_output.data()); + } + } +} + template struct Im2col; template <> diff --git a/onnxruntime/test/providers/cpu/nn/qlinearconv_op_test.cc b/onnxruntime/test/providers/cpu/nn/qlinearconv_op_test.cc index c4deb3553e..3bbb2b50a1 100644 --- a/onnxruntime/test/providers/cpu/nn/qlinearconv_op_test.cc +++ b/onnxruntime/test/providers/cpu/nn/qlinearconv_op_test.cc @@ -736,8 +736,21 @@ TEST(QLinearConvTest, Conv2D_U8S8_Groups_PerChannel) { test.Run(); } +TEST(QLinearConvTest, Conv1D_U8S8_Depthwise) { + for (int64_t channels : std::initializer_list{7, 8, 9, 16, 25, 64}) { + QLinearConvOpTester test; + test.GenerateRandomInput({1, channels, 25}, .03f, 12); + test.GenerateRandomWeights({channels, 1, 3}, .10f, 2); + test.GenerateRandomBias(); + test.SetPads({1, 1}); + test.SetGroups(channels); + test.SetOutputScaleAndZeroPoint(.21f, 88); + test.Run(); + } +} + TEST(QLinearConvTest, Conv2D_U8S8_Depthwise) { - for (int64_t channels : std::initializer_list{7, 8, 9, 16, 25}) { + for (int64_t channels : std::initializer_list{7, 8, 9, 16, 25, 64}) { QLinearConvOpTester test; test.GenerateRandomInput({1, channels, 25, 25}, .03f, 12); test.GenerateRandomWeights({channels, 1, 5, 5}, .10f, 0); @@ -750,7 +763,7 @@ TEST(QLinearConvTest, Conv2D_U8S8_Depthwise) { } TEST(QLinearConvTest, Conv2D_U8U8_Depthwise) { - for (int64_t channels : std::initializer_list{3, 8, 13, 24, 31}) { + for (int64_t channels : std::initializer_list{3, 8, 13, 24, 31, 64}) { QLinearConvOpTester test; test.GenerateRandomInput({1, channels, 25, 25}, .03f, 12); test.GenerateRandomWeights({channels, 1, 3, 3}, .10f, 167); @@ -775,13 +788,15 @@ TEST(QLinearConvTest, Conv2D_U8S8_DepthwisePointwise) { } TEST(QLinearConvTest, Conv3D_U8S8_Depthwise) { - QLinearConvOpTester test; - test.GenerateRandomInput({1, 16, 15, 11, 13}, .02f, 135); - test.GenerateRandomWeights({16, 1, 3, 3, 3}, .09f, 0); - test.GenerateRandomBias(); - test.SetGroups(16); - test.SetOutputScaleAndZeroPoint(.85f, 112); - test.Run(); + for (int64_t channels : std::initializer_list{6, 8, 31, 64}) { + QLinearConvOpTester test; + test.GenerateRandomInput({1, channels, 15, 11, 13}, .02f, 135); + test.GenerateRandomWeights({channels, 1, 3, 3, 3}, .09f, 0); + test.GenerateRandomBias(); + test.SetGroups(channels); + test.SetOutputScaleAndZeroPoint(.85f, 112); + test.Run(); + } } TEST(QLinearConvTest, Conv2D_U8S8_Requantize_NoBias) {