saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-24 19:43:35 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

MLAS: improve quantized depthwise convolution (#6513)

Browse source
This commit is contained in:
Tracy Sharpe 2021-02-01 21:22:27 -08:00 committed by GitHub
parent 588ddeb82f
commit 9a6e71574a
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
12 changed files with 835 additions and 373 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

12
cmake/onnxruntime_mlas.cmake
View file

@ -6,6 +6,7 @@ set(mlas_common_srcs
${ONNXRUNTIME_ROOT}/core/mlas/lib/threading.cpp
${ONNXRUNTIME_ROOT}/core/mlas/lib/sgemm.cpp
${ONNXRUNTIME_ROOT}/core/mlas/lib/qgemm.cpp
${ONNXRUNTIME_ROOT}/core/mlas/lib/qdwconv.cpp
${ONNXRUNTIME_ROOT}/core/mlas/lib/convolve.cpp
${ONNXRUNTIME_ROOT}/core/mlas/lib/pooling.cpp
${ONNXRUNTIME_ROOT}/core/mlas/lib/transpose.cpp
@ -59,13 +60,13 @@ if(MSVC)
elseif(onnxruntime_target_platform STREQUAL "x64")
enable_language(ASM_MASM)
set(mlas_platform_srcs_avx
${ONNXRUNTIME_ROOT}/core/mlas/lib/intrinsics/avx/min_max_elements.cpp
file(GLOB_RECURSE mlas_platform_srcs_avx CONFIGURE_DEPENDS
"${ONNXRUNTIME_ROOT}/core/mlas/lib/intrinsics/avx/*.cpp"
)
set_source_files_properties(${mlas_platform_srcs_avx} PROPERTIES COMPILE_FLAGS "/arch:AVX")
set(mlas_platform_srcs_avx2
${ONNXRUNTIME_ROOT}/core/mlas/lib/intrinsics/avx2/qladd_avx2.cpp
file(GLOB_RECURSE mlas_platform_srcs_avx2 CONFIGURE_DEPENDS
"${ONNXRUNTIME_ROOT}/core/mlas/lib/intrinsics/avx2/*.cpp"
)
set_source_files_properties(${mlas_platform_srcs_avx2} PROPERTIES COMPILE_FLAGS "/arch:AVX2")
@ -259,6 +260,7 @@ else()
${ONNXRUNTIME_ROOT}/core/mlas/lib/x86_64/TanhKernelFma3.S
${ONNXRUNTIME_ROOT}/core/mlas/lib/x86_64/ErfKernelFma3.S
${ONNXRUNTIME_ROOT}/core/mlas/lib/intrinsics/avx2/qladd_avx2.cpp
${ONNXRUNTIME_ROOT}/core/mlas/lib/intrinsics/avx2/qdwconv_avx2.cpp
)
set_source_files_properties(${mlas_platform_srcs_avx2} PROPERTIES COMPILE_FLAGS "-mavx2 -mfma")
@ -352,7 +354,7 @@ else()
endif()
add_library(onnxruntime_mlas STATIC ${mlas_common_srcs} ${mlas_platform_srcs})
target_include_directories(onnxruntime_mlas PRIVATE ${ONNXRUNTIME_ROOT}/core/mlas/inc ${ONNXRUNTIME_ROOT}/core/mlas/lib ${ONNXRUNTIME_ROOT}/core/mlas/lib/amd64)
target_include_directories(onnxruntime_mlas PRIVATE ${ONNXRUNTIME_ROOT}/core/mlas/inc ${ONNXRUNTIME_ROOT}/core/mlas/lib)
set_target_properties(onnxruntime_mlas PROPERTIES FOLDER "ONNXRuntime")
if (WIN32)
target_compile_options(onnxruntime_mlas PRIVATE "/wd6385")

8
onnxruntime/core/mlas/inc/mlas.h
View file

@ -422,14 +422,14 @@ MlasConv(
MLAS_THREADPOOL* ThreadPool
);
template<typename FilterType>
void
MLASCALL
MlasConvDepthwise(
const uint8_t* Input,
const uint8_t* const* Input,
uint8_t InputZeroPoint,
const FilterType* Filter,
FilterType FilterZeroPoint,
const uint8_t* Filter,
uint8_t FilterZeroPoint,
bool FilterIsSigned,
int32_t* Output,
size_t Channels,
size_t OutputCount,

207
onnxruntime/core/mlas/lib/convolve.cpp
View file

@ -1257,210 +1257,3 @@ Return Value:
*WorkingBufferSize = TargetThreadCount * MLAS_CONV_WORKING_BUFFER_SIZE_PER_THREAD;
}
}
template<typename FilterType>
void
MLASCALL
MlasConvDepthwise(
const uint8_t* Input,
uint8_t InputZeroPoint,
const FilterType* Filter,
FilterType FilterZeroPoint,
int32_t* Output,
size_t Channels,
size_t OutputCount,
size_t KernelSize
)
/*++
Routine Description:
This routine implements the depthwise convolution operation.
The input tensor is organized in channels last format (NHWC) after applying
the Im2col transform, so the length of each row of the input tensor is
Channels times KernelSize. The number of columns of the input tensor is
OutputCount.
The filter tensor is organized in HW1O format, so the length of each row of
the filter tensor is Channels. The number of columns of the filter tensor
is KernelSize.
Arguments:
Input - Supplies the input tensor in channels last format.
InputZeroPoint - Supplies the zero point offset of the input tensor.
Filter - Supplies the filter tensor.
FilterZeroPoint - Supplies the zero point offset of the filter tensor.
Output - Supplies the output tensor in channels last format.
Channels - Supplies the number of channels.
OutputCount - Supplies the number of channel sized output elements to
produce.
KernelSize - Supplies the total number of channel sized kernel elements to
consume.
Return Value:
None.
--*/
{
#if defined(MLAS_SSE2_INTRINSICS)
const __m128i ZeroVector = _mm_setzero_si128();
const __m128i InputZeroPointVector = _mm_set1_epi16(InputZeroPoint);
const __m128i FilterZeroPointVector = _mm_set1_epi16(FilterZeroPoint);
#elif defined(MLAS_NEON_INTRINSICS)
const uint8x8_t InputZeroPointVector = vdup_n_u8(InputZeroPoint);
const uint8x8_t FilterZeroPointVector = vdup_n_u8(uint8_t(FilterZeroPoint));
#endif
while (OutputCount > 0) {
size_t ChannelOffset = 0;
size_t c = Channels;
#if defined(MLAS_SSE2_INTRINSICS)
while (c >= 8) {
__m128i Accumulator0 = _mm_setzero_si128();
__m128i Accumulator1 = _mm_setzero_si128();
size_t ChannelKernelOffset = ChannelOffset;
for (size_t k = 0; k < KernelSize; k++) {
__m128i InputVector = _mm_loadl_epi64((const __m128i*)&Input[ChannelKernelOffset]);
__m128i FilterVector = _mm_loadl_epi64((const __m128i*)&Filter[ChannelKernelOffset]);
InputVector = _mm_unpacklo_epi8(InputVector, ZeroVector);
if (std::is_signed<FilterType>::value) {
FilterVector = _mm_srai_epi16(_mm_unpacklo_epi8(ZeroVector, FilterVector), 8);
} else {
FilterVector = _mm_unpacklo_epi8(FilterVector, ZeroVector);
}
InputVector = _mm_sub_epi16(InputVector, InputZeroPointVector);
FilterVector = _mm_sub_epi16(FilterVector, FilterZeroPointVector);
// N.B. Emulate PMULLD functionality on SSE2 by computing the low
// and high parts of the result and interleaving the results.
__m128i MultiplyLowWords = _mm_mullo_epi16(InputVector, FilterVector);
__m128i MultiplyHighWords = _mm_mulhi_epi16(InputVector, FilterVector);
__m128i Multiply0 = _mm_unpacklo_epi16(MultiplyLowWords, MultiplyHighWords);
__m128i Multiply1 = _mm_unpackhi_epi16(MultiplyLowWords, MultiplyHighWords);
Accumulator0 = _mm_add_epi32(Accumulator0, Multiply0);
Accumulator1 = _mm_add_epi32(Accumulator1, Multiply1);
ChannelKernelOffset += Channels;
}
_mm_storeu_si128((__m128i*)&Output[0], Accumulator0);
_mm_storeu_si128((__m128i*)&Output[4], Accumulator1);
Output += 8;
ChannelOffset += 8;
c -= 8;
}
#elif defined(MLAS_NEON_INTRINSICS)
while (c >= 8) {
int32x4_t Accumulator0 = vdupq_n_s32(0);
int32x4_t Accumulator1 = vdupq_n_s32(0);
size_t ChannelKernelOffset = ChannelOffset;
for (size_t k = 0; k < KernelSize; k++) {
uint8x8_t InputVector = vld1_u8(&Input[ChannelKernelOffset]);
uint8x8_t FilterVector = vld1_u8(reinterpret_cast<const uint8_t*>(&Filter[ChannelKernelOffset]));
int16x8_t InputVector16 = vreinterpretq_s16_u16(vsubl_u8(InputVector, InputZeroPointVector));
int16x8_t FilterVector16;
if (std::is_signed<FilterType>::value) {
FilterVector16 = vsubl_s8(vreinterpret_s8_u8(FilterVector), vreinterpret_s8_u8(FilterZeroPointVector));
} else {
FilterVector16 = vreinterpretq_s16_u16(vsubl_u8(FilterVector, FilterZeroPointVector));
}
Accumulator0 = vmlal_s16(Accumulator0, vget_low_s16(InputVector16), vget_low_s16(FilterVector16));
#if defined(MLAS_NEON64_INTRINSICS)
Accumulator1 = vmlal_high_s16(Accumulator1, InputVector16, FilterVector16);
#else
Accumulator1 = vmlal_s16(Accumulator1, vget_high_s16(InputVector16), vget_high_s16(FilterVector16));
#endif
ChannelKernelOffset += Channels;
}
vst1q_s32(&Output[0], Accumulator0);
vst1q_s32(&Output[4], Accumulator1);
Output += 8;
ChannelOffset += 8;
c -= 8;
}
#endif
while (c > 0) {
int32_t Accumulator = 0;
size_t ChannelKernelOffset = ChannelOffset;
for (size_t k = 0; k < KernelSize; k++) {
int32_t InputValue = int32_t(Input[ChannelKernelOffset]) - InputZeroPoint;
int32_t FilterValue = int32_t(Filter[ChannelKernelOffset]) - FilterZeroPoint;
Accumulator += InputValue * FilterValue;
ChannelKernelOffset += Channels;
}
*Output++ = Accumulator;
ChannelOffset += 1;
c -= 1;
}
Input += Channels * KernelSize;
OutputCount -= 1;
}
}
template
void
MLASCALL
MlasConvDepthwise<int8_t>(
const uint8_t* Input,
uint8_t InputZeroPoint,
const int8_t* Filter,
int8_t FilterZeroPoint,
int32_t* Output,
size_t Channels,
size_t OutputCount,
size_t KernelSize
);
template
void
MLASCALL
MlasConvDepthwise<uint8_t>(
const uint8_t* Input,
uint8_t InputZeroPoint,
const uint8_t* Filter,
uint8_t FilterZeroPoint,
int32_t* Output,
size_t Channels,
size_t OutputCount,
size_t KernelSize
);

4
onnxruntime/core/mlas/lib/intrinsics/avx/min_max_elements.cpp
View file

@ -14,7 +14,7 @@ Abstract:
--*/
#include "../mlasi.h"
#include "mlasi.h"
void
MLASCALL
@ -103,4 +103,4 @@ MlasReduceMinimumMaximumF32KernelAvx(
*Min = tmp_min;
*Max = tmp_max;
}
}

183
onnxruntime/core/mlas/lib/intrinsics/avx2/qdwconv_avx2.cpp Normal file
View file

@ -0,0 +1,183 @@
/*++
Copyright (c) Microsoft Corporation. All rights reserved.
Licensed under the MIT License.
Module Name:
qdwconv_avx2.cpp
Abstract:
This module implements the quantized integer depthwise convolution kernels.
This implementation uses AVX2 instructions.
--*/
#include "mlasi.h"
template<typename FilterType>
void
MLASCALL
MlasConvDepthwiseKernelAvx2(
const uint8_t* const* Input,
uint8_t InputZeroPoint,
const FilterType* Filter,
FilterType FilterZeroPoint,
int32_t* Output,
size_t Channels,
size_t OutputCount,
size_t KernelSize
)
{
const __m256i InputZeroPointVector = _mm256_set1_epi16(InputZeroPoint);
const __m256i FilterZeroPointVector = _mm256_set1_epi16(FilterZeroPoint);
while (OutputCount > 0) {
size_t ChannelOffset = 0;
size_t c = Channels;
while (c >= 16) {
__m256i Accumulator0 = _mm256_setzero_si256();
__m256i Accumulator1 = _mm256_setzero_si256();
size_t ChannelKernelOffset = ChannelOffset;
for (size_t k = 0; k < KernelSize; k++) {
__m256i InputVector = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)&Input[k][ChannelOffset]));
__m256i FilterVector;
if (std::is_signed<FilterType>::value) {
FilterVector = _mm256_cvtepi8_epi16(_mm_loadu_si128((const __m128i*)&Filter[ChannelKernelOffset]));
} else {
FilterVector = _mm256_cvtepu8_epi16(_mm_loadu_si128((const __m128i*)&Filter[ChannelKernelOffset]));
}
InputVector = _mm256_sub_epi16(InputVector, InputZeroPointVector);
FilterVector = _mm256_sub_epi16(FilterVector, FilterZeroPointVector);
// N.B. The original SSE2 implementation used PMULLW/PMULHW in
// order to emulate the SSE 4.1 PMULLD instruction, however this
// implementation ends up being faster for some CPUs than
// extending to 32-bits and using PMULLD.
__m256i MultiplyLowWords = _mm256_mullo_epi16(InputVector, FilterVector);
__m256i MultiplyHighWords = _mm256_mulhi_epi16(InputVector, FilterVector);
__m256i Multiply0 = _mm256_unpacklo_epi16(MultiplyLowWords, MultiplyHighWords);
__m256i Multiply1 = _mm256_unpackhi_epi16(MultiplyLowWords, MultiplyHighWords);
Accumulator0 = _mm256_add_epi32(Accumulator0, Multiply0);
Accumulator1 = _mm256_add_epi32(Accumulator1, Multiply1);
ChannelKernelOffset += Channels;
}
// N.B. The above interleaving of the intermediate results leaves
// the accumulators in a swizzled layout, because the interleaving
// is per 128-bit half of the __m256i register. Reorder the results
// now to get the expected sequential order.
__m256i Reorder0 = _mm256_permute2x128_si256(Accumulator0, Accumulator1, 0x20);
__m256i Reorder1 = _mm256_permute2x128_si256(Accumulator0, Accumulator1, 0x31);
_mm256_storeu_si256((__m256i*)&Output[0], Reorder0);
_mm256_storeu_si256((__m256i*)&Output[8], Reorder1);
Output += 16;
ChannelOffset += 16;
c -= 16;
}
if (c >= 8) {
__m128i Accumulator0 = _mm_setzero_si128();
__m128i Accumulator1 = _mm_setzero_si128();
size_t ChannelKernelOffset = ChannelOffset;
for (size_t k = 0; k < KernelSize; k++) {
__m128i InputVector = _mm_loadl_epi64((const __m128i*)&Input[k][ChannelOffset]);
__m128i FilterVector = _mm_loadl_epi64((const __m128i*)&Filter[ChannelKernelOffset]);
InputVector = _mm_cvtepu8_epi16(InputVector);
if (std::is_signed<FilterType>::value) {
FilterVector = _mm_cvtepi8_epi16(FilterVector);
} else {
FilterVector = _mm_cvtepu8_epi16(FilterVector);
}
InputVector = _mm_sub_epi16(InputVector, _mm256_castsi256_si128(InputZeroPointVector));
FilterVector = _mm_sub_epi16(FilterVector, _mm256_castsi256_si128(FilterZeroPointVector));
__m128i MultiplyLowWords = _mm_mullo_epi16(InputVector, FilterVector);
__m128i MultiplyHighWords = _mm_mulhi_epi16(InputVector, FilterVector);
__m128i Multiply0 = _mm_unpacklo_epi16(MultiplyLowWords, MultiplyHighWords);
__m128i Multiply1 = _mm_unpackhi_epi16(MultiplyLowWords, MultiplyHighWords);
Accumulator0 = _mm_add_epi32(Accumulator0, Multiply0);
Accumulator1 = _mm_add_epi32(Accumulator1, Multiply1);
ChannelKernelOffset += Channels;
}
_mm_storeu_si128((__m128i*)&Output[0], Accumulator0);
_mm_storeu_si128((__m128i*)&Output[4], Accumulator1);
Output += 8;
ChannelOffset += 8;
c -= 8;
}
while (c > 0) {
int32_t Accumulator = 0;
size_t ChannelKernelOffset = ChannelOffset;
for (size_t k = 0; k < KernelSize; k++) {
int32_t InputValue = int32_t(Input[k][ChannelOffset]) - InputZeroPoint;
int32_t FilterValue = int32_t(Filter[ChannelKernelOffset]) - FilterZeroPoint;
Accumulator += InputValue * FilterValue;
ChannelKernelOffset += Channels;
}
*Output++ = Accumulator;
ChannelOffset += 1;
c -= 1;
}
Input += KernelSize;
OutputCount -= 1;
}
}
template
void
MLASCALL
MlasConvDepthwiseKernelAvx2<int8_t>(
const uint8_t* const* Input,
uint8_t InputZeroPoint,
const int8_t* Filter,
int8_t FilterZeroPoint,
int32_t* Output,
size_t Channels,
size_t OutputCount,
size_t KernelSize
);
template
void
MLASCALL
MlasConvDepthwiseKernelAvx2<uint8_t>(
const uint8_t* const* Input,
uint8_t InputZeroPoint,
const uint8_t* Filter,
uint8_t FilterZeroPoint,
int32_t* Output,
size_t Channels,
size_t OutputCount,
size_t KernelSize
);

51
onnxruntime/core/mlas/lib/mlasi.h
View file

@ -513,6 +513,23 @@ void
typedef MLAS_QUANTIZE_LINEAR_S8_KERNEL* PMLAS_QUANTIZE_LINEAR_S8_KERNEL;
template<typename FilterType>
struct MLAS_U8X8_KERNEL
{
typedef
void
(MLASCALL DepthwiseKernel)(
const uint8_t* const* Input,
uint8_t InputZeroPoint,
const FilterType* Filter,
FilterType FilterZeroPoint,
int32_t* Output,
size_t Channels,
size_t OutputCount,
size_t KernelSize
);
};
extern "C" {
#if defined(MLAS_TARGET_AMD64_IX86)
@ -701,6 +718,38 @@ MlasGemmU8X8PackedOperation(
const MLAS_GEMM_U8X8_WORK_BLOCK* WorkBlock
);
//
// Quantized depthwise convolution kernels.
//
template<typename FilterType>
void
MLASCALL
MlasConvDepthwiseKernel(
const uint8_t* const* Input,
uint8_t InputZeroPoint,
const FilterType* Filter,
FilterType FilterZeroPoint,
int32_t* Output,
size_t Channels,
size_t OutputCount,
size_t KernelSize
);
template<typename FilterType>
void
MLASCALL
MlasConvDepthwiseKernelAvx2(
const uint8_t* const* Input,
uint8_t InputZeroPoint,
const FilterType* Filter,
FilterType FilterZeroPoint,
int32_t* Output,
size_t Channels,
size_t OutputCount,
size_t KernelSize
);
//
// Environment information class.
//
@ -733,6 +782,8 @@ struct MLAS_PLATFORM {
PMLAS_COMPUTE_UNARY_FLOAT_KERNEL ErfKernelRoutine;
PMLAS_QLINEAR_BINARY_OP_S8_KERNEL QLinearAddS8Kernel;
PMLAS_QLINEAR_BINARY_OP_U8_KERNEL QLinearAddU8Kernel;
MLAS_U8X8_KERNEL<int8_t>::DepthwiseKernel* ConvDepthwiseU8S8Kernel;
MLAS_U8X8_KERNEL<uint8_t>::DepthwiseKernel* ConvDepthwiseU8U8Kernel;
PMLAS_COMPUTE_UNARY_FLOAT_KERNEL ComputeExpF32Kernel;
PMLAS_COMPUTE_UNARY_FLOAT_KERNEL LogisticKernelRoutine;
PMLAS_COMPUTE_UNARY_FLOAT_KERNEL TanhKernelRoutine;

8
onnxruntime/core/mlas/lib/platform.cpp
View file

@ -142,6 +142,8 @@ Return Value:
this->QLinearAddU8Kernel = MlasQLinearAddU8Kernel;
this->QuantizeLinearS8Kernel = MlasQuantizeLinearS8Kernel;
this->QuantizeLinearU8Kernel = MlasQuantizeLinearU8Kernel;
this->ConvDepthwiseU8S8Kernel = MlasConvDepthwiseKernel<int8_t>;
this->ConvDepthwiseU8U8Kernel = MlasConvDepthwiseKernel<uint8_t>;
this->NchwcBlockSize = 8;
this->PreferredBufferAlignment = MLAS_DEFAULT_PREFERRED_BUFFER_ALIGNMENT;
@ -222,8 +224,10 @@ Return Value:
this->ErfKernelRoutine = MlasErfKernelFma3;
this->QLinearAddS8Kernel = MlasQLinearAddS8KernelAvx2;
this->QLinearAddU8Kernel = MlasQLinearAddU8KernelAvx2;
this->ConvDepthwiseU8S8Kernel = MlasConvDepthwiseKernelAvx2<int8_t>;
this->ConvDepthwiseU8U8Kernel = MlasConvDepthwiseKernelAvx2<uint8_t>;
this->ComputeSumExpF32Kernel = MlasComputeSumExpF32KernelFma3;
//
// Check if the processor supports AVXVNNI features.
//
@ -231,7 +235,7 @@ Return Value:
unsigned Cpuid7_1[4];
#if defined(_WIN32)
__cpuidex((int*)Cpuid7_1, 7, 1);
#else
#else
__cpuid_count(7, 1, Cpuid7_1[0], Cpuid7_1[1], Cpuid7_1[2], Cpuid7_1[3]);
#endif

277
onnxruntime/core/mlas/lib/qdwconv.cpp Normal file
View file

@ -0,0 +1,277 @@
/*++
Copyright (c) Microsoft Corporation. All rights reserved.
Licensed under the MIT License.
Module Name:
qdwconv.cpp
Abstract:
This module implements the quantized integer depthwise convolution routines.
--*/
#include "mlasi.h"
template<typename FilterType>
void
MLASCALL
MlasConvDepthwiseKernel(
const uint8_t* const* Input,
uint8_t InputZeroPoint,
const FilterType* Filter,
FilterType FilterZeroPoint,
int32_t* Output,
size_t Channels,
size_t OutputCount,
size_t KernelSize
)
{
#if defined(MLAS_SSE2_INTRINSICS)
const __m128i ZeroVector = _mm_setzero_si128();
const __m128i InputZeroPointVector = _mm_set1_epi16(InputZeroPoint);
const __m128i FilterZeroPointVector = _mm_set1_epi16(FilterZeroPoint);
#elif defined(MLAS_NEON_INTRINSICS)
const uint8x8_t InputZeroPointVector = vdup_n_u8(InputZeroPoint);
const uint8x8_t FilterZeroPointVector = vdup_n_u8(uint8_t(FilterZeroPoint));
#endif
while (OutputCount > 0) {
size_t ChannelOffset = 0;
size_t c = Channels;
#if defined(MLAS_SSE2_INTRINSICS)
while (c >= 8) {
__m128i Accumulator0 = _mm_setzero_si128();
__m128i Accumulator1 = _mm_setzero_si128();
size_t ChannelKernelOffset = ChannelOffset;
for (size_t k = 0; k < KernelSize; k++) {
__m128i InputVector = _mm_loadl_epi64((const __m128i*)&Input[k][ChannelOffset]);
__m128i FilterVector = _mm_loadl_epi64((const __m128i*)&Filter[ChannelKernelOffset]);
InputVector = _mm_unpacklo_epi8(InputVector, ZeroVector);
if (std::is_signed<FilterType>::value) {
FilterVector = _mm_srai_epi16(_mm_unpacklo_epi8(ZeroVector, FilterVector), 8);
} else {
FilterVector = _mm_unpacklo_epi8(FilterVector, ZeroVector);
}
InputVector = _mm_sub_epi16(InputVector, InputZeroPointVector);
FilterVector = _mm_sub_epi16(FilterVector, FilterZeroPointVector);
// N.B. Emulate PMULLD functionality on SSE2 by computing the low
// and high parts of the result and interleaving the results.
__m128i MultiplyLowWords = _mm_mullo_epi16(InputVector, FilterVector);
__m128i MultiplyHighWords = _mm_mulhi_epi16(InputVector, FilterVector);
__m128i Multiply0 = _mm_unpacklo_epi16(MultiplyLowWords, MultiplyHighWords);
__m128i Multiply1 = _mm_unpackhi_epi16(MultiplyLowWords, MultiplyHighWords);
Accumulator0 = _mm_add_epi32(Accumulator0, Multiply0);
Accumulator1 = _mm_add_epi32(Accumulator1, Multiply1);
ChannelKernelOffset += Channels;
}
_mm_storeu_si128((__m128i*)&Output[0], Accumulator0);
_mm_storeu_si128((__m128i*)&Output[4], Accumulator1);
Output += 8;
ChannelOffset += 8;
c -= 8;
}
#elif defined(MLAS_NEON_INTRINSICS)
while (c >= 8) {
int32x4_t Accumulator0 = vdupq_n_s32(0);
int32x4_t Accumulator1 = vdupq_n_s32(0);
size_t ChannelKernelOffset = ChannelOffset;
for (size_t k = 0; k < KernelSize; k++) {
uint8x8_t InputVector = vld1_u8(&Input[k][ChannelOffset]);
uint8x8_t FilterVector = vld1_u8(reinterpret_cast<const uint8_t*>(&Filter[ChannelKernelOffset]));
int16x8_t InputVector16 = vreinterpretq_s16_u16(vsubl_u8(InputVector, InputZeroPointVector));
int16x8_t FilterVector16;
if (std::is_signed<FilterType>::value) {
FilterVector16 = vsubl_s8(vreinterpret_s8_u8(FilterVector), vreinterpret_s8_u8(FilterZeroPointVector));
} else {
FilterVector16 = vreinterpretq_s16_u16(vsubl_u8(FilterVector, FilterZeroPointVector));
}
Accumulator0 = vmlal_s16(Accumulator0, vget_low_s16(InputVector16), vget_low_s16(FilterVector16));
#if defined(MLAS_NEON64_INTRINSICS)
Accumulator1 = vmlal_high_s16(Accumulator1, InputVector16, FilterVector16);
#else
Accumulator1 = vmlal_s16(Accumulator1, vget_high_s16(InputVector16), vget_high_s16(FilterVector16));
#endif
ChannelKernelOffset += Channels;
}
vst1q_s32(&Output[0], Accumulator0);
vst1q_s32(&Output[4], Accumulator1);
Output += 8;
ChannelOffset += 8;
c -= 8;
}
#endif
while (c > 0) {
int32_t Accumulator = 0;
size_t ChannelKernelOffset = ChannelOffset;
for (size_t k = 0; k < KernelSize; k++) {
int32_t InputValue = int32_t(Input[k][ChannelOffset]) - InputZeroPoint;
int32_t FilterValue = int32_t(Filter[ChannelKernelOffset]) - FilterZeroPoint;
Accumulator += InputValue * FilterValue;
ChannelKernelOffset += Channels;
}
*Output++ = Accumulator;
ChannelOffset += 1;
c -= 1;
}
Input += KernelSize;
OutputCount -= 1;
}
}
template
void
MLASCALL
MlasConvDepthwiseKernel(
const uint8_t* const* Input,
uint8_t InputZeroPoint,
const int8_t* Filter,
int8_t FilterZeroPoint,
int32_t* Output,
size_t Channels,
size_t OutputCount,
size_t KernelSize
);
template
void
MLASCALL
MlasConvDepthwiseKernel(
const uint8_t* const* Input,
uint8_t InputZeroPoint,
const uint8_t* Filter,
uint8_t FilterZeroPoint,
int32_t* Output,
size_t Channels,
size_t OutputCount,
size_t KernelSize
);
void
MLASCALL
MlasConvDepthwise(
const uint8_t* const* Input,
uint8_t InputZeroPoint,
const uint8_t* Filter,
uint8_t FilterZeroPoint,
bool FilterIsSigned,
int32_t* Output,
size_t Channels,
size_t OutputCount,
size_t KernelSize
)
/*++
Routine Description:
This routine implements the depthwise convolution operation.
The input tensor is organized in channels last format (NHWC) after applying
the Im2col transform, so the length of each row of the input tensor is
Channels times KernelSize. The number of columns of the input tensor is
OutputCount.
The filter tensor is organized in HW1O format, so the length of each row of
the filter tensor is Channels. The number of columns of the filter tensor
is KernelSize.
Arguments:
Input - Supplies an indirection buffer to the elements of the input tensor.
InputZeroPoint - Supplies the zero point offset of the input tensor.
Filter - Supplies the filter tensor.
FilterZeroPoint - Supplies the zero point offset of the filter tensor.
FilterIsSigned - Supplies true if the filter tensor is signed data, else
false if the filter tensor is unsigned data.
Output - Supplies the output tensor in channels last format.
Channels - Supplies the number of channels.
OutputCount - Supplies the number of channel sized output elements to
produce.
KernelSize - Supplies the total number of channel sized kernel elements to
consume.
Return Value:
None.
--*/
{
if (FilterIsSigned) {
#if defined(MLAS_TARGET_AMD64)
MlasPlatform.ConvDepthwiseU8S8Kernel(
#else
MlasConvDepthwiseKernel<int8_t>(
#endif
Input,
InputZeroPoint,
reinterpret_cast<const int8_t*>(Filter),
static_cast<int8_t>(FilterZeroPoint),
Output,
Channels,
OutputCount,
KernelSize
);
} else {
#if defined(MLAS_TARGET_AMD64)
MlasPlatform.ConvDepthwiseU8U8Kernel(
#else
MlasConvDepthwiseKernel<uint8_t>(
#endif
Input,
InputZeroPoint,
Filter,
FilterZeroPoint,
Output,
Channels,
OutputCount,
KernelSize
);
}
}

299
onnxruntime/core/providers/cpu/nn/qlinearconv.cc
View file

@ -292,11 +292,12 @@ Status QLinearConv::Compute(OpKernelContext* context) const {
// Weight tensor was not constant or prepacking is disabled.
reordered_W = static_cast<uint8_t*>(alloc->Alloc(SafeInt<size_t>(sizeof(uint8_t)) * W_shape.Size()));
reordered_W_buffer = BufferUniquePtr(reordered_W, BufferDeleter(alloc));
ReorderFilter(static_cast<const uint8_t*>(W->DataRaw()),
reordered_W,
static_cast<size_t>(M),
static_cast<size_t>(W_shape[1]),
static_cast<size_t>(kernel_size));
ReorderFilter(
static_cast<const uint8_t*>(W->DataRaw()),
reordered_W,
static_cast<size_t>(M),
static_cast<size_t>(W_shape[1]),
static_cast<size_t>(kernel_size));
}
}
@ -341,10 +342,17 @@ Status QLinearConv::Compute(OpKernelContext* context) const {
}
BufferUniquePtr col_buffer;
std::vector<uint8_t> padding_data;
// Pointwise convolutions can use the original input tensor in place,
// otherwise a temporary buffer is required for the im2col transform.
if (kernel_size != 1 || !conv_attrs_.HasStridesOneAndNoPadding()) {
if (is_depthwise_conv) {
// Allocate indirection buffer pointers and prepare a padding vector for
// the im2col transform.
auto* col_data = alloc->Alloc(SafeInt<size_t>(sizeof(const uint8_t*)) * kernel_size * output_image_size);
col_buffer = BufferUniquePtr(col_data, BufferDeleter(alloc));
padding_data.resize(static_cast<size_t>(C), X_zero_point_value);
} else if (kernel_size != 1 || !conv_attrs_.HasStridesOneAndNoPadding()) {
// Pointwise convolutions can use the original input tensor in place,
// otherwise a temporary buffer is required for the im2col transform.
int64_t group_col_buffer_size = (kernel_rank > 2) ? group_count * col_buffer_size : col_buffer_size;
auto* col_data = alloc->Alloc(SafeInt<size_t>(sizeof(uint8_t)) * group_col_buffer_size);
col_buffer = BufferUniquePtr(col_data, BufferDeleter(alloc));
@ -377,33 +385,32 @@ Status QLinearConv::Compute(OpKernelContext* context) const {
if (!channels_last_) {
// Transpose the input from channels first (NCHW) to channels last (NHWC).
MlasTranspose(Xdata,
static_cast<uint8_t*>(transpose_input_buffer.get()),
static_cast<size_t>(C),
static_cast<size_t>(input_image_size));
MlasTranspose(
Xdata,
static_cast<uint8_t*>(transpose_input_buffer.get()),
static_cast<size_t>(C),
static_cast<size_t>(input_image_size));
input_data = static_cast<uint8_t*>(transpose_input_buffer.get());
output_data = static_cast<uint8_t*>(transpose_output_buffer.get());
}
if (col_buffer) {
if (kernel_rank > 2) {
// Threaded implementation of ND convolution is not yet supported, so
// prepare all im2col transformations here.
for (int64_t group_id = 0; group_id < group_count; ++group_id) {
math::Im2col<uint8_t, StorageOrder::NHWC>()(
input_data + group_id * group_input_channels,
group_input_channels,
C,
input_shape.GetDims().data(),
output_shape.GetDims().data(),
kernel_shape.data(),
strides.data(),
dilations.data(),
pads.data(),
static_cast<int64_t>(kernel_rank),
static_cast<uint8_t*>(col_buffer.get()) + group_id * col_buffer_size,
X_zero_point_value);
}
// Threaded implementation of ND convolution is not yet supported, so
// prepare all im2col transformations here.
if (!is_depthwise_conv && col_buffer && kernel_rank > 2) {
for (int64_t group_id = 0; group_id < group_count; ++group_id) {
math::Im2col<uint8_t, StorageOrder::NHWC>()(
input_data + group_id * group_input_channels,
group_input_channels,
C,
input_shape.GetDims().data(),
output_shape.GetDims().data(),
kernel_shape.data(),
strides.data(),
dilations.data(),
pads.data(),
static_cast<int64_t>(kernel_rank),
static_cast<uint8_t*>(col_buffer.get()) + group_id * col_buffer_size,
X_zero_point_value);
}
}
@ -415,134 +422,144 @@ Status QLinearConv::Compute(OpKernelContext* context) const {
auto* worker_gemm_output = gemm_output + output_start * M;
auto* worker_requantize_output = output_data + output_start * M;
for (int64_t group_id = 0; group_id < group_count; ++group_id) {
// Prepare the im2col transformation or use the input buffer directly for
// pointwise convolutions.
const uint8_t* worker_gemm_input;
if (col_buffer) {
auto* worker_col_buffer = static_cast<uint8_t*>(col_buffer.get()) + output_start * kernel_dim;
if (kernel_rank == 2) {
math::Im2col<uint8_t, StorageOrder::NHWC>()(
input_data + group_id * group_input_channels,
group_input_channels,
C,
input_shape[0],
input_shape[1],
kernel_shape[0],
kernel_shape[1],
dilations[0],
dilations[1],
pads[0],
pads[1],
strides[0],
strides[1],
output_shape[1],
output_start,
output_count,
worker_col_buffer,
X_zero_point_value);
} else if (kernel_rank == 1) {
math::Im2col<uint8_t, StorageOrder::NHWC>()(
input_data + group_id * group_input_channels,
group_input_channels,
C,
1,
input_shape[0],
1,
kernel_shape[0],
1,
dilations[0],
0,
pads[0],
1,
strides[0],
output_shape[0],
output_start,
output_count,
worker_col_buffer,
X_zero_point_value);
if (is_depthwise_conv) {
auto* worker_col_buffer = static_cast<uint8_t const**>(col_buffer.get()) + output_start * kernel_size;
math::Im2col<uint8_t, StorageOrder::NHWC>()(
input_data,
C,
input_shape.GetDims().data(),
output_shape.GetDims().data(),
kernel_shape.data(),
strides.data(),
dilations.data(),
pads.data(),
static_cast<ptrdiff_t>(kernel_rank),
output_start,
output_count,
worker_col_buffer,
padding_data.data());
MlasConvDepthwise(
worker_col_buffer,
X_zero_point_value,
reordered_W,
W_zero_point_value,
is_W_signed,
worker_gemm_output,
static_cast<size_t>(M),
static_cast<size_t>(output_count),
static_cast<size_t>(kernel_size));
} else {
for (int64_t group_id = 0; group_id < group_count; ++group_id) {
// Prepare the im2col transformation or use the input buffer directly for
// pointwise convolutions.
const uint8_t* worker_gemm_input;
if (col_buffer) {
auto* worker_col_buffer = static_cast<uint8_t*>(col_buffer.get()) + output_start * kernel_dim;
if (kernel_rank == 2) {
math::Im2col<uint8_t, StorageOrder::NHWC>()(
input_data + group_id * group_input_channels,
group_input_channels,
C,
input_shape[0],
input_shape[1],
kernel_shape[0],
kernel_shape[1],
dilations[0],
dilations[1],
pads[0],
pads[1],
strides[0],
strides[1],
output_shape[1],
output_start,
output_count,
worker_col_buffer,
X_zero_point_value);
} else if (kernel_rank == 1) {
math::Im2col<uint8_t, StorageOrder::NHWC>()(
input_data + group_id * group_input_channels,
group_input_channels,
C,
1,
input_shape[0],
1,
kernel_shape[0],
1,
dilations[0],
0,
pads[0],
1,
strides[0],
output_shape[0],
output_start,
output_count,
worker_col_buffer,
X_zero_point_value);
} else {
// Use the im2col buffer prepared outside the thread, indexed by group.
worker_col_buffer += group_id * col_buffer_size;
}
worker_gemm_input = worker_col_buffer;
} else {
// Use the im2col buffer prepared outside the thread, indexed by group.
worker_col_buffer += group_id * col_buffer_size;
worker_gemm_input = input_data + output_start * kernel_dim;
}
worker_gemm_input = worker_col_buffer;
} else {
worker_gemm_input = input_data + output_start * kernel_dim;
}
if (is_depthwise_conv) {
if (is_W_signed) {
MlasConvDepthwise(worker_gemm_input,
X_zero_point_value,
reinterpret_cast<int8_t*>(reordered_W),
static_cast<int8_t>(W_zero_point_value),
worker_gemm_output,
static_cast<size_t>(M),
static_cast<size_t>(output_count),
static_cast<size_t>(kernel_size));
} else {
MlasConvDepthwise(worker_gemm_input,
X_zero_point_value,
reordered_W,
W_zero_point_value,
worker_gemm_output,
static_cast<size_t>(M),
static_cast<size_t>(output_count),
static_cast<size_t>(kernel_size));
}
} else {
#ifdef MLAS_SUPPORTS_PACKED_GEMM_U8X8
if (packed_W_buffer_) {
MlasGemm(static_cast<size_t>(output_count),
static_cast<size_t>(group_output_channels),
static_cast<size_t>(kernel_dim),
worker_gemm_input,
static_cast<size_t>(kernel_dim),
X_zero_point_value,
static_cast<const int8_t*>(packed_W_buffer_.get()) + group_id * packed_W_size_,
W_zero_point_value,
is_W_signed,
worker_gemm_output + group_id * group_output_channels,
static_cast<size_t>(M),
nullptr);
MlasGemm(
static_cast<size_t>(output_count),
static_cast<size_t>(group_output_channels),
static_cast<size_t>(kernel_dim),
worker_gemm_input,
static_cast<size_t>(kernel_dim),
X_zero_point_value,
static_cast<const int8_t*>(packed_W_buffer_.get()) + group_id * packed_W_size_,
W_zero_point_value,
is_W_signed,
worker_gemm_output + group_id * group_output_channels,
static_cast<size_t>(M),
nullptr);
} else
#endif
{
MlasGemm(static_cast<size_t>(output_count),
static_cast<size_t>(group_output_channels),
static_cast<size_t>(kernel_dim),
worker_gemm_input,
static_cast<size_t>(kernel_dim),
X_zero_point_value,
reordered_W + group_id * group_output_channels,
static_cast<size_t>(M),
W_zero_point_value,
is_W_signed,
worker_gemm_output + group_id * group_output_channels,
static_cast<size_t>(M),
nullptr);
MlasGemm(
static_cast<size_t>(output_count),
static_cast<size_t>(group_output_channels),
static_cast<size_t>(kernel_dim),
worker_gemm_input,
static_cast<size_t>(kernel_dim),
X_zero_point_value,
reordered_W + group_id * group_output_channels,
static_cast<size_t>(M),
W_zero_point_value,
is_W_signed,
worker_gemm_output + group_id * group_output_channels,
static_cast<size_t>(M),
nullptr);
}
}
}
MlasRequantizeOutput(worker_gemm_output,
worker_requantize_output,
Bdata,
static_cast<size_t>(output_count),
static_cast<size_t>(M),
output_scales.data(),
output_scales.size() > 1,
Y_zero_point_value);
MlasRequantizeOutput(
worker_gemm_output,
worker_requantize_output,
Bdata,
static_cast<size_t>(output_count),
static_cast<size_t>(M),
output_scales.data(),
output_scales.size() > 1,
Y_zero_point_value);
};
concurrency::ThreadPool::TrySimpleParallelFor(thread_pool, thread_count, conv_worker);
if (!channels_last_) {
// Transpose the output from channels last (NHWC) to channels first (NCHW).
MlasTranspose(output_data,
Ydata,
static_cast<size_t>(output_image_size),
static_cast<size_t>(M));
MlasTranspose(
output_data,
Ydata,
static_cast<size_t>(output_image_size),
static_cast<size_t>(M));
}
Xdata += X_offset;

16
onnxruntime/core/util/math.h
View file

@ -20,7 +20,6 @@
// still keep it simple, so all platforms would be able to support it fairly
// easily.
// We include the cblas header here so that we can obtain the macros from cblas.
extern "C" {
#include "core/framework/cblas.h"
@ -49,7 +48,6 @@ void Log(int N, const T* x, T* y, Provider* provider);
template <typename T, class Provider>
void Sqr(int N, const T* x, T* y, Provider* provider);
#define DECLARE_BINARY_OP(name) \
template <typename T, class Provider> \
void name(int N, const T* a, const T* b, T* y, Provider* provider); \
@ -240,6 +238,20 @@ struct Im2col<T, StorageOrder::NHWC> {
ptrdiff_t rank,
T* data_col,
T padding_value = 0);
void operator()(
const T* data_im,
int64_t input_channels,
const int64_t* input_shape,
const int64_t* output_shape,
const int64_t* kernel_shape,
const int64_t* stride,
const int64_t* dilation,
const int64_t* pad,
ptrdiff_t rank,
int64_t output_start,
int64_t output_count,
T const** data_indirection,
const T* padding_ptr);
};
template <typename T, class Provider, int order>

110
onnxruntime/core/util/math_cpu.cc
View file

@ -210,7 +210,6 @@ template void Gemv<double, CPUMathUtil>(const CBLAS_TRANSPOSE TransA, int M, int
SPECIALIZED_AXPY(float)
#undef SPECIALIZED_AXPY
#define DELEGATE_SIMPLE_UNARY_FUNCTION(T, Funcname, expr) \
template <> \
void Funcname<T, CPUMathUtil>(int N, const T* x, T* y, CPUMathUtil*) { \
@ -535,6 +534,115 @@ void Im2col<T, StorageOrder::NHWC>::operator()(
} while (NextPosition(rank, output_shape, d_output.data()));
}
template <typename T>
void Im2col<T, StorageOrder::NHWC>::operator()(
const T* data_im,
int64_t input_channels,
const int64_t* input_shape,
const int64_t* output_shape,
const int64_t* kernel_shape,
const int64_t* stride,
const int64_t* dilation,
const int64_t* pad,
ptrdiff_t rank,
int64_t output_start,
int64_t output_count,
T const** data_indirection,
const T* padding_ptr) {
if (rank == 1) {
int64_t stride_w = stride[0];
int64_t kernel_w = kernel_shape[0];
int64_t dilation_w = dilation[0];
int64_t pad_l = pad[0];
int64_t input_w = input_shape[0];
int64_t ow = output_start * stride_w;
while (output_count--) {
int64_t iw = ow - pad_l;
for (int64_t kw = 0; kw < kernel_w; kw++) {
const T* data_ptr = data_im + iw * input_channels;
data_indirection[kw] = is_a_ge_zero_and_a_lt_b(iw, input_w) ? data_ptr : padding_ptr;
iw += dilation_w;
}
data_indirection += kernel_w;
ow += stride_w;
}
} else if (rank == 2) {
int64_t stride_h = stride[0];
int64_t stride_w = stride[1];
int64_t kernel_h = kernel_shape[0];
int64_t kernel_w = kernel_shape[1];
int64_t dilation_h = dilation[0];
int64_t dilation_w = dilation[1];
int64_t pad_t = pad[0];
int64_t pad_l = pad[1];
int64_t input_h = input_shape[0];
int64_t input_w = input_shape[1];
int64_t output_w = output_shape[1];
int64_t oh = (output_start / output_w) * stride_h;
int64_t ow = (output_start % output_w) * stride_w;
int64_t ow_end = output_w * stride_w;
while (output_count--) {
for (int64_t kh = 0; kh < kernel_h; kh++) {
int64_t ih = kh * dilation_h + oh - pad_t;
if (is_a_ge_zero_and_a_lt_b(ih, input_h)) {
int64_t ihw = ih * input_w;
int64_t iw = ow - pad_l;
for (int64_t kw = 0; kw < kernel_w; kw++) {
const T* data_ptr = data_im + (ihw + iw) * input_channels;
data_indirection[kw] = is_a_ge_zero_and_a_lt_b(iw, input_w) ? data_ptr : padding_ptr;
iw += dilation_w;
}
} else {
std::fill_n(data_indirection, kernel_w, padding_ptr);
}
data_indirection += kernel_w;
}
ow += stride_w;
if (ow == ow_end) {
oh += stride_h;
ow = 0;
}
}
} else {
// iterate dimensions on output image shape (without Batch and Channel)
std::vector<int64_t> d_output(rank, 0);
// inner iterate dimensions on kernel shape (without output channel and input channel)
std::vector<int64_t> d_kernel(rank, 0);
// Skip ahead to the starting output index.
for (ptrdiff_t d_i = rank - 1; d_i >= 0; --d_i) {
d_output[d_i] = output_start % output_shape[d_i];
output_start /= output_shape[d_i];
}
while (output_count--) {
// Loop over spatial axes in reverse order to choose an index on kernel dimensions
do {
// Loop over spatial axes in forward order to compute the indices in the image
// and the inner col, and whether the index lies in the padding.
int64_t index_im = 0;
bool is_padding = false;
for (ptrdiff_t d_i = 0; d_i < rank; ++d_i) {
int64_t d_input = d_output[d_i] * stride[d_i] - pad[d_i] + d_kernel[d_i] * dilation[d_i];
is_padding |= !is_a_ge_zero_and_a_lt_b(d_input, input_shape[d_i]);
index_im *= input_shape[d_i];
index_im += d_input;
}
const T* data_ptr = data_im + index_im * input_channels;
*data_indirection++ = is_padding ? padding_ptr : data_ptr;
} while (NextPosition(rank, kernel_shape, d_kernel.data()));
// Loop over spatial axes along the output image shape
NextPosition(rank, output_shape, d_output.data());
}
}
}
template struct Im2col<uint8_t, StorageOrder::NHWC>;
template <>

33
onnxruntime/test/providers/cpu/nn/qlinearconv_op_test.cc
View file

@ -736,8 +736,21 @@ TEST(QLinearConvTest, Conv2D_U8S8_Groups_PerChannel) {
test.Run();
}
TEST(QLinearConvTest, Conv1D_U8S8_Depthwise) {
for (int64_t channels : std::initializer_list<int64_t>{7, 8, 9, 16, 25, 64}) {
QLinearConvOpTester<uint8_t, int8_t> test;
test.GenerateRandomInput({1, channels, 25}, .03f, 12);
test.GenerateRandomWeights({channels, 1, 3}, .10f, 2);
test.GenerateRandomBias();
test.SetPads({1, 1});
test.SetGroups(channels);
test.SetOutputScaleAndZeroPoint(.21f, 88);
test.Run();
}
}
TEST(QLinearConvTest, Conv2D_U8S8_Depthwise) {
for (int64_t channels : std::initializer_list<int64_t>{7, 8, 9, 16, 25}) {
for (int64_t channels : std::initializer_list<int64_t>{7, 8, 9, 16, 25, 64}) {
QLinearConvOpTester<uint8_t, int8_t> test;
test.GenerateRandomInput({1, channels, 25, 25}, .03f, 12);
test.GenerateRandomWeights({channels, 1, 5, 5}, .10f, 0);
@ -750,7 +763,7 @@ TEST(QLinearConvTest, Conv2D_U8S8_Depthwise) {
}
TEST(QLinearConvTest, Conv2D_U8U8_Depthwise) {
for (int64_t channels : std::initializer_list<int64_t>{3, 8, 13, 24, 31}) {
for (int64_t channels : std::initializer_list<int64_t>{3, 8, 13, 24, 31, 64}) {
QLinearConvOpTester<uint8_t, uint8_t> test;
test.GenerateRandomInput({1, channels, 25, 25}, .03f, 12);
test.GenerateRandomWeights({channels, 1, 3, 3}, .10f, 167);
@ -775,13 +788,15 @@ TEST(QLinearConvTest, Conv2D_U8S8_DepthwisePointwise) {
}
TEST(QLinearConvTest, Conv3D_U8S8_Depthwise) {
QLinearConvOpTester<uint8_t, int8_t> test;
test.GenerateRandomInput({1, 16, 15, 11, 13}, .02f, 135);
test.GenerateRandomWeights({16, 1, 3, 3, 3}, .09f, 0);
test.GenerateRandomBias();
test.SetGroups(16);
test.SetOutputScaleAndZeroPoint(.85f, 112);
test.Run();
for (int64_t channels : std::initializer_list<int64_t>{6, 8, 31, 64}) {
QLinearConvOpTester<uint8_t, int8_t> test;
test.GenerateRandomInput({1, channels, 15, 11, 13}, .02f, 135);
test.GenerateRandomWeights({channels, 1, 3, 3, 3}, .09f, 0);
test.GenerateRandomBias();
test.SetGroups(channels);
test.SetOutputScaleAndZeroPoint(.85f, 112);
test.Run();
}
}
TEST(QLinearConvTest, Conv2D_U8S8_Requantize_NoBias) {