From a0afd7303da0c6622c1d73bd6dda1bec1fae457c Mon Sep 17 00:00:00 2001 From: Yufeng Li Date: Mon, 29 Nov 2021 18:43:43 -0800 Subject: [PATCH] add int8_t support for pool operators (#9852) * add int8_t support for pool operators --- docs/OperatorKernels.md | 2 +- .../contrib_ops/cpu/cpu_contrib_kernels.cc | 6 +- .../cpu/qlinear_global_average_pool.cc | 45 +- .../cpu/qlinear_global_average_pool.h | 9 +- onnxruntime/contrib_ops/cpu/qlinear_pool.cc | 56 +- onnxruntime/contrib_ops/cpu/qlinear_pool.h | 7 + .../cpu/quantization/nhwc_max_pool.cc | 41 +- onnxruntime/core/mlas/inc/mlas.h | 22 +- onnxruntime/core/mlas/lib/pooling.cpp | 146 ++++- onnxruntime/core/mlas/lib/qlgavgpool.cpp | 587 +++++++++++++----- onnxruntime/core/mlas/lib/quantize.cpp | 117 +++- onnxruntime/core/mlas/lib/transpose.cpp | 15 + onnxruntime/core/util/math_cpu.cc | 1 + .../test/contrib_ops/nhwc_maxpool_op_test.cc | 62 +- .../qlinear_global_average_pool_test.cc | 154 +++-- .../test/contrib_ops/qlinear_pool_test.cc | 302 +++++++-- .../mlas/unittest/test_qlinear_gavgpool.cpp | 85 ++- .../kernel_def_hashes/contrib.cpu.json | 4 + 18 files changed, 1234 insertions(+), 427 deletions(-) diff --git a/docs/OperatorKernels.md b/docs/OperatorKernels.md index bf637d6405..a7ea71314f 100644 --- a/docs/OperatorKernels.md +++ b/docs/OperatorKernels.md @@ -400,7 +400,7 @@ Do not modify directly.* |MaxpoolWithMask|*in* X:**T**
*in* M:**tensor(int32)**
*out* Y:**T**|1+|**X** = tensor(float)| |MurmurHash3|*in* X:**T1**
*out* Y:**T2**|1+|**T1** = tensor(double), tensor(float), tensor(int32), tensor(int64), tensor(string), tensor(uint32), tensor(uint64)
**T2** = tensor(int32), tensor(uint32)| |NGramRepeatBlock|*in* input_ids:**Tid**
*in* scores:**T**
*out* scores_out:**T**|1+|**T** = tensor(float)
**Tid** = tensor(int64)| -|NhwcMaxPool|*in* x:**T**
*out* y:**T**|1+|**T** = tensor(uint8)| +|NhwcMaxPool|*in* x:**T**
*out* y:**T**|1+|**T** = tensor(int8), tensor(uint8)| |Pad|*in* data:**T**
*in* pads:**tensor(int64)**
*in* value:**T**
*out* output:**T**|1+|**T** = tensor(float)| |QAttention|*in* input:**T1**
*in* weight:**T2**
*in* bias:**T3**
*in* input_scale:**T3**
*in* weight_scale:**T3**
*in* mask_index:**T4**
*in* input_zero_point:**T1**
*in* weight_zero_point:**T2**
*in* past:**T3**
*out* output:**T3**
*out* present:**T3**|1+|**T1** = tensor(uint8)
**T2** = tensor(int8), tensor(uint8)
**T3** = tensor(float)
**T4** = tensor(int32)| |QEmbedLayerNormalization|*in* input_ids:**T1**
*in* segment_ids:**T1**
*in* word_embedding_quant:**T2**
*in* position_embedding_quant:**T2**
*in* segment_embedding:**T2**
*in* gamma_quant:**T2**
*in* beta_quant:**T2**
*in* mask:**T1**
*in* word_embedding_scale:**T**
*in* position_embedding_scale:**T**
*in* segment_embedding_scale:**T**
*in* gamma_scale:**T**
*in* beta_scale:**T**
*in* word_embedding_zero_point:**T2**
*in* position_embedding_zero_point:**T2**
*in* segment_embedding_zero_point:**T2**
*in* gamma_zero_point:**T2**
*in* beta_zero_point:**T2**
*out* layernorm_out:**T**
*out* mask_index_out:**T1**|1+|**T** = tensor(float)| diff --git a/onnxruntime/contrib_ops/cpu/cpu_contrib_kernels.cc b/onnxruntime/contrib_ops/cpu/cpu_contrib_kernels.cc index 702b5bcd67..ebf2c6c65d 100644 --- a/onnxruntime/contrib_ops/cpu/cpu_contrib_kernels.cc +++ b/onnxruntime/contrib_ops/cpu/cpu_contrib_kernels.cc @@ -72,7 +72,8 @@ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, MatMulIntegerToFloat); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, DynamicQuantizeLSTM); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, QLinearConv); -class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, NhwcMaxPool); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, int8_t, NhwcMaxPool); +class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, NhwcMaxPool); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, QEmbedLayerNormalization); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, QGemm); // ******** End: Quantization ******************* // @@ -160,7 +161,8 @@ Status RegisterQuantizationKernels(KernelRegistry& kernel_registry) { BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, - BuildKernelCreateInfo, + BuildKernelCreateInfo, + BuildKernelCreateInfo, BuildKernelCreateInfo, BuildKernelCreateInfo, }; diff --git a/onnxruntime/contrib_ops/cpu/qlinear_global_average_pool.cc b/onnxruntime/contrib_ops/cpu/qlinear_global_average_pool.cc index d8314d4cfd..ee63c23869 100644 --- a/onnxruntime/contrib_ops/cpu/qlinear_global_average_pool.cc +++ b/onnxruntime/contrib_ops/cpu/qlinear_global_average_pool.cc @@ -14,13 +14,14 @@ using onnxruntime::concurrency::ThreadPool; namespace onnxruntime { namespace contrib { +template Status ComputeQLinearGlobalAvgPool( - const uint8_t* x, + const T8Bits* x, float x_scale, - uint8_t x_zero_point, - uint8_t* y, + T8Bits x_zero_point, + T8Bits* y, float y_scale, - uint8_t y_zero_point, + T8Bits y_zero_point, int64_t N, int64_t C, int64_t image_size, @@ -28,8 +29,8 @@ Status ComputeQLinearGlobalAvgPool( concurrency::ThreadPool* tp) { if (!channels_last || C == 1) { auto worker = [=](std::ptrdiff_t first, std::ptrdiff_t last) { - const uint8_t* input = (const uint8_t*)(x + (first * image_size)); - uint8_t* output = (uint8_t*)(y + first); + const T8Bits* input = (const T8Bits*)(x + (first * image_size)); + T8Bits* output = (T8Bits*)(y + first); std::vector acc_buffer(MlasQLinearSafePaddingElementCount(sizeof(int32_t), last - first)); MlasQLinearGlobalAveragePoolNchw(input, x_scale, x_zero_point, output, y_scale, y_zero_point, last - first, image_size, acc_buffer.data()); }; @@ -37,17 +38,17 @@ Status ComputeQLinearGlobalAvgPool( tp, static_cast(N * C), {1.0 * image_size, 1.0, 8.0 * image_size}, worker); } else { auto worker = [=](std::ptrdiff_t first, std::ptrdiff_t last) { - const uint8_t* input = x + first * C * image_size; - uint8_t* output = y + first * C; + const T8Bits* input = x + first * C * image_size; + T8Bits* output = y + first * C; std::vector acc_buffer(MlasQLinearSafePaddingElementCount(sizeof(int32_t), C)); - std::vector zero_buffer(MlasQLinearSafePaddingElementCount(sizeof(uint8_t), C), 0); + std::vector zero_buffer(MlasQLinearSafePaddingElementCount(sizeof(T8Bits), C), 0); MlasQLinearGlobalAveragePoolNhwc( input, x_scale, x_zero_point, output, y_scale, y_zero_point, last - first, image_size, C, C, acc_buffer.data(), zero_buffer.data()); }; concurrency::ThreadPool::TryParallelFor( tp, static_cast(N), - {1.0 * image_size * C, 1.0 * C, 8.0 *image_size * C}, + {1.0 * image_size * C, 1.0 * C, 8.0 * image_size * C}, worker); } return Status::OK(); @@ -88,19 +89,25 @@ Status QLinearGlobalAveragePool::Compute(OpKernelContext* context) const { const float x_scale = *(tensor_x_scale->Data()); const float y_scale = *(tensor_y_scale->Data()); + auto dtype = X.GetElementType(); - switch (dtype) { - case ONNX_NAMESPACE::TensorProto_DataType_UINT8: - return ComputeQLinearGlobalAvgPool(X.Data(), x_scale, *(tensor_x_zero_point->Data()), - Y.MutableData(), y_scale, *(tensor_y_zero_point->Data()), - N, C, image_size, channels_last_, tp); - default: - ORT_THROW("Unsupported 'dtype' value: ", dtype); + if (dtype == ONNX_NAMESPACE::TensorProto_DataType_UINT8) { + return ComputeQLinearGlobalAvgPool(X.Data(), x_scale, *(tensor_x_zero_point->Data()), + Y.MutableData(), y_scale, *(tensor_y_zero_point->Data()), + N, C, image_size, channels_last_, tp); + } else { + return ComputeQLinearGlobalAvgPool(X.Data(), x_scale, *(tensor_x_zero_point->Data()), + Y.MutableData(), y_scale, *(tensor_y_zero_point->Data()), + N, C, image_size, channels_last_, tp); } } -ONNX_OPERATOR_KERNEL_EX(QLinearGlobalAveragePool, kMSDomain, 1, kCpuExecutionProvider, KernelDefBuilder(), QLinearGlobalAveragePool); +ONNX_OPERATOR_KERNEL_EX(QLinearGlobalAveragePool, + kMSDomain, + 1, + kCpuExecutionProvider, + KernelDefBuilder(), + QLinearGlobalAveragePool); } // namespace contrib - } // namespace onnxruntime diff --git a/onnxruntime/contrib_ops/cpu/qlinear_global_average_pool.h b/onnxruntime/contrib_ops/cpu/qlinear_global_average_pool.h index f46ed1c0e4..bc0fca6063 100644 --- a/onnxruntime/contrib_ops/cpu/qlinear_global_average_pool.h +++ b/onnxruntime/contrib_ops/cpu/qlinear_global_average_pool.h @@ -21,13 +21,14 @@ class QLinearGlobalAveragePool final : public OpKernel { bool channels_last_; }; +template Status ComputeQLinearGlobalAvgPool( - const uint8_t* x, + const T8Bits* x, float x_scale, - uint8_t x_zero_point, - uint8_t* y, + T8Bits x_zero_point, + T8Bits* y, float y_scale, - uint8_t y_zero_point, + T8Bits y_zero_point, int64_t N, int64_t C, int64_t image_size, diff --git a/onnxruntime/contrib_ops/cpu/qlinear_pool.cc b/onnxruntime/contrib_ops/cpu/qlinear_pool.cc index 154c4620b3..6953206a70 100644 --- a/onnxruntime/contrib_ops/cpu/qlinear_pool.cc +++ b/onnxruntime/contrib_ops/cpu/qlinear_pool.cc @@ -23,19 +23,15 @@ using concurrency::ThreadPool; namespace contrib { template -static inline float dequantize_value(T8Bits x, float x_scale, T8Bits x_zero_point); - -template -static inline T8Bits quantize_value(float y, float y_scale, T8Bits y_zero_point); - -template <> -inline float dequantize_value(uint8_t x, float x_scale, uint8_t x_zero_point) { +static inline float dequantize_value(T8Bits x, float x_scale, T8Bits x_zero_point) { return x_scale * (static_cast(x) - x_zero_point); } -template <> -inline uint8_t quantize_value(float y, float y_scale, uint8_t y_zero_point) { - return static_cast(std::max(0.0f, std::min(std::nearbyintf(y / y_scale + y_zero_point), 255.0f))); +template +static inline T8Bits quantize_value(float y, float y_scale, T8Bits y_zero_point) { + constexpr int32_t min_8bits = std::numeric_limits::lowest(); + constexpr int32_t max_8bits = std::numeric_limits::max(); + return static_cast(std::max(min_8bits, std::min(static_cast(std::nearbyintf(y / y_scale + y_zero_point)), max_8bits))); } static void SwitchDimsNchwNhwc(std::vector& dims, bool from_nchw_to_nhwc) { @@ -509,6 +505,15 @@ void dequantize_array(int64_t N, const T8Bits* input, float scale, T8Bits zero_p } Status QLinearAveragePool::Compute(OpKernelContext* context) const { + if (is_input_signed_) { + return ComputeImpl(context); + } else { + return ComputeImpl(context); + } +} + +template +Status QLinearAveragePool::ComputeImpl(OpKernelContext* context) const { const auto tensor_x_scale = context->Input(1); const auto tensor_x_zero_point = context->Input(2); const auto tensor_y_scale = context->Input(3); @@ -524,16 +529,12 @@ Status QLinearAveragePool::Compute(OpKernelContext* context) const { "input y_zero_point must be a scalar or 1D tensor of size 1 if given"); const auto* X = context->Input(0); - auto dtype = X->GetElementType(); - if (dtype != ONNX_NAMESPACE::TensorProto_DataType_UINT8) { - ORT_THROW("Unsupported 'dtype' in QLinear Pooling:", dtype); - } TensorShape x_shape = X->Shape(); const float x_scale = *(tensor_x_scale->Data()); const float y_scale = *(tensor_y_scale->Data()); - uint8_t x_zero_point = (tensor_x_zero_point ? *(tensor_x_zero_point->Data()) : (uint8_t)0); - uint8_t y_zero_point = (tensor_y_zero_point ? *(tensor_y_zero_point->Data()) : (uint8_t)0); + T8Bits x_zero_point = (tensor_x_zero_point ? *(tensor_x_zero_point->Data()) : (T8Bits)0); + T8Bits y_zero_point = (tensor_y_zero_point ? *(tensor_y_zero_point->Data()) : (T8Bits)0); ORT_RETURN_IF_NOT(x_shape.NumDimensions() >= 3, "Input dimension cannot be less than 3."); std::vector pads = pool_attrs_.pads; @@ -564,8 +565,8 @@ Status QLinearAveragePool::Compute(OpKernelContext* context) const { SwitchDimsNchwNhwc(output_dims, true); } Tensor* Y = context->Output(0, output_dims); - const auto* X_data = X->Data(); - auto* Y_data = Y->MutableData(); + const auto* X_data = X->Data(); + auto* Y_data = Y->MutableData(); ThreadPool* tp = context->GetOperatorThreadPool(); // Check for special case which could fall back to global average pool @@ -589,12 +590,12 @@ Status QLinearAveragePool::Compute(OpKernelContext* context) const { switch (kernel_shape.size()) { case 1: { if (channels_last_) { - QLinearPoolNhwc1DTask avg_pool_task_1d = { + QLinearPoolNhwc1DTask avg_pool_task_1d = { x_data_fp32, Y_data, y_scale, y_zero_point, channels, pooled_height, strides[0], height, kernel_shape, pads, pool_context_, pool_attrs_}; ThreadPool::TryParallelFor(tp, y_image_size * batch_count, avg_pool_task_1d.Cost(), avg_pool_task_1d); } else { - QLinearPool1DTask avg_pool_task_1d = { + QLinearPool1DTask avg_pool_task_1d = { x_data_fp32, Y_data, y_scale, y_zero_point, x_image_size, y_image_size, pooled_height, strides[0], height, kernel_shape, pads, pool_context_, pool_attrs_}; ThreadPool::TryParallelFor(tp, total_channels, avg_pool_task_1d.Cost(), avg_pool_task_1d); @@ -604,13 +605,13 @@ Status QLinearAveragePool::Compute(OpKernelContext* context) const { case 2: { if (channels_last_) { - QLinearPoolNhwc2DTask avg_pool_task_2d = { + QLinearPoolNhwc2DTask avg_pool_task_2d = { x_data_fp32, Y_data, y_scale, y_zero_point, x_image_size, y_image_size, kernel_size, channels, pooled_height, pooled_width, strides[0], strides[1], height, width, kernel_shape, pads, pool_context_, pool_attrs_}; ThreadPool::TryParallelFor(tp, y_image_size * batch_count, avg_pool_task_2d.Cost(), avg_pool_task_2d); } else { - QLinearPool2DTask avg_pool_task_2d = { + QLinearPool2DTask avg_pool_task_2d = { x_data_fp32, Y_data, y_scale, y_zero_point, x_image_size, y_image_size, pooled_height, pooled_width, strides[0], strides[1], height, width, kernel_shape, pads, pool_context_, pool_attrs_}; ThreadPool::TryParallelFor(tp, total_channels, avg_pool_task_2d.Cost(), avg_pool_task_2d); @@ -620,14 +621,14 @@ Status QLinearAveragePool::Compute(OpKernelContext* context) const { case 3: { if (channels_last_) { - QLinearPoolNhwc3DTask avg_pool_task_3d = { + QLinearPoolNhwc3DTask avg_pool_task_3d = { x_data_fp32, Y_data, y_scale, y_zero_point, x_image_size, y_image_size, kernel_size, channels, pooled_height, pooled_width, pooled_depth, strides[0], strides[1], strides[2], height, width, depth, kernel_shape, pads, pool_context_, pool_attrs_}; ThreadPool::TryParallelFor(tp, y_image_size * batch_count, avg_pool_task_3d.Cost(), avg_pool_task_3d); } else { - QLinearPool3DTask avg_pool_task_3d = { + QLinearPool3DTask avg_pool_task_3d = { x_data_fp32, Y_data, y_scale, y_zero_point, x_image_size, y_image_size, pooled_height, pooled_width, pooled_depth, strides[0], strides[1], strides[2], height, width, depth, kernel_shape, pads, pool_context_, pool_attrs_}; @@ -647,7 +648,12 @@ Status QLinearAveragePool::Compute(OpKernelContext* context) const { return Status::OK(); } -ONNX_OPERATOR_KERNEL_EX(QLinearAveragePool, kMSDomain, 1, kCpuExecutionProvider, KernelDefBuilder(), QLinearAveragePool); +ONNX_OPERATOR_KERNEL_EX(QLinearAveragePool, + kMSDomain, + 1, + kCpuExecutionProvider, + KernelDefBuilder(), + QLinearAveragePool); } // namespace contrib diff --git a/onnxruntime/contrib_ops/cpu/qlinear_pool.h b/onnxruntime/contrib_ops/cpu/qlinear_pool.h index 92285e4f78..ebb6be4a00 100644 --- a/onnxruntime/contrib_ops/cpu/qlinear_pool.h +++ b/onnxruntime/contrib_ops/cpu/qlinear_pool.h @@ -14,6 +14,9 @@ class QLinearAveragePool final : public OpKernel, public PoolBase { public: QLinearAveragePool(const OpKernelInfo& info) : OpKernel(info), PoolBase(info) { channels_last_ = (info.GetAttrOrDefault("channels_last", static_cast(0)) != 0); + + int32_t input_type = info.node().InputDefs()[0]->TypeAsProto()->tensor_type().elem_type(); + is_input_signed_ = ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT8 == input_type; } ~QLinearAveragePool() override = default; @@ -21,8 +24,12 @@ class QLinearAveragePool final : public OpKernel, public PoolBase { Status Compute(OpKernelContext* context) const override; private: + template + Status ComputeImpl(OpKernelContext* context) const; + PoolProcessContext pool_context_; bool channels_last_; + bool is_input_signed_; }; } // namespace contrib diff --git a/onnxruntime/contrib_ops/cpu/quantization/nhwc_max_pool.cc b/onnxruntime/contrib_ops/cpu/quantization/nhwc_max_pool.cc index 991f5f19e8..a6f297c7b5 100644 --- a/onnxruntime/contrib_ops/cpu/quantization/nhwc_max_pool.cc +++ b/onnxruntime/contrib_ops/cpu/quantization/nhwc_max_pool.cc @@ -11,6 +11,7 @@ namespace onnxruntime { namespace contrib { +template class NhwcMaxPool : public OpKernel { public: explicit NhwcMaxPool(const OpKernelInfo& info) : OpKernel(info), @@ -20,10 +21,11 @@ class NhwcMaxPool : public OpKernel { Status Compute(OpKernelContext* context) const override; private: - PoolAttributes pool_attrs_; + PoolAttributes pool_attrs_; }; -Status NhwcMaxPool::Compute(OpKernelContext* context) const { +template +Status NhwcMaxPool::Compute(OpKernelContext* context) const { const auto* X = context->Input(0); const TensorShape& input_shape = X->Shape(); @@ -73,17 +75,17 @@ Status NhwcMaxPool::Compute(OpKernelContext* context) const { AllocatorPtr alloc; ORT_RETURN_IF_ERROR(context->GetTempSpaceAllocator(&alloc)); int64_t col_buffer_batch_count = std::min(output_image_size, output_batch_count); - auto* col_data = alloc->Alloc(SafeInt(sizeof(const uint8_t*)) * kernel_size * col_buffer_batch_count); + auto* col_data = alloc->Alloc(SafeInt(sizeof(const T8Bits*)) * kernel_size * col_buffer_batch_count); BufferUniquePtr col_buffer(col_data, BufferDeleter(alloc)); - std::vector padding_data(static_cast(C), 0); + std::vector padding_data(static_cast(C), std::numeric_limits::lowest()); - const auto* Xdata = X->template Data(); - auto* Ydata = Y->template MutableData(); + const auto* Xdata = X->template Data(); + auto* Ydata = Y->template MutableData(); for (int64_t image_id = 0; image_id < N; ++image_id) { for (int64_t output_start = 0; output_start < output_image_size;) { int64_t output_count = std::min(output_image_size - output_start, output_batch_count); - math::Im2col()( + math::Im2col()( Xdata, C, input_shape.GetDims().data() + 1, @@ -95,10 +97,10 @@ Status NhwcMaxPool::Compute(OpKernelContext* context) const { static_cast(spatial_dims), output_start, output_count, - static_cast(col_buffer.get()), + static_cast(col_buffer.get()), padding_data.data()); MlasMaximumPool( - static_cast(col_buffer.get()), + static_cast(col_buffer.get()), Ydata, static_cast(C), static_cast(output_count), @@ -114,14 +116,19 @@ Status NhwcMaxPool::Compute(OpKernelContext* context) const { return Status::OK(); } -ONNX_OPERATOR_KERNEL_EX( - NhwcMaxPool, - kMSDomain, - 1, - kCpuExecutionProvider, - KernelDefBuilder() - .TypeConstraint("T", DataTypeImpl::GetTensorType()), - NhwcMaxPool); +#define REGISTER_NHWCMAXPOOL_TYPED_KERNEL(T) \ + ONNX_OPERATOR_TYPED_KERNEL_EX( \ + NhwcMaxPool, \ + kMSDomain, \ + 1, \ + T, \ + kCpuExecutionProvider, \ + KernelDefBuilder() \ + .TypeConstraint("T", DataTypeImpl::GetTensorType()), \ + NhwcMaxPool); + +REGISTER_NHWCMAXPOOL_TYPED_KERNEL(int8_t); +REGISTER_NHWCMAXPOOL_TYPED_KERNEL(uint8_t); } // namespace contrib } // namespace onnxruntime diff --git a/onnxruntime/core/mlas/inc/mlas.h b/onnxruntime/core/mlas/inc/mlas.h index 01f22c25b4..d3a5619c5c 100644 --- a/onnxruntime/core/mlas/inc/mlas.h +++ b/onnxruntime/core/mlas/inc/mlas.h @@ -786,11 +786,12 @@ MlasPool( MLAS_THREADPOOL* ThreadPool ); +template void MLASCALL MlasMaximumPool( - const uint8_t* const* Input, - uint8_t* Output, + const T8Bits* const* Input, + T8Bits* Output, size_t Channels, size_t OutputCount, size_t KernelSize @@ -1041,17 +1042,18 @@ MlasQuantizeLinear( * @param CountN * @return */ +template void MLASCALL MlasRequantizeOutput( const int32_t* Input, size_t InputLeadingDimension, - uint8_t* Output, + OutputType* Output, size_t OutputLeadingDimension, const int32_t* Bias, const float* Scale, bool PerColumnScale, - uint8_t ZeroPoint, + OutputType ZeroPoint, size_t StartM, size_t StartN, size_t CountM, @@ -1115,13 +1117,14 @@ MlasQLinearSafePaddingElementCount( size_t ElementCount ); +template void MLASCALL MlasQLinearGlobalAveragePoolNchw( - const uint8_t* Input, + const T8Bits* Input, float ScaleInput, int32_t ZeroPointInput, - uint8_t* Output, + T8Bits* Output, float ScaleOutput, int32_t ZeroPointOutput, size_t Channels, @@ -1129,13 +1132,14 @@ MlasQLinearGlobalAveragePoolNchw( int32_t* AccumulateBuffer ); +template void MLASCALL MlasQLinearGlobalAveragePoolNhwc( - const uint8_t* Input, + const T8Bits* Input, float ScaleInput, int32_t ZeroPointInput, - uint8_t* Output, + T8Bits* Output, float ScaleOutput, int32_t ZeroPointOutput, size_t Batch, @@ -1143,7 +1147,7 @@ MlasQLinearGlobalAveragePoolNhwc( size_t Stride, size_t Channels, int32_t* AccumulateBuffer, - const uint8_t* ZeroBuffer + const T8Bits* ZeroBuffer ); // diff --git a/onnxruntime/core/mlas/lib/pooling.cpp b/onnxruntime/core/mlas/lib/pooling.cpp index 0223da1ee6..649e137182 100644 --- a/onnxruntime/core/mlas/lib/pooling.cpp +++ b/onnxruntime/core/mlas/lib/pooling.cpp @@ -1304,11 +1304,12 @@ Return Value: #endif } +template void MLASCALL MlasMaximumPool( - const uint8_t* const* Input, - uint8_t* Output, + const T8Bits* const* Input, + T8Bits* Output, size_t Channels, size_t OutputCount, size_t KernelSize @@ -1352,6 +1353,10 @@ Return Value: size_t c = Channels; #if defined(MLAS_SSE2_INTRINSICS) + const __m128i BitFlipVector = _mm_set1_epi32(0x80808080); + if constexpr (std::is_unsigned::value) { + MLAS_UNREFERENCED_PARAMETER(BitFlipVector); + } while (c >= 32) { @@ -1363,10 +1368,20 @@ Return Value: __m128i InputVector0 = _mm_loadu_si128((const __m128i*)&Input[k][ChannelOffset]); __m128i InputVector1 = _mm_loadu_si128((const __m128i*)&Input[k][ChannelOffset + 16]); + if constexpr (std::is_signed::value) { + InputVector0 = _mm_xor_si128(InputVector0, BitFlipVector); + InputVector1 = _mm_xor_si128(InputVector1, BitFlipVector); + } + MaximumVector0 = _mm_max_epu8(MaximumVector0, InputVector0); MaximumVector1 = _mm_max_epu8(MaximumVector1, InputVector1); } + if constexpr (std::is_signed::value) { + MaximumVector0 = _mm_xor_si128(MaximumVector0, BitFlipVector); + MaximumVector1 = _mm_xor_si128(MaximumVector1, BitFlipVector); + } + _mm_storeu_si128((__m128i*)&Output[0], MaximumVector0); _mm_storeu_si128((__m128i*)&Output[16], MaximumVector1); Output += 32; @@ -1383,9 +1398,17 @@ Return Value: __m128i InputVector0 = _mm_loadu_si128((const __m128i*)&Input[k][ChannelOffset]); + if constexpr (std::is_signed::value){ + InputVector0 = _mm_xor_si128(InputVector0, BitFlipVector); + } + MaximumVector0 = _mm_max_epu8(MaximumVector0, InputVector0); } + if constexpr (std::is_signed::value) { + MaximumVector0 = _mm_xor_si128(MaximumVector0, BitFlipVector); + } + _mm_storeu_si128((__m128i*)&Output[0], MaximumVector0); Output += 16; @@ -1401,9 +1424,17 @@ Return Value: __m128i InputVector0 = _mm_loadl_epi64((const __m128i*)&Input[k][ChannelOffset]); + if constexpr (std::is_signed::value){ + InputVector0 = _mm_xor_si128(InputVector0, BitFlipVector); + } + MaximumVector0 = _mm_max_epu8(MaximumVector0, InputVector0); } + if constexpr (std::is_signed::value) { + MaximumVector0 = _mm_xor_si128(MaximumVector0, BitFlipVector); + } + _mm_storel_epi64((__m128i*)&Output[0], MaximumVector0); Output += 8; @@ -1415,20 +1446,40 @@ Return Value: while (c >= 32) { - uint8x16_t MaximumVector0 = vdupq_n_u8(0); - uint8x16_t MaximumVector1 = vdupq_n_u8(0); + if constexpr (std::is_signed::value){ - for (size_t k = 0; k < KernelSize; k++) { + int8x16_t MaximumVector0 = vdupq_n_s8(-128); + int8x16_t MaximumVector1 = vdupq_n_s8(-128); - uint8x16_t InputVector0 = vld1q_u8(&Input[k][ChannelOffset]); - uint8x16_t InputVector1 = vld1q_u8(&Input[k][ChannelOffset + 16]); + for (size_t k = 0; k < KernelSize; k++) { - MaximumVector0 = vmaxq_u8(MaximumVector0, InputVector0); - MaximumVector1 = vmaxq_u8(MaximumVector1, InputVector1); + int8x16_t InputVector0 = vld1q_s8(&Input[k][ChannelOffset]); + int8x16_t InputVector1 = vld1q_s8(&Input[k][ChannelOffset + 16]); + + MaximumVector0 = vmaxq_s8(MaximumVector0, InputVector0); + MaximumVector1 = vmaxq_s8(MaximumVector1, InputVector1); + } + + vst1q_s8(&Output[0], MaximumVector0); + vst1q_s8(&Output[16], MaximumVector1); + } else { + + uint8x16_t MaximumVector0 = vdupq_n_u8(0); + uint8x16_t MaximumVector1 = vdupq_n_u8(0); + + for (size_t k = 0; k < KernelSize; k++) { + + uint8x16_t InputVector0 = vld1q_u8(&Input[k][ChannelOffset]); + uint8x16_t InputVector1 = vld1q_u8(&Input[k][ChannelOffset + 16]); + + MaximumVector0 = vmaxq_u8(MaximumVector0, InputVector0); + MaximumVector1 = vmaxq_u8(MaximumVector1, InputVector1); + } + + vst1q_u8(&Output[0], MaximumVector0); + vst1q_u8(&Output[16], MaximumVector1); } - vst1q_u8(&Output[0], MaximumVector0); - vst1q_u8(&Output[16], MaximumVector1); Output += 32; ChannelOffset += 32; @@ -1437,16 +1488,30 @@ Return Value: while (c >= 16) { - uint8x16_t MaximumVector0 = vdupq_n_u8(0); + if constexpr (std::is_signed::value){ - for (size_t k = 0; k < KernelSize; k++) { + int8x16_t MaximumVector0 = vdupq_n_s8(-128); - uint8x16_t InputVector0 = vld1q_u8(&Input[k][ChannelOffset]); + for (size_t k = 0; k < KernelSize; k++) { - MaximumVector0 = vmaxq_u8(MaximumVector0, InputVector0); + int8x16_t InputVector0 = vld1q_s8(&Input[k][ChannelOffset]); + MaximumVector0 = vmaxq_s8(MaximumVector0, InputVector0); + } + + vst1q_s8(&Output[0], MaximumVector0); + } else { + + uint8x16_t MaximumVector0 = vdupq_n_u8(0); + + for (size_t k = 0; k < KernelSize; k++) { + + uint8x16_t InputVector0 = vld1q_u8(&Input[k][ChannelOffset]); + MaximumVector0 = vmaxq_u8(MaximumVector0, InputVector0); + } + + vst1q_u8(&Output[0], MaximumVector0); } - vst1q_u8(&Output[0], MaximumVector0); Output += 16; ChannelOffset += 16; @@ -1455,16 +1520,29 @@ Return Value: if (c >= 8) { - uint8x8_t MaximumVector0 = vdup_n_u8(0); + if constexpr (std::is_signed::value){ - for (size_t k = 0; k < KernelSize; k++) { + int8x8_t MaximumVector0 = vdup_n_s8(-128); - uint8x8_t InputVector0 = vld1_u8(&Input[k][ChannelOffset]); + for (size_t k = 0; k < KernelSize; k++) { - MaximumVector0 = vmax_u8(MaximumVector0, InputVector0); + int8x8_t InputVector0 = vld1_s8(&Input[k][ChannelOffset]); + MaximumVector0 = vmax_s8(MaximumVector0, InputVector0); + } + + vst1_s8(&Output[0], MaximumVector0); + } else { + + uint8x8_t MaximumVector0 = vdup_n_u8(0); + + for (size_t k = 0; k < KernelSize; k++) { + + uint8x8_t InputVector0 = vld1_u8(&Input[k][ChannelOffset]); + MaximumVector0 = vmax_u8(MaximumVector0, InputVector0); + } + vst1_u8(&Output[0], MaximumVector0); } - vst1_u8(&Output[0], MaximumVector0); Output += 8; ChannelOffset += 8; @@ -1475,13 +1553,13 @@ Return Value: while (c > 0) { - int32_t MaximumValue = 0; + int32_t MaximumValue = std::numeric_limits::lowest(); for (size_t k = 0; k < KernelSize; k++) { MaximumValue = std::max(MaximumValue, int32_t(Input[k][ChannelOffset])); } - *Output++ = uint8_t(MaximumValue); + *Output++ = T8Bits(MaximumValue); ChannelOffset += 1; c -= 1; @@ -1491,3 +1569,25 @@ Return Value: OutputCount -= 1; } } + +template +void +MLASCALL +MlasMaximumPool( + const int8_t* const* Input, + int8_t* Output, + size_t Channels, + size_t OutputCount, + size_t KernelSize + ); + +template +void +MLASCALL +MlasMaximumPool( + const uint8_t* const* Input, + uint8_t* Output, + size_t Channels, + size_t OutputCount, + size_t KernelSize + ); diff --git a/onnxruntime/core/mlas/lib/qlgavgpool.cpp b/onnxruntime/core/mlas/lib/qlgavgpool.cpp index d8972eecbf..38c777d34b 100644 --- a/onnxruntime/core/mlas/lib/qlgavgpool.cpp +++ b/onnxruntime/core/mlas/lib/qlgavgpool.cpp @@ -24,6 +24,7 @@ MlasQLinearSafePaddingElementCount( ) { if (!(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8 || ElementSize == 16)) { + #ifdef MLAS_NO_EXCEPTION abort(); #else @@ -42,6 +43,7 @@ CheckQLinearGlobalAveragePoolScaleAndSize( ) { if (ImageSize >= 0x1000000) { + #ifdef MLAS_NO_EXCEPTION abort(); #else @@ -51,6 +53,7 @@ CheckQLinearGlobalAveragePoolScaleAndSize( float scale = ScaleInput / (ScaleOutput * static_cast(ImageSize)); if (scale < 0x1.0p-32f || scale >= 256.0f) { + // In first case, the scale is too small, ScaleInput/ScaleOutput < 1/256 no matter what ImageSize // In second case, the scale is too large, ScaleInput/ScaleOutput >= 256 no matter what Image Size // both case make output value constant, and hence not meaningful. @@ -65,13 +68,14 @@ CheckQLinearGlobalAveragePoolScaleAndSize( #if defined(MLAS_NEON_INTRINSICS) +template void MLASCALL MlasQLinearGlobalAveragePoolNchw( - const uint8_t* Input, + const T8Bits* Input, float ScaleInput, int32_t ZeroPointInput, - uint8_t* Output, + T8Bits* Output, float ScaleOutput, int32_t ZeroPointOutput, size_t Channels, @@ -83,38 +87,65 @@ MlasQLinearGlobalAveragePoolNchw( int32_t bias[] = {-ZeroPointInput * static_cast(ImageSize), 0, 0, 0}; const int32x4_t vbias = vld1q_s32(bias); const int32x4_t vzero = vmovq_n_s32(0); + const uint8_t* InputU8 = (const uint8_t*)(Input); int32_t* sum_buffer = AccumulateBuffer; uint8_t tail_buffer[8] = {0, 0, 0, 0, 0, 0, 0, 0}; for (size_t c = Channels; c > 0; c--) { + int32x4_t vacc_lo = vbias; int32x4_t vacc_hi = vzero; auto Len = ImageSize; for (; Len >= 32; Len -= 32) { - const uint8x8_t vi0 = vld1_u8(Input); - const uint8x8_t vi1 = vld1_u8(Input + 8); - const uint8x8_t vi2 = vld1_u8(Input + 16); - const uint8x8_t vi3 = vld1_u8(Input + 24); - const uint16x8_t vs01 = vaddl_u8(vi0, vi1); - const uint16x8_t vs23 = vaddl_u8(vi2, vi3); - const int16x8_t vsum = vreinterpretq_s16_u16(vaddq_u16(vs01, vs23)); + const uint8x8_t vi0 = vld1_u8(InputU8); + const uint8x8_t vi1 = vld1_u8(InputU8 + 8); + const uint8x8_t vi2 = vld1_u8(InputU8 + 16); + const uint8x8_t vi3 = vld1_u8(InputU8 + 24); + + int16x8_t vsum; + if constexpr (std::is_signed::value) { + + const int16x8_t vs01 = vaddl_s8(vreinterpret_s8_u8(vi0), vreinterpret_s8_u8(vi1)); + const int16x8_t vs23 = vaddl_s8(vreinterpret_s8_u8(vi2), vreinterpret_s8_u8(vi3)); + vsum = vaddq_s16(vs01, vs23); + } else { + + const uint16x8_t vs01 = vaddl_u8(vi0, vi1); + const uint16x8_t vs23 = vaddl_u8(vi2, vi3); + vsum = vreinterpretq_s16_u16(vaddq_u16(vs01, vs23)); + } + vacc_lo = vaddw_s16(vacc_lo, vget_low_s16(vsum)); vacc_hi = vaddw_s16(vacc_hi, vget_high_s16(vsum)); - Input += 32; + InputU8 += 32; } for (; Len >= 8; Len -= 8) { - const int16x8_t vsum = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(Input))); + + int16x8_t vsum; + if constexpr (std::is_signed::value) { + vsum = vmovl_s8(vreinterpret_s8_u8(vld1_u8(InputU8))); + } else { + vsum = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(InputU8))); + } vacc_lo = vaddw_s16(vacc_lo, vget_low_s16(vsum)); vacc_hi = vaddw_s16(vacc_hi, vget_high_s16(vsum)); - Input += 8; + InputU8 += 8; } + if (Len > 0) { - memcpy(tail_buffer, Input, Len); - const int16x8_t vsum = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(tail_buffer))); + + memcpy(tail_buffer, InputU8, Len); + int16x8_t vsum; + if constexpr (std::is_signed::value) { + vsum = vmovl_s8(vreinterpret_s8_u8(vld1_u8(tail_buffer))); + } else { + vsum = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(tail_buffer))); + } + vacc_lo = vaddw_s16(vacc_lo, vget_low_s16(vsum)); vacc_hi = vaddw_s16(vacc_hi, vget_high_s16(vsum)); - Input += Len; + InputU8 += Len; } vacc_lo = vaddq_s32(vacc_lo, vacc_hi); @@ -123,23 +154,24 @@ MlasQLinearGlobalAveragePoolNchw( } MlasRequantizeOutput(AccumulateBuffer, Channels, Output, Channels, nullptr, &scale, false, - static_cast(ZeroPointOutput), 0, 0, 1, Channels); + static_cast(ZeroPointOutput), 0, 0, 1, Channels); } +template MLAS_FORCEINLINE void MlasQLinearGlobalAveragePoolNhwcSingleBatch( - const uint8_t* Input, - uint8_t* Output, - const uint8_t* LastOf8, + const T8Bits* Input, + T8Bits* Output, + const T8Bits* LastOf8, size_t ImageSize, size_t Channels, size_t Stride, int32_t Bias, float Scale, - uint8_t Output_zero_point, + T8Bits Output_zero_point, int32_t* AccumulateBuffer, - const uint8_t* ZeroBuffer + const T8Bits* ZeroBuffer ) { #define LOAD_FULL_CHANNELS() \ @@ -158,24 +190,35 @@ MlasQLinearGlobalAveragePoolNhwcSingleBatch( const uint8x8_t vi6 = vld1_u8(i6); \ i6 += 8 -#define CALCULATE_ACCUMULATE_VECTORS() \ - int32x4_t vacc_lo = finish_one_pass ? vld1q_s32(acc) : vbias; \ - int32x4_t vacc_hi = finish_one_pass ? vld1q_s32(acc + 4) : vbias; \ - const uint16x8_t vsum01 = vaddl_u8(vi0, vi1); \ - const uint16x8_t vsum23 = vaddl_u8(vi2, vi3); \ - const uint16x8_t vsum45 = vaddl_u8(vi4, vi5); \ - const uint16x8_t vsum016 = vaddw_u8(vsum01, vi6); \ - const uint16x8_t vsum2345 = vaddq_u16(vsum23, vsum45); \ - const int16x8_t vsum = vreinterpretq_s16_u16(vaddq_u16(vsum016, vsum2345)); \ - vacc_lo = vaddw_s16(vacc_lo, vget_low_s16(vsum)); \ +#define CALCULATE_ACCUMULATE_VECTORS() \ + int32x4_t vacc_lo = finish_one_pass ? vld1q_s32(acc) : vbias; \ + int32x4_t vacc_hi = finish_one_pass ? vld1q_s32(acc + 4) : vbias; \ + int16x8_t vsum; \ + if constexpr (std::is_signed::value) { \ + const int16x8_t vsum01 = vaddl_s8(vreinterpret_s8_u8(vi0), vreinterpret_s8_u8(vi1)); \ + const int16x8_t vsum23 = vaddl_s8(vreinterpret_s8_u8(vi2), vreinterpret_s8_u8(vi3)); \ + const int16x8_t vsum45 = vaddl_s8(vreinterpret_s8_u8(vi4), vreinterpret_s8_u8(vi5)); \ + const int16x8_t vsum016 = vaddw_s8(vsum01, vreinterpret_s8_u8(vi6)); \ + const int16x8_t vsum2345 = vaddq_s16(vsum23, vsum45); \ + vsum = vaddq_s16(vsum016, vsum2345); \ + } else { \ + const uint16x8_t vsum01 = vaddl_u8(vi0, vi1); \ + const uint16x8_t vsum23 = vaddl_u8(vi2, vi3); \ + const uint16x8_t vsum45 = vaddl_u8(vi4, vi5); \ + const uint16x8_t vsum016 = vaddw_u8(vsum01, vi6); \ + const uint16x8_t vsum2345 = vaddq_u16(vsum23, vsum45); \ + vsum = vreinterpretq_s16_u16(vaddq_u16(vsum016, vsum2345)); \ + } \ + vacc_lo = vaddw_s16(vacc_lo, vget_low_s16(vsum)); \ vacc_hi = vaddw_s16(vacc_hi, vget_high_s16(vsum)) - uint8_t tail[8] = { 0, 0, 0, 0, 0, 0, 0, 0 }; + uint8_t tail[8] = {0, 0, 0, 0, 0, 0, 0, 0}; const int32x4_t vbias = vld1q_dup_s32(&Bias); bool finish_one_pass = false; const size_t step_next_group = 7 * Stride - (Channels & ~size_t{7}); - const uint8_t* i0 = Input; + const uint8_t* LastOf8U8 = (const uint8_t*)LastOf8; + const uint8_t* i0 = (const uint8_t*)Input; const uint8_t* i1 = i0 + Stride; const uint8_t* i4 = i0 + Stride * 4; const uint8_t* i2 = i1 + Stride; @@ -184,9 +227,11 @@ MlasQLinearGlobalAveragePoolNhwcSingleBatch( const uint8_t* i6 = i5 + Stride; for (; ImageSize > 7; ImageSize -= 7) { + int32_t* acc = AccumulateBuffer; size_t c = Channels; for (; c >= 8; c -= 8) { + LOAD_FULL_CHANNELS(); CALCULATE_ACCUMULATE_VECTORS(); @@ -196,13 +241,14 @@ MlasQLinearGlobalAveragePoolNhwcSingleBatch( acc += 8; } if (c > 0) { - const uint8x8_t vi0 = vld1_u8(((i0 >= LastOf8) ? (const uint8_t*)memcpy(tail, i0, c) : i0)); - const uint8x8_t vi1 = vld1_u8(((i1 >= LastOf8) ? (const uint8_t*)memcpy(tail, i1, c) : i1)); - const uint8x8_t vi2 = vld1_u8(((i2 >= LastOf8) ? (const uint8_t*)memcpy(tail, i2, c) : i2)); - const uint8x8_t vi3 = vld1_u8(((i3 >= LastOf8) ? (const uint8_t*)memcpy(tail, i3, c) : i3)); - const uint8x8_t vi4 = vld1_u8(((i4 >= LastOf8) ? (const uint8_t*)memcpy(tail, i4, c) : i4)); - const uint8x8_t vi5 = vld1_u8(((i5 >= LastOf8) ? (const uint8_t*)memcpy(tail, i5, c) : i5)); - const uint8x8_t vi6 = vld1_u8(((i6 >= LastOf8) ? (const uint8_t*)memcpy(tail, i6, c) : i6)); + + const uint8x8_t vi0 = vld1_u8(((i0 >= LastOf8U8) ? (const uint8_t*)memcpy(tail, i0, c) : i0)); + const uint8x8_t vi1 = vld1_u8(((i1 >= LastOf8U8) ? (const uint8_t*)memcpy(tail, i1, c) : i1)); + const uint8x8_t vi2 = vld1_u8(((i2 >= LastOf8U8) ? (const uint8_t*)memcpy(tail, i2, c) : i2)); + const uint8x8_t vi3 = vld1_u8(((i3 >= LastOf8U8) ? (const uint8_t*)memcpy(tail, i3, c) : i3)); + const uint8x8_t vi4 = vld1_u8(((i4 >= LastOf8U8) ? (const uint8_t*)memcpy(tail, i4, c) : i4)); + const uint8x8_t vi5 = vld1_u8(((i5 >= LastOf8U8) ? (const uint8_t*)memcpy(tail, i5, c) : i5)); + const uint8x8_t vi6 = vld1_u8(((i6 >= LastOf8U8) ? (const uint8_t*)memcpy(tail, i6, c) : i6)); CALCULATE_ACCUMULATE_VECTORS(); @@ -221,19 +267,28 @@ MlasQLinearGlobalAveragePoolNhwcSingleBatch( } if (ImageSize > 0) { + switch (ImageSize) { - case 1: i1 = ZeroBuffer; /* fall through */ - case 2: i2 = ZeroBuffer; /* fall through */ - case 3: i3 = ZeroBuffer; /* fall through */ - case 4: i4 = ZeroBuffer; /* fall through */ - case 5: i5 = ZeroBuffer; /* fall through */ - case 6: i6 = ZeroBuffer; /* fall through */ - default: break; + case 1: + i1 = (const uint8_t*)ZeroBuffer; /* fall through */ + case 2: + i2 = (const uint8_t*)ZeroBuffer; /* fall through */ + case 3: + i3 = (const uint8_t*)ZeroBuffer; /* fall through */ + case 4: + i4 = (const uint8_t*)ZeroBuffer; /* fall through */ + case 5: + i5 = (const uint8_t*)ZeroBuffer; /* fall through */ + case 6: + i6 = (const uint8_t*)ZeroBuffer; /* fall through */ + default: + break; } int32_t* acc = AccumulateBuffer; size_t c = Channels; for (; c >= 8; c -= 8) { + LOAD_FULL_CHANNELS(); CALCULATE_ACCUMULATE_VECTORS(); @@ -244,13 +299,21 @@ MlasQLinearGlobalAveragePoolNhwcSingleBatch( } if (c > 0) { - const uint8x8_t vi0 = vld1_u8(((i0 >= LastOf8) ? (const uint8_t*)memcpy(tail, i0, c) : i0)); - const uint8x8_t vi1 = vld1_u8(((1 < ImageSize && i1 >= LastOf8) ? (const uint8_t*)memcpy(tail, i1, c) : i1)); - const uint8x8_t vi2 = vld1_u8(((2 < ImageSize && i2 >= LastOf8) ? (const uint8_t*)memcpy(tail, i2, c) : i2)); - const uint8x8_t vi3 = vld1_u8(((3 < ImageSize && i3 >= LastOf8) ? (const uint8_t*)memcpy(tail, i3, c) : i3)); - const uint8x8_t vi4 = vld1_u8(((4 < ImageSize && i4 >= LastOf8) ? (const uint8_t*)memcpy(tail, i4, c) : i4)); - const uint8x8_t vi5 = vld1_u8(((5 < ImageSize && i5 >= LastOf8) ? (const uint8_t*)memcpy(tail, i5, c) : i5)); - const uint8x8_t vi6 = vld1_u8(((6 < ImageSize && i6 >= LastOf8) ? (const uint8_t*)memcpy(tail, i6, c) : i6)); + + const uint8x8_t vi0 = + vld1_u8(((i0 >= LastOf8U8) ? (const uint8_t*)memcpy(tail, i0, c) : i0)); + const uint8x8_t vi1 = vld1_u8( + ((1 < ImageSize && i1 >= LastOf8U8) ? (const uint8_t*)memcpy(tail, i1, c) : i1)); + const uint8x8_t vi2 = vld1_u8( + ((2 < ImageSize && i2 >= LastOf8U8) ? (const uint8_t*)memcpy(tail, i2, c) : i2)); + const uint8x8_t vi3 = vld1_u8( + ((3 < ImageSize && i3 >= LastOf8U8) ? (const uint8_t*)memcpy(tail, i3, c) : i3)); + const uint8x8_t vi4 = vld1_u8( + ((4 < ImageSize && i4 >= LastOf8U8) ? (const uint8_t*)memcpy(tail, i4, c) : i4)); + const uint8x8_t vi5 = vld1_u8( + ((5 < ImageSize && i5 >= LastOf8U8) ? (const uint8_t*)memcpy(tail, i5, c) : i5)); + const uint8x8_t vi6 = vld1_u8( + ((6 < ImageSize && i6 >= LastOf8U8) ? (const uint8_t*)memcpy(tail, i6, c) : i6)); CALCULATE_ACCUMULATE_VECTORS(); @@ -264,13 +327,13 @@ MlasQLinearGlobalAveragePoolNhwcSingleBatch( #elif defined(MLAS_SSE2_INTRINSICS) -void -MLASCALL +template +void MLASCALL MlasQLinearGlobalAveragePoolNchw( - const uint8_t* Input, + const T8Bits* Input, float ScaleInput, int32_t ZeroPointInput, - uint8_t* Output, + T8Bits* Output, float ScaleOutput, int32_t ZeroPointOutput, size_t Channels, @@ -286,36 +349,73 @@ MlasQLinearGlobalAveragePoolNchw( int32_t* sum_buffer = AccumulateBuffer; for (size_t c = Channels; c > 0; c--) { + __m128i vacc_lo = vbias; __m128i vacc_hi = vzero; auto Len = ImageSize; for (; Len >= 32; Len -= 32) { + const __m128i vi0 = _mm_loadl_epi64((const __m128i*)Input); const __m128i vi1 = _mm_loadl_epi64((const __m128i*)(Input + 8)); const __m128i vi2 = _mm_loadl_epi64((const __m128i*)(Input + 16)); const __m128i vi3 = _mm_loadl_epi64((const __m128i*)(Input + 24)); - const __m128i vxi0 = _mm_unpacklo_epi8(vi0, vzero); - const __m128i vxi1 = _mm_unpacklo_epi8(vi1, vzero); - const __m128i vxi2 = _mm_unpacklo_epi8(vi2, vzero); - const __m128i vxi3 = _mm_unpacklo_epi8(vi3, vzero); + if constexpr (std::is_signed::value) { + + const __m128i vxi0 = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, vi0), 8); + const __m128i vxi1 = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, vi1), 8); + const __m128i vxi2 = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, vi2), 8); + const __m128i vxi3 = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, vi3), 8); + const __m128i vsum = _mm_add_epi16(_mm_add_epi16(vxi0, vxi1), + _mm_add_epi16(vxi2, vxi3)); + vacc_lo = _mm_add_epi32(vacc_lo, _mm_srai_epi32(_mm_unpacklo_epi16(vzero, vsum), 16)); + vacc_hi = _mm_add_epi32(vacc_hi, _mm_srai_epi32(_mm_unpackhi_epi16(vzero, vsum), 16)); + } else { + + const __m128i vxi0 = _mm_unpacklo_epi8(vi0, vzero); + const __m128i vxi1 = _mm_unpacklo_epi8(vi1, vzero); + const __m128i vxi2 = _mm_unpacklo_epi8(vi2, vzero); + const __m128i vxi3 = _mm_unpacklo_epi8(vi3, vzero); + const __m128i vsum = _mm_add_epi16(_mm_add_epi16(vxi0, vxi1), + _mm_add_epi16(vxi2, vxi3)); + vacc_lo = _mm_add_epi32(vacc_lo, _mm_unpacklo_epi16(vsum, vzero)); + vacc_hi = _mm_add_epi32(vacc_hi, _mm_unpackhi_epi16(vsum, vzero)); + } - const __m128i vsum = _mm_add_epi16(_mm_add_epi16(vxi0, vxi1), _mm_add_epi16(vxi2, vxi3)); - vacc_lo = _mm_add_epi32(vacc_lo, _mm_unpacklo_epi16(vsum, vzero)); - vacc_hi = _mm_add_epi32(vacc_hi, _mm_unpackhi_epi16(vsum, vzero)); Input += 32; } for (; Len >= 8; Len -= 8) { - const __m128i vsum = _mm_unpacklo_epi8(_mm_loadl_epi64((const __m128i*)Input), vzero); - vacc_lo = _mm_add_epi32(vacc_lo, _mm_unpacklo_epi16(vsum, vzero)); - vacc_hi = _mm_add_epi32(vacc_hi, _mm_unpackhi_epi16(vsum, vzero)); + + if constexpr (std::is_signed::value) { + + const __m128i vsum = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, _mm_loadl_epi64((const __m128i*)Input)), 8); + vacc_lo = _mm_add_epi32(vacc_lo, _mm_srai_epi32(_mm_unpacklo_epi16(vzero, vsum), 16)); + vacc_hi = _mm_add_epi32(vacc_hi, _mm_srai_epi32(_mm_unpackhi_epi16(vzero, vsum), 16)); + } else { + + const __m128i vsum = _mm_unpacklo_epi8(_mm_loadl_epi64((const __m128i*)Input), vzero); + vacc_lo = _mm_add_epi32(vacc_lo, _mm_unpacklo_epi16(vsum, vzero)); + vacc_hi = _mm_add_epi32(vacc_hi, _mm_unpackhi_epi16(vsum, vzero)); + } + Input += 8; } if (Len > 0) { + memcpy(buffer, Input, Len); - const __m128i vsum = _mm_unpacklo_epi8(_mm_loadl_epi64((const __m128i*)buffer), vzero); - vacc_lo = _mm_add_epi32(vacc_lo, _mm_unpacklo_epi16(vsum, vzero)); - vacc_hi = _mm_add_epi32(vacc_hi, _mm_unpackhi_epi16(vsum, vzero)); + + if constexpr (std::is_signed::value) { + + const __m128i vsum = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, _mm_loadl_epi64((const __m128i*)buffer)), 8); + vacc_lo = _mm_add_epi32(vacc_lo, _mm_srai_epi32(_mm_unpacklo_epi16(vzero, vsum), 16)); + vacc_hi = _mm_add_epi32(vacc_hi, _mm_srai_epi32(_mm_unpackhi_epi16(vzero, vsum), 16)); + } else { + + const __m128i vsum = _mm_unpacklo_epi8(_mm_loadl_epi64((const __m128i*)buffer), vzero); + vacc_lo = _mm_add_epi32(vacc_lo, _mm_unpacklo_epi16(vsum, vzero)); + vacc_hi = _mm_add_epi32(vacc_hi, _mm_unpackhi_epi16(vsum, vzero)); + } + Input += Len; } @@ -326,24 +426,27 @@ MlasQLinearGlobalAveragePoolNchw( vsums = _mm_add_epi32(vsums, vshuf); *sum_buffer++ = _mm_cvtsi128_si32(vsums); } + MlasRequantizeOutput(AccumulateBuffer, Channels, Output, Channels, nullptr, &scale, false, - static_cast(ZeroPointOutput), 0, 0, 1, Channels); + static_cast(ZeroPointOutput), 0, 0, 1, Channels); } +template MLAS_FORCEINLINE void MlasQLinearGlobalAveragePoolNhwcSingleBatch( - const uint8_t* Input, - uint8_t* Output, - const uint8_t* LastOf8, + const T8Bits* Input, + T8Bits* Output, + const T8Bits* LastOf8, size_t ImageSize, size_t Channels, size_t Stride, int32_t Bias, float Scale, - uint8_t Output_zero_point, + T8Bits Output_zero_point, int32_t* AccumulateBuffer, - const uint8_t* ZeroBuffer) + const T8Bits* ZeroBuffer + ) { #if defined(MLAS_TARGET_IX86) @@ -359,23 +462,39 @@ MlasQLinearGlobalAveragePoolNhwcSingleBatch( const __m128i vi3 = _mm_loadl_epi64((const __m128i*)i3); \ i3 += 8; -#define CALCULATE_ACCUMULATE_VECTORS() \ - __m128i vacc_lo = finish_one_pass ? _mm_loadu_si128((__m128i*)acc) : vbias; \ - __m128i vacc_hi = finish_one_pass ? _mm_loadu_si128(((__m128i*)acc) + 1) : vbias; \ - const __m128i vxi0 = _mm_unpacklo_epi8(vi0, vzero); \ - const __m128i vxi1 = _mm_unpacklo_epi8(vi1, vzero); \ - const __m128i vxi2 = _mm_unpacklo_epi8(vi2, vzero); \ - const __m128i vxi3 = _mm_unpacklo_epi8(vi3, vzero); \ - const __m128i vsum01 = _mm_add_epi16(vxi0, vxi1); \ - const __m128i vsum23 = _mm_add_epi16(vxi2, vxi3); \ - const __m128i vsum = _mm_add_epi16(vsum01, vsum23); \ - vacc_lo = _mm_add_epi32(vacc_lo, _mm_unpacklo_epi16(vsum, vzero)); \ - vacc_hi = _mm_add_epi32(vacc_hi, _mm_unpackhi_epi16(vsum, vzero)) +#define CALCULATE_ACCUMULATE_VECTORS() \ + __m128i vacc_lo = finish_one_pass ? _mm_loadu_si128((__m128i*)acc) : vbias; \ + __m128i vacc_hi = finish_one_pass ? _mm_loadu_si128(((__m128i*)acc) + 1) : vbias; \ + __m128i vxi0; \ + __m128i vxi1; \ + __m128i vxi2; \ + __m128i vxi3; \ + if constexpr (std::is_signed::value) { \ + vxi0 = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, vi0), 8); \ + vxi1 = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, vi1), 8); \ + vxi2 = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, vi2), 8); \ + vxi3 = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, vi3), 8); \ + } else { \ + vxi0 = _mm_unpacklo_epi8(vi0, vzero); \ + vxi1 = _mm_unpacklo_epi8(vi1, vzero); \ + vxi2 = _mm_unpacklo_epi8(vi2, vzero); \ + vxi3 = _mm_unpacklo_epi8(vi3, vzero); \ + } \ + __m128i vsum01 = _mm_add_epi16(vxi0, vxi1); \ + __m128i vsum23 = _mm_add_epi16(vxi2, vxi3); \ + __m128i vsum = _mm_add_epi16(vsum01, vsum23); \ + \ + if constexpr (std::is_signed::value) { \ + vacc_lo = _mm_add_epi32(vacc_lo, _mm_srai_epi32(_mm_unpacklo_epi16(vzero, vsum), 16)); \ + vacc_hi = _mm_add_epi32(vacc_hi, _mm_srai_epi32(_mm_unpackhi_epi16(vzero, vsum), 16)); \ + } else { \ + vacc_lo = _mm_add_epi32(vacc_lo, _mm_unpacklo_epi16(vsum, vzero)); \ + vacc_hi = _mm_add_epi32(vacc_hi, _mm_unpackhi_epi16(vsum, vzero)); \ + } #else constexpr size_t PixelsPerIteration = 7; - #define LOAD_FULL_CHANNELS() \ const __m128i vi0 = _mm_loadl_epi64((const __m128i*)i0); \ i0 += 8; \ @@ -392,47 +511,71 @@ MlasQLinearGlobalAveragePoolNhwcSingleBatch( const __m128i vi6 = _mm_loadl_epi64((const __m128i*)i6); \ i6 += 8 -#define CALCULATE_ACCUMULATE_VECTORS() \ - __m128i vacc_lo = finish_one_pass ? _mm_loadu_si128((__m128i*)acc) : vbias; \ - __m128i vacc_hi = finish_one_pass ? _mm_loadu_si128(((__m128i*)acc) + 1) : vbias; \ - const __m128i vxi0 = _mm_unpacklo_epi8(vi0, vzero); \ - const __m128i vxi1 = _mm_unpacklo_epi8(vi1, vzero); \ - const __m128i vxi2 = _mm_unpacklo_epi8(vi2, vzero); \ - const __m128i vxi3 = _mm_unpacklo_epi8(vi3, vzero); \ - const __m128i vxi4 = _mm_unpacklo_epi8(vi4, vzero); \ - const __m128i vxi5 = _mm_unpacklo_epi8(vi5, vzero); \ - const __m128i vxi6 = _mm_unpacklo_epi8(vi6, vzero); \ - const __m128i vsum01 = _mm_add_epi16(vxi0, vxi1); \ - const __m128i vsum23 = _mm_add_epi16(vxi2, vxi3); \ - const __m128i vsum45 = _mm_add_epi16(vxi4, vxi5); \ - const __m128i vsum016 = _mm_add_epi16(vsum01, vxi6); \ - const __m128i vsum2345 = _mm_add_epi16(vsum23, vsum45); \ - const __m128i vsum = _mm_add_epi16(vsum016, vsum2345); \ - vacc_lo = _mm_add_epi32(vacc_lo, _mm_unpacklo_epi16(vsum, vzero)); \ - vacc_hi = _mm_add_epi32(vacc_hi, _mm_unpackhi_epi16(vsum, vzero)) +#define CALCULATE_ACCUMULATE_VECTORS() \ + __m128i vacc_lo = finish_one_pass ? _mm_loadu_si128((__m128i*)acc) : vbias; \ + __m128i vacc_hi = finish_one_pass ? _mm_loadu_si128(((__m128i*)acc) + 1) : vbias; \ + __m128i vxi0; \ + __m128i vxi1; \ + __m128i vxi2; \ + __m128i vxi3; \ + __m128i vxi4; \ + __m128i vxi5; \ + __m128i vxi6; \ + if constexpr (std::is_signed::value) { \ + vxi0 = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, vi0), 8); \ + vxi1 = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, vi1), 8); \ + vxi2 = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, vi2), 8); \ + vxi3 = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, vi3), 8); \ + vxi4 = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, vi4), 8); \ + vxi5 = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, vi5), 8); \ + vxi6 = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, vi6), 8); \ + } else { \ + vxi0 = _mm_unpacklo_epi8(vi0, vzero); \ + vxi1 = _mm_unpacklo_epi8(vi1, vzero); \ + vxi2 = _mm_unpacklo_epi8(vi2, vzero); \ + vxi3 = _mm_unpacklo_epi8(vi3, vzero); \ + vxi4 = _mm_unpacklo_epi8(vi4, vzero); \ + vxi5 = _mm_unpacklo_epi8(vi5, vzero); \ + vxi6 = _mm_unpacklo_epi8(vi6, vzero); \ + } \ + const __m128i vsum01 = _mm_add_epi16(vxi0, vxi1); \ + const __m128i vsum23 = _mm_add_epi16(vxi2, vxi3); \ + const __m128i vsum45 = _mm_add_epi16(vxi4, vxi5); \ + const __m128i vsum016 = _mm_add_epi16(vsum01, vxi6); \ + const __m128i vsum2345 = _mm_add_epi16(vsum23, vsum45); \ + const __m128i vsum = _mm_add_epi16(vsum016, vsum2345); \ + if constexpr (std::is_signed::value) { \ + vacc_lo = _mm_add_epi32(vacc_lo, _mm_srai_epi32(_mm_unpacklo_epi16(vzero, vsum), 16)); \ + vacc_hi = _mm_add_epi32(vacc_hi, _mm_srai_epi32(_mm_unpackhi_epi16(vzero, vsum), 16)); \ + } else { \ + vacc_lo = _mm_add_epi32(vacc_lo, _mm_unpacklo_epi16(vsum, vzero)); \ + vacc_hi = _mm_add_epi32(vacc_hi, _mm_unpackhi_epi16(vsum, vzero)); \ + } #endif - uint8_t tail[8] = { 0, 0, 0, 0, 0, 0, 0, 0 }; + T8Bits tail[8] = {0, 0, 0, 0, 0, 0, 0, 0}; bool finish_one_pass = false; const __m128i vbias = _mm_set1_epi32(Bias); const __m128i vzero = _mm_setzero_si128(); size_t step_next_group = PixelsPerIteration * Stride - (Channels & ~size_t{7}); - const uint8_t* i0 = Input; - const uint8_t* i1 = i0 + Stride; - const uint8_t* i2 = i1 + Stride; - const uint8_t* i3 = i2 + Stride; + const T8Bits* i0 = Input; + const T8Bits* i1 = i0 + Stride; + const T8Bits* i2 = i1 + Stride; + const T8Bits* i3 = i2 + Stride; #if !defined(MLAS_TARGET_IX86) - const uint8_t* i4 = i0 + Stride * 4; - const uint8_t* i5 = i4 + Stride; - const uint8_t* i6 = i5 + Stride; + const T8Bits* i4 = i0 + Stride * 4; + const T8Bits* i5 = i4 + Stride; + const T8Bits* i6 = i5 + Stride; #endif for (; ImageSize > PixelsPerIteration; ImageSize -= PixelsPerIteration) { + int32_t* acc = AccumulateBuffer; size_t c = Channels; for (; c >= 8; c -= 8) { + LOAD_FULL_CHANNELS(); CALCULATE_ACCUMULATE_VECTORS(); @@ -442,14 +585,21 @@ MlasQLinearGlobalAveragePoolNhwcSingleBatch( acc += 8; } if (c > 0) { - const __m128i vi0 = _mm_loadl_epi64((const __m128i*)(i0 >= LastOf8 ? memcpy(tail, i0, c) : i0)); - const __m128i vi1 = _mm_loadl_epi64((const __m128i*)(i1 >= LastOf8 ? memcpy(tail, i1, c) : i1)); - const __m128i vi2 = _mm_loadl_epi64((const __m128i*)(i2 >= LastOf8 ? memcpy(tail, i2, c) : i2)); - const __m128i vi3 = _mm_loadl_epi64((const __m128i*)(i3 >= LastOf8 ? memcpy(tail, i3, c) : i3)); + const __m128i vi0 = + _mm_loadl_epi64((const __m128i*)(i0 >= LastOf8 ? memcpy(tail, i0, c) : i0)); + const __m128i vi1 = + _mm_loadl_epi64((const __m128i*)(i1 >= LastOf8 ? memcpy(tail, i1, c) : i1)); + const __m128i vi2 = + _mm_loadl_epi64((const __m128i*)(i2 >= LastOf8 ? memcpy(tail, i2, c) : i2)); + const __m128i vi3 = + _mm_loadl_epi64((const __m128i*)(i3 >= LastOf8 ? memcpy(tail, i3, c) : i3)); #if !defined(MLAS_TARGET_IX86) - const __m128i vi4 = _mm_loadl_epi64((const __m128i*)(i4 >= LastOf8 ? memcpy(tail, i4, c) : i4)); - const __m128i vi5 = _mm_loadl_epi64((const __m128i*)(i5 >= LastOf8 ? memcpy(tail, i5, c) : i5)); - const __m128i vi6 = _mm_loadl_epi64((const __m128i*)(i6 >= LastOf8 ? memcpy(tail, i6, c) : i6)); + const __m128i vi4 = + _mm_loadl_epi64((const __m128i*)(i4 >= LastOf8 ? memcpy(tail, i4, c) : i4)); + const __m128i vi5 = + _mm_loadl_epi64((const __m128i*)(i5 >= LastOf8 ? memcpy(tail, i5, c) : i5)); + const __m128i vi6 = + _mm_loadl_epi64((const __m128i*)(i6 >= LastOf8 ? memcpy(tail, i6, c) : i6)); #endif CALCULATE_ACCUMULATE_VECTORS(); @@ -473,26 +623,38 @@ MlasQLinearGlobalAveragePoolNhwcSingleBatch( if (ImageSize > 0) { #if defined(MLAS_TARGET_IX86) switch (ImageSize) { - case 1: i1 = ZeroBuffer; /* fall through */ - case 2: i2 = ZeroBuffer; /* fall through */ - case 3: i3 = ZeroBuffer; /* fall through */ - default: break; + case 1: + i1 = ZeroBuffer; /* fall through */ + case 2: + i2 = ZeroBuffer; /* fall through */ + case 3: + i3 = ZeroBuffer; /* fall through */ + default: + break; } #else switch (ImageSize) { - case 1: i1 = ZeroBuffer; /* fall through */ - case 2: i2 = ZeroBuffer; /* fall through */ - case 3: i3 = ZeroBuffer; /* fall through */ - case 4: i4 = ZeroBuffer; /* fall through */ - case 5: i5 = ZeroBuffer; /* fall through */ - case 6: i6 = ZeroBuffer; /* fall through */ - default: break; + case 1: + i1 = ZeroBuffer; /* fall through */ + case 2: + i2 = ZeroBuffer; /* fall through */ + case 3: + i3 = ZeroBuffer; /* fall through */ + case 4: + i4 = ZeroBuffer; /* fall through */ + case 5: + i5 = ZeroBuffer; /* fall through */ + case 6: + i6 = ZeroBuffer; /* fall through */ + default: + break; } #endif int32_t* acc = AccumulateBuffer; size_t c = Channels; for (; c >= 8; c -= 8) { + LOAD_FULL_CHANNELS(); CALCULATE_ACCUMULATE_VECTORS(); @@ -503,14 +665,21 @@ MlasQLinearGlobalAveragePoolNhwcSingleBatch( } if (c > 0) { - const __m128i vi0 = _mm_loadl_epi64((const __m128i*)(i0 >= LastOf8 ? memcpy(tail, i0, c) : i0)); - const __m128i vi1 = _mm_loadl_epi64((const __m128i*)(1 < ImageSize && i1 >= LastOf8 ? memcpy(tail, i1, c) : i1)); - const __m128i vi2 = _mm_loadl_epi64((const __m128i*)(2 < ImageSize && i2 >= LastOf8 ? memcpy(tail, i2, c) : i2)); - const __m128i vi3 = _mm_loadl_epi64((const __m128i*)(3 < ImageSize && i3 >= LastOf8 ? memcpy(tail, i3, c) : i3)); + const __m128i vi0 = + _mm_loadl_epi64((const __m128i*)(i0 >= LastOf8 ? memcpy(tail, i0, c) : i0)); + const __m128i vi1 = _mm_loadl_epi64( + (const __m128i*)(1 < ImageSize && i1 >= LastOf8 ? memcpy(tail, i1, c) : i1)); + const __m128i vi2 = _mm_loadl_epi64( + (const __m128i*)(2 < ImageSize && i2 >= LastOf8 ? memcpy(tail, i2, c) : i2)); + const __m128i vi3 = _mm_loadl_epi64( + (const __m128i*)(3 < ImageSize && i3 >= LastOf8 ? memcpy(tail, i3, c) : i3)); #if !defined(MLAS_TARGET_IX86) - const __m128i vi4 = _mm_loadl_epi64((const __m128i*)(4 < ImageSize && i4 >= LastOf8 ? memcpy(tail, i4, c) : i4)); - const __m128i vi5 = _mm_loadl_epi64((const __m128i*)(5 < ImageSize && i5 >= LastOf8 ? memcpy(tail, i5, c) : i5)); - const __m128i vi6 = _mm_loadl_epi64((const __m128i*)(6 < ImageSize && i6 >= LastOf8 ? memcpy(tail, i6, c) : i6)); + const __m128i vi4 = _mm_loadl_epi64( + (const __m128i*)(4 < ImageSize && i4 >= LastOf8 ? memcpy(tail, i4, c) : i4)); + const __m128i vi5 = _mm_loadl_epi64( + (const __m128i*)(5 < ImageSize && i5 >= LastOf8 ? memcpy(tail, i5, c) : i5)); + const __m128i vi6 = _mm_loadl_epi64( + (const __m128i*)(6 < ImageSize && i6 >= LastOf8 ? memcpy(tail, i6, c) : i6)); #endif CALCULATE_ACCUMULATE_VECTORS(); @@ -527,13 +696,14 @@ MlasQLinearGlobalAveragePoolNhwcSingleBatch( // Pure C++ Implementation +template void MLASCALL MlasQLinearGlobalAveragePoolNchw( - const uint8_t* Input, + const T8Bits* Input, float ScaleInput, int32_t ZeroPointInput, - uint8_t* Output, + T8Bits* Output, float ScaleOutput, int32_t ZeroPointOutput, size_t Channels, @@ -544,22 +714,26 @@ MlasQLinearGlobalAveragePoolNchw( float scale = CheckQLinearGlobalAveragePoolScaleAndSize(ScaleInput, ScaleOutput, ImageSize); int32_t bias = -ZeroPointInput * static_cast(ImageSize); for (; Channels > 0; Channels--) { + int32_t acc = bias; for (size_t i = 0; i < ImageSize; ++i) { - acc += static_cast(*Input++); + acc += static_cast(*Input++); } - int32_t v = static_cast(std::nearbyintf(acc * scale)) + ZeroPointOutput; - *Output++ = std::max(std::min(255, v), 0); + int32_t v = static_cast(std::nearbyintf(acc * scale)) + ZeroPointOutput; + v = std::min(static_cast(std::numeric_limits::max()), v); + v = std::max(static_cast(std::numeric_limits::lowest()), v); + *Output++ = static_cast(v); } } +template void MLASCALL MlasQLinearGlobalAveragePoolNhwc( - const uint8_t* Input, + const T8Bits* Input, float ScaleInput, int32_t ZeroPointInput, - uint8_t* Output, + T8Bits* Output, float ScaleOutput, int32_t ZeroPointOutput, size_t Batch, @@ -567,26 +741,33 @@ MlasQLinearGlobalAveragePoolNhwc( size_t Stride, size_t Channels, int32_t* AccumulateBuffer, - const uint8_t* /* ZeroBuffer */ + const T8Bits* ZeroBuffer ) { float scale = CheckQLinearGlobalAveragePoolScaleAndSize(ScaleInput, ScaleOutput, ImageSize); int32_t bias = -ZeroPointInput * static_cast(ImageSize); for (; Batch > 0; Batch--) { - const uint8_t* batch_input = Input; - uint8_t* batch_output = Output; + + const T8Bits* batch_input = Input; + T8Bits* batch_output = Output; Input += Stride * ImageSize; Output += Stride; std::fill_n(AccumulateBuffer, Channels, bias); for (size_t i = 0; i < ImageSize; ++i) { + for (size_t c = 0; c < Channels; ++c) { AccumulateBuffer[c] += static_cast(batch_input[c]); } + batch_input += Stride; } + for (size_t c = 0; c < Channels; ++c) { - int32_t v = static_cast(std::nearbyintf(AccumulateBuffer[c] * scale)) + ZeroPointOutput; - *batch_output++ = std::max(std::min(255, v), 0); + + int32_t v = static_cast(std::nearbyintf(AccumulateBuffer[c] * scale)) + ZeroPointOutput; + v = std::min(static_cast(std::numeric_limits::max()), v); + v = std::max(static_cast(std::numeric_limits::lowest()), v); + *batch_output++ = static_cast(v); } } } @@ -595,9 +776,91 @@ MlasQLinearGlobalAveragePoolNhwc( #if defined(MLAS_NEON_INTRINSICS) || defined(MLAS_SSE2_INTRINSICS) +template void MLASCALL MlasQLinearGlobalAveragePoolNhwc( + const T8Bits* Input, + float ScaleInput, + int32_t ZeroPointInput, + T8Bits* Output, + float ScaleOutput, + int32_t ZeroPointOutput, + size_t Batch, + size_t ImageSize, + size_t Stride, + size_t Channels, + int32_t* AccumulateBuffer, + const T8Bits* ZeroBuffer + ) +{ + float scale = CheckQLinearGlobalAveragePoolScaleAndSize(ScaleInput, ScaleOutput, ImageSize); + const int32_t bias = -ZeroPointInput * static_cast(ImageSize); + const T8Bits* inputLastOf8 = Input + (Batch * ImageSize * Stride - Stride + Channels) - 8; + + for (; Batch > 0; Batch--) { + MlasQLinearGlobalAveragePoolNhwcSingleBatch( + Input, Output, inputLastOf8, ImageSize, Channels, Stride, bias, scale, + static_cast(ZeroPointOutput), AccumulateBuffer, ZeroBuffer); + Input += ImageSize * Stride; + Output += Stride; + } +} + +#endif + +template +void +MLASCALL +MlasQLinearGlobalAveragePoolNchw( + const int8_t* Input, + float ScaleInput, + int32_t ZeroPointInput, + int8_t* Output, + float ScaleOutput, + int32_t ZeroPointOutput, + size_t Channels, + size_t ImageSize, + int32_t* AccumulateBuffer + ); + +template +void +MLASCALL +MlasQLinearGlobalAveragePoolNchw( + const uint8_t* Input, + float ScaleInput, + int32_t ZeroPointInput, + uint8_t* Output, + float ScaleOutput, + int32_t ZeroPointOutput, + size_t Channels, + size_t ImageSize, + int32_t* AccumulateBuffer + ); + +template +void +MLASCALL +MlasQLinearGlobalAveragePoolNhwc( + const int8_t* Input, + float ScaleInput, + int32_t ZeroPointInput, + int8_t* Output, + float ScaleOutput, + int32_t ZeroPointOutput, + size_t Batch, + size_t ImageSize, + size_t Stride, + size_t Channels, + int32_t* AccumulateBuffer, + const int8_t* ZeroBuffer + ); + +template +void +MLASCALL +MlasQLinearGlobalAveragePoolNhwc( const uint8_t* Input, float ScaleInput, int32_t ZeroPointInput, @@ -610,20 +873,4 @@ MlasQLinearGlobalAveragePoolNhwc( size_t Channels, int32_t* AccumulateBuffer, const uint8_t* ZeroBuffer - ) -{ - float scale = CheckQLinearGlobalAveragePoolScaleAndSize(ScaleInput, ScaleOutput, ImageSize); - const int32_t bias = -ZeroPointInput * static_cast(ImageSize); - const uint8_t* inputLastOf8 = Input + (Batch * ImageSize * Stride - Stride + Channels) - 8; - - for (; Batch > 0; Batch--) { - MlasQLinearGlobalAveragePoolNhwcSingleBatch( - Input, Output, inputLastOf8, ImageSize, Channels, Stride, - bias, scale, static_cast(ZeroPointOutput), - AccumulateBuffer, ZeroBuffer); - Input += ImageSize * Stride; - Output += Stride; - } -} - -#endif + ); diff --git a/onnxruntime/core/mlas/lib/quantize.cpp b/onnxruntime/core/mlas/lib/quantize.cpp index 01a5529fb6..632800e8fd 100644 --- a/onnxruntime/core/mlas/lib/quantize.cpp +++ b/onnxruntime/core/mlas/lib/quantize.cpp @@ -165,7 +165,7 @@ Return Value: --*/ { - constexpr int32_t MinimumValue = std::numeric_limits::min(); + constexpr int32_t MinimumValue = std::numeric_limits::lowest(); constexpr int32_t MaximumValue = std::numeric_limits::max(); auto ScaleVector = MlasBroadcastFloat32x4(Scale); @@ -315,7 +315,7 @@ Return Value: --*/ { - constexpr int32_t MinimumValue = std::numeric_limits::min(); + constexpr int32_t MinimumValue = std::numeric_limits::lowest(); constexpr int32_t MaximumValue = std::numeric_limits::max(); for (size_t n = 0; n < N; n++) { @@ -352,17 +352,18 @@ MlasQuantizeLinear( #if defined(MLAS_SSE2_INTRINSICS) +template void MLASCALL MlasRequantizeOutput( const int32_t* Input, size_t InputLeadingDimension, - uint8_t* Output, + OutputType* Output, size_t OutputLeadingDimension, const int32_t* Bias, const float* Scale, bool PerColumnScale, - uint8_t ZeroPoint, + OutputType ZeroPoint, size_t StartM, size_t StartN, size_t CountM, @@ -370,8 +371,8 @@ MlasRequantizeOutput( ) { const __m128 PerMatrixScaleVector = PerColumnScale ? _mm_setzero_ps() : _mm_load1_ps(Scale); - const __m128 MinimumValueVector = _mm_set1_ps(float(0 - ZeroPoint)); - const __m128 MaximumValueVector = _mm_set1_ps(float(255 - ZeroPoint)); + const __m128 MinimumValueVector = _mm_set1_ps(float(std::numeric_limits::lowest() - ZeroPoint)); + const __m128 MaximumValueVector = _mm_set1_ps(float(std::numeric_limits::max() - ZeroPoint)); const __m128i ZeroPointVector = _mm_set1_epi32(ZeroPoint); if (nullptr != Bias) { @@ -467,10 +468,23 @@ MlasRequantizeOutput( IntegerVector2 = _mm_add_epi32(IntegerVector2, ZeroPointVector); IntegerVector3 = _mm_add_epi32(IntegerVector3, ZeroPointVector); - __m128i WordVector0 = _mm_packus_epi16(IntegerVector0, IntegerVector1); - __m128i WordVector1 = _mm_packus_epi16(IntegerVector2, IntegerVector3); + __m128i WordVector0; + __m128i WordVector1; + __m128i ByteVector; - __m128i ByteVector = _mm_packus_epi16(WordVector0, WordVector1); + if (std::is_signed::value) { + + WordVector0 = _mm_packs_epi32(IntegerVector0, IntegerVector1); + WordVector1 = _mm_packs_epi32(IntegerVector2, IntegerVector3); + ByteVector = _mm_packs_epi16(WordVector0, WordVector1); + + } else { + + WordVector0 = _mm_packus_epi16(IntegerVector0, IntegerVector1); + WordVector1 = _mm_packus_epi16(IntegerVector2, IntegerVector3); + ByteVector = _mm_packus_epi16(WordVector0, WordVector1); + + } _mm_storeu_si128((__m128i*)RowOutput, ByteVector); RowOutput += 16; @@ -541,8 +555,17 @@ MlasRequantizeOutput( IntegerVector = _mm_cvtps_epi32(FloatVector); IntegerVector = _mm_add_epi32(IntegerVector, ZeroPointVector); - IntegerVector = _mm_packus_epi16(IntegerVector, IntegerVector); - IntegerVector = _mm_packus_epi16(IntegerVector, IntegerVector); + if (std::is_signed::value) { + + IntegerVector = _mm_packs_epi32(IntegerVector, IntegerVector); + IntegerVector = _mm_packs_epi16(IntegerVector, IntegerVector); + + } else { + + IntegerVector = _mm_packus_epi16(IntegerVector, IntegerVector); + IntegerVector = _mm_packus_epi16(IntegerVector, IntegerVector); + + } uint32_t OutputValue = uint32_t(_mm_cvtsi128_si32(IntegerVector)); @@ -570,17 +593,18 @@ MlasRequantizeOutput( #elif defined(MLAS_NEON64_INTRINSICS) +template void MLASCALL MlasRequantizeOutput( const int32_t* Input, size_t InputLeadingDimension, - uint8_t* Output, + OutputType* Output, size_t OutputLeadingDimension, const int32_t* Bias, const float* Scale, bool PerColumnScale, - uint8_t ZeroPoint, + OutputType ZeroPoint, size_t StartM, size_t StartN, size_t CountM, @@ -686,7 +710,7 @@ MlasRequantizeOutput( // // Pack the integers with saturation to 16-bit values and shift by - // the zero point, then pack the integers again to unsigned bytes. + // the zero point, then pack the integers again to bytes. // int16x8x2_t WordVector; @@ -697,7 +721,13 @@ MlasRequantizeOutput( WordVector.val[0] = vqaddq_s16(WordVector.val[0], ZeroPointVector); WordVector.val[1] = vqaddq_s16(WordVector.val[1], ZeroPointVector); - vst1q_u8(RowOutput, vqmovun_high_s16(vqmovun_s16(WordVector.val[0]), WordVector.val[1])); + if (std::is_signed::value) { + vst1q_s8(reinterpret_cast(RowOutput), + vqmovn_high_s16(vqmovn_s16(WordVector.val[0]), WordVector.val[1])); + } else { + vst1q_u8(reinterpret_cast(RowOutput), + vqmovun_high_s16(vqmovun_s16(WordVector.val[0]), WordVector.val[1])); + } RowOutput += 16; n -= 16; @@ -775,7 +805,13 @@ MlasRequantizeOutput( int16x8_t WordVector = vcombine_s16(vqmovn_s32(IntegerVector), vdup_n_s16(0)); WordVector = vqaddq_s16(WordVector, ZeroPointVector); - uint8x16_t ByteVector = vcombine_u8(vqmovun_s16(WordVector), vdup_n_u8(0)); + uint8x16_t ByteVector; + + if (std::is_signed::value) { + ByteVector = vcombine_u8(vreinterpret_u8_s8(vqmovn_s16(WordVector)), vdup_n_u8(0)); + } else { + ByteVector = vcombine_u8(vqmovun_s16(WordVector), vdup_n_u8(0)); + } if (n >= 4) { @@ -787,7 +823,7 @@ MlasRequantizeOutput( } else { - vst1q_lane_u8(RowOutput, ByteVector, 0); + vst1q_lane_u8(reinterpret_cast(RowOutput), ByteVector, 0); RowOutput += 1; n -= 1; @@ -802,17 +838,18 @@ MlasRequantizeOutput( #else +template void MLASCALL MlasRequantizeOutput( const int32_t* Input, size_t InputLeadingDimension, - uint8_t* Output, + OutputType* Output, size_t OutputLeadingDimension, const int32_t* Bias, const float* Scale, bool PerColumnScale, - uint8_t ZeroPoint, + OutputType ZeroPoint, size_t StartM, size_t StartN, size_t CountM, @@ -820,8 +857,8 @@ MlasRequantizeOutput( ) { const float PerMatrixScaleValue = PerColumnScale ? 0.0f : *Scale; - const float MinimumValue = float(0 - ZeroPoint); - const float MaximumValue = float(255 - ZeroPoint); + const float MinimumValue = float(std::numeric_limits::lowest() - ZeroPoint); + const float MaximumValue = float(std::numeric_limits::max() - ZeroPoint); if (nullptr != Bias) { Bias += StartN; @@ -872,7 +909,7 @@ MlasRequantizeOutput( IntegerValue = int32_t(MlasBitsOfFp32(FloatValue + MLAS_ROUNDING_BIAS_MAGIC)) - MLAS_ROUNDING_BIAS_MAGIC_BITS; - *RowOutput++ = uint8_t(IntegerValue + ZeroPoint); + *RowOutput++ = OutputType(IntegerValue + ZeroPoint); n -= 1; } @@ -885,6 +922,42 @@ MlasRequantizeOutput( #endif +template +void +MLASCALL +MlasRequantizeOutput( + const int32_t* Input, + size_t InputLeadingDimension, + int8_t* Output, + size_t OutputLeadingDimension, + const int32_t* Bias, + const float* Scale, + bool PerColumnScale, + int8_t ZeroPoint, + size_t StartM, + size_t StartN, + size_t CountM, + size_t CountN + ); + +template +void +MLASCALL +MlasRequantizeOutput( + const int32_t* Input, + size_t InputLeadingDimension, + uint8_t* Output, + size_t OutputLeadingDimension, + const int32_t* Bias, + const float* Scale, + bool PerColumnScale, + uint8_t ZeroPoint, + size_t StartM, + size_t StartN, + size_t CountM, + size_t CountN + ); + void MLASCALL MlasFindMinMaxElement( diff --git a/onnxruntime/core/mlas/lib/transpose.cpp b/onnxruntime/core/mlas/lib/transpose.cpp index 3f12773050..37181ec2f3 100644 --- a/onnxruntime/core/mlas/lib/transpose.cpp +++ b/onnxruntime/core/mlas/lib/transpose.cpp @@ -436,3 +436,18 @@ Return Value: n -= 1; } } + +void +MLASCALL +MlasTranspose( + const int8_t* Input, + int8_t* Output, + size_t M, + size_t N) +{ + MlasTranspose( + reinterpret_cast(Input), + reinterpret_cast(Output), + M, + N); +} \ No newline at end of file diff --git a/onnxruntime/core/util/math_cpu.cc b/onnxruntime/core/util/math_cpu.cc index eb4515f1f7..c09d885a23 100644 --- a/onnxruntime/core/util/math_cpu.cc +++ b/onnxruntime/core/util/math_cpu.cc @@ -650,6 +650,7 @@ void Im2col::operator()( } } +template struct Im2col; template struct Im2col; template <> diff --git a/onnxruntime/test/contrib_ops/nhwc_maxpool_op_test.cc b/onnxruntime/test/contrib_ops/nhwc_maxpool_op_test.cc index 8c52de8373..0a74a68a80 100644 --- a/onnxruntime/test/contrib_ops/nhwc_maxpool_op_test.cc +++ b/onnxruntime/test/contrib_ops/nhwc_maxpool_op_test.cc @@ -2,10 +2,12 @@ // Licensed under the MIT License. #include +#include + #include "core/util/math.h" +#include "core/mlas/inc/mlas.h" #include "gtest/gtest.h" #include "test/providers/provider_test_utils.h" -#include namespace onnxruntime { namespace test { @@ -70,7 +72,9 @@ class NhwcMaxPoolOpTester { Y_shape.push_back(batch_count); for (size_t n = 0; n < kernel_rank; n++) { Y_shape.push_back(((input_shape[n] + pads[n] + pads[kernel_rank + n]) - - (dilations[n] * (kernel_shape_[n] - 1) + 1)) / strides[n] + 1); + (dilations[n] * (kernel_shape_[n] - 1) + 1)) / + strides[n] + + 1); } Y_shape.push_back(channels); Y_data.resize(ShapeSize(Y_shape)); @@ -87,7 +91,7 @@ class NhwcMaxPoolOpTester { std::vector d_output(kernel_rank, 0); std::vector d_kernel(kernel_rank, 0); do { - std::fill_n(Ydata, channels, static_cast(0)); + std::fill_n(Ydata, channels, std::numeric_limits::lowest()); do { int64_t input_offset = 0; bool is_padding = false; @@ -163,7 +167,7 @@ class NhwcMaxPoolOpTester { }; TEST(NhwcMaxPoolContribOpTest, MaxPool1D) { - for (int64_t channels = 1; channels < 64; channels++) { + for (int64_t channels = 1; channels < 94; channels++) { NhwcMaxPoolOpTester test; test.GenerateRandomInput({1, 23, channels}); test.SetKernelShape({5}); @@ -173,7 +177,7 @@ TEST(NhwcMaxPoolContribOpTest, MaxPool1D) { } TEST(NhwcMaxPoolContribOpTest, MaxPool2D) { - for (int64_t channels = 1; channels < 64; channels++) { + for (int64_t channels = 1; channels < 94; channels++) { NhwcMaxPoolOpTester test; test.GenerateRandomInput({1, 15, 19, channels}); test.SetKernelShape({3, 5}); @@ -183,7 +187,7 @@ TEST(NhwcMaxPoolContribOpTest, MaxPool2D) { } TEST(NhwcMaxPoolContribOpTest, MaxPool3D) { - for (int64_t channels = 1; channels < 64; channels++) { + for (int64_t channels = 1; channels < 94; channels++) { NhwcMaxPoolOpTester test; test.GenerateRandomInput({1, 9, 13, 15, channels}); test.SetKernelShape({2, 4, 6}); @@ -208,5 +212,51 @@ TEST(NhwcMaxPoolContribOpTest, MaxPoolDilations) { test.Run(); } +TEST(NhwcMaxPoolContribOpTest, MaxPool1D_S8) { + for (int64_t channels = 1; channels < 94; channels++) { + NhwcMaxPoolOpTester test; + test.GenerateRandomInput({1, 23, channels}); + test.SetKernelShape({5}); + test.SetPads({2, 2}); + test.Run(); + } +} + +TEST(NhwcMaxPoolContribOpTest, MaxPool2D_S8) { + for (int64_t channels = 1; channels < 94; channels++) { + NhwcMaxPoolOpTester test; + test.GenerateRandomInput({1, 15, 19, channels}); + test.SetKernelShape({3, 5}); + test.SetPads({1, 1, 1, 1}); + test.Run(); + } +} + +TEST(NhwcMaxPoolContribOpTest, MaxPool3D_S8) { + for (int64_t channels = 1; channels < 94; channels++) { + NhwcMaxPoolOpTester test; + test.GenerateRandomInput({1, 9, 13, 15, channels}); + test.SetKernelShape({2, 4, 6}); + test.SetPads({0, 0, 0, 1, 1, 1}); + test.Run(); + } +} + +TEST(NhwcMaxPoolContribOpTest, MaxPoolStrides_S8) { + NhwcMaxPoolOpTester test; + test.GenerateRandomInput({4, 23, 19, 32}); + test.SetKernelShape({3, 3}); + test.SetStrides({2, 2}); + test.Run(); +} + +TEST(NhwcMaxPoolContribOpTest, MaxPoolDilations_S8) { + NhwcMaxPoolOpTester test; + test.GenerateRandomInput({4, 23, 19, 32}); + test.SetKernelShape({3, 3}); + test.SetDilations({2, 2}); + test.Run(); +} + } // namespace test } // namespace onnxruntime diff --git a/onnxruntime/test/contrib_ops/qlinear_global_average_pool_test.cc b/onnxruntime/test/contrib_ops/qlinear_global_average_pool_test.cc index 94fdef7b78..7fc057aa79 100644 --- a/onnxruntime/test/contrib_ops/qlinear_global_average_pool_test.cc +++ b/onnxruntime/test/contrib_ops/qlinear_global_average_pool_test.cc @@ -5,22 +5,24 @@ #include "test/common/tensor_op_test_utils.h" #include "test/providers/provider_test_utils.h" #include "core/providers/common.h" +#include "core/mlas/inc/mlas.h" namespace onnxruntime { namespace test { -static void CalculateGlobalAvgPoolU8( - const uint8_t* x, int64_t batch, int64_t hw, int64_t channel, bool channels_last, uint8_t* y, +template +static void CalculateGlobalAvgPool( + const T8Bits* x, int64_t batch, int64_t hw, int64_t channel, bool channels_last, T8Bits* y, int32_t x_zero_point, float x_scale, int32_t y_zero_point, float y_scale) { int32_t bias = -x_zero_point * gsl::narrow_cast(hw); int64_t stride_image = channels_last ? channel : 1; int64_t stride_channel = channels_last ? 1 : hw; for (int64_t b = 0; b < batch; ++b) { - const uint8_t* bx = x + b * hw * channel; - uint8_t* by = y + b * channel; + const T8Bits* bx = x + b * hw * channel; + T8Bits* by = y + b * channel; for (int64_t c = 0; c < channel; ++c) { - const uint8_t* ix = bx + c * stride_channel; + const T8Bits* ix = bx + c * stride_channel; int32_t sum = 0; for (int64_t i = 0; i < hw; ++i) { sum += static_cast(*ix); @@ -29,40 +31,41 @@ static void CalculateGlobalAvgPoolU8( sum += bias; int32_t r = static_cast(std::nearbyintf(x_scale * sum / static_cast(hw) / y_scale)); r += y_zero_point; - r = std::min(255, r); - r = std::max(0, r); - by[c] = static_cast(r); + r = std::min((int32_t)(std::numeric_limits::max()), r); + r = std::max((int32_t)(std::numeric_limits::lowest()), r); + by[c] = static_cast(r); } } } -void RunQLinearGlobalAveragePoolU8( +template +void RunQLinearGlobalAveragePool( bool channels_last, int64_t batch, int64_t channel, int64_t h, int64_t w, - uint8_t x_zero_point, float x_scale, uint8_t y_zero_point, float y_scale, int32_t seed = 0) { + T8Bits x_zero_point, float x_scale, T8Bits y_zero_point, float y_scale, int32_t seed = 0) { std::vector x_dims = channels_last ? std::vector{batch, h, w, channel} : std::vector{batch, channel, h, w}; std::vector y_dims = channels_last ? std::vector{batch, 1, 1, channel} : std::vector{batch, channel, 1, 1}; int64_t x_size = batch * channel * h * w; int64_t y_size = batch * channel; - std::vector x_data((size_t)x_size); - std::vector y_data((size_t)y_size); + std::vector x_data((size_t)x_size); + std::vector y_data((size_t)y_size); RandomValueGenerator random{seed ? optional{seed} : optional{}}; - std::vector tmp_x_data = random.Uniform(x_dims, 0, 255); - std::transform(tmp_x_data.begin(), tmp_x_data.end(), x_data.data(), [](int32_t v) -> uint8_t { - return static_cast(v); + std::vector tmp_x_data = random.Uniform(x_dims, std::numeric_limits::lowest(), std::numeric_limits::max()); + std::transform(tmp_x_data.begin(), tmp_x_data.end(), x_data.data(), [](int32_t v) -> T8Bits { + return static_cast(v); }); - CalculateGlobalAvgPoolU8(x_data.data(), batch, h * w, channel, channels_last, y_data.data(), + CalculateGlobalAvgPool(x_data.data(), batch, h * w, channel, channels_last, y_data.data(), x_zero_point, x_scale, y_zero_point, y_scale); OpTester test("QLinearGlobalAveragePool", 1, onnxruntime::kMSDomain); test.AddAttribute("channels_last", channels_last ? 1LL : 0LL); - test.AddInput("X", x_dims, x_data); + test.AddInput("X", x_dims, x_data); test.AddInput("x_scale", {}, {x_scale}); - test.AddInput("x_zero_point", {}, {x_zero_point}); + test.AddInput("x_zero_point", {}, {x_zero_point}); test.AddInput("y_scale", {}, {y_scale}); - test.AddInput("y_zero_point", {}, {y_zero_point}); - test.AddOutput("Y", y_dims, y_data); + test.AddInput("y_zero_point", {}, {y_zero_point}); + test.AddOutput("Y", y_dims, y_data); auto q8checker = [&](const std::vector& fetches, const std::string& provider_type) { const OrtValue& ort_value = fetches[0]; @@ -75,7 +78,7 @@ void RunQLinearGlobalAveragePoolU8( ORT_ENFORCE(y_shape == output_tensor.Shape(), "Expected output shape [" + y_shape.ToString() + "] did not match run output shape [" + output_tensor.Shape().ToString() + "] for Y @" + provider_type); - auto* output = output_tensor.Data(); + auto* output = output_tensor.Data(); auto size = static_cast(output_tensor.Shape().Size()); for (int i = 0; i < size; ++i) { int diff = abs(y_data[i] - output[i]); @@ -89,76 +92,149 @@ void RunQLinearGlobalAveragePoolU8( } TEST(QLinearGlobalAveragePool, Nhwc_1x1x32x32) { - RunQLinearGlobalAveragePoolU8(true, 1, 1, 32, 32, 128, 1.0, 64, 2.0); + RunQLinearGlobalAveragePool(true, 1, 1, 32, 32, 128, 1.0, 64, 2.0); } TEST(QLinearGlobalAveragePool, Nchw_1x32x32x1) { - RunQLinearGlobalAveragePoolU8(false, 1, 1, 32, 32, 128, 1.0, 64, 2.0); + RunQLinearGlobalAveragePool(false, 1, 1, 32, 32, 128, 1.0, 64, 2.0); } TEST(QLinearGlobalAveragePool, Nhwc_1x256x8x8) { - RunQLinearGlobalAveragePoolU8(true, 1, 256, 8, 8, 128, 1.0, 64, 3.0); + RunQLinearGlobalAveragePool(true, 1, 256, 8, 8, 128, 1.0, 64, 3.0); } TEST(QLinearGlobalAveragePool, Nchw_1x8x8x256) { - RunQLinearGlobalAveragePoolU8(false, 1, 256, 8, 8, 128, 1.0, 64, 3.0); + RunQLinearGlobalAveragePool(false, 1, 256, 8, 8, 128, 1.0, 64, 3.0); } TEST(QLinearGlobalAveragePool, Nhwc_1x255x7x7) { - RunQLinearGlobalAveragePoolU8(true, 1, 255, 7, 7, 128, 7.0, 128, 21.0); + RunQLinearGlobalAveragePool(true, 1, 255, 7, 7, 128, 7.0, 128, 21.0); } TEST(QLinearGlobalAveragePool, Nchw_1x7x7x255) { - RunQLinearGlobalAveragePoolU8(false, 1, 255, 7, 7, 128, 7.0, 128, 21.0); + RunQLinearGlobalAveragePool(false, 1, 255, 7, 7, 128, 7.0, 128, 21.0); } TEST(QLinearGlobalAveragePool, Nhwc_1x255x8x8) { - RunQLinearGlobalAveragePoolU8(true, 1, 255, 8, 8, 128, 1.0, 128, 2.0); + RunQLinearGlobalAveragePool(true, 1, 255, 8, 8, 128, 1.0, 128, 2.0); } TEST(QLinearGlobalAveragePool, Nchw_1x8x8x255) { - RunQLinearGlobalAveragePoolU8(false, 1, 255, 8, 8, 128, 1.0, 128, 2.0); + RunQLinearGlobalAveragePool(false, 1, 255, 8, 8, 128, 1.0, 128, 2.0); } TEST(QLinearGlobalAveragePool, Nhwc_1x256x7x7) { - RunQLinearGlobalAveragePoolU8(true, 1, 256, 7, 7, 128, 7.0, 128, 21.0); + RunQLinearGlobalAveragePool(true, 1, 256, 7, 7, 128, 7.0, 128, 21.0); } TEST(QLinearGlobalAveragePool, Nchw_1x7x7x256) { - RunQLinearGlobalAveragePoolU8(false, 1, 256, 7, 7, 128, 7.0, 128, 21.0); + RunQLinearGlobalAveragePool(false, 1, 256, 7, 7, 128, 7.0, 128, 21.0); } // tests for BatchSize > 1 TEST(QLinearGlobalAveragePool, Nhwc_3x256x8x8) { - RunQLinearGlobalAveragePoolU8(true, 3, 256, 8, 8, 128, 1.0, 64, 3.0); + RunQLinearGlobalAveragePool(true, 3, 256, 8, 8, 128, 1.0, 64, 3.0); } TEST(QLinearGlobalAveragePool, Nchw_3x8x8x256) { - RunQLinearGlobalAveragePoolU8(false, 3, 256, 8, 8, 128, 1.0, 64, 3.0); + RunQLinearGlobalAveragePool(false, 3, 256, 8, 8, 128, 1.0, 64, 3.0); } TEST(QLinearGlobalAveragePool, Nhwc_3x255x7x7) { - RunQLinearGlobalAveragePoolU8(true, 3, 255, 7, 7, 128, 7.0, 128, 21.0); + RunQLinearGlobalAveragePool(true, 3, 255, 7, 7, 128, 7.0, 128, 21.0); } TEST(QLinearGlobalAveragePool, Nchw_3x7x7x255) { - RunQLinearGlobalAveragePoolU8(false, 3, 255, 7, 7, 128, 7.0, 128, 21.0); + RunQLinearGlobalAveragePool(false, 3, 255, 7, 7, 128, 7.0, 128, 21.0); } TEST(QLinearGlobalAveragePool, Nhwc_3x255x8x8) { - RunQLinearGlobalAveragePoolU8(true, 3, 255, 8, 8, 128, 1.0, 128, 2.0); + RunQLinearGlobalAveragePool(true, 3, 255, 8, 8, 128, 1.0, 128, 2.0); } TEST(QLinearGlobalAveragePool, Nchw_3x8x8x255) { - RunQLinearGlobalAveragePoolU8(false, 3, 255, 8, 8, 128, 1.0, 128, 2.0); + RunQLinearGlobalAveragePool(false, 3, 255, 8, 8, 128, 1.0, 128, 2.0); } TEST(QLinearGlobalAveragePool, Nhwc_3x256x7x7) { - RunQLinearGlobalAveragePoolU8(true, 3, 256, 7, 7, 128, 7.0, 128, 21.0); + RunQLinearGlobalAveragePool(true, 3, 256, 7, 7, 128, 7.0, 128, 21.0); } TEST(QLinearGlobalAveragePool, Nchw_3x7x7x256) { - RunQLinearGlobalAveragePoolU8(false, 3, 256, 7, 7, 128, 7.0, 128, 21.0); + RunQLinearGlobalAveragePool(false, 3, 256, 7, 7, 128, 7.0, 128, 21.0); +} + +TEST(QLinearGlobalAveragePool, Nhwc_1x1x32x32_S8) { + RunQLinearGlobalAveragePool(true, 1, 1, 32, 32, 1, 1.0, -64, 2.0); +} + +TEST(QLinearGlobalAveragePool, Nchw_1x32x32x1_S8) { + RunQLinearGlobalAveragePool(false, 1, 1, 32, 32, 1, 1.0, 64, 2.0); +} + +TEST(QLinearGlobalAveragePool, Nhwc_1x256x8x8_S8) { + RunQLinearGlobalAveragePool(true, 1, 256, 8, 8, -1, 1.0, -64, 3.0); +} + +TEST(QLinearGlobalAveragePool, Nchw_1x8x8x256_S8) { + RunQLinearGlobalAveragePool(false, 1, 256, 8, 8, -1, 1.0, 64, 3.0); +} + +TEST(QLinearGlobalAveragePool, Nhwc_1x255x7x7_S8) { + RunQLinearGlobalAveragePool(true, 1, 255, 7, 7, 64, 7.0, 1, 21.0); +} + +TEST(QLinearGlobalAveragePool, Nchw_1x7x7x255_S8) { + RunQLinearGlobalAveragePool(false, 1, 255, 7, 7, 64, 7.0, -1, 21.0); +} + +TEST(QLinearGlobalAveragePool, Nhwc_1x255x8x8_S8) { + RunQLinearGlobalAveragePool(true, 1, 255, 8, 8, -64, 1.0, 1, 2.0); +} + +TEST(QLinearGlobalAveragePool, Nchw_1x8x8x255_S8) { + RunQLinearGlobalAveragePool(false, 1, 255, 8, 8, -64, 1.0, -1, 2.0); +} + +TEST(QLinearGlobalAveragePool, Nhwc_1x256x7x7_S8) { + RunQLinearGlobalAveragePool(true, 1, 256, 7, 7, -64, 7.0, 64, 21.0); +} + +TEST(QLinearGlobalAveragePool, Nchw_1x7x7x256_S8) { + RunQLinearGlobalAveragePool(false, 1, 256, 7, 7, 64, 7.0, -64, 21.0); +} + +// tests for BatchSize > 1 +TEST(QLinearGlobalAveragePool, Nhwc_3x256x8x8_S8) { + RunQLinearGlobalAveragePool(true, 3, 256, 8, 8, 1, 1.0, 64, 3.0); +} + +TEST(QLinearGlobalAveragePool, Nchw_3x8x8x256_S8) { + RunQLinearGlobalAveragePool(false, 3, 256, 8, 8, 1, 1.0, 64, 3.0); +} + +TEST(QLinearGlobalAveragePool, Nhwc_3x255x7x7_S8) { + RunQLinearGlobalAveragePool(true, 3, 255, 7, 7, 1, 7.0, -1, 21.0); +} + +TEST(QLinearGlobalAveragePool, Nchw_3x7x7x255_S8) { + RunQLinearGlobalAveragePool(false, 3, 255, 7, 7, 1, 7.0, -1, 21.0); +} + +TEST(QLinearGlobalAveragePool, Nhwc_3x255x8x8_S8) { + RunQLinearGlobalAveragePool(true, 3, 255, 8, 8, 1, 1.0, -1, 2.0); +} + +TEST(QLinearGlobalAveragePool, Nchw_3x8x8x255_S8) { + RunQLinearGlobalAveragePool(false, 3, 255, 8, 8, -1, 1.0, 1, 2.0); +} + +TEST(QLinearGlobalAveragePool, Nhwc_3x256x7x7_S8) { + RunQLinearGlobalAveragePool(true, 3, 256, 7, 7, -1, 7.0, 1, 21.0); +} + +TEST(QLinearGlobalAveragePool, Nchw_3x7x7x256_S8) { + RunQLinearGlobalAveragePool(false, 3, 256, 7, 7, -1, 7.0, 1, 21.0); } } // namespace test diff --git a/onnxruntime/test/contrib_ops/qlinear_pool_test.cc b/onnxruntime/test/contrib_ops/qlinear_pool_test.cc index 85b8c998e7..335c5eaeaf 100644 --- a/onnxruntime/test/contrib_ops/qlinear_pool_test.cc +++ b/onnxruntime/test/contrib_ops/qlinear_pool_test.cc @@ -6,6 +6,7 @@ #include "test/common/tensor_op_test_utils.h" #include "test/providers/provider_test_utils.h" #include "core/providers/common.h" +#include "core/mlas/inc/mlas.h" namespace onnxruntime { namespace test { @@ -46,14 +47,15 @@ struct DimIterator { int64_t index_; }; +template static void -CalculateAvgPoolNchwU8( - uint8_t* x, +CalculateAvgPoolNchw( + T8Bits* x, const std::vector x_dims, - const quantization::Params& x_params, - uint8_t* y, + const quantization::Params& x_params, + T8Bits* y, const std::vector y_dims, - const quantization::Params& y_params, + const quantization::Params& y_params, const std::vector kernel_shape, const std::vector strides, const std::vector pads, @@ -73,8 +75,8 @@ CalculateAvgPoolNchwU8( int64_t x_step = std::accumulate(x_img_dims.begin(), x_img_dims.end(), 1LL, std::multiplies()); for (int64_t b = 0; b < batch; ++b) { for (int64_t c = 0; c < channel; ++c) { - uint8_t* ybc = y + (b * channel + c) * y_step; - uint8_t* xbc = x + (b * channel + c) * x_step; + T8Bits* ybc = y + (b * channel + c) * y_step; + T8Bits* xbc = x + (b * channel + c) * x_step; DimIterator yit(y_img_dims); while (yit.has_next()) { @@ -103,14 +105,15 @@ CalculateAvgPoolNchwU8( } } auto y_offset = yit.next(); - auto y_u8 = QuantizeTestValue(y_value_sum / count, y_params); + auto y_u8 = QuantizeTestValue(y_value_sum / count, y_params); ybc[y_offset] = y_u8; } } } } -void RunQLinearAveragePoolNchwU8( +template +void RunQLinearAveragePoolNchw( const std::vector x_dims, const std::vector y_dims, const std::vector kernel_shape, @@ -119,17 +122,19 @@ void RunQLinearAveragePoolNchwU8( const int64_t count_include_pad = 0) { auto run_test = [&](bool only_x_not_initializer, bool x_y_same_zero_point) { float x_scale = 1.0f / 255.0f; - quantization::Params x_params(x_scale, /*zero_point=*/128); + T8Bits x_zero_point = (std::numeric_limits::lowest() + std::numeric_limits::max() - 5) / 2; + quantization::Params x_params(x_scale, x_zero_point); RandomValueGenerator random{}; std::vector x_data_fp32 = random.Uniform(x_dims, -0.5f, 0.5f); - std::vector x_data = QuantizeTestVector(x_data_fp32, x_params); + std::vector x_data = QuantizeTestVector(x_data_fp32, x_params); float y_scale = 1.0f / 255.0f; - uint8_t y_zero_point = x_y_same_zero_point ? x_params.zero_point : 100; - const quantization::Params y_params(y_scale, y_zero_point); + T8Bits y_zero_point_not_same = (std::numeric_limits::lowest() + std::numeric_limits::max() + 10) / 2; + T8Bits y_zero_point = x_y_same_zero_point ? x_params.zero_point : y_zero_point_not_same; + const quantization::Params y_params(y_scale, y_zero_point); int64_t y_size = std::accumulate(y_dims.begin(), y_dims.end(), 1LL, std::multiplies()); - std::vector y_data(y_size); - CalculateAvgPoolNchwU8( + std::vector y_data(y_size); + CalculateAvgPoolNchw( x_data.data(), x_dims, x_params, y_data.data(), y_dims, y_params, kernel_shape, strides, pads, count_include_pad); @@ -142,12 +147,12 @@ void RunQLinearAveragePoolNchwU8( test.AddAttribute("kernel_shape", kernel_shape); test.AddAttribute("count_include_pad", count_include_pad); - test.AddInput("X", x_dims, x_data); + test.AddInput("X", x_dims, x_data); test.AddInput("x_scale", {}, {x_scale}, only_x_not_initializer); - test.AddInput("x_zero_point", {}, {x_params.zero_point}, only_x_not_initializer); + test.AddInput("x_zero_point", {}, {x_params.zero_point}, only_x_not_initializer); test.AddInput("y_scale", {}, {y_scale}, only_x_not_initializer); - test.AddInput("y_zero_point", {}, {y_params.zero_point}, only_x_not_initializer); - test.AddOutput("Y", y_dims, y_data); + test.AddInput("y_zero_point", {}, {y_params.zero_point}, only_x_not_initializer); + test.AddOutput("Y", y_dims, y_data); auto q8checker = [&](const std::vector& fetches, const std::string& provider_type) { const OrtValue& ort_value = fetches[0]; @@ -160,7 +165,7 @@ void RunQLinearAveragePoolNchwU8( ORT_ENFORCE(y_shape == output_tensor.Shape(), "Expected output shape [" + y_shape.ToString() + "] did not match run output shape [" + output_tensor.Shape().ToString() + "] for Y @" + provider_type); - auto* output = output_tensor.Data(); + auto* output = output_tensor.Data(); auto size = static_cast(output_tensor.Shape().Size()); for (int i = 0; i < size; ++i) { int diff = abs(y_data[i] - output[i]); @@ -188,15 +193,16 @@ static std::vector dims_to_nhwc(const std::vector& nchw) { return nhwc; } -static std::vector transpose_to_nhwc(const std::vector& nchw_data, const std::vector& nchw_dims) { - std::vector nhwc_data(nchw_data.size()); +template +static std::vector transpose_to_nhwc(const std::vector& nchw_data, const std::vector& nchw_dims) { + std::vector nhwc_data(nchw_data.size()); auto batch_count = nchw_dims[0]; auto channels = nchw_dims[1]; int64_t image_size = std::accumulate(nchw_dims.begin() + 2, nchw_dims.end(), 1LL, std::multiplies()); for (int64_t b = 0; b < batch_count; b++) { - const uint8_t* nchw_image = nchw_data.data() + (b * channels * image_size); - uint8_t* nhwc_image = nhwc_data.data() + (b * channels * image_size); + const T8Bits* nchw_image = nchw_data.data() + (b * channels * image_size); + T8Bits* nhwc_image = nhwc_data.data() + (b * channels * image_size); for (int64_t img_index = 0; img_index < image_size; ++img_index) { for (int64_t c = 0; c < channels; c++) { *nhwc_image++ = nchw_image[c * image_size + img_index]; @@ -207,7 +213,8 @@ static std::vector transpose_to_nhwc(const std::vector& nchw_d return nhwc_data; } -void RunQLinearAveragePoolNhwcU8( +template +void RunQLinearAveragePoolNhwc( const std::vector x_dims, const std::vector y_dims, const std::vector kernel_shape, @@ -215,23 +222,25 @@ void RunQLinearAveragePoolNhwcU8( const std::vector pads, const int64_t count_include_pad = 0) { float x_scale = 1.0f / 255.0f; - const quantization::Params x_params(x_scale, /*zero_point=*/128); + T8Bits x_zero_point = (std::numeric_limits::lowest() + std::numeric_limits::max() - 5) / 2; + const quantization::Params x_params(x_scale, x_zero_point); RandomValueGenerator random{}; std::vector x_data_fp32 = random.Uniform(x_dims, -0.5f, 0.5f); - std::vector x_data = QuantizeTestVector(x_data_fp32, x_params); + std::vector x_data = QuantizeTestVector(x_data_fp32, x_params); float y_scale = 1.0f / 255.0f; - const quantization::Params y_params(y_scale, /*zero_point=*/100); + T8Bits y_zero_point = (std::numeric_limits::lowest() + std::numeric_limits::max() + 10) / 2; + const quantization::Params y_params(y_scale, y_zero_point); int64_t y_size = std::accumulate(y_dims.begin(), y_dims.end(), 1LL, std::multiplies()); - std::vector y_data(y_size); - CalculateAvgPoolNchwU8( + std::vector y_data(y_size); + CalculateAvgPoolNchw( x_data.data(), x_dims, x_params, y_data.data(), y_dims, y_params, kernel_shape, strides, pads, count_include_pad); // transpose the result - std::vector y_data_nhwc = transpose_to_nhwc(y_data, y_dims); - std::vector x_data_nhwc = transpose_to_nhwc(x_data, x_dims); + std::vector y_data_nhwc = transpose_to_nhwc(y_data, y_dims); + std::vector x_data_nhwc = transpose_to_nhwc(x_data, x_dims); auto x_dims_nhwc = dims_to_nhwc(x_dims); auto y_dims_nhwc = dims_to_nhwc(y_dims); @@ -244,12 +253,12 @@ void RunQLinearAveragePoolNhwcU8( test.AddAttribute("count_include_pad", count_include_pad); test.AddAttribute("channels_last", (int64_t)1LL); - test.AddInput("X", x_dims_nhwc, x_data_nhwc); + test.AddInput("X", x_dims_nhwc, x_data_nhwc); test.AddInput("x_scale", {}, {x_scale}); - test.AddInput("x_zero_point", {}, {x_params.zero_point}); + test.AddInput("x_zero_point", {}, {x_params.zero_point}); test.AddInput("y_scale", {}, {y_scale}); - test.AddInput("y_zero_point", {}, {y_params.zero_point}); - test.AddOutput("Y", y_dims_nhwc, y_data_nhwc); + test.AddInput("y_zero_point", {}, {y_params.zero_point}); + test.AddOutput("Y", y_dims_nhwc, y_data_nhwc); auto q8checker = [&](const std::vector& fetches, const std::string& provider_type) { const OrtValue& ort_value = fetches[0]; @@ -262,7 +271,7 @@ void RunQLinearAveragePoolNhwcU8( ORT_ENFORCE(y_shape == output_tensor.Shape(), "Expected output shape [" + y_shape.ToString() + "] did not match run output shape [" + output_tensor.Shape().ToString() + "] for Y @" + provider_type); - auto* output = output_tensor.Data(); + auto* output = output_tensor.Data(); auto size = static_cast(output_tensor.Shape().Size()); for (int i = 0; i < size; ++i) { int diff = abs(y_data_nhwc[i] - output[i]); @@ -278,7 +287,7 @@ void RunQLinearAveragePoolNhwcU8( } TEST(QLinearPoolTest, AveragePool1D_ExcludePadPixel) { - RunQLinearAveragePoolNchwU8( + RunQLinearAveragePoolNchw( {1, 1, 5}, // x shape {1, 1, 6}, // expected y shape {3}, // kernel shape @@ -288,7 +297,7 @@ TEST(QLinearPoolTest, AveragePool1D_ExcludePadPixel) { } TEST(QLinearPoolTest, AveragePool1D_IncludePadPixel) { - RunQLinearAveragePoolNchwU8( + RunQLinearAveragePoolNchw( {1, 1, 5}, // x shape {1, 1, 6}, // expected y shape {3}, // kernel shape @@ -298,7 +307,7 @@ TEST(QLinearPoolTest, AveragePool1D_IncludePadPixel) { } TEST(QLinearPoolTest, AveragePool2D_ExcludePadPixel) { - RunQLinearAveragePoolNchwU8( + RunQLinearAveragePoolNchw( {1, 1, 5, 7}, // x shape {1, 1, 6, 4}, // expected y shape {3, 4}, // kernel shape @@ -308,7 +317,7 @@ TEST(QLinearPoolTest, AveragePool2D_ExcludePadPixel) { } TEST(QLinearPoolTest, AveragePool2D_IncludePadPixel) { - RunQLinearAveragePoolNchwU8( + RunQLinearAveragePoolNchw( {1, 1, 5, 7}, // x shape {1, 1, 6, 4}, // expected y shape {3, 4}, // kernel shape @@ -318,7 +327,7 @@ TEST(QLinearPoolTest, AveragePool2D_IncludePadPixel) { } TEST(QLinearPoolTest, AveragePool2D_MultiChannel) { - RunQLinearAveragePoolNchwU8( + RunQLinearAveragePoolNchw( {1, 3, 5, 7}, // x shape {1, 3, 6, 4}, // expected y shape {3, 4}, // kernel shape @@ -328,7 +337,7 @@ TEST(QLinearPoolTest, AveragePool2D_MultiChannel) { } TEST(QLinearPoolTest, AveragePool3D_ExcludePadPixel) { - RunQLinearAveragePoolNchwU8( + RunQLinearAveragePoolNchw( {1, 1, 5, 7, 9}, // x shape {1, 1, 6, 4, 3}, // expected y shape {3, 4, 5}, // kernel shape @@ -338,7 +347,7 @@ TEST(QLinearPoolTest, AveragePool3D_ExcludePadPixel) { } TEST(QLinearPoolTest, AveragePool3D_IncludePadPixel) { - RunQLinearAveragePoolNchwU8( + RunQLinearAveragePoolNchw( {1, 1, 5, 7, 9}, // x shape {1, 1, 6, 4, 3}, // expected y shape {3, 4, 5}, // kernel shape @@ -351,7 +360,7 @@ TEST(QLinearPoolTest, AveragePool3D_IncludePadPixel) { * Channels last test **************************************************/ TEST(QLinearPoolTest, AveragePool1D_ExcludePadPixel_nhwc) { - RunQLinearAveragePoolNhwcU8( + RunQLinearAveragePoolNhwc( {1, 1, 5}, // x shape {1, 1, 6}, // expected y shape {3}, // kernel shape @@ -361,7 +370,7 @@ TEST(QLinearPoolTest, AveragePool1D_ExcludePadPixel_nhwc) { } TEST(QLinearPoolTest, AveragePool1D_IncludePadPixel_nhwc) { - RunQLinearAveragePoolNhwcU8( + RunQLinearAveragePoolNhwc( {1, 1, 5}, // x shape {1, 1, 6}, // expected y shape {3}, // kernel shape @@ -371,7 +380,7 @@ TEST(QLinearPoolTest, AveragePool1D_IncludePadPixel_nhwc) { } TEST(QLinearPoolTest, AveragePool2D_ExcludePadPixel_nhwc) { - RunQLinearAveragePoolNhwcU8( + RunQLinearAveragePoolNhwc( {1, 1, 5, 7}, // x shape {1, 1, 6, 4}, // expected y shape {3, 4}, // kernel shape @@ -381,7 +390,7 @@ TEST(QLinearPoolTest, AveragePool2D_ExcludePadPixel_nhwc) { } TEST(QLinearPoolTest, AveragePool2D_IncludePadPixel_nhwc) { - RunQLinearAveragePoolNhwcU8( + RunQLinearAveragePoolNhwc( {1, 1, 5, 7}, // x shape {1, 1, 6, 4}, // expected y shape {3, 4}, // kernel shape @@ -391,7 +400,7 @@ TEST(QLinearPoolTest, AveragePool2D_IncludePadPixel_nhwc) { } TEST(QLinearPoolTest, AveragePool2D_MultiChannel_nhwc) { - RunQLinearAveragePoolNhwcU8( + RunQLinearAveragePoolNhwc( {1, 3, 5, 7}, // x shape {1, 3, 6, 4}, // expected y shape {3, 4}, // kernel shape @@ -401,7 +410,7 @@ TEST(QLinearPoolTest, AveragePool2D_MultiChannel_nhwc) { } TEST(QLinearPoolTest, AveragePool3D_ExcludePadPixel_nhwc) { - RunQLinearAveragePoolNhwcU8( + RunQLinearAveragePoolNhwc( {1, 1, 5, 7, 9}, // x shape {1, 1, 6, 4, 3}, // expected y shape {3, 4, 5}, // kernel shape @@ -411,7 +420,7 @@ TEST(QLinearPoolTest, AveragePool3D_ExcludePadPixel_nhwc) { } TEST(QLinearPoolTest, AveragePool3D_IncludePadPixel_nhwc) { - RunQLinearAveragePoolNhwcU8( + RunQLinearAveragePoolNhwc( {1, 1, 5, 7, 9}, // x shape {1, 1, 6, 4, 3}, // expected y shape {3, 4, 5}, // kernel shape @@ -420,9 +429,8 @@ TEST(QLinearPoolTest, AveragePool3D_IncludePadPixel_nhwc) { 1); // count_include_pad } - TEST(QLinearPoolTest, AveragePool2D_BigImage) { - RunQLinearAveragePoolNchwU8( + RunQLinearAveragePoolNchw( {1, 1, 32, 64}, // x shape {1, 1, 32, 64}, // expected y shape {3, 3}, // kernel shape @@ -432,7 +440,7 @@ TEST(QLinearPoolTest, AveragePool2D_BigImage) { } TEST(QLinearPoolTest, AveragePool2D_BigImage_nhwc) { - RunQLinearAveragePoolNhwcU8( + RunQLinearAveragePoolNhwc( {1, 1, 32, 64}, // x shape {1, 1, 32, 64}, // expected y shape {3, 3}, // kernel shape @@ -442,7 +450,7 @@ TEST(QLinearPoolTest, AveragePool2D_BigImage_nhwc) { } TEST(QLinearPoolTest, AveragePool2D_Global) { - RunQLinearAveragePoolNchwU8( + RunQLinearAveragePoolNchw( {1, 2, 32, 16}, // x shape {1, 2, 1, 1}, // expected y shape {32, 16}, // kernel shape @@ -452,7 +460,7 @@ TEST(QLinearPoolTest, AveragePool2D_Global) { } TEST(QLinearPoolTest, AveragePool2D_Global_nhwc) { - RunQLinearAveragePoolNhwcU8( + RunQLinearAveragePoolNhwc( {1, 2, 32, 16}, // x shape {1, 2, 1, 1}, // expected y shape {32, 16}, // kernel shape @@ -461,6 +469,188 @@ TEST(QLinearPoolTest, AveragePool2D_Global_nhwc) { 1); // count_include_pad } +TEST(QLinearPoolTest, AveragePool1D_ExcludePadPixel_S8) { + RunQLinearAveragePoolNchw( + {1, 1, 5}, // x shape + {1, 1, 6}, // expected y shape + {3}, // kernel shape + {1}, // strides + {1, 2}, // pads + 0); // count_include_pad +} + +TEST(QLinearPoolTest, AveragePool1D_IncludePadPixel_S8) { + RunQLinearAveragePoolNchw( + {1, 1, 5}, // x shape + {1, 1, 6}, // expected y shape + {3}, // kernel shape + {1}, // strides + {1, 2}, // pads + 1); // count_include_pad +} + +TEST(QLinearPoolTest, AveragePool2D_ExcludePadPixel_S8) { + RunQLinearAveragePoolNchw( + {1, 1, 5, 7}, // x shape + {1, 1, 6, 4}, // expected y shape + {3, 4}, // kernel shape + {1, 2}, // strides + {1, 3, 2, 1}, // pads + 0); // count_include_pad +} + +TEST(QLinearPoolTest, AveragePool2D_IncludePadPixel_S8) { + RunQLinearAveragePoolNchw( + {1, 1, 5, 7}, // x shape + {1, 1, 6, 4}, // expected y shape + {3, 4}, // kernel shape + {1, 2}, // strides + {1, 3, 2, 1}, // pads + 1); // count_include_pad +} + +TEST(QLinearPoolTest, AveragePool2D_MultiChannel_S8) { + RunQLinearAveragePoolNchw( + {1, 3, 5, 7}, // x shape + {1, 3, 6, 4}, // expected y shape + {3, 4}, // kernel shape + {1, 2}, // strides + {1, 3, 2, 1}, // pads + 1); // count_include_pad +} + +TEST(QLinearPoolTest, AveragePool3D_ExcludePadPixel_S8) { + RunQLinearAveragePoolNchw( + {1, 1, 5, 7, 9}, // x shape + {1, 1, 6, 4, 3}, // expected y shape + {3, 4, 5}, // kernel shape + {1, 2, 3}, // strides + {1, 3, 2, 2, 1, 2}, // pads + 0); // count_include_pad +} + +TEST(QLinearPoolTest, AveragePool3D_IncludePadPixel_S8) { + RunQLinearAveragePoolNchw( + {1, 1, 5, 7, 9}, // x shape + {1, 1, 6, 4, 3}, // expected y shape + {3, 4, 5}, // kernel shape + {1, 2, 3}, // strides + {1, 3, 2, 2, 1, 2}, // pads + 1); // count_include_pad +} + +/************************************************* +* Channels last test +**************************************************/ +TEST(QLinearPoolTest, AveragePool1D_ExcludePadPixel_nhwc_S8) { + RunQLinearAveragePoolNhwc( + {1, 1, 5}, // x shape + {1, 1, 6}, // expected y shape + {3}, // kernel shape + {1}, // strides + {1, 2}, // pads + 0); // count_include_pad +} + +TEST(QLinearPoolTest, AveragePool1D_IncludePadPixel_nhwc_S8) { + RunQLinearAveragePoolNhwc( + {1, 1, 5}, // x shape + {1, 1, 6}, // expected y shape + {3}, // kernel shape + {1}, // strides + {1, 2}, // pads + 1); // count_include_pad +} + +TEST(QLinearPoolTest, AveragePool2D_ExcludePadPixel_nhwc_S8) { + RunQLinearAveragePoolNhwc( + {1, 1, 5, 7}, // x shape + {1, 1, 6, 4}, // expected y shape + {3, 4}, // kernel shape + {1, 2}, // strides + {1, 3, 2, 1}, // pads + 0); // count_include_pad +} + +TEST(QLinearPoolTest, AveragePool2D_IncludePadPixel_nhwc_S8) { + RunQLinearAveragePoolNhwc( + {1, 1, 5, 7}, // x shape + {1, 1, 6, 4}, // expected y shape + {3, 4}, // kernel shape + {1, 2}, // strides + {1, 3, 2, 1}, // pads + 1); // count_include_pad +} + +TEST(QLinearPoolTest, AveragePool2D_MultiChannel_nhwc_S8) { + RunQLinearAveragePoolNhwc( + {1, 3, 5, 7}, // x shape + {1, 3, 6, 4}, // expected y shape + {3, 4}, // kernel shape + {1, 2}, // strides + {1, 3, 2, 1}, // pads + 1); // count_include_pad +} + +TEST(QLinearPoolTest, AveragePool3D_ExcludePadPixel_nhwc_S8) { + RunQLinearAveragePoolNhwc( + {1, 1, 5, 7, 9}, // x shape + {1, 1, 6, 4, 3}, // expected y shape + {3, 4, 5}, // kernel shape + {1, 2, 3}, // strides + {1, 3, 2, 2, 1, 2}, // pads + 0); // count_include_pad +} + +TEST(QLinearPoolTest, AveragePool3D_IncludePadPixel_nhwc_S8) { + RunQLinearAveragePoolNhwc( + {1, 1, 5, 7, 9}, // x shape + {1, 1, 6, 4, 3}, // expected y shape + {3, 4, 5}, // kernel shape + {1, 2, 3}, // strides + {1, 3, 2, 2, 1, 2}, // pads + 1); // count_include_pad +} + +TEST(QLinearPoolTest, AveragePool2D_BigImage_S8) { + RunQLinearAveragePoolNchw( + {1, 1, 32, 64}, // x shape + {1, 1, 32, 64}, // expected y shape + {3, 3}, // kernel shape + {1, 1}, // strides + {1, 1, 1, 1}, // pads + 1); // count_include_pad +} + +TEST(QLinearPoolTest, AveragePool2D_BigImage_nhwc_S8) { + RunQLinearAveragePoolNhwc( + {1, 1, 32, 64}, // x shape + {1, 1, 32, 64}, // expected y shape + {3, 3}, // kernel shape + {1, 1}, // strides + {1, 1, 1, 1}, // pads + 1); // count_include_pad +} + +TEST(QLinearPoolTest, AveragePool2D_Global_S8) { + RunQLinearAveragePoolNchw( + {1, 2, 32, 16}, // x shape + {1, 2, 1, 1}, // expected y shape + {32, 16}, // kernel shape + {1, 1}, // strides + {0, 0, 0, 0}, // pads + 1); // count_include_pad +} + +TEST(QLinearPoolTest, AveragePool2D_Global_nhwc_S8) { + RunQLinearAveragePoolNhwc( + {1, 2, 32, 16}, // x shape + {1, 2, 1, 1}, // expected y shape + {32, 16}, // kernel shape + {1, 1}, // strides + {0, 0, 0, 0}, // pads + 1); // count_include_pad +} } // namespace test } // namespace onnxruntime diff --git a/onnxruntime/test/mlas/unittest/test_qlinear_gavgpool.cpp b/onnxruntime/test/mlas/unittest/test_qlinear_gavgpool.cpp index 936bac86c6..aeb13af5b9 100644 --- a/onnxruntime/test/mlas/unittest/test_qlinear_gavgpool.cpp +++ b/onnxruntime/test/mlas/unittest/test_qlinear_gavgpool.cpp @@ -3,24 +3,28 @@ #include "test_util.h" -class MlasQLinearGlobalAveragePoolU8Test : public MlasTestBase { - private: - MatrixGuardBuffer BufferInput; - MatrixGuardBuffer BufferOutput; - MatrixGuardBuffer BufferOutputReference; +#include - static void CalculateGlobalAvgPoolU8( - const uint8_t* x, int64_t batch, int64_t channel, int64_t hw, bool channel_last, - uint8_t* y, int32_t x_zero_point, float x_scale, int32_t y_zero_point, float y_scale) { +template +class MlasQLinearGlobalAveragePoolTest : public MlasTestBase { + private: + MatrixGuardBuffer BufferInput; + MatrixGuardBuffer BufferOutput; + MatrixGuardBuffer BufferOutputReference; + static const std::vector ZeroPoints; + + static void CalculateGlobalAvgPool( + const T8Bits* x, int64_t batch, int64_t channel, int64_t hw, bool channel_last, + T8Bits* y, int32_t x_zero_point, float x_scale, int32_t y_zero_point, float y_scale) { int32_t bias = -x_zero_point * static_cast(hw); int64_t stride_image = channel_last ? channel : 1; int64_t stride_channel = channel_last ? 1 : hw; for (int64_t b = 0; b < batch; ++b) { - const uint8_t* bx = x + b * hw * channel; - uint8_t* by = y + b * channel; + const T8Bits* bx = x + b * hw * channel; + T8Bits* by = y + b * channel; for (int64_t c = 0; c < channel; ++c) { - const uint8_t* ix = bx + c * stride_channel; + const T8Bits* ix = bx + c * stride_channel; int32_t sum = 0; for (int64_t i = 0; i < hw; ++i) { sum += static_cast(*ix); @@ -29,15 +33,15 @@ class MlasQLinearGlobalAveragePoolU8Test : public MlasTestBase { sum += bias; int32_t r = static_cast(std::nearbyintf(x_scale * sum / static_cast(hw) / y_scale)); r += y_zero_point; - r = std::min(255, r); - r = std::max(0, r); - by[c] = static_cast(r); + r = std::min((int32_t)(std::numeric_limits::max()), r); + r = std::max((int32_t)(std::numeric_limits::lowest()), r); + by[c] = static_cast(r); } } } static void CompareResultWithGold(size_t Batch, size_t Channel, - uint8_t* Output, uint8_t* OutputReference, std::string& info) { + T8Bits* Output, T8Bits* OutputReference, std::string& info) { size_t n = 0; for (size_t b = 0; b < Batch; ++b) { for (size_t c = 0; c < Channel; c++) { @@ -53,9 +57,9 @@ class MlasQLinearGlobalAveragePoolU8Test : public MlasTestBase { size_t Channel, size_t ImageSize, float InputScale, - uint8_t InputZeroPoint, + T8Bits InputZeroPoint, float OutputScale, - uint8_t OutputZeroPoint) { + T8Bits OutputZeroPoint) { std::stringstream ss; ss << (channel_last ? "Nhwc_" : "Nchw_"); ss << Batch << "x [C=" << Stride << "-" << Channel << "] x" << ImageSize << "-"; @@ -69,25 +73,25 @@ class MlasQLinearGlobalAveragePoolU8Test : public MlasTestBase { size_t Channel, size_t ImageSize, float InputScale, - uint8_t InputZeroPoint, + T8Bits InputZeroPoint, float OutputScale, - uint8_t OutputZeroPoint, + T8Bits OutputZeroPoint, int32_t UnalignedOffset = 0) { size_t N = Batch * Stride * ImageSize; size_t ResultLen = Batch * Stride; - uint8_t* Input = BufferInput.GetBuffer(N); - uint8_t* Output = BufferOutput.GetBuffer(ResultLen); - uint8_t* Gold = BufferOutputReference.GetBuffer(ResultLen); + T8Bits* Input = BufferInput.GetBuffer(N); + T8Bits* Output = BufferOutput.GetBuffer(ResultLen); + T8Bits* Gold = BufferOutputReference.GetBuffer(ResultLen); std::string test_info = GetTestInfo( channel_last, Batch, Stride, Channel, ImageSize, InputScale, InputZeroPoint, OutputScale, OutputZeroPoint); std::default_random_engine generator(static_cast(N)); - std::uniform_int_distribution distribution(0, 255); + std::uniform_int_distribution distribution(std::numeric_limits::lowest(), std::numeric_limits::max()); for (size_t n = 0; n < N; n++) { - Input[n] = static_cast(distribution(generator)); + Input[n] = static_cast(distribution(generator)); } - CalculateGlobalAvgPoolU8( + CalculateGlobalAvgPool( Input, Batch, Stride, ImageSize, channel_last, Gold, InputZeroPoint, InputScale, OutputZeroPoint, OutputScale); @@ -98,7 +102,7 @@ class MlasQLinearGlobalAveragePoolU8Test : public MlasTestBase { OutputScale, OutputZeroPoint, ResultLen, ImageSize, acc.data() + UnalignedOffset); } else { std::vector acc(MlasQLinearSafePaddingElementCount(sizeof(int32_t), Channel + UnalignedOffset)); - std::vector zero(MlasQLinearSafePaddingElementCount(sizeof(uint8_t), Channel + UnalignedOffset)); + std::vector zero(MlasQLinearSafePaddingElementCount(sizeof(T8Bits), Channel + UnalignedOffset)); if (Stride == Channel) { MlasQLinearGlobalAveragePoolNhwc( Input, InputScale, InputZeroPoint, Output, @@ -120,12 +124,12 @@ class MlasQLinearGlobalAveragePoolU8Test : public MlasTestBase { public: static const char* GetTestSuiteName() { - static const std::string suite_name("QLinearGlobalAvgPool"); + constexpr bool is_signed = std::is_signed::value; + static const std::string suite_name(is_signed ? "QLinearGlobalAvgPoolS8" : "QLinearGlobalAvgPoolU8"); return suite_name.c_str(); } void ExecuteShort(void) override { - static const uint8_t zero_points[] = {0, 18, 128, 231, 255}; static const float scales[] = {18.0f, 90.0f}; static const size_t Batch[] = {1, 3}; static const size_t Stride[] = {7, 8, 63, 256}; @@ -134,17 +138,17 @@ class MlasQLinearGlobalAveragePoolU8Test : public MlasTestBase { for (int channel_last = 0; channel_last <= 1; ++channel_last) { for (size_t b = 0; b < _countof(Batch); b++) { - for (size_t xzp = 0; xzp < _countof(zero_points); xzp++) { - for (size_t yzp = 0; yzp < _countof(zero_points); yzp++) { + for (size_t xzp = 0; xzp < ZeroPoints.size(); xzp++) { + for (size_t yzp = 0; yzp < ZeroPoints.size(); yzp++) { for (size_t xs = 0; xs < _countof(scales); ++xs) { for (size_t ys = 0; ys < _countof(scales); ++ys) { for (size_t i = 0; i < _countof(ImageSize); i++) { for (size_t s = 0; s < _countof(Stride); s++) { Test(channel_last != 0, Batch[b], Stride[s], Stride[s], ImageSize[i], - scales[xs], zero_points[xzp], scales[ys], zero_points[yzp], unalign_offset); + scales[xs], ZeroPoints[xzp], scales[ys], ZeroPoints[yzp], unalign_offset); if (channel_last == 1 && Stride[s] > 32) { Test(channel_last != 0, Batch[b], Stride[s], 32, ImageSize[i], - scales[xs], zero_points[xzp], scales[ys], zero_points[yzp], unalign_offset); + scales[xs], ZeroPoints[xzp], scales[ys], ZeroPoints[yzp], unalign_offset); } unalign_offset = (unalign_offset + 1) & 3; } @@ -158,8 +162,21 @@ class MlasQLinearGlobalAveragePoolU8Test : public MlasTestBase { } }; -template <> MlasQLinearGlobalAveragePoolU8Test* MlasTestFixture::mlas_tester(nullptr); +template <> +MlasQLinearGlobalAveragePoolTest* MlasTestFixture>::mlas_tester(nullptr); +template <> +MlasQLinearGlobalAveragePoolTest* MlasTestFixture>::mlas_tester(nullptr); + +template <> +const std::vector MlasQLinearGlobalAveragePoolTest::ZeroPoints = {-128, -110, 1, 103, 127}; + +template <> +const std::vector MlasQLinearGlobalAveragePoolTest::ZeroPoints = {0, 18, 128, 231, 255}; static UNUSED_VARIABLE bool added_to_main = AddTestRegister([](bool is_short_execute) { - return is_short_execute ? MlasDirectShortExecuteTests::RegisterShortExecute() : 0; + if (is_short_execute) { + return MlasDirectShortExecuteTests>::RegisterShortExecute() + + MlasDirectShortExecuteTests>::RegisterShortExecute(); + } + return (size_t)0; }); diff --git a/onnxruntime/test/testdata/kernel_def_hashes/contrib.cpu.json b/onnxruntime/test/testdata/kernel_def_hashes/contrib.cpu.json index bb0f31e904..3b1747a90f 100644 --- a/onnxruntime/test/testdata/kernel_def_hashes/contrib.cpu.json +++ b/onnxruntime/test/testdata/kernel_def_hashes/contrib.cpu.json @@ -27,6 +27,10 @@ "MeanVarianceNormalization ai.onnx CPUExecutionProvider", 13114085849278607104 ], + [ + "NhwcMaxPool com.microsoft CPUExecutionProvider", + 11773579655431087496 + ], [ "ParametricSoftplus ai.onnx CPUExecutionProvider", 17971715260566574960