diff --git a/docs/OperatorKernels.md b/docs/OperatorKernels.md
index bf637d6405..a7ea71314f 100644
--- a/docs/OperatorKernels.md
+++ b/docs/OperatorKernels.md
@@ -400,7 +400,7 @@ Do not modify directly.*
|MaxpoolWithMask|*in* X:**T**
*in* M:**tensor(int32)**
*out* Y:**T**|1+|**X** = tensor(float)|
|MurmurHash3|*in* X:**T1**
*out* Y:**T2**|1+|**T1** = tensor(double), tensor(float), tensor(int32), tensor(int64), tensor(string), tensor(uint32), tensor(uint64)
**T2** = tensor(int32), tensor(uint32)|
|NGramRepeatBlock|*in* input_ids:**Tid**
*in* scores:**T**
*out* scores_out:**T**|1+|**T** = tensor(float)
**Tid** = tensor(int64)|
-|NhwcMaxPool|*in* x:**T**
*out* y:**T**|1+|**T** = tensor(uint8)|
+|NhwcMaxPool|*in* x:**T**
*out* y:**T**|1+|**T** = tensor(int8), tensor(uint8)|
|Pad|*in* data:**T**
*in* pads:**tensor(int64)**
*in* value:**T**
*out* output:**T**|1+|**T** = tensor(float)|
|QAttention|*in* input:**T1**
*in* weight:**T2**
*in* bias:**T3**
*in* input_scale:**T3**
*in* weight_scale:**T3**
*in* mask_index:**T4**
*in* input_zero_point:**T1**
*in* weight_zero_point:**T2**
*in* past:**T3**
*out* output:**T3**
*out* present:**T3**|1+|**T1** = tensor(uint8)
**T2** = tensor(int8), tensor(uint8)
**T3** = tensor(float)
**T4** = tensor(int32)|
|QEmbedLayerNormalization|*in* input_ids:**T1**
*in* segment_ids:**T1**
*in* word_embedding_quant:**T2**
*in* position_embedding_quant:**T2**
*in* segment_embedding:**T2**
*in* gamma_quant:**T2**
*in* beta_quant:**T2**
*in* mask:**T1**
*in* word_embedding_scale:**T**
*in* position_embedding_scale:**T**
*in* segment_embedding_scale:**T**
*in* gamma_scale:**T**
*in* beta_scale:**T**
*in* word_embedding_zero_point:**T2**
*in* position_embedding_zero_point:**T2**
*in* segment_embedding_zero_point:**T2**
*in* gamma_zero_point:**T2**
*in* beta_zero_point:**T2**
*out* layernorm_out:**T**
*out* mask_index_out:**T1**|1+|**T** = tensor(float)|
diff --git a/onnxruntime/contrib_ops/cpu/cpu_contrib_kernels.cc b/onnxruntime/contrib_ops/cpu/cpu_contrib_kernels.cc
index 702b5bcd67..ebf2c6c65d 100644
--- a/onnxruntime/contrib_ops/cpu/cpu_contrib_kernels.cc
+++ b/onnxruntime/contrib_ops/cpu/cpu_contrib_kernels.cc
@@ -72,7 +72,8 @@ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1,
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, MatMulIntegerToFloat);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, DynamicQuantizeLSTM);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, QLinearConv);
-class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, NhwcMaxPool);
+class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, int8_t, NhwcMaxPool);
+class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, NhwcMaxPool);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, QEmbedLayerNormalization);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, QGemm);
// ******** End: Quantization ******************* //
@@ -160,7 +161,8 @@ Status RegisterQuantizationKernels(KernelRegistry& kernel_registry) {
BuildKernelCreateInfo,
BuildKernelCreateInfo,
BuildKernelCreateInfo,
- BuildKernelCreateInfo,
+ BuildKernelCreateInfo,
+ BuildKernelCreateInfo,
BuildKernelCreateInfo,
BuildKernelCreateInfo,
};
diff --git a/onnxruntime/contrib_ops/cpu/qlinear_global_average_pool.cc b/onnxruntime/contrib_ops/cpu/qlinear_global_average_pool.cc
index d8314d4cfd..ee63c23869 100644
--- a/onnxruntime/contrib_ops/cpu/qlinear_global_average_pool.cc
+++ b/onnxruntime/contrib_ops/cpu/qlinear_global_average_pool.cc
@@ -14,13 +14,14 @@ using onnxruntime::concurrency::ThreadPool;
namespace onnxruntime {
namespace contrib {
+template
Status ComputeQLinearGlobalAvgPool(
- const uint8_t* x,
+ const T8Bits* x,
float x_scale,
- uint8_t x_zero_point,
- uint8_t* y,
+ T8Bits x_zero_point,
+ T8Bits* y,
float y_scale,
- uint8_t y_zero_point,
+ T8Bits y_zero_point,
int64_t N,
int64_t C,
int64_t image_size,
@@ -28,8 +29,8 @@ Status ComputeQLinearGlobalAvgPool(
concurrency::ThreadPool* tp) {
if (!channels_last || C == 1) {
auto worker = [=](std::ptrdiff_t first, std::ptrdiff_t last) {
- const uint8_t* input = (const uint8_t*)(x + (first * image_size));
- uint8_t* output = (uint8_t*)(y + first);
+ const T8Bits* input = (const T8Bits*)(x + (first * image_size));
+ T8Bits* output = (T8Bits*)(y + first);
std::vector acc_buffer(MlasQLinearSafePaddingElementCount(sizeof(int32_t), last - first));
MlasQLinearGlobalAveragePoolNchw(input, x_scale, x_zero_point, output, y_scale, y_zero_point, last - first, image_size, acc_buffer.data());
};
@@ -37,17 +38,17 @@ Status ComputeQLinearGlobalAvgPool(
tp, static_cast(N * C), {1.0 * image_size, 1.0, 8.0 * image_size}, worker);
} else {
auto worker = [=](std::ptrdiff_t first, std::ptrdiff_t last) {
- const uint8_t* input = x + first * C * image_size;
- uint8_t* output = y + first * C;
+ const T8Bits* input = x + first * C * image_size;
+ T8Bits* output = y + first * C;
std::vector acc_buffer(MlasQLinearSafePaddingElementCount(sizeof(int32_t), C));
- std::vector zero_buffer(MlasQLinearSafePaddingElementCount(sizeof(uint8_t), C), 0);
+ std::vector zero_buffer(MlasQLinearSafePaddingElementCount(sizeof(T8Bits), C), 0);
MlasQLinearGlobalAveragePoolNhwc(
input, x_scale, x_zero_point, output, y_scale, y_zero_point,
last - first, image_size, C, C, acc_buffer.data(), zero_buffer.data());
};
concurrency::ThreadPool::TryParallelFor(
tp, static_cast(N),
- {1.0 * image_size * C, 1.0 * C, 8.0 *image_size * C},
+ {1.0 * image_size * C, 1.0 * C, 8.0 * image_size * C},
worker);
}
return Status::OK();
@@ -88,19 +89,25 @@ Status QLinearGlobalAveragePool::Compute(OpKernelContext* context) const {
const float x_scale = *(tensor_x_scale->Data());
const float y_scale = *(tensor_y_scale->Data());
+
auto dtype = X.GetElementType();
- switch (dtype) {
- case ONNX_NAMESPACE::TensorProto_DataType_UINT8:
- return ComputeQLinearGlobalAvgPool(X.Data(), x_scale, *(tensor_x_zero_point->Data()),
- Y.MutableData(), y_scale, *(tensor_y_zero_point->Data()),
- N, C, image_size, channels_last_, tp);
- default:
- ORT_THROW("Unsupported 'dtype' value: ", dtype);
+ if (dtype == ONNX_NAMESPACE::TensorProto_DataType_UINT8) {
+ return ComputeQLinearGlobalAvgPool(X.Data(), x_scale, *(tensor_x_zero_point->Data()),
+ Y.MutableData(), y_scale, *(tensor_y_zero_point->Data()),
+ N, C, image_size, channels_last_, tp);
+ } else {
+ return ComputeQLinearGlobalAvgPool(X.Data(), x_scale, *(tensor_x_zero_point->Data()),
+ Y.MutableData(), y_scale, *(tensor_y_zero_point->Data()),
+ N, C, image_size, channels_last_, tp);
}
}
-ONNX_OPERATOR_KERNEL_EX(QLinearGlobalAveragePool, kMSDomain, 1, kCpuExecutionProvider, KernelDefBuilder(), QLinearGlobalAveragePool);
+ONNX_OPERATOR_KERNEL_EX(QLinearGlobalAveragePool,
+ kMSDomain,
+ 1,
+ kCpuExecutionProvider,
+ KernelDefBuilder(),
+ QLinearGlobalAveragePool);
} // namespace contrib
-
} // namespace onnxruntime
diff --git a/onnxruntime/contrib_ops/cpu/qlinear_global_average_pool.h b/onnxruntime/contrib_ops/cpu/qlinear_global_average_pool.h
index f46ed1c0e4..bc0fca6063 100644
--- a/onnxruntime/contrib_ops/cpu/qlinear_global_average_pool.h
+++ b/onnxruntime/contrib_ops/cpu/qlinear_global_average_pool.h
@@ -21,13 +21,14 @@ class QLinearGlobalAveragePool final : public OpKernel {
bool channels_last_;
};
+template
Status ComputeQLinearGlobalAvgPool(
- const uint8_t* x,
+ const T8Bits* x,
float x_scale,
- uint8_t x_zero_point,
- uint8_t* y,
+ T8Bits x_zero_point,
+ T8Bits* y,
float y_scale,
- uint8_t y_zero_point,
+ T8Bits y_zero_point,
int64_t N,
int64_t C,
int64_t image_size,
diff --git a/onnxruntime/contrib_ops/cpu/qlinear_pool.cc b/onnxruntime/contrib_ops/cpu/qlinear_pool.cc
index 154c4620b3..6953206a70 100644
--- a/onnxruntime/contrib_ops/cpu/qlinear_pool.cc
+++ b/onnxruntime/contrib_ops/cpu/qlinear_pool.cc
@@ -23,19 +23,15 @@ using concurrency::ThreadPool;
namespace contrib {
template
-static inline float dequantize_value(T8Bits x, float x_scale, T8Bits x_zero_point);
-
-template
-static inline T8Bits quantize_value(float y, float y_scale, T8Bits y_zero_point);
-
-template <>
-inline float dequantize_value(uint8_t x, float x_scale, uint8_t x_zero_point) {
+static inline float dequantize_value(T8Bits x, float x_scale, T8Bits x_zero_point) {
return x_scale * (static_cast(x) - x_zero_point);
}
-template <>
-inline uint8_t quantize_value(float y, float y_scale, uint8_t y_zero_point) {
- return static_cast(std::max(0.0f, std::min(std::nearbyintf(y / y_scale + y_zero_point), 255.0f)));
+template
+static inline T8Bits quantize_value(float y, float y_scale, T8Bits y_zero_point) {
+ constexpr int32_t min_8bits = std::numeric_limits::lowest();
+ constexpr int32_t max_8bits = std::numeric_limits::max();
+ return static_cast(std::max(min_8bits, std::min(static_cast(std::nearbyintf(y / y_scale + y_zero_point)), max_8bits)));
}
static void SwitchDimsNchwNhwc(std::vector& dims, bool from_nchw_to_nhwc) {
@@ -509,6 +505,15 @@ void dequantize_array(int64_t N, const T8Bits* input, float scale, T8Bits zero_p
}
Status QLinearAveragePool::Compute(OpKernelContext* context) const {
+ if (is_input_signed_) {
+ return ComputeImpl(context);
+ } else {
+ return ComputeImpl(context);
+ }
+}
+
+template
+Status QLinearAveragePool::ComputeImpl(OpKernelContext* context) const {
const auto tensor_x_scale = context->Input(1);
const auto tensor_x_zero_point = context->Input(2);
const auto tensor_y_scale = context->Input(3);
@@ -524,16 +529,12 @@ Status QLinearAveragePool::Compute(OpKernelContext* context) const {
"input y_zero_point must be a scalar or 1D tensor of size 1 if given");
const auto* X = context->Input(0);
- auto dtype = X->GetElementType();
- if (dtype != ONNX_NAMESPACE::TensorProto_DataType_UINT8) {
- ORT_THROW("Unsupported 'dtype' in QLinear Pooling:", dtype);
- }
TensorShape x_shape = X->Shape();
const float x_scale = *(tensor_x_scale->Data());
const float y_scale = *(tensor_y_scale->Data());
- uint8_t x_zero_point = (tensor_x_zero_point ? *(tensor_x_zero_point->Data()) : (uint8_t)0);
- uint8_t y_zero_point = (tensor_y_zero_point ? *(tensor_y_zero_point->Data()) : (uint8_t)0);
+ T8Bits x_zero_point = (tensor_x_zero_point ? *(tensor_x_zero_point->Data()) : (T8Bits)0);
+ T8Bits y_zero_point = (tensor_y_zero_point ? *(tensor_y_zero_point->Data()) : (T8Bits)0);
ORT_RETURN_IF_NOT(x_shape.NumDimensions() >= 3, "Input dimension cannot be less than 3.");
std::vector pads = pool_attrs_.pads;
@@ -564,8 +565,8 @@ Status QLinearAveragePool::Compute(OpKernelContext* context) const {
SwitchDimsNchwNhwc(output_dims, true);
}
Tensor* Y = context->Output(0, output_dims);
- const auto* X_data = X->Data();
- auto* Y_data = Y->MutableData();
+ const auto* X_data = X->Data();
+ auto* Y_data = Y->MutableData();
ThreadPool* tp = context->GetOperatorThreadPool();
// Check for special case which could fall back to global average pool
@@ -589,12 +590,12 @@ Status QLinearAveragePool::Compute(OpKernelContext* context) const {
switch (kernel_shape.size()) {
case 1: {
if (channels_last_) {
- QLinearPoolNhwc1DTask avg_pool_task_1d = {
+ QLinearPoolNhwc1DTask avg_pool_task_1d = {
x_data_fp32, Y_data, y_scale, y_zero_point, channels,
pooled_height, strides[0], height, kernel_shape, pads, pool_context_, pool_attrs_};
ThreadPool::TryParallelFor(tp, y_image_size * batch_count, avg_pool_task_1d.Cost(), avg_pool_task_1d);
} else {
- QLinearPool1DTask avg_pool_task_1d = {
+ QLinearPool1DTask avg_pool_task_1d = {
x_data_fp32, Y_data, y_scale, y_zero_point, x_image_size, y_image_size,
pooled_height, strides[0], height, kernel_shape, pads, pool_context_, pool_attrs_};
ThreadPool::TryParallelFor(tp, total_channels, avg_pool_task_1d.Cost(), avg_pool_task_1d);
@@ -604,13 +605,13 @@ Status QLinearAveragePool::Compute(OpKernelContext* context) const {
case 2: {
if (channels_last_) {
- QLinearPoolNhwc2DTask avg_pool_task_2d = {
+ QLinearPoolNhwc2DTask avg_pool_task_2d = {
x_data_fp32, Y_data, y_scale, y_zero_point, x_image_size, y_image_size, kernel_size, channels,
pooled_height, pooled_width, strides[0], strides[1], height, width, kernel_shape, pads, pool_context_, pool_attrs_};
ThreadPool::TryParallelFor(tp, y_image_size * batch_count, avg_pool_task_2d.Cost(), avg_pool_task_2d);
} else {
- QLinearPool2DTask avg_pool_task_2d = {
+ QLinearPool2DTask avg_pool_task_2d = {
x_data_fp32, Y_data, y_scale, y_zero_point, x_image_size, y_image_size,
pooled_height, pooled_width, strides[0], strides[1], height, width, kernel_shape, pads, pool_context_, pool_attrs_};
ThreadPool::TryParallelFor(tp, total_channels, avg_pool_task_2d.Cost(), avg_pool_task_2d);
@@ -620,14 +621,14 @@ Status QLinearAveragePool::Compute(OpKernelContext* context) const {
case 3: {
if (channels_last_) {
- QLinearPoolNhwc3DTask avg_pool_task_3d = {
+ QLinearPoolNhwc3DTask avg_pool_task_3d = {
x_data_fp32, Y_data, y_scale, y_zero_point, x_image_size, y_image_size, kernel_size, channels,
pooled_height, pooled_width, pooled_depth, strides[0], strides[1], strides[2], height, width, depth,
kernel_shape, pads, pool_context_, pool_attrs_};
ThreadPool::TryParallelFor(tp, y_image_size * batch_count, avg_pool_task_3d.Cost(), avg_pool_task_3d);
} else {
- QLinearPool3DTask avg_pool_task_3d = {
+ QLinearPool3DTask avg_pool_task_3d = {
x_data_fp32, Y_data, y_scale, y_zero_point, x_image_size, y_image_size,
pooled_height, pooled_width, pooled_depth, strides[0], strides[1], strides[2], height, width, depth,
kernel_shape, pads, pool_context_, pool_attrs_};
@@ -647,7 +648,12 @@ Status QLinearAveragePool::Compute(OpKernelContext* context) const {
return Status::OK();
}
-ONNX_OPERATOR_KERNEL_EX(QLinearAveragePool, kMSDomain, 1, kCpuExecutionProvider, KernelDefBuilder(), QLinearAveragePool);
+ONNX_OPERATOR_KERNEL_EX(QLinearAveragePool,
+ kMSDomain,
+ 1,
+ kCpuExecutionProvider,
+ KernelDefBuilder(),
+ QLinearAveragePool);
} // namespace contrib
diff --git a/onnxruntime/contrib_ops/cpu/qlinear_pool.h b/onnxruntime/contrib_ops/cpu/qlinear_pool.h
index 92285e4f78..ebb6be4a00 100644
--- a/onnxruntime/contrib_ops/cpu/qlinear_pool.h
+++ b/onnxruntime/contrib_ops/cpu/qlinear_pool.h
@@ -14,6 +14,9 @@ class QLinearAveragePool final : public OpKernel, public PoolBase {
public:
QLinearAveragePool(const OpKernelInfo& info) : OpKernel(info), PoolBase(info) {
channels_last_ = (info.GetAttrOrDefault("channels_last", static_cast(0)) != 0);
+
+ int32_t input_type = info.node().InputDefs()[0]->TypeAsProto()->tensor_type().elem_type();
+ is_input_signed_ = ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT8 == input_type;
}
~QLinearAveragePool() override = default;
@@ -21,8 +24,12 @@ class QLinearAveragePool final : public OpKernel, public PoolBase {
Status Compute(OpKernelContext* context) const override;
private:
+ template
+ Status ComputeImpl(OpKernelContext* context) const;
+
PoolProcessContext pool_context_;
bool channels_last_;
+ bool is_input_signed_;
};
} // namespace contrib
diff --git a/onnxruntime/contrib_ops/cpu/quantization/nhwc_max_pool.cc b/onnxruntime/contrib_ops/cpu/quantization/nhwc_max_pool.cc
index 991f5f19e8..a6f297c7b5 100644
--- a/onnxruntime/contrib_ops/cpu/quantization/nhwc_max_pool.cc
+++ b/onnxruntime/contrib_ops/cpu/quantization/nhwc_max_pool.cc
@@ -11,6 +11,7 @@
namespace onnxruntime {
namespace contrib {
+template
class NhwcMaxPool : public OpKernel {
public:
explicit NhwcMaxPool(const OpKernelInfo& info) : OpKernel(info),
@@ -20,10 +21,11 @@ class NhwcMaxPool : public OpKernel {
Status Compute(OpKernelContext* context) const override;
private:
- PoolAttributes pool_attrs_;
+ PoolAttributes pool_attrs_;
};
-Status NhwcMaxPool::Compute(OpKernelContext* context) const {
+template
+Status NhwcMaxPool::Compute(OpKernelContext* context) const {
const auto* X = context->Input(0);
const TensorShape& input_shape = X->Shape();
@@ -73,17 +75,17 @@ Status NhwcMaxPool::Compute(OpKernelContext* context) const {
AllocatorPtr alloc;
ORT_RETURN_IF_ERROR(context->GetTempSpaceAllocator(&alloc));
int64_t col_buffer_batch_count = std::min(output_image_size, output_batch_count);
- auto* col_data = alloc->Alloc(SafeInt(sizeof(const uint8_t*)) * kernel_size * col_buffer_batch_count);
+ auto* col_data = alloc->Alloc(SafeInt(sizeof(const T8Bits*)) * kernel_size * col_buffer_batch_count);
BufferUniquePtr col_buffer(col_data, BufferDeleter(alloc));
- std::vector padding_data(static_cast(C), 0);
+ std::vector padding_data(static_cast(C), std::numeric_limits::lowest());
- const auto* Xdata = X->template Data();
- auto* Ydata = Y->template MutableData();
+ const auto* Xdata = X->template Data();
+ auto* Ydata = Y->template MutableData();
for (int64_t image_id = 0; image_id < N; ++image_id) {
for (int64_t output_start = 0; output_start < output_image_size;) {
int64_t output_count = std::min(output_image_size - output_start, output_batch_count);
- math::Im2col()(
+ math::Im2col()(
Xdata,
C,
input_shape.GetDims().data() + 1,
@@ -95,10 +97,10 @@ Status NhwcMaxPool::Compute(OpKernelContext* context) const {
static_cast(spatial_dims),
output_start,
output_count,
- static_cast(col_buffer.get()),
+ static_cast(col_buffer.get()),
padding_data.data());
MlasMaximumPool(
- static_cast(col_buffer.get()),
+ static_cast(col_buffer.get()),
Ydata,
static_cast(C),
static_cast(output_count),
@@ -114,14 +116,19 @@ Status NhwcMaxPool::Compute(OpKernelContext* context) const {
return Status::OK();
}
-ONNX_OPERATOR_KERNEL_EX(
- NhwcMaxPool,
- kMSDomain,
- 1,
- kCpuExecutionProvider,
- KernelDefBuilder()
- .TypeConstraint("T", DataTypeImpl::GetTensorType()),
- NhwcMaxPool);
+#define REGISTER_NHWCMAXPOOL_TYPED_KERNEL(T) \
+ ONNX_OPERATOR_TYPED_KERNEL_EX( \
+ NhwcMaxPool, \
+ kMSDomain, \
+ 1, \
+ T, \
+ kCpuExecutionProvider, \
+ KernelDefBuilder() \
+ .TypeConstraint("T", DataTypeImpl::GetTensorType()), \
+ NhwcMaxPool);
+
+REGISTER_NHWCMAXPOOL_TYPED_KERNEL(int8_t);
+REGISTER_NHWCMAXPOOL_TYPED_KERNEL(uint8_t);
} // namespace contrib
} // namespace onnxruntime
diff --git a/onnxruntime/core/mlas/inc/mlas.h b/onnxruntime/core/mlas/inc/mlas.h
index 01f22c25b4..d3a5619c5c 100644
--- a/onnxruntime/core/mlas/inc/mlas.h
+++ b/onnxruntime/core/mlas/inc/mlas.h
@@ -786,11 +786,12 @@ MlasPool(
MLAS_THREADPOOL* ThreadPool
);
+template
void
MLASCALL
MlasMaximumPool(
- const uint8_t* const* Input,
- uint8_t* Output,
+ const T8Bits* const* Input,
+ T8Bits* Output,
size_t Channels,
size_t OutputCount,
size_t KernelSize
@@ -1041,17 +1042,18 @@ MlasQuantizeLinear(
* @param CountN
* @return
*/
+template
void
MLASCALL
MlasRequantizeOutput(
const int32_t* Input,
size_t InputLeadingDimension,
- uint8_t* Output,
+ OutputType* Output,
size_t OutputLeadingDimension,
const int32_t* Bias,
const float* Scale,
bool PerColumnScale,
- uint8_t ZeroPoint,
+ OutputType ZeroPoint,
size_t StartM,
size_t StartN,
size_t CountM,
@@ -1115,13 +1117,14 @@ MlasQLinearSafePaddingElementCount(
size_t ElementCount
);
+template
void
MLASCALL
MlasQLinearGlobalAveragePoolNchw(
- const uint8_t* Input,
+ const T8Bits* Input,
float ScaleInput,
int32_t ZeroPointInput,
- uint8_t* Output,
+ T8Bits* Output,
float ScaleOutput,
int32_t ZeroPointOutput,
size_t Channels,
@@ -1129,13 +1132,14 @@ MlasQLinearGlobalAveragePoolNchw(
int32_t* AccumulateBuffer
);
+template
void
MLASCALL
MlasQLinearGlobalAveragePoolNhwc(
- const uint8_t* Input,
+ const T8Bits* Input,
float ScaleInput,
int32_t ZeroPointInput,
- uint8_t* Output,
+ T8Bits* Output,
float ScaleOutput,
int32_t ZeroPointOutput,
size_t Batch,
@@ -1143,7 +1147,7 @@ MlasQLinearGlobalAveragePoolNhwc(
size_t Stride,
size_t Channels,
int32_t* AccumulateBuffer,
- const uint8_t* ZeroBuffer
+ const T8Bits* ZeroBuffer
);
//
diff --git a/onnxruntime/core/mlas/lib/pooling.cpp b/onnxruntime/core/mlas/lib/pooling.cpp
index 0223da1ee6..649e137182 100644
--- a/onnxruntime/core/mlas/lib/pooling.cpp
+++ b/onnxruntime/core/mlas/lib/pooling.cpp
@@ -1304,11 +1304,12 @@ Return Value:
#endif
}
+template
void
MLASCALL
MlasMaximumPool(
- const uint8_t* const* Input,
- uint8_t* Output,
+ const T8Bits* const* Input,
+ T8Bits* Output,
size_t Channels,
size_t OutputCount,
size_t KernelSize
@@ -1352,6 +1353,10 @@ Return Value:
size_t c = Channels;
#if defined(MLAS_SSE2_INTRINSICS)
+ const __m128i BitFlipVector = _mm_set1_epi32(0x80808080);
+ if constexpr (std::is_unsigned::value) {
+ MLAS_UNREFERENCED_PARAMETER(BitFlipVector);
+ }
while (c >= 32) {
@@ -1363,10 +1368,20 @@ Return Value:
__m128i InputVector0 = _mm_loadu_si128((const __m128i*)&Input[k][ChannelOffset]);
__m128i InputVector1 = _mm_loadu_si128((const __m128i*)&Input[k][ChannelOffset + 16]);
+ if constexpr (std::is_signed::value) {
+ InputVector0 = _mm_xor_si128(InputVector0, BitFlipVector);
+ InputVector1 = _mm_xor_si128(InputVector1, BitFlipVector);
+ }
+
MaximumVector0 = _mm_max_epu8(MaximumVector0, InputVector0);
MaximumVector1 = _mm_max_epu8(MaximumVector1, InputVector1);
}
+ if constexpr (std::is_signed::value) {
+ MaximumVector0 = _mm_xor_si128(MaximumVector0, BitFlipVector);
+ MaximumVector1 = _mm_xor_si128(MaximumVector1, BitFlipVector);
+ }
+
_mm_storeu_si128((__m128i*)&Output[0], MaximumVector0);
_mm_storeu_si128((__m128i*)&Output[16], MaximumVector1);
Output += 32;
@@ -1383,9 +1398,17 @@ Return Value:
__m128i InputVector0 = _mm_loadu_si128((const __m128i*)&Input[k][ChannelOffset]);
+ if constexpr (std::is_signed::value){
+ InputVector0 = _mm_xor_si128(InputVector0, BitFlipVector);
+ }
+
MaximumVector0 = _mm_max_epu8(MaximumVector0, InputVector0);
}
+ if constexpr (std::is_signed::value) {
+ MaximumVector0 = _mm_xor_si128(MaximumVector0, BitFlipVector);
+ }
+
_mm_storeu_si128((__m128i*)&Output[0], MaximumVector0);
Output += 16;
@@ -1401,9 +1424,17 @@ Return Value:
__m128i InputVector0 = _mm_loadl_epi64((const __m128i*)&Input[k][ChannelOffset]);
+ if constexpr (std::is_signed::value){
+ InputVector0 = _mm_xor_si128(InputVector0, BitFlipVector);
+ }
+
MaximumVector0 = _mm_max_epu8(MaximumVector0, InputVector0);
}
+ if constexpr (std::is_signed::value) {
+ MaximumVector0 = _mm_xor_si128(MaximumVector0, BitFlipVector);
+ }
+
_mm_storel_epi64((__m128i*)&Output[0], MaximumVector0);
Output += 8;
@@ -1415,20 +1446,40 @@ Return Value:
while (c >= 32) {
- uint8x16_t MaximumVector0 = vdupq_n_u8(0);
- uint8x16_t MaximumVector1 = vdupq_n_u8(0);
+ if constexpr (std::is_signed::value){
- for (size_t k = 0; k < KernelSize; k++) {
+ int8x16_t MaximumVector0 = vdupq_n_s8(-128);
+ int8x16_t MaximumVector1 = vdupq_n_s8(-128);
- uint8x16_t InputVector0 = vld1q_u8(&Input[k][ChannelOffset]);
- uint8x16_t InputVector1 = vld1q_u8(&Input[k][ChannelOffset + 16]);
+ for (size_t k = 0; k < KernelSize; k++) {
- MaximumVector0 = vmaxq_u8(MaximumVector0, InputVector0);
- MaximumVector1 = vmaxq_u8(MaximumVector1, InputVector1);
+ int8x16_t InputVector0 = vld1q_s8(&Input[k][ChannelOffset]);
+ int8x16_t InputVector1 = vld1q_s8(&Input[k][ChannelOffset + 16]);
+
+ MaximumVector0 = vmaxq_s8(MaximumVector0, InputVector0);
+ MaximumVector1 = vmaxq_s8(MaximumVector1, InputVector1);
+ }
+
+ vst1q_s8(&Output[0], MaximumVector0);
+ vst1q_s8(&Output[16], MaximumVector1);
+ } else {
+
+ uint8x16_t MaximumVector0 = vdupq_n_u8(0);
+ uint8x16_t MaximumVector1 = vdupq_n_u8(0);
+
+ for (size_t k = 0; k < KernelSize; k++) {
+
+ uint8x16_t InputVector0 = vld1q_u8(&Input[k][ChannelOffset]);
+ uint8x16_t InputVector1 = vld1q_u8(&Input[k][ChannelOffset + 16]);
+
+ MaximumVector0 = vmaxq_u8(MaximumVector0, InputVector0);
+ MaximumVector1 = vmaxq_u8(MaximumVector1, InputVector1);
+ }
+
+ vst1q_u8(&Output[0], MaximumVector0);
+ vst1q_u8(&Output[16], MaximumVector1);
}
- vst1q_u8(&Output[0], MaximumVector0);
- vst1q_u8(&Output[16], MaximumVector1);
Output += 32;
ChannelOffset += 32;
@@ -1437,16 +1488,30 @@ Return Value:
while (c >= 16) {
- uint8x16_t MaximumVector0 = vdupq_n_u8(0);
+ if constexpr (std::is_signed::value){
- for (size_t k = 0; k < KernelSize; k++) {
+ int8x16_t MaximumVector0 = vdupq_n_s8(-128);
- uint8x16_t InputVector0 = vld1q_u8(&Input[k][ChannelOffset]);
+ for (size_t k = 0; k < KernelSize; k++) {
- MaximumVector0 = vmaxq_u8(MaximumVector0, InputVector0);
+ int8x16_t InputVector0 = vld1q_s8(&Input[k][ChannelOffset]);
+ MaximumVector0 = vmaxq_s8(MaximumVector0, InputVector0);
+ }
+
+ vst1q_s8(&Output[0], MaximumVector0);
+ } else {
+
+ uint8x16_t MaximumVector0 = vdupq_n_u8(0);
+
+ for (size_t k = 0; k < KernelSize; k++) {
+
+ uint8x16_t InputVector0 = vld1q_u8(&Input[k][ChannelOffset]);
+ MaximumVector0 = vmaxq_u8(MaximumVector0, InputVector0);
+ }
+
+ vst1q_u8(&Output[0], MaximumVector0);
}
- vst1q_u8(&Output[0], MaximumVector0);
Output += 16;
ChannelOffset += 16;
@@ -1455,16 +1520,29 @@ Return Value:
if (c >= 8) {
- uint8x8_t MaximumVector0 = vdup_n_u8(0);
+ if constexpr (std::is_signed::value){
- for (size_t k = 0; k < KernelSize; k++) {
+ int8x8_t MaximumVector0 = vdup_n_s8(-128);
- uint8x8_t InputVector0 = vld1_u8(&Input[k][ChannelOffset]);
+ for (size_t k = 0; k < KernelSize; k++) {
- MaximumVector0 = vmax_u8(MaximumVector0, InputVector0);
+ int8x8_t InputVector0 = vld1_s8(&Input[k][ChannelOffset]);
+ MaximumVector0 = vmax_s8(MaximumVector0, InputVector0);
+ }
+
+ vst1_s8(&Output[0], MaximumVector0);
+ } else {
+
+ uint8x8_t MaximumVector0 = vdup_n_u8(0);
+
+ for (size_t k = 0; k < KernelSize; k++) {
+
+ uint8x8_t InputVector0 = vld1_u8(&Input[k][ChannelOffset]);
+ MaximumVector0 = vmax_u8(MaximumVector0, InputVector0);
+ }
+ vst1_u8(&Output[0], MaximumVector0);
}
- vst1_u8(&Output[0], MaximumVector0);
Output += 8;
ChannelOffset += 8;
@@ -1475,13 +1553,13 @@ Return Value:
while (c > 0) {
- int32_t MaximumValue = 0;
+ int32_t MaximumValue = std::numeric_limits::lowest();
for (size_t k = 0; k < KernelSize; k++) {
MaximumValue = std::max(MaximumValue, int32_t(Input[k][ChannelOffset]));
}
- *Output++ = uint8_t(MaximumValue);
+ *Output++ = T8Bits(MaximumValue);
ChannelOffset += 1;
c -= 1;
@@ -1491,3 +1569,25 @@ Return Value:
OutputCount -= 1;
}
}
+
+template
+void
+MLASCALL
+MlasMaximumPool(
+ const int8_t* const* Input,
+ int8_t* Output,
+ size_t Channels,
+ size_t OutputCount,
+ size_t KernelSize
+ );
+
+template
+void
+MLASCALL
+MlasMaximumPool(
+ const uint8_t* const* Input,
+ uint8_t* Output,
+ size_t Channels,
+ size_t OutputCount,
+ size_t KernelSize
+ );
diff --git a/onnxruntime/core/mlas/lib/qlgavgpool.cpp b/onnxruntime/core/mlas/lib/qlgavgpool.cpp
index d8972eecbf..38c777d34b 100644
--- a/onnxruntime/core/mlas/lib/qlgavgpool.cpp
+++ b/onnxruntime/core/mlas/lib/qlgavgpool.cpp
@@ -24,6 +24,7 @@ MlasQLinearSafePaddingElementCount(
)
{
if (!(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8 || ElementSize == 16)) {
+
#ifdef MLAS_NO_EXCEPTION
abort();
#else
@@ -42,6 +43,7 @@ CheckQLinearGlobalAveragePoolScaleAndSize(
)
{
if (ImageSize >= 0x1000000) {
+
#ifdef MLAS_NO_EXCEPTION
abort();
#else
@@ -51,6 +53,7 @@ CheckQLinearGlobalAveragePoolScaleAndSize(
float scale = ScaleInput / (ScaleOutput * static_cast(ImageSize));
if (scale < 0x1.0p-32f || scale >= 256.0f) {
+
// In first case, the scale is too small, ScaleInput/ScaleOutput < 1/256 no matter what ImageSize
// In second case, the scale is too large, ScaleInput/ScaleOutput >= 256 no matter what Image Size
// both case make output value constant, and hence not meaningful.
@@ -65,13 +68,14 @@ CheckQLinearGlobalAveragePoolScaleAndSize(
#if defined(MLAS_NEON_INTRINSICS)
+template
void
MLASCALL
MlasQLinearGlobalAveragePoolNchw(
- const uint8_t* Input,
+ const T8Bits* Input,
float ScaleInput,
int32_t ZeroPointInput,
- uint8_t* Output,
+ T8Bits* Output,
float ScaleOutput,
int32_t ZeroPointOutput,
size_t Channels,
@@ -83,38 +87,65 @@ MlasQLinearGlobalAveragePoolNchw(
int32_t bias[] = {-ZeroPointInput * static_cast(ImageSize), 0, 0, 0};
const int32x4_t vbias = vld1q_s32(bias);
const int32x4_t vzero = vmovq_n_s32(0);
+ const uint8_t* InputU8 = (const uint8_t*)(Input);
int32_t* sum_buffer = AccumulateBuffer;
uint8_t tail_buffer[8] = {0, 0, 0, 0, 0, 0, 0, 0};
for (size_t c = Channels; c > 0; c--) {
+
int32x4_t vacc_lo = vbias;
int32x4_t vacc_hi = vzero;
auto Len = ImageSize;
for (; Len >= 32; Len -= 32) {
- const uint8x8_t vi0 = vld1_u8(Input);
- const uint8x8_t vi1 = vld1_u8(Input + 8);
- const uint8x8_t vi2 = vld1_u8(Input + 16);
- const uint8x8_t vi3 = vld1_u8(Input + 24);
- const uint16x8_t vs01 = vaddl_u8(vi0, vi1);
- const uint16x8_t vs23 = vaddl_u8(vi2, vi3);
- const int16x8_t vsum = vreinterpretq_s16_u16(vaddq_u16(vs01, vs23));
+ const uint8x8_t vi0 = vld1_u8(InputU8);
+ const uint8x8_t vi1 = vld1_u8(InputU8 + 8);
+ const uint8x8_t vi2 = vld1_u8(InputU8 + 16);
+ const uint8x8_t vi3 = vld1_u8(InputU8 + 24);
+
+ int16x8_t vsum;
+ if constexpr (std::is_signed::value) {
+
+ const int16x8_t vs01 = vaddl_s8(vreinterpret_s8_u8(vi0), vreinterpret_s8_u8(vi1));
+ const int16x8_t vs23 = vaddl_s8(vreinterpret_s8_u8(vi2), vreinterpret_s8_u8(vi3));
+ vsum = vaddq_s16(vs01, vs23);
+ } else {
+
+ const uint16x8_t vs01 = vaddl_u8(vi0, vi1);
+ const uint16x8_t vs23 = vaddl_u8(vi2, vi3);
+ vsum = vreinterpretq_s16_u16(vaddq_u16(vs01, vs23));
+ }
+
vacc_lo = vaddw_s16(vacc_lo, vget_low_s16(vsum));
vacc_hi = vaddw_s16(vacc_hi, vget_high_s16(vsum));
- Input += 32;
+ InputU8 += 32;
}
for (; Len >= 8; Len -= 8) {
- const int16x8_t vsum = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(Input)));
+
+ int16x8_t vsum;
+ if constexpr (std::is_signed::value) {
+ vsum = vmovl_s8(vreinterpret_s8_u8(vld1_u8(InputU8)));
+ } else {
+ vsum = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(InputU8)));
+ }
vacc_lo = vaddw_s16(vacc_lo, vget_low_s16(vsum));
vacc_hi = vaddw_s16(vacc_hi, vget_high_s16(vsum));
- Input += 8;
+ InputU8 += 8;
}
+
if (Len > 0) {
- memcpy(tail_buffer, Input, Len);
- const int16x8_t vsum = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(tail_buffer)));
+
+ memcpy(tail_buffer, InputU8, Len);
+ int16x8_t vsum;
+ if constexpr (std::is_signed::value) {
+ vsum = vmovl_s8(vreinterpret_s8_u8(vld1_u8(tail_buffer)));
+ } else {
+ vsum = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(tail_buffer)));
+ }
+
vacc_lo = vaddw_s16(vacc_lo, vget_low_s16(vsum));
vacc_hi = vaddw_s16(vacc_hi, vget_high_s16(vsum));
- Input += Len;
+ InputU8 += Len;
}
vacc_lo = vaddq_s32(vacc_lo, vacc_hi);
@@ -123,23 +154,24 @@ MlasQLinearGlobalAveragePoolNchw(
}
MlasRequantizeOutput(AccumulateBuffer, Channels, Output, Channels, nullptr, &scale, false,
- static_cast(ZeroPointOutput), 0, 0, 1, Channels);
+ static_cast(ZeroPointOutput), 0, 0, 1, Channels);
}
+template
MLAS_FORCEINLINE
void
MlasQLinearGlobalAveragePoolNhwcSingleBatch(
- const uint8_t* Input,
- uint8_t* Output,
- const uint8_t* LastOf8,
+ const T8Bits* Input,
+ T8Bits* Output,
+ const T8Bits* LastOf8,
size_t ImageSize,
size_t Channels,
size_t Stride,
int32_t Bias,
float Scale,
- uint8_t Output_zero_point,
+ T8Bits Output_zero_point,
int32_t* AccumulateBuffer,
- const uint8_t* ZeroBuffer
+ const T8Bits* ZeroBuffer
)
{
#define LOAD_FULL_CHANNELS() \
@@ -158,24 +190,35 @@ MlasQLinearGlobalAveragePoolNhwcSingleBatch(
const uint8x8_t vi6 = vld1_u8(i6); \
i6 += 8
-#define CALCULATE_ACCUMULATE_VECTORS() \
- int32x4_t vacc_lo = finish_one_pass ? vld1q_s32(acc) : vbias; \
- int32x4_t vacc_hi = finish_one_pass ? vld1q_s32(acc + 4) : vbias; \
- const uint16x8_t vsum01 = vaddl_u8(vi0, vi1); \
- const uint16x8_t vsum23 = vaddl_u8(vi2, vi3); \
- const uint16x8_t vsum45 = vaddl_u8(vi4, vi5); \
- const uint16x8_t vsum016 = vaddw_u8(vsum01, vi6); \
- const uint16x8_t vsum2345 = vaddq_u16(vsum23, vsum45); \
- const int16x8_t vsum = vreinterpretq_s16_u16(vaddq_u16(vsum016, vsum2345)); \
- vacc_lo = vaddw_s16(vacc_lo, vget_low_s16(vsum)); \
+#define CALCULATE_ACCUMULATE_VECTORS() \
+ int32x4_t vacc_lo = finish_one_pass ? vld1q_s32(acc) : vbias; \
+ int32x4_t vacc_hi = finish_one_pass ? vld1q_s32(acc + 4) : vbias; \
+ int16x8_t vsum; \
+ if constexpr (std::is_signed::value) { \
+ const int16x8_t vsum01 = vaddl_s8(vreinterpret_s8_u8(vi0), vreinterpret_s8_u8(vi1)); \
+ const int16x8_t vsum23 = vaddl_s8(vreinterpret_s8_u8(vi2), vreinterpret_s8_u8(vi3)); \
+ const int16x8_t vsum45 = vaddl_s8(vreinterpret_s8_u8(vi4), vreinterpret_s8_u8(vi5)); \
+ const int16x8_t vsum016 = vaddw_s8(vsum01, vreinterpret_s8_u8(vi6)); \
+ const int16x8_t vsum2345 = vaddq_s16(vsum23, vsum45); \
+ vsum = vaddq_s16(vsum016, vsum2345); \
+ } else { \
+ const uint16x8_t vsum01 = vaddl_u8(vi0, vi1); \
+ const uint16x8_t vsum23 = vaddl_u8(vi2, vi3); \
+ const uint16x8_t vsum45 = vaddl_u8(vi4, vi5); \
+ const uint16x8_t vsum016 = vaddw_u8(vsum01, vi6); \
+ const uint16x8_t vsum2345 = vaddq_u16(vsum23, vsum45); \
+ vsum = vreinterpretq_s16_u16(vaddq_u16(vsum016, vsum2345)); \
+ } \
+ vacc_lo = vaddw_s16(vacc_lo, vget_low_s16(vsum)); \
vacc_hi = vaddw_s16(vacc_hi, vget_high_s16(vsum))
- uint8_t tail[8] = { 0, 0, 0, 0, 0, 0, 0, 0 };
+ uint8_t tail[8] = {0, 0, 0, 0, 0, 0, 0, 0};
const int32x4_t vbias = vld1q_dup_s32(&Bias);
bool finish_one_pass = false;
const size_t step_next_group = 7 * Stride - (Channels & ~size_t{7});
- const uint8_t* i0 = Input;
+ const uint8_t* LastOf8U8 = (const uint8_t*)LastOf8;
+ const uint8_t* i0 = (const uint8_t*)Input;
const uint8_t* i1 = i0 + Stride;
const uint8_t* i4 = i0 + Stride * 4;
const uint8_t* i2 = i1 + Stride;
@@ -184,9 +227,11 @@ MlasQLinearGlobalAveragePoolNhwcSingleBatch(
const uint8_t* i6 = i5 + Stride;
for (; ImageSize > 7; ImageSize -= 7) {
+
int32_t* acc = AccumulateBuffer;
size_t c = Channels;
for (; c >= 8; c -= 8) {
+
LOAD_FULL_CHANNELS();
CALCULATE_ACCUMULATE_VECTORS();
@@ -196,13 +241,14 @@ MlasQLinearGlobalAveragePoolNhwcSingleBatch(
acc += 8;
}
if (c > 0) {
- const uint8x8_t vi0 = vld1_u8(((i0 >= LastOf8) ? (const uint8_t*)memcpy(tail, i0, c) : i0));
- const uint8x8_t vi1 = vld1_u8(((i1 >= LastOf8) ? (const uint8_t*)memcpy(tail, i1, c) : i1));
- const uint8x8_t vi2 = vld1_u8(((i2 >= LastOf8) ? (const uint8_t*)memcpy(tail, i2, c) : i2));
- const uint8x8_t vi3 = vld1_u8(((i3 >= LastOf8) ? (const uint8_t*)memcpy(tail, i3, c) : i3));
- const uint8x8_t vi4 = vld1_u8(((i4 >= LastOf8) ? (const uint8_t*)memcpy(tail, i4, c) : i4));
- const uint8x8_t vi5 = vld1_u8(((i5 >= LastOf8) ? (const uint8_t*)memcpy(tail, i5, c) : i5));
- const uint8x8_t vi6 = vld1_u8(((i6 >= LastOf8) ? (const uint8_t*)memcpy(tail, i6, c) : i6));
+
+ const uint8x8_t vi0 = vld1_u8(((i0 >= LastOf8U8) ? (const uint8_t*)memcpy(tail, i0, c) : i0));
+ const uint8x8_t vi1 = vld1_u8(((i1 >= LastOf8U8) ? (const uint8_t*)memcpy(tail, i1, c) : i1));
+ const uint8x8_t vi2 = vld1_u8(((i2 >= LastOf8U8) ? (const uint8_t*)memcpy(tail, i2, c) : i2));
+ const uint8x8_t vi3 = vld1_u8(((i3 >= LastOf8U8) ? (const uint8_t*)memcpy(tail, i3, c) : i3));
+ const uint8x8_t vi4 = vld1_u8(((i4 >= LastOf8U8) ? (const uint8_t*)memcpy(tail, i4, c) : i4));
+ const uint8x8_t vi5 = vld1_u8(((i5 >= LastOf8U8) ? (const uint8_t*)memcpy(tail, i5, c) : i5));
+ const uint8x8_t vi6 = vld1_u8(((i6 >= LastOf8U8) ? (const uint8_t*)memcpy(tail, i6, c) : i6));
CALCULATE_ACCUMULATE_VECTORS();
@@ -221,19 +267,28 @@ MlasQLinearGlobalAveragePoolNhwcSingleBatch(
}
if (ImageSize > 0) {
+
switch (ImageSize) {
- case 1: i1 = ZeroBuffer; /* fall through */
- case 2: i2 = ZeroBuffer; /* fall through */
- case 3: i3 = ZeroBuffer; /* fall through */
- case 4: i4 = ZeroBuffer; /* fall through */
- case 5: i5 = ZeroBuffer; /* fall through */
- case 6: i6 = ZeroBuffer; /* fall through */
- default: break;
+ case 1:
+ i1 = (const uint8_t*)ZeroBuffer; /* fall through */
+ case 2:
+ i2 = (const uint8_t*)ZeroBuffer; /* fall through */
+ case 3:
+ i3 = (const uint8_t*)ZeroBuffer; /* fall through */
+ case 4:
+ i4 = (const uint8_t*)ZeroBuffer; /* fall through */
+ case 5:
+ i5 = (const uint8_t*)ZeroBuffer; /* fall through */
+ case 6:
+ i6 = (const uint8_t*)ZeroBuffer; /* fall through */
+ default:
+ break;
}
int32_t* acc = AccumulateBuffer;
size_t c = Channels;
for (; c >= 8; c -= 8) {
+
LOAD_FULL_CHANNELS();
CALCULATE_ACCUMULATE_VECTORS();
@@ -244,13 +299,21 @@ MlasQLinearGlobalAveragePoolNhwcSingleBatch(
}
if (c > 0) {
- const uint8x8_t vi0 = vld1_u8(((i0 >= LastOf8) ? (const uint8_t*)memcpy(tail, i0, c) : i0));
- const uint8x8_t vi1 = vld1_u8(((1 < ImageSize && i1 >= LastOf8) ? (const uint8_t*)memcpy(tail, i1, c) : i1));
- const uint8x8_t vi2 = vld1_u8(((2 < ImageSize && i2 >= LastOf8) ? (const uint8_t*)memcpy(tail, i2, c) : i2));
- const uint8x8_t vi3 = vld1_u8(((3 < ImageSize && i3 >= LastOf8) ? (const uint8_t*)memcpy(tail, i3, c) : i3));
- const uint8x8_t vi4 = vld1_u8(((4 < ImageSize && i4 >= LastOf8) ? (const uint8_t*)memcpy(tail, i4, c) : i4));
- const uint8x8_t vi5 = vld1_u8(((5 < ImageSize && i5 >= LastOf8) ? (const uint8_t*)memcpy(tail, i5, c) : i5));
- const uint8x8_t vi6 = vld1_u8(((6 < ImageSize && i6 >= LastOf8) ? (const uint8_t*)memcpy(tail, i6, c) : i6));
+
+ const uint8x8_t vi0 =
+ vld1_u8(((i0 >= LastOf8U8) ? (const uint8_t*)memcpy(tail, i0, c) : i0));
+ const uint8x8_t vi1 = vld1_u8(
+ ((1 < ImageSize && i1 >= LastOf8U8) ? (const uint8_t*)memcpy(tail, i1, c) : i1));
+ const uint8x8_t vi2 = vld1_u8(
+ ((2 < ImageSize && i2 >= LastOf8U8) ? (const uint8_t*)memcpy(tail, i2, c) : i2));
+ const uint8x8_t vi3 = vld1_u8(
+ ((3 < ImageSize && i3 >= LastOf8U8) ? (const uint8_t*)memcpy(tail, i3, c) : i3));
+ const uint8x8_t vi4 = vld1_u8(
+ ((4 < ImageSize && i4 >= LastOf8U8) ? (const uint8_t*)memcpy(tail, i4, c) : i4));
+ const uint8x8_t vi5 = vld1_u8(
+ ((5 < ImageSize && i5 >= LastOf8U8) ? (const uint8_t*)memcpy(tail, i5, c) : i5));
+ const uint8x8_t vi6 = vld1_u8(
+ ((6 < ImageSize && i6 >= LastOf8U8) ? (const uint8_t*)memcpy(tail, i6, c) : i6));
CALCULATE_ACCUMULATE_VECTORS();
@@ -264,13 +327,13 @@ MlasQLinearGlobalAveragePoolNhwcSingleBatch(
#elif defined(MLAS_SSE2_INTRINSICS)
-void
-MLASCALL
+template
+void MLASCALL
MlasQLinearGlobalAveragePoolNchw(
- const uint8_t* Input,
+ const T8Bits* Input,
float ScaleInput,
int32_t ZeroPointInput,
- uint8_t* Output,
+ T8Bits* Output,
float ScaleOutput,
int32_t ZeroPointOutput,
size_t Channels,
@@ -286,36 +349,73 @@ MlasQLinearGlobalAveragePoolNchw(
int32_t* sum_buffer = AccumulateBuffer;
for (size_t c = Channels; c > 0; c--) {
+
__m128i vacc_lo = vbias;
__m128i vacc_hi = vzero;
auto Len = ImageSize;
for (; Len >= 32; Len -= 32) {
+
const __m128i vi0 = _mm_loadl_epi64((const __m128i*)Input);
const __m128i vi1 = _mm_loadl_epi64((const __m128i*)(Input + 8));
const __m128i vi2 = _mm_loadl_epi64((const __m128i*)(Input + 16));
const __m128i vi3 = _mm_loadl_epi64((const __m128i*)(Input + 24));
- const __m128i vxi0 = _mm_unpacklo_epi8(vi0, vzero);
- const __m128i vxi1 = _mm_unpacklo_epi8(vi1, vzero);
- const __m128i vxi2 = _mm_unpacklo_epi8(vi2, vzero);
- const __m128i vxi3 = _mm_unpacklo_epi8(vi3, vzero);
+ if constexpr (std::is_signed::value) {
+
+ const __m128i vxi0 = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, vi0), 8);
+ const __m128i vxi1 = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, vi1), 8);
+ const __m128i vxi2 = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, vi2), 8);
+ const __m128i vxi3 = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, vi3), 8);
+ const __m128i vsum = _mm_add_epi16(_mm_add_epi16(vxi0, vxi1),
+ _mm_add_epi16(vxi2, vxi3));
+ vacc_lo = _mm_add_epi32(vacc_lo, _mm_srai_epi32(_mm_unpacklo_epi16(vzero, vsum), 16));
+ vacc_hi = _mm_add_epi32(vacc_hi, _mm_srai_epi32(_mm_unpackhi_epi16(vzero, vsum), 16));
+ } else {
+
+ const __m128i vxi0 = _mm_unpacklo_epi8(vi0, vzero);
+ const __m128i vxi1 = _mm_unpacklo_epi8(vi1, vzero);
+ const __m128i vxi2 = _mm_unpacklo_epi8(vi2, vzero);
+ const __m128i vxi3 = _mm_unpacklo_epi8(vi3, vzero);
+ const __m128i vsum = _mm_add_epi16(_mm_add_epi16(vxi0, vxi1),
+ _mm_add_epi16(vxi2, vxi3));
+ vacc_lo = _mm_add_epi32(vacc_lo, _mm_unpacklo_epi16(vsum, vzero));
+ vacc_hi = _mm_add_epi32(vacc_hi, _mm_unpackhi_epi16(vsum, vzero));
+ }
- const __m128i vsum = _mm_add_epi16(_mm_add_epi16(vxi0, vxi1), _mm_add_epi16(vxi2, vxi3));
- vacc_lo = _mm_add_epi32(vacc_lo, _mm_unpacklo_epi16(vsum, vzero));
- vacc_hi = _mm_add_epi32(vacc_hi, _mm_unpackhi_epi16(vsum, vzero));
Input += 32;
}
for (; Len >= 8; Len -= 8) {
- const __m128i vsum = _mm_unpacklo_epi8(_mm_loadl_epi64((const __m128i*)Input), vzero);
- vacc_lo = _mm_add_epi32(vacc_lo, _mm_unpacklo_epi16(vsum, vzero));
- vacc_hi = _mm_add_epi32(vacc_hi, _mm_unpackhi_epi16(vsum, vzero));
+
+ if constexpr (std::is_signed::value) {
+
+ const __m128i vsum = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, _mm_loadl_epi64((const __m128i*)Input)), 8);
+ vacc_lo = _mm_add_epi32(vacc_lo, _mm_srai_epi32(_mm_unpacklo_epi16(vzero, vsum), 16));
+ vacc_hi = _mm_add_epi32(vacc_hi, _mm_srai_epi32(_mm_unpackhi_epi16(vzero, vsum), 16));
+ } else {
+
+ const __m128i vsum = _mm_unpacklo_epi8(_mm_loadl_epi64((const __m128i*)Input), vzero);
+ vacc_lo = _mm_add_epi32(vacc_lo, _mm_unpacklo_epi16(vsum, vzero));
+ vacc_hi = _mm_add_epi32(vacc_hi, _mm_unpackhi_epi16(vsum, vzero));
+ }
+
Input += 8;
}
if (Len > 0) {
+
memcpy(buffer, Input, Len);
- const __m128i vsum = _mm_unpacklo_epi8(_mm_loadl_epi64((const __m128i*)buffer), vzero);
- vacc_lo = _mm_add_epi32(vacc_lo, _mm_unpacklo_epi16(vsum, vzero));
- vacc_hi = _mm_add_epi32(vacc_hi, _mm_unpackhi_epi16(vsum, vzero));
+
+ if constexpr (std::is_signed::value) {
+
+ const __m128i vsum = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, _mm_loadl_epi64((const __m128i*)buffer)), 8);
+ vacc_lo = _mm_add_epi32(vacc_lo, _mm_srai_epi32(_mm_unpacklo_epi16(vzero, vsum), 16));
+ vacc_hi = _mm_add_epi32(vacc_hi, _mm_srai_epi32(_mm_unpackhi_epi16(vzero, vsum), 16));
+ } else {
+
+ const __m128i vsum = _mm_unpacklo_epi8(_mm_loadl_epi64((const __m128i*)buffer), vzero);
+ vacc_lo = _mm_add_epi32(vacc_lo, _mm_unpacklo_epi16(vsum, vzero));
+ vacc_hi = _mm_add_epi32(vacc_hi, _mm_unpackhi_epi16(vsum, vzero));
+ }
+
Input += Len;
}
@@ -326,24 +426,27 @@ MlasQLinearGlobalAveragePoolNchw(
vsums = _mm_add_epi32(vsums, vshuf);
*sum_buffer++ = _mm_cvtsi128_si32(vsums);
}
+
MlasRequantizeOutput(AccumulateBuffer, Channels, Output, Channels, nullptr, &scale, false,
- static_cast(ZeroPointOutput), 0, 0, 1, Channels);
+ static_cast(ZeroPointOutput), 0, 0, 1, Channels);
}
+template
MLAS_FORCEINLINE
void
MlasQLinearGlobalAveragePoolNhwcSingleBatch(
- const uint8_t* Input,
- uint8_t* Output,
- const uint8_t* LastOf8,
+ const T8Bits* Input,
+ T8Bits* Output,
+ const T8Bits* LastOf8,
size_t ImageSize,
size_t Channels,
size_t Stride,
int32_t Bias,
float Scale,
- uint8_t Output_zero_point,
+ T8Bits Output_zero_point,
int32_t* AccumulateBuffer,
- const uint8_t* ZeroBuffer)
+ const T8Bits* ZeroBuffer
+ )
{
#if defined(MLAS_TARGET_IX86)
@@ -359,23 +462,39 @@ MlasQLinearGlobalAveragePoolNhwcSingleBatch(
const __m128i vi3 = _mm_loadl_epi64((const __m128i*)i3); \
i3 += 8;
-#define CALCULATE_ACCUMULATE_VECTORS() \
- __m128i vacc_lo = finish_one_pass ? _mm_loadu_si128((__m128i*)acc) : vbias; \
- __m128i vacc_hi = finish_one_pass ? _mm_loadu_si128(((__m128i*)acc) + 1) : vbias; \
- const __m128i vxi0 = _mm_unpacklo_epi8(vi0, vzero); \
- const __m128i vxi1 = _mm_unpacklo_epi8(vi1, vzero); \
- const __m128i vxi2 = _mm_unpacklo_epi8(vi2, vzero); \
- const __m128i vxi3 = _mm_unpacklo_epi8(vi3, vzero); \
- const __m128i vsum01 = _mm_add_epi16(vxi0, vxi1); \
- const __m128i vsum23 = _mm_add_epi16(vxi2, vxi3); \
- const __m128i vsum = _mm_add_epi16(vsum01, vsum23); \
- vacc_lo = _mm_add_epi32(vacc_lo, _mm_unpacklo_epi16(vsum, vzero)); \
- vacc_hi = _mm_add_epi32(vacc_hi, _mm_unpackhi_epi16(vsum, vzero))
+#define CALCULATE_ACCUMULATE_VECTORS() \
+ __m128i vacc_lo = finish_one_pass ? _mm_loadu_si128((__m128i*)acc) : vbias; \
+ __m128i vacc_hi = finish_one_pass ? _mm_loadu_si128(((__m128i*)acc) + 1) : vbias; \
+ __m128i vxi0; \
+ __m128i vxi1; \
+ __m128i vxi2; \
+ __m128i vxi3; \
+ if constexpr (std::is_signed::value) { \
+ vxi0 = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, vi0), 8); \
+ vxi1 = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, vi1), 8); \
+ vxi2 = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, vi2), 8); \
+ vxi3 = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, vi3), 8); \
+ } else { \
+ vxi0 = _mm_unpacklo_epi8(vi0, vzero); \
+ vxi1 = _mm_unpacklo_epi8(vi1, vzero); \
+ vxi2 = _mm_unpacklo_epi8(vi2, vzero); \
+ vxi3 = _mm_unpacklo_epi8(vi3, vzero); \
+ } \
+ __m128i vsum01 = _mm_add_epi16(vxi0, vxi1); \
+ __m128i vsum23 = _mm_add_epi16(vxi2, vxi3); \
+ __m128i vsum = _mm_add_epi16(vsum01, vsum23); \
+ \
+ if constexpr (std::is_signed::value) { \
+ vacc_lo = _mm_add_epi32(vacc_lo, _mm_srai_epi32(_mm_unpacklo_epi16(vzero, vsum), 16)); \
+ vacc_hi = _mm_add_epi32(vacc_hi, _mm_srai_epi32(_mm_unpackhi_epi16(vzero, vsum), 16)); \
+ } else { \
+ vacc_lo = _mm_add_epi32(vacc_lo, _mm_unpacklo_epi16(vsum, vzero)); \
+ vacc_hi = _mm_add_epi32(vacc_hi, _mm_unpackhi_epi16(vsum, vzero)); \
+ }
#else
constexpr size_t PixelsPerIteration = 7;
-
#define LOAD_FULL_CHANNELS() \
const __m128i vi0 = _mm_loadl_epi64((const __m128i*)i0); \
i0 += 8; \
@@ -392,47 +511,71 @@ MlasQLinearGlobalAveragePoolNhwcSingleBatch(
const __m128i vi6 = _mm_loadl_epi64((const __m128i*)i6); \
i6 += 8
-#define CALCULATE_ACCUMULATE_VECTORS() \
- __m128i vacc_lo = finish_one_pass ? _mm_loadu_si128((__m128i*)acc) : vbias; \
- __m128i vacc_hi = finish_one_pass ? _mm_loadu_si128(((__m128i*)acc) + 1) : vbias; \
- const __m128i vxi0 = _mm_unpacklo_epi8(vi0, vzero); \
- const __m128i vxi1 = _mm_unpacklo_epi8(vi1, vzero); \
- const __m128i vxi2 = _mm_unpacklo_epi8(vi2, vzero); \
- const __m128i vxi3 = _mm_unpacklo_epi8(vi3, vzero); \
- const __m128i vxi4 = _mm_unpacklo_epi8(vi4, vzero); \
- const __m128i vxi5 = _mm_unpacklo_epi8(vi5, vzero); \
- const __m128i vxi6 = _mm_unpacklo_epi8(vi6, vzero); \
- const __m128i vsum01 = _mm_add_epi16(vxi0, vxi1); \
- const __m128i vsum23 = _mm_add_epi16(vxi2, vxi3); \
- const __m128i vsum45 = _mm_add_epi16(vxi4, vxi5); \
- const __m128i vsum016 = _mm_add_epi16(vsum01, vxi6); \
- const __m128i vsum2345 = _mm_add_epi16(vsum23, vsum45); \
- const __m128i vsum = _mm_add_epi16(vsum016, vsum2345); \
- vacc_lo = _mm_add_epi32(vacc_lo, _mm_unpacklo_epi16(vsum, vzero)); \
- vacc_hi = _mm_add_epi32(vacc_hi, _mm_unpackhi_epi16(vsum, vzero))
+#define CALCULATE_ACCUMULATE_VECTORS() \
+ __m128i vacc_lo = finish_one_pass ? _mm_loadu_si128((__m128i*)acc) : vbias; \
+ __m128i vacc_hi = finish_one_pass ? _mm_loadu_si128(((__m128i*)acc) + 1) : vbias; \
+ __m128i vxi0; \
+ __m128i vxi1; \
+ __m128i vxi2; \
+ __m128i vxi3; \
+ __m128i vxi4; \
+ __m128i vxi5; \
+ __m128i vxi6; \
+ if constexpr (std::is_signed::value) { \
+ vxi0 = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, vi0), 8); \
+ vxi1 = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, vi1), 8); \
+ vxi2 = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, vi2), 8); \
+ vxi3 = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, vi3), 8); \
+ vxi4 = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, vi4), 8); \
+ vxi5 = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, vi5), 8); \
+ vxi6 = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, vi6), 8); \
+ } else { \
+ vxi0 = _mm_unpacklo_epi8(vi0, vzero); \
+ vxi1 = _mm_unpacklo_epi8(vi1, vzero); \
+ vxi2 = _mm_unpacklo_epi8(vi2, vzero); \
+ vxi3 = _mm_unpacklo_epi8(vi3, vzero); \
+ vxi4 = _mm_unpacklo_epi8(vi4, vzero); \
+ vxi5 = _mm_unpacklo_epi8(vi5, vzero); \
+ vxi6 = _mm_unpacklo_epi8(vi6, vzero); \
+ } \
+ const __m128i vsum01 = _mm_add_epi16(vxi0, vxi1); \
+ const __m128i vsum23 = _mm_add_epi16(vxi2, vxi3); \
+ const __m128i vsum45 = _mm_add_epi16(vxi4, vxi5); \
+ const __m128i vsum016 = _mm_add_epi16(vsum01, vxi6); \
+ const __m128i vsum2345 = _mm_add_epi16(vsum23, vsum45); \
+ const __m128i vsum = _mm_add_epi16(vsum016, vsum2345); \
+ if constexpr (std::is_signed::value) { \
+ vacc_lo = _mm_add_epi32(vacc_lo, _mm_srai_epi32(_mm_unpacklo_epi16(vzero, vsum), 16)); \
+ vacc_hi = _mm_add_epi32(vacc_hi, _mm_srai_epi32(_mm_unpackhi_epi16(vzero, vsum), 16)); \
+ } else { \
+ vacc_lo = _mm_add_epi32(vacc_lo, _mm_unpacklo_epi16(vsum, vzero)); \
+ vacc_hi = _mm_add_epi32(vacc_hi, _mm_unpackhi_epi16(vsum, vzero)); \
+ }
#endif
- uint8_t tail[8] = { 0, 0, 0, 0, 0, 0, 0, 0 };
+ T8Bits tail[8] = {0, 0, 0, 0, 0, 0, 0, 0};
bool finish_one_pass = false;
const __m128i vbias = _mm_set1_epi32(Bias);
const __m128i vzero = _mm_setzero_si128();
size_t step_next_group = PixelsPerIteration * Stride - (Channels & ~size_t{7});
- const uint8_t* i0 = Input;
- const uint8_t* i1 = i0 + Stride;
- const uint8_t* i2 = i1 + Stride;
- const uint8_t* i3 = i2 + Stride;
+ const T8Bits* i0 = Input;
+ const T8Bits* i1 = i0 + Stride;
+ const T8Bits* i2 = i1 + Stride;
+ const T8Bits* i3 = i2 + Stride;
#if !defined(MLAS_TARGET_IX86)
- const uint8_t* i4 = i0 + Stride * 4;
- const uint8_t* i5 = i4 + Stride;
- const uint8_t* i6 = i5 + Stride;
+ const T8Bits* i4 = i0 + Stride * 4;
+ const T8Bits* i5 = i4 + Stride;
+ const T8Bits* i6 = i5 + Stride;
#endif
for (; ImageSize > PixelsPerIteration; ImageSize -= PixelsPerIteration) {
+
int32_t* acc = AccumulateBuffer;
size_t c = Channels;
for (; c >= 8; c -= 8) {
+
LOAD_FULL_CHANNELS();
CALCULATE_ACCUMULATE_VECTORS();
@@ -442,14 +585,21 @@ MlasQLinearGlobalAveragePoolNhwcSingleBatch(
acc += 8;
}
if (c > 0) {
- const __m128i vi0 = _mm_loadl_epi64((const __m128i*)(i0 >= LastOf8 ? memcpy(tail, i0, c) : i0));
- const __m128i vi1 = _mm_loadl_epi64((const __m128i*)(i1 >= LastOf8 ? memcpy(tail, i1, c) : i1));
- const __m128i vi2 = _mm_loadl_epi64((const __m128i*)(i2 >= LastOf8 ? memcpy(tail, i2, c) : i2));
- const __m128i vi3 = _mm_loadl_epi64((const __m128i*)(i3 >= LastOf8 ? memcpy(tail, i3, c) : i3));
+ const __m128i vi0 =
+ _mm_loadl_epi64((const __m128i*)(i0 >= LastOf8 ? memcpy(tail, i0, c) : i0));
+ const __m128i vi1 =
+ _mm_loadl_epi64((const __m128i*)(i1 >= LastOf8 ? memcpy(tail, i1, c) : i1));
+ const __m128i vi2 =
+ _mm_loadl_epi64((const __m128i*)(i2 >= LastOf8 ? memcpy(tail, i2, c) : i2));
+ const __m128i vi3 =
+ _mm_loadl_epi64((const __m128i*)(i3 >= LastOf8 ? memcpy(tail, i3, c) : i3));
#if !defined(MLAS_TARGET_IX86)
- const __m128i vi4 = _mm_loadl_epi64((const __m128i*)(i4 >= LastOf8 ? memcpy(tail, i4, c) : i4));
- const __m128i vi5 = _mm_loadl_epi64((const __m128i*)(i5 >= LastOf8 ? memcpy(tail, i5, c) : i5));
- const __m128i vi6 = _mm_loadl_epi64((const __m128i*)(i6 >= LastOf8 ? memcpy(tail, i6, c) : i6));
+ const __m128i vi4 =
+ _mm_loadl_epi64((const __m128i*)(i4 >= LastOf8 ? memcpy(tail, i4, c) : i4));
+ const __m128i vi5 =
+ _mm_loadl_epi64((const __m128i*)(i5 >= LastOf8 ? memcpy(tail, i5, c) : i5));
+ const __m128i vi6 =
+ _mm_loadl_epi64((const __m128i*)(i6 >= LastOf8 ? memcpy(tail, i6, c) : i6));
#endif
CALCULATE_ACCUMULATE_VECTORS();
@@ -473,26 +623,38 @@ MlasQLinearGlobalAveragePoolNhwcSingleBatch(
if (ImageSize > 0) {
#if defined(MLAS_TARGET_IX86)
switch (ImageSize) {
- case 1: i1 = ZeroBuffer; /* fall through */
- case 2: i2 = ZeroBuffer; /* fall through */
- case 3: i3 = ZeroBuffer; /* fall through */
- default: break;
+ case 1:
+ i1 = ZeroBuffer; /* fall through */
+ case 2:
+ i2 = ZeroBuffer; /* fall through */
+ case 3:
+ i3 = ZeroBuffer; /* fall through */
+ default:
+ break;
}
#else
switch (ImageSize) {
- case 1: i1 = ZeroBuffer; /* fall through */
- case 2: i2 = ZeroBuffer; /* fall through */
- case 3: i3 = ZeroBuffer; /* fall through */
- case 4: i4 = ZeroBuffer; /* fall through */
- case 5: i5 = ZeroBuffer; /* fall through */
- case 6: i6 = ZeroBuffer; /* fall through */
- default: break;
+ case 1:
+ i1 = ZeroBuffer; /* fall through */
+ case 2:
+ i2 = ZeroBuffer; /* fall through */
+ case 3:
+ i3 = ZeroBuffer; /* fall through */
+ case 4:
+ i4 = ZeroBuffer; /* fall through */
+ case 5:
+ i5 = ZeroBuffer; /* fall through */
+ case 6:
+ i6 = ZeroBuffer; /* fall through */
+ default:
+ break;
}
#endif
int32_t* acc = AccumulateBuffer;
size_t c = Channels;
for (; c >= 8; c -= 8) {
+
LOAD_FULL_CHANNELS();
CALCULATE_ACCUMULATE_VECTORS();
@@ -503,14 +665,21 @@ MlasQLinearGlobalAveragePoolNhwcSingleBatch(
}
if (c > 0) {
- const __m128i vi0 = _mm_loadl_epi64((const __m128i*)(i0 >= LastOf8 ? memcpy(tail, i0, c) : i0));
- const __m128i vi1 = _mm_loadl_epi64((const __m128i*)(1 < ImageSize && i1 >= LastOf8 ? memcpy(tail, i1, c) : i1));
- const __m128i vi2 = _mm_loadl_epi64((const __m128i*)(2 < ImageSize && i2 >= LastOf8 ? memcpy(tail, i2, c) : i2));
- const __m128i vi3 = _mm_loadl_epi64((const __m128i*)(3 < ImageSize && i3 >= LastOf8 ? memcpy(tail, i3, c) : i3));
+ const __m128i vi0 =
+ _mm_loadl_epi64((const __m128i*)(i0 >= LastOf8 ? memcpy(tail, i0, c) : i0));
+ const __m128i vi1 = _mm_loadl_epi64(
+ (const __m128i*)(1 < ImageSize && i1 >= LastOf8 ? memcpy(tail, i1, c) : i1));
+ const __m128i vi2 = _mm_loadl_epi64(
+ (const __m128i*)(2 < ImageSize && i2 >= LastOf8 ? memcpy(tail, i2, c) : i2));
+ const __m128i vi3 = _mm_loadl_epi64(
+ (const __m128i*)(3 < ImageSize && i3 >= LastOf8 ? memcpy(tail, i3, c) : i3));
#if !defined(MLAS_TARGET_IX86)
- const __m128i vi4 = _mm_loadl_epi64((const __m128i*)(4 < ImageSize && i4 >= LastOf8 ? memcpy(tail, i4, c) : i4));
- const __m128i vi5 = _mm_loadl_epi64((const __m128i*)(5 < ImageSize && i5 >= LastOf8 ? memcpy(tail, i5, c) : i5));
- const __m128i vi6 = _mm_loadl_epi64((const __m128i*)(6 < ImageSize && i6 >= LastOf8 ? memcpy(tail, i6, c) : i6));
+ const __m128i vi4 = _mm_loadl_epi64(
+ (const __m128i*)(4 < ImageSize && i4 >= LastOf8 ? memcpy(tail, i4, c) : i4));
+ const __m128i vi5 = _mm_loadl_epi64(
+ (const __m128i*)(5 < ImageSize && i5 >= LastOf8 ? memcpy(tail, i5, c) : i5));
+ const __m128i vi6 = _mm_loadl_epi64(
+ (const __m128i*)(6 < ImageSize && i6 >= LastOf8 ? memcpy(tail, i6, c) : i6));
#endif
CALCULATE_ACCUMULATE_VECTORS();
@@ -527,13 +696,14 @@ MlasQLinearGlobalAveragePoolNhwcSingleBatch(
// Pure C++ Implementation
+template
void
MLASCALL
MlasQLinearGlobalAveragePoolNchw(
- const uint8_t* Input,
+ const T8Bits* Input,
float ScaleInput,
int32_t ZeroPointInput,
- uint8_t* Output,
+ T8Bits* Output,
float ScaleOutput,
int32_t ZeroPointOutput,
size_t Channels,
@@ -544,22 +714,26 @@ MlasQLinearGlobalAveragePoolNchw(
float scale = CheckQLinearGlobalAveragePoolScaleAndSize(ScaleInput, ScaleOutput, ImageSize);
int32_t bias = -ZeroPointInput * static_cast(ImageSize);
for (; Channels > 0; Channels--) {
+
int32_t acc = bias;
for (size_t i = 0; i < ImageSize; ++i) {
- acc += static_cast(*Input++);
+ acc += static_cast(*Input++);
}
- int32_t v = static_cast(std::nearbyintf(acc * scale)) + ZeroPointOutput;
- *Output++ = std::max(std::min(255, v), 0);
+ int32_t v = static_cast(std::nearbyintf(acc * scale)) + ZeroPointOutput;
+ v = std::min(static_cast(std::numeric_limits::max()), v);
+ v = std::max(static_cast(std::numeric_limits::lowest()), v);
+ *Output++ = static_cast(v);
}
}
+template
void
MLASCALL
MlasQLinearGlobalAveragePoolNhwc(
- const uint8_t* Input,
+ const T8Bits* Input,
float ScaleInput,
int32_t ZeroPointInput,
- uint8_t* Output,
+ T8Bits* Output,
float ScaleOutput,
int32_t ZeroPointOutput,
size_t Batch,
@@ -567,26 +741,33 @@ MlasQLinearGlobalAveragePoolNhwc(
size_t Stride,
size_t Channels,
int32_t* AccumulateBuffer,
- const uint8_t* /* ZeroBuffer */
+ const T8Bits* ZeroBuffer
)
{
float scale = CheckQLinearGlobalAveragePoolScaleAndSize(ScaleInput, ScaleOutput, ImageSize);
int32_t bias = -ZeroPointInput * static_cast(ImageSize);
for (; Batch > 0; Batch--) {
- const uint8_t* batch_input = Input;
- uint8_t* batch_output = Output;
+
+ const T8Bits* batch_input = Input;
+ T8Bits* batch_output = Output;
Input += Stride * ImageSize;
Output += Stride;
std::fill_n(AccumulateBuffer, Channels, bias);
for (size_t i = 0; i < ImageSize; ++i) {
+
for (size_t c = 0; c < Channels; ++c) {
AccumulateBuffer[c] += static_cast(batch_input[c]);
}
+
batch_input += Stride;
}
+
for (size_t c = 0; c < Channels; ++c) {
- int32_t v = static_cast(std::nearbyintf(AccumulateBuffer[c] * scale)) + ZeroPointOutput;
- *batch_output++ = std::max(std::min(255, v), 0);
+
+ int32_t v = static_cast(std::nearbyintf(AccumulateBuffer[c] * scale)) + ZeroPointOutput;
+ v = std::min(static_cast(std::numeric_limits::max()), v);
+ v = std::max(static_cast(std::numeric_limits::lowest()), v);
+ *batch_output++ = static_cast(v);
}
}
}
@@ -595,9 +776,91 @@ MlasQLinearGlobalAveragePoolNhwc(
#if defined(MLAS_NEON_INTRINSICS) || defined(MLAS_SSE2_INTRINSICS)
+template
void
MLASCALL
MlasQLinearGlobalAveragePoolNhwc(
+ const T8Bits* Input,
+ float ScaleInput,
+ int32_t ZeroPointInput,
+ T8Bits* Output,
+ float ScaleOutput,
+ int32_t ZeroPointOutput,
+ size_t Batch,
+ size_t ImageSize,
+ size_t Stride,
+ size_t Channels,
+ int32_t* AccumulateBuffer,
+ const T8Bits* ZeroBuffer
+ )
+{
+ float scale = CheckQLinearGlobalAveragePoolScaleAndSize(ScaleInput, ScaleOutput, ImageSize);
+ const int32_t bias = -ZeroPointInput * static_cast(ImageSize);
+ const T8Bits* inputLastOf8 = Input + (Batch * ImageSize * Stride - Stride + Channels) - 8;
+
+ for (; Batch > 0; Batch--) {
+ MlasQLinearGlobalAveragePoolNhwcSingleBatch(
+ Input, Output, inputLastOf8, ImageSize, Channels, Stride, bias, scale,
+ static_cast(ZeroPointOutput), AccumulateBuffer, ZeroBuffer);
+ Input += ImageSize * Stride;
+ Output += Stride;
+ }
+}
+
+#endif
+
+template
+void
+MLASCALL
+MlasQLinearGlobalAveragePoolNchw(
+ const int8_t* Input,
+ float ScaleInput,
+ int32_t ZeroPointInput,
+ int8_t* Output,
+ float ScaleOutput,
+ int32_t ZeroPointOutput,
+ size_t Channels,
+ size_t ImageSize,
+ int32_t* AccumulateBuffer
+ );
+
+template
+void
+MLASCALL
+MlasQLinearGlobalAveragePoolNchw(
+ const uint8_t* Input,
+ float ScaleInput,
+ int32_t ZeroPointInput,
+ uint8_t* Output,
+ float ScaleOutput,
+ int32_t ZeroPointOutput,
+ size_t Channels,
+ size_t ImageSize,
+ int32_t* AccumulateBuffer
+ );
+
+template
+void
+MLASCALL
+MlasQLinearGlobalAveragePoolNhwc(
+ const int8_t* Input,
+ float ScaleInput,
+ int32_t ZeroPointInput,
+ int8_t* Output,
+ float ScaleOutput,
+ int32_t ZeroPointOutput,
+ size_t Batch,
+ size_t ImageSize,
+ size_t Stride,
+ size_t Channels,
+ int32_t* AccumulateBuffer,
+ const int8_t* ZeroBuffer
+ );
+
+template
+void
+MLASCALL
+MlasQLinearGlobalAveragePoolNhwc(
const uint8_t* Input,
float ScaleInput,
int32_t ZeroPointInput,
@@ -610,20 +873,4 @@ MlasQLinearGlobalAveragePoolNhwc(
size_t Channels,
int32_t* AccumulateBuffer,
const uint8_t* ZeroBuffer
- )
-{
- float scale = CheckQLinearGlobalAveragePoolScaleAndSize(ScaleInput, ScaleOutput, ImageSize);
- const int32_t bias = -ZeroPointInput * static_cast(ImageSize);
- const uint8_t* inputLastOf8 = Input + (Batch * ImageSize * Stride - Stride + Channels) - 8;
-
- for (; Batch > 0; Batch--) {
- MlasQLinearGlobalAveragePoolNhwcSingleBatch(
- Input, Output, inputLastOf8, ImageSize, Channels, Stride,
- bias, scale, static_cast(ZeroPointOutput),
- AccumulateBuffer, ZeroBuffer);
- Input += ImageSize * Stride;
- Output += Stride;
- }
-}
-
-#endif
+ );
diff --git a/onnxruntime/core/mlas/lib/quantize.cpp b/onnxruntime/core/mlas/lib/quantize.cpp
index 01a5529fb6..632800e8fd 100644
--- a/onnxruntime/core/mlas/lib/quantize.cpp
+++ b/onnxruntime/core/mlas/lib/quantize.cpp
@@ -165,7 +165,7 @@ Return Value:
--*/
{
- constexpr int32_t MinimumValue = std::numeric_limits::min();
+ constexpr int32_t MinimumValue = std::numeric_limits::lowest();
constexpr int32_t MaximumValue = std::numeric_limits::max();
auto ScaleVector = MlasBroadcastFloat32x4(Scale);
@@ -315,7 +315,7 @@ Return Value:
--*/
{
- constexpr int32_t MinimumValue = std::numeric_limits::min();
+ constexpr int32_t MinimumValue = std::numeric_limits::lowest();
constexpr int32_t MaximumValue = std::numeric_limits::max();
for (size_t n = 0; n < N; n++) {
@@ -352,17 +352,18 @@ MlasQuantizeLinear(
#if defined(MLAS_SSE2_INTRINSICS)
+template
void
MLASCALL
MlasRequantizeOutput(
const int32_t* Input,
size_t InputLeadingDimension,
- uint8_t* Output,
+ OutputType* Output,
size_t OutputLeadingDimension,
const int32_t* Bias,
const float* Scale,
bool PerColumnScale,
- uint8_t ZeroPoint,
+ OutputType ZeroPoint,
size_t StartM,
size_t StartN,
size_t CountM,
@@ -370,8 +371,8 @@ MlasRequantizeOutput(
)
{
const __m128 PerMatrixScaleVector = PerColumnScale ? _mm_setzero_ps() : _mm_load1_ps(Scale);
- const __m128 MinimumValueVector = _mm_set1_ps(float(0 - ZeroPoint));
- const __m128 MaximumValueVector = _mm_set1_ps(float(255 - ZeroPoint));
+ const __m128 MinimumValueVector = _mm_set1_ps(float(std::numeric_limits::lowest() - ZeroPoint));
+ const __m128 MaximumValueVector = _mm_set1_ps(float(std::numeric_limits::max() - ZeroPoint));
const __m128i ZeroPointVector = _mm_set1_epi32(ZeroPoint);
if (nullptr != Bias) {
@@ -467,10 +468,23 @@ MlasRequantizeOutput(
IntegerVector2 = _mm_add_epi32(IntegerVector2, ZeroPointVector);
IntegerVector3 = _mm_add_epi32(IntegerVector3, ZeroPointVector);
- __m128i WordVector0 = _mm_packus_epi16(IntegerVector0, IntegerVector1);
- __m128i WordVector1 = _mm_packus_epi16(IntegerVector2, IntegerVector3);
+ __m128i WordVector0;
+ __m128i WordVector1;
+ __m128i ByteVector;
- __m128i ByteVector = _mm_packus_epi16(WordVector0, WordVector1);
+ if (std::is_signed::value) {
+
+ WordVector0 = _mm_packs_epi32(IntegerVector0, IntegerVector1);
+ WordVector1 = _mm_packs_epi32(IntegerVector2, IntegerVector3);
+ ByteVector = _mm_packs_epi16(WordVector0, WordVector1);
+
+ } else {
+
+ WordVector0 = _mm_packus_epi16(IntegerVector0, IntegerVector1);
+ WordVector1 = _mm_packus_epi16(IntegerVector2, IntegerVector3);
+ ByteVector = _mm_packus_epi16(WordVector0, WordVector1);
+
+ }
_mm_storeu_si128((__m128i*)RowOutput, ByteVector);
RowOutput += 16;
@@ -541,8 +555,17 @@ MlasRequantizeOutput(
IntegerVector = _mm_cvtps_epi32(FloatVector);
IntegerVector = _mm_add_epi32(IntegerVector, ZeroPointVector);
- IntegerVector = _mm_packus_epi16(IntegerVector, IntegerVector);
- IntegerVector = _mm_packus_epi16(IntegerVector, IntegerVector);
+ if (std::is_signed::value) {
+
+ IntegerVector = _mm_packs_epi32(IntegerVector, IntegerVector);
+ IntegerVector = _mm_packs_epi16(IntegerVector, IntegerVector);
+
+ } else {
+
+ IntegerVector = _mm_packus_epi16(IntegerVector, IntegerVector);
+ IntegerVector = _mm_packus_epi16(IntegerVector, IntegerVector);
+
+ }
uint32_t OutputValue = uint32_t(_mm_cvtsi128_si32(IntegerVector));
@@ -570,17 +593,18 @@ MlasRequantizeOutput(
#elif defined(MLAS_NEON64_INTRINSICS)
+template
void
MLASCALL
MlasRequantizeOutput(
const int32_t* Input,
size_t InputLeadingDimension,
- uint8_t* Output,
+ OutputType* Output,
size_t OutputLeadingDimension,
const int32_t* Bias,
const float* Scale,
bool PerColumnScale,
- uint8_t ZeroPoint,
+ OutputType ZeroPoint,
size_t StartM,
size_t StartN,
size_t CountM,
@@ -686,7 +710,7 @@ MlasRequantizeOutput(
//
// Pack the integers with saturation to 16-bit values and shift by
- // the zero point, then pack the integers again to unsigned bytes.
+ // the zero point, then pack the integers again to bytes.
//
int16x8x2_t WordVector;
@@ -697,7 +721,13 @@ MlasRequantizeOutput(
WordVector.val[0] = vqaddq_s16(WordVector.val[0], ZeroPointVector);
WordVector.val[1] = vqaddq_s16(WordVector.val[1], ZeroPointVector);
- vst1q_u8(RowOutput, vqmovun_high_s16(vqmovun_s16(WordVector.val[0]), WordVector.val[1]));
+ if (std::is_signed::value) {
+ vst1q_s8(reinterpret_cast(RowOutput),
+ vqmovn_high_s16(vqmovn_s16(WordVector.val[0]), WordVector.val[1]));
+ } else {
+ vst1q_u8(reinterpret_cast(RowOutput),
+ vqmovun_high_s16(vqmovun_s16(WordVector.val[0]), WordVector.val[1]));
+ }
RowOutput += 16;
n -= 16;
@@ -775,7 +805,13 @@ MlasRequantizeOutput(
int16x8_t WordVector = vcombine_s16(vqmovn_s32(IntegerVector), vdup_n_s16(0));
WordVector = vqaddq_s16(WordVector, ZeroPointVector);
- uint8x16_t ByteVector = vcombine_u8(vqmovun_s16(WordVector), vdup_n_u8(0));
+ uint8x16_t ByteVector;
+
+ if (std::is_signed::value) {
+ ByteVector = vcombine_u8(vreinterpret_u8_s8(vqmovn_s16(WordVector)), vdup_n_u8(0));
+ } else {
+ ByteVector = vcombine_u8(vqmovun_s16(WordVector), vdup_n_u8(0));
+ }
if (n >= 4) {
@@ -787,7 +823,7 @@ MlasRequantizeOutput(
} else {
- vst1q_lane_u8(RowOutput, ByteVector, 0);
+ vst1q_lane_u8(reinterpret_cast(RowOutput), ByteVector, 0);
RowOutput += 1;
n -= 1;
@@ -802,17 +838,18 @@ MlasRequantizeOutput(
#else
+template
void
MLASCALL
MlasRequantizeOutput(
const int32_t* Input,
size_t InputLeadingDimension,
- uint8_t* Output,
+ OutputType* Output,
size_t OutputLeadingDimension,
const int32_t* Bias,
const float* Scale,
bool PerColumnScale,
- uint8_t ZeroPoint,
+ OutputType ZeroPoint,
size_t StartM,
size_t StartN,
size_t CountM,
@@ -820,8 +857,8 @@ MlasRequantizeOutput(
)
{
const float PerMatrixScaleValue = PerColumnScale ? 0.0f : *Scale;
- const float MinimumValue = float(0 - ZeroPoint);
- const float MaximumValue = float(255 - ZeroPoint);
+ const float MinimumValue = float(std::numeric_limits::lowest() - ZeroPoint);
+ const float MaximumValue = float(std::numeric_limits::max() - ZeroPoint);
if (nullptr != Bias) {
Bias += StartN;
@@ -872,7 +909,7 @@ MlasRequantizeOutput(
IntegerValue = int32_t(MlasBitsOfFp32(FloatValue + MLAS_ROUNDING_BIAS_MAGIC)) -
MLAS_ROUNDING_BIAS_MAGIC_BITS;
- *RowOutput++ = uint8_t(IntegerValue + ZeroPoint);
+ *RowOutput++ = OutputType(IntegerValue + ZeroPoint);
n -= 1;
}
@@ -885,6 +922,42 @@ MlasRequantizeOutput(
#endif
+template
+void
+MLASCALL
+MlasRequantizeOutput(
+ const int32_t* Input,
+ size_t InputLeadingDimension,
+ int8_t* Output,
+ size_t OutputLeadingDimension,
+ const int32_t* Bias,
+ const float* Scale,
+ bool PerColumnScale,
+ int8_t ZeroPoint,
+ size_t StartM,
+ size_t StartN,
+ size_t CountM,
+ size_t CountN
+ );
+
+template
+void
+MLASCALL
+MlasRequantizeOutput(
+ const int32_t* Input,
+ size_t InputLeadingDimension,
+ uint8_t* Output,
+ size_t OutputLeadingDimension,
+ const int32_t* Bias,
+ const float* Scale,
+ bool PerColumnScale,
+ uint8_t ZeroPoint,
+ size_t StartM,
+ size_t StartN,
+ size_t CountM,
+ size_t CountN
+ );
+
void
MLASCALL
MlasFindMinMaxElement(
diff --git a/onnxruntime/core/mlas/lib/transpose.cpp b/onnxruntime/core/mlas/lib/transpose.cpp
index 3f12773050..37181ec2f3 100644
--- a/onnxruntime/core/mlas/lib/transpose.cpp
+++ b/onnxruntime/core/mlas/lib/transpose.cpp
@@ -436,3 +436,18 @@ Return Value:
n -= 1;
}
}
+
+void
+MLASCALL
+MlasTranspose(
+ const int8_t* Input,
+ int8_t* Output,
+ size_t M,
+ size_t N)
+{
+ MlasTranspose(
+ reinterpret_cast(Input),
+ reinterpret_cast(Output),
+ M,
+ N);
+}
\ No newline at end of file
diff --git a/onnxruntime/core/util/math_cpu.cc b/onnxruntime/core/util/math_cpu.cc
index eb4515f1f7..c09d885a23 100644
--- a/onnxruntime/core/util/math_cpu.cc
+++ b/onnxruntime/core/util/math_cpu.cc
@@ -650,6 +650,7 @@ void Im2col::operator()(
}
}
+template struct Im2col;
template struct Im2col;
template <>
diff --git a/onnxruntime/test/contrib_ops/nhwc_maxpool_op_test.cc b/onnxruntime/test/contrib_ops/nhwc_maxpool_op_test.cc
index 8c52de8373..0a74a68a80 100644
--- a/onnxruntime/test/contrib_ops/nhwc_maxpool_op_test.cc
+++ b/onnxruntime/test/contrib_ops/nhwc_maxpool_op_test.cc
@@ -2,10 +2,12 @@
// Licensed under the MIT License.
#include
+#include
+
#include "core/util/math.h"
+#include "core/mlas/inc/mlas.h"
#include "gtest/gtest.h"
#include "test/providers/provider_test_utils.h"
-#include
namespace onnxruntime {
namespace test {
@@ -70,7 +72,9 @@ class NhwcMaxPoolOpTester {
Y_shape.push_back(batch_count);
for (size_t n = 0; n < kernel_rank; n++) {
Y_shape.push_back(((input_shape[n] + pads[n] + pads[kernel_rank + n]) -
- (dilations[n] * (kernel_shape_[n] - 1) + 1)) / strides[n] + 1);
+ (dilations[n] * (kernel_shape_[n] - 1) + 1)) /
+ strides[n] +
+ 1);
}
Y_shape.push_back(channels);
Y_data.resize(ShapeSize(Y_shape));
@@ -87,7 +91,7 @@ class NhwcMaxPoolOpTester {
std::vector d_output(kernel_rank, 0);
std::vector d_kernel(kernel_rank, 0);
do {
- std::fill_n(Ydata, channels, static_cast(0));
+ std::fill_n(Ydata, channels, std::numeric_limits::lowest());
do {
int64_t input_offset = 0;
bool is_padding = false;
@@ -163,7 +167,7 @@ class NhwcMaxPoolOpTester {
};
TEST(NhwcMaxPoolContribOpTest, MaxPool1D) {
- for (int64_t channels = 1; channels < 64; channels++) {
+ for (int64_t channels = 1; channels < 94; channels++) {
NhwcMaxPoolOpTester test;
test.GenerateRandomInput({1, 23, channels});
test.SetKernelShape({5});
@@ -173,7 +177,7 @@ TEST(NhwcMaxPoolContribOpTest, MaxPool1D) {
}
TEST(NhwcMaxPoolContribOpTest, MaxPool2D) {
- for (int64_t channels = 1; channels < 64; channels++) {
+ for (int64_t channels = 1; channels < 94; channels++) {
NhwcMaxPoolOpTester test;
test.GenerateRandomInput({1, 15, 19, channels});
test.SetKernelShape({3, 5});
@@ -183,7 +187,7 @@ TEST(NhwcMaxPoolContribOpTest, MaxPool2D) {
}
TEST(NhwcMaxPoolContribOpTest, MaxPool3D) {
- for (int64_t channels = 1; channels < 64; channels++) {
+ for (int64_t channels = 1; channels < 94; channels++) {
NhwcMaxPoolOpTester test;
test.GenerateRandomInput({1, 9, 13, 15, channels});
test.SetKernelShape({2, 4, 6});
@@ -208,5 +212,51 @@ TEST(NhwcMaxPoolContribOpTest, MaxPoolDilations) {
test.Run();
}
+TEST(NhwcMaxPoolContribOpTest, MaxPool1D_S8) {
+ for (int64_t channels = 1; channels < 94; channels++) {
+ NhwcMaxPoolOpTester test;
+ test.GenerateRandomInput({1, 23, channels});
+ test.SetKernelShape({5});
+ test.SetPads({2, 2});
+ test.Run();
+ }
+}
+
+TEST(NhwcMaxPoolContribOpTest, MaxPool2D_S8) {
+ for (int64_t channels = 1; channels < 94; channels++) {
+ NhwcMaxPoolOpTester test;
+ test.GenerateRandomInput({1, 15, 19, channels});
+ test.SetKernelShape({3, 5});
+ test.SetPads({1, 1, 1, 1});
+ test.Run();
+ }
+}
+
+TEST(NhwcMaxPoolContribOpTest, MaxPool3D_S8) {
+ for (int64_t channels = 1; channels < 94; channels++) {
+ NhwcMaxPoolOpTester test;
+ test.GenerateRandomInput({1, 9, 13, 15, channels});
+ test.SetKernelShape({2, 4, 6});
+ test.SetPads({0, 0, 0, 1, 1, 1});
+ test.Run();
+ }
+}
+
+TEST(NhwcMaxPoolContribOpTest, MaxPoolStrides_S8) {
+ NhwcMaxPoolOpTester test;
+ test.GenerateRandomInput({4, 23, 19, 32});
+ test.SetKernelShape({3, 3});
+ test.SetStrides({2, 2});
+ test.Run();
+}
+
+TEST(NhwcMaxPoolContribOpTest, MaxPoolDilations_S8) {
+ NhwcMaxPoolOpTester test;
+ test.GenerateRandomInput({4, 23, 19, 32});
+ test.SetKernelShape({3, 3});
+ test.SetDilations({2, 2});
+ test.Run();
+}
+
} // namespace test
} // namespace onnxruntime
diff --git a/onnxruntime/test/contrib_ops/qlinear_global_average_pool_test.cc b/onnxruntime/test/contrib_ops/qlinear_global_average_pool_test.cc
index 94fdef7b78..7fc057aa79 100644
--- a/onnxruntime/test/contrib_ops/qlinear_global_average_pool_test.cc
+++ b/onnxruntime/test/contrib_ops/qlinear_global_average_pool_test.cc
@@ -5,22 +5,24 @@
#include "test/common/tensor_op_test_utils.h"
#include "test/providers/provider_test_utils.h"
#include "core/providers/common.h"
+#include "core/mlas/inc/mlas.h"
namespace onnxruntime {
namespace test {
-static void CalculateGlobalAvgPoolU8(
- const uint8_t* x, int64_t batch, int64_t hw, int64_t channel, bool channels_last, uint8_t* y,
+template
+static void CalculateGlobalAvgPool(
+ const T8Bits* x, int64_t batch, int64_t hw, int64_t channel, bool channels_last, T8Bits* y,
int32_t x_zero_point, float x_scale, int32_t y_zero_point, float y_scale) {
int32_t bias = -x_zero_point * gsl::narrow_cast(hw);
int64_t stride_image = channels_last ? channel : 1;
int64_t stride_channel = channels_last ? 1 : hw;
for (int64_t b = 0; b < batch; ++b) {
- const uint8_t* bx = x + b * hw * channel;
- uint8_t* by = y + b * channel;
+ const T8Bits* bx = x + b * hw * channel;
+ T8Bits* by = y + b * channel;
for (int64_t c = 0; c < channel; ++c) {
- const uint8_t* ix = bx + c * stride_channel;
+ const T8Bits* ix = bx + c * stride_channel;
int32_t sum = 0;
for (int64_t i = 0; i < hw; ++i) {
sum += static_cast(*ix);
@@ -29,40 +31,41 @@ static void CalculateGlobalAvgPoolU8(
sum += bias;
int32_t r = static_cast(std::nearbyintf(x_scale * sum / static_cast(hw) / y_scale));
r += y_zero_point;
- r = std::min(255, r);
- r = std::max(0, r);
- by[c] = static_cast(r);
+ r = std::min((int32_t)(std::numeric_limits::max()), r);
+ r = std::max((int32_t)(std::numeric_limits::lowest()), r);
+ by[c] = static_cast(r);
}
}
}
-void RunQLinearGlobalAveragePoolU8(
+template
+void RunQLinearGlobalAveragePool(
bool channels_last, int64_t batch, int64_t channel, int64_t h, int64_t w,
- uint8_t x_zero_point, float x_scale, uint8_t y_zero_point, float y_scale, int32_t seed = 0) {
+ T8Bits x_zero_point, float x_scale, T8Bits y_zero_point, float y_scale, int32_t seed = 0) {
std::vector x_dims = channels_last ? std::vector{batch, h, w, channel} : std::vector{batch, channel, h, w};
std::vector y_dims = channels_last ? std::vector{batch, 1, 1, channel} : std::vector{batch, channel, 1, 1};
int64_t x_size = batch * channel * h * w;
int64_t y_size = batch * channel;
- std::vector x_data((size_t)x_size);
- std::vector y_data((size_t)y_size);
+ std::vector x_data((size_t)x_size);
+ std::vector y_data((size_t)y_size);
RandomValueGenerator random{seed ? optional{seed} : optional{}};
- std::vector tmp_x_data = random.Uniform(x_dims, 0, 255);
- std::transform(tmp_x_data.begin(), tmp_x_data.end(), x_data.data(), [](int32_t v) -> uint8_t {
- return static_cast(v);
+ std::vector tmp_x_data = random.Uniform(x_dims, std::numeric_limits::lowest(), std::numeric_limits::max());
+ std::transform(tmp_x_data.begin(), tmp_x_data.end(), x_data.data(), [](int32_t v) -> T8Bits {
+ return static_cast(v);
});
- CalculateGlobalAvgPoolU8(x_data.data(), batch, h * w, channel, channels_last, y_data.data(),
+ CalculateGlobalAvgPool(x_data.data(), batch, h * w, channel, channels_last, y_data.data(),
x_zero_point, x_scale, y_zero_point, y_scale);
OpTester test("QLinearGlobalAveragePool", 1, onnxruntime::kMSDomain);
test.AddAttribute("channels_last", channels_last ? 1LL : 0LL);
- test.AddInput("X", x_dims, x_data);
+ test.AddInput("X", x_dims, x_data);
test.AddInput("x_scale", {}, {x_scale});
- test.AddInput("x_zero_point", {}, {x_zero_point});
+ test.AddInput("x_zero_point", {}, {x_zero_point});
test.AddInput("y_scale", {}, {y_scale});
- test.AddInput("y_zero_point", {}, {y_zero_point});
- test.AddOutput("Y", y_dims, y_data);
+ test.AddInput