saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-30 20:18:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

add int8_t support for pool operators (#9852)

Browse source 
* add int8_t support for pool operators
This commit is contained in:
Yufeng Li 2021-11-29 18:43:43 -08:00 committed by GitHub
parent ec9b0ed800
commit a0afd7303d
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
18 changed files with 1234 additions and 427 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
docs/OperatorKernels.md
View file

@ -400,7 +400,7 @@ Do not modify directly.*
|MaxpoolWithMask|*in* X:**T**<br> *in* M:**tensor(int32)**<br> *out* Y:**T**|1+|**X** = tensor(float)|
|MurmurHash3|*in* X:**T1**<br> *out* Y:**T2**|1+|**T1** = tensor(double), tensor(float), tensor(int32), tensor(int64), tensor(string), tensor(uint32), tensor(uint64)<br/> **T2** = tensor(int32), tensor(uint32)|
|NGramRepeatBlock|*in* input_ids:**Tid**<br> *in* scores:**T**<br> *out* scores_out:**T**|1+|**T** = tensor(float)<br/> **Tid** = tensor(int64)|
|NhwcMaxPool|*in* x:**T**<br> *out* y:**T**|1+|**T** = tensor(uint8)|
|NhwcMaxPool|*in* x:**T**<br> *out* y:**T**|1+|**T** = tensor(int8), tensor(uint8)|
|Pad|*in* data:**T**<br> *in* pads:**tensor(int64)**<br> *in* value:**T**<br> *out* output:**T**|1+|**T** = tensor(float)|
|QAttention|*in* input:**T1**<br> *in* weight:**T2**<br> *in* bias:**T3**<br> *in* input_scale:**T3**<br> *in* weight_scale:**T3**<br> *in* mask_index:**T4**<br> *in* input_zero_point:**T1**<br> *in* weight_zero_point:**T2**<br> *in* past:**T3**<br> *out* output:**T3**<br> *out* present:**T3**|1+|**T1** = tensor(uint8)<br/> **T2** = tensor(int8), tensor(uint8)<br/> **T3** = tensor(float)<br/> **T4** = tensor(int32)|
|QEmbedLayerNormalization|*in* input_ids:**T1**<br> *in* segment_ids:**T1**<br> *in* word_embedding_quant:**T2**<br> *in* position_embedding_quant:**T2**<br> *in* segment_embedding:**T2**<br> *in* gamma_quant:**T2**<br> *in* beta_quant:**T2**<br> *in* mask:**T1**<br> *in* word_embedding_scale:**T**<br> *in* position_embedding_scale:**T**<br> *in* segment_embedding_scale:**T**<br> *in* gamma_scale:**T**<br> *in* beta_scale:**T**<br> *in* word_embedding_zero_point:**T2**<br> *in* position_embedding_zero_point:**T2**<br> *in* segment_embedding_zero_point:**T2**<br> *in* gamma_zero_point:**T2**<br> *in* beta_zero_point:**T2**<br> *out* layernorm_out:**T**<br> *out* mask_index_out:**T1**|1+|**T** = tensor(float)|

6
onnxruntime/contrib_ops/cpu/cpu_contrib_kernels.cc
View file

@ -72,7 +72,8 @@ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1,
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, MatMulIntegerToFloat);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, DynamicQuantizeLSTM);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, QLinearConv);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, NhwcMaxPool);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, int8_t, NhwcMaxPool);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, NhwcMaxPool);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, QEmbedLayerNormalization);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, QGemm);
// ******** End: Quantization ******************* //
@ -160,7 +161,8 @@ Status RegisterQuantizationKernels(KernelRegistry& kernel_registry) {
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, MatMulIntegerToFloat)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, DynamicQuantizeLSTM)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, QLinearConv)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, NhwcMaxPool)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, int8_t, NhwcMaxPool)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, NhwcMaxPool)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, float, QEmbedLayerNormalization)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kMSDomain, 1, uint8_t, QGemm)>,
};

45
onnxruntime/contrib_ops/cpu/qlinear_global_average_pool.cc
View file

@ -14,13 +14,14 @@ using onnxruntime::concurrency::ThreadPool;
namespace onnxruntime {
namespace contrib {
template <typename T8Bits>
Status ComputeQLinearGlobalAvgPool(
const uint8_t* x,
const T8Bits* x,
float x_scale,
uint8_t x_zero_point,
uint8_t* y,
T8Bits x_zero_point,
T8Bits* y,
float y_scale,
uint8_t y_zero_point,
T8Bits y_zero_point,
int64_t N,
int64_t C,
int64_t image_size,
@ -28,8 +29,8 @@ Status ComputeQLinearGlobalAvgPool(
concurrency::ThreadPool* tp) {
if (!channels_last || C == 1) {
auto worker = [=](std::ptrdiff_t first, std::ptrdiff_t last) {
const uint8_t* input = (const uint8_t*)(x + (first * image_size));
uint8_t* output = (uint8_t*)(y + first);
const T8Bits* input = (const T8Bits*)(x + (first * image_size));
T8Bits* output = (T8Bits*)(y + first);
std::vector<int32_t> acc_buffer(MlasQLinearSafePaddingElementCount(sizeof(int32_t), last - first));
MlasQLinearGlobalAveragePoolNchw(input, x_scale, x_zero_point, output, y_scale, y_zero_point, last - first, image_size, acc_buffer.data());
};
@ -37,17 +38,17 @@ Status ComputeQLinearGlobalAvgPool(
tp, static_cast<std::ptrdiff_t>(N * C), {1.0 * image_size, 1.0, 8.0 * image_size}, worker);
} else {
auto worker = [=](std::ptrdiff_t first, std::ptrdiff_t last) {
const uint8_t* input = x + first * C * image_size;
uint8_t* output = y + first * C;
const T8Bits* input = x + first * C * image_size;
T8Bits* output = y + first * C;
std::vector<int32_t> acc_buffer(MlasQLinearSafePaddingElementCount(sizeof(int32_t), C));
std::vector<uint8_t> zero_buffer(MlasQLinearSafePaddingElementCount(sizeof(uint8_t), C), 0);
std::vector<T8Bits> zero_buffer(MlasQLinearSafePaddingElementCount(sizeof(T8Bits), C), 0);
MlasQLinearGlobalAveragePoolNhwc(
input, x_scale, x_zero_point, output, y_scale, y_zero_point,
last - first, image_size, C, C, acc_buffer.data(), zero_buffer.data());
};
concurrency::ThreadPool::TryParallelFor(
tp, static_cast<std::ptrdiff_t>(N),
{1.0 * image_size * C, 1.0 * C, 8.0 *image_size * C},
{1.0 * image_size * C, 1.0 * C, 8.0 * image_size * C},
worker);
}
return Status::OK();
@ -88,19 +89,25 @@ Status QLinearGlobalAveragePool::Compute(OpKernelContext* context) const {
const float x_scale = *(tensor_x_scale->Data<float>());
const float y_scale = *(tensor_y_scale->Data<float>());
auto dtype = X.GetElementType();
switch (dtype) {
case ONNX_NAMESPACE::TensorProto_DataType_UINT8:
return ComputeQLinearGlobalAvgPool(X.Data<uint8_t>(), x_scale, *(tensor_x_zero_point->Data<uint8_t>()),
Y.MutableData<uint8_t>(), y_scale, *(tensor_y_zero_point->Data<uint8_t>()),
N, C, image_size, channels_last_, tp);
default:
ORT_THROW("Unsupported 'dtype' value: ", dtype);
if (dtype == ONNX_NAMESPACE::TensorProto_DataType_UINT8) {
return ComputeQLinearGlobalAvgPool(X.Data<uint8_t>(), x_scale, *(tensor_x_zero_point->Data<uint8_t>()),
Y.MutableData<uint8_t>(), y_scale, *(tensor_y_zero_point->Data<uint8_t>()),
N, C, image_size, channels_last_, tp);
} else {
return ComputeQLinearGlobalAvgPool(X.Data<int8_t>(), x_scale, *(tensor_x_zero_point->Data<int8_t>()),
Y.MutableData<int8_t>(), y_scale, *(tensor_y_zero_point->Data<int8_t>()),
N, C, image_size, channels_last_, tp);
}
}
ONNX_OPERATOR_KERNEL_EX(QLinearGlobalAveragePool, kMSDomain, 1, kCpuExecutionProvider, KernelDefBuilder(), QLinearGlobalAveragePool);
ONNX_OPERATOR_KERNEL_EX(QLinearGlobalAveragePool,
kMSDomain,
1,
kCpuExecutionProvider,
KernelDefBuilder(),
QLinearGlobalAveragePool);
} // namespace contrib
} // namespace onnxruntime

9
onnxruntime/contrib_ops/cpu/qlinear_global_average_pool.h
View file

@ -21,13 +21,14 @@ class QLinearGlobalAveragePool final : public OpKernel {
bool channels_last_;
};
template<typename T8Bits>
Status ComputeQLinearGlobalAvgPool(
const uint8_t* x,
const T8Bits* x,
float x_scale,
uint8_t x_zero_point,
uint8_t* y,
T8Bits x_zero_point,
T8Bits* y,
float y_scale,
uint8_t y_zero_point,
T8Bits y_zero_point,
int64_t N,
int64_t C,
int64_t image_size,

56
onnxruntime/contrib_ops/cpu/qlinear_pool.cc
View file

@ -23,19 +23,15 @@ using concurrency::ThreadPool;
namespace contrib {
template <typename T8Bits>
static inline float dequantize_value(T8Bits x, float x_scale, T8Bits x_zero_point);
template <typename T8Bits>
static inline T8Bits quantize_value(float y, float y_scale, T8Bits y_zero_point);
template <>
inline float dequantize_value<uint8_t>(uint8_t x, float x_scale, uint8_t x_zero_point) {
static inline float dequantize_value(T8Bits x, float x_scale, T8Bits x_zero_point) {
return x_scale * (static_cast<int>(x) - x_zero_point);
}
template <>
inline uint8_t quantize_value<uint8_t>(float y, float y_scale, uint8_t y_zero_point) {
return static_cast<uint8_t>(std::max(0.0f, std::min(std::nearbyintf(y / y_scale + y_zero_point), 255.0f)));
template <typename T8Bits>
static inline T8Bits quantize_value(float y, float y_scale, T8Bits y_zero_point) {
constexpr int32_t min_8bits = std::numeric_limits<T8Bits>::lowest();
constexpr int32_t max_8bits = std::numeric_limits<T8Bits>::max();
return static_cast<T8Bits>(std::max(min_8bits, std::min(static_cast<int32_t>(std::nearbyintf(y / y_scale + y_zero_point)), max_8bits)));
}
static void SwitchDimsNchwNhwc(std::vector<int64_t>& dims, bool from_nchw_to_nhwc) {
@ -509,6 +505,15 @@ void dequantize_array(int64_t N, const T8Bits* input, float scale, T8Bits zero_p
}
Status QLinearAveragePool::Compute(OpKernelContext* context) const {
if (is_input_signed_) {
return ComputeImpl<int8_t>(context);
} else {
return ComputeImpl<uint8_t>(context);
}
}
template <typename T8Bits>
Status QLinearAveragePool::ComputeImpl(OpKernelContext* context) const {
const auto tensor_x_scale = context->Input<Tensor>(1);
const auto tensor_x_zero_point = context->Input<Tensor>(2);
const auto tensor_y_scale = context->Input<Tensor>(3);
@ -524,16 +529,12 @@ Status QLinearAveragePool::Compute(OpKernelContext* context) const {
"input y_zero_point must be a scalar or 1D tensor of size 1 if given");
const auto* X = context->Input<Tensor>(0);
auto dtype = X->GetElementType();
if (dtype != ONNX_NAMESPACE::TensorProto_DataType_UINT8) {
ORT_THROW("Unsupported 'dtype' in QLinear Pooling:", dtype);
}
TensorShape x_shape = X->Shape();
const float x_scale = *(tensor_x_scale->Data<float>());
const float y_scale = *(tensor_y_scale->Data<float>());
uint8_t x_zero_point = (tensor_x_zero_point ? *(tensor_x_zero_point->Data<uint8_t>()) : (uint8_t)0);
uint8_t y_zero_point = (tensor_y_zero_point ? *(tensor_y_zero_point->Data<uint8_t>()) : (uint8_t)0);
T8Bits x_zero_point = (tensor_x_zero_point ? *(tensor_x_zero_point->Data<T8Bits>()) : (T8Bits)0);
T8Bits y_zero_point = (tensor_y_zero_point ? *(tensor_y_zero_point->Data<T8Bits>()) : (T8Bits)0);
ORT_RETURN_IF_NOT(x_shape.NumDimensions() >= 3, "Input dimension cannot be less than 3.");
std::vector<int64_t> pads = pool_attrs_.pads;
@ -564,8 +565,8 @@ Status QLinearAveragePool::Compute(OpKernelContext* context) const {
SwitchDimsNchwNhwc(output_dims, true);
}
Tensor* Y = context->Output(0, output_dims);
const auto* X_data = X->Data<uint8_t>();
auto* Y_data = Y->MutableData<uint8_t>();
const auto* X_data = X->Data<T8Bits>();
auto* Y_data = Y->MutableData<T8Bits>();
ThreadPool* tp = context->GetOperatorThreadPool();
// Check for special case which could fall back to global average pool
@ -589,12 +590,12 @@ Status QLinearAveragePool::Compute(OpKernelContext* context) const {
switch (kernel_shape.size()) {
case 1: {
if (channels_last_) {
QLinearPoolNhwc1DTask<uint8_t, onnxruntime::AveragePool> avg_pool_task_1d = {
QLinearPoolNhwc1DTask<T8Bits, onnxruntime::AveragePool> avg_pool_task_1d = {
x_data_fp32, Y_data, y_scale, y_zero_point, channels,
pooled_height, strides[0], height, kernel_shape, pads, pool_context_, pool_attrs_};
ThreadPool::TryParallelFor(tp, y_image_size * batch_count, avg_pool_task_1d.Cost(), avg_pool_task_1d);
} else {
QLinearPool1DTask<uint8_t, onnxruntime::AveragePool> avg_pool_task_1d = {
QLinearPool1DTask<T8Bits, onnxruntime::AveragePool> avg_pool_task_1d = {
x_data_fp32, Y_data, y_scale, y_zero_point, x_image_size, y_image_size,
pooled_height, strides[0], height, kernel_shape, pads, pool_context_, pool_attrs_};
ThreadPool::TryParallelFor(tp, total_channels, avg_pool_task_1d.Cost(), avg_pool_task_1d);
@ -604,13 +605,13 @@ Status QLinearAveragePool::Compute(OpKernelContext* context) const {
case 2: {
if (channels_last_) {
QLinearPoolNhwc2DTask<uint8_t, onnxruntime::AveragePool> avg_pool_task_2d = {
QLinearPoolNhwc2DTask<T8Bits, onnxruntime::AveragePool> avg_pool_task_2d = {
x_data_fp32, Y_data, y_scale, y_zero_point, x_image_size, y_image_size, kernel_size, channels,
pooled_height, pooled_width, strides[0], strides[1], height, width, kernel_shape, pads, pool_context_, pool_attrs_};
ThreadPool::TryParallelFor(tp, y_image_size * batch_count, avg_pool_task_2d.Cost(), avg_pool_task_2d);
} else {
QLinearPool2DTask<uint8_t, onnxruntime::AveragePool> avg_pool_task_2d = {
QLinearPool2DTask<T8Bits, onnxruntime::AveragePool> avg_pool_task_2d = {
x_data_fp32, Y_data, y_scale, y_zero_point, x_image_size, y_image_size,
pooled_height, pooled_width, strides[0], strides[1], height, width, kernel_shape, pads, pool_context_, pool_attrs_};
ThreadPool::TryParallelFor(tp, total_channels, avg_pool_task_2d.Cost(), avg_pool_task_2d);
@ -620,14 +621,14 @@ Status QLinearAveragePool::Compute(OpKernelContext* context) const {
case 3: {
if (channels_last_) {
QLinearPoolNhwc3DTask<uint8_t, onnxruntime::AveragePool> avg_pool_task_3d = {
QLinearPoolNhwc3DTask<T8Bits, onnxruntime::AveragePool> avg_pool_task_3d = {
x_data_fp32, Y_data, y_scale, y_zero_point, x_image_size, y_image_size, kernel_size, channels,
pooled_height, pooled_width, pooled_depth, strides[0], strides[1], strides[2], height, width, depth,
kernel_shape, pads, pool_context_, pool_attrs_};
ThreadPool::TryParallelFor(tp, y_image_size * batch_count, avg_pool_task_3d.Cost(), avg_pool_task_3d);
} else {
QLinearPool3DTask<uint8_t, onnxruntime::AveragePool> avg_pool_task_3d = {
QLinearPool3DTask<T8Bits, onnxruntime::AveragePool> avg_pool_task_3d = {
x_data_fp32, Y_data, y_scale, y_zero_point, x_image_size, y_image_size,
pooled_height, pooled_width, pooled_depth, strides[0], strides[1], strides[2], height, width, depth,
kernel_shape, pads, pool_context_, pool_attrs_};
@ -647,7 +648,12 @@ Status QLinearAveragePool::Compute(OpKernelContext* context) const {
return Status::OK();
}
ONNX_OPERATOR_KERNEL_EX(QLinearAveragePool, kMSDomain, 1, kCpuExecutionProvider, KernelDefBuilder(), QLinearAveragePool);
ONNX_OPERATOR_KERNEL_EX(QLinearAveragePool,
kMSDomain,
1,
kCpuExecutionProvider,
KernelDefBuilder(),
QLinearAveragePool);
} // namespace contrib

7
onnxruntime/contrib_ops/cpu/qlinear_pool.h
View file

@ -14,6 +14,9 @@ class QLinearAveragePool final : public OpKernel, public PoolBase {
public:
QLinearAveragePool(const OpKernelInfo& info) : OpKernel(info), PoolBase(info) {
channels_last_ = (info.GetAttrOrDefault<int64_t>("channels_last", static_cast<int64_t>(0)) != 0);
int32_t input_type = info.node().InputDefs()[0]->TypeAsProto()->tensor_type().elem_type();
is_input_signed_ = ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT8 == input_type;
}
~QLinearAveragePool() override = default;
@ -21,8 +24,12 @@ class QLinearAveragePool final : public OpKernel, public PoolBase {
Status Compute(OpKernelContext* context) const override;
private:
template <typename T8Bits>
Status ComputeImpl(OpKernelContext* context) const;
PoolProcessContext pool_context_;
bool channels_last_;
bool is_input_signed_;
};
} // namespace contrib

41
onnxruntime/contrib_ops/cpu/quantization/nhwc_max_pool.cc
View file

@ -11,6 +11,7 @@
namespace onnxruntime {
namespace contrib {
template <typename T8Bits>
class NhwcMaxPool : public OpKernel {
public:
explicit NhwcMaxPool(const OpKernelInfo& info) : OpKernel(info),
@ -20,10 +21,11 @@ class NhwcMaxPool : public OpKernel {
Status Compute(OpKernelContext* context) const override;
private:
PoolAttributes pool_attrs_;
PoolAttributes pool_attrs_;
};
Status NhwcMaxPool::Compute(OpKernelContext* context) const {
template <typename T8Bits>
Status NhwcMaxPool<T8Bits>::Compute(OpKernelContext* context) const {
const auto* X = context->Input<Tensor>(0);
const TensorShape& input_shape = X->Shape();
@ -73,17 +75,17 @@ Status NhwcMaxPool::Compute(OpKernelContext* context) const {
AllocatorPtr alloc;
ORT_RETURN_IF_ERROR(context->GetTempSpaceAllocator(&alloc));
int64_t col_buffer_batch_count = std::min(output_image_size, output_batch_count);
auto* col_data = alloc->Alloc(SafeInt<size_t>(sizeof(const uint8_t*)) * kernel_size * col_buffer_batch_count);
auto* col_data = alloc->Alloc(SafeInt<size_t>(sizeof(const T8Bits*)) * kernel_size * col_buffer_batch_count);
BufferUniquePtr col_buffer(col_data, BufferDeleter(alloc));
std::vector<uint8_t> padding_data(static_cast<size_t>(C), 0);
std::vector<T8Bits> padding_data(static_cast<size_t>(C), std::numeric_limits<T8Bits>::lowest());
const auto* Xdata = X->template Data<uint8_t>();
auto* Ydata = Y->template MutableData<uint8_t>();
const auto* Xdata = X->template Data<T8Bits>();
auto* Ydata = Y->template MutableData<T8Bits>();
for (int64_t image_id = 0; image_id < N; ++image_id) {
for (int64_t output_start = 0; output_start < output_image_size;) {
int64_t output_count = std::min(output_image_size - output_start, output_batch_count);
math::Im2col<uint8_t, StorageOrder::NHWC>()(
math::Im2col<T8Bits, StorageOrder::NHWC>()(
Xdata,
C,
input_shape.GetDims().data() + 1,
@ -95,10 +97,10 @@ Status NhwcMaxPool::Compute(OpKernelContext* context) const {
static_cast<ptrdiff_t>(spatial_dims),
output_start,
output_count,
static_cast<uint8_t const**>(col_buffer.get()),
static_cast<T8Bits const**>(col_buffer.get()),
padding_data.data());
MlasMaximumPool(
static_cast<uint8_t const**>(col_buffer.get()),
static_cast<T8Bits const**>(col_buffer.get()),
Ydata,
static_cast<size_t>(C),
static_cast<size_t>(output_count),
@ -114,14 +116,19 @@ Status NhwcMaxPool::Compute(OpKernelContext* context) const {
return Status::OK();
}
ONNX_OPERATOR_KERNEL_EX(
NhwcMaxPool,
kMSDomain,
1,
kCpuExecutionProvider,
KernelDefBuilder()
.TypeConstraint("T", DataTypeImpl::GetTensorType<uint8_t>()),
NhwcMaxPool);
#define REGISTER_NHWCMAXPOOL_TYPED_KERNEL(T) \
ONNX_OPERATOR_TYPED_KERNEL_EX( \
NhwcMaxPool, \
kMSDomain, \
1, \
T, \
kCpuExecutionProvider, \
KernelDefBuilder() \
.TypeConstraint("T", DataTypeImpl::GetTensorType<T>()), \
NhwcMaxPool<T>);
REGISTER_NHWCMAXPOOL_TYPED_KERNEL(int8_t);
REGISTER_NHWCMAXPOOL_TYPED_KERNEL(uint8_t);
} // namespace contrib
} // namespace onnxruntime

22
onnxruntime/core/mlas/inc/mlas.h
View file

@ -786,11 +786,12 @@ MlasPool(
MLAS_THREADPOOL* ThreadPool
);
template<typename T8Bits>
void
MLASCALL
MlasMaximumPool(
const uint8_t* const* Input,
uint8_t* Output,
const T8Bits* const* Input,
T8Bits* Output,
size_t Channels,
size_t OutputCount,
size_t KernelSize
@ -1041,17 +1042,18 @@ MlasQuantizeLinear(
* @param CountN
* @return
*/
template<typename OutputType>
void
MLASCALL
MlasRequantizeOutput(
const int32_t* Input,
size_t InputLeadingDimension,
uint8_t* Output,
OutputType* Output,
size_t OutputLeadingDimension,
const int32_t* Bias,
const float* Scale,
bool PerColumnScale,
uint8_t ZeroPoint,
OutputType ZeroPoint,
size_t StartM,
size_t StartN,
size_t CountM,
@ -1115,13 +1117,14 @@ MlasQLinearSafePaddingElementCount(
size_t ElementCount
);
template<typename T8Bits>
void
MLASCALL
MlasQLinearGlobalAveragePoolNchw(
const uint8_t* Input,
const T8Bits* Input,
float ScaleInput,
int32_t ZeroPointInput,
uint8_t* Output,
T8Bits* Output,
float ScaleOutput,
int32_t ZeroPointOutput,
size_t Channels,
@ -1129,13 +1132,14 @@ MlasQLinearGlobalAveragePoolNchw(
int32_t* AccumulateBuffer
);
template <typename T8Bits>
void
MLASCALL
MlasQLinearGlobalAveragePoolNhwc(
const uint8_t* Input,
const T8Bits* Input,
float ScaleInput,
int32_t ZeroPointInput,
uint8_t* Output,
T8Bits* Output,
float ScaleOutput,
int32_t ZeroPointOutput,
size_t Batch,
@ -1143,7 +1147,7 @@ MlasQLinearGlobalAveragePoolNhwc(
size_t Stride,
size_t Channels,
int32_t* AccumulateBuffer,
const uint8_t* ZeroBuffer
const T8Bits* ZeroBuffer
);
//

146
onnxruntime/core/mlas/lib/pooling.cpp
View file

@ -1304,11 +1304,12 @@ Return Value:
#endif
}
template<typename T8Bits>
void
MLASCALL
MlasMaximumPool(
const uint8_t* const* Input,
uint8_t* Output,
const T8Bits* const* Input,
T8Bits* Output,
size_t Channels,
size_t OutputCount,
size_t KernelSize
@ -1352,6 +1353,10 @@ Return Value:
size_t c = Channels;
#if defined(MLAS_SSE2_INTRINSICS)
const __m128i BitFlipVector = _mm_set1_epi32(0x80808080);
if constexpr (std::is_unsigned<T8Bits>::value) {
MLAS_UNREFERENCED_PARAMETER(BitFlipVector);
}
while (c >= 32) {
@ -1363,10 +1368,20 @@ Return Value:
__m128i InputVector0 = _mm_loadu_si128((const __m128i*)&Input[k][ChannelOffset]);
__m128i InputVector1 = _mm_loadu_si128((const __m128i*)&Input[k][ChannelOffset + 16]);
if constexpr (std::is_signed<T8Bits>::value) {
InputVector0 = _mm_xor_si128(InputVector0, BitFlipVector);
InputVector1 = _mm_xor_si128(InputVector1, BitFlipVector);
}
MaximumVector0 = _mm_max_epu8(MaximumVector0, InputVector0);
MaximumVector1 = _mm_max_epu8(MaximumVector1, InputVector1);
}
if constexpr (std::is_signed<T8Bits>::value) {
MaximumVector0 = _mm_xor_si128(MaximumVector0, BitFlipVector);
MaximumVector1 = _mm_xor_si128(MaximumVector1, BitFlipVector);
}
_mm_storeu_si128((__m128i*)&Output[0], MaximumVector0);
_mm_storeu_si128((__m128i*)&Output[16], MaximumVector1);
Output += 32;
@ -1383,9 +1398,17 @@ Return Value:
__m128i InputVector0 = _mm_loadu_si128((const __m128i*)&Input[k][ChannelOffset]);
if constexpr (std::is_signed<T8Bits>::value){
InputVector0 = _mm_xor_si128(InputVector0, BitFlipVector);
}
MaximumVector0 = _mm_max_epu8(MaximumVector0, InputVector0);
}
if constexpr (std::is_signed<T8Bits>::value) {
MaximumVector0 = _mm_xor_si128(MaximumVector0, BitFlipVector);
}
_mm_storeu_si128((__m128i*)&Output[0], MaximumVector0);
Output += 16;
@ -1401,9 +1424,17 @@ Return Value:
__m128i InputVector0 = _mm_loadl_epi64((const __m128i*)&Input[k][ChannelOffset]);
if constexpr (std::is_signed<T8Bits>::value){
InputVector0 = _mm_xor_si128(InputVector0, BitFlipVector);
}
MaximumVector0 = _mm_max_epu8(MaximumVector0, InputVector0);
}
if constexpr (std::is_signed<T8Bits>::value) {
MaximumVector0 = _mm_xor_si128(MaximumVector0, BitFlipVector);
}
_mm_storel_epi64((__m128i*)&Output[0], MaximumVector0);
Output += 8;
@ -1415,20 +1446,40 @@ Return Value:
while (c >= 32) {
uint8x16_t MaximumVector0 = vdupq_n_u8(0);
uint8x16_t MaximumVector1 = vdupq_n_u8(0);
if constexpr (std::is_signed<T8Bits>::value){
for (size_t k = 0; k < KernelSize; k++) {
int8x16_t MaximumVector0 = vdupq_n_s8(-128);
int8x16_t MaximumVector1 = vdupq_n_s8(-128);
uint8x16_t InputVector0 = vld1q_u8(&Input[k][ChannelOffset]);
uint8x16_t InputVector1 = vld1q_u8(&Input[k][ChannelOffset + 16]);
for (size_t k = 0; k < KernelSize; k++) {
MaximumVector0 = vmaxq_u8(MaximumVector0, InputVector0);
MaximumVector1 = vmaxq_u8(MaximumVector1, InputVector1);
int8x16_t InputVector0 = vld1q_s8(&Input[k][ChannelOffset]);
int8x16_t InputVector1 = vld1q_s8(&Input[k][ChannelOffset + 16]);
MaximumVector0 = vmaxq_s8(MaximumVector0, InputVector0);
MaximumVector1 = vmaxq_s8(MaximumVector1, InputVector1);
}
vst1q_s8(&Output[0], MaximumVector0);
vst1q_s8(&Output[16], MaximumVector1);
} else {
uint8x16_t MaximumVector0 = vdupq_n_u8(0);
uint8x16_t MaximumVector1 = vdupq_n_u8(0);
for (size_t k = 0; k < KernelSize; k++) {
uint8x16_t InputVector0 = vld1q_u8(&Input[k][ChannelOffset]);
uint8x16_t InputVector1 = vld1q_u8(&Input[k][ChannelOffset + 16]);
MaximumVector0 = vmaxq_u8(MaximumVector0, InputVector0);
MaximumVector1 = vmaxq_u8(MaximumVector1, InputVector1);
}
vst1q_u8(&Output[0], MaximumVector0);
vst1q_u8(&Output[16], MaximumVector1);
}
vst1q_u8(&Output[0], MaximumVector0);
vst1q_u8(&Output[16], MaximumVector1);
Output += 32;
ChannelOffset += 32;
@ -1437,16 +1488,30 @@ Return Value:
while (c >= 16) {
uint8x16_t MaximumVector0 = vdupq_n_u8(0);
if constexpr (std::is_signed<T8Bits>::value){
for (size_t k = 0; k < KernelSize; k++) {
int8x16_t MaximumVector0 = vdupq_n_s8(-128);
uint8x16_t InputVector0 = vld1q_u8(&Input[k][ChannelOffset]);
for (size_t k = 0; k < KernelSize; k++) {
MaximumVector0 = vmaxq_u8(MaximumVector0, InputVector0);
int8x16_t InputVector0 = vld1q_s8(&Input[k][ChannelOffset]);
MaximumVector0 = vmaxq_s8(MaximumVector0, InputVector0);
}
vst1q_s8(&Output[0], MaximumVector0);
} else {
uint8x16_t MaximumVector0 = vdupq_n_u8(0);
for (size_t k = 0; k < KernelSize; k++) {
uint8x16_t InputVector0 = vld1q_u8(&Input[k][ChannelOffset]);
MaximumVector0 = vmaxq_u8(MaximumVector0, InputVector0);
}
vst1q_u8(&Output[0], MaximumVector0);
}
vst1q_u8(&Output[0], MaximumVector0);
Output += 16;
ChannelOffset += 16;
@ -1455,16 +1520,29 @@ Return Value:
if (c >= 8) {
uint8x8_t MaximumVector0 = vdup_n_u8(0);
if constexpr (std::is_signed<T8Bits>::value){
for (size_t k = 0; k < KernelSize; k++) {
int8x8_t MaximumVector0 = vdup_n_s8(-128);
uint8x8_t InputVector0 = vld1_u8(&Input[k][ChannelOffset]);
for (size_t k = 0; k < KernelSize; k++) {
MaximumVector0 = vmax_u8(MaximumVector0, InputVector0);
int8x8_t InputVector0 = vld1_s8(&Input[k][ChannelOffset]);
MaximumVector0 = vmax_s8(MaximumVector0, InputVector0);
}
vst1_s8(&Output[0], MaximumVector0);
} else {
uint8x8_t MaximumVector0 = vdup_n_u8(0);
for (size_t k = 0; k < KernelSize; k++) {
uint8x8_t InputVector0 = vld1_u8(&Input[k][ChannelOffset]);
MaximumVector0 = vmax_u8(MaximumVector0, InputVector0);
}
vst1_u8(&Output[0], MaximumVector0);
}
vst1_u8(&Output[0], MaximumVector0);
Output += 8;
ChannelOffset += 8;
@ -1475,13 +1553,13 @@ Return Value:
while (c > 0) {
int32_t MaximumValue = 0;
int32_t MaximumValue = std::numeric_limits<T8Bits>::lowest();
for (size_t k = 0; k < KernelSize; k++) {
MaximumValue = std::max(MaximumValue, int32_t(Input[k][ChannelOffset]));
}
*Output++ = uint8_t(MaximumValue);
*Output++ = T8Bits(MaximumValue);
ChannelOffset += 1;
c -= 1;
@ -1491,3 +1569,25 @@ Return Value:
OutputCount -= 1;
}
}
template
void
MLASCALL
MlasMaximumPool<int8_t>(
const int8_t* const* Input,
int8_t* Output,
size_t Channels,
size_t OutputCount,
size_t KernelSize
);
template
void
MLASCALL
MlasMaximumPool<uint8_t>(
const uint8_t* const* Input,
uint8_t* Output,
size_t Channels,
size_t OutputCount,
size_t KernelSize
);

587
onnxruntime/core/mlas/lib/qlgavgpool.cpp
View file

@ -24,6 +24,7 @@ MlasQLinearSafePaddingElementCount(
)
{
if (!(ElementSize == 1 || ElementSize == 2 || ElementSize == 4 || ElementSize == 8 || ElementSize == 16)) {
#ifdef MLAS_NO_EXCEPTION
abort();
#else
@ -42,6 +43,7 @@ CheckQLinearGlobalAveragePoolScaleAndSize(
)
{
if (ImageSize >= 0x1000000) {
#ifdef MLAS_NO_EXCEPTION
abort();
#else
@ -51,6 +53,7 @@ CheckQLinearGlobalAveragePoolScaleAndSize(
float scale = ScaleInput / (ScaleOutput * static_cast<float>(ImageSize));
if (scale < 0x1.0p-32f || scale >= 256.0f) {
// In first case, the scale is too small, ScaleInput/ScaleOutput < 1/256 no matter what ImageSize
// In second case, the scale is too large, ScaleInput/ScaleOutput >= 256 no matter what Image Size
// both case make output value constant, and hence not meaningful.
@ -65,13 +68,14 @@ CheckQLinearGlobalAveragePoolScaleAndSize(
#if defined(MLAS_NEON_INTRINSICS)
template <typename T8Bits>
void
MLASCALL
MlasQLinearGlobalAveragePoolNchw(
const uint8_t* Input,
const T8Bits* Input,
float ScaleInput,
int32_t ZeroPointInput,
uint8_t* Output,
T8Bits* Output,
float ScaleOutput,
int32_t ZeroPointOutput,
size_t Channels,
@ -83,38 +87,65 @@ MlasQLinearGlobalAveragePoolNchw(
int32_t bias[] = {-ZeroPointInput * static_cast<int32_t>(ImageSize), 0, 0, 0};
const int32x4_t vbias = vld1q_s32(bias);
const int32x4_t vzero = vmovq_n_s32(0);
const uint8_t* InputU8 = (const uint8_t*)(Input);
int32_t* sum_buffer = AccumulateBuffer;
uint8_t tail_buffer[8] = {0, 0, 0, 0, 0, 0, 0, 0};
for (size_t c = Channels; c > 0; c--) {
int32x4_t vacc_lo = vbias;
int32x4_t vacc_hi = vzero;
auto Len = ImageSize;
for (; Len >= 32; Len -= 32) {
const uint8x8_t vi0 = vld1_u8(Input);
const uint8x8_t vi1 = vld1_u8(Input + 8);
const uint8x8_t vi2 = vld1_u8(Input + 16);
const uint8x8_t vi3 = vld1_u8(Input + 24);
const uint16x8_t vs01 = vaddl_u8(vi0, vi1);
const uint16x8_t vs23 = vaddl_u8(vi2, vi3);
const int16x8_t vsum = vreinterpretq_s16_u16(vaddq_u16(vs01, vs23));
const uint8x8_t vi0 = vld1_u8(InputU8);
const uint8x8_t vi1 = vld1_u8(InputU8 + 8);
const uint8x8_t vi2 = vld1_u8(InputU8 + 16);
const uint8x8_t vi3 = vld1_u8(InputU8 + 24);
int16x8_t vsum;
if constexpr (std::is_signed<T8Bits>::value) {
const int16x8_t vs01 = vaddl_s8(vreinterpret_s8_u8(vi0), vreinterpret_s8_u8(vi1));
const int16x8_t vs23 = vaddl_s8(vreinterpret_s8_u8(vi2), vreinterpret_s8_u8(vi3));
vsum = vaddq_s16(vs01, vs23);
} else {
const uint16x8_t vs01 = vaddl_u8(vi0, vi1);
const uint16x8_t vs23 = vaddl_u8(vi2, vi3);
vsum = vreinterpretq_s16_u16(vaddq_u16(vs01, vs23));
}
vacc_lo = vaddw_s16(vacc_lo, vget_low_s16(vsum));
vacc_hi = vaddw_s16(vacc_hi, vget_high_s16(vsum));
Input += 32;
InputU8 += 32;
}
for (; Len >= 8; Len -= 8) {
const int16x8_t vsum = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(Input)));
int16x8_t vsum;
if constexpr (std::is_signed<T8Bits>::value) {
vsum = vmovl_s8(vreinterpret_s8_u8(vld1_u8(InputU8)));
} else {
vsum = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(InputU8)));
}
vacc_lo = vaddw_s16(vacc_lo, vget_low_s16(vsum));
vacc_hi = vaddw_s16(vacc_hi, vget_high_s16(vsum));
Input += 8;
InputU8 += 8;
}
if (Len > 0) {
memcpy(tail_buffer, Input, Len);
const int16x8_t vsum = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(tail_buffer)));
memcpy(tail_buffer, InputU8, Len);
int16x8_t vsum;
if constexpr (std::is_signed<T8Bits>::value) {
vsum = vmovl_s8(vreinterpret_s8_u8(vld1_u8(tail_buffer)));
} else {
vsum = vreinterpretq_s16_u16(vmovl_u8(vld1_u8(tail_buffer)));
}
vacc_lo = vaddw_s16(vacc_lo, vget_low_s16(vsum));
vacc_hi = vaddw_s16(vacc_hi, vget_high_s16(vsum));
Input += Len;
InputU8 += Len;
}
vacc_lo = vaddq_s32(vacc_lo, vacc_hi);
@ -123,23 +154,24 @@ MlasQLinearGlobalAveragePoolNchw(
}
MlasRequantizeOutput(AccumulateBuffer, Channels, Output, Channels, nullptr, &scale, false,
static_cast<uint8_t>(ZeroPointOutput), 0, 0, 1, Channels);
static_cast<T8Bits>(ZeroPointOutput), 0, 0, 1, Channels);
}
template <typename T8Bits>
MLAS_FORCEINLINE
void
MlasQLinearGlobalAveragePoolNhwcSingleBatch(
const uint8_t* Input,
uint8_t* Output,
const uint8_t* LastOf8,
const T8Bits* Input,
T8Bits* Output,
const T8Bits* LastOf8,
size_t ImageSize,
size_t Channels,
size_t Stride,
int32_t Bias,
float Scale,
uint8_t Output_zero_point,
T8Bits Output_zero_point,
int32_t* AccumulateBuffer,
const uint8_t* ZeroBuffer
const T8Bits* ZeroBuffer
)
{
#define LOAD_FULL_CHANNELS() \
@ -158,24 +190,35 @@ MlasQLinearGlobalAveragePoolNhwcSingleBatch(
const uint8x8_t vi6 = vld1_u8(i6); \
i6 += 8
#define CALCULATE_ACCUMULATE_VECTORS() \
int32x4_t vacc_lo = finish_one_pass ? vld1q_s32(acc) : vbias; \
int32x4_t vacc_hi = finish_one_pass ? vld1q_s32(acc + 4) : vbias; \
const uint16x8_t vsum01 = vaddl_u8(vi0, vi1); \
const uint16x8_t vsum23 = vaddl_u8(vi2, vi3); \
const uint16x8_t vsum45 = vaddl_u8(vi4, vi5); \
const uint16x8_t vsum016 = vaddw_u8(vsum01, vi6); \
const uint16x8_t vsum2345 = vaddq_u16(vsum23, vsum45); \
const int16x8_t vsum = vreinterpretq_s16_u16(vaddq_u16(vsum016, vsum2345)); \
vacc_lo = vaddw_s16(vacc_lo, vget_low_s16(vsum)); \
#define CALCULATE_ACCUMULATE_VECTORS() \
int32x4_t vacc_lo = finish_one_pass ? vld1q_s32(acc) : vbias; \
int32x4_t vacc_hi = finish_one_pass ? vld1q_s32(acc + 4) : vbias; \
int16x8_t vsum; \
if constexpr (std::is_signed<T8Bits>::value) { \
const int16x8_t vsum01 = vaddl_s8(vreinterpret_s8_u8(vi0), vreinterpret_s8_u8(vi1)); \
const int16x8_t vsum23 = vaddl_s8(vreinterpret_s8_u8(vi2), vreinterpret_s8_u8(vi3)); \
const int16x8_t vsum45 = vaddl_s8(vreinterpret_s8_u8(vi4), vreinterpret_s8_u8(vi5)); \
const int16x8_t vsum016 = vaddw_s8(vsum01, vreinterpret_s8_u8(vi6)); \
const int16x8_t vsum2345 = vaddq_s16(vsum23, vsum45); \
vsum = vaddq_s16(vsum016, vsum2345); \
} else { \
const uint16x8_t vsum01 = vaddl_u8(vi0, vi1); \
const uint16x8_t vsum23 = vaddl_u8(vi2, vi3); \
const uint16x8_t vsum45 = vaddl_u8(vi4, vi5); \
const uint16x8_t vsum016 = vaddw_u8(vsum01, vi6); \
const uint16x8_t vsum2345 = vaddq_u16(vsum23, vsum45); \
vsum = vreinterpretq_s16_u16(vaddq_u16(vsum016, vsum2345)); \
} \
vacc_lo = vaddw_s16(vacc_lo, vget_low_s16(vsum)); \
vacc_hi = vaddw_s16(vacc_hi, vget_high_s16(vsum))
uint8_t tail[8] = { 0, 0, 0, 0, 0, 0, 0, 0 };
uint8_t tail[8] = {0, 0, 0, 0, 0, 0, 0, 0};
const int32x4_t vbias = vld1q_dup_s32(&Bias);
bool finish_one_pass = false;
const size_t step_next_group = 7 * Stride - (Channels & ~size_t{7});
const uint8_t* i0 = Input;
const uint8_t* LastOf8U8 = (const uint8_t*)LastOf8;
const uint8_t* i0 = (const uint8_t*)Input;
const uint8_t* i1 = i0 + Stride;
const uint8_t* i4 = i0 + Stride * 4;
const uint8_t* i2 = i1 + Stride;
@ -184,9 +227,11 @@ MlasQLinearGlobalAveragePoolNhwcSingleBatch(
const uint8_t* i6 = i5 + Stride;
for (; ImageSize > 7; ImageSize -= 7) {
int32_t* acc = AccumulateBuffer;
size_t c = Channels;
for (; c >= 8; c -= 8) {
LOAD_FULL_CHANNELS();
CALCULATE_ACCUMULATE_VECTORS();
@ -196,13 +241,14 @@ MlasQLinearGlobalAveragePoolNhwcSingleBatch(
acc += 8;
}
if (c > 0) {
const uint8x8_t vi0 = vld1_u8(((i0 >= LastOf8) ? (const uint8_t*)memcpy(tail, i0, c) : i0));
const uint8x8_t vi1 = vld1_u8(((i1 >= LastOf8) ? (const uint8_t*)memcpy(tail, i1, c) : i1));
const uint8x8_t vi2 = vld1_u8(((i2 >= LastOf8) ? (const uint8_t*)memcpy(tail, i2, c) : i2));
const uint8x8_t vi3 = vld1_u8(((i3 >= LastOf8) ? (const uint8_t*)memcpy(tail, i3, c) : i3));
const uint8x8_t vi4 = vld1_u8(((i4 >= LastOf8) ? (const uint8_t*)memcpy(tail, i4, c) : i4));
const uint8x8_t vi5 = vld1_u8(((i5 >= LastOf8) ? (const uint8_t*)memcpy(tail, i5, c) : i5));
const uint8x8_t vi6 = vld1_u8(((i6 >= LastOf8) ? (const uint8_t*)memcpy(tail, i6, c) : i6));
const uint8x8_t vi0 = vld1_u8(((i0 >= LastOf8U8) ? (const uint8_t*)memcpy(tail, i0, c) : i0));
const uint8x8_t vi1 = vld1_u8(((i1 >= LastOf8U8) ? (const uint8_t*)memcpy(tail, i1, c) : i1));
const uint8x8_t vi2 = vld1_u8(((i2 >= LastOf8U8) ? (const uint8_t*)memcpy(tail, i2, c) : i2));
const uint8x8_t vi3 = vld1_u8(((i3 >= LastOf8U8) ? (const uint8_t*)memcpy(tail, i3, c) : i3));
const uint8x8_t vi4 = vld1_u8(((i4 >= LastOf8U8) ? (const uint8_t*)memcpy(tail, i4, c) : i4));
const uint8x8_t vi5 = vld1_u8(((i5 >= LastOf8U8) ? (const uint8_t*)memcpy(tail, i5, c) : i5));
const uint8x8_t vi6 = vld1_u8(((i6 >= LastOf8U8) ? (const uint8_t*)memcpy(tail, i6, c) : i6));
CALCULATE_ACCUMULATE_VECTORS();
@ -221,19 +267,28 @@ MlasQLinearGlobalAveragePoolNhwcSingleBatch(
}
if (ImageSize > 0) {
switch (ImageSize) {
case 1: i1 = ZeroBuffer; /* fall through */
case 2: i2 = ZeroBuffer; /* fall through */
case 3: i3 = ZeroBuffer; /* fall through */
case 4: i4 = ZeroBuffer; /* fall through */
case 5: i5 = ZeroBuffer; /* fall through */
case 6: i6 = ZeroBuffer; /* fall through */
default: break;
case 1:
i1 = (const uint8_t*)ZeroBuffer; /* fall through */
case 2:
i2 = (const uint8_t*)ZeroBuffer; /* fall through */
case 3:
i3 = (const uint8_t*)ZeroBuffer; /* fall through */
case 4:
i4 = (const uint8_t*)ZeroBuffer; /* fall through */
case 5:
i5 = (const uint8_t*)ZeroBuffer; /* fall through */
case 6:
i6 = (const uint8_t*)ZeroBuffer; /* fall through */
default:
break;
}
int32_t* acc = AccumulateBuffer;
size_t c = Channels;
for (; c >= 8; c -= 8) {
LOAD_FULL_CHANNELS();
CALCULATE_ACCUMULATE_VECTORS();
@ -244,13 +299,21 @@ MlasQLinearGlobalAveragePoolNhwcSingleBatch(
}
if (c > 0) {
const uint8x8_t vi0 = vld1_u8(((i0 >= LastOf8) ? (const uint8_t*)memcpy(tail, i0, c) : i0));
const uint8x8_t vi1 = vld1_u8(((1 < ImageSize && i1 >= LastOf8) ? (const uint8_t*)memcpy(tail, i1, c) : i1));
const uint8x8_t vi2 = vld1_u8(((2 < ImageSize && i2 >= LastOf8) ? (const uint8_t*)memcpy(tail, i2, c) : i2));
const uint8x8_t vi3 = vld1_u8(((3 < ImageSize && i3 >= LastOf8) ? (const uint8_t*)memcpy(tail, i3, c) : i3));
const uint8x8_t vi4 = vld1_u8(((4 < ImageSize && i4 >= LastOf8) ? (const uint8_t*)memcpy(tail, i4, c) : i4));
const uint8x8_t vi5 = vld1_u8(((5 < ImageSize && i5 >= LastOf8) ? (const uint8_t*)memcpy(tail, i5, c) : i5));
const uint8x8_t vi6 = vld1_u8(((6 < ImageSize && i6 >= LastOf8) ? (const uint8_t*)memcpy(tail, i6, c) : i6));
const uint8x8_t vi0 =
vld1_u8(((i0 >= LastOf8U8) ? (const uint8_t*)memcpy(tail, i0, c) : i0));
const uint8x8_t vi1 = vld1_u8(
((1 < ImageSize && i1 >= LastOf8U8) ? (const uint8_t*)memcpy(tail, i1, c) : i1));
const uint8x8_t vi2 = vld1_u8(
((2 < ImageSize && i2 >= LastOf8U8) ? (const uint8_t*)memcpy(tail, i2, c) : i2));
const uint8x8_t vi3 = vld1_u8(
((3 < ImageSize && i3 >= LastOf8U8) ? (const uint8_t*)memcpy(tail, i3, c) : i3));
const uint8x8_t vi4 = vld1_u8(
((4 < ImageSize && i4 >= LastOf8U8) ? (const uint8_t*)memcpy(tail, i4, c) : i4));
const uint8x8_t vi5 = vld1_u8(
((5 < ImageSize && i5 >= LastOf8U8) ? (const uint8_t*)memcpy(tail, i5, c) : i5));
const uint8x8_t vi6 = vld1_u8(
((6 < ImageSize && i6 >= LastOf8U8) ? (const uint8_t*)memcpy(tail, i6, c) : i6));
CALCULATE_ACCUMULATE_VECTORS();
@ -264,13 +327,13 @@ MlasQLinearGlobalAveragePoolNhwcSingleBatch(
#elif defined(MLAS_SSE2_INTRINSICS)
void
MLASCALL
template <typename T8Bits>
void MLASCALL
MlasQLinearGlobalAveragePoolNchw(
const uint8_t* Input,
const T8Bits* Input,
float ScaleInput,
int32_t ZeroPointInput,
uint8_t* Output,
T8Bits* Output,
float ScaleOutput,
int32_t ZeroPointOutput,
size_t Channels,
@ -286,36 +349,73 @@ MlasQLinearGlobalAveragePoolNchw(
int32_t* sum_buffer = AccumulateBuffer;
for (size_t c = Channels; c > 0; c--) {
__m128i vacc_lo = vbias;
__m128i vacc_hi = vzero;
auto Len = ImageSize;
for (; Len >= 32; Len -= 32) {
const __m128i vi0 = _mm_loadl_epi64((const __m128i*)Input);
const __m128i vi1 = _mm_loadl_epi64((const __m128i*)(Input + 8));
const __m128i vi2 = _mm_loadl_epi64((const __m128i*)(Input + 16));
const __m128i vi3 = _mm_loadl_epi64((const __m128i*)(Input + 24));
const __m128i vxi0 = _mm_unpacklo_epi8(vi0, vzero);
const __m128i vxi1 = _mm_unpacklo_epi8(vi1, vzero);
const __m128i vxi2 = _mm_unpacklo_epi8(vi2, vzero);
const __m128i vxi3 = _mm_unpacklo_epi8(vi3, vzero);
if constexpr (std::is_signed<T8Bits>::value) {
const __m128i vxi0 = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, vi0), 8);
const __m128i vxi1 = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, vi1), 8);
const __m128i vxi2 = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, vi2), 8);
const __m128i vxi3 = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, vi3), 8);
const __m128i vsum = _mm_add_epi16(_mm_add_epi16(vxi0, vxi1),
_mm_add_epi16(vxi2, vxi3));
vacc_lo = _mm_add_epi32(vacc_lo, _mm_srai_epi32(_mm_unpacklo_epi16(vzero, vsum), 16));
vacc_hi = _mm_add_epi32(vacc_hi, _mm_srai_epi32(_mm_unpackhi_epi16(vzero, vsum), 16));
} else {
const __m128i vxi0 = _mm_unpacklo_epi8(vi0, vzero);
const __m128i vxi1 = _mm_unpacklo_epi8(vi1, vzero);
const __m128i vxi2 = _mm_unpacklo_epi8(vi2, vzero);
const __m128i vxi3 = _mm_unpacklo_epi8(vi3, vzero);
const __m128i vsum = _mm_add_epi16(_mm_add_epi16(vxi0, vxi1),
_mm_add_epi16(vxi2, vxi3));
vacc_lo = _mm_add_epi32(vacc_lo, _mm_unpacklo_epi16(vsum, vzero));
vacc_hi = _mm_add_epi32(vacc_hi, _mm_unpackhi_epi16(vsum, vzero));
}
const __m128i vsum = _mm_add_epi16(_mm_add_epi16(vxi0, vxi1), _mm_add_epi16(vxi2, vxi3));
vacc_lo = _mm_add_epi32(vacc_lo, _mm_unpacklo_epi16(vsum, vzero));
vacc_hi = _mm_add_epi32(vacc_hi, _mm_unpackhi_epi16(vsum, vzero));
Input += 32;
}
for (; Len >= 8; Len -= 8) {
const __m128i vsum = _mm_unpacklo_epi8(_mm_loadl_epi64((const __m128i*)Input), vzero);
vacc_lo = _mm_add_epi32(vacc_lo, _mm_unpacklo_epi16(vsum, vzero));
vacc_hi = _mm_add_epi32(vacc_hi, _mm_unpackhi_epi16(vsum, vzero));
if constexpr (std::is_signed<T8Bits>::value) {
const __m128i vsum = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, _mm_loadl_epi64((const __m128i*)Input)), 8);
vacc_lo = _mm_add_epi32(vacc_lo, _mm_srai_epi32(_mm_unpacklo_epi16(vzero, vsum), 16));
vacc_hi = _mm_add_epi32(vacc_hi, _mm_srai_epi32(_mm_unpackhi_epi16(vzero, vsum), 16));
} else {
const __m128i vsum = _mm_unpacklo_epi8(_mm_loadl_epi64((const __m128i*)Input), vzero);
vacc_lo = _mm_add_epi32(vacc_lo, _mm_unpacklo_epi16(vsum, vzero));
vacc_hi = _mm_add_epi32(vacc_hi, _mm_unpackhi_epi16(vsum, vzero));
}
Input += 8;
}
if (Len > 0) {
memcpy(buffer, Input, Len);
const __m128i vsum = _mm_unpacklo_epi8(_mm_loadl_epi64((const __m128i*)buffer), vzero);
vacc_lo = _mm_add_epi32(vacc_lo, _mm_unpacklo_epi16(vsum, vzero));
vacc_hi = _mm_add_epi32(vacc_hi, _mm_unpackhi_epi16(vsum, vzero));
if constexpr (std::is_signed<T8Bits>::value) {
const __m128i vsum = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, _mm_loadl_epi64((const __m128i*)buffer)), 8);
vacc_lo = _mm_add_epi32(vacc_lo, _mm_srai_epi32(_mm_unpacklo_epi16(vzero, vsum), 16));
vacc_hi = _mm_add_epi32(vacc_hi, _mm_srai_epi32(_mm_unpackhi_epi16(vzero, vsum), 16));
} else {
const __m128i vsum = _mm_unpacklo_epi8(_mm_loadl_epi64((const __m128i*)buffer), vzero);
vacc_lo = _mm_add_epi32(vacc_lo, _mm_unpacklo_epi16(vsum, vzero));
vacc_hi = _mm_add_epi32(vacc_hi, _mm_unpackhi_epi16(vsum, vzero));
}
Input += Len;
}
@ -326,24 +426,27 @@ MlasQLinearGlobalAveragePoolNchw(
vsums = _mm_add_epi32(vsums, vshuf);
*sum_buffer++ = _mm_cvtsi128_si32(vsums);
}
MlasRequantizeOutput(AccumulateBuffer, Channels, Output, Channels, nullptr, &scale, false,
static_cast<uint8_t>(ZeroPointOutput), 0, 0, 1, Channels);
static_cast<T8Bits>(ZeroPointOutput), 0, 0, 1, Channels);
}
template <typename T8Bits>
MLAS_FORCEINLINE
void
MlasQLinearGlobalAveragePoolNhwcSingleBatch(
const uint8_t* Input,
uint8_t* Output,
const uint8_t* LastOf8,
const T8Bits* Input,
T8Bits* Output,
const T8Bits* LastOf8,
size_t ImageSize,
size_t Channels,
size_t Stride,
int32_t Bias,
float Scale,
uint8_t Output_zero_point,
T8Bits Output_zero_point,
int32_t* AccumulateBuffer,
const uint8_t* ZeroBuffer)
const T8Bits* ZeroBuffer
)
{
#if defined(MLAS_TARGET_IX86)
@ -359,23 +462,39 @@ MlasQLinearGlobalAveragePoolNhwcSingleBatch(
const __m128i vi3 = _mm_loadl_epi64((const __m128i*)i3); \
i3 += 8;
#define CALCULATE_ACCUMULATE_VECTORS() \
__m128i vacc_lo = finish_one_pass ? _mm_loadu_si128((__m128i*)acc) : vbias; \
__m128i vacc_hi = finish_one_pass ? _mm_loadu_si128(((__m128i*)acc) + 1) : vbias; \
const __m128i vxi0 = _mm_unpacklo_epi8(vi0, vzero); \
const __m128i vxi1 = _mm_unpacklo_epi8(vi1, vzero); \
const __m128i vxi2 = _mm_unpacklo_epi8(vi2, vzero); \
const __m128i vxi3 = _mm_unpacklo_epi8(vi3, vzero); \
const __m128i vsum01 = _mm_add_epi16(vxi0, vxi1); \
const __m128i vsum23 = _mm_add_epi16(vxi2, vxi3); \
const __m128i vsum = _mm_add_epi16(vsum01, vsum23); \
vacc_lo = _mm_add_epi32(vacc_lo, _mm_unpacklo_epi16(vsum, vzero)); \
vacc_hi = _mm_add_epi32(vacc_hi, _mm_unpackhi_epi16(vsum, vzero))
#define CALCULATE_ACCUMULATE_VECTORS() \
__m128i vacc_lo = finish_one_pass ? _mm_loadu_si128((__m128i*)acc) : vbias; \
__m128i vacc_hi = finish_one_pass ? _mm_loadu_si128(((__m128i*)acc) + 1) : vbias; \
__m128i vxi0; \
__m128i vxi1; \
__m128i vxi2; \
__m128i vxi3; \
if constexpr (std::is_signed<T8Bits>::value) { \
vxi0 = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, vi0), 8); \
vxi1 = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, vi1), 8); \
vxi2 = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, vi2), 8); \
vxi3 = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, vi3), 8); \
} else { \
vxi0 = _mm_unpacklo_epi8(vi0, vzero); \
vxi1 = _mm_unpacklo_epi8(vi1, vzero); \
vxi2 = _mm_unpacklo_epi8(vi2, vzero); \
vxi3 = _mm_unpacklo_epi8(vi3, vzero); \
} \
__m128i vsum01 = _mm_add_epi16(vxi0, vxi1); \
__m128i vsum23 = _mm_add_epi16(vxi2, vxi3); \
__m128i vsum = _mm_add_epi16(vsum01, vsum23); \
\
if constexpr (std::is_signed<T8Bits>::value) { \
vacc_lo = _mm_add_epi32(vacc_lo, _mm_srai_epi32(_mm_unpacklo_epi16(vzero, vsum), 16)); \
vacc_hi = _mm_add_epi32(vacc_hi, _mm_srai_epi32(_mm_unpackhi_epi16(vzero, vsum), 16)); \
} else { \
vacc_lo = _mm_add_epi32(vacc_lo, _mm_unpacklo_epi16(vsum, vzero)); \
vacc_hi = _mm_add_epi32(vacc_hi, _mm_unpackhi_epi16(vsum, vzero)); \
}
#else
constexpr size_t PixelsPerIteration = 7;
#define LOAD_FULL_CHANNELS() \
const __m128i vi0 = _mm_loadl_epi64((const __m128i*)i0); \
i0 += 8; \
@ -392,47 +511,71 @@ MlasQLinearGlobalAveragePoolNhwcSingleBatch(
const __m128i vi6 = _mm_loadl_epi64((const __m128i*)i6); \
i6 += 8
#define CALCULATE_ACCUMULATE_VECTORS() \
__m128i vacc_lo = finish_one_pass ? _mm_loadu_si128((__m128i*)acc) : vbias; \
__m128i vacc_hi = finish_one_pass ? _mm_loadu_si128(((__m128i*)acc) + 1) : vbias; \
const __m128i vxi0 = _mm_unpacklo_epi8(vi0, vzero); \
const __m128i vxi1 = _mm_unpacklo_epi8(vi1, vzero); \
const __m128i vxi2 = _mm_unpacklo_epi8(vi2, vzero); \
const __m128i vxi3 = _mm_unpacklo_epi8(vi3, vzero); \
const __m128i vxi4 = _mm_unpacklo_epi8(vi4, vzero); \
const __m128i vxi5 = _mm_unpacklo_epi8(vi5, vzero); \
const __m128i vxi6 = _mm_unpacklo_epi8(vi6, vzero); \
const __m128i vsum01 = _mm_add_epi16(vxi0, vxi1); \
const __m128i vsum23 = _mm_add_epi16(vxi2, vxi3); \
const __m128i vsum45 = _mm_add_epi16(vxi4, vxi5); \
const __m128i vsum016 = _mm_add_epi16(vsum01, vxi6); \
const __m128i vsum2345 = _mm_add_epi16(vsum23, vsum45); \
const __m128i vsum = _mm_add_epi16(vsum016, vsum2345); \
vacc_lo = _mm_add_epi32(vacc_lo, _mm_unpacklo_epi16(vsum, vzero)); \
vacc_hi = _mm_add_epi32(vacc_hi, _mm_unpackhi_epi16(vsum, vzero))
#define CALCULATE_ACCUMULATE_VECTORS() \
__m128i vacc_lo = finish_one_pass ? _mm_loadu_si128((__m128i*)acc) : vbias; \
__m128i vacc_hi = finish_one_pass ? _mm_loadu_si128(((__m128i*)acc) + 1) : vbias; \
__m128i vxi0; \
__m128i vxi1; \
__m128i vxi2; \
__m128i vxi3; \
__m128i vxi4; \
__m128i vxi5; \
__m128i vxi6; \
if constexpr (std::is_signed<T8Bits>::value) { \
vxi0 = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, vi0), 8); \
vxi1 = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, vi1), 8); \
vxi2 = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, vi2), 8); \
vxi3 = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, vi3), 8); \
vxi4 = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, vi4), 8); \
vxi5 = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, vi5), 8); \
vxi6 = _mm_srai_epi16(_mm_unpacklo_epi8(vzero, vi6), 8); \
} else { \
vxi0 = _mm_unpacklo_epi8(vi0, vzero); \
vxi1 = _mm_unpacklo_epi8(vi1, vzero); \
vxi2 = _mm_unpacklo_epi8(vi2, vzero); \
vxi3 = _mm_unpacklo_epi8(vi3, vzero); \
vxi4 = _mm_unpacklo_epi8(vi4, vzero); \
vxi5 = _mm_unpacklo_epi8(vi5, vzero); \
vxi6 = _mm_unpacklo_epi8(vi6, vzero); \
} \
const __m128i vsum01 = _mm_add_epi16(vxi0, vxi1); \
const __m128i vsum23 = _mm_add_epi16(vxi2, vxi3); \
const __m128i vsum45 = _mm_add_epi16(vxi4, vxi5); \
const __m128i vsum016 = _mm_add_epi16(vsum01, vxi6); \
const __m128i vsum2345 = _mm_add_epi16(vsum23, vsum45); \
const __m128i vsum = _mm_add_epi16(vsum016, vsum2345); \
if constexpr (std::is_signed<T8Bits>::value) { \
vacc_lo = _mm_add_epi32(vacc_lo, _mm_srai_epi32(_mm_unpacklo_epi16(vzero, vsum), 16)); \
vacc_hi = _mm_add_epi32(vacc_hi, _mm_srai_epi32(_mm_unpackhi_epi16(vzero, vsum), 16)); \
} else { \
vacc_lo = _mm_add_epi32(vacc_lo, _mm_unpacklo_epi16(vsum, vzero)); \
vacc_hi = _mm_add_epi32(vacc_hi, _mm_unpackhi_epi16(vsum, vzero)); \
}
#endif
uint8_t tail[8] = { 0, 0, 0, 0, 0, 0, 0, 0 };
T8Bits tail[8] = {0, 0, 0, 0, 0, 0, 0, 0};
bool finish_one_pass = false;
const __m128i vbias = _mm_set1_epi32(Bias);
const __m128i vzero = _mm_setzero_si128();
size_t step_next_group = PixelsPerIteration * Stride - (Channels & ~size_t{7});
const uint8_t* i0 = Input;
const uint8_t* i1 = i0 + Stride;
const uint8_t* i2 = i1 + Stride;
const uint8_t* i3 = i2 + Stride;
const T8Bits* i0 = Input;
const T8Bits* i1 = i0 + Stride;
const T8Bits* i2 = i1 + Stride;
const T8Bits* i3 = i2 + Stride;
#if !defined(MLAS_TARGET_IX86)
const uint8_t* i4 = i0 + Stride * 4;
const uint8_t* i5 = i4 + Stride;
const uint8_t* i6 = i5 + Stride;
const T8Bits* i4 = i0 + Stride * 4;
const T8Bits* i5 = i4 + Stride;
const T8Bits* i6 = i5 + Stride;
#endif
for (; ImageSize > PixelsPerIteration; ImageSize -= PixelsPerIteration) {
int32_t* acc = AccumulateBuffer;
size_t c = Channels;
for (; c >= 8; c -= 8) {
LOAD_FULL_CHANNELS();
CALCULATE_ACCUMULATE_VECTORS();
@ -442,14 +585,21 @@ MlasQLinearGlobalAveragePoolNhwcSingleBatch(
acc += 8;
}
if (c > 0) {
const __m128i vi0 = _mm_loadl_epi64((const __m128i*)(i0 >= LastOf8 ? memcpy(tail, i0, c) : i0));
const __m128i vi1 = _mm_loadl_epi64((const __m128i*)(i1 >= LastOf8 ? memcpy(tail, i1, c) : i1));
const __m128i vi2 = _mm_loadl_epi64((const __m128i*)(i2 >= LastOf8 ? memcpy(tail, i2, c) : i2));
const __m128i vi3 = _mm_loadl_epi64((const __m128i*)(i3 >= LastOf8 ? memcpy(tail, i3, c) : i3));
const __m128i vi0 =
_mm_loadl_epi64((const __m128i*)(i0 >= LastOf8 ? memcpy(tail, i0, c) : i0));
const __m128i vi1 =
_mm_loadl_epi64((const __m128i*)(i1 >= LastOf8 ? memcpy(tail, i1, c) : i1));
const __m128i vi2 =
_mm_loadl_epi64((const __m128i*)(i2 >= LastOf8 ? memcpy(tail, i2, c) : i2));
const __m128i vi3 =
_mm_loadl_epi64((const __m128i*)(i3 >= LastOf8 ? memcpy(tail, i3, c) : i3));
#if !defined(MLAS_TARGET_IX86)
const __m128i vi4 = _mm_loadl_epi64((const __m128i*)(i4 >= LastOf8 ? memcpy(tail, i4, c) : i4));
const __m128i vi5 = _mm_loadl_epi64((const __m128i*)(i5 >= LastOf8 ? memcpy(tail, i5, c) : i5));
const __m128i vi6 = _mm_loadl_epi64((const __m128i*)(i6 >= LastOf8 ? memcpy(tail, i6, c) : i6));
const __m128i vi4 =
_mm_loadl_epi64((const __m128i*)(i4 >= LastOf8 ? memcpy(tail, i4, c) : i4));
const __m128i vi5 =
_mm_loadl_epi64((const __m128i*)(i5 >= LastOf8 ? memcpy(tail, i5, c) : i5));
const __m128i vi6 =
_mm_loadl_epi64((const __m128i*)(i6 >= LastOf8 ? memcpy(tail, i6, c) : i6));
#endif
CALCULATE_ACCUMULATE_VECTORS();
@ -473,26 +623,38 @@ MlasQLinearGlobalAveragePoolNhwcSingleBatch(
if (ImageSize > 0) {
#if defined(MLAS_TARGET_IX86)
switch (ImageSize) {
case 1: i1 = ZeroBuffer; /* fall through */
case 2: i2 = ZeroBuffer; /* fall through */
case 3: i3 = ZeroBuffer; /* fall through */
default: break;
case 1:
i1 = ZeroBuffer; /* fall through */
case 2:
i2 = ZeroBuffer; /* fall through */
case 3:
i3 = ZeroBuffer; /* fall through */
default:
break;
}
#else
switch (ImageSize) {
case 1: i1 = ZeroBuffer; /* fall through */
case 2: i2 = ZeroBuffer; /* fall through */
case 3: i3 = ZeroBuffer; /* fall through */
case 4: i4 = ZeroBuffer; /* fall through */
case 5: i5 = ZeroBuffer; /* fall through */
case 6: i6 = ZeroBuffer; /* fall through */
default: break;
case 1:
i1 = ZeroBuffer; /* fall through */
case 2:
i2 = ZeroBuffer; /* fall through */
case 3:
i3 = ZeroBuffer; /* fall through */
case 4:
i4 = ZeroBuffer; /* fall through */
case 5:
i5 = ZeroBuffer; /* fall through */
case 6:
i6 = ZeroBuffer; /* fall through */
default:
break;
}
#endif
int32_t* acc = AccumulateBuffer;
size_t c = Channels;
for (; c >= 8; c -= 8) {
LOAD_FULL_CHANNELS();
CALCULATE_ACCUMULATE_VECTORS();
@ -503,14 +665,21 @@ MlasQLinearGlobalAveragePoolNhwcSingleBatch(
}
if (c > 0) {
const __m128i vi0 = _mm_loadl_epi64((const __m128i*)(i0 >= LastOf8 ? memcpy(tail, i0, c) : i0));
const __m128i vi1 = _mm_loadl_epi64((const __m128i*)(1 < ImageSize && i1 >= LastOf8 ? memcpy(tail, i1, c) : i1));
const __m128i vi2 = _mm_loadl_epi64((const __m128i*)(2 < ImageSize && i2 >= LastOf8 ? memcpy(tail, i2, c) : i2));
const __m128i vi3 = _mm_loadl_epi64((const __m128i*)(3 < ImageSize && i3 >= LastOf8 ? memcpy(tail, i3, c) : i3));
const __m128i vi0 =
_mm_loadl_epi64((const __m128i*)(i0 >= LastOf8 ? memcpy(tail, i0, c) : i0));
const __m128i vi1 = _mm_loadl_epi64(
(const __m128i*)(1 < ImageSize && i1 >= LastOf8 ? memcpy(tail, i1, c) : i1));
const __m128i vi2 = _mm_loadl_epi64(
(const __m128i*)(2 < ImageSize && i2 >= LastOf8 ? memcpy(tail, i2, c) : i2));
const __m128i vi3 = _mm_loadl_epi64(
(const __m128i*)(3 < ImageSize && i3 >= LastOf8 ? memcpy(tail, i3, c) : i3));
#if !defined(MLAS_TARGET_IX86)
const __m128i vi4 = _mm_loadl_epi64((const __m128i*)(4 < ImageSize && i4 >= LastOf8 ? memcpy(tail, i4, c) : i4));
const __m128i vi5 = _mm_loadl_epi64((const __m128i*)(5 < ImageSize && i5 >= LastOf8 ? memcpy(tail, i5, c) : i5));
const __m128i vi6 = _mm_loadl_epi64((const __m128i*)(6 < ImageSize && i6 >= LastOf8 ? memcpy(tail, i6, c) : i6));
const __m128i vi4 = _mm_loadl_epi64(
(const __m128i*)(4 < ImageSize && i4 >= LastOf8 ? memcpy(tail, i4, c) : i4));
const __m128i vi5 = _mm_loadl_epi64(
(const __m128i*)(5 < ImageSize && i5 >= LastOf8 ? memcpy(tail, i5, c) : i5));
const __m128i vi6 = _mm_loadl_epi64(
(const __m128i*)(6 < ImageSize && i6 >= LastOf8 ? memcpy(tail, i6, c) : i6));
#endif
CALCULATE_ACCUMULATE_VECTORS();
@ -527,13 +696,14 @@ MlasQLinearGlobalAveragePoolNhwcSingleBatch(
// Pure C++ Implementation
template <typename T8Bits>
void
MLASCALL
MlasQLinearGlobalAveragePoolNchw(
const uint8_t* Input,
const T8Bits* Input,
float ScaleInput,
int32_t ZeroPointInput,
uint8_t* Output,
T8Bits* Output,
float ScaleOutput,
int32_t ZeroPointOutput,
size_t Channels,
@ -544,22 +714,26 @@ MlasQLinearGlobalAveragePoolNchw(
float scale = CheckQLinearGlobalAveragePoolScaleAndSize(ScaleInput, ScaleOutput, ImageSize);
int32_t bias = -ZeroPointInput * static_cast<int32_t>(ImageSize);
for (; Channels > 0; Channels--) {
int32_t acc = bias;
for (size_t i = 0; i < ImageSize; ++i) {
acc += static_cast<int>(*Input++);
acc += static_cast<int32_t>(*Input++);
}
int32_t v = static_cast<int>(std::nearbyintf(acc * scale)) + ZeroPointOutput;
*Output++ = std::max(std::min(255, v), 0);
int32_t v = static_cast<int32_t>(std::nearbyintf(acc * scale)) + ZeroPointOutput;
v = std::min(static_cast<int32_t>(std::numeric_limits<T8Bits>::max()), v);
v = std::max(static_cast<int32_t>(std::numeric_limits<T8Bits>::lowest()), v);
*Output++ = static_cast<T8Bits>(v);
}
}
template <typename T8Bits>
void
MLASCALL
MlasQLinearGlobalAveragePoolNhwc(
const uint8_t* Input,
const T8Bits* Input,
float ScaleInput,
int32_t ZeroPointInput,
uint8_t* Output,
T8Bits* Output,
float ScaleOutput,
int32_t ZeroPointOutput,
size_t Batch,
@ -567,26 +741,33 @@ MlasQLinearGlobalAveragePoolNhwc(
size_t Stride,
size_t Channels,
int32_t* AccumulateBuffer,
const uint8_t* /* ZeroBuffer */
const T8Bits* ZeroBuffer
)
{
float scale = CheckQLinearGlobalAveragePoolScaleAndSize(ScaleInput, ScaleOutput, ImageSize);
int32_t bias = -ZeroPointInput * static_cast<int32_t>(ImageSize);
for (; Batch > 0; Batch--) {
const uint8_t* batch_input = Input;
uint8_t* batch_output = Output;
const T8Bits* batch_input = Input;
T8Bits* batch_output = Output;
Input += Stride * ImageSize;
Output += Stride;
std::fill_n(AccumulateBuffer, Channels, bias);
for (size_t i = 0; i < ImageSize; ++i) {
for (size_t c = 0; c < Channels; ++c) {
AccumulateBuffer[c] += static_cast<int>(batch_input[c]);
}
batch_input += Stride;
}
for (size_t c = 0; c < Channels; ++c) {
int32_t v = static_cast<int>(std::nearbyintf(AccumulateBuffer[c] * scale)) + ZeroPointOutput;
*batch_output++ = std::max(std::min(255, v), 0);
int32_t v = static_cast<int32_t>(std::nearbyintf(AccumulateBuffer[c] * scale)) + ZeroPointOutput;
v = std::min(static_cast<int32_t>(std::numeric_limits<T8Bits>::max()), v);
v = std::max(static_cast<int32_t>(std::numeric_limits<T8Bits>::lowest()), v);
*batch_output++ = static_cast<T8Bits>(v);
}
}
}
@ -595,9 +776,91 @@ MlasQLinearGlobalAveragePoolNhwc(
#if defined(MLAS_NEON_INTRINSICS) || defined(MLAS_SSE2_INTRINSICS)
template <typename T8Bits>
void
MLASCALL
MlasQLinearGlobalAveragePoolNhwc(
const T8Bits* Input,
float ScaleInput,
int32_t ZeroPointInput,
T8Bits* Output,
float ScaleOutput,
int32_t ZeroPointOutput,
size_t Batch,
size_t ImageSize,
size_t Stride,
size_t Channels,
int32_t* AccumulateBuffer,
const T8Bits* ZeroBuffer
)
{
float scale = CheckQLinearGlobalAveragePoolScaleAndSize(ScaleInput, ScaleOutput, ImageSize);
const int32_t bias = -ZeroPointInput * static_cast<int32_t>(ImageSize);
const T8Bits* inputLastOf8 = Input + (Batch * ImageSize * Stride - Stride + Channels) - 8;
for (; Batch > 0; Batch--) {
MlasQLinearGlobalAveragePoolNhwcSingleBatch(
Input, Output, inputLastOf8, ImageSize, Channels, Stride, bias, scale,
static_cast<T8Bits>(ZeroPointOutput), AccumulateBuffer, ZeroBuffer);
Input += ImageSize * Stride;
Output += Stride;
}
}
#endif
template
void
MLASCALL
MlasQLinearGlobalAveragePoolNchw<int8_t>(
const int8_t* Input,
float ScaleInput,
int32_t ZeroPointInput,
int8_t* Output,
float ScaleOutput,
int32_t ZeroPointOutput,
size_t Channels,
size_t ImageSize,
int32_t* AccumulateBuffer
);
template
void
MLASCALL
MlasQLinearGlobalAveragePoolNchw<uint8_t>(
const uint8_t* Input,
float ScaleInput,
int32_t ZeroPointInput,
uint8_t* Output,
float ScaleOutput,
int32_t ZeroPointOutput,
size_t Channels,
size_t ImageSize,
int32_t* AccumulateBuffer
);
template
void
MLASCALL
MlasQLinearGlobalAveragePoolNhwc<int8_t>(
const int8_t* Input,
float ScaleInput,
int32_t ZeroPointInput,
int8_t* Output,
float ScaleOutput,
int32_t ZeroPointOutput,
size_t Batch,
size_t ImageSize,
size_t Stride,
size_t Channels,
int32_t* AccumulateBuffer,
const int8_t* ZeroBuffer
);
template
void
MLASCALL
MlasQLinearGlobalAveragePoolNhwc<uint8_t>(
const uint8_t* Input,
float ScaleInput,
int32_t ZeroPointInput,
@ -610,20 +873,4 @@ MlasQLinearGlobalAveragePoolNhwc(
size_t Channels,
int32_t* AccumulateBuffer,
const uint8_t* ZeroBuffer
)
{
float scale = CheckQLinearGlobalAveragePoolScaleAndSize(ScaleInput, ScaleOutput, ImageSize);
const int32_t bias = -ZeroPointInput * static_cast<int32_t>(ImageSize);
const uint8_t* inputLastOf8 = Input + (Batch * ImageSize * Stride - Stride + Channels) - 8;
for (; Batch > 0; Batch--) {
MlasQLinearGlobalAveragePoolNhwcSingleBatch(
Input, Output, inputLastOf8, ImageSize, Channels, Stride,
bias, scale, static_cast<uint8_t>(ZeroPointOutput),
AccumulateBuffer, ZeroBuffer);
Input += ImageSize * Stride;
Output += Stride;
}
}
#endif
);

117
onnxruntime/core/mlas/lib/quantize.cpp
View file

@ -165,7 +165,7 @@ Return Value:
--*/
{
constexpr int32_t MinimumValue = std::numeric_limits<OutputType>::min();
constexpr int32_t MinimumValue = std::numeric_limits<OutputType>::lowest();
constexpr int32_t MaximumValue = std::numeric_limits<OutputType>::max();
auto ScaleVector = MlasBroadcastFloat32x4(Scale);
@ -315,7 +315,7 @@ Return Value:
--*/
{
constexpr int32_t MinimumValue = std::numeric_limits<OutputType>::min();
constexpr int32_t MinimumValue = std::numeric_limits<OutputType>::lowest();
constexpr int32_t MaximumValue = std::numeric_limits<OutputType>::max();
for (size_t n = 0; n < N; n++) {
@ -352,17 +352,18 @@ MlasQuantizeLinear<uint8_t>(
#if defined(MLAS_SSE2_INTRINSICS)
template <typename OutputType>
void
MLASCALL
MlasRequantizeOutput(
const int32_t* Input,
size_t InputLeadingDimension,
uint8_t* Output,
OutputType* Output,
size_t OutputLeadingDimension,
const int32_t* Bias,
const float* Scale,
bool PerColumnScale,
uint8_t ZeroPoint,
OutputType ZeroPoint,
size_t StartM,
size_t StartN,
size_t CountM,
@ -370,8 +371,8 @@ MlasRequantizeOutput(
)
{
const __m128 PerMatrixScaleVector = PerColumnScale ? _mm_setzero_ps() : _mm_load1_ps(Scale);
const __m128 MinimumValueVector = _mm_set1_ps(float(0 - ZeroPoint));
const __m128 MaximumValueVector = _mm_set1_ps(float(255 - ZeroPoint));
const __m128 MinimumValueVector = _mm_set1_ps(float(std::numeric_limits<OutputType>::lowest() - ZeroPoint));
const __m128 MaximumValueVector = _mm_set1_ps(float(std::numeric_limits<OutputType>::max() - ZeroPoint));
const __m128i ZeroPointVector = _mm_set1_epi32(ZeroPoint);
if (nullptr != Bias) {
@ -467,10 +468,23 @@ MlasRequantizeOutput(
IntegerVector2 = _mm_add_epi32(IntegerVector2, ZeroPointVector);
IntegerVector3 = _mm_add_epi32(IntegerVector3, ZeroPointVector);
__m128i WordVector0 = _mm_packus_epi16(IntegerVector0, IntegerVector1);
__m128i WordVector1 = _mm_packus_epi16(IntegerVector2, IntegerVector3);
__m128i WordVector0;
__m128i WordVector1;
__m128i ByteVector;
__m128i ByteVector = _mm_packus_epi16(WordVector0, WordVector1);
if (std::is_signed<OutputType>::value) {
WordVector0 = _mm_packs_epi32(IntegerVector0, IntegerVector1);
WordVector1 = _mm_packs_epi32(IntegerVector2, IntegerVector3);
ByteVector = _mm_packs_epi16(WordVector0, WordVector1);
} else {
WordVector0 = _mm_packus_epi16(IntegerVector0, IntegerVector1);
WordVector1 = _mm_packus_epi16(IntegerVector2, IntegerVector3);
ByteVector = _mm_packus_epi16(WordVector0, WordVector1);
}
_mm_storeu_si128((__m128i*)RowOutput, ByteVector);
RowOutput += 16;
@ -541,8 +555,17 @@ MlasRequantizeOutput(
IntegerVector = _mm_cvtps_epi32(FloatVector);
IntegerVector = _mm_add_epi32(IntegerVector, ZeroPointVector);
IntegerVector = _mm_packus_epi16(IntegerVector, IntegerVector);
IntegerVector = _mm_packus_epi16(IntegerVector, IntegerVector);
if (std::is_signed<OutputType>::value) {
IntegerVector = _mm_packs_epi32(IntegerVector, IntegerVector);
IntegerVector = _mm_packs_epi16(IntegerVector, IntegerVector);
} else {
IntegerVector = _mm_packus_epi16(IntegerVector, IntegerVector);
IntegerVector = _mm_packus_epi16(IntegerVector, IntegerVector);
}
uint32_t OutputValue = uint32_t(_mm_cvtsi128_si32(IntegerVector));
@ -570,17 +593,18 @@ MlasRequantizeOutput(
#elif defined(MLAS_NEON64_INTRINSICS)
template<typename OutputType>
void
MLASCALL
MlasRequantizeOutput(
const int32_t* Input,
size_t InputLeadingDimension,
uint8_t* Output,
OutputType* Output,
size_t OutputLeadingDimension,
const int32_t* Bias,
const float* Scale,
bool PerColumnScale,
uint8_t ZeroPoint,
OutputType ZeroPoint,
size_t StartM,
size_t StartN,
size_t CountM,
@ -686,7 +710,7 @@ MlasRequantizeOutput(
//
// Pack the integers with saturation to 16-bit values and shift by
// the zero point, then pack the integers again to unsigned bytes.
// the zero point, then pack the integers again to bytes.
//
int16x8x2_t WordVector;
@ -697,7 +721,13 @@ MlasRequantizeOutput(
WordVector.val[0] = vqaddq_s16(WordVector.val[0], ZeroPointVector);
WordVector.val[1] = vqaddq_s16(WordVector.val[1], ZeroPointVector);
vst1q_u8(RowOutput, vqmovun_high_s16(vqmovun_s16(WordVector.val[0]), WordVector.val[1]));
if (std::is_signed<OutputType>::value) {
vst1q_s8(reinterpret_cast<int8_t*>(RowOutput),
vqmovn_high_s16(vqmovn_s16(WordVector.val[0]), WordVector.val[1]));
} else {
vst1q_u8(reinterpret_cast<uint8_t*>(RowOutput),
vqmovun_high_s16(vqmovun_s16(WordVector.val[0]), WordVector.val[1]));
}
RowOutput += 16;
n -= 16;
@ -775,7 +805,13 @@ MlasRequantizeOutput(
int16x8_t WordVector = vcombine_s16(vqmovn_s32(IntegerVector), vdup_n_s16(0));
WordVector = vqaddq_s16(WordVector, ZeroPointVector);
uint8x16_t ByteVector = vcombine_u8(vqmovun_s16(WordVector), vdup_n_u8(0));
uint8x16_t ByteVector;
if (std::is_signed<OutputType>::value) {
ByteVector = vcombine_u8(vreinterpret_u8_s8(vqmovn_s16(WordVector)), vdup_n_u8(0));
} else {
ByteVector = vcombine_u8(vqmovun_s16(WordVector), vdup_n_u8(0));
}
if (n >= 4) {
@ -787,7 +823,7 @@ MlasRequantizeOutput(
} else {
vst1q_lane_u8(RowOutput, ByteVector, 0);
vst1q_lane_u8(reinterpret_cast<uint8_t*>(RowOutput), ByteVector, 0);
RowOutput += 1;
n -= 1;
@ -802,17 +838,18 @@ MlasRequantizeOutput(
#else
template <typename OutputType>
void
MLASCALL
MlasRequantizeOutput(
const int32_t* Input,
size_t InputLeadingDimension,
uint8_t* Output,
OutputType* Output,
size_t OutputLeadingDimension,
const int32_t* Bias,
const float* Scale,
bool PerColumnScale,
uint8_t ZeroPoint,
OutputType ZeroPoint,
size_t StartM,
size_t StartN,
size_t CountM,
@ -820,8 +857,8 @@ MlasRequantizeOutput(
)
{
const float PerMatrixScaleValue = PerColumnScale ? 0.0f : *Scale;
const float MinimumValue = float(0 - ZeroPoint);
const float MaximumValue = float(255 - ZeroPoint);
const float MinimumValue = float(std::numeric_limits<OutputType>::lowest() - ZeroPoint);
const float MaximumValue = float(std::numeric_limits<OutputType>::max() - ZeroPoint);
if (nullptr != Bias) {
Bias += StartN;
@ -872,7 +909,7 @@ MlasRequantizeOutput(
IntegerValue = int32_t(MlasBitsOfFp32(FloatValue + MLAS_ROUNDING_BIAS_MAGIC)) -
MLAS_ROUNDING_BIAS_MAGIC_BITS;
*RowOutput++ = uint8_t(IntegerValue + ZeroPoint);
*RowOutput++ = OutputType(IntegerValue + ZeroPoint);
n -= 1;
}
@ -885,6 +922,42 @@ MlasRequantizeOutput(
#endif
template
void
MLASCALL
MlasRequantizeOutput<int8_t>(
const int32_t* Input,
size_t InputLeadingDimension,
int8_t* Output,
size_t OutputLeadingDimension,
const int32_t* Bias,
const float* Scale,
bool PerColumnScale,
int8_t ZeroPoint,
size_t StartM,
size_t StartN,
size_t CountM,
size_t CountN
);
template
void
MLASCALL
MlasRequantizeOutput<uint8_t>(
const int32_t* Input,
size_t InputLeadingDimension,
uint8_t* Output,
size_t OutputLeadingDimension,
const int32_t* Bias,
const float* Scale,
bool PerColumnScale,
uint8_t ZeroPoint,
size_t StartM,
size_t StartN,
size_t CountM,
size_t CountN
);
void
MLASCALL
MlasFindMinMaxElement(

15
onnxruntime/core/mlas/lib/transpose.cpp
View file

@ -436,3 +436,18 @@ Return Value:
n -= 1;
}
}
void
MLASCALL
MlasTranspose(
const int8_t* Input,
int8_t* Output,
size_t M,
size_t N)
{
MlasTranspose(
reinterpret_cast<const uint8_t*>(Input),
reinterpret_cast<uint8_t*>(Output),
M,
N);
}

1
onnxruntime/core/util/math_cpu.cc
View file

@ -650,6 +650,7 @@ void Im2col<T, StorageOrder::NHWC>::operator()(
}
}
template struct Im2col<int8_t, StorageOrder::NHWC>;
template struct Im2col<uint8_t, StorageOrder::NHWC>;
template <>

62
onnxruntime/test/contrib_ops/nhwc_maxpool_op_test.cc
View file

@ -2,10 +2,12 @@
// Licensed under the MIT License.
#include <algorithm>
#include <random>
#include "core/util/math.h"
#include "core/mlas/inc/mlas.h"
#include "gtest/gtest.h"
#include "test/providers/provider_test_utils.h"
#include <random>
namespace onnxruntime {
namespace test {
@ -70,7 +72,9 @@ class NhwcMaxPoolOpTester {
Y_shape.push_back(batch_count);
for (size_t n = 0; n < kernel_rank; n++) {
Y_shape.push_back(((input_shape[n] + pads[n] + pads[kernel_rank + n]) -
(dilations[n] * (kernel_shape_[n] - 1) + 1)) / strides[n] + 1);
(dilations[n] * (kernel_shape_[n] - 1) + 1)) /
strides[n] +
1);
}
Y_shape.push_back(channels);
Y_data.resize(ShapeSize(Y_shape));
@ -87,7 +91,7 @@ class NhwcMaxPoolOpTester {
std::vector<int64_t> d_output(kernel_rank, 0);
std::vector<int64_t> d_kernel(kernel_rank, 0);
do {
std::fill_n(Ydata, channels, static_cast<T>(0));
std::fill_n(Ydata, channels, std::numeric_limits<T>::lowest());
do {
int64_t input_offset = 0;
bool is_padding = false;
@ -163,7 +167,7 @@ class NhwcMaxPoolOpTester {
};
TEST(NhwcMaxPoolContribOpTest, MaxPool1D) {
for (int64_t channels = 1; channels < 64; channels++) {
for (int64_t channels = 1; channels < 94; channels++) {
NhwcMaxPoolOpTester<uint8_t> test;
test.GenerateRandomInput({1, 23, channels});
test.SetKernelShape({5});
@ -173,7 +177,7 @@ TEST(NhwcMaxPoolContribOpTest, MaxPool1D) {
}
TEST(NhwcMaxPoolContribOpTest, MaxPool2D) {
for (int64_t channels = 1; channels < 64; channels++) {
for (int64_t channels = 1; channels < 94; channels++) {
NhwcMaxPoolOpTester<uint8_t> test;
test.GenerateRandomInput({1, 15, 19, channels});
test.SetKernelShape({3, 5});
@ -183,7 +187,7 @@ TEST(NhwcMaxPoolContribOpTest, MaxPool2D) {
}
TEST(NhwcMaxPoolContribOpTest, MaxPool3D) {
for (int64_t channels = 1; channels < 64; channels++) {
for (int64_t channels = 1; channels < 94; channels++) {
NhwcMaxPoolOpTester<uint8_t> test;
test.GenerateRandomInput({1, 9, 13, 15, channels});
test.SetKernelShape({2, 4, 6});
@ -208,5 +212,51 @@ TEST(NhwcMaxPoolContribOpTest, MaxPoolDilations) {
test.Run();
}
TEST(NhwcMaxPoolContribOpTest, MaxPool1D_S8) {
for (int64_t channels = 1; channels < 94; channels++) {
NhwcMaxPoolOpTester<int8_t> test;
test.GenerateRandomInput({1, 23, channels});
test.SetKernelShape({5});
test.SetPads({2, 2});
test.Run();
}
}
TEST(NhwcMaxPoolContribOpTest, MaxPool2D_S8) {
for (int64_t channels = 1; channels < 94; channels++) {
NhwcMaxPoolOpTester<int8_t> test;
test.GenerateRandomInput({1, 15, 19, channels});
test.SetKernelShape({3, 5});
test.SetPads({1, 1, 1, 1});
test.Run();
}
}
TEST(NhwcMaxPoolContribOpTest, MaxPool3D_S8) {
for (int64_t channels = 1; channels < 94; channels++) {
NhwcMaxPoolOpTester<int8_t> test;
test.GenerateRandomInput({1, 9, 13, 15, channels});
test.SetKernelShape({2, 4, 6});
test.SetPads({0, 0, 0, 1, 1, 1});
test.Run();
}
}
TEST(NhwcMaxPoolContribOpTest, MaxPoolStrides_S8) {
NhwcMaxPoolOpTester<int8_t> test;
test.GenerateRandomInput({4, 23, 19, 32});
test.SetKernelShape({3, 3});
test.SetStrides({2, 2});
test.Run();
}
TEST(NhwcMaxPoolContribOpTest, MaxPoolDilations_S8) {
NhwcMaxPoolOpTester<int8_t> test;
test.GenerateRandomInput({4, 23, 19, 32});
test.SetKernelShape({3, 3});
test.SetDilations({2, 2});
test.Run();
}
} // namespace test
} // namespace onnxruntime

154
onnxruntime/test/contrib_ops/qlinear_global_average_pool_test.cc
View file

@ -5,22 +5,24 @@
#include "test/common/tensor_op_test_utils.h"
#include "test/providers/provider_test_utils.h"
#include "core/providers/common.h"
#include "core/mlas/inc/mlas.h"
namespace onnxruntime {
namespace test {
static void CalculateGlobalAvgPoolU8(
const uint8_t* x, int64_t batch, int64_t hw, int64_t channel, bool channels_last, uint8_t* y,
template <typename T8Bits>
static void CalculateGlobalAvgPool(
const T8Bits* x, int64_t batch, int64_t hw, int64_t channel, bool channels_last, T8Bits* y,
int32_t x_zero_point, float x_scale, int32_t y_zero_point, float y_scale) {
int32_t bias = -x_zero_point * gsl::narrow_cast<int32_t>(hw);
int64_t stride_image = channels_last ? channel : 1;
int64_t stride_channel = channels_last ? 1 : hw;
for (int64_t b = 0; b < batch; ++b) {
const uint8_t* bx = x + b * hw * channel;
uint8_t* by = y + b * channel;
const T8Bits* bx = x + b * hw * channel;
T8Bits* by = y + b * channel;
for (int64_t c = 0; c < channel; ++c) {
const uint8_t* ix = bx + c * stride_channel;
const T8Bits* ix = bx + c * stride_channel;
int32_t sum = 0;
for (int64_t i = 0; i < hw; ++i) {
sum += static_cast<int32_t>(*ix);
@ -29,40 +31,41 @@ static void CalculateGlobalAvgPoolU8(
sum += bias;
int32_t r = static_cast<int32_t>(std::nearbyintf(x_scale * sum / static_cast<float>(hw) / y_scale));
r += y_zero_point;
r = std::min(255, r);
r = std::max(0, r);
by[c] = static_cast<uint8_t>(r);
r = std::min((int32_t)(std::numeric_limits<T8Bits>::max()), r);
r = std::max((int32_t)(std::numeric_limits<T8Bits>::lowest()), r);
by[c] = static_cast<T8Bits>(r);
}
}
}
void RunQLinearGlobalAveragePoolU8(
template <typename T8Bits = uint8_t>
void RunQLinearGlobalAveragePool(
bool channels_last, int64_t batch, int64_t channel, int64_t h, int64_t w,
uint8_t x_zero_point, float x_scale, uint8_t y_zero_point, float y_scale, int32_t seed = 0) {
T8Bits x_zero_point, float x_scale, T8Bits y_zero_point, float y_scale, int32_t seed = 0) {
std::vector<int64_t> x_dims = channels_last ? std::vector<int64_t>{batch, h, w, channel} : std::vector<int64_t>{batch, channel, h, w};
std::vector<int64_t> y_dims = channels_last ? std::vector<int64_t>{batch, 1, 1, channel} : std::vector<int64_t>{batch, channel, 1, 1};
int64_t x_size = batch * channel * h * w;
int64_t y_size = batch * channel;
std::vector<uint8_t> x_data((size_t)x_size);
std::vector<uint8_t> y_data((size_t)y_size);
std::vector<T8Bits> x_data((size_t)x_size);
std::vector<T8Bits> y_data((size_t)y_size);
RandomValueGenerator random{seed ? optional<RandomValueGenerator::RandomSeedType>{seed} : optional<RandomValueGenerator::RandomSeedType>{}};
std::vector<int> tmp_x_data = random.Uniform<int32_t>(x_dims, 0, 255);
std::transform(tmp_x_data.begin(), tmp_x_data.end(), x_data.data(), [](int32_t v) -> uint8_t {
return static_cast<uint8_t>(v);
std::vector<int> tmp_x_data = random.Uniform<int32_t>(x_dims, std::numeric_limits<T8Bits>::lowest(), std::numeric_limits<T8Bits>::max());
std::transform(tmp_x_data.begin(), tmp_x_data.end(), x_data.data(), [](int32_t v) -> T8Bits {
return static_cast<T8Bits>(v);
});
CalculateGlobalAvgPoolU8(x_data.data(), batch, h * w, channel, channels_last, y_data.data(),
CalculateGlobalAvgPool(x_data.data(), batch, h * w, channel, channels_last, y_data.data(),
x_zero_point, x_scale, y_zero_point, y_scale);
OpTester test("QLinearGlobalAveragePool", 1, onnxruntime::kMSDomain);
test.AddAttribute<int64_t>("channels_last", channels_last ? 1LL : 0LL);
test.AddInput<uint8_t>("X", x_dims, x_data);
test.AddInput<T8Bits>("X", x_dims, x_data);
test.AddInput<float>("x_scale", {}, {x_scale});
test.AddInput<uint8_t>("x_zero_point", {}, {x_zero_point});
test.AddInput<T8Bits>("x_zero_point", {}, {x_zero_point});
test.AddInput<float>("y_scale", {}, {y_scale});
test.AddInput<uint8_t>("y_zero_point", {}, {y_zero_point});
test.AddOutput<uint8_t>("Y", y_dims, y_data);
test.AddInput<T8Bits>("y_zero_point", {}, {y_zero_point});
test.AddOutput<T8Bits>("Y", y_dims, y_data);
auto q8checker = [&](const std::vector<OrtValue>& fetches, const std::string& provider_type) {
const OrtValue& ort_value = fetches[0];
@ -75,7 +78,7 @@ void RunQLinearGlobalAveragePoolU8(
ORT_ENFORCE(y_shape == output_tensor.Shape(),
"Expected output shape [" + y_shape.ToString() + "] did not match run output shape [" +
output_tensor.Shape().ToString() + "] for Y @" + provider_type);
auto* output = output_tensor.Data<uint8_t>();
auto* output = output_tensor.Data<T8Bits>();
auto size = static_cast<int>(output_tensor.Shape().Size());
for (int i = 0; i < size; ++i) {
int diff = abs(y_data[i] - output[i]);
@ -89,76 +92,149 @@ void RunQLinearGlobalAveragePoolU8(
}
TEST(QLinearGlobalAveragePool, Nhwc_1x1x32x32) {
RunQLinearGlobalAveragePoolU8(true, 1, 1, 32, 32, 128, 1.0, 64, 2.0);
RunQLinearGlobalAveragePool<uint8_t>(true, 1, 1, 32, 32, 128, 1.0, 64, 2.0);
}
TEST(QLinearGlobalAveragePool, Nchw_1x32x32x1) {
RunQLinearGlobalAveragePoolU8(false, 1, 1, 32, 32, 128, 1.0, 64, 2.0);
RunQLinearGlobalAveragePool<uint8_t>(false, 1, 1, 32, 32, 128, 1.0, 64, 2.0);
}
TEST(QLinearGlobalAveragePool, Nhwc_1x256x8x8) {
RunQLinearGlobalAveragePoolU8(true, 1, 256, 8, 8, 128, 1.0, 64, 3.0);
RunQLinearGlobalAveragePool<uint8_t>(true, 1, 256, 8, 8, 128, 1.0, 64, 3.0);
}
TEST(QLinearGlobalAveragePool, Nchw_1x8x8x256) {
RunQLinearGlobalAveragePoolU8(false, 1, 256, 8, 8, 128, 1.0, 64, 3.0);
RunQLinearGlobalAveragePool<uint8_t>(false, 1, 256, 8, 8, 128, 1.0, 64, 3.0);
}
TEST(QLinearGlobalAveragePool, Nhwc_1x255x7x7) {
RunQLinearGlobalAveragePoolU8(true, 1, 255, 7, 7, 128, 7.0, 128, 21.0);
RunQLinearGlobalAveragePool<uint8_t>(true, 1, 255, 7, 7, 128, 7.0, 128, 21.0);
}
TEST(QLinearGlobalAveragePool, Nchw_1x7x7x255) {
RunQLinearGlobalAveragePoolU8(false, 1, 255, 7, 7, 128, 7.0, 128, 21.0);
RunQLinearGlobalAveragePool<uint8_t>(false, 1, 255, 7, 7, 128, 7.0, 128, 21.0);
}
TEST(QLinearGlobalAveragePool, Nhwc_1x255x8x8) {
RunQLinearGlobalAveragePoolU8(true, 1, 255, 8, 8, 128, 1.0, 128, 2.0);
RunQLinearGlobalAveragePool<uint8_t>(true, 1, 255, 8, 8, 128, 1.0, 128, 2.0);
}
TEST(QLinearGlobalAveragePool, Nchw_1x8x8x255) {
RunQLinearGlobalAveragePoolU8(false, 1, 255, 8, 8, 128, 1.0, 128, 2.0);
RunQLinearGlobalAveragePool<uint8_t>(false, 1, 255, 8, 8, 128, 1.0, 128, 2.0);
}
TEST(QLinearGlobalAveragePool, Nhwc_1x256x7x7) {
RunQLinearGlobalAveragePoolU8(true, 1, 256, 7, 7, 128, 7.0, 128, 21.0);
RunQLinearGlobalAveragePool<uint8_t>(true, 1, 256, 7, 7, 128, 7.0, 128, 21.0);
}
TEST(QLinearGlobalAveragePool, Nchw_1x7x7x256) {
RunQLinearGlobalAveragePoolU8(false, 1, 256, 7, 7, 128, 7.0, 128, 21.0);
RunQLinearGlobalAveragePool<uint8_t>(false, 1, 256, 7, 7, 128, 7.0, 128, 21.0);
}
// tests for BatchSize > 1
TEST(QLinearGlobalAveragePool, Nhwc_3x256x8x8) {
RunQLinearGlobalAveragePoolU8(true, 3, 256, 8, 8, 128, 1.0, 64, 3.0);
RunQLinearGlobalAveragePool<uint8_t>(true, 3, 256, 8, 8, 128, 1.0, 64, 3.0);
}
TEST(QLinearGlobalAveragePool, Nchw_3x8x8x256) {
RunQLinearGlobalAveragePoolU8(false, 3, 256, 8, 8, 128, 1.0, 64, 3.0);
RunQLinearGlobalAveragePool<uint8_t>(false, 3, 256, 8, 8, 128, 1.0, 64, 3.0);
}
TEST(QLinearGlobalAveragePool, Nhwc_3x255x7x7) {
RunQLinearGlobalAveragePoolU8(true, 3, 255, 7, 7, 128, 7.0, 128, 21.0);
RunQLinearGlobalAveragePool<uint8_t>(true, 3, 255, 7, 7, 128, 7.0, 128, 21.0);
}
TEST(QLinearGlobalAveragePool, Nchw_3x7x7x255) {
RunQLinearGlobalAveragePoolU8(false, 3, 255, 7, 7, 128, 7.0, 128, 21.0);
RunQLinearGlobalAveragePool<uint8_t>(false, 3, 255, 7, 7, 128, 7.0, 128, 21.0);
}
TEST(QLinearGlobalAveragePool, Nhwc_3x255x8x8) {
RunQLinearGlobalAveragePoolU8(true, 3, 255, 8, 8, 128, 1.0, 128, 2.0);
RunQLinearGlobalAveragePool<uint8_t>(true, 3, 255, 8, 8, 128, 1.0, 128, 2.0);
}
TEST(QLinearGlobalAveragePool, Nchw_3x8x8x255) {
RunQLinearGlobalAveragePoolU8(false, 3, 255, 8, 8, 128, 1.0, 128, 2.0);
RunQLinearGlobalAveragePool<uint8_t>(false, 3, 255, 8, 8, 128, 1.0, 128, 2.0);
}
TEST(QLinearGlobalAveragePool, Nhwc_3x256x7x7) {
RunQLinearGlobalAveragePoolU8(true, 3, 256, 7, 7, 128, 7.0, 128, 21.0);
RunQLinearGlobalAveragePool<uint8_t>(true, 3, 256, 7, 7, 128, 7.0, 128, 21.0);
}
TEST(QLinearGlobalAveragePool, Nchw_3x7x7x256) {
RunQLinearGlobalAveragePoolU8(false, 3, 256, 7, 7, 128, 7.0, 128, 21.0);
RunQLinearGlobalAveragePool<uint8_t>(false, 3, 256, 7, 7, 128, 7.0, 128, 21.0);
}
TEST(QLinearGlobalAveragePool, Nhwc_1x1x32x32_S8) {
RunQLinearGlobalAveragePool<int8_t>(true, 1, 1, 32, 32, 1, 1.0, -64, 2.0);
}
TEST(QLinearGlobalAveragePool, Nchw_1x32x32x1_S8) {
RunQLinearGlobalAveragePool<int8_t>(false, 1, 1, 32, 32, 1, 1.0, 64, 2.0);
}
TEST(QLinearGlobalAveragePool, Nhwc_1x256x8x8_S8) {
RunQLinearGlobalAveragePool<int8_t>(true, 1, 256, 8, 8, -1, 1.0, -64, 3.0);
}
TEST(QLinearGlobalAveragePool, Nchw_1x8x8x256_S8) {
RunQLinearGlobalAveragePool<int8_t>(false, 1, 256, 8, 8, -1, 1.0, 64, 3.0);
}
TEST(QLinearGlobalAveragePool, Nhwc_1x255x7x7_S8) {
RunQLinearGlobalAveragePool<int8_t>(true, 1, 255, 7, 7, 64, 7.0, 1, 21.0);
}
TEST(QLinearGlobalAveragePool, Nchw_1x7x7x255_S8) {
RunQLinearGlobalAveragePool<int8_t>(false, 1, 255, 7, 7, 64, 7.0, -1, 21.0);
}
TEST(QLinearGlobalAveragePool, Nhwc_1x255x8x8_S8) {
RunQLinearGlobalAveragePool<int8_t>(true, 1, 255, 8, 8, -64, 1.0, 1, 2.0);
}
TEST(QLinearGlobalAveragePool, Nchw_1x8x8x255_S8) {
RunQLinearGlobalAveragePool<int8_t>(false, 1, 255, 8, 8, -64, 1.0, -1, 2.0);
}
TEST(QLinearGlobalAveragePool, Nhwc_1x256x7x7_S8) {
RunQLinearGlobalAveragePool<int8_t>(true, 1, 256, 7, 7, -64, 7.0, 64, 21.0);
}
TEST(QLinearGlobalAveragePool, Nchw_1x7x7x256_S8) {
RunQLinearGlobalAveragePool<int8_t>(false, 1, 256, 7, 7, 64, 7.0, -64, 21.0);
}
// tests for BatchSize > 1
TEST(QLinearGlobalAveragePool, Nhwc_3x256x8x8_S8) {
RunQLinearGlobalAveragePool<int8_t>(true, 3, 256, 8, 8, 1, 1.0, 64, 3.0);
}
TEST(QLinearGlobalAveragePool, Nchw_3x8x8x256_S8) {
RunQLinearGlobalAveragePool<int8_t>(false, 3, 256, 8, 8, 1, 1.0, 64, 3.0);
}
TEST(QLinearGlobalAveragePool, Nhwc_3x255x7x7_S8) {
RunQLinearGlobalAveragePool<int8_t>(true, 3, 255, 7, 7, 1, 7.0, -1, 21.0);
}
TEST(QLinearGlobalAveragePool, Nchw_3x7x7x255_S8) {
RunQLinearGlobalAveragePool<int8_t>(false, 3, 255, 7, 7, 1, 7.0, -1, 21.0);
}
TEST(QLinearGlobalAveragePool, Nhwc_3x255x8x8_S8) {
RunQLinearGlobalAveragePool<int8_t>(true, 3, 255, 8, 8, 1, 1.0, -1, 2.0);
}
TEST(QLinearGlobalAveragePool, Nchw_3x8x8x255_S8) {
RunQLinearGlobalAveragePool<int8_t>(false, 3, 255, 8, 8, -1, 1.0, 1, 2.0);
}
TEST(QLinearGlobalAveragePool, Nhwc_3x256x7x7_S8) {
RunQLinearGlobalAveragePool<int8_t>(true, 3, 256, 7, 7, -1, 7.0, 1, 21.0);
}
TEST(QLinearGlobalAveragePool, Nchw_3x7x7x256_S8) {
RunQLinearGlobalAveragePool<int8_t>(false, 3, 256, 7, 7, -1, 7.0, 1, 21.0);
}
} // namespace test

302
onnxruntime/test/contrib_ops/qlinear_pool_test.cc
View file

@ -6,6 +6,7 @@
#include "test/common/tensor_op_test_utils.h"
#include "test/providers/provider_test_utils.h"
#include "core/providers/common.h"
#include "core/mlas/inc/mlas.h"
namespace onnxruntime {
namespace test {
@ -46,14 +47,15 @@ struct DimIterator {
int64_t index_;
};
template <typename T8Bits>
static void
CalculateAvgPoolNchwU8(
uint8_t* x,
CalculateAvgPoolNchw(
T8Bits* x,
const std::vector<int64_t> x_dims,
const quantization::Params<uint8_t>& x_params,
uint8_t* y,
const quantization::Params<T8Bits>& x_params,
T8Bits* y,
const std::vector<int64_t> y_dims,
const quantization::Params<uint8_t>& y_params,
const quantization::Params<T8Bits>& y_params,
const std::vector<int64_t> kernel_shape,
const std::vector<int64_t> strides,
const std::vector<int64_t> pads,
@ -73,8 +75,8 @@ CalculateAvgPoolNchwU8(
int64_t x_step = std::accumulate(x_img_dims.begin(), x_img_dims.end(), 1LL, std::multiplies<int64_t>());
for (int64_t b = 0; b < batch; ++b) {
for (int64_t c = 0; c < channel; ++c) {
uint8_t* ybc = y + (b * channel + c) * y_step;
uint8_t* xbc = x + (b * channel + c) * x_step;
T8Bits* ybc = y + (b * channel + c) * y_step;
T8Bits* xbc = x + (b * channel + c) * x_step;
DimIterator yit(y_img_dims);
while (yit.has_next()) {
@ -103,14 +105,15 @@ CalculateAvgPoolNchwU8(
}
}
auto y_offset = yit.next();
auto y_u8 = QuantizeTestValue<uint8_t>(y_value_sum / count, y_params);
auto y_u8 = QuantizeTestValue<T8Bits>(y_value_sum / count, y_params);
ybc[y_offset] = y_u8;
}
}
}
}
void RunQLinearAveragePoolNchwU8(
template <typename T8Bits = uint8_t>
void RunQLinearAveragePoolNchw(
const std::vector<int64_t> x_dims,
const std::vector<int64_t> y_dims,
const std::vector<int64_t> kernel_shape,
@ -119,17 +122,19 @@ void RunQLinearAveragePoolNchwU8(
const int64_t count_include_pad = 0) {
auto run_test = [&](bool only_x_not_initializer, bool x_y_same_zero_point) {
float x_scale = 1.0f / 255.0f;
quantization::Params<uint8_t> x_params(x_scale, /*zero_point=*/128);
T8Bits x_zero_point = (std::numeric_limits<T8Bits>::lowest() + std::numeric_limits<T8Bits>::max() - 5) / 2;
quantization::Params<T8Bits> x_params(x_scale, x_zero_point);
RandomValueGenerator random{};
std::vector<float> x_data_fp32 = random.Uniform<float>(x_dims, -0.5f, 0.5f);
std::vector<uint8_t> x_data = QuantizeTestVector<uint8_t>(x_data_fp32, x_params);
std::vector<T8Bits> x_data = QuantizeTestVector<T8Bits>(x_data_fp32, x_params);
float y_scale = 1.0f / 255.0f;
uint8_t y_zero_point = x_y_same_zero_point ? x_params.zero_point : 100;
const quantization::Params<uint8_t> y_params(y_scale, y_zero_point);
T8Bits y_zero_point_not_same = (std::numeric_limits<T8Bits>::lowest() + std::numeric_limits<T8Bits>::max() + 10) / 2;
T8Bits y_zero_point = x_y_same_zero_point ? x_params.zero_point : y_zero_point_not_same;
const quantization::Params<T8Bits> y_params(y_scale, y_zero_point);
int64_t y_size = std::accumulate(y_dims.begin(), y_dims.end(), 1LL, std::multiplies<int64_t>());
std::vector<uint8_t> y_data(y_size);
CalculateAvgPoolNchwU8(
std::vector<T8Bits> y_data(y_size);
CalculateAvgPoolNchw(
x_data.data(), x_dims, x_params,
y_data.data(), y_dims, y_params,
kernel_shape, strides, pads, count_include_pad);
@ -142,12 +147,12 @@ void RunQLinearAveragePoolNchwU8(
test.AddAttribute("kernel_shape", kernel_shape);
test.AddAttribute("count_include_pad", count_include_pad);
test.AddInput<uint8_t>("X", x_dims, x_data);
test.AddInput<T8Bits>("X", x_dims, x_data);
test.AddInput<float>("x_scale", {}, {x_scale}, only_x_not_initializer);
test.AddInput<uint8_t>("x_zero_point", {}, {x_params.zero_point}, only_x_not_initializer);
test.AddInput<T8Bits>("x_zero_point", {}, {x_params.zero_point}, only_x_not_initializer);
test.AddInput<float>("y_scale", {}, {y_scale}, only_x_not_initializer);
test.AddInput<uint8_t>("y_zero_point", {}, {y_params.zero_point}, only_x_not_initializer);
test.AddOutput<uint8_t>("Y", y_dims, y_data);
test.AddInput<T8Bits>("y_zero_point", {}, {y_params.zero_point}, only_x_not_initializer);
test.AddOutput<T8Bits>("Y", y_dims, y_data);
auto q8checker = [&](const std::vector<OrtValue>& fetches, const std::string& provider_type) {
const OrtValue& ort_value = fetches[0];
@ -160,7 +165,7 @@ void RunQLinearAveragePoolNchwU8(
ORT_ENFORCE(y_shape == output_tensor.Shape(),
"Expected output shape [" + y_shape.ToString() + "] did not match run output shape [" +
output_tensor.Shape().ToString() + "] for Y @" + provider_type);
auto* output = output_tensor.Data<uint8_t>();
auto* output = output_tensor.Data<T8Bits>();
auto size = static_cast<int>(output_tensor.Shape().Size());
for (int i = 0; i < size; ++i) {
int diff = abs(y_data[i] - output[i]);
@ -188,15 +193,16 @@ static std::vector<int64_t> dims_to_nhwc(const std::vector<int64_t>& nchw) {
return nhwc;
}
static std::vector<uint8_t> transpose_to_nhwc(const std::vector<uint8_t>& nchw_data, const std::vector<int64_t>& nchw_dims) {
std::vector<uint8_t> nhwc_data(nchw_data.size());
template <typename T8Bits>
static std::vector<T8Bits> transpose_to_nhwc(const std::vector<T8Bits>& nchw_data, const std::vector<int64_t>& nchw_dims) {
std::vector<T8Bits> nhwc_data(nchw_data.size());
auto batch_count = nchw_dims[0];
auto channels = nchw_dims[1];
int64_t image_size = std::accumulate(nchw_dims.begin() + 2, nchw_dims.end(), 1LL, std::multiplies<int64_t>());
for (int64_t b = 0; b < batch_count; b++) {
const uint8_t* nchw_image = nchw_data.data() + (b * channels * image_size);
uint8_t* nhwc_image = nhwc_data.data() + (b * channels * image_size);
const T8Bits* nchw_image = nchw_data.data() + (b * channels * image_size);
T8Bits* nhwc_image = nhwc_data.data() + (b * channels * image_size);
for (int64_t img_index = 0; img_index < image_size; ++img_index) {
for (int64_t c = 0; c < channels; c++) {
*nhwc_image++ = nchw_image[c * image_size + img_index];
@ -207,7 +213,8 @@ static std::vector<uint8_t> transpose_to_nhwc(const std::vector<uint8_t>& nchw_d
return nhwc_data;
}
void RunQLinearAveragePoolNhwcU8(
template <typename T8Bits = uint8_t>
void RunQLinearAveragePoolNhwc(
const std::vector<int64_t> x_dims,
const std::vector<int64_t> y_dims,
const std::vector<int64_t> kernel_shape,
@ -215,23 +222,25 @@ void RunQLinearAveragePoolNhwcU8(
const std::vector<int64_t> pads,
const int64_t count_include_pad = 0) {
float x_scale = 1.0f / 255.0f;
const quantization::Params<uint8_t> x_params(x_scale, /*zero_point=*/128);
T8Bits x_zero_point = (std::numeric_limits<T8Bits>::lowest() + std::numeric_limits<T8Bits>::max() - 5) / 2;
const quantization::Params<T8Bits> x_params(x_scale, x_zero_point);
RandomValueGenerator random{};
std::vector<float> x_data_fp32 = random.Uniform<float>(x_dims, -0.5f, 0.5f);
std::vector<uint8_t> x_data = QuantizeTestVector<uint8_t>(x_data_fp32, x_params);
std::vector<T8Bits> x_data = QuantizeTestVector<T8Bits>(x_data_fp32, x_params);
float y_scale = 1.0f / 255.0f;
const quantization::Params<uint8_t> y_params(y_scale, /*zero_point=*/100);
T8Bits y_zero_point = (std::numeric_limits<T8Bits>::lowest() + std::numeric_limits<T8Bits>::max() + 10) / 2;
const quantization::Params<T8Bits> y_params(y_scale, y_zero_point);
int64_t y_size = std::accumulate(y_dims.begin(), y_dims.end(), 1LL, std::multiplies<int64_t>());
std::vector<uint8_t> y_data(y_size);
CalculateAvgPoolNchwU8(
std::vector<T8Bits> y_data(y_size);
CalculateAvgPoolNchw(
x_data.data(), x_dims, x_params,
y_data.data(), y_dims, y_params,
kernel_shape, strides, pads, count_include_pad);
// transpose the result
std::vector<uint8_t> y_data_nhwc = transpose_to_nhwc(y_data, y_dims);
std::vector<uint8_t> x_data_nhwc = transpose_to_nhwc(x_data, x_dims);
std::vector<T8Bits> y_data_nhwc = transpose_to_nhwc(y_data, y_dims);
std::vector<T8Bits> x_data_nhwc = transpose_to_nhwc(x_data, x_dims);
auto x_dims_nhwc = dims_to_nhwc(x_dims);
auto y_dims_nhwc = dims_to_nhwc(y_dims);
@ -244,12 +253,12 @@ void RunQLinearAveragePoolNhwcU8(
test.AddAttribute("count_include_pad", count_include_pad);
test.AddAttribute("channels_last", (int64_t)1LL);
test.AddInput<uint8_t>("X", x_dims_nhwc, x_data_nhwc);
test.AddInput<T8Bits>("X", x_dims_nhwc, x_data_nhwc);
test.AddInput<float>("x_scale", {}, {x_scale});
test.AddInput<uint8_t>("x_zero_point", {}, {x_params.zero_point});
test.AddInput<T8Bits>("x_zero_point", {}, {x_params.zero_point});
test.AddInput<float>("y_scale", {}, {y_scale});
test.AddInput<uint8_t>("y_zero_point", {}, {y_params.zero_point});
test.AddOutput<uint8_t>("Y", y_dims_nhwc, y_data_nhwc);
test.AddInput<T8Bits>("y_zero_point", {}, {y_params.zero_point});
test.AddOutput<T8Bits>("Y", y_dims_nhwc, y_data_nhwc);
auto q8checker = [&](const std::vector<OrtValue>& fetches, const std::string& provider_type) {
const OrtValue& ort_value = fetches[0];
@ -262,7 +271,7 @@ void RunQLinearAveragePoolNhwcU8(
ORT_ENFORCE(y_shape == output_tensor.Shape(),
"Expected output shape [" + y_shape.ToString() + "] did not match run output shape [" +
output_tensor.Shape().ToString() + "] for Y @" + provider_type);
auto* output = output_tensor.Data<uint8_t>();
auto* output = output_tensor.Data<T8Bits>();
auto size = static_cast<int>(output_tensor.Shape().Size());
for (int i = 0; i < size; ++i) {
int diff = abs(y_data_nhwc[i] - output[i]);
@ -278,7 +287,7 @@ void RunQLinearAveragePoolNhwcU8(
}
TEST(QLinearPoolTest, AveragePool1D_ExcludePadPixel) {
RunQLinearAveragePoolNchwU8(
RunQLinearAveragePoolNchw(
{1, 1, 5}, // x shape
{1, 1, 6}, // expected y shape
{3}, // kernel shape
@ -288,7 +297,7 @@ TEST(QLinearPoolTest, AveragePool1D_ExcludePadPixel) {
}
TEST(QLinearPoolTest, AveragePool1D_IncludePadPixel) {
RunQLinearAveragePoolNchwU8(
RunQLinearAveragePoolNchw(
{1, 1, 5}, // x shape
{1, 1, 6}, // expected y shape
{3}, // kernel shape
@ -298,7 +307,7 @@ TEST(QLinearPoolTest, AveragePool1D_IncludePadPixel) {
}
TEST(QLinearPoolTest, AveragePool2D_ExcludePadPixel) {
RunQLinearAveragePoolNchwU8(
RunQLinearAveragePoolNchw(
{1, 1, 5, 7}, // x shape
{1, 1, 6, 4}, // expected y shape
{3, 4}, // kernel shape
@ -308,7 +317,7 @@ TEST(QLinearPoolTest, AveragePool2D_ExcludePadPixel) {
}
TEST(QLinearPoolTest, AveragePool2D_IncludePadPixel) {
RunQLinearAveragePoolNchwU8(
RunQLinearAveragePoolNchw(
{1, 1, 5, 7}, // x shape
{1, 1, 6, 4}, // expected y shape
{3, 4}, // kernel shape
@ -318,7 +327,7 @@ TEST(QLinearPoolTest, AveragePool2D_IncludePadPixel) {
}
TEST(QLinearPoolTest, AveragePool2D_MultiChannel) {
RunQLinearAveragePoolNchwU8(
RunQLinearAveragePoolNchw(
{1, 3, 5, 7}, // x shape
{1, 3, 6, 4}, // expected y shape
{3, 4}, // kernel shape
@ -328,7 +337,7 @@ TEST(QLinearPoolTest, AveragePool2D_MultiChannel) {
}
TEST(QLinearPoolTest, AveragePool3D_ExcludePadPixel) {
RunQLinearAveragePoolNchwU8(
RunQLinearAveragePoolNchw(
{1, 1, 5, 7, 9}, // x shape
{1, 1, 6, 4, 3}, // expected y shape
{3, 4, 5}, // kernel shape
@ -338,7 +347,7 @@ TEST(QLinearPoolTest, AveragePool3D_ExcludePadPixel) {
}
TEST(QLinearPoolTest, AveragePool3D_IncludePadPixel) {
RunQLinearAveragePoolNchwU8(
RunQLinearAveragePoolNchw(
{1, 1, 5, 7, 9}, // x shape
{1, 1, 6, 4, 3}, // expected y shape
{3, 4, 5}, // kernel shape
@ -351,7 +360,7 @@ TEST(QLinearPoolTest, AveragePool3D_IncludePadPixel) {
* Channels last test
**************************************************/
TEST(QLinearPoolTest, AveragePool1D_ExcludePadPixel_nhwc) {
RunQLinearAveragePoolNhwcU8(
RunQLinearAveragePoolNhwc(
{1, 1, 5}, // x shape
{1, 1, 6}, // expected y shape
{3}, // kernel shape
@ -361,7 +370,7 @@ TEST(QLinearPoolTest, AveragePool1D_ExcludePadPixel_nhwc) {
}
TEST(QLinearPoolTest, AveragePool1D_IncludePadPixel_nhwc) {
RunQLinearAveragePoolNhwcU8(
RunQLinearAveragePoolNhwc(
{1, 1, 5}, // x shape
{1, 1, 6}, // expected y shape
{3}, // kernel shape
@ -371,7 +380,7 @@ TEST(QLinearPoolTest, AveragePool1D_IncludePadPixel_nhwc) {
}
TEST(QLinearPoolTest, AveragePool2D_ExcludePadPixel_nhwc) {
RunQLinearAveragePoolNhwcU8(
RunQLinearAveragePoolNhwc(
{1, 1, 5, 7}, // x shape
{1, 1, 6, 4}, // expected y shape
{3, 4}, // kernel shape
@ -381,7 +390,7 @@ TEST(QLinearPoolTest, AveragePool2D_ExcludePadPixel_nhwc) {
}
TEST(QLinearPoolTest, AveragePool2D_IncludePadPixel_nhwc) {
RunQLinearAveragePoolNhwcU8(
RunQLinearAveragePoolNhwc(
{1, 1, 5, 7}, // x shape
{1, 1, 6, 4}, // expected y shape
{3, 4}, // kernel shape
@ -391,7 +400,7 @@ TEST(QLinearPoolTest, AveragePool2D_IncludePadPixel_nhwc) {
}
TEST(QLinearPoolTest, AveragePool2D_MultiChannel_nhwc) {
RunQLinearAveragePoolNhwcU8(
RunQLinearAveragePoolNhwc(
{1, 3, 5, 7}, // x shape
{1, 3, 6, 4}, // expected y shape
{3, 4}, // kernel shape
@ -401,7 +410,7 @@ TEST(QLinearPoolTest, AveragePool2D_MultiChannel_nhwc) {
}
TEST(QLinearPoolTest, AveragePool3D_ExcludePadPixel_nhwc) {
RunQLinearAveragePoolNhwcU8(
RunQLinearAveragePoolNhwc(
{1, 1, 5, 7, 9}, // x shape
{1, 1, 6, 4, 3}, // expected y shape
{3, 4, 5}, // kernel shape
@ -411,7 +420,7 @@ TEST(QLinearPoolTest, AveragePool3D_ExcludePadPixel_nhwc) {
}
TEST(QLinearPoolTest, AveragePool3D_IncludePadPixel_nhwc) {
RunQLinearAveragePoolNhwcU8(
RunQLinearAveragePoolNhwc(
{1, 1, 5, 7, 9}, // x shape
{1, 1, 6, 4, 3}, // expected y shape
{3, 4, 5}, // kernel shape
@ -420,9 +429,8 @@ TEST(QLinearPoolTest, AveragePool3D_IncludePadPixel_nhwc) {
1); // count_include_pad
}
TEST(QLinearPoolTest, AveragePool2D_BigImage) {
RunQLinearAveragePoolNchwU8(
RunQLinearAveragePoolNchw(
{1, 1, 32, 64}, // x shape
{1, 1, 32, 64}, // expected y shape
{3, 3}, // kernel shape
@ -432,7 +440,7 @@ TEST(QLinearPoolTest, AveragePool2D_BigImage) {
}
TEST(QLinearPoolTest, AveragePool2D_BigImage_nhwc) {
RunQLinearAveragePoolNhwcU8(
RunQLinearAveragePoolNhwc(
{1, 1, 32, 64}, // x shape
{1, 1, 32, 64}, // expected y shape
{3, 3}, // kernel shape
@ -442,7 +450,7 @@ TEST(QLinearPoolTest, AveragePool2D_BigImage_nhwc) {
}
TEST(QLinearPoolTest, AveragePool2D_Global) {
RunQLinearAveragePoolNchwU8(
RunQLinearAveragePoolNchw(
{1, 2, 32, 16}, // x shape
{1, 2, 1, 1}, // expected y shape
{32, 16}, // kernel shape
@ -452,7 +460,7 @@ TEST(QLinearPoolTest, AveragePool2D_Global) {
}
TEST(QLinearPoolTest, AveragePool2D_Global_nhwc) {
RunQLinearAveragePoolNhwcU8(
RunQLinearAveragePoolNhwc(
{1, 2, 32, 16}, // x shape
{1, 2, 1, 1}, // expected y shape
{32, 16}, // kernel shape
@ -461,6 +469,188 @@ TEST(QLinearPoolTest, AveragePool2D_Global_nhwc) {
1); // count_include_pad
}
TEST(QLinearPoolTest, AveragePool1D_ExcludePadPixel_S8) {
RunQLinearAveragePoolNchw<int8_t>(
{1, 1, 5}, // x shape
{1, 1, 6}, // expected y shape
{3}, // kernel shape
{1}, // strides
{1, 2}, // pads
0); // count_include_pad
}
TEST(QLinearPoolTest, AveragePool1D_IncludePadPixel_S8) {
RunQLinearAveragePoolNchw<int8_t>(
{1, 1, 5}, // x shape
{1, 1, 6}, // expected y shape
{3}, // kernel shape
{1}, // strides
{1, 2}, // pads
1); // count_include_pad
}
TEST(QLinearPoolTest, AveragePool2D_ExcludePadPixel_S8) {
RunQLinearAveragePoolNchw<int8_t>(
{1, 1, 5, 7}, // x shape
{1, 1, 6, 4}, // expected y shape
{3, 4}, // kernel shape
{1, 2}, // strides
{1, 3, 2, 1}, // pads
0); // count_include_pad
}
TEST(QLinearPoolTest, AveragePool2D_IncludePadPixel_S8) {
RunQLinearAveragePoolNchw<int8_t>(
{1, 1, 5, 7}, // x shape
{1, 1, 6, 4}, // expected y shape
{3, 4}, // kernel shape
{1, 2}, // strides
{1, 3, 2, 1}, // pads
1); // count_include_pad
}
TEST(QLinearPoolTest, AveragePool2D_MultiChannel_S8) {
RunQLinearAveragePoolNchw<int8_t>(
{1, 3, 5, 7}, // x shape
{1, 3, 6, 4}, // expected y shape
{3, 4}, // kernel shape
{1, 2}, // strides
{1, 3, 2, 1}, // pads
1); // count_include_pad
}
TEST(QLinearPoolTest, AveragePool3D_ExcludePadPixel_S8) {
RunQLinearAveragePoolNchw<int8_t>(
{1, 1, 5, 7, 9}, // x shape
{1, 1, 6, 4, 3}, // expected y shape
{3, 4, 5}, // kernel shape
{1, 2, 3}, // strides
{1, 3, 2, 2, 1, 2}, // pads
0); // count_include_pad
}
TEST(QLinearPoolTest, AveragePool3D_IncludePadPixel_S8) {
RunQLinearAveragePoolNchw<int8_t>(
{1, 1, 5, 7, 9}, // x shape
{1, 1, 6, 4, 3}, // expected y shape
{3, 4, 5}, // kernel shape
{1, 2, 3}, // strides
{1, 3, 2, 2, 1, 2}, // pads
1); // count_include_pad
}
/*************************************************
* Channels last test
**************************************************/
TEST(QLinearPoolTest, AveragePool1D_ExcludePadPixel_nhwc_S8) {
RunQLinearAveragePoolNhwc<int8_t>(
{1, 1, 5}, // x shape
{1, 1, 6}, // expected y shape
{3}, // kernel shape
{1}, // strides
{1, 2}, // pads
0); // count_include_pad
}
TEST(QLinearPoolTest, AveragePool1D_IncludePadPixel_nhwc_S8) {
RunQLinearAveragePoolNhwc<int8_t>(
{1, 1, 5}, // x shape
{1, 1, 6}, // expected y shape
{3}, // kernel shape
{1}, // strides
{1, 2}, // pads
1); // count_include_pad
}
TEST(QLinearPoolTest, AveragePool2D_ExcludePadPixel_nhwc_S8) {
RunQLinearAveragePoolNhwc<int8_t>(
{1, 1, 5, 7}, // x shape
{1, 1, 6, 4}, // expected y shape
{3, 4}, // kernel shape
{1, 2}, // strides
{1, 3, 2, 1}, // pads
0); // count_include_pad
}
TEST(QLinearPoolTest, AveragePool2D_IncludePadPixel_nhwc_S8) {
RunQLinearAveragePoolNhwc<int8_t>(
{1, 1, 5, 7}, // x shape
{1, 1, 6, 4}, // expected y shape
{3, 4}, // kernel shape
{1, 2}, // strides
{1, 3, 2, 1}, // pads
1); // count_include_pad
}
TEST(QLinearPoolTest, AveragePool2D_MultiChannel_nhwc_S8) {
RunQLinearAveragePoolNhwc<int8_t>(
{1, 3, 5, 7}, // x shape
{1, 3, 6, 4}, // expected y shape
{3, 4}, // kernel shape
{1, 2}, // strides
{1, 3, 2, 1}, // pads
1); // count_include_pad
}
TEST(QLinearPoolTest, AveragePool3D_ExcludePadPixel_nhwc_S8) {
RunQLinearAveragePoolNhwc<int8_t>(
{1, 1, 5, 7, 9}, // x shape
{1, 1, 6, 4, 3}, // expected y shape
{3, 4, 5}, // kernel shape
{1, 2, 3}, // strides
{1, 3, 2, 2, 1, 2}, // pads
0); // count_include_pad
}
TEST(QLinearPoolTest, AveragePool3D_IncludePadPixel_nhwc_S8) {
RunQLinearAveragePoolNhwc<int8_t>(
{1, 1, 5, 7, 9}, // x shape
{1, 1, 6, 4, 3}, // expected y shape
{3, 4, 5}, // kernel shape
{1, 2, 3}, // strides
{1, 3, 2, 2, 1, 2}, // pads
1); // count_include_pad
}
TEST(QLinearPoolTest, AveragePool2D_BigImage_S8) {
RunQLinearAveragePoolNchw<int8_t>(
{1, 1, 32, 64}, // x shape
{1, 1, 32, 64}, // expected y shape
{3, 3}, // kernel shape
{1, 1}, // strides
{1, 1, 1, 1}, // pads
1); // count_include_pad
}
TEST(QLinearPoolTest, AveragePool2D_BigImage_nhwc_S8) {
RunQLinearAveragePoolNhwc<int8_t>(
{1, 1, 32, 64}, // x shape
{1, 1, 32, 64}, // expected y shape
{3, 3}, // kernel shape
{1, 1}, // strides
{1, 1, 1, 1}, // pads
1); // count_include_pad
}
TEST(QLinearPoolTest, AveragePool2D_Global_S8) {
RunQLinearAveragePoolNchw<int8_t>(
{1, 2, 32, 16}, // x shape
{1, 2, 1, 1}, // expected y shape
{32, 16}, // kernel shape
{1, 1}, // strides
{0, 0, 0, 0}, // pads
1); // count_include_pad
}
TEST(QLinearPoolTest, AveragePool2D_Global_nhwc_S8) {
RunQLinearAveragePoolNhwc<int8_t>(
{1, 2, 32, 16}, // x shape
{1, 2, 1, 1}, // expected y shape
{32, 16}, // kernel shape
{1, 1}, // strides
{0, 0, 0, 0}, // pads
1); // count_include_pad
}
} // namespace test
} // namespace onnxruntime

85
onnxruntime/test/mlas/unittest/test_qlinear_gavgpool.cpp
View file

@ -3,24 +3,28 @@
#include "test_util.h"
class MlasQLinearGlobalAveragePoolU8Test : public MlasTestBase {
private:
MatrixGuardBuffer<uint8_t> BufferInput;
MatrixGuardBuffer<uint8_t> BufferOutput;
MatrixGuardBuffer<uint8_t> BufferOutputReference;
#include <vector>
static void CalculateGlobalAvgPoolU8(
const uint8_t* x, int64_t batch, int64_t channel, int64_t hw, bool channel_last,
uint8_t* y, int32_t x_zero_point, float x_scale, int32_t y_zero_point, float y_scale) {
template <typename T8Bits>
class MlasQLinearGlobalAveragePoolTest : public MlasTestBase {
private:
MatrixGuardBuffer<T8Bits> BufferInput;
MatrixGuardBuffer<T8Bits> BufferOutput;
MatrixGuardBuffer<T8Bits> BufferOutputReference;
static const std::vector<T8Bits> ZeroPoints;
static void CalculateGlobalAvgPool(
const T8Bits* x, int64_t batch, int64_t channel, int64_t hw, bool channel_last,
T8Bits* y, int32_t x_zero_point, float x_scale, int32_t y_zero_point, float y_scale) {
int32_t bias = -x_zero_point * static_cast<int32_t>(hw);
int64_t stride_image = channel_last ? channel : 1;
int64_t stride_channel = channel_last ? 1 : hw;
for (int64_t b = 0; b < batch; ++b) {
const uint8_t* bx = x + b * hw * channel;
uint8_t* by = y + b * channel;
const T8Bits* bx = x + b * hw * channel;
T8Bits* by = y + b * channel;
for (int64_t c = 0; c < channel; ++c) {
const uint8_t* ix = bx + c * stride_channel;
const T8Bits* ix = bx + c * stride_channel;
int32_t sum = 0;
for (int64_t i = 0; i < hw; ++i) {
sum += static_cast<int32_t>(*ix);
@ -29,15 +33,15 @@ class MlasQLinearGlobalAveragePoolU8Test : public MlasTestBase {
sum += bias;
int32_t r = static_cast<int32_t>(std::nearbyintf(x_scale * sum / static_cast<float>(hw) / y_scale));
r += y_zero_point;
r = std::min(255, r);
r = std::max(0, r);
by[c] = static_cast<uint8_t>(r);
r = std::min((int32_t)(std::numeric_limits<T8Bits>::max()), r);
r = std::max((int32_t)(std::numeric_limits<T8Bits>::lowest()), r);
by[c] = static_cast<T8Bits>(r);
}
}
}
static void CompareResultWithGold(size_t Batch, size_t Channel,
uint8_t* Output, uint8_t* OutputReference, std::string& info) {
T8Bits* Output, T8Bits* OutputReference, std::string& info) {
size_t n = 0;
for (size_t b = 0; b < Batch; ++b) {
for (size_t c = 0; c < Channel; c++) {
@ -53,9 +57,9 @@ class MlasQLinearGlobalAveragePoolU8Test : public MlasTestBase {
size_t Channel,
size_t ImageSize,
float InputScale,
uint8_t InputZeroPoint,
T8Bits InputZeroPoint,
float OutputScale,
uint8_t OutputZeroPoint) {
T8Bits OutputZeroPoint) {
std::stringstream ss;
ss << (channel_last ? "Nhwc_" : "Nchw_");
ss << Batch << "x [C=" << Stride << "-" << Channel << "] x" << ImageSize << "-";
@ -69,25 +73,25 @@ class MlasQLinearGlobalAveragePoolU8Test : public MlasTestBase {
size_t Channel,
size_t ImageSize,
float InputScale,
uint8_t InputZeroPoint,
T8Bits InputZeroPoint,
float OutputScale,
uint8_t OutputZeroPoint,
T8Bits OutputZeroPoint,
int32_t UnalignedOffset = 0) {
size_t N = Batch * Stride * ImageSize;
size_t ResultLen = Batch * Stride;
uint8_t* Input = BufferInput.GetBuffer(N);
uint8_t* Output = BufferOutput.GetBuffer(ResultLen);
uint8_t* Gold = BufferOutputReference.GetBuffer(ResultLen);
T8Bits* Input = BufferInput.GetBuffer(N);
T8Bits* Output = BufferOutput.GetBuffer(ResultLen);
T8Bits* Gold = BufferOutputReference.GetBuffer(ResultLen);
std::string test_info = GetTestInfo(
channel_last, Batch, Stride, Channel, ImageSize,
InputScale, InputZeroPoint, OutputScale, OutputZeroPoint);
std::default_random_engine generator(static_cast<unsigned>(N));
std::uniform_int_distribution<int> distribution(0, 255);
std::uniform_int_distribution<int> distribution(std::numeric_limits<T8Bits>::lowest(), std::numeric_limits<T8Bits>::max());
for (size_t n = 0; n < N; n++) {
Input[n] = static_cast<uint8_t>(distribution(generator));
Input[n] = static_cast<T8Bits>(distribution(generator));
}
CalculateGlobalAvgPoolU8(
CalculateGlobalAvgPool(
Input, Batch, Stride, ImageSize, channel_last,
Gold, InputZeroPoint, InputScale, OutputZeroPoint, OutputScale);
@ -98,7 +102,7 @@ class MlasQLinearGlobalAveragePoolU8Test : public MlasTestBase {
OutputScale, OutputZeroPoint, ResultLen, ImageSize, acc.data() + UnalignedOffset);
} else {
std::vector<int32_t> acc(MlasQLinearSafePaddingElementCount(sizeof(int32_t), Channel + UnalignedOffset));
std::vector<uint8_t> zero(MlasQLinearSafePaddingElementCount(sizeof(uint8_t), Channel + UnalignedOffset));
std::vector<T8Bits> zero(MlasQLinearSafePaddingElementCount(sizeof(T8Bits), Channel + UnalignedOffset));
if (Stride == Channel) {
MlasQLinearGlobalAveragePoolNhwc(
Input, InputScale, InputZeroPoint, Output,
@ -120,12 +124,12 @@ class MlasQLinearGlobalAveragePoolU8Test : public MlasTestBase {
public:
static const char* GetTestSuiteName() {
static const std::string suite_name("QLinearGlobalAvgPool");
constexpr bool is_signed = std::is_signed<T8Bits>::value;
static const std::string suite_name(is_signed ? "QLinearGlobalAvgPoolS8" : "QLinearGlobalAvgPoolU8");
return suite_name.c_str();
}
void ExecuteShort(void) override {
static const uint8_t zero_points[] = {0, 18, 128, 231, 255};
static const float scales[] = {18.0f, 90.0f};
static const size_t Batch[] = {1, 3};
static const size_t Stride[] = {7, 8, 63, 256};
@ -134,17 +138,17 @@ class MlasQLinearGlobalAveragePoolU8Test : public MlasTestBase {
for (int channel_last = 0; channel_last <= 1; ++channel_last) {
for (size_t b = 0; b < _countof(Batch); b++) {
for (size_t xzp = 0; xzp < _countof(zero_points); xzp++) {
for (size_t yzp = 0; yzp < _countof(zero_points); yzp++) {
for (size_t xzp = 0; xzp < ZeroPoints.size(); xzp++) {
for (size_t yzp = 0; yzp < ZeroPoints.size(); yzp++) {
for (size_t xs = 0; xs < _countof(scales); ++xs) {
for (size_t ys = 0; ys < _countof(scales); ++ys) {
for (size_t i = 0; i < _countof(ImageSize); i++) {
for (size_t s = 0; s < _countof(Stride); s++) {
Test(channel_last != 0, Batch[b], Stride[s], Stride[s], ImageSize[i],
scales[xs], zero_points[xzp], scales[ys], zero_points[yzp], unalign_offset);
scales[xs], ZeroPoints[xzp], scales[ys], ZeroPoints[yzp], unalign_offset);
if (channel_last == 1 && Stride[s] > 32) {
Test(channel_last != 0, Batch[b], Stride[s], 32, ImageSize[i],
scales[xs], zero_points[xzp], scales[ys], zero_points[yzp], unalign_offset);
scales[xs], ZeroPoints[xzp], scales[ys], ZeroPoints[yzp], unalign_offset);
}
unalign_offset = (unalign_offset + 1) & 3;
}
@ -158,8 +162,21 @@ class MlasQLinearGlobalAveragePoolU8Test : public MlasTestBase {
}
};
template <> MlasQLinearGlobalAveragePoolU8Test* MlasTestFixture<MlasQLinearGlobalAveragePoolU8Test>::mlas_tester(nullptr);
template <>
MlasQLinearGlobalAveragePoolTest<int8_t>* MlasTestFixture<MlasQLinearGlobalAveragePoolTest<int8_t>>::mlas_tester(nullptr);
template <>
MlasQLinearGlobalAveragePoolTest<uint8_t>* MlasTestFixture<MlasQLinearGlobalAveragePoolTest<uint8_t>>::mlas_tester(nullptr);
template <>
const std::vector<int8_t> MlasQLinearGlobalAveragePoolTest<int8_t>::ZeroPoints = {-128, -110, 1, 103, 127};
template <>
const std::vector<uint8_t> MlasQLinearGlobalAveragePoolTest<uint8_t>::ZeroPoints = {0, 18, 128, 231, 255};
static UNUSED_VARIABLE bool added_to_main = AddTestRegister([](bool is_short_execute) {
return is_short_execute ? MlasDirectShortExecuteTests<MlasQLinearGlobalAveragePoolU8Test>::RegisterShortExecute() : 0;
if (is_short_execute) {
return MlasDirectShortExecuteTests<MlasQLinearGlobalAveragePoolTest<int8_t>>::RegisterShortExecute() +
MlasDirectShortExecuteTests<MlasQLinearGlobalAveragePoolTest<uint8_t>>::RegisterShortExecute();
}
return (size_t)0;
});

4
onnxruntime/test/testdata/kernel_def_hashes/contrib.cpu.json vendored
View file

@ -27,6 +27,10 @@
"MeanVarianceNormalization ai.onnx CPUExecutionProvider",
13114085849278607104
],
[
"NhwcMaxPool com.microsoft CPUExecutionProvider",
11773579655431087496
],
[
"ParametricSoftplus ai.onnx CPUExecutionProvider",
17971715260566574960