support AveragePool19 and Pad19 (#15597)

This commit is contained in:
liqun Fu 2023-05-15 10:46:24 -07:00 committed by GitHub
parent 22bc020dcd
commit a8d9b29cd2
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
10 changed files with 578 additions and 21 deletions

View file

@ -36,7 +36,8 @@ Do not modify directly.*
|Asinh|*in* input:**T**<br> *out* output:**T**|9+|**T** = tensor(float)|
|Atan|*in* input:**T**<br> *out* output:**T**|7+|**T** = tensor(float)|
|Atanh|*in* input:**T**<br> *out* output:**T**|9+|**T** = tensor(float)|
|AveragePool|*in* X:**T**<br> *out* Y:**T**|11+|**T** = tensor(float)|
|AveragePool|*in* X:**T**<br> *out* Y:**T**|19+|**T** = tensor(float)|
|||[11, 18]|**T** = tensor(float)|
|||10|**T** = tensor(float)|
|||[7, 9]|**T** = tensor(float)|
|BatchNormalization|*in* X:**T**<br> *in* scale:**T**<br> *in* B:**T**<br> *in* input_mean:**U**<br> *in* input_var:**U**<br> *out* Y:**T**<br> *out* running_mean:**U**<br> *out* running_var:**U**<br><br>or<br><br>*in* X:**T**<br> *in* scale:**T**<br> *in* B:**T**<br> *in* mean:**T**<br> *in* var:**T**<br> *out* Y:**T**<br> *out* mean:**T**<br> *out* var:**T**<br> *out* saved_mean:**T**<br> *out* saved_var:**T**<br><br>or<br><br>*in* X:**T**<br> *in* scale:**T1**<br> *in* B:**T1**<br> *in* input_mean:**T2**<br> *in* input_var:**T2**<br> *out* Y:**T**<br> *out* running_mean:**T2**<br> *out* running_var:**T2**|15+|**T** = tensor(double), tensor(float)<br/> **T1** = tensor(double), tensor(float)<br/> **T2** = tensor(double), tensor(float)|
@ -225,7 +226,8 @@ Do not modify directly.*
|PRelu|*in* X:**T**<br> *in* slope:**T**<br> *out* Y:**T**|16+|**T** = tensor(float)|
|||[9, 15]|**T** = tensor(float)|
|||[7, 8]|**T** = tensor(float)|
|Pad|*in* data:**T**<br> *in* pads:**tensor(int64)**<br> *in* constant_value:**T**<br> *in* axes:**Tind**<br> *out* output:**T**<br><br>or<br><br>*in* data:**T**<br> *in* pads:**tensor(int64)**<br> *in* constant_value:**T**<br> *out* output:**T**<br><br>or<br><br>*in* data:**T**<br> *out* output:**T**|18+|**T** = tensor(bool), tensor(double), tensor(float), tensor(int32), tensor(int64), tensor(int8), tensor(uint32), tensor(uint64), tensor(uint8)|
|Pad|*in* data:**T**<br> *in* pads:**tensor(int64)**<br> *in* constant_value:**T**<br> *in* axes:**Tind**<br> *out* output:**T**<br><br>or<br><br>*in* data:**T**<br> *in* pads:**tensor(int64)**<br> *in* constant_value:**T**<br> *out* output:**T**<br><br>or<br><br>*in* data:**T**<br> *out* output:**T**|19+|**T** = tensor(bool), tensor(double), tensor(float), tensor(int32), tensor(int64), tensor(int8), tensor(uint32), tensor(uint64), tensor(uint8)|
|||18|**T** = tensor(bool), tensor(double), tensor(float), tensor(int32), tensor(int64), tensor(int8), tensor(uint32), tensor(uint64), tensor(uint8)|
|||[13, 17]|**T** = tensor(bool), tensor(double), tensor(float), tensor(int32), tensor(int64), tensor(int8), tensor(uint32), tensor(uint64), tensor(uint8)|
|||[11, 12]|**T** = tensor(double), tensor(float), tensor(int32), tensor(int64), tensor(int8), tensor(uint32), tensor(uint64), tensor(uint8)|
|||[2, 10]|**T** = tensor(double), tensor(float)|

View file

@ -460,13 +460,13 @@ class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDoma
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Det);
class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, ScatterElements);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, NonMaxSuppression);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, AveragePool);
class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 18, AveragePool);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, MaxUnpool);
class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 17, LpPool);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Conv);
#ifdef MLAS_F16VEC_INTRINSICS_SUPPORTED
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, MLFloat16, Conv);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, MLFloat16, AveragePool);
class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 18, MLFloat16, AveragePool);
#endif
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, ConvTranspose);
class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, If);
@ -913,15 +913,22 @@ class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain,
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 18, uint16_t, BitwiseXor);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 18, uint32_t, BitwiseXor);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 18, uint64_t, BitwiseXor);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 18, Pad);
class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 18, 18, Pad);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 18, ScatterND);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 18, ScatterElements);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 18, Split);
#if !defined(DISABLE_OPTIONAL_TYPE)
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 18, OptionalHasElement);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 18, OptionalGetElement);
#endif
// Opset 19
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 19, AveragePool);
#ifdef MLAS_F16VEC_INTRINSICS_SUPPORTED
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 19, MLFloat16, AveragePool);
#endif
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 19, Pad);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 19, float, Resize);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 19, int32_t, Resize);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 19, int8_t, Resize);
@ -1534,7 +1541,7 @@ Status RegisterOnnxOperatorKernels(KernelRegistry& kernel_registry) {
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 12, ScatterElements)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11,
NonMaxSuppression)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, AveragePool)>,
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 18, AveragePool)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, MaxUnpool)>,
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 17, LpPool)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, Conv)>,
@ -2292,7 +2299,7 @@ Status RegisterOnnxOperatorKernels(KernelRegistry& kernel_registry) {
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 18, uint16_t, BitwiseXor)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 18, uint32_t, BitwiseXor)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 18, uint64_t, BitwiseXor)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 18, Pad)>,
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 18, 18, Pad)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 18, ScatterND)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 18, ScatterElements)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 18, Split)>,
@ -2300,6 +2307,10 @@ Status RegisterOnnxOperatorKernels(KernelRegistry& kernel_registry) {
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 18, OptionalHasElement)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 18, OptionalGetElement)>,
#endif
// Opset 19
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 19, AveragePool)>,
BuildKernelCreateInfo<ONNX_OPERATOR_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 19, Pad)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 19, float, Resize)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 19, int32_t, Resize)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 19, int8_t, Resize)>,
@ -2321,7 +2332,8 @@ Status RegisterFp16Kernels(KernelRegistry& kernel_registry) {
static const BuildKernelCreateInfoFn function_table[] = {
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 1, MLFloat16, GlobalAveragePool)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, MLFloat16, Conv)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, MLFloat16, AveragePool)>,
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 11, 18, MLFloat16, AveragePool)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 19, MLFloat16, AveragePool)>,
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 8, 11, MLFloat16, MaxPool)>,
BuildKernelCreateInfo<ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 12, MLFloat16, MaxPool)>,
BuildKernelCreateInfo<ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kCpuExecutionProvider, kOnnxDomain, 6, 12, MLFloat16, Relu)>,

View file

@ -231,9 +231,15 @@ ONNX_CPU_OPERATOR_TYPED_KERNEL(
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<MLFloat16>()),
PoolFp16);
ONNX_CPU_OPERATOR_VERSIONED_TYPED_KERNEL(
AveragePool, 11, 18,
MLFloat16,
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<MLFloat16>()),
PoolFp16);
ONNX_CPU_OPERATOR_TYPED_KERNEL(
AveragePool,
11,
19,
MLFloat16,
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<MLFloat16>()),
PoolFp16);

View file

@ -249,6 +249,81 @@ Status MaxPoolV8::ComputeImpl(OpKernelContext* context) const {
return Status::OK();
}
template <typename T>
Status AveragePoolV19<T>::Compute(OpKernelContext* context) const {
concurrency::ThreadPool* tp = context->GetOperatorThreadPool();
bool need_dilation = false;
for (auto n : pool_attrs_.dilations) {
need_dilation |= n > 1;
}
const auto* X = context->Input<Tensor>(0);
const TensorShape& x_shape = X->Shape();
ORT_RETURN_IF_NOT(x_shape.NumDimensions() >= 3, "Input dimension cannot be less than 3.");
auto pads = pool_attrs_.pads;
auto kernel_shape = pool_attrs_.kernel_shape;
auto output_dims = pool_attrs_.SetOutputSize(x_shape, x_shape[1], &pads);
Tensor* Y = context->Output(0, output_dims);
const auto* X_data = X->Data<T>();
auto* Y_data = Y->MutableData<T>();
// The main loop
int64_t channels = x_shape[1];
int64_t height = x_shape[2];
int64_t width = kernel_shape.size() > 1 ? x_shape[3] : 1;
int64_t depth = kernel_shape.size() > 2 ? x_shape[4] : 1;
int64_t pooled_height = output_dims[2];
int64_t pooled_width = kernel_shape.size() > 1 ? output_dims[3] : 1;
int64_t pooled_depth = kernel_shape.size() > 2 ? output_dims[4] : 1;
const int64_t total_channels = x_shape[0] * channels;
switch (kernel_shape.size()) {
case 1: {
int64_t x_step = height;
int64_t y_step = pooled_height;
const int64_t dilation_h = pool_attrs_.dilations[0];
RunLoop<AveragePool1DTask<T>>(tp, onnxruntime::narrow<size_t>(total_channels),
{X_data, Y_data, x_step, y_step, dilation_h, pooled_height, stride_h(),
height, kernel_shape, pads, pool_attrs_.count_include_pad, p_});
break;
}
case 2: {
int64_t x_step = height * width;
int64_t y_step = pooled_height * pooled_width;
const int64_t dilation_h = pool_attrs_.dilations[0];
const int64_t dilation_w = pool_attrs_.dilations[1];
RunLoop<AveragePool2DTask<T>>(
tp, onnxruntime::narrow<size_t>(total_channels),
{X_data, Y_data, x_step, y_step, dilation_h, dilation_w, pooled_height, pooled_width, stride_h(),
stride_w(), height, width, kernel_shape, pads, pool_attrs_.count_include_pad, p_});
break;
}
case 3: {
int64_t x_step = height * width * depth;
int64_t y_step = pooled_height * pooled_width * pooled_depth;
const int64_t dilation_h = pool_attrs_.dilations[0];
const int64_t dilation_w = pool_attrs_.dilations[1];
const int64_t dilation_d = pool_attrs_.dilations[2];
RunLoop<AveragePool3DTask<T>>(tp, onnxruntime::narrow<size_t>(total_channels),
{X_data, Y_data, x_step, y_step,
dilation_h, dilation_w, dilation_d, pooled_height, pooled_width,
pooled_depth, stride_h(), stride_w(), stride_d(), height,
width, depth, kernel_shape, pads, pool_attrs_.count_include_pad, p_});
break;
}
default:
return Status(ONNXRUNTIME, INVALID_ARGUMENT, "Unsupported kernel dimension : " + std::to_string(kernel_shape.size()));
}
return Status::OK();
}
template <typename T>
Status LpPoolV18<T>::Compute(OpKernelContext* context) const {
concurrency::ThreadPool* tp = context->GetOperatorThreadPool();
@ -332,8 +407,16 @@ ONNX_CPU_OPERATOR_VERSIONED_KERNEL(AveragePool, 10, 10,
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<float>()),
Pool<float, AveragePool>);
ONNX_CPU_OPERATOR_KERNEL(AveragePool, 11, KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<float>()),
Pool<float, AveragePool>);
ONNX_CPU_OPERATOR_VERSIONED_KERNEL(AveragePool, 11, 18,
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<float>()),
Pool<float, AveragePool>);
ONNX_CPU_OPERATOR_KERNEL(AveragePool, 19,
KernelDefBuilder()
.TypeConstraint(
"T",
DataTypeImpl::GetTensorType<float>()),
AveragePoolV19<float>);
ONNX_CPU_OPERATOR_VERSIONED_KERNEL(MaxPool, 1, 7,
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<float>()),

View file

@ -25,6 +25,20 @@ class Pool : public OpKernel, public PoolBase {
PoolProcessContext pool_context_;
};
// For averagepool v19 and beyond
// version 19: Added dilations
template <typename T>
class AveragePoolV19 : public OpKernel, public PoolBase {
public:
AveragePoolV19(const OpKernelInfo& info) : OpKernel(info), PoolBase(info) {
}
Status Compute(OpKernelContext* context) const override;
private:
int64_t p_;
};
// For maxpool v8 and beyond
// version 8: Added storage_order And Indices
// version 10: Added ceil_mode

View file

@ -376,6 +376,199 @@ struct MaxPool3DTask {
}
};
template <typename T>
struct AveragePool1DTask final {
const T* X_data;
T* Y_data;
int64_t x_step;
int64_t y_step;
int64_t dilation_h;
int64_t pooled_height;
int64_t stride_h;
int64_t height;
gsl::span<const int64_t> kernel_shape;
gsl::span<const int64_t> pads;
bool count_include_pad;
int64_t p;
TensorOpCost Cost() {
double loop_count = static_cast<double>(pooled_height * kernel_shape[0]);
return TensorOpCost{loop_count, loop_count, loop_count};
}
void operator()(std::ptrdiff_t begin, std::ptrdiff_t end) const {
for (std::ptrdiff_t c = begin; c < end; ++c) {
operator()(c);
}
}
void operator()(std::ptrdiff_t c) const {
const T* x_d = X_data + c * x_step;
T* y_d = Y_data + c * y_step;
for (int64_t ph = 0; ph < pooled_height; ++ph) {
int64_t hstart = ph * stride_h - pads[0];
int64_t hend = hstart + kernel_shape[0] * dilation_h;
y_d[ph] = 0;
int total_elements = 0;
for (int64_t h = hstart; h < hend; h += dilation_h) {
if (math::is_a_ge_zero_and_a_lt_b(h, height)) {
y_d[ph] += x_d[h];
total_elements++;
}
}
if (total_elements > 0) {
if (count_include_pad) {
y_d[ph] /= (1 + (hend - hstart - 1) / dilation_h);
} else {
y_d[ph] /= total_elements;
}
}
}
}
};
template <typename T>
struct AveragePool2DTask final {
const T* X_data;
T* Y_data;
int64_t x_step;
int64_t y_step;
int64_t dilation_h;
int64_t dilation_w;
int64_t pooled_height;
int64_t pooled_width;
int64_t stride_h;
int64_t stride_w;
int64_t height;
int64_t width;
gsl::span<const int64_t> kernel_shape;
gsl::span<const int64_t> pads;
bool count_include_pad;
int64_t p;
TensorOpCost Cost() {
double loop_count = static_cast<double>(pooled_height * pooled_width * kernel_shape[0] * kernel_shape[1]);
return TensorOpCost{loop_count, loop_count, loop_count};
}
void operator()(std::ptrdiff_t begin, std::ptrdiff_t end) const {
for (std::ptrdiff_t c = begin; c < end; ++c) {
operator()(c);
}
}
void operator()(std::ptrdiff_t c) const {
const T* x_d = X_data + c * x_step;
T* y_d = Y_data + c * y_step;
for (int64_t ph = 0; ph < pooled_height; ++ph) {
int64_t hstart = ph * stride_h - pads[0];
int64_t hend = hstart + kernel_shape[0] * dilation_h;
for (int64_t pw = 0; pw < pooled_width; ++pw) {
int64_t wstart = pw * stride_w - pads[1];
int64_t wend = wstart + kernel_shape[1] * dilation_w;
const int64_t pool_index = ph * pooled_width + pw;
y_d[pool_index] = 0;
int total_elements = 0;
for (int64_t h = hstart; h < hend; h += dilation_h) {
if (math::is_a_ge_zero_and_a_lt_b(h, height)) {
for (int64_t w = wstart; w < wend; w += dilation_w) {
if (math::is_a_ge_zero_and_a_lt_b(w, width)) {
const int64_t input_index = h * width + w;
y_d[pool_index] += x_d[input_index];
total_elements++;
}
}
}
}
if (total_elements > 0) {
if (count_include_pad) {
y_d[pool_index] /= ((1 + (hend - hstart - 1) / dilation_h) * (1 + (wend - wstart - 1) / dilation_w));
} else {
y_d[pool_index] /= total_elements;
}
}
}
}
}
};
template <typename T>
struct AveragePool3DTask {
const T* X_data;
T* Y_data;
int64_t x_step;
int64_t y_step;
int64_t dilation_h;
int64_t dilation_w;
int64_t dilation_d;
int64_t pooled_height;
int64_t pooled_width;
int64_t pooled_depth;
int64_t stride_h;
int64_t stride_w;
int64_t stride_d;
int64_t height;
int64_t width;
int64_t depth;
gsl::span<const int64_t> kernel_shape;
gsl::span<const int64_t> pads;
bool count_include_pad;
int64_t p;
void operator()(std::ptrdiff_t begin, std::ptrdiff_t end) const {
for (std::ptrdiff_t c = begin; c < end; ++c) {
operator()(c);
}
}
TensorOpCost Cost() {
double loop_count = static_cast<double>(pooled_height * pooled_width * pooled_depth * kernel_shape[0] *
kernel_shape[1] * kernel_shape[2]);
return TensorOpCost{loop_count, loop_count, loop_count};
}
void operator()(std::ptrdiff_t c) const {
const T* x_d = X_data + c * x_step;
T* y_d = Y_data + c * y_step;
for (int64_t ph = 0; ph < pooled_height; ++ph) {
int64_t hstart = ph * stride_h - pads[0];
int64_t hend = hstart + kernel_shape[0] * dilation_h;
for (int64_t pw = 0; pw < pooled_width; ++pw) {
int64_t wstart = pw * stride_w - pads[1];
int64_t wend = wstart + kernel_shape[1] * dilation_w;
for (int64_t pd = 0; pd < pooled_depth; ++pd) {
int64_t dstart = pd * stride_d - pads[2];
int64_t dend = dstart + kernel_shape[2] * dilation_d;
const int64_t pool_index = ph * pooled_width * pooled_depth + pw * pooled_depth + pd;
y_d[pool_index] = 0;
int total_elements = 0;
for (int64_t h = hstart; h < hend; h += dilation_h) {
if (math::is_a_ge_zero_and_a_lt_b(h, height)) {
for (int64_t w = wstart; w < wend; w += dilation_w) {
if (math::is_a_ge_zero_and_a_lt_b(w, width)) {
for (int64_t d = dstart; d < dend; d += dilation_d) {
if (math::is_a_ge_zero_and_a_lt_b(d, depth)) {
const int64_t input_index = h * width * depth + w * depth + d;
y_d[pool_index] += x_d[input_index];
total_elements++;
}
}
}
}
}
}
if (total_elements > 0) {
if (count_include_pad) {
y_d[pool_index] /= ((1 + (hend - hstart - 1) / dilation_h) * (1 + (wend - wstart - 1) / dilation_w) * (1 + (dend - dstart - 1) / dilation_d));
} else {
y_d[pool_index] /= total_elements;
}
}
}
}
}
}
};
template <typename T>
struct LpPool1DTask final {
const T* X_data;

View file

@ -79,12 +79,26 @@ ORT_SPECIFY_OP_KERNEL_ARG_DEFAULT_TYPES(
uint8_t,
bool);
ORT_SPECIFY_OP_KERNEL_ARG_DEFAULT_TYPES(
kCpuExecutionProvider, kOnnxDomain, Pad, 19, Input, 0,
float,
double,
int32_t,
int64_t,
uint32_t,
uint64_t,
int8_t,
uint8_t,
bool);
ORT_SPECIFY_OP_KERNEL_ARG_REQUIRED_TYPES(
kCpuExecutionProvider, kOnnxDomain, Pad, 11, Input, 0, int32_t, int64_t);
ORT_SPECIFY_OP_KERNEL_ARG_REQUIRED_TYPES(
kCpuExecutionProvider, kOnnxDomain, Pad, 13, Input, 0, int32_t, int64_t);
ORT_SPECIFY_OP_KERNEL_ARG_REQUIRED_TYPES(
kCpuExecutionProvider, kOnnxDomain, Pad, 18, Input, 0, int32_t, int64_t);
ORT_SPECIFY_OP_KERNEL_ARG_REQUIRED_TYPES(
kCpuExecutionProvider, kOnnxDomain, Pad, 19, Input, 0, int32_t, int64_t);
} // namespace op_kernel_type_control
using EnabledPad2Types = ORT_OP_KERNEL_ARG_ENABLED_TYPE_LIST(
@ -95,6 +109,8 @@ using EnabledPad13Types = ORT_OP_KERNEL_ARG_ENABLED_TYPE_LIST(
kCpuExecutionProvider, kOnnxDomain, Pad, 13, Input, 0);
using EnabledPad18Types = ORT_OP_KERNEL_ARG_ENABLED_TYPE_LIST(
kCpuExecutionProvider, kOnnxDomain, Pad, 18, Input, 0);
using EnabledPad19Types = ORT_OP_KERNEL_ARG_ENABLED_TYPE_LIST(
kCpuExecutionProvider, kOnnxDomain, Pad, 19, Input, 0);
using AllEnabledPadTypes =
utils::TypeSetUnion<
@ -131,15 +147,24 @@ ONNX_CPU_OPERATOR_VERSIONED_KERNEL(
BuildKernelDefConstraintsFromTypeList<EnabledPad13Types>()),
Pad);
ONNX_CPU_OPERATOR_KERNEL(
ONNX_CPU_OPERATOR_VERSIONED_KERNEL(
Pad,
18,
18, 18,
KernelDefBuilder()
.TypeConstraint(
"T",
BuildKernelDefConstraintsFromTypeList<EnabledPad18Types>()),
Pad);
ONNX_CPU_OPERATOR_KERNEL(
Pad,
19,
KernelDefBuilder()
.TypeConstraint(
"T",
BuildKernelDefConstraintsFromTypeList<EnabledPad19Types>()),
Pad);
using PadsVector = PadBase::PadsVector;
// This is the general padding method to n-dimensionally do edge or reflection padding (based on the inputDelta values)
@ -426,6 +451,7 @@ static Status PadImpl(OpKernelContext* ctx,
break;
case Mode::Reflect:
case Mode::Wrap:
// Loop over the output tensor, writing out padding between the blocks of copied data
// On loop entry, 'pad' is already set to the first continuous block of padding, and
// after every pass through the inner loop it gets set to the next continuous pad size.
@ -438,12 +464,46 @@ static Status PadImpl(OpKernelContext* ctx,
int64_t prePad = reshaped_pad[inner_axis];
int64_t postPad = reshaped_pad[inner_axis + new_dims_count];
if (inner_no_pad_size == 1) {
PadInnermostAxis(axisStart - prePad, axisStart + prePad, -1 /* inputDelta */, onnxruntime::narrow<size_t>(prePad));
PadInnermostAxis(output, output - 2, -1 /* inputDelta */, onnxruntime::narrow<size_t>(postPad));
if (mode == Mode::Reflect) {
PadInnermostAxis(axisStart - prePad, axisStart + prePad, -1 /* inputDelta */, onnxruntime::narrow<size_t>(prePad));
PadInnermostAxis(output, output - 2, -1 /* inputDelta */, onnxruntime::narrow<size_t>(postPad));
} else {
PadInnermostAxis(axisStart - prePad, output - prePad, 1 /* inputDelta */, onnxruntime::narrow<size_t>(prePad));
PadInnermostAxis(output, axisStart, 1 /* inputDelta */, onnxruntime::narrow<size_t>(postPad));
}
} else {
// When inner_most axis(es) do not need pad, Above PadInnermostAxis() do not fit for Reflect mode.
PadAxis(axisStart - prePad, axisStart + prePad, 1, -ptrdiff_t(inner_no_pad_size * 2), inner_no_pad_size, onnxruntime::narrow<size_t>(pads[inner_axis]));
PadAxis(output, output - 2 * inner_no_pad_size, 1, -ptrdiff_t(inner_no_pad_size * 2), inner_no_pad_size, onnxruntime::narrow<size_t>(pads[inner_axis + data_rank]));
if (mode == Mode::Reflect) {
PadAxis(
axisStart - prePad,
axisStart + prePad,
1,
-ptrdiff_t(inner_no_pad_size * 2),
inner_no_pad_size,
onnxruntime::narrow<size_t>(pads[inner_axis]));
PadAxis(
output,
output - 2 * inner_no_pad_size,
1,
-ptrdiff_t(inner_no_pad_size * 2),
inner_no_pad_size,
onnxruntime::narrow<size_t>(pads[inner_axis + data_rank]));
} else {
PadAxis(
axisStart - prePad,
output - pads[inner_axis] * inner_no_pad_size,
1,
0,
inner_no_pad_size,
onnxruntime::narrow<size_t>(pads[inner_axis]));
PadAxis(
output,
axisStart,
1,
0,
inner_no_pad_size,
onnxruntime::narrow<size_t>(pads[inner_axis + data_rank]));
}
}
output += postPad;
alignSkip = onnxruntime::narrow<size_t>(prePad);
@ -454,9 +514,37 @@ static Status PadImpl(OpKernelContext* ctx,
T* axisStart = output - inner_pitch * input_extents[input_counters.Axis()];
int64_t prePad = reshaped_pad[input_counters.Axis()];
int64_t postPad = reshaped_pad[input_counters.Axis() + new_dims_count];
PadAxis(axisStart - prePad * inner_pitch, axisStart + prePad * inner_pitch, 1, -inner_pitch * 2,
inner_pitch, onnxruntime::narrow<size_t>(prePad));
PadAxis(output, output - 2 * inner_pitch, 1, -inner_pitch * 2, inner_pitch, onnxruntime::narrow<size_t>(postPad));
if (mode == Mode::Reflect) {
PadAxis(
axisStart - prePad * inner_pitch,
axisStart + prePad * inner_pitch,
1,
-inner_pitch * 2,
inner_pitch,
onnxruntime::narrow<size_t>(prePad));
PadAxis(
output,
output - 2 * inner_pitch,
1,
-inner_pitch * 2,
inner_pitch,
onnxruntime::narrow<size_t>(postPad));
} else {
PadAxis(
axisStart - prePad * inner_pitch,
output - prePad * inner_pitch,
1,
0,
inner_pitch,
onnxruntime::narrow<size_t>(prePad));
PadAxis(
output,
axisStart,
1,
0,
inner_pitch,
onnxruntime::narrow<size_t>(postPad));
}
output += inner_pitch * postPad;
alignSkip += inner_pitch * SafeInt<size_t>(prePad);
}

View file

@ -10,7 +10,8 @@ namespace onnxruntime {
enum class Mode : int {
Constant = 0,
Reflect,
Edge
Edge,
Wrap
};
class PadBase {
@ -32,6 +33,8 @@ class PadBase {
mode_ = Mode::Reflect;
else if (mode == "edge")
mode_ = Mode::Edge;
else if (mode == "wrap")
mode_ = Mode::Wrap;
else
ORT_THROW("Invalid 'mode' attribute value");
}

View file

@ -938,6 +938,35 @@ TEST(PoolTest, AveragePool_10_ceil1_2d) {
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider, kAclExecutionProvider});
}
TEST(PoolTest, AveragePool_19_dilation_2d) {
// TODO: Unskip when fixed #41968513
if (DefaultDmlExecutionProvider().get() != nullptr) {
GTEST_SKIP() << "Skipping because of the following error: MLOperatorAuthorImpl.cpp(2100): The parameter is incorrect.";
}
OpTester test("AveragePool", 19);
test.AddAttribute("auto_pad", "");
test.AddAttribute("strides", std::vector<int64_t>{1, 1});
test.AddAttribute("dilations", std::vector<int64_t>{2, 2});
test.AddAttribute("pads", vector<int64_t>{0, 0, 0, 0});
test.AddAttribute("kernel_shape", vector<int64_t>{2, 2});
test.AddAttribute("ceil_mode", (int64_t)1);
std::vector<float> x_vals = {
1, 3, 2, 4,
5, 7, 6, 8,
9, 11, 10, 12,
13, 15, 14, 16};
std::vector<int64_t> x_dims = {1, 1, 4, 4};
std::vector<int64_t> expected_dims = {1, 1, 2, 2};
std::vector<float> expected_vals = {5.5f, 7.5f, 9.5f, 11.5f};
test.AddInput<float>("X", x_dims, x_vals);
test.AddOutput<float>("Y", expected_dims, expected_vals);
test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider, kAclExecutionProvider, kOpenVINOExecutionProvider});
}
TEST(PoolTest, GlobalAveragePool) {
OpTester test("GlobalAveragePool");

View file

@ -173,6 +173,19 @@ TYPED_TEST(PadOpTest, Pad_Reflect_1D) {
"reflect");
}
TYPED_TEST(PadOpTest, Pad_Wrap_1D) {
using T = TypeParam;
RunOnnxOpsetTypedTest<T, 19>({3, 2},
{T(1), T(2), T(3), T(4), T(5), T(6)},
{0, 1, 0, 1},
false,
T(0),
false,
{3, 4},
{T(2), T(1), T(2), T(1), T(4), T(3), T(4), T(3), T(6), T(5), T(6), T(5)},
"wrap");
}
TYPED_TEST(PadOpTest, Pad_Edge_1D) {
using T = TypeParam;
RunAllOpsetAllDomainPadTests<T>({3, 2},
@ -363,6 +376,27 @@ TYPED_TEST(PadOpTest, Pad_Reflect_2D) {
"reflect");
}
TYPED_TEST(PadOpTest, Pad_Wrap_2D) {
using T = TypeParam;
RunOnnxOpsetTypedTest<T, 19>({3, 3},
{T(11), T(21), T(31),
T(12), T(22), T(32),
T(13), T(23), T(33)},
{2, 2, 2, 2},
false,
T(0),
false,
{7, 7},
{T(22), T(32), T(12), T(22), T(32), T(12), T(22),
T(23), T(33), T(13), T(23), T(33), T(13), T(23),
T(21), T(31), T(11), T(21), T(31), T(11), T(21),
T(22), T(32), T(12), T(22), T(32), T(12), T(22),
T(23), T(33), T(13), T(23), T(33), T(13), T(23),
T(21), T(31), T(11), T(21), T(31), T(11), T(21),
T(22), T(32), T(12), T(22), T(32), T(12), T(22)},
"wrap");
}
TYPED_TEST(PadOpTest, Pad_Constant_3D_Inner_No_Padding) {
using T = TypeParam;
RunAllOpsetAllDomainPadTests<T>({3, 2, 5},
@ -540,6 +574,99 @@ TYPED_TEST(PadOpTest, Pad_Reflect_3D_Inner_No_Padding) {
"reflect");
}
TYPED_TEST(PadOpTest, Pad_wrap_3D_Inner_No_Padding) {
using T = TypeParam;
RunOnnxOpsetTypedTest<T, 19>({3, 2, 5},
{T(1), T(2), T(3), T(4), T(5),
T(6), T(7), T(8), T(9), T(10),
T(11), T(12), T(13), T(14), T(15),
T(16), T(17), T(18), T(19), T(20),
T(21), T(22), T(23), T(24), T(25),
T(26), T(27), T(28), T(29), T(30)},
{1, 1, 0, 1, 1, 0},
false,
T(0),
false,
{5, 4, 5},
{T(26), T(27), T(28), T(29), T(30),
T(21), T(22), T(23), T(24), T(25),
T(26), T(27), T(28), T(29), T(30),
T(21), T(22), T(23), T(24), T(25),
T(6), T(7), T(8), T(9), T(10),
T(1), T(2), T(3), T(4), T(5),
T(6), T(7), T(8), T(9), T(10),
T(1), T(2), T(3), T(4), T(5),
T(16), T(17), T(18), T(19), T(20),
T(11), T(12), T(13), T(14), T(15),
T(16), T(17), T(18), T(19), T(20),
T(11), T(12), T(13), T(14), T(15),
T(26), T(27), T(28), T(29), T(30),
T(21), T(22), T(23), T(24), T(25),
T(26), T(27), T(28), T(29), T(30),
T(21), T(22), T(23), T(24), T(25),
T(6), T(7), T(8), T(9), T(10),
T(1), T(2), T(3), T(4), T(5),
T(6), T(7), T(8), T(9), T(10),
T(1), T(2), T(3), T(4), T(5)},
"wrap");
}
TYPED_TEST(PadOpTest, Pad_wrap_3D_Inner_No_Padding2) {
using T = TypeParam;
RunOnnxOpsetTypedTest<T, 19>({3, 2, 5},
{T(1), T(2), T(3), T(4), T(5),
T(6), T(7), T(8), T(9), T(10),
T(11), T(12), T(13), T(14), T(15),
T(16), T(17), T(18), T(19), T(20),
T(21), T(22), T(23), T(24), T(25),
T(26), T(27), T(28), T(29), T(30)},
{1, 2, 0, 1, 2, 0},
false,
T(0),
false,
{5, 6, 5},
{T(21), T(22), T(23), T(24), T(25),
T(26), T(27), T(28), T(29), T(30),
T(21), T(22), T(23), T(24), T(25),
T(26), T(27), T(28), T(29), T(30),
T(21), T(22), T(23), T(24), T(25),
T(26), T(27), T(28), T(29), T(30),
T(1), T(2), T(3), T(4), T(5),
T(6), T(7), T(8), T(9), T(10),
T(1), T(2), T(3), T(4), T(5),
T(6), T(7), T(8), T(9), T(10),
T(1), T(2), T(3), T(4), T(5),
T(6), T(7), T(8), T(9), T(10),
T(11), T(12), T(13), T(14), T(15),
T(16), T(17), T(18), T(19), T(20),
T(11), T(12), T(13), T(14), T(15),
T(16), T(17), T(18), T(19), T(20),
T(11), T(12), T(13), T(14), T(15),
T(16), T(17), T(18), T(19), T(20),
T(21), T(22), T(23), T(24), T(25),
T(26), T(27), T(28), T(29), T(30),
T(21), T(22), T(23), T(24), T(25),
T(26), T(27), T(28), T(29), T(30),
T(21), T(22), T(23), T(24), T(25),
T(26), T(27), T(28), T(29), T(30),
T(1), T(2), T(3), T(4), T(5),
T(6), T(7), T(8), T(9), T(10),
T(1), T(2), T(3), T(4), T(5),
T(6), T(7), T(8), T(9), T(10),
T(1), T(2), T(3), T(4), T(5),
T(6), T(7), T(8), T(9), T(10)},
"wrap");
}
TYPED_TEST(PadOpTest, Pad_Reflect_3D_Last_Pad_Slice_Inner_No_Padding) {
using T = TypeParam;
RunAllOpsetAllDomainPadTests<T>({2, 3, 5},