saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-29 20:14:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add more F16 kernels of XNNPack (#22381)

Browse source 
### Description
1. Add Gemm, MatMul, Softmax, AveragePool and  Resize F16 kernels

This PR has included all changes in #22378


[AB#51066](https://aiinfra.visualstudio.com/6a833879-cd9b-44a4-a9de-adc2d818f13c/_workitems/edit/51066)

[AB#51026](https://aiinfra.visualstudio.com/6a833879-cd9b-44a4-a9de-adc2d818f13c/_workitems/edit/51026)

2. Matrix B must be const and martrix A and B dim_size shoule NOT bigger
than 2 in XNNPack, so I added 2 tests in matmul_test.cc to make sure
it's really tested. (that is, compute() must be called.)
### Motivation and Context
This commit is contained in:
Yi Zhang 2024-10-14 17:41:59 +08:00 committed by GitHub
parent 72cc72cc21
commit caa67439b5
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
19 changed files with 335 additions and 204 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
onnxruntime/core/framework/kernel_registry.cc
View file

@ -6,6 +6,7 @@
#include <algorithm>
#include <memory>
#include <numeric>
#include <string>
#include <unordered_map>
#include "core/framework/kernel_type_str_resolver.h"
@ -310,9 +311,12 @@ Status KernelRegistry::Register(KernelCreateInfo&& create_info) {
for (auto i = range.first; i != range.second; ++i) {
if (i->second.kernel_def &&
i->second.kernel_def->IsConflict(*create_info.kernel_def)) {
return Status(common::ONNXRUNTIME, common::FAIL,
"Failed to add kernel for " + key +
": Conflicting with a registered kernel with op versions.");
int since_version = i->second.kernel_def->SinceVersion().first;
std::string since_version_str = std::to_string(since_version);
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL,
"Failed to add kernel for ", key,
": Conflicting with a registered kernel with op versions. the since version is: ",
since_version_str);
}
}

11
onnxruntime/core/providers/xnnpack/detail/node_support_checker.cc
View file

@ -90,6 +90,17 @@ const NodeUnit* ClipReluChecker(const NodeUnit& node_unit,
} // namespace
bool NodeSupportChecker::IsNodeSupported(const NodeUnit& nodeunit) {
#ifndef XNNPACK_FP16_SUPPORTED
// check whether the hardware support XNNPack FP16
// Note. In CI, ios pipeline on ADO doesn't support XNNPack FP16. Because ADO mac pool is still x64.
const auto& inputs = nodeunit.Inputs();
const auto& x_arg = inputs[0].node_arg;
const auto* x_type = x_arg.TypeAsProto();
if (x_type == nullptr || x_type->tensor_type().elem_type() == ONNX_NAMESPACE::TensorProto_DataType_FLOAT16) {
return false;
}
#endif
static std::unordered_map<std::string, CheckerFn> checkers{
{"Conv", Conv::IsOnnxNodeSupported},
{"ConvTranspose", ConvTranspose::IsOnnxNodeSupported},

115
onnxruntime/core/providers/xnnpack/math/gemm.cc
View file

@ -4,6 +4,7 @@
#include "gemm.h"
#include "core/framework/transpose_helper.h"
#include "core/providers/utils.h"
#include "core/providers/xnnpack/xnnpack_init.h"
namespace onnxruntime {
namespace xnnpack {
@ -37,7 +38,8 @@ bool Gemm::IsOnnxNodeSupported(const NodeUnit& node_unit, const GraphViewer& gra
const auto* A_type = A_arg->TypeAsProto();
if (A_type == nullptr ||
A_type->tensor_type().elem_type() != ONNX_NAMESPACE::TensorProto_DataType_FLOAT) {
(A_type->tensor_type().elem_type() != ONNX_NAMESPACE::TensorProto_DataType_FLOAT &&
A_type->tensor_type().elem_type() != ONNX_NAMESPACE::TensorProto_DataType_FLOAT16)) {
break;
}
@ -74,19 +76,26 @@ bool Gemm::IsOnnxNodeSupported(const NodeUnit& node_unit, const GraphViewer& gra
supported = true;
} while (false);
return supported;
}
Gemm::Gemm(const OpKernelInfo& info) : GemmBase(info), XnnpackKernel(info, /*enable_caches*/ true) {
const auto& node{Node()};
info.GetAttrOrDefault<float>("alpha", &alpha_, 1.f);
info.GetAttrOrDefault<float>("beta", &beta_, 1.f);
const auto& node{Node()};
const auto& input_defs = node.InputDefs();
const auto* shapeA = input_defs[0]->Shape();
const auto* shapeB = input_defs[1]->Shape();
const NodeArg& X = *input_defs[0];
auto input_dtype = X.TypeAsProto()->tensor_type().elem_type();
if (input_dtype == ONNX_NAMESPACE::TensorProto_DataType_FLOAT) {
op_compute_type_ = OpComputeType::op_compute_type_fp32;
} else if (input_dtype == ONNX_NAMESPACE::TensorProto_DataType_FLOAT16) {
op_compute_type_ = OpComputeType::op_compute_type_fp16;
}
const NodeArg* C_arg = input_defs.size() == 2 ? nullptr : input_defs[2];
C_matrix_exists_ = C_arg && C_arg->Exists();
@ -127,32 +136,49 @@ Status Gemm::PrePack(const Tensor& tensor, int input_idx, AllocatorPtr,
// flags - 1 - for no transpose - 0 for transpose
uint32_t flags = trans_B_ == CblasTrans ? 0 : XNN_FLAG_TRANSPOSE_WEIGHTS;
float output_min = clip_min_max_ ? clip_min_max_->first : -INFINITY;
float output_max = clip_min_max_ ? clip_min_max_->second : INFINITY;
const float* bias_Data = nullptr;
if (C_matrix_exists_) {
bias_Data = tensor.Data<float>();
}
auto code_cache = GetCodeCache();
auto weights_cache = GetWeightsCache();
xnn_status status = xnn_status::xnn_status_uninitialized;
struct xnn_operator* p = nullptr;
status = xnn_create_fully_connected_nc_f32(
trans_B_ == CblasNoTrans ? B_->Shape()[0] : B_->Shape()[1], // size_t input_channels,
trans_B_ == CblasNoTrans ? B_->Shape()[1] : B_->Shape()[0], // size_t output_channels,
trans_B_ == CblasNoTrans ? B_->Shape()[0] : B_->Shape()[1], // size_t input_stride,
trans_B_ == CblasNoTrans ? B_->Shape()[1] : B_->Shape()[0], // size_t output_stride,
B_->Data<float>(), // const float* kernel,
bias_Data, // const float* bias,
output_min, output_max,
flags,
GetCodeCache(), GetWeightsCache(),
&p);
float foutput_min = clip_min_max_ ? clip_min_max_->first : -INFINITY;
float foutput_max = clip_min_max_ ? clip_min_max_->second : INFINITY;
if (op_compute_type_ == OpComputeType::op_compute_type_fp32) {
const float* bias_data = nullptr;
if (C_matrix_exists_) {
bias_data = tensor.Data<float>();
}
status = xnn_create_fully_connected_nc_f32(
trans_B_ == CblasNoTrans ? B_->Shape()[0] : B_->Shape()[1], // size_t input_channels,
trans_B_ == CblasNoTrans ? B_->Shape()[1] : B_->Shape()[0], // size_t output_channels,
trans_B_ == CblasNoTrans ? B_->Shape()[0] : B_->Shape()[1], // size_t input_stride,
trans_B_ == CblasNoTrans ? B_->Shape()[1] : B_->Shape()[0], // size_t output_stride,
B_->Data<float>(), // const float* kernel,
bias_data, // const float* bias,
foutput_min, foutput_max,
flags,
code_cache, weights_cache,
&p);
} else if (op_compute_type_ == OpComputeType::op_compute_type_fp16) {
const MLFloat16* bias_data = nullptr;
if (C_matrix_exists_) {
bias_data = tensor.Data<MLFloat16>();
}
status = xnn_create_fully_connected_nc_f16(
trans_B_ == CblasNoTrans ? B_->Shape()[0] : B_->Shape()[1], // size_t input_channels,
trans_B_ == CblasNoTrans ? B_->Shape()[1] : B_->Shape()[0], // size_t output_channels,
trans_B_ == CblasNoTrans ? B_->Shape()[0] : B_->Shape()[1], // size_t input_stride,
trans_B_ == CblasNoTrans ? B_->Shape()[1] : B_->Shape()[0], // size_t output_stride,
B_->Data<MLFloat16>(), // const MLFloat16* kernel,
bias_data, // const float* bias,
foutput_min, foutput_max,
flags,
code_cache, weights_cache,
&p);
}
if (status != xnn_status_success) {
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "xnn_create_fully_connected_nc_f32 returned ", status);
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "xnn_create_fully_connected_nc_",
OpTypeToString(op_compute_type_), " returned ", status);
}
op0_.reset(p);
@ -169,19 +195,30 @@ Status Gemm::Compute(OpKernelContext* context) const {
return Status::OK();
}
xnn_status status = xnn_reshape_fully_connected_nc_f32(op0_.get(),
// Number of rows to multiply
trans_A_ == CblasNoTrans ? M_ : K_,
threadpool);
auto reshape_func = xnn_reshape_fully_connected_nc_f32;
if (op_compute_type_ == OpComputeType::op_compute_type_fp16) {
reshape_func = xnn_reshape_fully_connected_nc_f16;
}
xnn_status status = reshape_func(op0_.get(),
// Number of rows to multiply
trans_A_ == CblasNoTrans ? M_ : K_,
threadpool);
if (status != xnn_status_success) {
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "xnn_reshape_fully_connected_nc_f32 returned ", status);
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "xnn_reshape_fully_connected_nc_",
OpTypeToString(op_compute_type_), " returned ", status);
}
status = xnn_setup_fully_connected_nc_f32(op0_.get(), A->Data<float>(), Y->MutableData<float>());
status = xnn_status_invalid_state;
if (op_compute_type_ == op_compute_type_fp32) {
status = xnn_setup_fully_connected_nc_f32(op0_.get(), A->Data<float>(), Y->MutableData<float>());
} else if (op_compute_type_ == OpComputeType::op_compute_type_fp16) {
status = xnn_setup_fully_connected_nc_f16(op0_.get(), A->Data<MLFloat16>(), Y->MutableData<MLFloat16>());
}
if (status != xnn_status_success) {
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "xnn_setup_fully_connected_nc_f32 returned ", status);
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "xnn_setup_fully_connected_nc_",
OpTypeToString(op_compute_type_), " returned ", status);
}
status = xnn_run_operator(op0_.get(), nullptr);
@ -193,19 +230,23 @@ Status Gemm::Compute(OpKernelContext* context) const {
}
ONNX_OPERATOR_VERSIONED_KERNEL_EX(Gemm, kOnnxDomain, 7, 8, kXnnpackExecutionProvider,
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<float>()),
KernelDefBuilder().TypeConstraint("T", {DataTypeImpl::GetTensorType<float>(),
DataTypeImpl::GetTensorType<MLFloat16>()}),
Gemm);
ONNX_OPERATOR_VERSIONED_KERNEL_EX(Gemm, kOnnxDomain, 9, 10, kXnnpackExecutionProvider,
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<float>()),
KernelDefBuilder().TypeConstraint("T", {DataTypeImpl::GetTensorType<float>(),
DataTypeImpl::GetTensorType<MLFloat16>()}),
Gemm);
ONNX_OPERATOR_VERSIONED_KERNEL_EX(Gemm, kOnnxDomain, 11, 12, kXnnpackExecutionProvider,
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<float>()),
KernelDefBuilder().TypeConstraint("T", {DataTypeImpl::GetTensorType<float>(),
DataTypeImpl::GetTensorType<MLFloat16>()}),
Gemm);
ONNX_OPERATOR_KERNEL_EX(Gemm, kOnnxDomain, 13, kXnnpackExecutionProvider,
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<float>()),
KernelDefBuilder().TypeConstraint("T", {DataTypeImpl::GetTensorType<float>(),
DataTypeImpl::GetTensorType<MLFloat16>()}),
Gemm);
} // namespace xnnpack

2
onnxruntime/core/providers/xnnpack/math/gemm.h
View file

@ -41,6 +41,8 @@ class Gemm : protected GemmBase, public XnnpackKernel {
float alpha_;
float beta_;
OpComputeType op_compute_type_ = OpComputeType::op_compute_type_invalid;
};
} // namespace xnnpack

110
onnxruntime/core/providers/xnnpack/math/matmul.cc
View file

@ -3,6 +3,7 @@
#include "matmul.h"
#include "core/providers/cpu/math/matmul_helper.h"
#include "core/providers/xnnpack/xnnpack_init.h"
// Todo -
// 1. Integrate activation layers - Cliping & Relu
@ -34,7 +35,8 @@ bool MatMul::IsOnnxNodeSupported(const NodeUnit& node_unit, const GraphViewer& g
const auto* A_shape = A_arg.Shape();
const auto* B_shape = B_arg.Shape();
if (A_type->tensor_type().elem_type() != ONNX_NAMESPACE::TensorProto_DataType_FLOAT) {
if (A_type->tensor_type().elem_type() != ONNX_NAMESPACE::TensorProto_DataType_FLOAT &&
A_type->tensor_type().elem_type() != ONNX_NAMESPACE::TensorProto_DataType_FLOAT16) {
break;
}
@ -62,7 +64,18 @@ bool MatMul::IsOnnxNodeSupported(const NodeUnit& node_unit, const GraphViewer& g
return supported;
}
MatMul::MatMul(const OpKernelInfo& info) : XnnpackKernel(info, /*enable_caches*/ true) {}
MatMul::MatMul(const OpKernelInfo& info) : XnnpackKernel(info, /*enable_caches*/ true) {
const auto& node{Node()};
const auto& input_defs = node.InputDefs();
const NodeArg& X = *input_defs[0];
auto input_dtype = X.TypeAsProto()->tensor_type().elem_type();
op_type_str_ = DataTypeImpl::ToString(DataTypeImpl::TypeFromProto(*X.TypeAsProto()));
if (input_dtype == ONNX_NAMESPACE::TensorProto_DataType_FLOAT) {
op_type_ = OpComputeType::op_compute_type_fp32;
} else if (input_dtype == ONNX_NAMESPACE::TensorProto_DataType_FLOAT16) {
op_type_ = OpComputeType::op_compute_type_fp16;
}
}
Status MatMul::PrePack(const Tensor& tensor, int input_idx, AllocatorPtr alloc,
/*out*/ bool& is_packed,
@ -78,8 +91,7 @@ Status MatMul::PrePack(const Tensor& tensor, int input_idx, AllocatorPtr alloc,
is_packed = true;
uint32_t flags = XNN_FLAG_TRANSPOSE_WEIGHTS;
float output_min = -INFINITY;
float output_max = INFINITY;
xnn_status status = xnn_status::xnn_status_uninitialized;
struct xnn_operator* p = nullptr;
@ -88,27 +100,49 @@ Status MatMul::PrePack(const Tensor& tensor, int input_idx, AllocatorPtr alloc,
if (b_shape_.NumDimensions() == 1) {
shape_broadcast.push_back(1);
}
status = xnn_create_fully_connected_nc_f32(
shape_broadcast[0], // size_t input_channels,
shape_broadcast[1], // size_t output_channels,
shape_broadcast[0], // size_t input_stride,
shape_broadcast[1], // size_t output_stride,
tensor.Data<float>(), // const float* kernel,
nullptr, // const float* bias,
output_min,
output_max,
flags,
#ifdef XNN_CACHE_ENABLE
GetCodeCache(),
GetWeightsCache(),
xnn_code_cache_t code_cache = GetCodeCache();
xnn_weights_cache_t weight_cache = GetWeightsCache();
#else
nullptr,
nullptr,
xnn_code_cache_t code_cache = nullptr;
xnn_weights_cache_t weight_cache = nullptr;
#endif
&p);
float foutput_min = -INFINITY;
float foutput_max = INFINITY;
if (op_type_ == OpComputeType::op_compute_type_fp32) {
status = xnn_create_fully_connected_nc_f32(
shape_broadcast[0], // size_t input_channels,
shape_broadcast[1], // size_t output_channels,
shape_broadcast[0], // size_t input_stride,
shape_broadcast[1], // size_t output_stride,
tensor.Data<float>(), // const float* kernel,
nullptr, // const float* bias,
foutput_min,
foutput_max,
flags,
code_cache,
weight_cache,
&p);
} else if (op_type_ == OpComputeType::op_compute_type_fp16) {
status = xnn_create_fully_connected_nc_f16(
shape_broadcast[0], // size_t input_channels,
shape_broadcast[1], // size_t output_channels,
shape_broadcast[0], // size_t input_stride,
shape_broadcast[1], // size_t output_stride,
tensor.Data<MLFloat16>(), // const MLFloat16* kernel,
nullptr, // const MLFloat16* bias,
foutput_min,
foutput_max,
flags,
code_cache,
weight_cache,
&p);
}
if (status != xnn_status_success) {
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "xnn_create_fully_connected_nc_f32 returned ", status);
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "xnn_create_fully_connected_nc_", op_type_str_, " returned ", status);
}
op0_.reset(p);
@ -118,24 +152,35 @@ Status MatMul::PrePack(const Tensor& tensor, int input_idx, AllocatorPtr alloc,
Status MatMul::Compute(OpKernelContext* ctx) const {
const Tensor* a = ctx->Input<Tensor>(0);
pthreadpool_t threadpool = GetThreadPool();
MatMulComputeHelper helper;
ORT_RETURN_IF_ERROR(helper.Compute(a->Shape(), b_shape_));
Tensor* y = ctx->Output(0, helper.OutputShape());
if (y->Shape().Size() == 0)
return Status::OK();
auto* y_data = y->MutableData<float>();
xnn_status status = xnn_status_success;
xnn_status status = xnn_reshape_fully_connected_nc_f32(op0_.get(), a->Shape()[0], threadpool);
if (status != xnn_status_success) {
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "xnn_reshape_fully_connected_nc_f32 returned ", status);
pthreadpool_t threadpool = GetThreadPool();
if (op_type_ == OpComputeType::op_compute_type_fp32) {
status = xnn_reshape_fully_connected_nc_f32(op0_.get(), a->Shape()[0], threadpool);
} else if (op_type_ == OpComputeType::op_compute_type_fp16) {
status = xnn_reshape_fully_connected_nc_f16(op0_.get(), a->Shape()[0], threadpool);
}
status = xnn_setup_fully_connected_nc_f32(op0_.get(), a->Data<float>(), y_data);
if (status != xnn_status_success) {
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "xnn_setup_fully_connected_nc_f32 returned ", status);
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "xnn_reshape_fully_connected_nc_", op_type_str_, " returned ", status);
}
if (op_type_ == OpComputeType::op_compute_type_fp32) {
auto* y_data = y->MutableData<float>();
status = xnn_setup_fully_connected_nc_f32(op0_.get(), a->Data<float>(), y_data);
} else if (op_type_ == OpComputeType::op_compute_type_fp16) {
auto* y_data = y->MutableData<MLFloat16>();
status = xnn_setup_fully_connected_nc_f16(op0_.get(), a->Data<MLFloat16>(), y_data);
}
if (status != xnn_status_success) {
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "xnn_setup_fully_connected_nc_", op_type_str_, " returned ", status);
}
status = xnn_run_operator(op0_.get(), nullptr);
@ -146,15 +191,18 @@ Status MatMul::Compute(OpKernelContext* ctx) const {
}
ONNX_OPERATOR_VERSIONED_KERNEL_EX(MatMul, kOnnxDomain, 1, 8, kXnnpackExecutionProvider,
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<float>()),
KernelDefBuilder().TypeConstraint("T", {DataTypeImpl::GetTensorType<float>(),
DataTypeImpl::GetTensorType<MLFloat16>()}),
MatMul);
ONNX_OPERATOR_VERSIONED_KERNEL_EX(MatMul, kOnnxDomain, 9, 12, kXnnpackExecutionProvider,
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<float>()),
KernelDefBuilder().TypeConstraint("T", {DataTypeImpl::GetTensorType<float>(),
DataTypeImpl::GetTensorType<MLFloat16>()}),
MatMul);
ONNX_OPERATOR_KERNEL_EX(MatMul, kOnnxDomain, 13, kXnnpackExecutionProvider,
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<float>()),
KernelDefBuilder().TypeConstraint("T", {DataTypeImpl::GetTensorType<float>(),
DataTypeImpl::GetTensorType<MLFloat16>()}),
MatMul);
} // namespace xnnpack

3
onnxruntime/core/providers/xnnpack/math/matmul.h
View file

@ -31,6 +31,9 @@ class MatMul : public XnnpackKernel {
BufferUniquePtr packed_b_;
AllocatorPtr myAlloc;
OpComputeType op_type_ = OpComputeType::op_compute_type_invalid;
std::string op_type_str_ = "";
XnnpackOperator op0_ = nullptr;
};

36
onnxruntime/core/providers/xnnpack/math/softmax.cc
View file

@ -6,8 +6,9 @@
#include <utility>
#include "core/framework/op_kernel.h"
#include "core/providers/cpu/math/softmax_shared.h"
#include "core/optimizer/initializer.h"
#include "core/providers/cpu/math/softmax_shared.h"
#include "core/providers/xnnpack/xnnpack_init.h"
namespace onnxruntime {
namespace xnnpack {
@ -70,6 +71,7 @@ bool Softmax::IsOnnxNodeSupported(const NodeUnit& node_unit,
const auto* x_type = x_arg.TypeAsProto();
if (x_type == nullptr ||
(x_type->tensor_type().elem_type() != ONNX_NAMESPACE::TensorProto_DataType_FLOAT &&
x_type->tensor_type().elem_type() != ONNX_NAMESPACE::TensorProto_DataType_FLOAT16 &&
x_type->tensor_type().elem_type() != ONNX_NAMESPACE::TensorProto_DataType_UINT8)) {
break;
}
@ -120,14 +122,16 @@ Softmax::Softmax(const OpKernelInfo& info) : XnnpackKernel{info} {
ORT_ENFORCE(GetType(*input_defs[0], x_dtype));
if (x_dtype == ONNX_NAMESPACE::TensorProto_DataType_FLOAT) {
op_type_ = OpComputeType::op_compute_type_fp32;
} else if (x_dtype == ONNX_NAMESPACE::TensorProto_DataType_FLOAT16) {
op_type_ = OpComputeType::op_compute_type_fp16;
} else if (x_dtype == ONNX_NAMESPACE::TensorProto_DataType_UINT8) {
op_type_ = OpComputeType::op_compute_type_qu8;
} else {
auto stype = DataTypeImpl::ToString(DataTypeImpl::TypeFromProto(*input_defs[0]->TypeAsProto()));
ORT_THROW("unsupported Conv in softmax, we have FLOAT|UINT8, but got ", stype);
ORT_THROW("unsupported compute type in softmax, we have FLOAT|FLOAT16|UINT8, but got ", stype);
}
if (op_type_ == OpComputeType::op_compute_type_fp32) {
if (op_type_ == OpComputeType::op_compute_type_fp32 || op_type_ == OpComputeType::op_compute_type_fp16) {
opset_ = node.SinceVersion();
} else {
// Qlinearsoftmax's opset keep 1, we have to parse it by "opset"
@ -176,6 +180,10 @@ Softmax::Softmax(const OpKernelInfo& info) : XnnpackKernel{info} {
xstatus = xnn_create_softmax_nc_f32(
0, // flags,
&p);
} else if (op_type_ == OpComputeType::op_compute_type_fp16) {
xstatus = xnn_create_softmax_nc_f16(
0, // flags,
&p);
}
ORT_ENFORCE(xstatus == xnn_status_success, "xnn_create_softmax_nc_",
@ -200,8 +208,13 @@ Status Softmax::Compute(OpKernelContext* ctx) const {
// const size_t D = X_shape.SizeFromDimension(axis_); // the step D is 1
xnn_status status = xnn_status_invalid_state;
auto reshape_fn = op_type_ == OpComputeType::op_compute_type_qu8 ? xnn_reshape_softmax_nc_qu8
: xnn_reshape_softmax_nc_f32;
auto reshape_fn = xnn_reshape_softmax_nc_f32;
if (op_type_ == OpComputeType::op_compute_type_fp16) {
reshape_fn = xnn_reshape_softmax_nc_f16;
} else if (op_type_ == OpComputeType::op_compute_type_qu8) {
reshape_fn = xnn_reshape_softmax_nc_qu8;
}
status = reshape_fn(op0_.get(), channel_dim_, channel_dim_, channel_dim_, N, threadpool);
if (status != xnn_status_success) {
@ -211,8 +224,10 @@ Status Softmax::Compute(OpKernelContext* ctx) const {
if (op_type_ == OpComputeType::op_compute_type_qu8) {
status = xnn_setup_softmax_nc_qu8(op0_.get(), X->Data<uint8_t>(), Y->MutableData<uint8_t>());
} else {
} else if (op_type_ == op_compute_type_fp32) {
status = xnn_setup_softmax_nc_f32(op0_.get(), X->Data<float>(), Y->MutableData<float>());
} else if (op_type_ == op_compute_type_fp16) {
status = xnn_setup_softmax_nc_f16(op0_.get(), X->Data<MLFloat16>(), Y->MutableData<MLFloat16>());
}
if (status != xnn_status_success) {
@ -229,15 +244,18 @@ Status Softmax::Compute(OpKernelContext* ctx) const {
}
ONNX_OPERATOR_VERSIONED_KERNEL_EX(Softmax, kOnnxDomain, 1, 10, kXnnpackExecutionProvider,
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<float>()),
KernelDefBuilder().TypeConstraint("T", {DataTypeImpl::GetTensorType<float>(),
DataTypeImpl::GetTensorType<MLFloat16>()}),
Softmax);
ONNX_OPERATOR_VERSIONED_KERNEL_EX(Softmax, kOnnxDomain, 11, 12, kXnnpackExecutionProvider,
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<float>()),
KernelDefBuilder().TypeConstraint("T", {DataTypeImpl::GetTensorType<float>(),
DataTypeImpl::GetTensorType<MLFloat16>()}),
Softmax);
ONNX_OPERATOR_KERNEL_EX(Softmax, kOnnxDomain, 13, kXnnpackExecutionProvider,
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<float>()),
KernelDefBuilder().TypeConstraint("T", {DataTypeImpl::GetTensorType<float>(),
DataTypeImpl::GetTensorType<MLFloat16>()}),
Softmax);
ONNX_OPERATOR_KERNEL_EX(QLinearSoftmax, kDynamicDomainByCreate, 1, kXnnpackExecutionProvider,

44
onnxruntime/core/providers/xnnpack/nn/average_pool.cc
View file

@ -42,6 +42,12 @@ Status CreateXnnpackKernel(const PoolAttributes& pool_attrs,
pooling_height, pooling_width,
stride_height, stride_width,
foutput_min, foutput_max, flags, &p);
} else if (avgpool_type == OpComputeType::op_compute_type_fp16) {
status = xnn_create_average_pooling2d_nhwc_f16(input_padding_top, input_padding_right,
input_padding_bottom, input_padding_left,
pooling_height, pooling_width,
stride_height, stride_width,
foutput_min, foutput_max, flags, &p);
} else if (avgpool_type == OpComputeType::op_compute_type_qu8) {
const float output_scale = quant_param[1].first[0];
const uint8_t output_zero_point = quant_param[1].second;
@ -89,6 +95,11 @@ bool AveragePool::IsOnnxNodeSupported(const NodeUnit& node_unit,
// share the common checks here for fp32 and quant-op
const auto& inputs = node_unit.Inputs();
// use do {} while(false) so it's easier to set a breakpoint on the return
static const ComputeTypeSet compute_type_set = {
ONNX_NAMESPACE::TensorProto_DataType_FLOAT,
ONNX_NAMESPACE::TensorProto_DataType_FLOAT16,
ONNX_NAMESPACE::TensorProto_DataType_UINT8,
};
do {
if (node_unit.SinceVersion() < 7) {
break;
@ -105,8 +116,7 @@ bool AveragePool::IsOnnxNodeSupported(const NodeUnit& node_unit,
// we only support float and u8 currently
const auto* x_type = x_arg.TypeAsProto();
if (x_type == nullptr ||
(x_type->tensor_type().elem_type() != ONNX_NAMESPACE::TensorProto_DataType_FLOAT &&
x_type->tensor_type().elem_type() != ONNX_NAMESPACE::TensorProto_DataType_UINT8)) {
!IsComputeTypeSupported(x_type->tensor_type().elem_type(), compute_type_set)) {
break;
}
@ -197,13 +207,12 @@ AveragePool::AveragePool(const OpKernelInfo& info)
const auto& input_dtype = X_arg.TypeAsProto()->tensor_type().elem_type();
if (input_dtype == ONNX_NAMESPACE::TensorProto_DataType_FLOAT) {
avgpool_type_ = OpComputeType::op_compute_type_fp32;
} else if (input_dtype == ONNX_NAMESPACE::TensorProto_DataType_FLOAT16) {
avgpool_type_ = OpComputeType::op_compute_type_fp16;
} else if (input_dtype == ONNX_NAMESPACE::TensorProto_DataType_UINT8) {
// the order of input tensor, x,x_scale, x_zp, y_scale, y_zp
quant_param = ParseQuantParamForOp(info, input_dtype, 1);
avgpool_type_ = OpComputeType::op_compute_type_qu8;
} else {
auto stype = DataTypeImpl::ToString(DataTypeImpl::TypeFromProto(*X_arg.TypeAsProto()));
ORT_THROW("unsupported AveragePool in XnnpackEP, we have FLOAT|UINT8, but got ", stype);
}
struct xnn_operator* p;
auto ret = CreateXnnpackKernel(pool_attrs_, clip_min_max_, p,
@ -241,9 +250,12 @@ Status AveragePool::Compute(OpKernelContext* context) const {
std::unique_ptr<void, decltype(deallocator)> workspace(nullptr, deallocator);
auto reshape_fn = (avgpool_type_ == OpComputeType::op_compute_type_fp32)
? xnn_reshape_average_pooling2d_nhwc_f32
: xnn_reshape_average_pooling2d_nhwc_qu8;
auto reshape_fn = xnn_reshape_average_pooling2d_nhwc_f32;
if (avgpool_type_ == OpComputeType::op_compute_type_fp16) {
reshape_fn = xnn_reshape_average_pooling2d_nhwc_f16;
} else if (avgpool_type_ == OpComputeType::op_compute_type_qu8) {
reshape_fn = xnn_reshape_average_pooling2d_nhwc_qu8;
}
auto status = reshape_fn(op0_.get(), N, H, W, C, C, C,
&workspace_size, &workspace_alignment,
@ -260,7 +272,9 @@ Status AveragePool::Compute(OpKernelContext* context) const {
if (avgpool_type_ == OpComputeType::op_compute_type_fp32) {
status = xnn_setup_average_pooling2d_nhwc_f32(op0_.get(), workspace.get(),
X.Data<float>(), Y.MutableData<float>());
} else if (avgpool_type_ == OpComputeType::op_compute_type_fp16) {
status = xnn_setup_average_pooling2d_nhwc_f16(op0_.get(), workspace.get(),
X.Data<MLFloat16>(), Y.MutableData<MLFloat16>());
} else if (avgpool_type_ == OpComputeType::op_compute_type_qu8) {
status = xnn_setup_average_pooling2d_nhwc_qu8(op0_.get(), workspace.get(),
X.Data<uint8_t>(), Y.MutableData<uint8_t>());
@ -282,25 +296,29 @@ Status AveragePool::Compute(OpKernelContext* context) const {
ONNX_OPERATOR_VERSIONED_KERNEL_EX(
AveragePool, kMSInternalNHWCDomain, 7, 9,
kXnnpackExecutionProvider,
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<float>()),
KernelDefBuilder().TypeConstraint("T", {DataTypeImpl::GetTensorType<float>(),
DataTypeImpl::GetTensorType<MLFloat16>()}),
AveragePool);
ONNX_OPERATOR_VERSIONED_KERNEL_EX(
AveragePool, kMSInternalNHWCDomain, 10, 10,
kXnnpackExecutionProvider,
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<float>()),
KernelDefBuilder().TypeConstraint("T", {DataTypeImpl::GetTensorType<float>(),
DataTypeImpl::GetTensorType<MLFloat16>()}),
AveragePool);
ONNX_OPERATOR_VERSIONED_KERNEL_EX(
AveragePool, kMSInternalNHWCDomain, 11, 18,
kXnnpackExecutionProvider,
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<float>()),
KernelDefBuilder().TypeConstraint("T", {DataTypeImpl::GetTensorType<float>(),
DataTypeImpl::GetTensorType<MLFloat16>()}),
AveragePool);
ONNX_OPERATOR_KERNEL_EX(
AveragePool, kMSInternalNHWCDomain, 19,
kXnnpackExecutionProvider,
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<float>()),
KernelDefBuilder().TypeConstraint("T", {DataTypeImpl::GetTensorType<float>(),
DataTypeImpl::GetTensorType<MLFloat16>()}),
AveragePool);
ONNX_OPERATOR_KERNEL_EX(

19
onnxruntime/core/providers/xnnpack/nn/conv.cc
View file

@ -148,21 +148,18 @@ Status Conv::Compute(OpKernelContext* context) const {
}
ONNX_OPERATOR_VERSIONED_KERNEL_EX(Conv, kMSInternalNHWCDomain, 1, 10, kXnnpackExecutionProvider,
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<float>()),
KernelDefBuilder().TypeConstraint("T", {
DataTypeImpl::GetTensorType<float>(),
DataTypeImpl::GetTensorType<MLFloat16>(),
}),
Conv);
ONNX_OPERATOR_KERNEL_EX(Conv, kMSInternalNHWCDomain, 11, kXnnpackExecutionProvider,
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<float>()),
KernelDefBuilder().TypeConstraint("T", {
DataTypeImpl::GetTensorType<float>(),
DataTypeImpl::GetTensorType<MLFloat16>(),
}),
Conv);
#ifdef XNNPACK_FP16_SUPPORTED
ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX(Conv, kMSInternalNHWCDomain, 1, 10, MLFloat16, kXnnpackExecutionProvider,
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<MLFloat16>()),
Conv);
ONNX_OPERATOR_TYPED_KERNEL_EX(Conv, kMSInternalNHWCDomain, 11, MLFloat16, kXnnpackExecutionProvider,
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<MLFloat16>()),
Conv);
#endif
ONNX_OPERATOR_TYPED_KERNEL_EX(
QLinearConv,

6
onnxruntime/core/providers/xnnpack/nn/conv_base.cc
View file

@ -83,10 +83,6 @@ Status CreateXnnpackKernel(const ConvAttributes& conv_attrs,
&p);
} else if (conv_type == OpComputeType::op_compute_type_fp16) {
const auto* B_data = Bias ? Bias->Data<MLFloat16>() : nullptr;
// 65504 is the max value of float16
// https://en.wikipedia.org/wiki/Half-precision_floating-point_format
const float output_min = clip_min_max ? clip_min_max->first : -65504.0f;
const float output_max = clip_min_max ? clip_min_max->second : 65504.0f;
auto create_func = is_transpose ? xnn_create_deconvolution2d_nhwc_f16
: xnn_create_convolution2d_nhwc_f16;
status = create_func(
@ -99,7 +95,7 @@ Status CreateXnnpackKernel(const ConvAttributes& conv_attrs,
group_output_channels,
C, M, // input channel stride, output channel stride
Weight.Data<MLFloat16>(), B_data, // kernel, bias
output_min, output_max,
foutput_min, foutput_max,
flags,
code_cache, weights_cache,
&p);

19
onnxruntime/core/providers/xnnpack/nn/conv_transpose.cc
View file

@ -170,12 +170,14 @@ Status ConvTranspose::Compute(OpKernelContext* context) const {
ONNX_OPERATOR_VERSIONED_KERNEL_EX(ConvTranspose, kMSInternalNHWCDomain, 1, 10, kXnnpackExecutionProvider,
KernelDefBuilder().TypeConstraint(
"T", DataTypeImpl::GetTensorType<float>()),
"T", {DataTypeImpl::GetTensorType<float>(),
DataTypeImpl::GetTensorType<MLFloat16>()}),
ConvTranspose);
ONNX_OPERATOR_KERNEL_EX(ConvTranspose, kMSInternalNHWCDomain, 11, kXnnpackExecutionProvider,
KernelDefBuilder().TypeConstraint(
"T", DataTypeImpl::GetTensorType<float>()),
"T", {DataTypeImpl::GetTensorType<float>(),
DataTypeImpl::GetTensorType<MLFloat16>()}),
ConvTranspose);
ONNX_OPERATOR_KERNEL_EX(QLinearConvTranspose, kMSInternalNHWCDomain, 1, kXnnpackExecutionProvider,
@ -186,18 +188,5 @@ ONNX_OPERATOR_KERNEL_EX(QLinearConvTranspose, kMSInternalNHWCDomain, 1, kXnnpack
DataTypeImpl::GetTensorType<int8_t>()}),
ConvTranspose);
#ifdef XNNPACK_FP16_SUPPORTED
ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX(ConvTranspose, kMSInternalNHWCDomain, 1, 10, MLFloat16,
kXnnpackExecutionProvider,
KernelDefBuilder().TypeConstraint(
"T", DataTypeImpl::GetTensorType<MLFloat16>()),
ConvTranspose);
ONNX_OPERATOR_TYPED_KERNEL_EX(ConvTranspose, kMSInternalNHWCDomain, 11, MLFloat16, kXnnpackExecutionProvider,
KernelDefBuilder().TypeConstraint(
"T", DataTypeImpl::GetTensorType<MLFloat16>()),
ConvTranspose);
#endif
} // namespace xnnpack
} // namespace onnxruntime

26
onnxruntime/core/providers/xnnpack/nn/max_pool.cc
View file

@ -200,14 +200,12 @@ MaxPool::MaxPool(const OpKernelInfo& info)
output_min, output_max, flags, &p);
} else if (input_dtype == ONNX_NAMESPACE::TensorProto_DataType_FLOAT16) {
maxpool_type_ = OpComputeType::op_compute_type_fp16;
const float output_min = clip_min_max_ ? clip_min_max_->first : -65504.0f;
const float output_max = clip_min_max_ ? clip_min_max_->first : 65504.0f;
status = xnn_create_max_pooling2d_nhwc_f16(input_padding_top, input_padding_right,
input_padding_bottom, input_padding_left,
pooling_height, pooling_width,
stride_height, stride_width,
dilation_height, dilation_width,
output_min, output_max, flags, &p);
foutput_min, foutput_max, flags, &p);
} else {
auto stype = DataTypeImpl::ToString(DataTypeImpl::TypeFromProto(*X_arg.TypeAsProto()));
ORT_THROW("unsupported Conv in maxpool, we have FLOAT|UINT8|FLOAT16, but got ", stype);
@ -282,18 +280,21 @@ Status MaxPool::Compute(OpKernelContext* context) const {
ONNX_OPERATOR_VERSIONED_KERNEL_EX(MaxPool, kMSInternalNHWCDomain, 8, 9, kXnnpackExecutionProvider,
KernelDefBuilder().TypeConstraint("T", {DataTypeImpl::GetTensorType<float>(),
DataTypeImpl::GetTensorType<MLFloat16>(),
DataTypeImpl::GetTensorType<uint8_t>(),
DataTypeImpl::GetTensorType<int8_t>()}),
MaxPool);
ONNX_OPERATOR_VERSIONED_KERNEL_EX(MaxPool, kMSInternalNHWCDomain, 10, 10, kXnnpackExecutionProvider,
KernelDefBuilder().TypeConstraint("T", {DataTypeImpl::GetTensorType<float>(),
DataTypeImpl::GetTensorType<MLFloat16>(),
DataTypeImpl::GetTensorType<uint8_t>(),
DataTypeImpl::GetTensorType<int8_t>()}),
MaxPool);
ONNX_OPERATOR_VERSIONED_KERNEL_EX(MaxPool, kMSInternalNHWCDomain, 11, 11, kXnnpackExecutionProvider,
KernelDefBuilder().TypeConstraint("T", {DataTypeImpl::GetTensorType<float>(),
DataTypeImpl::GetTensorType<MLFloat16>(),
DataTypeImpl::GetTensorType<uint8_t>(),
DataTypeImpl::GetTensorType<int8_t>()}),
MaxPool);
@ -301,27 +302,10 @@ ONNX_OPERATOR_VERSIONED_KERNEL_EX(MaxPool, kMSInternalNHWCDomain, 11, 11, kXnnpa
ONNX_OPERATOR_KERNEL_EX(MaxPool, kMSInternalNHWCDomain, 12, kXnnpackExecutionProvider,
KernelDefBuilder()
.TypeConstraint("T", {DataTypeImpl::GetTensorType<float>(),
DataTypeImpl::GetTensorType<MLFloat16>(),
DataTypeImpl::GetTensorType<uint8_t>(),
DataTypeImpl::GetTensorType<int8_t>()}),
MaxPool);
#ifdef XNNPACK_FP16_SUPPORTED
ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX(MaxPool, kMSInternalNHWCDomain, 8, 9, MLFloat16, kXnnpackExecutionProvider,
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<MLFloat16>()),
MaxPool);
ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX(MaxPool, kMSInternalNHWCDomain, 10, 10, MLFloat16, kXnnpackExecutionProvider,
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<MLFloat16>()),
MaxPool);
ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX(MaxPool, kMSInternalNHWCDomain, 11, 11, MLFloat16, kXnnpackExecutionProvider,
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<MLFloat16>()),
MaxPool);
ONNX_OPERATOR_TYPED_KERNEL_EX(MaxPool, kMSInternalNHWCDomain, 12, MLFloat16, kXnnpackExecutionProvider,
KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType<MLFloat16>()),
MaxPool);
#endif
} // namespace xnnpack
} // namespace onnxruntime

22
onnxruntime/core/providers/xnnpack/tensor/resize.cc
View file

@ -29,9 +29,7 @@ bool Resize::IsOnnxNodeSupported(const NodeUnit& node_unit,
const auto& x_arg = inputs[0].node_arg;
const auto* x_type = x_arg.TypeAsProto();
if (x_type == nullptr || (x_type->tensor_type().elem_type() != ONNX_NAMESPACE::TensorProto_DataType_FLOAT &&
x_type->tensor_type().elem_type() != ONNX_NAMESPACE::TensorProto_DataType_UINT8 &&
x_type->tensor_type().elem_type() != ONNX_NAMESPACE::TensorProto_DataType_INT8)) {
if (x_type == nullptr || !IsComputeTypeSupported(x_type->tensor_type().elem_type())) {
break;
}
@ -181,6 +179,9 @@ Resize::Resize(const OpKernelInfo& info) : UpsampleBase(info), XnnpackKernel{inf
case ONNX_NAMESPACE::TensorProto_DataType_FLOAT:
op_type_ = OpComputeType::op_compute_type_fp32;
break;
case ONNX_NAMESPACE::TensorProto_DataType_FLOAT16:
op_type_ = OpComputeType::op_compute_type_fp16;
break;
case ONNX_NAMESPACE::TensorProto_DataType_UINT8:
op_type_ = OpComputeType::op_compute_type_qu8;
break;
@ -189,7 +190,7 @@ Resize::Resize(const OpKernelInfo& info) : UpsampleBase(info), XnnpackKernel{inf
break;
default:
auto stype = DataTypeImpl::ToString(DataTypeImpl::TypeFromProto(*input_defs[0]->TypeAsProto()));
ORT_THROW("unsupported op in Resize, we have FLOAT|UINT8|INT8, but get ", stype);
ORT_THROW("unsupported op in Resize, we have FLOAT|FLOAT16|UINT8|INT8, but get ", stype);
}
const auto* x_shape = input_defs[0]->Shape();
@ -229,6 +230,8 @@ Resize::Resize(const OpKernelInfo& info) : UpsampleBase(info), XnnpackKernel{inf
auto out_w = output_dims_[2];
if (op_type_ == OpComputeType::op_compute_type_fp32) {
xstatus = xnn_create_resize_bilinear2d_nhwc_f32(out_h, out_w, flags, &p);
} else if (op_type_ == OpComputeType::op_compute_type_fp16) {
xstatus = xnn_create_resize_bilinear2d_nhwc_f16(out_h, out_w, flags, &p);
} else if (op_type_ == OpComputeType::op_compute_type_qu8) {
xstatus = xnn_create_resize_bilinear2d_nhwc_u8(out_h, out_w, flags, &p);
} else {
@ -261,7 +264,9 @@ Status Resize::ComputeInternal(OpKernelContext* ctx, const Tensor* input,
std::unique_ptr<void, decltype(deallocator)> workspace(nullptr, deallocator);
auto reshape_fn = xnn_reshape_resize_bilinear2d_nhwc_f32;
if (op_type_ == OpComputeType::op_compute_type_qu8) {
if (op_type_ == OpComputeType::op_compute_type_fp16) {
reshape_fn = xnn_reshape_resize_bilinear2d_nhwc_f16;
} else if (op_type_ == OpComputeType::op_compute_type_qu8) {
reshape_fn = xnn_reshape_resize_bilinear2d_nhwc_u8;
} else if (op_type_ == OpComputeType::op_compute_type_qs8) {
reshape_fn = xnn_reshape_resize_bilinear2d_nhwc_s8;
@ -279,6 +284,9 @@ Status Resize::ComputeInternal(OpKernelContext* ctx, const Tensor* input,
if (op_type_ == OpComputeType::op_compute_type_fp32) {
status = xnn_setup_resize_bilinear2d_nhwc_f32(op0_.get(), workspace.get(), input->Data<float>(),
output->MutableData<float>());
} else if (op_type_ == OpComputeType::op_compute_type_fp16) {
status = xnn_setup_resize_bilinear2d_nhwc_f16(op0_.get(), workspace.get(), input->Data<MLFloat16>(),
output->MutableData<MLFloat16>());
} else if (op_type_ == OpComputeType::op_compute_type_qu8) {
status = xnn_setup_resize_bilinear2d_nhwc_u8(op0_.get(), workspace.get(), input->Data<uint8_t>(),
output->MutableData<uint8_t>());
@ -327,22 +335,26 @@ Status Resize::Compute(OpKernelContext* ctx) const {
ONNX_OPERATOR_VERSIONED_KERNEL_EX(Resize, kMSInternalNHWCDomain, 10, 10, kXnnpackExecutionProvider,
KernelDefBuilder().TypeConstraint("T", {DataTypeImpl::GetTensorType<float>(),
DataTypeImpl::GetTensorType<MLFloat16>(),
DataTypeImpl::GetTensorType<uint8_t>(),
DataTypeImpl::GetTensorType<int8_t>()}),
Resize);
ONNX_OPERATOR_VERSIONED_KERNEL_EX(Resize, kMSInternalNHWCDomain, 11, 12, kXnnpackExecutionProvider,
KernelDefBuilder().TypeConstraint("T1", {DataTypeImpl::GetTensorType<float>(),
DataTypeImpl::GetTensorType<MLFloat16>(),
DataTypeImpl::GetTensorType<uint8_t>(),
DataTypeImpl::GetTensorType<int8_t>()}),
Resize);
ONNX_OPERATOR_VERSIONED_KERNEL_EX(Resize, kMSInternalNHWCDomain, 13, 17, kXnnpackExecutionProvider,
KernelDefBuilder().TypeConstraint("T1", {DataTypeImpl::GetTensorType<float>(),
DataTypeImpl::GetTensorType<MLFloat16>(),
DataTypeImpl::GetTensorType<uint8_t>(),
DataTypeImpl::GetTensorType<int8_t>()}),
Resize);
ONNX_OPERATOR_VERSIONED_KERNEL_EX(Resize, kMSInternalNHWCDomain, 18, 18, kXnnpackExecutionProvider,
KernelDefBuilder().TypeConstraint("T1", {DataTypeImpl::GetTensorType<float>(),
DataTypeImpl::GetTensorType<MLFloat16>(),
DataTypeImpl::GetTensorType<uint8_t>(),
DataTypeImpl::GetTensorType<int8_t>()}),
Resize);

39
onnxruntime/core/providers/xnnpack/xnnpack_execution_provider.cc
View file

@ -31,10 +31,6 @@ KernelCreateInfo BuildKernelCreateInfo<void>() {
BuildKernelCreateInfo< \
ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, Domain, Start, End, Op)>
#define KERNEL_CREATE_INFO_VERSIONED_TYPED(Start, End, Type, Op, Domain) \
BuildKernelCreateInfo< \
ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, Domain, Start, End, Type, Op)>
#define KERNEL_CREATE_INFO(Start, Op, Domain) \
BuildKernelCreateInfo< \
ONNX_OPERATOR_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, Domain, Start, Op)>
@ -43,19 +39,6 @@ KernelCreateInfo BuildKernelCreateInfo<void>() {
BuildKernelCreateInfo< \
ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, Domain, Start, Type, Op)>
#ifdef XNNPACK_FP16_SUPPORTED
#define CLASS_ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME_FP16(provider, domain, startver, endver, name) \
class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, domain, \
startver, endver, MLFloat16, name)
#define CLASS_ONNX_OPERATOR_KERNEL_CLASS_NAME_FP16(provider, domain, startver, name) \
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, domain, startver, \
MLFloat16, name)
#else
#define CLASS_ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME_FP16(provider, domain, startver, endver, name)
#define CLASS_ONNX_OPERATOR_KERNEL_CLASS_NAME_FP16(provider, domain, startver, name)
#endif
// Layout sensitive operators in NHWC domain
class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 7, 9, AveragePool);
class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 10, 10, AveragePool);
@ -64,14 +47,9 @@ class ONNX_OPERATOR_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kMSInternalNHWC
class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 1, 10, Conv);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 11, Conv);
CLASS_ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME_FP16(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 1, 10, Conv);
CLASS_ONNX_OPERATOR_KERNEL_CLASS_NAME_FP16(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 11, Conv);
class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 1, 10, ConvTranspose);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 11, ConvTranspose);
CLASS_ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME_FP16(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 1, 10,
ConvTranspose);
CLASS_ONNX_OPERATOR_KERNEL_CLASS_NAME_FP16(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 11, ConvTranspose);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 10, uint8_t, QLinearConv);
class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 10, int8_t, QLinearConv);
@ -90,10 +68,6 @@ class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kMSIn
class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 10, 10, MaxPool);
class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 11, 11, MaxPool);
class ONNX_OPERATOR_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 12, MaxPool);
CLASS_ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME_FP16(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 8, 9, MaxPool);
CLASS_ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME_FP16(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 10, 10, MaxPool);
CLASS_ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME_FP16(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 11, 11, MaxPool);
CLASS_ONNX_OPERATOR_KERNEL_CLASS_NAME_FP16(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 12, MaxPool);
// ONNX operators
class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kOnnxDomain, 7, 8, Gemm);
@ -164,19 +138,6 @@ std::unique_ptr<KernelRegistry> RegisterKernels() {
KERNEL_CREATE_INFO_TYPED(10, int8_t, QLinearConv, kMSInternalNHWCDomain),
KERNEL_CREATE_INFO(1, QLinearSoftmax, kDynamicDomainByCreate),
#ifdef XNNPACK_FP16_SUPPORTED
KERNEL_CREATE_INFO_VERSIONED_TYPED(1, 10, MLFloat16, Conv, kMSInternalNHWCDomain),
KERNEL_CREATE_INFO_TYPED(11, MLFloat16, Conv, kMSInternalNHWCDomain),
KERNEL_CREATE_INFO_VERSIONED_TYPED(1, 10, MLFloat16, ConvTranspose, kMSInternalNHWCDomain),
KERNEL_CREATE_INFO_TYPED(11, MLFloat16, ConvTranspose, kMSInternalNHWCDomain),
KERNEL_CREATE_INFO_VERSIONED_TYPED(8, 9, MLFloat16, MaxPool, kMSInternalNHWCDomain),
KERNEL_CREATE_INFO_VERSIONED_TYPED(10, 10, MLFloat16, MaxPool, kMSInternalNHWCDomain),
KERNEL_CREATE_INFO_VERSIONED_TYPED(11, 11, MLFloat16, MaxPool, kMSInternalNHWCDomain),
KERNEL_CREATE_INFO_TYPED(12, MLFloat16, MaxPool, kMSInternalNHWCDomain),
#endif
};
for (auto& function_table_entry : function_table) {

2
onnxruntime/test/providers/cpu/math/gemm_test.cc
View file

@ -25,7 +25,7 @@ const constexpr auto run_with_tunable_op = &run_options;
} // namespace
// Only CUDA, ROCM and CoreML kernels have float 16 support
// Only CUDA, ROCM, CoreML and XNNPack kernels have float 16 support
TEST(GemmOpTest, GemmNoTrans_f16) {
#ifdef USE_CUDA
int min_cuda_architecture = 530;

67
onnxruntime/test/providers/cpu/math/matmul_test.cc
View file

@ -2,6 +2,7 @@
// Licensed under the MIT License.
#include "gtest/gtest.h"
#include "test/providers/provider_test_utils.h"
#include "test/providers/run_options_config_keys.h"
#include "test/common/dnnl_op_test_utils.h"
@ -144,6 +145,25 @@ std::vector<MatMulTestData<T>> GenerateTestCases() {
return test_cases;
}
template <>
std::vector<MatMulTestData<MLFloat16>> GenerateTestCases() {
std::vector<MatMulTestData<MLFloat16>> test_cases;
// test 2D expected_vals
std::vector<int64_t> expected_vals = {42, 48, 54, 114, 136, 158, 186, 224, 262};
std::vector<MLFloat16> expected_vals_fp16(expected_vals.size());
std::transform(expected_vals.begin(), expected_vals.end(), expected_vals_fp16.begin(),
[](int64_t num) { return MLFloat16(float(num)); });
test_cases.push_back(
{"test 2D MLfloat16",
{3, 4},
{4, 3},
{3, 3},
expected_vals_fp16});
return test_cases;
}
template <typename T>
void RunMatMulTest(int32_t opset_version, bool is_a_constant, bool is_b_constant) {
for (auto t : GenerateTestCases<T>()) {
@ -191,6 +211,32 @@ TEST(MathOpTest, MatMulFloatType) {
RunMatMulTest<float>(7, false, false);
}
// To Test XNNPACK, Matrix B must be constant
TEST(MathOpTest, MatMulFloatType_ConstantB) {
// TODO: Unskip when fixed #41968513
if (DefaultDmlExecutionProvider().get() != nullptr) {
GTEST_SKIP() << "Skipping because of the following error: Assertion failed: m_bufferTensorDesc.TotalTensorSizeInBytes >= ComputeByteSizeFromDimensions(nonBroadcastDimensions, dataType)";
}
RunMatMulTest<float>(7, false, true);
}
#if defined(USE_CUDA) || defined(USE_ROCM) || defined(COREML_ENABLE_MLPROGRAM) || defined(USE_XNNPACK)
TEST(MathOpTest, MatMulFloat16_ConstantB) {
#ifdef USE_CUDA
int min_cuda_architecture = 530;
if (!HasCudaEnvironment(min_cuda_architecture)) {
LOGS_DEFAULT(WARNING) << "Hardware NOT support FP16";
return;
}
#endif
// TODO: Unskip when fixed #41968513
if (DefaultDmlExecutionProvider().get() != nullptr) {
GTEST_SKIP() << "Skipping because of the following error: Assertion failed: m_bufferTensorDesc.TotalTensorSizeInBytes >= ComputeByteSizeFromDimensions(nonBroadcastDimensions, dataType)";
}
RunMatMulTest<MLFloat16>(7, false, true);
}
#endif
TEST(MathOpTest, MatMulDoubleType) {
RunMatMulTest<double>(7);
}
@ -246,7 +292,7 @@ TEST(MathOpTest, MatMulZeroKInt32Type) {
RunMatMulZeroKTest<int32_t>();
}
#if defined(USE_CUDA) || defined(USE_ROCM) || defined(COREML_ENABLE_MLPROGRAM)
#if defined(USE_CUDA) || defined(USE_ROCM) || defined(COREML_ENABLE_MLPROGRAM) || defined(USE_XNNPACK)
TEST(MathOpTest, MatMul_Float16) {
#ifdef USE_CUDA
int min_cuda_architecture = 530;
@ -255,8 +301,6 @@ TEST(MathOpTest, MatMul_Float16) {
return;
}
#endif
OpTester test("MatMul", 14);
std::vector<float> A{1.0f, 2.0f, 3.0f, 4.0f,
-1.0f, -2.0f, -3.0f, -4.0f};
std::vector<float> B(12, 1.0f);
@ -270,12 +314,17 @@ TEST(MathOpTest, MatMul_Float16) {
ConvertFloatToMLFloat16(B.data(), f_B.data(), 12);
ConvertFloatToMLFloat16(Y.data(), f_Y.data(), 6);
test.AddInput<MLFloat16>("A", {2, 4}, f_A);
test.AddInput<MLFloat16>("B", {4, 3}, f_B);
test.AddOutput<MLFloat16>("Y", {2, 3}, f_Y);
test.ConfigExcludeEps({kTensorrtExecutionProvider}) // TensorRT: fp16 is not supported
.Config(run_with_tunable_op)
.RunWithConfig();
for (int i = 0; i < 2; i++) {
// it needs Matrix B as constant to test XNNPack
bool b_is_constant = i == 0 ? false : true;
OpTester test("MatMul", 14);
test.AddInput<MLFloat16>("A", {2, 4}, f_A);
test.AddInput<MLFloat16>("B", {4, 3}, f_B, b_is_constant);
test.AddOutput<MLFloat16>("Y", {2, 3}, f_Y);
test.ConfigExcludeEps({kTensorrtExecutionProvider}) // TensorRT: fp16 is not supported
.Config(run_with_tunable_op)
.RunWithConfig();
}
}
#endif

2
onnxruntime/test/providers/cpu/math/softmax_test.cc
View file

@ -49,7 +49,7 @@ TEST(SoftmaxOperator, Simple) {
RunTest(x_vals, expected_vals, dimensions);
}
#if defined(USE_CUDA) || defined(USE_ROCM)
#if defined(USE_CUDA) || defined(USE_ROCM) || defined(USE_XNNPACK)
TEST(SoftmaxOperator, Simple_fp16) {
#ifdef USE_CUDA
int min_cuda_architecture = 530;

4
onnxruntime/test/providers/cpu/nn/conv_fp16_test.cc
View file

@ -2,10 +2,8 @@
// Licensed under the MIT License.
#include "core/mlas/inc/mlas.h"
#include "core/providers/xnnpack/xnnpack_init.h"
// XNNPACK_FP16_SUPPORTED scope is too big, so add USE_XNNPACK to avoid the FP16 tests enabled for other EPs
#if defined(MLAS_F16VEC_INTRINSICS_SUPPORTED) || defined(COREML_ENABLE_MLPROGRAM) || (defined(USE_XNNPACK) && defined(XNNPACK_FP16_SUPPORTED))
#if defined(MLAS_F16VEC_INTRINSICS_SUPPORTED) || defined(COREML_ENABLE_MLPROGRAM) || defined(USE_XNNPACK)
#include "gtest/gtest.h"
#include "test/providers/provider_test_utils.h"

2
onnxruntime/test/providers/cpu/nn/pool_fp16_op_test.cc
View file

@ -3,7 +3,7 @@
#include "core/mlas/inc/mlas.h"
#if defined(MLAS_F16VEC_INTRINSICS_SUPPORTED) || defined(COREML_ENABLE_MLPROGRAM) || defined(XNNPACK_FP16_SUPPORTED)
#if defined(MLAS_F16VEC_INTRINSICS_SUPPORTED) || defined(COREML_ENABLE_MLPROGRAM) || defined(USE_XNNPACK)
#include "core/providers/cpu/nn/pool.h"
#include "gtest/gtest.h"