From caa67439b5287445c40d9253ba17a62b8ee6254d Mon Sep 17 00:00:00 2001 From: Yi Zhang Date: Mon, 14 Oct 2024 17:41:59 +0800 Subject: [PATCH] Add more F16 kernels of XNNPack (#22381) ### Description 1. Add Gemm, MatMul, Softmax, AveragePool and Resize F16 kernels This PR has included all changes in #22378 [AB#51066](https://aiinfra.visualstudio.com/6a833879-cd9b-44a4-a9de-adc2d818f13c/_workitems/edit/51066) [AB#51026](https://aiinfra.visualstudio.com/6a833879-cd9b-44a4-a9de-adc2d818f13c/_workitems/edit/51026) 2. Matrix B must be const and martrix A and B dim_size shoule NOT bigger than 2 in XNNPack, so I added 2 tests in matmul_test.cc to make sure it's really tested. (that is, compute() must be called.) ### Motivation and Context --- onnxruntime/core/framework/kernel_registry.cc | 10 +- .../xnnpack/detail/node_support_checker.cc | 11 ++ .../core/providers/xnnpack/math/gemm.cc | 115 ++++++++++++------ .../core/providers/xnnpack/math/gemm.h | 2 + .../core/providers/xnnpack/math/matmul.cc | 110 ++++++++++++----- .../core/providers/xnnpack/math/matmul.h | 3 + .../core/providers/xnnpack/math/softmax.cc | 36 ++++-- .../core/providers/xnnpack/nn/average_pool.cc | 44 +++++-- onnxruntime/core/providers/xnnpack/nn/conv.cc | 19 ++- .../core/providers/xnnpack/nn/conv_base.cc | 6 +- .../providers/xnnpack/nn/conv_transpose.cc | 19 +-- .../core/providers/xnnpack/nn/max_pool.cc | 26 +--- .../core/providers/xnnpack/tensor/resize.cc | 22 +++- .../xnnpack/xnnpack_execution_provider.cc | 39 ------ .../test/providers/cpu/math/gemm_test.cc | 2 +- .../test/providers/cpu/math/matmul_test.cc | 67 ++++++++-- .../test/providers/cpu/math/softmax_test.cc | 2 +- .../test/providers/cpu/nn/conv_fp16_test.cc | 4 +- .../providers/cpu/nn/pool_fp16_op_test.cc | 2 +- 19 files changed, 335 insertions(+), 204 deletions(-) diff --git a/onnxruntime/core/framework/kernel_registry.cc b/onnxruntime/core/framework/kernel_registry.cc index d695e0e04c..cbbe0f86b8 100644 --- a/onnxruntime/core/framework/kernel_registry.cc +++ b/onnxruntime/core/framework/kernel_registry.cc @@ -6,6 +6,7 @@ #include #include #include +#include #include #include "core/framework/kernel_type_str_resolver.h" @@ -310,9 +311,12 @@ Status KernelRegistry::Register(KernelCreateInfo&& create_info) { for (auto i = range.first; i != range.second; ++i) { if (i->second.kernel_def && i->second.kernel_def->IsConflict(*create_info.kernel_def)) { - return Status(common::ONNXRUNTIME, common::FAIL, - "Failed to add kernel for " + key + - ": Conflicting with a registered kernel with op versions."); + int since_version = i->second.kernel_def->SinceVersion().first; + std::string since_version_str = std::to_string(since_version); + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, + "Failed to add kernel for ", key, + ": Conflicting with a registered kernel with op versions. the since version is: ", + since_version_str); } } diff --git a/onnxruntime/core/providers/xnnpack/detail/node_support_checker.cc b/onnxruntime/core/providers/xnnpack/detail/node_support_checker.cc index e2d71cda68..a0968dbc38 100644 --- a/onnxruntime/core/providers/xnnpack/detail/node_support_checker.cc +++ b/onnxruntime/core/providers/xnnpack/detail/node_support_checker.cc @@ -90,6 +90,17 @@ const NodeUnit* ClipReluChecker(const NodeUnit& node_unit, } // namespace bool NodeSupportChecker::IsNodeSupported(const NodeUnit& nodeunit) { +#ifndef XNNPACK_FP16_SUPPORTED + // check whether the hardware support XNNPack FP16 + // Note. In CI, ios pipeline on ADO doesn't support XNNPack FP16. Because ADO mac pool is still x64. + const auto& inputs = nodeunit.Inputs(); + const auto& x_arg = inputs[0].node_arg; + const auto* x_type = x_arg.TypeAsProto(); + if (x_type == nullptr || x_type->tensor_type().elem_type() == ONNX_NAMESPACE::TensorProto_DataType_FLOAT16) { + return false; + } +#endif + static std::unordered_map checkers{ {"Conv", Conv::IsOnnxNodeSupported}, {"ConvTranspose", ConvTranspose::IsOnnxNodeSupported}, diff --git a/onnxruntime/core/providers/xnnpack/math/gemm.cc b/onnxruntime/core/providers/xnnpack/math/gemm.cc index f7b736b0ff..35a06cb7eb 100644 --- a/onnxruntime/core/providers/xnnpack/math/gemm.cc +++ b/onnxruntime/core/providers/xnnpack/math/gemm.cc @@ -4,6 +4,7 @@ #include "gemm.h" #include "core/framework/transpose_helper.h" #include "core/providers/utils.h" +#include "core/providers/xnnpack/xnnpack_init.h" namespace onnxruntime { namespace xnnpack { @@ -37,7 +38,8 @@ bool Gemm::IsOnnxNodeSupported(const NodeUnit& node_unit, const GraphViewer& gra const auto* A_type = A_arg->TypeAsProto(); if (A_type == nullptr || - A_type->tensor_type().elem_type() != ONNX_NAMESPACE::TensorProto_DataType_FLOAT) { + (A_type->tensor_type().elem_type() != ONNX_NAMESPACE::TensorProto_DataType_FLOAT && + A_type->tensor_type().elem_type() != ONNX_NAMESPACE::TensorProto_DataType_FLOAT16)) { break; } @@ -74,19 +76,26 @@ bool Gemm::IsOnnxNodeSupported(const NodeUnit& node_unit, const GraphViewer& gra supported = true; } while (false); - return supported; } Gemm::Gemm(const OpKernelInfo& info) : GemmBase(info), XnnpackKernel(info, /*enable_caches*/ true) { - const auto& node{Node()}; - info.GetAttrOrDefault("alpha", &alpha_, 1.f); info.GetAttrOrDefault("beta", &beta_, 1.f); + const auto& node{Node()}; const auto& input_defs = node.InputDefs(); const auto* shapeA = input_defs[0]->Shape(); const auto* shapeB = input_defs[1]->Shape(); + + const NodeArg& X = *input_defs[0]; + auto input_dtype = X.TypeAsProto()->tensor_type().elem_type(); + if (input_dtype == ONNX_NAMESPACE::TensorProto_DataType_FLOAT) { + op_compute_type_ = OpComputeType::op_compute_type_fp32; + } else if (input_dtype == ONNX_NAMESPACE::TensorProto_DataType_FLOAT16) { + op_compute_type_ = OpComputeType::op_compute_type_fp16; + } + const NodeArg* C_arg = input_defs.size() == 2 ? nullptr : input_defs[2]; C_matrix_exists_ = C_arg && C_arg->Exists(); @@ -127,32 +136,49 @@ Status Gemm::PrePack(const Tensor& tensor, int input_idx, AllocatorPtr, // flags - 1 - for no transpose - 0 for transpose uint32_t flags = trans_B_ == CblasTrans ? 0 : XNN_FLAG_TRANSPOSE_WEIGHTS; - - float output_min = clip_min_max_ ? clip_min_max_->first : -INFINITY; - float output_max = clip_min_max_ ? clip_min_max_->second : INFINITY; - - const float* bias_Data = nullptr; - - if (C_matrix_exists_) { - bias_Data = tensor.Data(); - } - + auto code_cache = GetCodeCache(); + auto weights_cache = GetWeightsCache(); xnn_status status = xnn_status::xnn_status_uninitialized; struct xnn_operator* p = nullptr; - status = xnn_create_fully_connected_nc_f32( - trans_B_ == CblasNoTrans ? B_->Shape()[0] : B_->Shape()[1], // size_t input_channels, - trans_B_ == CblasNoTrans ? B_->Shape()[1] : B_->Shape()[0], // size_t output_channels, - trans_B_ == CblasNoTrans ? B_->Shape()[0] : B_->Shape()[1], // size_t input_stride, - trans_B_ == CblasNoTrans ? B_->Shape()[1] : B_->Shape()[0], // size_t output_stride, - B_->Data(), // const float* kernel, - bias_Data, // const float* bias, - output_min, output_max, - flags, - GetCodeCache(), GetWeightsCache(), - &p); + float foutput_min = clip_min_max_ ? clip_min_max_->first : -INFINITY; + float foutput_max = clip_min_max_ ? clip_min_max_->second : INFINITY; + if (op_compute_type_ == OpComputeType::op_compute_type_fp32) { + const float* bias_data = nullptr; + if (C_matrix_exists_) { + bias_data = tensor.Data(); + } + status = xnn_create_fully_connected_nc_f32( + trans_B_ == CblasNoTrans ? B_->Shape()[0] : B_->Shape()[1], // size_t input_channels, + trans_B_ == CblasNoTrans ? B_->Shape()[1] : B_->Shape()[0], // size_t output_channels, + trans_B_ == CblasNoTrans ? B_->Shape()[0] : B_->Shape()[1], // size_t input_stride, + trans_B_ == CblasNoTrans ? B_->Shape()[1] : B_->Shape()[0], // size_t output_stride, + B_->Data(), // const float* kernel, + bias_data, // const float* bias, + foutput_min, foutput_max, + flags, + code_cache, weights_cache, + &p); + } else if (op_compute_type_ == OpComputeType::op_compute_type_fp16) { + const MLFloat16* bias_data = nullptr; + if (C_matrix_exists_) { + bias_data = tensor.Data(); + } + status = xnn_create_fully_connected_nc_f16( + trans_B_ == CblasNoTrans ? B_->Shape()[0] : B_->Shape()[1], // size_t input_channels, + trans_B_ == CblasNoTrans ? B_->Shape()[1] : B_->Shape()[0], // size_t output_channels, + trans_B_ == CblasNoTrans ? B_->Shape()[0] : B_->Shape()[1], // size_t input_stride, + trans_B_ == CblasNoTrans ? B_->Shape()[1] : B_->Shape()[0], // size_t output_stride, + B_->Data(), // const MLFloat16* kernel, + bias_data, // const float* bias, + foutput_min, foutput_max, + flags, + code_cache, weights_cache, + &p); + } if (status != xnn_status_success) { - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "xnn_create_fully_connected_nc_f32 returned ", status); + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "xnn_create_fully_connected_nc_", + OpTypeToString(op_compute_type_), " returned ", status); } op0_.reset(p); @@ -169,19 +195,30 @@ Status Gemm::Compute(OpKernelContext* context) const { return Status::OK(); } - xnn_status status = xnn_reshape_fully_connected_nc_f32(op0_.get(), - // Number of rows to multiply - trans_A_ == CblasNoTrans ? M_ : K_, - threadpool); + auto reshape_func = xnn_reshape_fully_connected_nc_f32; + if (op_compute_type_ == OpComputeType::op_compute_type_fp16) { + reshape_func = xnn_reshape_fully_connected_nc_f16; + } + xnn_status status = reshape_func(op0_.get(), + // Number of rows to multiply + trans_A_ == CblasNoTrans ? M_ : K_, + threadpool); if (status != xnn_status_success) { - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "xnn_reshape_fully_connected_nc_f32 returned ", status); + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "xnn_reshape_fully_connected_nc_", + OpTypeToString(op_compute_type_), " returned ", status); } - status = xnn_setup_fully_connected_nc_f32(op0_.get(), A->Data(), Y->MutableData()); + status = xnn_status_invalid_state; + if (op_compute_type_ == op_compute_type_fp32) { + status = xnn_setup_fully_connected_nc_f32(op0_.get(), A->Data(), Y->MutableData()); + } else if (op_compute_type_ == OpComputeType::op_compute_type_fp16) { + status = xnn_setup_fully_connected_nc_f16(op0_.get(), A->Data(), Y->MutableData()); + } if (status != xnn_status_success) { - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "xnn_setup_fully_connected_nc_f32 returned ", status); + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "xnn_setup_fully_connected_nc_", + OpTypeToString(op_compute_type_), " returned ", status); } status = xnn_run_operator(op0_.get(), nullptr); @@ -193,19 +230,23 @@ Status Gemm::Compute(OpKernelContext* context) const { } ONNX_OPERATOR_VERSIONED_KERNEL_EX(Gemm, kOnnxDomain, 7, 8, kXnnpackExecutionProvider, - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + KernelDefBuilder().TypeConstraint("T", {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), Gemm); ONNX_OPERATOR_VERSIONED_KERNEL_EX(Gemm, kOnnxDomain, 9, 10, kXnnpackExecutionProvider, - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + KernelDefBuilder().TypeConstraint("T", {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), Gemm); ONNX_OPERATOR_VERSIONED_KERNEL_EX(Gemm, kOnnxDomain, 11, 12, kXnnpackExecutionProvider, - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + KernelDefBuilder().TypeConstraint("T", {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), Gemm); ONNX_OPERATOR_KERNEL_EX(Gemm, kOnnxDomain, 13, kXnnpackExecutionProvider, - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + KernelDefBuilder().TypeConstraint("T", {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), Gemm); } // namespace xnnpack diff --git a/onnxruntime/core/providers/xnnpack/math/gemm.h b/onnxruntime/core/providers/xnnpack/math/gemm.h index 6d11a8531c..954aab0698 100644 --- a/onnxruntime/core/providers/xnnpack/math/gemm.h +++ b/onnxruntime/core/providers/xnnpack/math/gemm.h @@ -41,6 +41,8 @@ class Gemm : protected GemmBase, public XnnpackKernel { float alpha_; float beta_; + + OpComputeType op_compute_type_ = OpComputeType::op_compute_type_invalid; }; } // namespace xnnpack diff --git a/onnxruntime/core/providers/xnnpack/math/matmul.cc b/onnxruntime/core/providers/xnnpack/math/matmul.cc index e90aa11c9d..44a6fb4ee8 100644 --- a/onnxruntime/core/providers/xnnpack/math/matmul.cc +++ b/onnxruntime/core/providers/xnnpack/math/matmul.cc @@ -3,6 +3,7 @@ #include "matmul.h" #include "core/providers/cpu/math/matmul_helper.h" +#include "core/providers/xnnpack/xnnpack_init.h" // Todo - // 1. Integrate activation layers - Cliping & Relu @@ -34,7 +35,8 @@ bool MatMul::IsOnnxNodeSupported(const NodeUnit& node_unit, const GraphViewer& g const auto* A_shape = A_arg.Shape(); const auto* B_shape = B_arg.Shape(); - if (A_type->tensor_type().elem_type() != ONNX_NAMESPACE::TensorProto_DataType_FLOAT) { + if (A_type->tensor_type().elem_type() != ONNX_NAMESPACE::TensorProto_DataType_FLOAT && + A_type->tensor_type().elem_type() != ONNX_NAMESPACE::TensorProto_DataType_FLOAT16) { break; } @@ -62,7 +64,18 @@ bool MatMul::IsOnnxNodeSupported(const NodeUnit& node_unit, const GraphViewer& g return supported; } -MatMul::MatMul(const OpKernelInfo& info) : XnnpackKernel(info, /*enable_caches*/ true) {} +MatMul::MatMul(const OpKernelInfo& info) : XnnpackKernel(info, /*enable_caches*/ true) { + const auto& node{Node()}; + const auto& input_defs = node.InputDefs(); + const NodeArg& X = *input_defs[0]; + auto input_dtype = X.TypeAsProto()->tensor_type().elem_type(); + op_type_str_ = DataTypeImpl::ToString(DataTypeImpl::TypeFromProto(*X.TypeAsProto())); + if (input_dtype == ONNX_NAMESPACE::TensorProto_DataType_FLOAT) { + op_type_ = OpComputeType::op_compute_type_fp32; + } else if (input_dtype == ONNX_NAMESPACE::TensorProto_DataType_FLOAT16) { + op_type_ = OpComputeType::op_compute_type_fp16; + } +} Status MatMul::PrePack(const Tensor& tensor, int input_idx, AllocatorPtr alloc, /*out*/ bool& is_packed, @@ -78,8 +91,7 @@ Status MatMul::PrePack(const Tensor& tensor, int input_idx, AllocatorPtr alloc, is_packed = true; uint32_t flags = XNN_FLAG_TRANSPOSE_WEIGHTS; - float output_min = -INFINITY; - float output_max = INFINITY; + xnn_status status = xnn_status::xnn_status_uninitialized; struct xnn_operator* p = nullptr; @@ -88,27 +100,49 @@ Status MatMul::PrePack(const Tensor& tensor, int input_idx, AllocatorPtr alloc, if (b_shape_.NumDimensions() == 1) { shape_broadcast.push_back(1); } - status = xnn_create_fully_connected_nc_f32( - shape_broadcast[0], // size_t input_channels, - shape_broadcast[1], // size_t output_channels, - shape_broadcast[0], // size_t input_stride, - shape_broadcast[1], // size_t output_stride, - tensor.Data(), // const float* kernel, - nullptr, // const float* bias, - output_min, - output_max, - flags, + #ifdef XNN_CACHE_ENABLE - GetCodeCache(), - GetWeightsCache(), + xnn_code_cache_t code_cache = GetCodeCache(); + xnn_weights_cache_t weight_cache = GetWeightsCache(); #else - nullptr, - nullptr, + xnn_code_cache_t code_cache = nullptr; + xnn_weights_cache_t weight_cache = nullptr; #endif - &p); + + float foutput_min = -INFINITY; + float foutput_max = INFINITY; + if (op_type_ == OpComputeType::op_compute_type_fp32) { + status = xnn_create_fully_connected_nc_f32( + shape_broadcast[0], // size_t input_channels, + shape_broadcast[1], // size_t output_channels, + shape_broadcast[0], // size_t input_stride, + shape_broadcast[1], // size_t output_stride, + tensor.Data(), // const float* kernel, + nullptr, // const float* bias, + foutput_min, + foutput_max, + flags, + code_cache, + weight_cache, + &p); + } else if (op_type_ == OpComputeType::op_compute_type_fp16) { + status = xnn_create_fully_connected_nc_f16( + shape_broadcast[0], // size_t input_channels, + shape_broadcast[1], // size_t output_channels, + shape_broadcast[0], // size_t input_stride, + shape_broadcast[1], // size_t output_stride, + tensor.Data(), // const MLFloat16* kernel, + nullptr, // const MLFloat16* bias, + foutput_min, + foutput_max, + flags, + code_cache, + weight_cache, + &p); + } if (status != xnn_status_success) { - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "xnn_create_fully_connected_nc_f32 returned ", status); + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "xnn_create_fully_connected_nc_", op_type_str_, " returned ", status); } op0_.reset(p); @@ -118,24 +152,35 @@ Status MatMul::PrePack(const Tensor& tensor, int input_idx, AllocatorPtr alloc, Status MatMul::Compute(OpKernelContext* ctx) const { const Tensor* a = ctx->Input(0); - pthreadpool_t threadpool = GetThreadPool(); MatMulComputeHelper helper; ORT_RETURN_IF_ERROR(helper.Compute(a->Shape(), b_shape_)); Tensor* y = ctx->Output(0, helper.OutputShape()); - if (y->Shape().Size() == 0) return Status::OK(); - auto* y_data = y->MutableData(); + xnn_status status = xnn_status_success; - xnn_status status = xnn_reshape_fully_connected_nc_f32(op0_.get(), a->Shape()[0], threadpool); - if (status != xnn_status_success) { - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "xnn_reshape_fully_connected_nc_f32 returned ", status); + pthreadpool_t threadpool = GetThreadPool(); + if (op_type_ == OpComputeType::op_compute_type_fp32) { + status = xnn_reshape_fully_connected_nc_f32(op0_.get(), a->Shape()[0], threadpool); + } else if (op_type_ == OpComputeType::op_compute_type_fp16) { + status = xnn_reshape_fully_connected_nc_f16(op0_.get(), a->Shape()[0], threadpool); } - status = xnn_setup_fully_connected_nc_f32(op0_.get(), a->Data(), y_data); if (status != xnn_status_success) { - return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "xnn_setup_fully_connected_nc_f32 returned ", status); + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "xnn_reshape_fully_connected_nc_", op_type_str_, " returned ", status); + } + + if (op_type_ == OpComputeType::op_compute_type_fp32) { + auto* y_data = y->MutableData(); + status = xnn_setup_fully_connected_nc_f32(op0_.get(), a->Data(), y_data); + } else if (op_type_ == OpComputeType::op_compute_type_fp16) { + auto* y_data = y->MutableData(); + status = xnn_setup_fully_connected_nc_f16(op0_.get(), a->Data(), y_data); + } + + if (status != xnn_status_success) { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "xnn_setup_fully_connected_nc_", op_type_str_, " returned ", status); } status = xnn_run_operator(op0_.get(), nullptr); @@ -146,15 +191,18 @@ Status MatMul::Compute(OpKernelContext* ctx) const { } ONNX_OPERATOR_VERSIONED_KERNEL_EX(MatMul, kOnnxDomain, 1, 8, kXnnpackExecutionProvider, - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + KernelDefBuilder().TypeConstraint("T", {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), MatMul); ONNX_OPERATOR_VERSIONED_KERNEL_EX(MatMul, kOnnxDomain, 9, 12, kXnnpackExecutionProvider, - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + KernelDefBuilder().TypeConstraint("T", {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), MatMul); ONNX_OPERATOR_KERNEL_EX(MatMul, kOnnxDomain, 13, kXnnpackExecutionProvider, - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + KernelDefBuilder().TypeConstraint("T", {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), MatMul); } // namespace xnnpack diff --git a/onnxruntime/core/providers/xnnpack/math/matmul.h b/onnxruntime/core/providers/xnnpack/math/matmul.h index b76e42c4d3..188cc73189 100644 --- a/onnxruntime/core/providers/xnnpack/math/matmul.h +++ b/onnxruntime/core/providers/xnnpack/math/matmul.h @@ -31,6 +31,9 @@ class MatMul : public XnnpackKernel { BufferUniquePtr packed_b_; AllocatorPtr myAlloc; + OpComputeType op_type_ = OpComputeType::op_compute_type_invalid; + std::string op_type_str_ = ""; + XnnpackOperator op0_ = nullptr; }; diff --git a/onnxruntime/core/providers/xnnpack/math/softmax.cc b/onnxruntime/core/providers/xnnpack/math/softmax.cc index 43e3ac193d..15e260889b 100644 --- a/onnxruntime/core/providers/xnnpack/math/softmax.cc +++ b/onnxruntime/core/providers/xnnpack/math/softmax.cc @@ -6,8 +6,9 @@ #include #include "core/framework/op_kernel.h" -#include "core/providers/cpu/math/softmax_shared.h" #include "core/optimizer/initializer.h" +#include "core/providers/cpu/math/softmax_shared.h" +#include "core/providers/xnnpack/xnnpack_init.h" namespace onnxruntime { namespace xnnpack { @@ -70,6 +71,7 @@ bool Softmax::IsOnnxNodeSupported(const NodeUnit& node_unit, const auto* x_type = x_arg.TypeAsProto(); if (x_type == nullptr || (x_type->tensor_type().elem_type() != ONNX_NAMESPACE::TensorProto_DataType_FLOAT && + x_type->tensor_type().elem_type() != ONNX_NAMESPACE::TensorProto_DataType_FLOAT16 && x_type->tensor_type().elem_type() != ONNX_NAMESPACE::TensorProto_DataType_UINT8)) { break; } @@ -120,14 +122,16 @@ Softmax::Softmax(const OpKernelInfo& info) : XnnpackKernel{info} { ORT_ENFORCE(GetType(*input_defs[0], x_dtype)); if (x_dtype == ONNX_NAMESPACE::TensorProto_DataType_FLOAT) { op_type_ = OpComputeType::op_compute_type_fp32; + } else if (x_dtype == ONNX_NAMESPACE::TensorProto_DataType_FLOAT16) { + op_type_ = OpComputeType::op_compute_type_fp16; } else if (x_dtype == ONNX_NAMESPACE::TensorProto_DataType_UINT8) { op_type_ = OpComputeType::op_compute_type_qu8; } else { auto stype = DataTypeImpl::ToString(DataTypeImpl::TypeFromProto(*input_defs[0]->TypeAsProto())); - ORT_THROW("unsupported Conv in softmax, we have FLOAT|UINT8, but got ", stype); + ORT_THROW("unsupported compute type in softmax, we have FLOAT|FLOAT16|UINT8, but got ", stype); } - if (op_type_ == OpComputeType::op_compute_type_fp32) { + if (op_type_ == OpComputeType::op_compute_type_fp32 || op_type_ == OpComputeType::op_compute_type_fp16) { opset_ = node.SinceVersion(); } else { // Qlinearsoftmax's opset keep 1, we have to parse it by "opset" @@ -176,6 +180,10 @@ Softmax::Softmax(const OpKernelInfo& info) : XnnpackKernel{info} { xstatus = xnn_create_softmax_nc_f32( 0, // flags, &p); + } else if (op_type_ == OpComputeType::op_compute_type_fp16) { + xstatus = xnn_create_softmax_nc_f16( + 0, // flags, + &p); } ORT_ENFORCE(xstatus == xnn_status_success, "xnn_create_softmax_nc_", @@ -200,8 +208,13 @@ Status Softmax::Compute(OpKernelContext* ctx) const { // const size_t D = X_shape.SizeFromDimension(axis_); // the step D is 1 xnn_status status = xnn_status_invalid_state; - auto reshape_fn = op_type_ == OpComputeType::op_compute_type_qu8 ? xnn_reshape_softmax_nc_qu8 - : xnn_reshape_softmax_nc_f32; + auto reshape_fn = xnn_reshape_softmax_nc_f32; + if (op_type_ == OpComputeType::op_compute_type_fp16) { + reshape_fn = xnn_reshape_softmax_nc_f16; + } else if (op_type_ == OpComputeType::op_compute_type_qu8) { + reshape_fn = xnn_reshape_softmax_nc_qu8; + } + status = reshape_fn(op0_.get(), channel_dim_, channel_dim_, channel_dim_, N, threadpool); if (status != xnn_status_success) { @@ -211,8 +224,10 @@ Status Softmax::Compute(OpKernelContext* ctx) const { if (op_type_ == OpComputeType::op_compute_type_qu8) { status = xnn_setup_softmax_nc_qu8(op0_.get(), X->Data(), Y->MutableData()); - } else { + } else if (op_type_ == op_compute_type_fp32) { status = xnn_setup_softmax_nc_f32(op0_.get(), X->Data(), Y->MutableData()); + } else if (op_type_ == op_compute_type_fp16) { + status = xnn_setup_softmax_nc_f16(op0_.get(), X->Data(), Y->MutableData()); } if (status != xnn_status_success) { @@ -229,15 +244,18 @@ Status Softmax::Compute(OpKernelContext* ctx) const { } ONNX_OPERATOR_VERSIONED_KERNEL_EX(Softmax, kOnnxDomain, 1, 10, kXnnpackExecutionProvider, - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + KernelDefBuilder().TypeConstraint("T", {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), Softmax); ONNX_OPERATOR_VERSIONED_KERNEL_EX(Softmax, kOnnxDomain, 11, 12, kXnnpackExecutionProvider, - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + KernelDefBuilder().TypeConstraint("T", {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), Softmax); ONNX_OPERATOR_KERNEL_EX(Softmax, kOnnxDomain, 13, kXnnpackExecutionProvider, - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + KernelDefBuilder().TypeConstraint("T", {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), Softmax); ONNX_OPERATOR_KERNEL_EX(QLinearSoftmax, kDynamicDomainByCreate, 1, kXnnpackExecutionProvider, diff --git a/onnxruntime/core/providers/xnnpack/nn/average_pool.cc b/onnxruntime/core/providers/xnnpack/nn/average_pool.cc index b31b5a9489..1c8ed556e9 100644 --- a/onnxruntime/core/providers/xnnpack/nn/average_pool.cc +++ b/onnxruntime/core/providers/xnnpack/nn/average_pool.cc @@ -42,6 +42,12 @@ Status CreateXnnpackKernel(const PoolAttributes& pool_attrs, pooling_height, pooling_width, stride_height, stride_width, foutput_min, foutput_max, flags, &p); + } else if (avgpool_type == OpComputeType::op_compute_type_fp16) { + status = xnn_create_average_pooling2d_nhwc_f16(input_padding_top, input_padding_right, + input_padding_bottom, input_padding_left, + pooling_height, pooling_width, + stride_height, stride_width, + foutput_min, foutput_max, flags, &p); } else if (avgpool_type == OpComputeType::op_compute_type_qu8) { const float output_scale = quant_param[1].first[0]; const uint8_t output_zero_point = quant_param[1].second; @@ -89,6 +95,11 @@ bool AveragePool::IsOnnxNodeSupported(const NodeUnit& node_unit, // share the common checks here for fp32 and quant-op const auto& inputs = node_unit.Inputs(); // use do {} while(false) so it's easier to set a breakpoint on the return + static const ComputeTypeSet compute_type_set = { + ONNX_NAMESPACE::TensorProto_DataType_FLOAT, + ONNX_NAMESPACE::TensorProto_DataType_FLOAT16, + ONNX_NAMESPACE::TensorProto_DataType_UINT8, + }; do { if (node_unit.SinceVersion() < 7) { break; @@ -105,8 +116,7 @@ bool AveragePool::IsOnnxNodeSupported(const NodeUnit& node_unit, // we only support float and u8 currently const auto* x_type = x_arg.TypeAsProto(); if (x_type == nullptr || - (x_type->tensor_type().elem_type() != ONNX_NAMESPACE::TensorProto_DataType_FLOAT && - x_type->tensor_type().elem_type() != ONNX_NAMESPACE::TensorProto_DataType_UINT8)) { + !IsComputeTypeSupported(x_type->tensor_type().elem_type(), compute_type_set)) { break; } @@ -197,13 +207,12 @@ AveragePool::AveragePool(const OpKernelInfo& info) const auto& input_dtype = X_arg.TypeAsProto()->tensor_type().elem_type(); if (input_dtype == ONNX_NAMESPACE::TensorProto_DataType_FLOAT) { avgpool_type_ = OpComputeType::op_compute_type_fp32; + } else if (input_dtype == ONNX_NAMESPACE::TensorProto_DataType_FLOAT16) { + avgpool_type_ = OpComputeType::op_compute_type_fp16; } else if (input_dtype == ONNX_NAMESPACE::TensorProto_DataType_UINT8) { // the order of input tensor, x,x_scale, x_zp, y_scale, y_zp quant_param = ParseQuantParamForOp(info, input_dtype, 1); avgpool_type_ = OpComputeType::op_compute_type_qu8; - } else { - auto stype = DataTypeImpl::ToString(DataTypeImpl::TypeFromProto(*X_arg.TypeAsProto())); - ORT_THROW("unsupported AveragePool in XnnpackEP, we have FLOAT|UINT8, but got ", stype); } struct xnn_operator* p; auto ret = CreateXnnpackKernel(pool_attrs_, clip_min_max_, p, @@ -241,9 +250,12 @@ Status AveragePool::Compute(OpKernelContext* context) const { std::unique_ptr workspace(nullptr, deallocator); - auto reshape_fn = (avgpool_type_ == OpComputeType::op_compute_type_fp32) - ? xnn_reshape_average_pooling2d_nhwc_f32 - : xnn_reshape_average_pooling2d_nhwc_qu8; + auto reshape_fn = xnn_reshape_average_pooling2d_nhwc_f32; + if (avgpool_type_ == OpComputeType::op_compute_type_fp16) { + reshape_fn = xnn_reshape_average_pooling2d_nhwc_f16; + } else if (avgpool_type_ == OpComputeType::op_compute_type_qu8) { + reshape_fn = xnn_reshape_average_pooling2d_nhwc_qu8; + } auto status = reshape_fn(op0_.get(), N, H, W, C, C, C, &workspace_size, &workspace_alignment, @@ -260,7 +272,9 @@ Status AveragePool::Compute(OpKernelContext* context) const { if (avgpool_type_ == OpComputeType::op_compute_type_fp32) { status = xnn_setup_average_pooling2d_nhwc_f32(op0_.get(), workspace.get(), X.Data(), Y.MutableData()); - + } else if (avgpool_type_ == OpComputeType::op_compute_type_fp16) { + status = xnn_setup_average_pooling2d_nhwc_f16(op0_.get(), workspace.get(), + X.Data(), Y.MutableData()); } else if (avgpool_type_ == OpComputeType::op_compute_type_qu8) { status = xnn_setup_average_pooling2d_nhwc_qu8(op0_.get(), workspace.get(), X.Data(), Y.MutableData()); @@ -282,25 +296,29 @@ Status AveragePool::Compute(OpKernelContext* context) const { ONNX_OPERATOR_VERSIONED_KERNEL_EX( AveragePool, kMSInternalNHWCDomain, 7, 9, kXnnpackExecutionProvider, - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + KernelDefBuilder().TypeConstraint("T", {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), AveragePool); ONNX_OPERATOR_VERSIONED_KERNEL_EX( AveragePool, kMSInternalNHWCDomain, 10, 10, kXnnpackExecutionProvider, - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + KernelDefBuilder().TypeConstraint("T", {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), AveragePool); ONNX_OPERATOR_VERSIONED_KERNEL_EX( AveragePool, kMSInternalNHWCDomain, 11, 18, kXnnpackExecutionProvider, - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + KernelDefBuilder().TypeConstraint("T", {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), AveragePool); ONNX_OPERATOR_KERNEL_EX( AveragePool, kMSInternalNHWCDomain, 19, kXnnpackExecutionProvider, - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + KernelDefBuilder().TypeConstraint("T", {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), AveragePool); ONNX_OPERATOR_KERNEL_EX( diff --git a/onnxruntime/core/providers/xnnpack/nn/conv.cc b/onnxruntime/core/providers/xnnpack/nn/conv.cc index 6e404c6259..4e6b308e28 100644 --- a/onnxruntime/core/providers/xnnpack/nn/conv.cc +++ b/onnxruntime/core/providers/xnnpack/nn/conv.cc @@ -148,21 +148,18 @@ Status Conv::Compute(OpKernelContext* context) const { } ONNX_OPERATOR_VERSIONED_KERNEL_EX(Conv, kMSInternalNHWCDomain, 1, 10, kXnnpackExecutionProvider, - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + KernelDefBuilder().TypeConstraint("T", { + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + }), Conv); ONNX_OPERATOR_KERNEL_EX(Conv, kMSInternalNHWCDomain, 11, kXnnpackExecutionProvider, - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), + KernelDefBuilder().TypeConstraint("T", { + DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), + }), Conv); -#ifdef XNNPACK_FP16_SUPPORTED -ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX(Conv, kMSInternalNHWCDomain, 1, 10, MLFloat16, kXnnpackExecutionProvider, - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), - Conv); - -ONNX_OPERATOR_TYPED_KERNEL_EX(Conv, kMSInternalNHWCDomain, 11, MLFloat16, kXnnpackExecutionProvider, - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), - Conv); -#endif ONNX_OPERATOR_TYPED_KERNEL_EX( QLinearConv, diff --git a/onnxruntime/core/providers/xnnpack/nn/conv_base.cc b/onnxruntime/core/providers/xnnpack/nn/conv_base.cc index d6b9a541fb..e0723c0e76 100644 --- a/onnxruntime/core/providers/xnnpack/nn/conv_base.cc +++ b/onnxruntime/core/providers/xnnpack/nn/conv_base.cc @@ -83,10 +83,6 @@ Status CreateXnnpackKernel(const ConvAttributes& conv_attrs, &p); } else if (conv_type == OpComputeType::op_compute_type_fp16) { const auto* B_data = Bias ? Bias->Data() : nullptr; - // 65504 is the max value of float16 - // https://en.wikipedia.org/wiki/Half-precision_floating-point_format - const float output_min = clip_min_max ? clip_min_max->first : -65504.0f; - const float output_max = clip_min_max ? clip_min_max->second : 65504.0f; auto create_func = is_transpose ? xnn_create_deconvolution2d_nhwc_f16 : xnn_create_convolution2d_nhwc_f16; status = create_func( @@ -99,7 +95,7 @@ Status CreateXnnpackKernel(const ConvAttributes& conv_attrs, group_output_channels, C, M, // input channel stride, output channel stride Weight.Data(), B_data, // kernel, bias - output_min, output_max, + foutput_min, foutput_max, flags, code_cache, weights_cache, &p); diff --git a/onnxruntime/core/providers/xnnpack/nn/conv_transpose.cc b/onnxruntime/core/providers/xnnpack/nn/conv_transpose.cc index b399311cd8..b6930a5fc9 100644 --- a/onnxruntime/core/providers/xnnpack/nn/conv_transpose.cc +++ b/onnxruntime/core/providers/xnnpack/nn/conv_transpose.cc @@ -170,12 +170,14 @@ Status ConvTranspose::Compute(OpKernelContext* context) const { ONNX_OPERATOR_VERSIONED_KERNEL_EX(ConvTranspose, kMSInternalNHWCDomain, 1, 10, kXnnpackExecutionProvider, KernelDefBuilder().TypeConstraint( - "T", DataTypeImpl::GetTensorType()), + "T", {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), ConvTranspose); ONNX_OPERATOR_KERNEL_EX(ConvTranspose, kMSInternalNHWCDomain, 11, kXnnpackExecutionProvider, KernelDefBuilder().TypeConstraint( - "T", DataTypeImpl::GetTensorType()), + "T", {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType()}), ConvTranspose); ONNX_OPERATOR_KERNEL_EX(QLinearConvTranspose, kMSInternalNHWCDomain, 1, kXnnpackExecutionProvider, @@ -186,18 +188,5 @@ ONNX_OPERATOR_KERNEL_EX(QLinearConvTranspose, kMSInternalNHWCDomain, 1, kXnnpack DataTypeImpl::GetTensorType()}), ConvTranspose); -#ifdef XNNPACK_FP16_SUPPORTED -ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX(ConvTranspose, kMSInternalNHWCDomain, 1, 10, MLFloat16, - kXnnpackExecutionProvider, - KernelDefBuilder().TypeConstraint( - "T", DataTypeImpl::GetTensorType()), - ConvTranspose); - -ONNX_OPERATOR_TYPED_KERNEL_EX(ConvTranspose, kMSInternalNHWCDomain, 11, MLFloat16, kXnnpackExecutionProvider, - KernelDefBuilder().TypeConstraint( - "T", DataTypeImpl::GetTensorType()), - ConvTranspose); - -#endif } // namespace xnnpack } // namespace onnxruntime diff --git a/onnxruntime/core/providers/xnnpack/nn/max_pool.cc b/onnxruntime/core/providers/xnnpack/nn/max_pool.cc index 0d1d9ee179..6742e51e55 100644 --- a/onnxruntime/core/providers/xnnpack/nn/max_pool.cc +++ b/onnxruntime/core/providers/xnnpack/nn/max_pool.cc @@ -200,14 +200,12 @@ MaxPool::MaxPool(const OpKernelInfo& info) output_min, output_max, flags, &p); } else if (input_dtype == ONNX_NAMESPACE::TensorProto_DataType_FLOAT16) { maxpool_type_ = OpComputeType::op_compute_type_fp16; - const float output_min = clip_min_max_ ? clip_min_max_->first : -65504.0f; - const float output_max = clip_min_max_ ? clip_min_max_->first : 65504.0f; status = xnn_create_max_pooling2d_nhwc_f16(input_padding_top, input_padding_right, input_padding_bottom, input_padding_left, pooling_height, pooling_width, stride_height, stride_width, dilation_height, dilation_width, - output_min, output_max, flags, &p); + foutput_min, foutput_max, flags, &p); } else { auto stype = DataTypeImpl::ToString(DataTypeImpl::TypeFromProto(*X_arg.TypeAsProto())); ORT_THROW("unsupported Conv in maxpool, we have FLOAT|UINT8|FLOAT16, but got ", stype); @@ -282,18 +280,21 @@ Status MaxPool::Compute(OpKernelContext* context) const { ONNX_OPERATOR_VERSIONED_KERNEL_EX(MaxPool, kMSInternalNHWCDomain, 8, 9, kXnnpackExecutionProvider, KernelDefBuilder().TypeConstraint("T", {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType()}), MaxPool); ONNX_OPERATOR_VERSIONED_KERNEL_EX(MaxPool, kMSInternalNHWCDomain, 10, 10, kXnnpackExecutionProvider, KernelDefBuilder().TypeConstraint("T", {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType()}), MaxPool); ONNX_OPERATOR_VERSIONED_KERNEL_EX(MaxPool, kMSInternalNHWCDomain, 11, 11, kXnnpackExecutionProvider, KernelDefBuilder().TypeConstraint("T", {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType()}), MaxPool); @@ -301,27 +302,10 @@ ONNX_OPERATOR_VERSIONED_KERNEL_EX(MaxPool, kMSInternalNHWCDomain, 11, 11, kXnnpa ONNX_OPERATOR_KERNEL_EX(MaxPool, kMSInternalNHWCDomain, 12, kXnnpackExecutionProvider, KernelDefBuilder() .TypeConstraint("T", {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType()}), MaxPool); -#ifdef XNNPACK_FP16_SUPPORTED -ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX(MaxPool, kMSInternalNHWCDomain, 8, 9, MLFloat16, kXnnpackExecutionProvider, - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), - MaxPool); - -ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX(MaxPool, kMSInternalNHWCDomain, 10, 10, MLFloat16, kXnnpackExecutionProvider, - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), - MaxPool); - -ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_EX(MaxPool, kMSInternalNHWCDomain, 11, 11, MLFloat16, kXnnpackExecutionProvider, - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), - MaxPool); - -ONNX_OPERATOR_TYPED_KERNEL_EX(MaxPool, kMSInternalNHWCDomain, 12, MLFloat16, kXnnpackExecutionProvider, - KernelDefBuilder().TypeConstraint("T", DataTypeImpl::GetTensorType()), - MaxPool); -#endif - } // namespace xnnpack } // namespace onnxruntime diff --git a/onnxruntime/core/providers/xnnpack/tensor/resize.cc b/onnxruntime/core/providers/xnnpack/tensor/resize.cc index db5648d5d6..45c292a36e 100644 --- a/onnxruntime/core/providers/xnnpack/tensor/resize.cc +++ b/onnxruntime/core/providers/xnnpack/tensor/resize.cc @@ -29,9 +29,7 @@ bool Resize::IsOnnxNodeSupported(const NodeUnit& node_unit, const auto& x_arg = inputs[0].node_arg; const auto* x_type = x_arg.TypeAsProto(); - if (x_type == nullptr || (x_type->tensor_type().elem_type() != ONNX_NAMESPACE::TensorProto_DataType_FLOAT && - x_type->tensor_type().elem_type() != ONNX_NAMESPACE::TensorProto_DataType_UINT8 && - x_type->tensor_type().elem_type() != ONNX_NAMESPACE::TensorProto_DataType_INT8)) { + if (x_type == nullptr || !IsComputeTypeSupported(x_type->tensor_type().elem_type())) { break; } @@ -181,6 +179,9 @@ Resize::Resize(const OpKernelInfo& info) : UpsampleBase(info), XnnpackKernel{inf case ONNX_NAMESPACE::TensorProto_DataType_FLOAT: op_type_ = OpComputeType::op_compute_type_fp32; break; + case ONNX_NAMESPACE::TensorProto_DataType_FLOAT16: + op_type_ = OpComputeType::op_compute_type_fp16; + break; case ONNX_NAMESPACE::TensorProto_DataType_UINT8: op_type_ = OpComputeType::op_compute_type_qu8; break; @@ -189,7 +190,7 @@ Resize::Resize(const OpKernelInfo& info) : UpsampleBase(info), XnnpackKernel{inf break; default: auto stype = DataTypeImpl::ToString(DataTypeImpl::TypeFromProto(*input_defs[0]->TypeAsProto())); - ORT_THROW("unsupported op in Resize, we have FLOAT|UINT8|INT8, but get ", stype); + ORT_THROW("unsupported op in Resize, we have FLOAT|FLOAT16|UINT8|INT8, but get ", stype); } const auto* x_shape = input_defs[0]->Shape(); @@ -229,6 +230,8 @@ Resize::Resize(const OpKernelInfo& info) : UpsampleBase(info), XnnpackKernel{inf auto out_w = output_dims_[2]; if (op_type_ == OpComputeType::op_compute_type_fp32) { xstatus = xnn_create_resize_bilinear2d_nhwc_f32(out_h, out_w, flags, &p); + } else if (op_type_ == OpComputeType::op_compute_type_fp16) { + xstatus = xnn_create_resize_bilinear2d_nhwc_f16(out_h, out_w, flags, &p); } else if (op_type_ == OpComputeType::op_compute_type_qu8) { xstatus = xnn_create_resize_bilinear2d_nhwc_u8(out_h, out_w, flags, &p); } else { @@ -261,7 +264,9 @@ Status Resize::ComputeInternal(OpKernelContext* ctx, const Tensor* input, std::unique_ptr workspace(nullptr, deallocator); auto reshape_fn = xnn_reshape_resize_bilinear2d_nhwc_f32; - if (op_type_ == OpComputeType::op_compute_type_qu8) { + if (op_type_ == OpComputeType::op_compute_type_fp16) { + reshape_fn = xnn_reshape_resize_bilinear2d_nhwc_f16; + } else if (op_type_ == OpComputeType::op_compute_type_qu8) { reshape_fn = xnn_reshape_resize_bilinear2d_nhwc_u8; } else if (op_type_ == OpComputeType::op_compute_type_qs8) { reshape_fn = xnn_reshape_resize_bilinear2d_nhwc_s8; @@ -279,6 +284,9 @@ Status Resize::ComputeInternal(OpKernelContext* ctx, const Tensor* input, if (op_type_ == OpComputeType::op_compute_type_fp32) { status = xnn_setup_resize_bilinear2d_nhwc_f32(op0_.get(), workspace.get(), input->Data(), output->MutableData()); + } else if (op_type_ == OpComputeType::op_compute_type_fp16) { + status = xnn_setup_resize_bilinear2d_nhwc_f16(op0_.get(), workspace.get(), input->Data(), + output->MutableData()); } else if (op_type_ == OpComputeType::op_compute_type_qu8) { status = xnn_setup_resize_bilinear2d_nhwc_u8(op0_.get(), workspace.get(), input->Data(), output->MutableData()); @@ -327,22 +335,26 @@ Status Resize::Compute(OpKernelContext* ctx) const { ONNX_OPERATOR_VERSIONED_KERNEL_EX(Resize, kMSInternalNHWCDomain, 10, 10, kXnnpackExecutionProvider, KernelDefBuilder().TypeConstraint("T", {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType()}), Resize); ONNX_OPERATOR_VERSIONED_KERNEL_EX(Resize, kMSInternalNHWCDomain, 11, 12, kXnnpackExecutionProvider, KernelDefBuilder().TypeConstraint("T1", {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType()}), Resize); ONNX_OPERATOR_VERSIONED_KERNEL_EX(Resize, kMSInternalNHWCDomain, 13, 17, kXnnpackExecutionProvider, KernelDefBuilder().TypeConstraint("T1", {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType()}), Resize); ONNX_OPERATOR_VERSIONED_KERNEL_EX(Resize, kMSInternalNHWCDomain, 18, 18, kXnnpackExecutionProvider, KernelDefBuilder().TypeConstraint("T1", {DataTypeImpl::GetTensorType(), + DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType(), DataTypeImpl::GetTensorType()}), Resize); diff --git a/onnxruntime/core/providers/xnnpack/xnnpack_execution_provider.cc b/onnxruntime/core/providers/xnnpack/xnnpack_execution_provider.cc index 4515a31eb0..12e567e708 100644 --- a/onnxruntime/core/providers/xnnpack/xnnpack_execution_provider.cc +++ b/onnxruntime/core/providers/xnnpack/xnnpack_execution_provider.cc @@ -31,10 +31,6 @@ KernelCreateInfo BuildKernelCreateInfo() { BuildKernelCreateInfo< \ ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, Domain, Start, End, Op)> -#define KERNEL_CREATE_INFO_VERSIONED_TYPED(Start, End, Type, Op, Domain) \ - BuildKernelCreateInfo< \ - ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, Domain, Start, End, Type, Op)> - #define KERNEL_CREATE_INFO(Start, Op, Domain) \ BuildKernelCreateInfo< \ ONNX_OPERATOR_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, Domain, Start, Op)> @@ -43,19 +39,6 @@ KernelCreateInfo BuildKernelCreateInfo() { BuildKernelCreateInfo< \ ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, Domain, Start, Type, Op)> -#ifdef XNNPACK_FP16_SUPPORTED -#define CLASS_ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME_FP16(provider, domain, startver, endver, name) \ - class ONNX_OPERATOR_VERSIONED_TYPED_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, domain, \ - startver, endver, MLFloat16, name) - -#define CLASS_ONNX_OPERATOR_KERNEL_CLASS_NAME_FP16(provider, domain, startver, name) \ - class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, domain, startver, \ - MLFloat16, name) -#else -#define CLASS_ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME_FP16(provider, domain, startver, endver, name) -#define CLASS_ONNX_OPERATOR_KERNEL_CLASS_NAME_FP16(provider, domain, startver, name) -#endif - // Layout sensitive operators in NHWC domain class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 7, 9, AveragePool); class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 10, 10, AveragePool); @@ -64,14 +47,9 @@ class ONNX_OPERATOR_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kMSInternalNHWC class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 1, 10, Conv); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 11, Conv); -CLASS_ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME_FP16(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 1, 10, Conv); -CLASS_ONNX_OPERATOR_KERNEL_CLASS_NAME_FP16(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 11, Conv); class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 1, 10, ConvTranspose); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 11, ConvTranspose); -CLASS_ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME_FP16(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 1, 10, - ConvTranspose); -CLASS_ONNX_OPERATOR_KERNEL_CLASS_NAME_FP16(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 11, ConvTranspose); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 10, uint8_t, QLinearConv); class ONNX_OPERATOR_TYPED_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 10, int8_t, QLinearConv); @@ -90,10 +68,6 @@ class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kMSIn class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 10, 10, MaxPool); class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 11, 11, MaxPool); class ONNX_OPERATOR_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 12, MaxPool); -CLASS_ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME_FP16(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 8, 9, MaxPool); -CLASS_ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME_FP16(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 10, 10, MaxPool); -CLASS_ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME_FP16(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 11, 11, MaxPool); -CLASS_ONNX_OPERATOR_KERNEL_CLASS_NAME_FP16(kXnnpackExecutionProvider, kMSInternalNHWCDomain, 12, MaxPool); // ONNX operators class ONNX_OPERATOR_VERSIONED_KERNEL_CLASS_NAME(kXnnpackExecutionProvider, kOnnxDomain, 7, 8, Gemm); @@ -164,19 +138,6 @@ std::unique_ptr RegisterKernels() { KERNEL_CREATE_INFO_TYPED(10, int8_t, QLinearConv, kMSInternalNHWCDomain), KERNEL_CREATE_INFO(1, QLinearSoftmax, kDynamicDomainByCreate), - -#ifdef XNNPACK_FP16_SUPPORTED - KERNEL_CREATE_INFO_VERSIONED_TYPED(1, 10, MLFloat16, Conv, kMSInternalNHWCDomain), - KERNEL_CREATE_INFO_TYPED(11, MLFloat16, Conv, kMSInternalNHWCDomain), - - KERNEL_CREATE_INFO_VERSIONED_TYPED(1, 10, MLFloat16, ConvTranspose, kMSInternalNHWCDomain), - KERNEL_CREATE_INFO_TYPED(11, MLFloat16, ConvTranspose, kMSInternalNHWCDomain), - - KERNEL_CREATE_INFO_VERSIONED_TYPED(8, 9, MLFloat16, MaxPool, kMSInternalNHWCDomain), - KERNEL_CREATE_INFO_VERSIONED_TYPED(10, 10, MLFloat16, MaxPool, kMSInternalNHWCDomain), - KERNEL_CREATE_INFO_VERSIONED_TYPED(11, 11, MLFloat16, MaxPool, kMSInternalNHWCDomain), - KERNEL_CREATE_INFO_TYPED(12, MLFloat16, MaxPool, kMSInternalNHWCDomain), -#endif }; for (auto& function_table_entry : function_table) { diff --git a/onnxruntime/test/providers/cpu/math/gemm_test.cc b/onnxruntime/test/providers/cpu/math/gemm_test.cc index 66408e6adf..d0069a0069 100644 --- a/onnxruntime/test/providers/cpu/math/gemm_test.cc +++ b/onnxruntime/test/providers/cpu/math/gemm_test.cc @@ -25,7 +25,7 @@ const constexpr auto run_with_tunable_op = &run_options; } // namespace -// Only CUDA, ROCM and CoreML kernels have float 16 support +// Only CUDA, ROCM, CoreML and XNNPack kernels have float 16 support TEST(GemmOpTest, GemmNoTrans_f16) { #ifdef USE_CUDA int min_cuda_architecture = 530; diff --git a/onnxruntime/test/providers/cpu/math/matmul_test.cc b/onnxruntime/test/providers/cpu/math/matmul_test.cc index a7d2281ac1..701a70e49c 100644 --- a/onnxruntime/test/providers/cpu/math/matmul_test.cc +++ b/onnxruntime/test/providers/cpu/math/matmul_test.cc @@ -2,6 +2,7 @@ // Licensed under the MIT License. #include "gtest/gtest.h" + #include "test/providers/provider_test_utils.h" #include "test/providers/run_options_config_keys.h" #include "test/common/dnnl_op_test_utils.h" @@ -144,6 +145,25 @@ std::vector> GenerateTestCases() { return test_cases; } +template <> +std::vector> GenerateTestCases() { + std::vector> test_cases; + + // test 2D expected_vals + std::vector expected_vals = {42, 48, 54, 114, 136, 158, 186, 224, 262}; + std::vector expected_vals_fp16(expected_vals.size()); + std::transform(expected_vals.begin(), expected_vals.end(), expected_vals_fp16.begin(), + [](int64_t num) { return MLFloat16(float(num)); }); + test_cases.push_back( + {"test 2D MLfloat16", + {3, 4}, + {4, 3}, + {3, 3}, + expected_vals_fp16}); + + return test_cases; +} + template void RunMatMulTest(int32_t opset_version, bool is_a_constant, bool is_b_constant) { for (auto t : GenerateTestCases()) { @@ -191,6 +211,32 @@ TEST(MathOpTest, MatMulFloatType) { RunMatMulTest(7, false, false); } +// To Test XNNPACK, Matrix B must be constant +TEST(MathOpTest, MatMulFloatType_ConstantB) { + // TODO: Unskip when fixed #41968513 + if (DefaultDmlExecutionProvider().get() != nullptr) { + GTEST_SKIP() << "Skipping because of the following error: Assertion failed: m_bufferTensorDesc.TotalTensorSizeInBytes >= ComputeByteSizeFromDimensions(nonBroadcastDimensions, dataType)"; + } + RunMatMulTest(7, false, true); +} + +#if defined(USE_CUDA) || defined(USE_ROCM) || defined(COREML_ENABLE_MLPROGRAM) || defined(USE_XNNPACK) +TEST(MathOpTest, MatMulFloat16_ConstantB) { +#ifdef USE_CUDA + int min_cuda_architecture = 530; + if (!HasCudaEnvironment(min_cuda_architecture)) { + LOGS_DEFAULT(WARNING) << "Hardware NOT support FP16"; + return; + } +#endif + // TODO: Unskip when fixed #41968513 + if (DefaultDmlExecutionProvider().get() != nullptr) { + GTEST_SKIP() << "Skipping because of the following error: Assertion failed: m_bufferTensorDesc.TotalTensorSizeInBytes >= ComputeByteSizeFromDimensions(nonBroadcastDimensions, dataType)"; + } + RunMatMulTest(7, false, true); +} +#endif + TEST(MathOpTest, MatMulDoubleType) { RunMatMulTest(7); } @@ -246,7 +292,7 @@ TEST(MathOpTest, MatMulZeroKInt32Type) { RunMatMulZeroKTest(); } -#if defined(USE_CUDA) || defined(USE_ROCM) || defined(COREML_ENABLE_MLPROGRAM) +#if defined(USE_CUDA) || defined(USE_ROCM) || defined(COREML_ENABLE_MLPROGRAM) || defined(USE_XNNPACK) TEST(MathOpTest, MatMul_Float16) { #ifdef USE_CUDA int min_cuda_architecture = 530; @@ -255,8 +301,6 @@ TEST(MathOpTest, MatMul_Float16) { return; } #endif - OpTester test("MatMul", 14); - std::vector A{1.0f, 2.0f, 3.0f, 4.0f, -1.0f, -2.0f, -3.0f, -4.0f}; std::vector B(12, 1.0f); @@ -270,12 +314,17 @@ TEST(MathOpTest, MatMul_Float16) { ConvertFloatToMLFloat16(B.data(), f_B.data(), 12); ConvertFloatToMLFloat16(Y.data(), f_Y.data(), 6); - test.AddInput("A", {2, 4}, f_A); - test.AddInput("B", {4, 3}, f_B); - test.AddOutput("Y", {2, 3}, f_Y); - test.ConfigExcludeEps({kTensorrtExecutionProvider}) // TensorRT: fp16 is not supported - .Config(run_with_tunable_op) - .RunWithConfig(); + for (int i = 0; i < 2; i++) { + // it needs Matrix B as constant to test XNNPack + bool b_is_constant = i == 0 ? false : true; + OpTester test("MatMul", 14); + test.AddInput("A", {2, 4}, f_A); + test.AddInput("B", {4, 3}, f_B, b_is_constant); + test.AddOutput("Y", {2, 3}, f_Y); + test.ConfigExcludeEps({kTensorrtExecutionProvider}) // TensorRT: fp16 is not supported + .Config(run_with_tunable_op) + .RunWithConfig(); + } } #endif diff --git a/onnxruntime/test/providers/cpu/math/softmax_test.cc b/onnxruntime/test/providers/cpu/math/softmax_test.cc index 6eb72255bd..6f7930f722 100644 --- a/onnxruntime/test/providers/cpu/math/softmax_test.cc +++ b/onnxruntime/test/providers/cpu/math/softmax_test.cc @@ -49,7 +49,7 @@ TEST(SoftmaxOperator, Simple) { RunTest(x_vals, expected_vals, dimensions); } -#if defined(USE_CUDA) || defined(USE_ROCM) +#if defined(USE_CUDA) || defined(USE_ROCM) || defined(USE_XNNPACK) TEST(SoftmaxOperator, Simple_fp16) { #ifdef USE_CUDA int min_cuda_architecture = 530; diff --git a/onnxruntime/test/providers/cpu/nn/conv_fp16_test.cc b/onnxruntime/test/providers/cpu/nn/conv_fp16_test.cc index 5cd43d21aa..4253e36e02 100644 --- a/onnxruntime/test/providers/cpu/nn/conv_fp16_test.cc +++ b/onnxruntime/test/providers/cpu/nn/conv_fp16_test.cc @@ -2,10 +2,8 @@ // Licensed under the MIT License. #include "core/mlas/inc/mlas.h" -#include "core/providers/xnnpack/xnnpack_init.h" -// XNNPACK_FP16_SUPPORTED scope is too big, so add USE_XNNPACK to avoid the FP16 tests enabled for other EPs -#if defined(MLAS_F16VEC_INTRINSICS_SUPPORTED) || defined(COREML_ENABLE_MLPROGRAM) || (defined(USE_XNNPACK) && defined(XNNPACK_FP16_SUPPORTED)) +#if defined(MLAS_F16VEC_INTRINSICS_SUPPORTED) || defined(COREML_ENABLE_MLPROGRAM) || defined(USE_XNNPACK) #include "gtest/gtest.h" #include "test/providers/provider_test_utils.h" diff --git a/onnxruntime/test/providers/cpu/nn/pool_fp16_op_test.cc b/onnxruntime/test/providers/cpu/nn/pool_fp16_op_test.cc index 46eb1180f4..d4e0af5011 100644 --- a/onnxruntime/test/providers/cpu/nn/pool_fp16_op_test.cc +++ b/onnxruntime/test/providers/cpu/nn/pool_fp16_op_test.cc @@ -3,7 +3,7 @@ #include "core/mlas/inc/mlas.h" -#if defined(MLAS_F16VEC_INTRINSICS_SUPPORTED) || defined(COREML_ENABLE_MLPROGRAM) || defined(XNNPACK_FP16_SUPPORTED) +#if defined(MLAS_F16VEC_INTRINSICS_SUPPORTED) || defined(COREML_ENABLE_MLPROGRAM) || defined(USE_XNNPACK) #include "core/providers/cpu/nn/pool.h" #include "gtest/gtest.h"