From 77b7619a3d81e619014bb714ece8b5e8c44f0788 Mon Sep 17 00:00:00 2001 From: Adrian Lizarraga Date: Fri, 19 Apr 2024 21:49:39 -0700 Subject: [PATCH] [QNN EP] Support float16 BatchNormalization on the HTP backend (#20391) ### Description - Adds support for float16 BatchNormalization to the HTP backend. - Fixes float32 support for BatchNormalization on the HTP backend when `enable_htp_fp16_precision` is enabled. ### Motivation and Context Support more models on the QNN HTP backend. --- .../opbuilder/batch_norm_op_builder.cc | 124 ++++++++++-------- .../test/providers/qnn/batch_norm_htp_test.cc | 108 ++++++++++++--- .../test/providers/qnn/qnn_test_utils.cc | 16 +++ .../test/providers/qnn/qnn_test_utils.h | 4 + 4 files changed, 175 insertions(+), 77 deletions(-) diff --git a/onnxruntime/core/providers/qnn/builder/opbuilder/batch_norm_op_builder.cc b/onnxruntime/core/providers/qnn/builder/opbuilder/batch_norm_op_builder.cc index 70ad00b90c..1cd09a157c 100644 --- a/onnxruntime/core/providers/qnn/builder/opbuilder/batch_norm_op_builder.cc +++ b/onnxruntime/core/providers/qnn/builder/opbuilder/batch_norm_op_builder.cc @@ -7,6 +7,7 @@ #include "core/providers/common.h" #include "core/providers/shared/utils/utils.h" +#include "core/framework/float16.h" #include "core/framework/tensorprotoutils.h" #include "core/providers/qnn/builder/qnn_model_wrapper.h" #include "core/providers/qnn/builder/qnn_utils.h" @@ -87,9 +88,13 @@ class BatchNormOpBuilder : public BaseOpBuilder { offset += sizeof(float); break; } + case QNN_DATATYPE_FLOAT_16: { + value = static_cast(reinterpret_cast(raw_ptr)->ToFloat()); + offset += sizeof(MLFloat16); + break; + } case QNN_DATATYPE_BOOL_8: case QNN_DATATYPE_STRING: - case QNN_DATATYPE_FLOAT_16: default: ORT_RETURN_IF(true, "Qnn Data Type: %d not supported yet.", qnn_data_type); } @@ -102,60 +107,64 @@ class BatchNormOpBuilder : public BaseOpBuilder { switch (qnn_data_type) { case QNN_DATATYPE_INT_8: case QNN_DATATYPE_SFIXED_POINT_8: { - ORT_ENFORCE(channel == static_cast(raw_ptr_length / sizeof(int8_t)), - "initializer size not match Qnn data type."); + ORT_RETURN_IF_NOT(channel == static_cast(raw_ptr_length / sizeof(int8_t)), + "initializer size not match Qnn data type."); break; } case QNN_DATATYPE_INT_16: case QNN_DATATYPE_SFIXED_POINT_16: { - ORT_ENFORCE(channel == static_cast(raw_ptr_length / sizeof(int16_t)), - "initializer size not match Qnn data type."); + ORT_RETURN_IF_NOT(channel == static_cast(raw_ptr_length / sizeof(int16_t)), + "initializer size not match Qnn data type."); break; } case QNN_DATATYPE_INT_32: case QNN_DATATYPE_SFIXED_POINT_32: { - ORT_ENFORCE(channel == static_cast(raw_ptr_length / sizeof(int32_t)), - "initializer size not match Qnn data type."); + ORT_RETURN_IF_NOT(channel == static_cast(raw_ptr_length / sizeof(int32_t)), + "initializer size not match Qnn data type."); break; } case QNN_DATATYPE_INT_64: { - ORT_ENFORCE(channel == static_cast(raw_ptr_length / sizeof(int64_t)), - "initializer size not match Qnn data type."); + ORT_RETURN_IF_NOT(channel == static_cast(raw_ptr_length / sizeof(int64_t)), + "initializer size not match Qnn data type."); break; } case QNN_DATATYPE_UINT_8: case QNN_DATATYPE_UFIXED_POINT_8: { - ORT_ENFORCE(channel == static_cast(raw_ptr_length / sizeof(uint8_t)), - "initializer size not match Qnn data type."); + ORT_RETURN_IF_NOT(channel == static_cast(raw_ptr_length / sizeof(uint8_t)), + "initializer size not match Qnn data type."); break; } case QNN_DATATYPE_UINT_16: case QNN_DATATYPE_UFIXED_POINT_16: { - ORT_ENFORCE(channel == static_cast(raw_ptr_length / sizeof(uint16_t)), - "initializer size not match Qnn data type."); + ORT_RETURN_IF_NOT(channel == static_cast(raw_ptr_length / sizeof(uint16_t)), + "initializer size not match Qnn data type."); break; } case QNN_DATATYPE_UINT_32: case QNN_DATATYPE_UFIXED_POINT_32: { - ORT_ENFORCE(channel == static_cast(raw_ptr_length / sizeof(uint32_t)), - "initializer size not match Qnn data type."); + ORT_RETURN_IF_NOT(channel == static_cast(raw_ptr_length / sizeof(uint32_t)), + "initializer size not match Qnn data type."); break; } case QNN_DATATYPE_UINT_64: { - ORT_ENFORCE(channel == static_cast(raw_ptr_length / sizeof(uint64_t)), - "initializer size not match Qnn data type."); + ORT_RETURN_IF_NOT(channel == static_cast(raw_ptr_length / sizeof(uint64_t)), + "initializer size not match Qnn data type."); break; } case QNN_DATATYPE_FLOAT_32: { - ORT_ENFORCE(channel == static_cast(raw_ptr_length / sizeof(float)), - "initializer size not match Qnn data type."); + ORT_RETURN_IF_NOT(channel == static_cast(raw_ptr_length / sizeof(float)), + "initializer size not match Qnn data type."); + break; + } + case QNN_DATATYPE_FLOAT_16: { + ORT_RETURN_IF_NOT(channel == static_cast(raw_ptr_length / sizeof(MLFloat16)), + "initializer size not match Qnn data type."); break; } case QNN_DATATYPE_BOOL_8: case QNN_DATATYPE_STRING: - case QNN_DATATYPE_FLOAT_16: default: - ORT_RETURN_IF(true, "Qnn Data Type: %d not supported yet.", qnn_data_type); + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Qnn Data Type: ", qnn_data_type, " is not supported yet."); } return Status::OK(); } @@ -236,6 +245,14 @@ class BatchNormOpBuilder : public BaseOpBuilder { } break; } + case QNN_DATATYPE_FLOAT_16: { + raw_tensor.resize(double_tensor.size() * sizeof(MLFloat16)); + MLFloat16* raw_ptr = reinterpret_cast(raw_tensor.data()); + for (size_t i = 0; i < double_tensor.size(); ++i) { + raw_ptr[i] = MLFloat16(static_cast(double_tensor[i])); + } + break; + } case QNN_DATATYPE_UFIXED_POINT_32: case QNN_DATATYPE_UFIXED_POINT_16: case QNN_DATATYPE_UFIXED_POINT_8: @@ -244,15 +261,13 @@ class BatchNormOpBuilder : public BaseOpBuilder { case QNN_DATATYPE_SFIXED_POINT_8: case QNN_DATATYPE_BOOL_8: case QNN_DATATYPE_STRING: - case QNN_DATATYPE_FLOAT_16: default: - ORT_RETURN_IF(true, "Qnn Data Type: %d not supported yet.", qnn_data_type); + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Qnn Data Type: ", qnn_data_type, " is not supported yet."); } return Status::OK(); } Status PreprocessMean(const TensorInfo& mean_info, - const bool is_npu_backend, const uint8_t* mean_raw_ptr, const size_t mean_raw_ptr_length, std::vector& mean_out) const { @@ -260,7 +275,9 @@ class BatchNormOpBuilder : public BaseOpBuilder { uint32_t channel = mean_info.shape[0]; mean_out.resize(channel); ORT_RETURN_IF_ERROR(AssertUnpackedTensorSize(mean_info.qnn_data_type, channel, mean_raw_ptr_length)); - ORT_RETURN_IF_NOT(!is_npu_backend || mean_info.quant_param.IsPerTensor(), + + const bool is_quantized = mean_info.quant_param.IsQuantized(); + ORT_RETURN_IF_NOT(!is_quantized || mean_info.quant_param.IsPerTensor(), "BatchNormalization's input_mean does not support per-channel quantization"); int i = 0; int offset = 0; @@ -268,16 +285,15 @@ class BatchNormOpBuilder : public BaseOpBuilder { for (; i < static_cast(channel); ++i) { double mean_value = 0.0; ORT_RETURN_IF_ERROR(GetValueOnQnnDataType(mean_info.qnn_data_type, mean_raw_ptr + offset, mean_value, offset)); - mean_out[i] = (is_npu_backend) ? utils::Dequantize(quant_param.scaleOffsetEncoding.offset, - quant_param.scaleOffsetEncoding.scale, - mean_value) - : mean_value; + mean_out[i] = (is_quantized) ? utils::Dequantize(quant_param.scaleOffsetEncoding.offset, + quant_param.scaleOffsetEncoding.scale, + mean_value) + : mean_value; } return Status::OK(); } Status PreprocessStd(const TensorInfo& var_info, - const bool is_npu_backend, const uint8_t* var_raw_ptr, const size_t var_raw_ptr_length, const float epsilon, @@ -286,7 +302,9 @@ class BatchNormOpBuilder : public BaseOpBuilder { uint32_t channel = var_info.shape[0]; std_out.resize(channel); ORT_RETURN_IF_ERROR(AssertUnpackedTensorSize(var_info.qnn_data_type, channel, var_raw_ptr_length)); - ORT_RETURN_IF_NOT(!is_npu_backend || var_info.quant_param.IsPerTensor(), + + const bool is_quantized = var_info.quant_param.IsQuantized(); + ORT_RETURN_IF_NOT(!is_quantized || var_info.quant_param.IsPerTensor(), "BatchNormalization's input_var does not support per-channel quantization"); int i = 0; int offset = 0; @@ -294,17 +312,16 @@ class BatchNormOpBuilder : public BaseOpBuilder { for (; i < static_cast(channel); ++i) { double var_value = 0.0; ORT_RETURN_IF_ERROR(GetValueOnQnnDataType(var_info.qnn_data_type, var_raw_ptr + offset, var_value, offset)); - std_out[i] = (is_npu_backend) ? utils::Dequantize(quant_param.scaleOffsetEncoding.offset, - quant_param.scaleOffsetEncoding.scale, - var_value) - : var_value; + std_out[i] = (is_quantized) ? utils::Dequantize(quant_param.scaleOffsetEncoding.offset, + quant_param.scaleOffsetEncoding.scale, + var_value) + : var_value; std_out[i] = std::sqrt(std_out[i] + static_cast(epsilon)); } return Status::OK(); } Status PreprocessScale(const TensorInfo& scale_info, - const bool is_npu_backend, const uint8_t* scale_raw_ptr, const size_t scale_raw_ptr_length, const std::vector& std_double_tensor, @@ -315,7 +332,9 @@ class BatchNormOpBuilder : public BaseOpBuilder { uint32_t channel = scale_info.shape[0]; scale_out.resize(channel); ORT_RETURN_IF_ERROR(AssertUnpackedTensorSize(scale_info.qnn_data_type, channel, scale_raw_ptr_length)); - ORT_RETURN_IF_NOT(!is_npu_backend || scale_info.quant_param.IsPerTensor(), + + const bool is_quantized = scale_info.quant_param.IsQuantized(); + ORT_RETURN_IF_NOT(!is_quantized || scale_info.quant_param.IsPerTensor(), "BatchNormalization's scale input does not support per-channel quantization"); int i = 0; int offset = 0; @@ -323,10 +342,10 @@ class BatchNormOpBuilder : public BaseOpBuilder { for (; i < static_cast(channel); ++i) { double scale_value = 0.0; ORT_RETURN_IF_ERROR(GetValueOnQnnDataType(scale_info.qnn_data_type, scale_raw_ptr + offset, scale_value, offset)); - scale_out[i] = (is_npu_backend) ? utils::Dequantize(quant_param.scaleOffsetEncoding.offset, - quant_param.scaleOffsetEncoding.scale, - scale_value) - : scale_value; + scale_out[i] = (is_quantized) ? utils::Dequantize(quant_param.scaleOffsetEncoding.offset, + quant_param.scaleOffsetEncoding.scale, + scale_value) + : scale_value; scale_out[i] = scale_out[i] / std_double_tensor[i]; rmax = std::max(rmax, scale_out[i]); rmin = std::min(rmin, scale_out[i]); @@ -335,7 +354,6 @@ class BatchNormOpBuilder : public BaseOpBuilder { } Status PreprocessBias(const TensorInfo& bias_info, - const bool is_npu_backend, const uint8_t* bias_raw_ptr, const size_t bias_raw_ptr_length, const std::vector& scale_double_tensor, @@ -347,7 +365,9 @@ class BatchNormOpBuilder : public BaseOpBuilder { uint32_t channel = bias_info.shape[0]; bias_out.resize(channel); ORT_RETURN_IF_ERROR(AssertUnpackedTensorSize(bias_info.qnn_data_type, channel, bias_raw_ptr_length)); - ORT_RETURN_IF_NOT(!is_npu_backend || bias_info.quant_param.IsPerTensor(), + + const bool is_quantized = bias_info.quant_param.IsQuantized(); + ORT_RETURN_IF_NOT(!is_quantized || bias_info.quant_param.IsPerTensor(), "BatchNormalization's bias input does not support per-channel quantization"); int i = 0; int offset = 0; @@ -355,10 +375,10 @@ class BatchNormOpBuilder : public BaseOpBuilder { for (; i < static_cast(channel); ++i) { double bias_value = 0.0; ORT_RETURN_IF_ERROR(GetValueOnQnnDataType(bias_info.qnn_data_type, bias_raw_ptr + offset, bias_value, offset)); - bias_out[i] = (is_npu_backend) ? utils::Dequantize(quant_param.scaleOffsetEncoding.offset, - quant_param.scaleOffsetEncoding.scale, - bias_value) - : bias_value; + bias_out[i] = (is_quantized) ? utils::Dequantize(quant_param.scaleOffsetEncoding.offset, + quant_param.scaleOffsetEncoding.scale, + bias_value) + : bias_value; bias_out[i] = bias_out[i] - (mean_double_tensor[i] * scale_double_tensor[i]); rmax = std::max(rmax, bias_out[i]); rmin = std::min(rmin, bias_out[i]); @@ -367,13 +387,12 @@ class BatchNormOpBuilder : public BaseOpBuilder { } Status Postprocess(const TensorInfo& info, - const bool is_npu_backend, const std::vector& double_tensor, const double rmax, const double rmin, QnnQuantParamsWrapper& quant_param, std::vector& raw_tensor) const { - if (is_npu_backend) { + if (info.quant_param.IsQuantized()) { raw_tensor.resize(double_tensor.size()); float scale = 0.0f; int zero_point = 0; @@ -474,7 +493,6 @@ Status BatchNormOpBuilder::ProcessInputs(QnnModelWrapper& qnn_model_wrapper, ORT_UNUSED_PARAMETER(logger); const auto& inputs = node_unit.Inputs(); - bool is_npu_backend = IsNpuBackend(qnn_model_wrapper.GetQnnBackendType()); // // Input 0 // @@ -527,18 +545,15 @@ Status BatchNormOpBuilder::ProcessInputs(QnnModelWrapper& qnn_model_wrapper, // Calculate and convert new scale, new bias, mean and std to double array (may be dequantized) ORT_RETURN_IF_ERROR(PreprocessMean(mean_info, - is_npu_backend, mean_unpacked_tensor.data(), mean_unpacked_tensor.size(), mean_double_tensor)); ORT_RETURN_IF_ERROR(PreprocessStd(var_info, - is_npu_backend, var_unpacked_tensor.data(), var_unpacked_tensor.size(), epsilon, std_double_tensor)); ORT_RETURN_IF_ERROR(PreprocessScale(scale_info, - is_npu_backend, scale_unpacked_tensor.data(), scale_unpacked_tensor.size(), std_double_tensor, @@ -546,7 +561,6 @@ Status BatchNormOpBuilder::ProcessInputs(QnnModelWrapper& qnn_model_wrapper, scale_rmin, scale_double_tensor)); ORT_RETURN_IF_ERROR(PreprocessBias(bias_info, - is_npu_backend, bias_unpacked_tensor.data(), bias_unpacked_tensor.size(), scale_double_tensor, @@ -559,7 +573,6 @@ Status BatchNormOpBuilder::ProcessInputs(QnnModelWrapper& qnn_model_wrapper, std::vector scale_raw_tensor; QnnQuantParamsWrapper scale_quant_param = scale_info.quant_param; ORT_RETURN_IF_ERROR(Postprocess(scale_info, - is_npu_backend, scale_double_tensor, scale_rmax, scale_rmin, @@ -577,7 +590,6 @@ Status BatchNormOpBuilder::ProcessInputs(QnnModelWrapper& qnn_model_wrapper, std::vector bias_raw_tensor; QnnQuantParamsWrapper bias_quant_param = bias_info.quant_param; ORT_RETURN_IF_ERROR(Postprocess(bias_info, - is_npu_backend, bias_double_tensor, bias_rmax, bias_rmin, diff --git a/onnxruntime/test/providers/qnn/batch_norm_htp_test.cc b/onnxruntime/test/providers/qnn/batch_norm_htp_test.cc index bf36922f88..023a6078ff 100644 --- a/onnxruntime/test/providers/qnn/batch_norm_htp_test.cc +++ b/onnxruntime/test/providers/qnn/batch_norm_htp_test.cc @@ -5,6 +5,7 @@ #include #include "core/graph/graph.h" +#include "core/framework/float16.h" #include "test/optimizer/qdq_test_utils.h" #include "test/providers/qnn/qnn_test_utils.h" @@ -17,8 +18,9 @@ namespace test { // Computes the mean and variance of inputs within a channel. // Requires an input with rank >= 3 -static void ComputeChannelMeanAndVar(const std::vector& input_data, const std::vector& input_shape, - std::vector& mean_vals, std::vector& var_vals) { +template +static void ComputeChannelMeanAndVar(const std::vector& input_data, const std::vector& input_shape, + std::vector& mean_vals, std::vector& var_vals) { const size_t input_rank = input_shape.size(); const size_t num_batches = input_shape[0]; const size_t num_channels = input_shape[1]; @@ -32,8 +34,8 @@ static void ComputeChannelMeanAndVar(const std::vector& input_data, const assert(mean_vals.size() == num_channels); assert(var_vals.size() == num_channels); for (size_t i = 0; i < num_channels; i++) { - mean_vals[i] = 0.0f; - var_vals[i] = 0.0f; + mean_vals[i] = FLOAT_TYPE{}; + var_vals[i] = FLOAT_TYPE{}; } // Compute running sum of elements within each channel. The running sum is stored in the mean_vals array directly. @@ -44,14 +46,14 @@ static void ComputeChannelMeanAndVar(const std::vector& input_data, const const size_t chan_start = batch_start + (c * channel_stride); for (size_t i = chan_start; i < chan_start + channel_stride; i++) { - mean_vals[c] += input_data[i]; + mean_vals[c] = FLOAT_TYPE(mean_vals[c] + input_data[i]); } } } // Divide sums by the number of elements in a channel to get the mean. for (size_t c = 0; c < num_channels; c++) { - mean_vals[c] /= static_cast(num_batches * channel_stride); + mean_vals[c] = FLOAT_TYPE(mean_vals[c] / FLOAT_TYPE(static_cast(num_batches * channel_stride))); } // Compute running sum of deviations from mean within each channel. The running sum is stored in the var_vals array directly. @@ -62,21 +64,22 @@ static void ComputeChannelMeanAndVar(const std::vector& input_data, const const size_t chan_start = batch_start + (c * channel_stride); for (size_t i = chan_start; i < chan_start + channel_stride; i++) { - const float deviation = input_data[i] - mean_vals[c]; - var_vals[c] += (deviation * deviation); + const FLOAT_TYPE deviation = FLOAT_TYPE(input_data[i] - mean_vals[c]); + var_vals[c] = FLOAT_TYPE(var_vals[c] + (deviation * deviation)); } } } // Divide sums by the number of elements in a channel to get the variance. for (size_t c = 0; c < num_channels; c++) { - var_vals[c] /= static_cast(num_batches * channel_stride); + var_vals[c] = FLOAT_TYPE(var_vals[c] / FLOAT_TYPE(static_cast(num_batches * channel_stride))); } } -static GetTestModelFn BuildBatchNormTestCase(const TestInputDef& input_def, - const TestInputDef& scale_def, - const TestInputDef& bias_def) { +template +static GetTestModelFn BuildBatchNormTestCase(const TestInputDef& input_def, + const TestInputDef& scale_def, + const TestInputDef& bias_def) { ORT_ENFORCE(input_def.IsRawData()); // Need raw data to compute mean and variance inputs. ORT_ENFORCE(input_def.GetShape().size() > 2); // Need at least rank 3 data for convenience. @@ -85,16 +88,16 @@ static GetTestModelFn BuildBatchNormTestCase(const TestInputDef& input_de const auto& input_data = input_def.GetRawData(); const int64_t num_channels = input_shape[1]; - NodeArg* input = MakeTestInput(builder, input_def); - NodeArg* scale = MakeTestInput(builder, scale_def); - NodeArg* bias = MakeTestInput(builder, bias_def); + NodeArg* input = MakeTestInput(builder, input_def); + NodeArg* scale = MakeTestInput(builder, scale_def); + NodeArg* bias = MakeTestInput(builder, bias_def); - std::vector mean_vals(num_channels); - std::vector var_vals(num_channels); - ComputeChannelMeanAndVar(input_data, input_shape, mean_vals, var_vals); + std::vector mean_vals(num_channels); + std::vector var_vals(num_channels); + ComputeChannelMeanAndVar(input_data, input_shape, mean_vals, var_vals); - NodeArg* mean = builder.MakeInitializer({num_channels}, mean_vals); - NodeArg* var = builder.MakeInitializer({num_channels}, var_vals); + NodeArg* mean = builder.MakeInitializer({num_channels}, mean_vals); + NodeArg* var = builder.MakeInitializer({num_channels}, var_vals); NodeArg* output = builder.MakeOutput(); builder.AddNode("BatchNormalization", {input, scale, bias, mean, var}, {output}); }; @@ -171,6 +174,29 @@ static void RunBatchNormQDQTest(const TestInputDef& input_def, expected_ep_assignment); } +static void RunBatchNormFP16Test(const TestInputDef& input_def, + const TestInputDef& scale_def, + const TestInputDef& bias_def, + ExpectedEPNodeAssignment expected_ep_assignment) { + ProviderOptions provider_options; +#if defined(_WIN32) + provider_options["backend_path"] = "QnnHtp.dll"; +#else + provider_options["backend_path"] = "libQnnHtp.so"; +#endif + + TestInputDef input_fp16_def = ConvertToFP16InputDef(input_def); + TestInputDef scale_fp16_def = ConvertToFP16InputDef(scale_def); + TestInputDef bias_fp16_def = ConvertToFP16InputDef(bias_def); + + // Runs model with DQ-> InstanceNorm -> Q and compares the outputs of the CPU and QNN EPs. + TestFp16ModelAccuracy(BuildBatchNormTestCase(input_def, scale_def, bias_def), + BuildBatchNormTestCase(input_fp16_def, scale_fp16_def, bias_fp16_def), + provider_options, + 11, + expected_ep_assignment); +} + // TODO: FIX TRANSLATION!!! // Check that QNN compiles DQ -> BatchNormalization -> Q as a single unit. // Use an input of rank 3. @@ -183,7 +209,6 @@ TEST_F(QnnHTPBackendTests, BatchNorm1D) { ExpectedEPNodeAssignment::All); } -// TODO: FIX TRANSLATION!!! // Check that QNN compiles DQ -> BatchNormalization -> Q as a single unit. // Use an input of rank 4. TEST_F(QnnHTPBackendTests, BatchNorm2D) { @@ -197,6 +222,47 @@ TEST_F(QnnHTPBackendTests, BatchNorm2D) { ExpectedEPNodeAssignment::All); } +// Test FP16 BatchNormalization on the HTP backend. +TEST_F(QnnHTPBackendTests, BatchNorm_FP16) { + constexpr int64_t num_channels = 2; + std::vector input_data = {-8.0f, -6.0f, -4.0f, -2.0f, 0.0f, 1.1f, 3.3f, 8.0f, + -7.0f, -5.0f, -3.0f, -1.0f, 0.0f, 2.1f, 4.3f, 7.0f}; + + RunBatchNormFP16Test(TestInputDef({2, num_channels, 2, 2}, false, input_data), // Input data + TestInputDef({num_channels}, true, {1.0f, 2.0f}), // Scale initializer + TestInputDef({num_channels}, true, {1.1f, 2.1f}), // Bias initializer + ExpectedEPNodeAssignment::All); +} + +// Test FP32 BatchNormalization on the HTP backend with the enable_htp_fp16_precision option enabled +// to run it with fp16 precision. +TEST_F(QnnHTPBackendTests, BatchNorm_FP32_as_FP16) { + ProviderOptions provider_options; + +#if defined(_WIN32) + provider_options["backend_path"] = "QnnHtp.dll"; +#else + provider_options["backend_path"] = "libQnnHtp.so"; +#endif + + provider_options["enable_htp_fp16_precision"] = "1"; + + constexpr int64_t num_channels = 2; + std::vector input_data = {-8.0f, -6.0f, -4.0f, -2.0f, 0.0f, 1.1f, 3.3f, 8.0f, + -7.0f, -5.0f, -3.0f, -1.0f, 0.0f, 2.1f, 4.3f, 7.0f}; + + auto input_def = TestInputDef({2, num_channels, 2, 2}, false, input_data); + auto scale_def = TestInputDef({num_channels}, true, {1.0f, 2.0f}); + auto bias_def = TestInputDef({num_channels}, true, {1.1f, 2.1f}); + auto model_fn = BuildBatchNormTestCase(input_def, scale_def, bias_def); + + RunQnnModelTest(model_fn, + provider_options, + 13, // opset + ExpectedEPNodeAssignment::All, + 0.01f); // abs err +} + // Check that QNN compiles DQ -> BatchNormalization -> Q as a single unit. // Use an input of rank 5. QNN BatchNormalization doesn't support 5D on HTP TEST_F(QnnHTPBackendTests, BatchNorm3D) { diff --git a/onnxruntime/test/providers/qnn/qnn_test_utils.cc b/onnxruntime/test/providers/qnn/qnn_test_utils.cc index f5ebe45a07..afaa5a341d 100644 --- a/onnxruntime/test/providers/qnn/qnn_test_utils.cc +++ b/onnxruntime/test/providers/qnn/qnn_test_utils.cc @@ -65,6 +65,22 @@ std::vector GetSequentialFloatData(const std::vector& shape, flo return data; } +TestInputDef ConvertToFP16InputDef(const TestInputDef& input_def) { + if (input_def.IsRawData()) { + std::vector input_data_fp16; + input_data_fp16.reserve(input_def.GetRawData().size()); + for (float f32_val : input_def.GetRawData()) { + input_data_fp16.push_back(MLFloat16(f32_val)); + } + + return TestInputDef(input_def.GetShape(), input_def.IsInitializer(), input_data_fp16); + } else { + auto rand_data = input_def.GetRandomDataInfo(); + return TestInputDef(input_def.GetShape(), input_def.IsInitializer(), + MLFloat16(rand_data.min), MLFloat16(rand_data.max)); + } +} + void TryEnableQNNSaver(ProviderOptions& qnn_options) { // Allow dumping QNN API calls to file by setting an environment variable that enables the QNN Saver backend. constexpr auto kEnableQNNSaverEnvironmentVariableName = "ORT_UNIT_TEST_ENABLE_QNN_SAVER"; diff --git a/onnxruntime/test/providers/qnn/qnn_test_utils.h b/onnxruntime/test/providers/qnn/qnn_test_utils.h index c474e98924..942914d0af 100644 --- a/onnxruntime/test/providers/qnn/qnn_test_utils.h +++ b/onnxruntime/test/providers/qnn/qnn_test_utils.h @@ -10,6 +10,7 @@ #include #include "core/framework/provider_options.h" #include "core/framework/tensor_shape.h" +#include "core/framework/float16.h" #include "core/util/qmath.h" #include "test/optimizer/qdq_test_utils.h" @@ -267,6 +268,9 @@ struct TestInputDef { std::pair range_override_; }; +// Convert a float input definition to a float16 input definition. +TestInputDef ConvertToFP16InputDef(const TestInputDef& input_def); + template inline QuantParams GetTestInputQuantParams(const TestInputDef& input_def, bool symmetric = false) { const std::pair frange = input_def.GetRange();