saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-27 20:02:15 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

[QNN EP] Support float16 BatchNormalization on the HTP backend (#20391)

Browse source 
### Description
- Adds support for float16 BatchNormalization to the HTP backend.
- Fixes float32 support for BatchNormalization on the HTP backend when
`enable_htp_fp16_precision` is enabled.


### Motivation and Context
Support more models on the QNN HTP backend.
This commit is contained in:
Adrian Lizarraga 2024-04-19 21:49:39 -07:00 committed by GitHub
parent 8fbb8a149f
commit 77b7619a3d
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
4 changed files with 175 additions and 77 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

124
onnxruntime/core/providers/qnn/builder/opbuilder/batch_norm_op_builder.cc
View file

@ -7,6 +7,7 @@
#include "core/providers/common.h"
#include "core/providers/shared/utils/utils.h"
#include "core/framework/float16.h"
#include "core/framework/tensorprotoutils.h"
#include "core/providers/qnn/builder/qnn_model_wrapper.h"
#include "core/providers/qnn/builder/qnn_utils.h"
@ -87,9 +88,13 @@ class BatchNormOpBuilder : public BaseOpBuilder {
offset += sizeof(float);
break;
}
case QNN_DATATYPE_FLOAT_16: {
value = static_cast<double>(reinterpret_cast<const MLFloat16*>(raw_ptr)->ToFloat());
offset += sizeof(MLFloat16);
break;
}
case QNN_DATATYPE_BOOL_8:
case QNN_DATATYPE_STRING:
case QNN_DATATYPE_FLOAT_16:
default:
ORT_RETURN_IF(true, "Qnn Data Type: %d not supported yet.", qnn_data_type);
}
@ -102,60 +107,64 @@ class BatchNormOpBuilder : public BaseOpBuilder {
switch (qnn_data_type) {
case QNN_DATATYPE_INT_8:
case QNN_DATATYPE_SFIXED_POINT_8: {
ORT_ENFORCE(channel == static_cast<uint32_t>(raw_ptr_length / sizeof(int8_t)),
"initializer size not match Qnn data type.");
ORT_RETURN_IF_NOT(channel == static_cast<uint32_t>(raw_ptr_length / sizeof(int8_t)),
"initializer size not match Qnn data type.");
break;
}
case QNN_DATATYPE_INT_16:
case QNN_DATATYPE_SFIXED_POINT_16: {
ORT_ENFORCE(channel == static_cast<uint32_t>(raw_ptr_length / sizeof(int16_t)),
"initializer size not match Qnn data type.");
ORT_RETURN_IF_NOT(channel == static_cast<uint32_t>(raw_ptr_length / sizeof(int16_t)),
"initializer size not match Qnn data type.");
break;
}
case QNN_DATATYPE_INT_32:
case QNN_DATATYPE_SFIXED_POINT_32: {
ORT_ENFORCE(channel == static_cast<uint32_t>(raw_ptr_length / sizeof(int32_t)),
"initializer size not match Qnn data type.");
ORT_RETURN_IF_NOT(channel == static_cast<uint32_t>(raw_ptr_length / sizeof(int32_t)),
"initializer size not match Qnn data type.");
break;
}
case QNN_DATATYPE_INT_64: {
ORT_ENFORCE(channel == static_cast<uint32_t>(raw_ptr_length / sizeof(int64_t)),
"initializer size not match Qnn data type.");
ORT_RETURN_IF_NOT(channel == static_cast<uint32_t>(raw_ptr_length / sizeof(int64_t)),
"initializer size not match Qnn data type.");
break;
}
case QNN_DATATYPE_UINT_8:
case QNN_DATATYPE_UFIXED_POINT_8: {
ORT_ENFORCE(channel == static_cast<uint32_t>(raw_ptr_length / sizeof(uint8_t)),
"initializer size not match Qnn data type.");
ORT_RETURN_IF_NOT(channel == static_cast<uint32_t>(raw_ptr_length / sizeof(uint8_t)),
"initializer size not match Qnn data type.");
break;
}
case QNN_DATATYPE_UINT_16:
case QNN_DATATYPE_UFIXED_POINT_16: {
ORT_ENFORCE(channel == static_cast<uint32_t>(raw_ptr_length / sizeof(uint16_t)),
"initializer size not match Qnn data type.");
ORT_RETURN_IF_NOT(channel == static_cast<uint32_t>(raw_ptr_length / sizeof(uint16_t)),
"initializer size not match Qnn data type.");
break;
}
case QNN_DATATYPE_UINT_32:
case QNN_DATATYPE_UFIXED_POINT_32: {
ORT_ENFORCE(channel == static_cast<uint32_t>(raw_ptr_length / sizeof(uint32_t)),
"initializer size not match Qnn data type.");
ORT_RETURN_IF_NOT(channel == static_cast<uint32_t>(raw_ptr_length / sizeof(uint32_t)),
"initializer size not match Qnn data type.");
break;
}
case QNN_DATATYPE_UINT_64: {
ORT_ENFORCE(channel == static_cast<uint32_t>(raw_ptr_length / sizeof(uint64_t)),
"initializer size not match Qnn data type.");
ORT_RETURN_IF_NOT(channel == static_cast<uint32_t>(raw_ptr_length / sizeof(uint64_t)),
"initializer size not match Qnn data type.");
break;
}
case QNN_DATATYPE_FLOAT_32: {
ORT_ENFORCE(channel == static_cast<uint32_t>(raw_ptr_length / sizeof(float)),
"initializer size not match Qnn data type.");
ORT_RETURN_IF_NOT(channel == static_cast<uint32_t>(raw_ptr_length / sizeof(float)),
"initializer size not match Qnn data type.");
break;
}
case QNN_DATATYPE_FLOAT_16: {
ORT_RETURN_IF_NOT(channel == static_cast<uint32_t>(raw_ptr_length / sizeof(MLFloat16)),
"initializer size not match Qnn data type.");
break;
}
case QNN_DATATYPE_BOOL_8:
case QNN_DATATYPE_STRING:
case QNN_DATATYPE_FLOAT_16:
default:
ORT_RETURN_IF(true, "Qnn Data Type: %d not supported yet.", qnn_data_type);
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Qnn Data Type: ", qnn_data_type, " is not supported yet.");
}
return Status::OK();
}
@ -236,6 +245,14 @@ class BatchNormOpBuilder : public BaseOpBuilder {
}
break;
}
case QNN_DATATYPE_FLOAT_16: {
raw_tensor.resize(double_tensor.size() * sizeof(MLFloat16));
MLFloat16* raw_ptr = reinterpret_cast<MLFloat16*>(raw_tensor.data());
for (size_t i = 0; i < double_tensor.size(); ++i) {
raw_ptr[i] = MLFloat16(static_cast<float>(double_tensor[i]));
}
break;
}
case QNN_DATATYPE_UFIXED_POINT_32:
case QNN_DATATYPE_UFIXED_POINT_16:
case QNN_DATATYPE_UFIXED_POINT_8:
@ -244,15 +261,13 @@ class BatchNormOpBuilder : public BaseOpBuilder {
case QNN_DATATYPE_SFIXED_POINT_8:
case QNN_DATATYPE_BOOL_8:
case QNN_DATATYPE_STRING:
case QNN_DATATYPE_FLOAT_16:
default:
ORT_RETURN_IF(true, "Qnn Data Type: %d not supported yet.", qnn_data_type);
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Qnn Data Type: ", qnn_data_type, " is not supported yet.");
}
return Status::OK();
}
Status PreprocessMean(const TensorInfo& mean_info,
const bool is_npu_backend,
const uint8_t* mean_raw_ptr,
const size_t mean_raw_ptr_length,
std::vector<double>& mean_out) const {
@ -260,7 +275,9 @@ class BatchNormOpBuilder : public BaseOpBuilder {
uint32_t channel = mean_info.shape[0];
mean_out.resize(channel);
ORT_RETURN_IF_ERROR(AssertUnpackedTensorSize(mean_info.qnn_data_type, channel, mean_raw_ptr_length));
ORT_RETURN_IF_NOT(!is_npu_backend || mean_info.quant_param.IsPerTensor(),
const bool is_quantized = mean_info.quant_param.IsQuantized();
ORT_RETURN_IF_NOT(!is_quantized || mean_info.quant_param.IsPerTensor(),
"BatchNormalization's input_mean does not support per-channel quantization");
int i = 0;
int offset = 0;
@ -268,16 +285,15 @@ class BatchNormOpBuilder : public BaseOpBuilder {
for (; i < static_cast<int>(channel); ++i) {
double mean_value = 0.0;
ORT_RETURN_IF_ERROR(GetValueOnQnnDataType(mean_info.qnn_data_type, mean_raw_ptr + offset, mean_value, offset));
mean_out[i] = (is_npu_backend) ? utils::Dequantize(quant_param.scaleOffsetEncoding.offset,
quant_param.scaleOffsetEncoding.scale,
mean_value)
: mean_value;
mean_out[i] = (is_quantized) ? utils::Dequantize(quant_param.scaleOffsetEncoding.offset,
quant_param.scaleOffsetEncoding.scale,
mean_value)
: mean_value;
}
return Status::OK();
}
Status PreprocessStd(const TensorInfo& var_info,
const bool is_npu_backend,
const uint8_t* var_raw_ptr,
const size_t var_raw_ptr_length,
const float epsilon,
@ -286,7 +302,9 @@ class BatchNormOpBuilder : public BaseOpBuilder {
uint32_t channel = var_info.shape[0];
std_out.resize(channel);
ORT_RETURN_IF_ERROR(AssertUnpackedTensorSize(var_info.qnn_data_type, channel, var_raw_ptr_length));
ORT_RETURN_IF_NOT(!is_npu_backend || var_info.quant_param.IsPerTensor(),
const bool is_quantized = var_info.quant_param.IsQuantized();
ORT_RETURN_IF_NOT(!is_quantized || var_info.quant_param.IsPerTensor(),
"BatchNormalization's input_var does not support per-channel quantization");
int i = 0;
int offset = 0;
@ -294,17 +312,16 @@ class BatchNormOpBuilder : public BaseOpBuilder {
for (; i < static_cast<int>(channel); ++i) {
double var_value = 0.0;
ORT_RETURN_IF_ERROR(GetValueOnQnnDataType(var_info.qnn_data_type, var_raw_ptr + offset, var_value, offset));
std_out[i] = (is_npu_backend) ? utils::Dequantize(quant_param.scaleOffsetEncoding.offset,
quant_param.scaleOffsetEncoding.scale,
var_value)
: var_value;
std_out[i] = (is_quantized) ? utils::Dequantize(quant_param.scaleOffsetEncoding.offset,
quant_param.scaleOffsetEncoding.scale,
var_value)
: var_value;
std_out[i] = std::sqrt(std_out[i] + static_cast<double>(epsilon));
}
return Status::OK();
}
Status PreprocessScale(const TensorInfo& scale_info,
const bool is_npu_backend,
const uint8_t* scale_raw_ptr,
const size_t scale_raw_ptr_length,
const std::vector<double>& std_double_tensor,
@ -315,7 +332,9 @@ class BatchNormOpBuilder : public BaseOpBuilder {
uint32_t channel = scale_info.shape[0];
scale_out.resize(channel);
ORT_RETURN_IF_ERROR(AssertUnpackedTensorSize(scale_info.qnn_data_type, channel, scale_raw_ptr_length));
ORT_RETURN_IF_NOT(!is_npu_backend || scale_info.quant_param.IsPerTensor(),
const bool is_quantized = scale_info.quant_param.IsQuantized();
ORT_RETURN_IF_NOT(!is_quantized || scale_info.quant_param.IsPerTensor(),
"BatchNormalization's scale input does not support per-channel quantization");
int i = 0;
int offset = 0;
@ -323,10 +342,10 @@ class BatchNormOpBuilder : public BaseOpBuilder {
for (; i < static_cast<int>(channel); ++i) {
double scale_value = 0.0;
ORT_RETURN_IF_ERROR(GetValueOnQnnDataType(scale_info.qnn_data_type, scale_raw_ptr + offset, scale_value, offset));
scale_out[i] = (is_npu_backend) ? utils::Dequantize(quant_param.scaleOffsetEncoding.offset,
quant_param.scaleOffsetEncoding.scale,
scale_value)
: scale_value;
scale_out[i] = (is_quantized) ? utils::Dequantize(quant_param.scaleOffsetEncoding.offset,
quant_param.scaleOffsetEncoding.scale,
scale_value)
: scale_value;
scale_out[i] = scale_out[i] / std_double_tensor[i];
rmax = std::max(rmax, scale_out[i]);
rmin = std::min(rmin, scale_out[i]);
@ -335,7 +354,6 @@ class BatchNormOpBuilder : public BaseOpBuilder {
}
Status PreprocessBias(const TensorInfo& bias_info,
const bool is_npu_backend,
const uint8_t* bias_raw_ptr,
const size_t bias_raw_ptr_length,
const std::vector<double>& scale_double_tensor,
@ -347,7 +365,9 @@ class BatchNormOpBuilder : public BaseOpBuilder {
uint32_t channel = bias_info.shape[0];
bias_out.resize(channel);
ORT_RETURN_IF_ERROR(AssertUnpackedTensorSize(bias_info.qnn_data_type, channel, bias_raw_ptr_length));
ORT_RETURN_IF_NOT(!is_npu_backend || bias_info.quant_param.IsPerTensor(),
const bool is_quantized = bias_info.quant_param.IsQuantized();
ORT_RETURN_IF_NOT(!is_quantized || bias_info.quant_param.IsPerTensor(),
"BatchNormalization's bias input does not support per-channel quantization");
int i = 0;
int offset = 0;
@ -355,10 +375,10 @@ class BatchNormOpBuilder : public BaseOpBuilder {
for (; i < static_cast<int>(channel); ++i) {
double bias_value = 0.0;
ORT_RETURN_IF_ERROR(GetValueOnQnnDataType(bias_info.qnn_data_type, bias_raw_ptr + offset, bias_value, offset));
bias_out[i] = (is_npu_backend) ? utils::Dequantize(quant_param.scaleOffsetEncoding.offset,
quant_param.scaleOffsetEncoding.scale,
bias_value)
: bias_value;
bias_out[i] = (is_quantized) ? utils::Dequantize(quant_param.scaleOffsetEncoding.offset,
quant_param.scaleOffsetEncoding.scale,
bias_value)
: bias_value;
bias_out[i] = bias_out[i] - (mean_double_tensor[i] * scale_double_tensor[i]);
rmax = std::max(rmax, bias_out[i]);
rmin = std::min(rmin, bias_out[i]);
@ -367,13 +387,12 @@ class BatchNormOpBuilder : public BaseOpBuilder {
}
Status Postprocess(const TensorInfo& info,
const bool is_npu_backend,
const std::vector<double>& double_tensor,
const double rmax,
const double rmin,
QnnQuantParamsWrapper& quant_param,
std::vector<uint8_t>& raw_tensor) const {
if (is_npu_backend) {
if (info.quant_param.IsQuantized()) {
raw_tensor.resize(double_tensor.size());
float scale = 0.0f;
int zero_point = 0;
@ -474,7 +493,6 @@ Status BatchNormOpBuilder::ProcessInputs(QnnModelWrapper& qnn_model_wrapper,
ORT_UNUSED_PARAMETER(logger);
const auto& inputs = node_unit.Inputs();
bool is_npu_backend = IsNpuBackend(qnn_model_wrapper.GetQnnBackendType());
//
// Input 0
//
@ -527,18 +545,15 @@ Status BatchNormOpBuilder::ProcessInputs(QnnModelWrapper& qnn_model_wrapper,
// Calculate and convert new scale, new bias, mean and std to double array (may be dequantized)
ORT_RETURN_IF_ERROR(PreprocessMean(mean_info,
is_npu_backend,
mean_unpacked_tensor.data(),
mean_unpacked_tensor.size(),
mean_double_tensor));
ORT_RETURN_IF_ERROR(PreprocessStd(var_info,
is_npu_backend,
var_unpacked_tensor.data(),
var_unpacked_tensor.size(),
epsilon,
std_double_tensor));
ORT_RETURN_IF_ERROR(PreprocessScale(scale_info,
is_npu_backend,
scale_unpacked_tensor.data(),
scale_unpacked_tensor.size(),
std_double_tensor,
@ -546,7 +561,6 @@ Status BatchNormOpBuilder::ProcessInputs(QnnModelWrapper& qnn_model_wrapper,
scale_rmin,
scale_double_tensor));
ORT_RETURN_IF_ERROR(PreprocessBias(bias_info,
is_npu_backend,
bias_unpacked_tensor.data(),
bias_unpacked_tensor.size(),
scale_double_tensor,
@ -559,7 +573,6 @@ Status BatchNormOpBuilder::ProcessInputs(QnnModelWrapper& qnn_model_wrapper,
std::vector<uint8_t> scale_raw_tensor;
QnnQuantParamsWrapper scale_quant_param = scale_info.quant_param;
ORT_RETURN_IF_ERROR(Postprocess(scale_info,
is_npu_backend,
scale_double_tensor,
scale_rmax,
scale_rmin,
@ -577,7 +590,6 @@ Status BatchNormOpBuilder::ProcessInputs(QnnModelWrapper& qnn_model_wrapper,
std::vector<uint8_t> bias_raw_tensor;
QnnQuantParamsWrapper bias_quant_param = bias_info.quant_param;
ORT_RETURN_IF_ERROR(Postprocess(bias_info,
is_npu_backend,
bias_double_tensor,
bias_rmax,
bias_rmin,

108
onnxruntime/test/providers/qnn/batch_norm_htp_test.cc
View file

@ -5,6 +5,7 @@
#include <string>
#include "core/graph/graph.h"
#include "core/framework/float16.h"
#include "test/optimizer/qdq_test_utils.h"
#include "test/providers/qnn/qnn_test_utils.h"
@ -17,8 +18,9 @@ namespace test {
// Computes the mean and variance of inputs within a channel.
// Requires an input with rank >= 3
static void ComputeChannelMeanAndVar(const std::vector<float>& input_data, const std::vector<int64_t>& input_shape,
std::vector<float>& mean_vals, std::vector<float>& var_vals) {
template <typename FLOAT_TYPE>
static void ComputeChannelMeanAndVar(const std::vector<FLOAT_TYPE>& input_data, const std::vector<int64_t>& input_shape,
std::vector<FLOAT_TYPE>& mean_vals, std::vector<FLOAT_TYPE>& var_vals) {
const size_t input_rank = input_shape.size();
const size_t num_batches = input_shape[0];
const size_t num_channels = input_shape[1];
@ -32,8 +34,8 @@ static void ComputeChannelMeanAndVar(const std::vector<float>& input_data, const
assert(mean_vals.size() == num_channels);
assert(var_vals.size() == num_channels);
for (size_t i = 0; i < num_channels; i++) {
mean_vals[i] = 0.0f;
var_vals[i] = 0.0f;
mean_vals[i] = FLOAT_TYPE{};
var_vals[i] = FLOAT_TYPE{};
}
// Compute running sum of elements within each channel. The running sum is stored in the mean_vals array directly.
@ -44,14 +46,14 @@ static void ComputeChannelMeanAndVar(const std::vector<float>& input_data, const
const size_t chan_start = batch_start + (c * channel_stride);
for (size_t i = chan_start; i < chan_start + channel_stride; i++) {
mean_vals[c] += input_data[i];
mean_vals[c] = FLOAT_TYPE(mean_vals[c] + input_data[i]);
}
}
}
// Divide sums by the number of elements in a channel to get the mean.
for (size_t c = 0; c < num_channels; c++) {
mean_vals[c] /= static_cast<float>(num_batches * channel_stride);
mean_vals[c] = FLOAT_TYPE(mean_vals[c] / FLOAT_TYPE(static_cast<float>(num_batches * channel_stride)));
}
// Compute running sum of deviations from mean within each channel. The running sum is stored in the var_vals array directly.
@ -62,21 +64,22 @@ static void ComputeChannelMeanAndVar(const std::vector<float>& input_data, const
const size_t chan_start = batch_start + (c * channel_stride);
for (size_t i = chan_start; i < chan_start + channel_stride; i++) {
const float deviation = input_data[i] - mean_vals[c];
var_vals[c] += (deviation * deviation);
const FLOAT_TYPE deviation = FLOAT_TYPE(input_data[i] - mean_vals[c]);
var_vals[c] = FLOAT_TYPE(var_vals[c] + (deviation * deviation));
}
}
}
// Divide sums by the number of elements in a channel to get the variance.
for (size_t c = 0; c < num_channels; c++) {
var_vals[c] /= static_cast<float>(num_batches * channel_stride);
var_vals[c] = FLOAT_TYPE(var_vals[c] / FLOAT_TYPE(static_cast<float>(num_batches * channel_stride)));
}
}
static GetTestModelFn BuildBatchNormTestCase(const TestInputDef<float>& input_def,
const TestInputDef<float>& scale_def,
const TestInputDef<float>& bias_def) {
template <typename FLOAT_TYPE>
static GetTestModelFn BuildBatchNormTestCase(const TestInputDef<FLOAT_TYPE>& input_def,
const TestInputDef<FLOAT_TYPE>& scale_def,
const TestInputDef<FLOAT_TYPE>& bias_def) {
ORT_ENFORCE(input_def.IsRawData()); // Need raw data to compute mean and variance inputs.
ORT_ENFORCE(input_def.GetShape().size() > 2); // Need at least rank 3 data for convenience.
@ -85,16 +88,16 @@ static GetTestModelFn BuildBatchNormTestCase(const TestInputDef<float>& input_de
const auto& input_data = input_def.GetRawData();
const int64_t num_channels = input_shape[1];
NodeArg* input = MakeTestInput(builder, input_def);
NodeArg* scale = MakeTestInput(builder, scale_def);
NodeArg* bias = MakeTestInput(builder, bias_def);
NodeArg* input = MakeTestInput<FLOAT_TYPE>(builder, input_def);
NodeArg* scale = MakeTestInput<FLOAT_TYPE>(builder, scale_def);
NodeArg* bias = MakeTestInput<FLOAT_TYPE>(builder, bias_def);
std::vector<float> mean_vals(num_channels);
std::vector<float> var_vals(num_channels);
ComputeChannelMeanAndVar(input_data, input_shape, mean_vals, var_vals);
std::vector<FLOAT_TYPE> mean_vals(num_channels);
std::vector<FLOAT_TYPE> var_vals(num_channels);
ComputeChannelMeanAndVar<FLOAT_TYPE>(input_data, input_shape, mean_vals, var_vals);
NodeArg* mean = builder.MakeInitializer<float>({num_channels}, mean_vals);
NodeArg* var = builder.MakeInitializer<float>({num_channels}, var_vals);
NodeArg* mean = builder.MakeInitializer<FLOAT_TYPE>({num_channels}, mean_vals);
NodeArg* var = builder.MakeInitializer<FLOAT_TYPE>({num_channels}, var_vals);
NodeArg* output = builder.MakeOutput();
builder.AddNode("BatchNormalization", {input, scale, bias, mean, var}, {output});
};
@ -171,6 +174,29 @@ static void RunBatchNormQDQTest(const TestInputDef<float>& input_def,
expected_ep_assignment);
}
static void RunBatchNormFP16Test(const TestInputDef<float>& input_def,
const TestInputDef<float>& scale_def,
const TestInputDef<float>& bias_def,
ExpectedEPNodeAssignment expected_ep_assignment) {
ProviderOptions provider_options;
#if defined(_WIN32)
provider_options["backend_path"] = "QnnHtp.dll";
#else
provider_options["backend_path"] = "libQnnHtp.so";
#endif
TestInputDef<MLFloat16> input_fp16_def = ConvertToFP16InputDef(input_def);
TestInputDef<MLFloat16> scale_fp16_def = ConvertToFP16InputDef(scale_def);
TestInputDef<MLFloat16> bias_fp16_def = ConvertToFP16InputDef(bias_def);
// Runs model with DQ-> InstanceNorm -> Q and compares the outputs of the CPU and QNN EPs.
TestFp16ModelAccuracy(BuildBatchNormTestCase<float>(input_def, scale_def, bias_def),
BuildBatchNormTestCase<MLFloat16>(input_fp16_def, scale_fp16_def, bias_fp16_def),
provider_options,
11,
expected_ep_assignment);
}
// TODO: FIX TRANSLATION!!!
// Check that QNN compiles DQ -> BatchNormalization -> Q as a single unit.
// Use an input of rank 3.
@ -183,7 +209,6 @@ TEST_F(QnnHTPBackendTests, BatchNorm1D) {
ExpectedEPNodeAssignment::All);
}
// TODO: FIX TRANSLATION!!!
// Check that QNN compiles DQ -> BatchNormalization -> Q as a single unit.
// Use an input of rank 4.
TEST_F(QnnHTPBackendTests, BatchNorm2D) {
@ -197,6 +222,47 @@ TEST_F(QnnHTPBackendTests, BatchNorm2D) {
ExpectedEPNodeAssignment::All);
}
// Test FP16 BatchNormalization on the HTP backend.
TEST_F(QnnHTPBackendTests, BatchNorm_FP16) {
constexpr int64_t num_channels = 2;
std::vector<float> input_data = {-8.0f, -6.0f, -4.0f, -2.0f, 0.0f, 1.1f, 3.3f, 8.0f,
-7.0f, -5.0f, -3.0f, -1.0f, 0.0f, 2.1f, 4.3f, 7.0f};
RunBatchNormFP16Test(TestInputDef<float>({2, num_channels, 2, 2}, false, input_data), // Input data
TestInputDef<float>({num_channels}, true, {1.0f, 2.0f}), // Scale initializer
TestInputDef<float>({num_channels}, true, {1.1f, 2.1f}), // Bias initializer
ExpectedEPNodeAssignment::All);
}
// Test FP32 BatchNormalization on the HTP backend with the enable_htp_fp16_precision option enabled
// to run it with fp16 precision.
TEST_F(QnnHTPBackendTests, BatchNorm_FP32_as_FP16) {
ProviderOptions provider_options;
#if defined(_WIN32)
provider_options["backend_path"] = "QnnHtp.dll";
#else
provider_options["backend_path"] = "libQnnHtp.so";
#endif
provider_options["enable_htp_fp16_precision"] = "1";
constexpr int64_t num_channels = 2;
std::vector<float> input_data = {-8.0f, -6.0f, -4.0f, -2.0f, 0.0f, 1.1f, 3.3f, 8.0f,
-7.0f, -5.0f, -3.0f, -1.0f, 0.0f, 2.1f, 4.3f, 7.0f};
auto input_def = TestInputDef<float>({2, num_channels, 2, 2}, false, input_data);
auto scale_def = TestInputDef<float>({num_channels}, true, {1.0f, 2.0f});
auto bias_def = TestInputDef<float>({num_channels}, true, {1.1f, 2.1f});
auto model_fn = BuildBatchNormTestCase<float>(input_def, scale_def, bias_def);
RunQnnModelTest(model_fn,
provider_options,
13, // opset
ExpectedEPNodeAssignment::All,
0.01f); // abs err
}
// Check that QNN compiles DQ -> BatchNormalization -> Q as a single unit.
// Use an input of rank 5. QNN BatchNormalization doesn't support 5D on HTP
TEST_F(QnnHTPBackendTests, BatchNorm3D) {

16
onnxruntime/test/providers/qnn/qnn_test_utils.cc
View file

@ -65,6 +65,22 @@ std::vector<float> GetSequentialFloatData(const std::vector<int64_t>& shape, flo
return data;
}
TestInputDef<MLFloat16> ConvertToFP16InputDef(const TestInputDef<float>& input_def) {
if (input_def.IsRawData()) {
std::vector<MLFloat16> input_data_fp16;
input_data_fp16.reserve(input_def.GetRawData().size());
for (float f32_val : input_def.GetRawData()) {
input_data_fp16.push_back(MLFloat16(f32_val));
}
return TestInputDef<MLFloat16>(input_def.GetShape(), input_def.IsInitializer(), input_data_fp16);
} else {
auto rand_data = input_def.GetRandomDataInfo();
return TestInputDef<MLFloat16>(input_def.GetShape(), input_def.IsInitializer(),
MLFloat16(rand_data.min), MLFloat16(rand_data.max));
}
}
void TryEnableQNNSaver(ProviderOptions& qnn_options) {
// Allow dumping QNN API calls to file by setting an environment variable that enables the QNN Saver backend.
constexpr auto kEnableQNNSaverEnvironmentVariableName = "ORT_UNIT_TEST_ENABLE_QNN_SAVER";

4
onnxruntime/test/providers/qnn/qnn_test_utils.h
View file

@ -10,6 +10,7 @@
#include <unordered_map>
#include "core/framework/provider_options.h"
#include "core/framework/tensor_shape.h"
#include "core/framework/float16.h"
#include "core/util/qmath.h"
#include "test/optimizer/qdq_test_utils.h"
@ -267,6 +268,9 @@ struct TestInputDef {
std::pair<T, T> range_override_;
};
// Convert a float input definition to a float16 input definition.
TestInputDef<MLFloat16> ConvertToFP16InputDef(const TestInputDef<float>& input_def);
template <typename QType>
inline QuantParams<QType> GetTestInputQuantParams(const TestInputDef<float>& input_def, bool symmetric = false) {
const std::pair<float, float> frange = input_def.GetRange();