saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-29 20:14:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

[QNN EP] Conv ConvTranspose 3D support (#20507)

Browse source 
### Description
Support Conv ConvTranspose 3D for QNN EP
This commit is contained in:
Hector Li 2024-05-03 08:55:31 -07:00 committed by GitHub
parent 030a9611c2
commit e540423179
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
10 changed files with 684 additions and 216 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

19
onnxruntime/core/providers/common.h
View file

@ -146,6 +146,25 @@ inline Status ComputePadAndOutputShape(const int64_t in_dim,
return Status::OK();
}
constexpr inline int64_t ComputeTotalPad(int64_t in_size, int64_t stride, int64_t adj,
int64_t kernel, int64_t dilation, int64_t out_size) {
return std::max<int64_t>(0, (in_size - 1) * stride + adj + (kernel - 1) * dilation + 1 - out_size);
}
inline void DistributePadding(AutoPadType pad_type, const int64_t& total_pad,
int64_t& pad_head, int64_t& pad_tail) {
if (pad_type == AutoPadType::SAME_UPPER) {
// pad more on tail when total_pad is odd.
pad_head = total_pad / 2;
pad_tail = total_pad - total_pad / 2;
} else {
// When pad_type is NOTSET, SAME_LOWER or VALID,
// pad more on head when total_pad is odd.
pad_head = total_pad - total_pad / 2;
pad_tail = total_pad / 2;
}
}
// Note: This helper function will not have overflow protection
template <template <typename...> class Container, typename T>
T Product(const Container<T>& c) {

19
onnxruntime/core/providers/cpu/nn/conv_transpose_attributes.h
View file

@ -192,25 +192,6 @@ struct ConvTransposeAttributes : public ConvAttributes {
TensorShapeVector output_shape;
private:
int64_t ComputeTotalPad(int64_t in_size, int64_t stride, int64_t adj,
int64_t kernel, int64_t dilation, int64_t out_size) const {
return std::max<int64_t>(0, (in_size - 1) * stride + adj + (kernel - 1) * dilation + 1 - out_size);
}
void DistributePadding(AutoPadType pad_type, const int64_t& total_pad,
int64_t& pad_head, int64_t& pad_tail) const {
if (pad_type == AutoPadType::SAME_UPPER) {
// pad more on tail when total_pad is odd.
pad_head = total_pad / 2;
pad_tail = total_pad - total_pad / 2;
} else {
// When pad_type is NOTSET, SAME_LOWER or VALID,
// pad more on head when total_pad is odd.
pad_head = total_pad - total_pad / 2;
pad_tail = total_pad / 2;
}
}
void ComputeTransposePadAndOutputShape(
const int64_t in_size,
const int64_t stride,

56
onnxruntime/core/providers/qnn/builder/opbuilder/base_op_builder.h
View file

@ -105,7 +105,6 @@ class BaseOpBuilder : public IOpBuilder {
QnnQuantParamsWrapper& quant_param) const ORT_MUST_USE_RESULT;
static const std::string& GetQnnOpType(const std::string& onnx_op_type) {
// TODO: Use QNN operator names defined in "QnnOpDef.h"
static const std::unordered_map<std::string, std::string> onnx_op_type_to_qnn_op_type = {
{"Add", QNN_OP_ELEMENT_WISE_ADD},
{"Mul", QNN_OP_ELEMENT_WISE_MULTIPLY},
@ -218,22 +217,40 @@ class BaseOpBuilder : public IOpBuilder {
// NCHW shape to HWCN shape, required for Conv weight
Status NchwShapeToHwcn(const std::vector<uint32_t>& nchw_shape, std::vector<uint32_t>& hwcn_shape) const {
ORT_ENFORCE(nchw_shape.size() == 4, "shape should have 4 dimension NCHW.");
hwcn_shape[0] = nchw_shape[2];
hwcn_shape[1] = nchw_shape[3];
hwcn_shape[2] = nchw_shape[1];
hwcn_shape[3] = nchw_shape[0];
if (nchw_shape.size() == 4) {
hwcn_shape[0] = nchw_shape[2];
hwcn_shape[1] = nchw_shape[3];
hwcn_shape[2] = nchw_shape[1];
hwcn_shape[3] = nchw_shape[0];
} else if (nchw_shape.size() == 5) {
hwcn_shape[0] = nchw_shape[2];
hwcn_shape[1] = nchw_shape[3];
hwcn_shape[2] = nchw_shape[4];
hwcn_shape[3] = nchw_shape[1];
hwcn_shape[4] = nchw_shape[0];
} else {
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Unsupported rank! only support 4 or 5.");
}
return Status::OK();
}
// CNHW shape to HWCN shape, required for Conv weight
Status CnhwShapeToHwcn(const std::vector<uint32_t>& cnhw_shape, std::vector<uint32_t>& hwcn_shape) const {
ORT_ENFORCE(cnhw_shape.size() == 4, "shape should have 4 dimension CNHW.");
hwcn_shape[0] = cnhw_shape[2];
hwcn_shape[1] = cnhw_shape[3];
hwcn_shape[2] = cnhw_shape[0];
hwcn_shape[3] = cnhw_shape[1];
if (cnhw_shape.size() == 4) {
hwcn_shape[0] = cnhw_shape[2];
hwcn_shape[1] = cnhw_shape[3];
hwcn_shape[2] = cnhw_shape[0];
hwcn_shape[3] = cnhw_shape[1];
} else if (cnhw_shape.size() == 5) {
hwcn_shape[0] = cnhw_shape[2];
hwcn_shape[1] = cnhw_shape[3];
hwcn_shape[2] = cnhw_shape[4];
hwcn_shape[3] = cnhw_shape[0];
hwcn_shape[4] = cnhw_shape[1];
} else {
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Unsupported rank! only support 4 or 5.");
}
return Status::OK();
}
@ -244,14 +261,18 @@ class BaseOpBuilder : public IOpBuilder {
Status TransposeFromNchwToHwcn(const QnnModelWrapper& qnn_model_wrapper,
const onnx::TensorProto& initializer,
std::vector<uint8_t>& transposed_data) const {
return TransposeInitializer(qnn_model_wrapper, initializer, nchw2hwcn_perm, transposed_data);
std::vector<uint8_t>& transposed_data,
bool is_3d = false) const {
auto& perm = is_3d ? nchw2hwcn_perm_3d : nchw2hwcn_perm;
return TransposeInitializer(qnn_model_wrapper, initializer, perm, transposed_data);
}
Status TransposeFromCnhwToHwcn(const QnnModelWrapper& qnn_model_wrapper,
const onnx::TensorProto& initializer,
std::vector<uint8_t>& transposed_data) const {
return TransposeInitializer(qnn_model_wrapper, initializer, cnhw2hwcn_perm, transposed_data);
std::vector<uint8_t>& transposed_data,
bool is_3d = false) const {
auto& perm = is_3d ? cnhw2hwcn_perm_3d : cnhw2hwcn_perm;
return TransposeInitializer(qnn_model_wrapper, initializer, perm, transposed_data);
}
Status TwoDimensionTranspose(const QnnModelWrapper& qnn_model_wrapper,
@ -311,11 +332,6 @@ class BaseOpBuilder : public IOpBuilder {
private:
std::string op_builder_type_;
protected:
const std::vector<size_t> nchw2nhwc_perm{0, 2, 3, 1};
const std::vector<size_t> nchw2hwcn_perm{2, 3, 1, 0};
const std::vector<size_t> cnhw2hwcn_perm{2, 3, 0, 1};
};
// Type that holds information about an ONNX attribute.

245
onnxruntime/core/providers/qnn/builder/opbuilder/conv_op_builder.cc
View file

@ -53,11 +53,11 @@ class ConvOpBuilder : public BaseOpBuilder {
const logging::Logger& logger,
std::vector<std::string>& input_names,
bool do_op_validation) const ORT_MUST_USE_RESULT;
Status ProcessConv2DInputs(QnnModelWrapper& qnn_model_wrapper,
const NodeUnit& node_unit,
const logging::Logger& logger,
std::vector<std::string>& input_names,
bool do_op_validation) const ORT_MUST_USE_RESULT;
Status ProcessConv2D3DInputs(QnnModelWrapper& qnn_model_wrapper,
const NodeUnit& node_unit,
const logging::Logger& logger,
std::vector<std::string>& input_names,
bool do_op_validation) const ORT_MUST_USE_RESULT;
Status ProcessAttributesAndOutputs(QnnModelWrapper& qnn_model_wrapper,
const NodeUnit& node_unit,
std::vector<std::string>&& input_names,
@ -87,8 +87,8 @@ Status ConvOpBuilder::IsOpSupported(QnnModelWrapper& qnn_model_wrapper,
const auto& input_0 = inputs[0];
std::vector<uint32_t> input_shape;
ORT_RETURN_IF_NOT(qnn_model_wrapper.GetOnnxShape(input_0.node_arg, input_shape), "Cannot get shape");
if (input_shape.size() != 4 && input_shape.size() != 3) {
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "QNN Conv only supports 2D (rank 4) or 1D (rank 3) inputs.");
if (input_shape.size() != 5 && input_shape.size() != 4 && input_shape.size() != 3) {
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "QNN Conv only supports 3D(rank 5), 2D (rank 4) or 1D (rank 3) inputs.");
}
ONNX_NAMESPACE::DataType input_data_type = input_0.node_arg.Type();
@ -160,20 +160,20 @@ Status ConvOpBuilder::ProcessInputs(QnnModelWrapper& qnn_model_wrapper,
ORT_RETURN_IF_NOT(qnn_model_wrapper.GetOnnxShape(inputs[0].node_arg, input0_shape),
"QNN EP: Cannot get shape for first input");
const bool is_1d_conv = input0_shape.size() == 3;
if (is_1d_conv) {
if (input0_shape.size() == 3) {
return ProcessConv1DInputs(qnn_model_wrapper, node_unit, logger, input_names, do_op_validation);
} else if (input0_shape.size() == 4 || input0_shape.size() == 5) {
return ProcessConv2D3DInputs(qnn_model_wrapper, node_unit, logger, input_names, do_op_validation);
}
return ProcessConv2DInputs(qnn_model_wrapper, node_unit, logger, input_names, do_op_validation);
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "QNN Conv only supports 3D(rank 5), 2D (rank 4) or 1D (rank 3) inputs.");
}
Status ConvOpBuilder::ProcessConv2DInputs(QnnModelWrapper& qnn_model_wrapper,
const NodeUnit& node_unit,
const logging::Logger& logger,
std::vector<std::string>& input_names,
bool do_op_validation) const {
Status ConvOpBuilder::ProcessConv2D3DInputs(QnnModelWrapper& qnn_model_wrapper,
const NodeUnit& node_unit,
const logging::Logger& logger,
std::vector<std::string>& input_names,
bool do_op_validation) const {
const auto& inputs = node_unit.Inputs();
const size_t num_inputs = inputs.size();
OnnxConvType conv_type = {};
@ -209,20 +209,27 @@ Status ConvOpBuilder::ProcessConv2DInputs(QnnModelWrapper& qnn_model_wrapper,
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "QNN EP: Unexpected convolution op type: ", node_unit.OpType().c_str());
}
bool is_3d = (input_info.shape.size() == 5);
std::vector<uint8_t> unpacked_tensor;
if (input_info.is_initializer) {
// Get transposed initializer bytes.
if (conv_type == OnnxConvType::kConv) {
ORT_RETURN_IF_ERROR(TransposeFromNchwToHwcn(qnn_model_wrapper, *input_info.initializer_tensor, unpacked_tensor));
ORT_RETURN_IF_ERROR(TransposeFromNchwToHwcn(qnn_model_wrapper, *input_info.initializer_tensor, unpacked_tensor, is_3d));
} else if (conv_type == OnnxConvType::kConvTranspose) {
ORT_RETURN_IF_ERROR(TransposeFromCnhwToHwcn(qnn_model_wrapper, *input_info.initializer_tensor, unpacked_tensor));
ORT_RETURN_IF_ERROR(TransposeFromCnhwToHwcn(qnn_model_wrapper, *input_info.initializer_tensor, unpacked_tensor, is_3d));
} else {
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "QNN EP: Unexpected convolution op type: ", node_unit.OpType().c_str());
}
// Transpose quantization parameter's axis if this is using per-channel quantization.
if (input_info.quant_param.IsPerChannel()) {
const std::vector<size_t>& perm = conv_type == OnnxConvType::kConv ? nchw2hwcn_perm : cnhw2hwcn_perm;
std::vector<size_t> perm;
if (is_3d) {
perm = conv_type == OnnxConvType::kConv ? nchw2hwcn_perm_3d : cnhw2hwcn_perm_3d;
} else {
perm = conv_type == OnnxConvType::kConv ? nchw2hwcn_perm : cnhw2hwcn_perm;
}
std::vector<size_t> perm_inv(perm.size());
ORT_RETURN_IF_ERROR(utils::InvertPerm<size_t>(perm, perm_inv));
ORT_RETURN_IF_ERROR(input_info.quant_param.HandleTranspose<size_t>(perm_inv));
@ -243,7 +250,9 @@ Status ConvOpBuilder::ProcessConv2DInputs(QnnModelWrapper& qnn_model_wrapper,
input_info.qnn_data_type,
input_info.quant_param,
do_op_validation,
is_graph_input));
is_graph_input,
false,
is_3d));
} else if (conv_type == OnnxConvType::kConvTranspose) {
ORT_RETURN_IF_ERROR(qnn_model_wrapper.AddCnhwToHwcnTranspose(node_unit.Index(),
input1_name,
@ -253,7 +262,9 @@ Status ConvOpBuilder::ProcessConv2DInputs(QnnModelWrapper& qnn_model_wrapper,
input_info.qnn_data_type,
input_info.quant_param,
do_op_validation,
is_graph_input));
is_graph_input,
false,
is_3d));
} else {
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "QNN EP: Unexpected convolution op type: ", node_unit.OpType().c_str());
}
@ -484,80 +495,48 @@ Status ConvOpBuilder::ProcessConv1DInputs(QnnModelWrapper& qnn_model_wrapper,
return Status::OK();
}
static Status GetAutoPadding(std::vector<uint32_t>& pads, const std::string& auto_pad, OnnxConvType conv_type,
const std::array<uint32_t, 2>& strides, const std::array<uint32_t, 2>& dilations,
const std::array<uint32_t, 2>& input_dims, const std::array<uint32_t, 2>& filter_dims,
const std::array<uint32_t, 2>& output_dims, const std::array<uint32_t, 2>& output_padding) {
constexpr size_t HEIGHT_IDX = 0;
constexpr size_t WIDTH_IDX = 1;
std::array<uint32_t, 2> total_padding = {};
if (conv_type == OnnxConvType::kConv) {
// dilated_filter_height = (shape(in[1])[height] - 1) * dilation[0] + 1
// height_out = floor((pad_amount[0,0] + shape(in[0])[height] + pad_amount[0,1] - dilated_filter_height) / stride[0] + 1)
//
// Set total_height_padding equal to pad_amount[0,0] + pad_amount[0,1] and solve for it.
uint32_t dilated_filter_height = (filter_dims[HEIGHT_IDX] - 1) * dilations[HEIGHT_IDX] + 1;
total_padding[HEIGHT_IDX] = (output_dims[HEIGHT_IDX] - 1) * strides[HEIGHT_IDX] + dilated_filter_height - input_dims[HEIGHT_IDX]; // Total height padding
// dilated_filter_width = (shape(in[1])[width] - 1) * dilation[1] + 1
// width_out = floor((pad_amount[1,0] + shape(in[0])[width] + pad_amount[1,1] - dilated_filter_width) / stride[1] + 1)
//
// Set total_width_padding equal to pad_amount[1,0] + pad_amount[1,1] and solve for it.
uint32_t dilated_filter_width = (filter_dims[WIDTH_IDX] - 1) * dilations[WIDTH_IDX] + 1;
total_padding[WIDTH_IDX] = (output_dims[WIDTH_IDX] - 1) * strides[WIDTH_IDX] + dilated_filter_width - input_dims[WIDTH_IDX]; // Total width padding
} else if (conv_type == OnnxConvType::kConvTranspose) {
// height_out = floor(stride[0] * (shape(in[0])[height] - 1) + shape(in[1])[height] - pad_amount[0,0] - pad_amount[0,1] + output_padding[0])
//
// Set total_height_padding equal to pad_amount[0,0] + pad_amount[0,1] and solve for it.
total_padding[HEIGHT_IDX] = strides[HEIGHT_IDX] * (input_dims[HEIGHT_IDX] - 1) + output_padding[HEIGHT_IDX] + filter_dims[HEIGHT_IDX] - output_dims[HEIGHT_IDX];
// width_out = floor(stride[1] * (shape(in[0])[width] - 1) + shape(in[1])[width] - pad_amount[1,0] - pad_amount[1,1] + output_padding[1])
//
// Set total_width_padding equal to pad_amount[1,0] + pad_amount[1,1] and solve for it.
total_padding[WIDTH_IDX] = strides[WIDTH_IDX] * (input_dims[WIDTH_IDX] - 1) + output_padding[WIDTH_IDX] + filter_dims[WIDTH_IDX] - output_dims[WIDTH_IDX];
} else {
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "QNN EP: Unexpected conv op type when computing auto-padding");
}
pads.resize(4); // Make room.
if (auto_pad == "SAME_UPPER") {
pads[0] = total_padding[0] / 2;
pads[1] = total_padding[1] / 2;
pads[2] = total_padding[0] - pads[0];
pads[3] = total_padding[1] - pads[1];
} else if (auto_pad == "SAME_LOWER") {
pads[2] = total_padding[0] / 2;
pads[3] = total_padding[1] / 2;
pads[0] = total_padding[0] - pads[2];
pads[1] = total_padding[1] - pads[3];
} else {
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "QNN EP: Cannot calculate auto-padding for unsupported auto_pad setting: ",
auto_pad.c_str());
}
return Status::OK();
}
Status ConvOpBuilder::ProcessAttributesAndOutputs(QnnModelWrapper& qnn_model_wrapper,
const NodeUnit& node_unit,
std::vector<std::string>&& input_names,
const logging::Logger& logger,
bool do_op_validation) const {
ORT_UNUSED_PARAMETER(do_op_validation);
const auto& outputs = node_unit.Outputs();
std::vector<uint32_t> output_shape;
ORT_RETURN_IF_NOT(qnn_model_wrapper.GetOnnxShape(outputs[0].node_arg, output_shape), "Cannot get shape");
const bool is_1d_conv = output_shape.size() == 3;
const bool is_3d_conv = output_shape.size() == 5;
OnnxConvType conv_type = {};
ORT_RETURN_IF_ERROR(GetOnnxConvType(node_unit.OpType(), conv_type));
NodeAttrHelper node_helper(node_unit);
std::vector<std::string> param_tensor_names;
const auto& input_0 = node_unit.Inputs()[0];
const auto& input_1 = node_unit.Inputs()[1];
std::vector<uint32_t> input_0_shape; // NHW[D]C
std::vector<uint32_t> input_1_shape; // NCHW[D]
ORT_RETURN_IF_NOT(qnn_model_wrapper.GetOnnxShape(input_0.node_arg, input_0_shape), "Cannot get shape");
ORT_RETURN_IF_NOT(qnn_model_wrapper.GetOnnxShape(input_1.node_arg, input_1_shape), "Cannot get shape");
// Kernel shape
std::vector<uint32_t> kernel_shape;
kernel_shape = node_helper.Get("kernel_shape", kernel_shape);
if (kernel_shape.empty()) { // infer from weight shape
kernel_shape.assign(input_1_shape.begin() + 2, input_1_shape.end());
}
if (is_1d_conv) {
// insert Hight = 1 for 1D
kernel_shape.insert(kernel_shape.begin(), 1);
}
// Dilations parameter
std::vector<uint32_t> dilations = {1, 1};
std::vector<uint32_t> dilations;
dilations.assign(kernel_shape.size(), 1);
if (conv_type == OnnxConvType::kConv) {
dilations = node_helper.Get("dilations", std::vector<uint32_t>{1, 1});
dilations = node_helper.Get("dilations", dilations);
// Handle 1D conv by setting height dilation to 1.
if (dilations.size() == 1) {
@ -574,7 +553,9 @@ Status ConvOpBuilder::ProcessAttributesAndOutputs(QnnModelWrapper& qnn_model_wra
}
// Strides parameter.
auto strides = node_helper.Get("strides", std::vector<uint32_t>{1, 1});
std::vector<uint32_t> strides;
strides.assign(kernel_shape.size(), 1);
strides = node_helper.Get("strides", strides);
{
// Handle 1D conv by setting the height stride to 1.
if (strides.size() == 1) {
@ -591,10 +572,10 @@ Status ConvOpBuilder::ProcessAttributesAndOutputs(QnnModelWrapper& qnn_model_wra
}
// Output padding parameter. (Only for ConvTranspose)
std::vector<uint32_t> output_padding = {0, 0};
std::vector<uint32_t> output_padding;
output_padding.assign(kernel_shape.size(), 0);
if (conv_type == OnnxConvType::kConvTranspose) {
output_padding = node_helper.Get("output_padding", std::vector<uint32_t>{0, 0});
output_padding = node_helper.Get("output_padding", output_padding);
// Handle 1D conv.
if (output_padding.size() == 1) {
@ -610,54 +591,45 @@ Status ConvOpBuilder::ProcessAttributesAndOutputs(QnnModelWrapper& qnn_model_wra
qnn_model_wrapper.AddParamWrapper(std::move(output_padding_paramwrapper));
}
const auto& outputs = node_unit.Outputs();
const auto& output_name = outputs[0].node_arg.Name();
std::vector<uint32_t> output_shape;
ORT_RETURN_IF_NOT(qnn_model_wrapper.GetOnnxShape(outputs[0].node_arg, output_shape), "Cannot get shape");
const bool is_1d_conv = output_shape.size() == 3;
// Pads attribute
{
std::vector<uint32_t> pads = node_helper.Get("pads", std::vector<uint32_t>({0, 0, 0, 0}));
std::vector<uint32_t> pads;
pads.assign(kernel_shape.size() * 2, 0);
pads = node_helper.Get("pads", pads);
auto auto_pad = node_helper.Get("auto_pad", std::string("NOTSET"));
ORT_RETURN_IF(auto_pad != "NOTSET" && auto_pad != "SAME_LOWER" && auto_pad != "SAME_UPPER",
"QNN Conv operators do not support 'auto_pad' value: ", auto_pad.c_str());
if (auto_pad != "NOTSET") {
const auto& input_0 = node_unit.Inputs()[0];
const auto& input_1 = node_unit.Inputs()[1];
std::vector<uint32_t> input_0_shape; // NHWC
std::vector<uint32_t> input_1_shape; // NCHW
ORT_RETURN_IF_NOT(qnn_model_wrapper.GetOnnxShape(input_0.node_arg, input_0_shape), "Cannot get shape");
ORT_RETURN_IF_NOT(qnn_model_wrapper.GetOnnxShape(input_1.node_arg, input_1_shape), "Cannot get shape");
std::array<uint32_t, 2> input_dims = {};
std::array<uint32_t, 2> filter_dims = {};
std::array<uint32_t, 2> output_dims = {};
auto pad_type = StringToAutoPadType(auto_pad);
// skip N, C, input0 shape NHWC
std::vector<uint32_t> input_dims(input_0_shape.begin() + 1, input_0_shape.end() - 1);
std::vector<uint32_t> output_dims(output_shape.begin() + 1, output_shape.end() - 1);
if (is_1d_conv) {
input_dims[0] = 1;
input_dims[1] = input_0_shape[1];
filter_dims[0] = 1;
filter_dims[1] = input_1_shape[2];
output_dims[0] = 1;
output_dims[1] = output_shape[1];
} else {
input_dims[0] = input_0_shape[1];
input_dims[1] = input_0_shape[2];
filter_dims[0] = input_1_shape[2];
filter_dims[1] = input_1_shape[3];
output_dims[0] = output_shape[1];
output_dims[1] = output_shape[2];
// insert Hight = 1 for 1D
input_dims.insert(input_dims.begin(), 1);
output_dims.insert(output_dims.begin(), 1);
}
size_t rank = input_dims.size();
for (size_t dim = 0; dim < rank; ++dim) {
int64_t pad_head = pads[dim];
int64_t pad_tail = pads[rank + dim];
if (conv_type == OnnxConvType::kConv) {
ORT_RETURN_IF_ERROR(onnxruntime::ComputePad(input_dims[dim],
strides[dim],
kernel_shape[dim],
dilations[dim],
pad_type,
pad_head,
pad_tail));
} else if (conv_type == OnnxConvType::kConvTranspose) {
auto total_pad = ComputeTotalPad(input_dims[dim], strides[dim], output_padding[dim],
kernel_shape[dim], dilations[dim], output_dims[dim]);
DistributePadding(pad_type, total_pad, pad_head, pad_tail);
}
pads[dim] = narrow<uint32_t>(pad_head);
pads[rank + dim] = narrow<uint32_t>(pad_tail);
}
ORT_RETURN_IF_ERROR(GetAutoPadding(pads, auto_pad, conv_type, {strides[0], strides[1]}, {dilations[0], dilations[1]},
input_dims, filter_dims, output_dims, {output_padding[0], output_padding[1]}));
} else {
// Handle 1D conv by setting padding for height to 0.
if (pads.size() == 2) {
@ -672,8 +644,9 @@ Status ConvOpBuilder::ProcessAttributesAndOutputs(QnnModelWrapper& qnn_model_wra
}
ReArranagePads(pads);
uint32_t pad_size = narrow<uint32_t>(pads.size() / 2);
QnnParamWrapper pad_amount_paramwrapper(node_unit.Index(), node_unit.Name(), QNN_OP_CONV_2D_PARAM_PAD_AMOUNT,
{2, 2}, std::move(pads));
{pad_size, 2}, std::move(pads));
param_tensor_names.push_back(pad_amount_paramwrapper.GetParamTensorName());
qnn_model_wrapper.AddParamWrapper(std::move(pad_amount_paramwrapper));
}
@ -683,7 +656,9 @@ Status ConvOpBuilder::ProcessAttributesAndOutputs(QnnModelWrapper& qnn_model_wra
uint32_t num_input_channels = 0;
ORT_RETURN_IF_ERROR(GetInputChannelNumber(qnn_model_wrapper, node_unit, num_input_channels));
const bool is_depthwise_conv2d = (conv_type == OnnxConvType::kConv) && (num_input_channels == num_output_channels) &&
// There's DepthWiseConv2d, but no DepthWiseConv3d
const bool is_depthwise_conv2d = (!is_3d_conv) && (conv_type == OnnxConvType::kConv) &&
(num_input_channels == num_output_channels) &&
(group == num_output_channels);
if (!is_depthwise_conv2d) { // DepthWiseConv2d does not need a group parameter.
@ -697,7 +672,16 @@ Status ConvOpBuilder::ProcessAttributesAndOutputs(QnnModelWrapper& qnn_model_wra
LOGS(logger, VERBOSE) << "Using DepthWiseConv2d instead of Conv2d for node " << node_unit.Name();
}
const std::string& output_node_type = is_depthwise_conv2d ? QNN_OP_DEPTH_WISE_CONV_2D : GetQnnOpType(node_unit.OpType());
std::string output_node_type;
if (is_3d_conv) {
if (conv_type == OnnxConvType::kConv) {
output_node_type = QNN_OP_CONV_3D;
} else {
output_node_type = QNN_OP_TRANSPOSE_CONV_3D;
}
} else {
output_node_type = is_depthwise_conv2d ? QNN_OP_DEPTH_WISE_CONV_2D : GetQnnOpType(node_unit.OpType());
}
QnnQuantParamsWrapper output_quantize_param;
ORT_RETURN_IF_ERROR(output_quantize_param.Init(qnn_model_wrapper, outputs[0]));
@ -707,6 +691,7 @@ Status ConvOpBuilder::ProcessAttributesAndOutputs(QnnModelWrapper& qnn_model_wra
Qnn_DataType_t qnn_data_type = QNN_DATATYPE_FLOAT_32;
ORT_RETURN_IF_ERROR(utils::GetQnnDataType(is_quantized_tensor, type_proto, qnn_data_type));
const auto& output_name = outputs[0].node_arg.Name();
if (is_1d_conv) {
const bool is_graph_output = qnn_model_wrapper.IsGraphOutput(output_name);
std::vector<uint32_t> output_shape_2d = {
@ -724,7 +709,8 @@ Status ConvOpBuilder::ProcessAttributesAndOutputs(QnnModelWrapper& qnn_model_wra
output_node_type,
std::move(input_names),
{conv_output_name},
std::move(param_tensor_names)),
std::move(param_tensor_names),
do_op_validation),
"Failed to add node.");
// Add Reshape to convert QNN Conv2d/TransposeConv2d/DepthWiseConv2d output back to 1D.
@ -748,7 +734,8 @@ Status ConvOpBuilder::ProcessAttributesAndOutputs(QnnModelWrapper& qnn_model_wra
output_node_type,
std::move(input_names),
{output_name},
std::move(param_tensor_names)),
std::move(param_tensor_names),
do_op_validation),
"Failed to add node.");
}

11
onnxruntime/core/providers/qnn/builder/opbuilder/simple_op_builder.cc
View file

@ -323,17 +323,6 @@ Status SimpleOpBuilder::ProcessAttributesAndOutputs(QnnModelWrapper& qnn_model_w
if (node_unit.Domain() != kMSInternalNHWCDomain && (op_type == "DepthToSpace" || op_type == "SpaceToDepth" || op_type == "GridSample")) {
return Status::OK();
}
// Explicitly skip the Op validation for Q & DQ node with 5D because of QNN bug.
// TODO (hecli), remove once QNN v2.17 is ready
if (op_type == "QuantizeLinear" || op_type == "DequantizeLinear") {
std::vector<uint32_t> input_shape;
ORT_RETURN_IF_NOT(qnn_model_wrapper.GetOnnxShape(node_unit.Inputs()[0].node_arg, input_shape),
"QNN EP: Cannot get input shape");
if (input_shape.size() == 5) {
return Status::OK();
}
}
}
std::vector<std::string> param_tensor_names;

5
onnxruntime/core/providers/qnn/builder/qnn_def.h
View file

@ -497,5 +497,10 @@ typedef struct GraphConfigInfo {
const QnnGraph_Config_t** graphConfigs;
} GraphConfigInfo_t;
static const std::vector<size_t> nchw2hwcn_perm{2, 3, 1, 0};
static const std::vector<size_t> nchw2hwcn_perm_3d{2, 3, 4, 1, 0};
static const std::vector<size_t> cnhw2hwcn_perm{2, 3, 0, 1};
static const std::vector<size_t> cnhw2hwcn_perm_3d{2, 3, 4, 0, 1};
} // namespace qnn
} // namespace onnxruntime

26
onnxruntime/core/providers/qnn/builder/qnn_model_wrapper.h
View file

@ -165,10 +165,18 @@ class QnnModelWrapper {
const QnnQuantParamsWrapper& quantize_param,
bool do_op_validation,
bool is_for_input = true,
bool is_for_output = false) {
bool is_for_output = false,
bool is_3d = false) {
LOGS(logger_, VERBOSE) << "Add NCHW->HWCN Transpose node after Conv weight input: " << input_name
<< " -> " << output_name;
return AddTransposeNode(node_index, input_name, output_name, input_shape, nchw2hwcn_perm_, output_shape,
auto perm = is_3d ? nchw2hwcn_perm_3d : nchw2hwcn_perm;
std::vector<uint32_t> transpose_perm;
transpose_perm.resize(perm.size());
std::transform(perm.begin(), perm.end(),
transpose_perm.begin(), [](size_t item) -> uint32_t {
return narrow<uint32_t>(item);
});
return AddTransposeNode(node_index, input_name, output_name, input_shape, transpose_perm, output_shape,
tensor_data_type, quantize_param, do_op_validation, is_for_input, is_for_output);
}
@ -182,10 +190,18 @@ class QnnModelWrapper {
const QnnQuantParamsWrapper& quantize_param,
bool do_op_validation,
bool is_for_input = true,
bool is_for_output = false) {
bool is_for_output = false,
bool is_3d = false) {
LOGS(logger_, VERBOSE) << "Add CNHW->HWCN Transpose node after ConvTranspose weight input: " << input_name
<< " -> " << output_name;
return AddTransposeNode(node_index, input_name, output_name, input_shape, cnhw2hwcn_perm_, output_shape,
auto perm = is_3d ? cnhw2hwcn_perm_3d : cnhw2hwcn_perm;
std::vector<uint32_t> transpose_perm;
transpose_perm.resize(perm.size());
std::transform(perm.begin(), perm.end(),
transpose_perm.begin(), [](size_t item) -> uint32_t {
return narrow<uint32_t>(item);
});
return AddTransposeNode(node_index, input_name, output_name, input_shape, transpose_perm, output_shape,
tensor_data_type, quantize_param, do_op_validation, is_for_input, is_for_output);
}
@ -258,8 +274,6 @@ class QnnModelWrapper {
const std::unordered_map<std::string, size_t>& input_index_map_;
const std::unordered_map<std::string, size_t>& output_index_map_;
const std::unordered_set<std::string>& initializer_lookup_;
const std::vector<uint32_t> nchw2hwcn_perm_{2, 3, 1, 0};
const std::vector<uint32_t> cnhw2hwcn_perm_{2, 3, 0, 1};
QnnBackendType qnn_backend_type_ = QnnBackendType::CPU;
}; // QnnModelWrapper

6
onnxruntime/core/providers/qnn/builder/qnn_utils.cc
View file

@ -328,6 +328,12 @@ std::ostream& operator<<(std::ostream& out, const Qnn_Tensor_t& tensor) {
} else if (GetQnnTensorDataType(tensor) == QNN_DATATYPE_INT_32 ||
GetQnnTensorDataType(tensor) == QNN_DATATYPE_SFIXED_POINT_32) {
operator<< <int32_t>(out, GetQnnTensorClientBuf(tensor));
} else if (GetQnnTensorDataType(tensor) == QNN_DATATYPE_UINT_16 ||
GetQnnTensorDataType(tensor) == QNN_DATATYPE_UFIXED_POINT_16) {
operator<< <uint16_t>(out, GetQnnTensorClientBuf(tensor));
} else if (GetQnnTensorDataType(tensor) == QNN_DATATYPE_INT_16 ||
GetQnnTensorDataType(tensor) == QNN_DATATYPE_SFIXED_POINT_16) {
operator<< <int16_t>(out, GetQnnTensorClientBuf(tensor));
} else if (GetQnnTensorDataType(tensor) == QNN_DATATYPE_UINT_8 ||
GetQnnTensorDataType(tensor) == QNN_DATATYPE_UFIXED_POINT_8) {
operator<< <uint8_t>(out, GetQnnTensorClientBuf(tensor));

3
onnxruntime/test/onnx/main.cc
View file

@ -812,7 +812,8 @@ select from 'TF8', 'TF16', 'UINT8', 'FLOAT', 'ITENSOR'. \n)");
ORT_TSTR("sce_NCd1d2d3_sum_weight_high_ii"),
ORT_TSTR("sce_NCd1d2d3_sum_weight_high_ii_expanded"),
ORT_TSTR("sce_none_weights_log_prob_expanded"),
ORT_TSTR("sce_none_weights_expanded")};
ORT_TSTR("sce_none_weights_expanded"),
ORT_TSTR("convtranspose_3d")};
std::unordered_set<std::basic_string<ORTCHAR_T>> all_disabled_tests(std::begin(immutable_broken_tests), std::end(immutable_broken_tests));

510
onnxruntime/test/providers/qnn/conv_test.cc
View file

@ -322,6 +322,17 @@ TEST_F(QnnCPUBackendTests, DISABLED_Convf32_dynamic_bias) {
1, // default group
"NOTSET", // No auto-padding
ExpectedEPNodeAssignment::All);
RunCPUConvOpTest("Conv",
TestInputDef<float>({1, 1, 3, 3, 3}, false, 0.0f, 10.0f), // Random dynamic input
TestInputDef<float>({2, 1, 2, 2, 2}, true, 0.0f, 1.0f), // Random static weights
TestInputDef<float>({2}, false, -1.0f, 1.0f), // Random dynamic bias
{1, 1, 1}, // default strides
{0, 0, 0, 0, 0, 0}, // default pads
{1, 1, 1}, // default dilations
1, // default group
"NOTSET", // No auto-padding
ExpectedEPNodeAssignment::All);
}
// Check that QNN compiles DQ -> Conv -> Q as a single unit.
@ -337,6 +348,17 @@ TEST_F(QnnCPUBackendTests, Convf32_bias_initializer) {
1, // default group
"NOTSET", // No auto-padding
ExpectedEPNodeAssignment::All);
RunCPUConvOpTest("Conv",
TestInputDef<float>({1, 1, 3, 3, 3}, false, 0.0f, 10.0f), // Random dynamic input
TestInputDef<float>({2, 1, 2, 2, 2}, true, 0.0f, 1.0f), // Random static weights
TestInputDef<float>({2}, true, -1.0f, 1.0f), // Random static bias
{1, 1, 1}, // default strides
{0, 0, 0, 0, 0, 0}, // default pads
{1, 1, 1}, // default dilations
1, // default group
"NOTSET", // No auto-padding
ExpectedEPNodeAssignment::All);
}
// Tests Conv's auto_pad value "SAME_UPPER" (compares to CPU EP).
@ -351,6 +373,17 @@ TEST_F(QnnCPUBackendTests, Convf32_AutoPadUpper) {
1, // default group
"SAME_UPPER", // auto_pad
ExpectedEPNodeAssignment::All);
RunCPUConvOpTest("Conv",
TestInputDef<float>({1, 1, 3, 3, 3}, false, -3.0f, 3.0f), // Random dynamic input
TestInputDef<float>({2, 1, 2, 2, 2}, true, -1.0f, 1.0f), // Random static weights
TestInputDef<float>({2}, true, -1.0f, 1.0f), // Random static bias
{1, 1, 1}, // strides
{}, // pads
{1, 1, 1}, // dilations
1, // default group
"SAME_UPPER", // auto_pad
ExpectedEPNodeAssignment::All);
}
// Tests ConvTranspose's auto_pad value "SAME_UPPER" (compares to CPU EP).
@ -365,6 +398,17 @@ TEST_F(QnnCPUBackendTests, ConvTransposef32_AutoPadUpper) {
1, // default group
"SAME_UPPER", // auto_pad
ExpectedEPNodeAssignment::All);
RunCPUConvOpTest("ConvTranspose",
TestInputDef<float>({1, 1, 3, 3, 3}, false, -3.0f, 3.0f), // Random dynamic input
TestInputDef<float>({1, 2, 2, 2, 2}, true, -1.0f, 1.0f), // Random static weights
TestInputDef<float>({2}, true, -1.0f, 1.0f), // Random static bias
{1, 1, 1}, // strides
{}, // pads
{1, 1, 1}, // dilations
1, // default group
"SAME_UPPER", // auto_pad
ExpectedEPNodeAssignment::All);
}
// Tests Conv's auto_pad value "SAME_LOWER" (compares to CPU EP).
@ -379,6 +423,17 @@ TEST_F(QnnCPUBackendTests, Convf32_AutoPadLower) {
1, // default group
"SAME_LOWER", // auto_pad
ExpectedEPNodeAssignment::All);
RunCPUConvOpTest("Conv",
TestInputDef<float>({1, 1, 3, 3, 3}, false, -3.0f, 3.0f), // Random dynamic input
TestInputDef<float>({2, 1, 2, 2, 2}, false, -1.0f, 1.0f), // Random dynamic weights
TestInputDef<float>({2}, true, -1.0f, 1.0f), // Random static bias
{1, 1, 1}, // strides
{}, // pads
{1, 1, 1}, // dilations
1, // default group
"SAME_LOWER", // auto_pad
ExpectedEPNodeAssignment::All);
}
// Tests ConvTranspose's auto_pad value "SAME_LOWER" (compares to CPU EP).
@ -395,6 +450,23 @@ TEST_F(QnnCPUBackendTests, ConvTransposef32_AutoPadLower) {
ExpectedEPNodeAssignment::All);
}
// Tests ConvTranspose's auto_pad value "SAME_LOWER" (compares to CPU EP).
// Exception from graphFinalize
// Exception thrown at 0x00007FFFB7651630 (QnnCpu.dll) in onnxruntime_test_all.exe:
// 0xC0000005: Access violation reading location 0x0000000000000000.
TEST_F(QnnCPUBackendTests, DISABLED_ConvTranspose3D_f32_AutoPadLower) {
RunCPUConvOpTest("ConvTranspose",
TestInputDef<float>({1, 1, 3, 3, 3}, false, -3.0f, 3.0f), // Random dynamic input
TestInputDef<float>({1, 2, 2, 2, 2}, false, -1.0f, 1.0f), // Random dynamic weights
TestInputDef<float>({2}, true, -1.0f, 1.0f), // Random static bias
{1, 1, 1}, // strides
{}, // pads
{1, 1, 1}, // dilations
1, // default group
"SAME_LOWER", // auto_pad
ExpectedEPNodeAssignment::All);
}
// large input,output, pads
TEST_F(QnnCPUBackendTests, Convf32_large_input1_pad_bias_initializer) {
RunCPUConvOpTest("Conv",
@ -409,6 +481,19 @@ TEST_F(QnnCPUBackendTests, Convf32_large_input1_pad_bias_initializer) {
ExpectedEPNodeAssignment::All,
13,
1e-4f);
RunCPUConvOpTest("Conv",
TestInputDef<float>({1, 3, 60, 452, 20}, false, 0.0f, 10.0f), // Random dynamic input
TestInputDef<float>({16, 3, 3, 3, 3}, true, 0.0f, 1.0f), // Random dynamic weights
TestInputDef<float>({16}, true, -1.0f, 1.0f), // Random static bias
{1, 1, 1},
{1, 1, 1, 1, 1, 1},
{1, 1, 1},
1, // default group
"NOTSET",
ExpectedEPNodeAssignment::All,
13,
2e-4f);
}
TEST_F(QnnCPUBackendTests, Convf32_large_input2_nopad_bias_initializer) {
@ -432,6 +517,19 @@ TEST_F(QnnCPUBackendTests, Convf32_large_input2_nopad_bias_initializer) {
ExpectedEPNodeAssignment::All,
13, // opset
fp32_abs_err);
RunCPUConvOpTest("Conv",
TestInputDef<float>({1, 32, 16, 113, 12}, false, -3.0f, 3.0f), // Random dynamic input
TestInputDef<float>({16, 32, 1, 1, 1}, false, -1.0f, 1.0f), // Random dynamic weights
TestInputDef<float>({16}, true, -1.0f, 1.0f), // Random static bias
{1, 1, 1},
{0, 0, 0, 0, 0, 0},
{1, 1, 1},
1, // default group
"NOTSET",
ExpectedEPNodeAssignment::All,
13, // opset
fp32_abs_err);
}
// Test 1D Conv with static weights (implemented in QNN EP as 2D convolution with height of 1).
@ -581,6 +679,21 @@ TEST_F(QnnHTPBackendTests, ConvU8U8S32_bias_dynamic_input) {
13, // opset
// Need tolerance of 0.413% of output range after QNN SDK 2.17
QDQTolerance(0.00413f));
RunHTPConvOpTest<uint8_t, uint8_t>("Conv",
TestInputDef<float>({1, 1, 5, 5, 5}, false, 0.0f, 10.0f), // Random dynamic input
TestInputDef<float>({1, 1, 3, 3, 3}, true, -10.0f, 10.0f), // Random static input
TestInputDef<float>({1}, false, {2.0f}), // Dynamic bias
{1, 1, 1}, // Strides
{0, 0, 0, 0, 0, 0}, // Pads
{1, 1, 1}, // Dilations
1, // default group
"NOTSET",
ExpectedEPNodeAssignment::All,
false, // use_qdq_contrib_ops
13, // opset
// Need tolerance of 0.413% of output range after QNN SDK 2.17
QDQTolerance(0.00413f));
}
// Test per-channel QDQ Conv. in0: u8, in1 (weight): s8, in2 (bias): s32, out: u8
@ -610,6 +723,41 @@ TEST_F(QnnHTPBackendTests, ConvU8S8S32_PerChannel) {
13); // opset
}
// Test per-channel QDQ Conv. in0: u8, in1 (weight): s8, in2 (bias): s32, out: u8
// \QNN\HTP\HTP\src\hexagon\prepare\graph_prepare.cc:203:ERROR:could not create op: q::QNN_Conv3d_w_scale
// \QNN\HTP\HTP\src\hexagon\prepare\graph_prepare.cc:1187:ERROR:Op 0x1a preparation failed with err:-1
// QnnDsp <E> "Conv" generated: could not create op
// QnnDsp <E> RouterWindows graph prepare failed 12
// QnnDsp <E> Failed to finalize graph (id: 1) with err 1002
// QnnDsp <V> Wake up free backend 1 thread(s)
// QnnDsp <I> QnnGraph_finalize done. status 0x3ea
// onnxruntime::qnn::QnnModel::FinalizeGraphs] Failed to finalize QNN graph.
TEST_F(QnnHTPBackendTests, DISABLED_Conv3D_U8S8S32_PerChannel) {
std::vector<int64_t> input_shape = {1, 2, 4, 4, 4};
std::vector<int64_t> weight_shape = {3, 2, 2, 2, 2};
std::vector<int64_t> bias_shape = {3};
TestInputDef<float> input_def(input_shape, false,
GetFloatDataInRange(-10.0f, 10.0f, TensorShape(input_shape).Size()));
TestInputDef<float> weight_def(weight_shape, true,
GetFloatDataInRange(-1.0f, 5.0f, TensorShape(weight_shape).Size()));
TestInputDef<float> bias_def(bias_shape, true,
GetFloatDataInRange(-1.0f, 1.0f, TensorShape(bias_shape).Size()));
RunHTPConvOpPerChannelTest<uint8_t, int8_t>("Conv",
input_def,
weight_def,
bias_def,
{1, 1, 1}, // Strides
{0, 0, 0, 0, 0, 0}, // Pads
{1, 1, 1}, // Dilations
1, // default group
"NOTSET",
ExpectedEPNodeAssignment::All,
false,
13);
}
// Test per-channel QDQ Conv that maps to QNN's DepthwiseConv2d (input_chans == output_chans == group).
// in0: u8, in1 (weight): s8, in2 (bias): s32, out: u8
TEST_F(QnnHTPBackendTests, ConvDepthwiseU8S8S32_PerChannel) {
@ -638,6 +786,41 @@ TEST_F(QnnHTPBackendTests, ConvDepthwiseU8S8S32_PerChannel) {
13); // opset
}
// Conv3D per-channel
// \QNN\HTP\HTP\src\hexagon\prepare\graph_prepare.cc:203:ERROR:could not create op: q::QNN_Conv3d_w_scale
// \QNN\HTP\HTP\src\hexagon\prepare\graph_prepare.cc:1187:ERROR:Op 0x1a preparation failed with err:-1
// QnnDsp <E> "Conv" generated: could not create op
// QnnDsp <E> RouterWindows graph prepare failed 12
// QnnDsp <E> Failed to finalize graph (id: 1) with err 1002
// QnnDsp <V> Wake up free backend 1 thread(s)
// QnnDsp <I> QnnGraph_finalize done. status 0x3ea
// onnxruntime::qnn::QnnModel::FinalizeGraphs] Failed to finalize QNN graph.
TEST_F(QnnHTPBackendTests, DISABLED_Conv3D_U8S8S32_PerChannel2) {
std::vector<int64_t> input_shape = {1, 2, 4, 4, 4};
std::vector<int64_t> weight_shape = {2, 1, 2, 2, 2};
std::vector<int64_t> bias_shape = {2};
TestInputDef<float> input_def(input_shape, false,
GetFloatDataInRange(-10.0f, 10.0f, TensorShape(input_shape).Size()));
TestInputDef<float> weight_def(weight_shape, true,
GetFloatDataInRange(-1.0f, 5.0f, TensorShape(weight_shape).Size()));
TestInputDef<float> bias_def(bias_shape, true,
GetFloatDataInRange(-1.0f, 1.0f, TensorShape(bias_shape).Size()));
RunHTPConvOpPerChannelTest<uint8_t, int8_t>("Conv",
input_def,
weight_def,
bias_def,
{1, 1, 1}, // Strides
{0, 0, 0, 0, 0, 0}, // Pads
{1, 1, 1}, // Dilations
2, // group
"NOTSET",
ExpectedEPNodeAssignment::All,
false,
13);
}
// Test per-channel QDQ ConvTranspose. in0: u8, in1 (weight): s8, in2 (bias): s32, out: u8
TEST_F(QnnHTPBackendTests, ConvTransposeU8S8S32_PerChannel) {
std::vector<int64_t> input_shape = {1, 2, 4, 4};
@ -665,6 +848,34 @@ TEST_F(QnnHTPBackendTests, ConvTransposeU8S8S32_PerChannel) {
13); // opset
}
// ConvTranspose3D per-channel
// Disable it for 2.21 since it failed, re-enabled it for 2.22
TEST_F(QnnHTPBackendTests, DISABLED_ConvTranspose3D_U8S8S32_PerChannel) {
std::vector<int64_t> input_shape = {1, 2, 4, 4, 4};
std::vector<int64_t> weight_shape = {2, 3, 2, 2, 2};
std::vector<int64_t> bias_shape = {3};
TestInputDef<float> input_def(input_shape, false,
GetFloatDataInRange(-10.0f, 10.0f, TensorShape(input_shape).Size()));
TestInputDef<float> weight_def(weight_shape, true,
GetFloatDataInRange(-1.0f, 5.0f, TensorShape(weight_shape).Size()));
TestInputDef<float> bias_def(bias_shape, true,
GetFloatDataInRange(-1.0f, 1.0f, TensorShape(bias_shape).Size()));
RunHTPConvOpPerChannelTest<uint8_t, int8_t>("ConvTranspose",
input_def,
weight_def,
bias_def,
{1, 1, 1}, // Strides
{0, 0, 0, 0, 0, 0}, // Pads
{1, 1, 1}, // Dilations
1, // default group
"NOTSET",
ExpectedEPNodeAssignment::All,
false,
13);
}
// Test per-channel QDQ Conv. in0: u16, in1 (weight): s8, in2 (bias): s32, out: u16
TEST_F(QnnHTPBackendTests, ConvU16S8S32_PerChannel) {
std::vector<int64_t> input_shape = {1, 2, 4, 4};
@ -692,6 +903,41 @@ TEST_F(QnnHTPBackendTests, ConvU16S8S32_PerChannel) {
13); // opset
}
// Conv3D per-channel
// \QNN\HTP\HTP\src\hexagon\prepare\graph_prepare.cc:203:ERROR:could not create op: q::QNN_Conv3d_w_scale
// \QNN\HTP\HTP\src\hexagon\prepare\graph_prepare.cc:1187:ERROR:Op 0x1a preparation failed with err:-1
// QnnDsp <E> "Conv" generated: could not create op
// QnnDsp <E> RouterWindows graph prepare failed 12
// QnnDsp <E> Failed to finalize graph (id: 1) with err 1002
// QnnDsp <V> Wake up free backend 1 thread(s)
// QnnDsp <I> QnnGraph_finalize done. status 0x3ea
// onnxruntime::qnn::QnnModel::FinalizeGraphs] Failed to finalize QNN graph.
TEST_F(QnnHTPBackendTests, DISABLED_Conv3D_U16S8S32_PerChannel) {
std::vector<int64_t> input_shape = {1, 2, 4, 4, 4};
std::vector<int64_t> weight_shape = {3, 2, 2, 2, 2};
std::vector<int64_t> bias_shape = {3};
TestInputDef<float> input_def(input_shape, false,
GetFloatDataInRange(-10.0f, 10.0f, TensorShape(input_shape).Size()));
TestInputDef<float> weight_def(weight_shape, true,
GetFloatDataInRange(-1.0f, 5.0f, TensorShape(weight_shape).Size()));
TestInputDef<float> bias_def(bias_shape, true,
GetFloatDataInRange(-1.0f, 1.0f, TensorShape(bias_shape).Size()));
RunHTPConvOpPerChannelTest<uint16_t, int8_t>("Conv",
input_def,
weight_def,
bias_def,
{1, 1, 1}, // Strides
{0, 0, 0, 0, 0, 0}, // Pads
{1, 1, 1}, // Dilations
1, // default group
"NOTSET",
ExpectedEPNodeAssignment::All,
true,
13);
}
// Test per-channel QDQ ConvTranspose. in0: u16, in1 (weight): s8, in2 (bias): s32, out: u16
TEST_F(QnnHTPBackendTests, ConvTransposeU16S8S32_PerChannel) {
std::vector<int64_t> input_shape = {1, 2, 4, 4};
@ -719,6 +965,33 @@ TEST_F(QnnHTPBackendTests, ConvTransposeU16S8S32_PerChannel) {
13); // opset
}
// Disable it for 2.21, re-enable it for 2.22
TEST_F(QnnHTPBackendTests, DISABLED_ConvTranspose3D_U16S8S32_PerChannel) {
std::vector<int64_t> input_shape = {1, 2, 4, 4, 4};
std::vector<int64_t> weight_shape = {2, 3, 2, 2, 2};
std::vector<int64_t> bias_shape = {3};
TestInputDef<float> input_def(input_shape, false,
GetFloatDataInRange(-10.0f, 10.0f, TensorShape(input_shape).Size()));
TestInputDef<float> weight_def(weight_shape, true,
GetFloatDataInRange(-1.0f, 5.0f, TensorShape(weight_shape).Size()));
TestInputDef<float> bias_def(bias_shape, true,
GetFloatDataInRange(-1.0f, 1.0f, TensorShape(bias_shape).Size()));
RunHTPConvOpPerChannelTest<uint16_t, int8_t>("ConvTranspose",
input_def,
weight_def,
bias_def,
{1, 1, 1}, // Strides
{0, 0, 0, 0, 0, 0}, // Pads
{1, 1, 1}, // Dilations
1, // default group
"NOTSET",
ExpectedEPNodeAssignment::All,
true,
13);
}
// Test per-channel QDQ Conv that maps to QNN's DepthwiseConv2d (input_chans == output_chans == group).
// in0: u16, in1 (weight): s8, in2 (bias): s32, out: u16
TEST_F(QnnHTPBackendTests, ConvDepthwiseU16S8S32_PerChannel) {
@ -747,6 +1020,41 @@ TEST_F(QnnHTPBackendTests, ConvDepthwiseU16S8S32_PerChannel) {
13); // opset
}
// Test per-channel QDQ Conv3D
// \QNN\HTP\HTP\src\hexagon\prepare\graph_prepare.cc:203:ERROR:could not create op: q::QNN_Conv3d_w_scale
// \QNN\HTP\HTP\src\hexagon\prepare\graph_prepare.cc:1187:ERROR:Op 0x1a preparation failed with err:-1
// QnnDsp <E> "Conv" generated: could not create op
// QnnDsp <E> RouterWindows graph prepare failed 12
// QnnDsp <E> Failed to finalize graph (id: 1) with err 1002
// QnnDsp <V> Wake up free backend 1 thread(s)
// QnnDsp <I> QnnGraph_finalize done. status 0x3ea
// onnxruntime::qnn::QnnModel::FinalizeGraphs] Failed to finalize QNN graph.
TEST_F(QnnHTPBackendTests, DISABLED_Conv3D_U16S8S32_PerChannel2) {
std::vector<int64_t> input_shape = {1, 2, 4, 4, 4};
std::vector<int64_t> weight_shape = {2, 1, 2, 2, 2};
std::vector<int64_t> bias_shape = {2};
TestInputDef<float> input_def(input_shape, false,
GetFloatDataInRange(-10.0f, 10.0f, TensorShape(input_shape).Size()));
TestInputDef<float> weight_def(weight_shape, true,
GetFloatDataInRange(-1.0f, 5.0f, TensorShape(weight_shape).Size()));
TestInputDef<float> bias_def(bias_shape, true,
GetFloatDataInRange(-1.0f, 1.0f, TensorShape(bias_shape).Size()));
RunHTPConvOpPerChannelTest<uint16_t, int8_t>("Conv",
input_def,
weight_def,
bias_def,
{1, 1, 1}, // Strides
{0, 0, 0, 0, 0, 0}, // Pads
{1, 1, 1}, // Dilations
2, // default group
"NOTSET",
ExpectedEPNodeAssignment::All,
true,
13);
}
// Tests 16-bit QDQ Conv with dynamic weights and bias (uses QNN's Conv2d)
// TODO: Inaccuracy detected for output 'output', element 0.
// Output quant params: scale=0.0040235077030956745, zero_point=0.
@ -829,11 +1137,6 @@ TEST_F(QnnHTPBackendTests, DISABLED_DepthwiseConvU16S16S32_NoBias) {
// Tests 16-bit activations, 8-bit static weights QDQ Conv with static bias.
// Uses QNN's DepthwiseConv2d operator.
// TODO: Inaccuracy detected for output 'output', element 8.
// Output quant params: scale=0.0027466239407658577, zero_point=10194.
// Expected val: 152
// QNN QDQ val: 151.8004150390625 (err 0.1995849609375)
// CPU QDQ val: 151.9981689453125 (err 0.0018310546875)
TEST_F(QnnHTPBackendTests, DepthwiseConvU16U8S32_StaticBias) {
std::vector<float> input_data = GetFloatDataInRange(-10.0f, 10.0f, 25);
std::vector<float> weight_data = GetFloatDataInRange(-1.0f, 5.0f, 9);
@ -849,14 +1152,24 @@ TEST_F(QnnHTPBackendTests, DepthwiseConvU16U8S32_StaticBias) {
ExpectedEPNodeAssignment::All,
true, // Use com.microsoft QDQ ops for 16-bit
13);
std::vector<float> input_data_3d = GetFloatDataInRange(-10.0f, 10.0f, 125);
std::vector<float> weight_data_3d = GetFloatDataInRange(-1.0f, 5.0f, 27);
RunHTPConvOpTest<uint16_t, uint8_t>("Conv",
TestInputDef<float>({1, 1, 5, 5, 5}, false, input_data_3d), // Input
TestInputDef<float>({1, 1, 3, 3, 3}, true, weight_data_3d), // Weights
TestInputDef<float>({1}, true, {2.0f}), // Bias
{1, 1, 1}, // Strides
{0, 0, 0, 0, 0, 0}, // Pads
{1, 1, 1}, // Dilations
1, // default group
"NOTSET",
ExpectedEPNodeAssignment::All,
true, // Use com.microsoft QDQ ops for 16-bit
13);
}
// Tests 16-bit activations, 8-bit static weights QDQ Conv with static bias.
// TODO: Inaccuracy detected for output 'output', element 0.
// Output quant params: scale=0.0040235077030956745, zero_point=0.
// Expected val: 87.354057312011719
// QNN QDQ val: 87.559577941894531 (err 0.2055206298828125)
// CPU QDQ val: 87.398635864257812 (err 0.04457855224609375)
TEST_F(QnnHTPBackendTests, ConvU16U8S32_StaticBias) {
std::vector<float> input_data = GetFloatDataInRange(-10.0f, 10.0f, 50);
std::vector<float> weight_data = GetFloatDataInRange(-1.0f, 5.0f, 18);
@ -872,15 +1185,25 @@ TEST_F(QnnHTPBackendTests, ConvU16U8S32_StaticBias) {
ExpectedEPNodeAssignment::All,
true, // Use com.microsoft QDQ ops for 16-bit
13);
std::vector<float> input_data_3d = GetFloatDataInRange(-10.0f, 10.0f, 150);
std::vector<float> weight_data_3d = GetFloatDataInRange(-1.0f, 5.0f, 36);
RunHTPConvOpTest<uint16_t, uint8_t>("Conv",
TestInputDef<float>({1, 2, 5, 5, 3}, false, input_data_3d), // Input
TestInputDef<float>({1, 2, 3, 3, 2}, true, weight_data_3d), // Weights
TestInputDef<float>({1}, true, {2.0f}), // Bias
{1, 1, 1}, // Strides
{0, 0, 0, 0, 0, 0}, // Pads
{1, 1, 1}, // Dilations
1, // default group
"NOTSET",
ExpectedEPNodeAssignment::All,
true, // Use com.microsoft QDQ ops for 16-bit
13);
}
// Tests 16-bit activations, 8-bit static weights QDQ Conv with dynamic bias.
// Uses QNN's DepthwiseConv2d operator.
// TODO: Inaccuracy detected for output 'output', element 1.
// Output quant params: scale=0.0027466239407658577, zero_point=10194.
// Expected val: -13.000001907348633
// QNN QDQ val: -13.095903396606445 (err 0.0959014892578125)
// CPU QDQ val: -12.999771118164062 (err 0.0002307891845703125)
TEST_F(QnnHTPBackendTests, DepthwiseConvU16U8S32_DynamicBias) {
std::vector<float> input_data = GetFloatDataInRange(-10.0f, 10.0f, 25);
std::vector<float> weight_data = GetFloatDataInRange(-1.0f, 5.0f, 9);
@ -896,14 +1219,24 @@ TEST_F(QnnHTPBackendTests, DepthwiseConvU16U8S32_DynamicBias) {
ExpectedEPNodeAssignment::All,
true, // Use com.microsoft QDQ ops for 16-bit
13);
std::vector<float> input_data_3d = GetFloatDataInRange(-10.0f, 10.0f, 75);
std::vector<float> weight_data_3d = GetFloatDataInRange(-1.0f, 5.0f, 27);
RunHTPConvOpTest<uint16_t, uint8_t>("Conv",
TestInputDef<float>({1, 1, 5, 5, 3}, false, input_data_3d), // Input
TestInputDef<float>({1, 1, 3, 3, 3}, true, weight_data_3d), // Weights
TestInputDef<float>({1}, false, {2.0f}), // Bias
{1, 1, 1}, // Strides
{0, 0, 0, 0, 0, 0}, // Pads
{1, 1, 1}, // Dilations
1, // default group
"NOTSET",
ExpectedEPNodeAssignment::All,
true, // Use com.microsoft QDQ ops for 16-bit
13);
}
// Tests 16-bit activations, 8-bit static weights QDQ Conv with dynamic bias.
// TODO: Inaccuracy detected for output 'output', element 0.
// Output quant params: scale=0.0040235077030956745, zero_point=0.
// Expected val: 87.354057312011719
// QNN QDQ val: 87.559577941894531 (err 0.2055206298828125)
// CPU QDQ val: 87.398635864257812 (err 0.04457855224609375)
TEST_F(QnnHTPBackendTests, ConvU16U8S32_DynamicBias) {
std::vector<float> input_data = GetFloatDataInRange(-10.0f, 10.0f, 50);
std::vector<float> weight_data = GetFloatDataInRange(-1.0f, 5.0f, 18);
@ -919,14 +1252,24 @@ TEST_F(QnnHTPBackendTests, ConvU16U8S32_DynamicBias) {
ExpectedEPNodeAssignment::All,
true, // Use com.microsoft QDQ ops for 16-bit
13);
std::vector<float> input_data_3d = GetFloatDataInRange(-10.0f, 10.0f, 150);
std::vector<float> weight_data_3d = GetFloatDataInRange(-1.0f, 5.0f, 36);
RunHTPConvOpTest<uint16_t, uint8_t>("Conv",
TestInputDef<float>({1, 2, 5, 5, 3}, false, input_data_3d), // Input
TestInputDef<float>({1, 2, 3, 3, 2}, true, weight_data_3d), // Weights
TestInputDef<float>({1}, false, {2.0f}), // Bias
{1, 1, 1}, // Strides
{0, 0, 0, 0, 0, 0}, // Pads
{1, 1, 1}, // Dilations
1, // default group
"NOTSET",
ExpectedEPNodeAssignment::All,
true, // Use com.microsoft QDQ ops for 16-bit
13);
}
// Tests 16-bit activations, 8-bit static weights QDQ Conv with no bias
// TODO: Inaccuracy detected for output 'output', element 7.
// Output quant params: scale=0.0039929896593093872, zero_point=0.
// Expected val: 246.98667907714844
// QNN QDQ val: 247.82090759277344 (err 0.834228515625)
// CPU QDQ val: 247.24192810058594 (err 0.2552490234375)
TEST_F(QnnHTPBackendTests, ConvU16U8S32_NoBias) {
std::vector<float> input_data = GetFloatDataInRange(-10.0f, 10.0f, 50);
std::vector<float> weight_data = GetFloatDataInRange(-1.0f, 5.0f, 18);
@ -942,15 +1285,25 @@ TEST_F(QnnHTPBackendTests, ConvU16U8S32_NoBias) {
ExpectedEPNodeAssignment::All,
true, // Use com.microsoft QDQ ops for 16-bit
13);
std::vector<float> input_data_3d = GetFloatDataInRange(-10.0f, 10.0f, 150);
std::vector<float> weight_data_3d = GetFloatDataInRange(-1.0f, 5.0f, 36);
RunHTPConvOpTest<uint16_t, uint8_t>("Conv",
TestInputDef<float>({1, 2, 5, 5, 3}, false, input_data_3d), // Input
TestInputDef<float>({1, 2, 3, 3, 2}, true, weight_data_3d), // Weights
TestInputDef<float>(), // Bias
{1, 1, 1}, // Strides
{0, 0, 0, 0, 0, 0}, // Pads
{1, 1, 1}, // Dilations
1, // default group
"NOTSET",
ExpectedEPNodeAssignment::All,
true, // Use com.microsoft QDQ ops for 16-bit
13);
}
// Tests 16-bit activations, 8-bit static weights QDQ Conv with no bias
// Uses QNN's DepthwiseConv2d operator.
// TODO: Inaccuracy detected for output 'output', element 8.
// Output quant params: scale=0.0027466239407658577, zero_point=10923.
// Expected val: 150
// QNN QDQ val: 149.80087280273438 (err 0.199127197265625)
// CPU QDQ val: 149.99862670898438 (err 0.001373291015625)
TEST_F(QnnHTPBackendTests, DepthwiseConvU16U8S32_NoBias) {
std::vector<float> input_data = GetFloatDataInRange(-10.0f, 10.0f, 25);
std::vector<float> weight_data = GetFloatDataInRange(-1.0f, 5.0f, 9);
@ -966,6 +1319,21 @@ TEST_F(QnnHTPBackendTests, DepthwiseConvU16U8S32_NoBias) {
ExpectedEPNodeAssignment::All,
true, // Use com.microsoft QDQ ops for 16-bit
13);
std::vector<float> input_data_3d = GetFloatDataInRange(-10.0f, 10.0f, 75);
std::vector<float> weight_data_3d = GetFloatDataInRange(-1.0f, 5.0f, 18);
RunHTPConvOpTest<uint16_t, uint8_t>("Conv",
TestInputDef<float>({1, 1, 5, 5, 3}, false, input_data_3d), // Input
TestInputDef<float>({1, 1, 3, 3, 2}, true, weight_data_3d), // Weights
TestInputDef<float>(), // Bias
{1, 1, 1}, // Strides
{0, 0, 0, 0, 0, 0}, // Pads
{1, 1, 1}, // Dilations
1, // default group
"NOTSET",
ExpectedEPNodeAssignment::All,
true, // Use com.microsoft QDQ ops for 16-bit
13);
}
// Test that dynamic weights with default bias works for Conv. This was previously not working
@ -981,6 +1349,17 @@ TEST_F(QnnHTPBackendTests, ConvU8U8S32_DynamicWeight_NoBias) {
1, // default group
"NOTSET",
ExpectedEPNodeAssignment::All);
RunHTPConvOpTest<uint8_t, uint8_t>("Conv",
TestInputDef<float>({1, 3, 32, 32, 32}, false, -10.0f, 10.0f), // Input
TestInputDef<float>({1, 3, 4, 4, 4}, false, -10.0f, 10.0f), // Weights
TestInputDef<float>(), // Bias
{1, 1, 1}, // Strides
{0, 0, 0, 0, 0, 0}, // Pads
{1, 1, 1}, // Dilations
1, // default group
"NOTSET",
ExpectedEPNodeAssignment::All);
}
// Test that dynamic weights with default bias works for ConvTranspose. This was previously not working
@ -998,6 +1377,23 @@ TEST_F(QnnHTPBackendTests, ConvTransposeU8U8S32_DynamicWeight_NoBias) {
ExpectedEPNodeAssignment::All);
}
// QNN op validation crash. Run correctly if by pass the QNN op validation
// Exception from backendValidateOpConfig:
// Exception thrown at 0x00007FFF9E0128B0 (QnnHtpPrepare.dll) in onnxruntime_test_all.exe:
// 0xC0000005: Access violation reading location 0x7079745F656C706D.
TEST_F(QnnHTPBackendTests, DISABLED_ConvTranspose3D_U8U8S32_DynamicWeight_NoBias) {
RunHTPConvOpTest<uint8_t, uint8_t>("ConvTranspose",
TestInputDef<float>({1, 3, 32, 32, 32}, false, -10.0f, 10.0f), // Input
TestInputDef<float>({3, 1, 4, 4, 4}, false, -10.0f, 10.0f), // Weights
TestInputDef<float>(), // Bias
{1, 1, 1}, // Strides
{0, 0, 0, 0, 0, 0}, // Pads
{1, 1, 1}, // Dilations
1, // default group
"NOTSET",
ExpectedEPNodeAssignment::All);
}
// Check that QNN compiles DQ -> Conv -> Q as a single unit.
// Tests bias as an initializer.
TEST_F(QnnHTPBackendTests, ConvU8U8S32_bias_initializer) {
@ -1015,6 +1411,21 @@ TEST_F(QnnHTPBackendTests, ConvU8U8S32_bias_initializer) {
13, // opset
// Need tolerance of 0.413% of output range after QNN SDK 2.17
QDQTolerance(0.00413f));
RunHTPConvOpTest<uint8_t, uint8_t>("Conv",
TestInputDef<float>({1, 1, 5, 5, 5}, false, 0.0f, 10.0f), // Random dynamic input
TestInputDef<float>({1, 1, 3, 3, 3}, true, -10.0f, 10.0f), // Random static weight
TestInputDef<float>({1}, true, {2.0f}), // Initializer bias
{1, 1, 1}, // Strides
{0, 0, 0, 0, 0, 0}, // Pads
{1, 1, 1}, // Dilations
1, // default group
"NOTSET",
ExpectedEPNodeAssignment::All,
false, // use_qdq_contrib_ops
13, // opset
// Need tolerance of 0.413% of output range after QNN SDK 2.17
QDQTolerance(0.00413f));
}
// Tests 1D Conv with bias as an initializer.
@ -1061,6 +1472,19 @@ TEST_F(QnnHTPBackendTests, ConvU8U8S32_AutoPadUpper) {
ExpectedEPNodeAssignment::All,
false, // use_contrib_qdq
13);
RunHTPConvOpTest<uint8_t, uint8_t>("Conv",
TestInputDef<float>({1, 1, 5, 5, 5}, false, 0.f, 10.f), // Dynamic input
TestInputDef<float>({1, 1, 4, 4, 4}, true, -1.f, 1.f), // Static weights
TestInputDef<float>({1}, true, {1.0f}), // Initializer bias
{1, 1, 1}, // strides
{}, // pads
{1, 1, 1}, // dilations
1, // default group
"SAME_UPPER", // auto_pad
ExpectedEPNodeAssignment::All,
false, // use_contrib_qdq
13);
}
// Tests Conv1d auto_pad value "SAME_UPPER" on HTP backend (compares to CPU EP).
@ -1111,6 +1535,19 @@ TEST_F(QnnHTPBackendTests, ConvU8U8S32_AutoPadLower) {
ExpectedEPNodeAssignment::All,
false, // use_contrib_qdq
13);
RunHTPConvOpTest<uint8_t, uint8_t>("Conv",
TestInputDef<float>({1, 1, 5, 5, 5}, false, 0.f, 10.f), // Dynamic input
TestInputDef<float>({1, 1, 4, 4, 4}, true, -1.f, 1.f), // Static weights
TestInputDef<float>({1}, true, {1.0f}), // Initializer bias
{1, 1, 1}, // strides
{}, // pads
{1, 1, 1}, // dilations
1, // default group
"SAME_LOWER", // auto_pad
ExpectedEPNodeAssignment::All,
false, // use_contrib_qdq
13);
}
// Tests ConvTranspose's auto_pad value "SAME_LOWER" on HTP backend (compares to CPU EP).
@ -1127,6 +1564,19 @@ TEST_F(QnnHTPBackendTests, ConvTransposeU8U8S32_AutoPadLower) {
ExpectedEPNodeAssignment::All,
false, // use_contrib_qdq
13);
RunHTPConvOpTest<uint8_t, uint8_t>("ConvTranspose",
TestInputDef<float>({1, 1, 5, 5, 5}, false, 0.f, 10.f), // Dynamic input
TestInputDef<float>({1, 1, 4, 4, 4}, true, -1.f, 1.f), // Static weights
TestInputDef<float>({1}, true, {1.0f}), // Initializer bias
{1, 1, 1}, // strides
{}, // pads
{1, 1, 1}, // dilations
1, // default group
"SAME_LOWER", // auto_pad
ExpectedEPNodeAssignment::All,
false, // use_contrib_qdq
13);
}
// Tests Conv1d auto_pad value "SAME_LOWER" on HTP backend (compares to CPU EP).