saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-29 20:14:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

[QNN EP] Initial INT4 support (#21171)

Browse source 
### Description
- Adds support for int4 quantized weights (per-tensor and per-channel)
on QNN EP
- Adds test script that creates an INT4 qdq model with a Conv
- Adds a unit tests demonstrating accuracy issues.



### Motivation and Context
This is the next step in being able to run models that use 4-bit
quantized weights on QNN EP.
This commit is contained in:
Adrian Lizarraga 2024-07-10 10:03:53 -07:00 committed by GitHub
parent 1b82d835d8
commit 5753f8da8c
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
12 changed files with 522 additions and 42 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

3
onnxruntime/core/providers/qnn/builder/opbuilder/conv_op_builder.cc
View file

@ -127,7 +127,8 @@ Status ConvOpBuilder::IsOpSupported(QnnModelWrapper& qnn_model_wrapper,
int32_t elem_data_type = 0;
ORT_RETURN_IF_ERROR(utils::GetOnnxTensorElemDataType(input_1.node_arg, elem_data_type));
const bool is_signed_type = (elem_data_type == ONNX_NAMESPACE::TensorProto_DataType_INT8) ||
const bool is_signed_type = (elem_data_type == ONNX_NAMESPACE::TensorProto_DataType_INT4) ||
(elem_data_type == ONNX_NAMESPACE::TensorProto_DataType_INT8) ||
(elem_data_type == ONNX_NAMESPACE::TensorProto_DataType_INT16);
ORT_RETURN_IF_NOT(is_signed_type, "Conv weights must be of a signed quantized type if quantized per-channel");

39
onnxruntime/core/providers/qnn/builder/qnn_model_wrapper.cc
View file

@ -5,6 +5,8 @@
#include <cstdlib>
#include <cstring>
#include <numeric>
#include <utility>
#include <vector>
#include "qnn_model_wrapper.h"
#include "core/common/safeint.h"
@ -313,7 +315,8 @@ bool QnnModelWrapper::GetOnnxShape(const NodeArg& node_arg, std::vector<uint32_t
}
Status QnnModelWrapper::UnpackZeroPoints(const std::string& initializer_name,
std::vector<int32_t>& zero_points) const {
/*out*/ std::vector<int32_t>& zero_points,
/*out*/ int32_t& onnx_data_type) const {
const auto& graph_initializers = GetInitializerTensors();
auto iter = graph_initializers.find(initializer_name);
ORT_RETURN_IF(iter == graph_initializers.end(), "Unable to find initializer for zero-point(s): ",
@ -323,13 +326,14 @@ Status QnnModelWrapper::UnpackZeroPoints(const std::string& initializer_name,
ORT_RETURN_IF_NOT(zp_tensor_proto->has_data_type(), "Expected zero-point initializer ", initializer_name.c_str(),
" to have a proto data type.");
const int32_t onnx_data_type = zp_tensor_proto->data_type();
onnx_data_type = zp_tensor_proto->data_type();
std::vector<uint8_t> initializer_bytes;
ORT_RETURN_IF_ERROR(UnpackInitializerData(*zp_tensor_proto, initializer_bytes));
switch (onnx_data_type) {
// QNN use -offset for some reason
case ONNX_NAMESPACE::TensorProto_DataType_INT4: // INT4 zero-points are unpacked as 8-bit values for QNN
case ONNX_NAMESPACE::TensorProto_DataType_INT8: {
auto int8_span = ReinterpretAsSpan<const int8_t>(gsl::make_span(initializer_bytes));
std::transform(int8_span.begin(), int8_span.end(), std::back_inserter(zero_points),
@ -338,6 +342,7 @@ Status QnnModelWrapper::UnpackZeroPoints(const std::string& initializer_name,
});
break;
}
case ONNX_NAMESPACE::TensorProto_DataType_UINT4: // UINT4 zero-points are unpacked as 8-bit values for QNN
case ONNX_NAMESPACE::TensorProto_DataType_UINT8: {
auto uint8_span = ReinterpretAsSpan<const uint8_t>(gsl::make_span(initializer_bytes));
std::transform(uint8_span.begin(), uint8_span.end(), std::back_inserter(zero_points),
@ -584,10 +589,36 @@ void QnnModelWrapper::GetGraphInputOutputTensorWrapper(const std::vector<std::st
Status QnnModelWrapper::UnpackInitializerData(const ONNX_NAMESPACE::TensorProto& initializer,
std::vector<uint8_t>& unpacked_tensor) const {
if (initializer.data_location() == onnx::TensorProto_DataLocation_EXTERNAL) {
return onnxruntime::utils::UnpackInitializerData(initializer, graph_viewer_.ModelPath(), unpacked_tensor);
ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(initializer, graph_viewer_.ModelPath(),
unpacked_tensor));
} else {
ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(initializer, unpacked_tensor));
}
return onnxruntime::utils::UnpackInitializerData(initializer, unpacked_tensor);
int32_t onnx_data_type = initializer.data_type();
// If this is an int4, we need to unpack it because QNN treats int4 as a full int8.
if (onnx_data_type == ONNX_NAMESPACE::TensorProto_DataType_INT4) {
TensorShape shape = onnxruntime::utils::GetTensorShapeFromTensorProto(initializer);
const size_t num_elems = shape.Size();
std::vector<uint8_t> packed_int4_bytes = std::move(unpacked_tensor);
unpacked_tensor = std::vector<uint8_t>(num_elems);
auto dst = gsl::make_span(reinterpret_cast<int8_t*>(unpacked_tensor.data()), unpacked_tensor.size());
auto src = gsl::make_span(reinterpret_cast<const Int4x2*>(packed_int4_bytes.data()), packed_int4_bytes.size());
ORT_RETURN_IF_NOT(Int4x2::Unpack(dst, src), "Failed to unpack Tensor<Int4x2> for QNN");
} else if (onnx_data_type == ONNX_NAMESPACE::TensorProto_DataType_UINT4) {
TensorShape shape = onnxruntime::utils::GetTensorShapeFromTensorProto(initializer);
const size_t num_elems = shape.Size();
std::vector<uint8_t> packed_int4_bytes = std::move(unpacked_tensor);
unpacked_tensor = std::vector<uint8_t>(num_elems);
auto dst = gsl::make_span(reinterpret_cast<uint8_t*>(unpacked_tensor.data()), unpacked_tensor.size());
auto src = gsl::make_span(reinterpret_cast<const UInt4x2*>(packed_int4_bytes.data()), packed_int4_bytes.size());
ORT_RETURN_IF_NOT(UInt4x2::Unpack(dst, src), "Failed to unpack Tensor<UInt4x2> for QNN");
}
return Status::OK();
}
} // namespace qnn

4
onnxruntime/core/providers/qnn/builder/qnn_model_wrapper.h
View file

@ -216,7 +216,9 @@ class QnnModelWrapper {
Status UnpackScales(const std::string& initializer_name, std::vector<float>& scales) const;
// Unpack zero-points from initializer and convert to int32_t (1 zero-point for per-tensor, > 1 for per-channel).
Status UnpackZeroPoints(const std::string& initializer_name, std::vector<int32_t>& zero_points) const;
Status UnpackZeroPoints(const std::string& initializer_name,
/*out*/ std::vector<int32_t>& zero_points,
/*out*/ int32_t& onnx_data_type) const;
// Checks if a tensor in the ONNX graph is per-channel quantized.
Status IsPerChannelQuantized(const onnxruntime::NodeUnitIODef& io_def,

140
onnxruntime/core/providers/qnn/builder/qnn_quant_params_wrapper.cc
View file

@ -9,6 +9,9 @@
#include "QnnTypes.h"
#include "core/providers/qnn/builder/qnn_model_wrapper.h"
#define ALIGN_PTR_UP(ptr, align, type) \
reinterpret_cast<type>((reinterpret_cast<std::uintptr_t>(ptr) + (align)-1) & ~((align)-1))
namespace onnxruntime {
namespace qnn {
@ -38,9 +41,10 @@ QnnQuantParamsWrapper QnnQuantParamsWrapper::Copy() const {
return QnnQuantParamsWrapper(*this);
}
// Initializes by copying from a Qnn_QuantizeParams_t.
Status QnnQuantParamsWrapper::Init(const Qnn_QuantizeParams_t& params) {
if (scale_offset_data_) {
scale_offset_data_.reset(nullptr);
if (per_channel_data_) {
per_channel_data_.reset(nullptr);
params_ = QNN_QUANTIZE_PARAMS_INIT;
}
@ -51,6 +55,7 @@ Status QnnQuantParamsWrapper::Init(const Qnn_QuantizeParams_t& params) {
switch (params.quantizationEncoding) {
case QNN_QUANTIZATION_ENCODING_SCALE_OFFSET:
case QNN_QUANTIZATION_ENCODING_BW_SCALE_OFFSET:
params_ = params;
break;
case QNN_QUANTIZATION_ENCODING_AXIS_SCALE_OFFSET: {
@ -63,15 +68,49 @@ Status QnnQuantParamsWrapper::Init(const Qnn_QuantizeParams_t& params) {
const uint32_t num_elems = params.axisScaleOffsetEncoding.numScaleOffsets;
if (num_elems > 0) {
scale_offset_data_ = std::make_unique<Qnn_ScaleOffset_t[]>(num_elems);
gsl::span<Qnn_ScaleOffset_t> src_span(params.axisScaleOffsetEncoding.scaleOffset, num_elems);
std::copy(src_span.begin(), src_span.end(), scale_offset_data_.get());
params_.axisScaleOffsetEncoding.scaleOffset = scale_offset_data_.get();
const size_t num_bytes = num_elems * sizeof(Qnn_ScaleOffset_t);
constexpr std::uintptr_t align = alignof(Qnn_ScaleOffset_t);
per_channel_data_ = std::make_unique<char[]>(num_bytes + align);
Qnn_ScaleOffset_t* aligned_dst = ALIGN_PTR_UP(per_channel_data_.get(), align, Qnn_ScaleOffset_t*);
std::memcpy(aligned_dst, params.axisScaleOffsetEncoding.scaleOffset, num_bytes);
params_.axisScaleOffsetEncoding.scaleOffset = aligned_dst;
} else {
params_.axisScaleOffsetEncoding.scaleOffset = nullptr;
}
break;
}
case QNN_QUANTIZATION_ENCODING_BW_AXIS_SCALE_OFFSET: {
const uint32_t num_elems = params.bwAxisScaleOffsetEncoding.numElements;
params_.encodingDefinition = params.encodingDefinition;
params_.quantizationEncoding = params.quantizationEncoding;
params_.bwAxisScaleOffsetEncoding.axis = params.bwAxisScaleOffsetEncoding.axis;
params_.bwAxisScaleOffsetEncoding.bitwidth = params.bwAxisScaleOffsetEncoding.bitwidth;
params_.bwAxisScaleOffsetEncoding.numElements = num_elems;
// Deep copy the scales[] and offsets[] arrays
if (num_elems > 0) {
const size_t num_scale_bytes = num_elems * sizeof(float);
const size_t num_zp_bytes = num_elems * sizeof(int32_t);
const size_t num_bytes = num_scale_bytes + num_zp_bytes;
constexpr std::uintptr_t align = alignof(float);
static_assert(alignof(float) == alignof(int32_t));
per_channel_data_ = std::make_unique<char[]>(num_bytes + align);
char* scales_begin = ALIGN_PTR_UP(per_channel_data_.get(), align, char*);
char* zps_begin = scales_begin + num_scale_bytes;
std::memcpy(scales_begin, params.bwAxisScaleOffsetEncoding.scales, num_scale_bytes);
std::memcpy(zps_begin, params.bwAxisScaleOffsetEncoding.offsets, num_zp_bytes);
params_.bwAxisScaleOffsetEncoding.scales = reinterpret_cast<float*>(scales_begin);
params_.bwAxisScaleOffsetEncoding.offsets = reinterpret_cast<int32_t*>(zps_begin);
} else {
params_.bwAxisScaleOffsetEncoding.scales = nullptr;
params_.bwAxisScaleOffsetEncoding.offsets = nullptr;
}
break;
}
default:
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Unsupported QNN quantization encoding: ", params.quantizationEncoding);
}
@ -79,11 +118,13 @@ Status QnnQuantParamsWrapper::Init(const Qnn_QuantizeParams_t& params) {
return Status::OK();
}
// Initialize this object from a (potentially) quantized ONNX tensor.
// QnnModelWrapper provides utilities for unpacking scale and zero-point ONNX initializers.
Status QnnQuantParamsWrapper::Init(const QnnModelWrapper& qnn_model_wrapper, const NodeUnitIODef& io_def) {
const std::optional<NodeUnitIODef::QuantParam>& ort_quant_params = io_def.quant_param;
if (scale_offset_data_) {
scale_offset_data_.reset(nullptr);
if (per_channel_data_) {
per_channel_data_.reset(nullptr);
params_ = QNN_QUANTIZE_PARAMS_INIT;
}
@ -98,17 +139,25 @@ Status QnnQuantParamsWrapper::Init(const QnnModelWrapper& qnn_model_wrapper, con
ORT_RETURN_IF_ERROR(qnn_model_wrapper.UnpackScales(ort_quant_params->scale.Name(), scales));
bool is_int4_type = false;
if (ort_quant_params->zero_point != nullptr) {
ORT_RETURN_IF_ERROR(qnn_model_wrapper.UnpackZeroPoints(ort_quant_params->zero_point->Name(), zero_points));
int32_t onnx_tp_type = 0;
ORT_RETURN_IF_ERROR(qnn_model_wrapper.UnpackZeroPoints(ort_quant_params->zero_point->Name(), zero_points,
onnx_tp_type));
is_int4_type = (onnx_tp_type == ONNX_NAMESPACE::TensorProto_DataType_INT4) ||
(onnx_tp_type == ONNX_NAMESPACE::TensorProto_DataType_UINT4);
}
const bool is_per_tensor = scales.size() == 1;
if (is_per_tensor) {
// QNN uses different structs to represent quantization parameters depending on
// - per-tensor vs per-channel
// - int4 vs not int4
if (is_per_tensor && !is_int4_type) {
params_.encodingDefinition = QNN_DEFINITION_DEFINED;
params_.quantizationEncoding = QNN_QUANTIZATION_ENCODING_SCALE_OFFSET;
// Parse scale & zero_point
params_.scaleOffsetEncoding.scale = scales[0];
if (ort_quant_params->zero_point != nullptr) {
@ -117,8 +166,62 @@ Status QnnQuantParamsWrapper::Init(const QnnModelWrapper& qnn_model_wrapper, con
} else {
params_.scaleOffsetEncoding.offset = 0;
}
} else {
// Per-channel quantization.
} else if (is_per_tensor && is_int4_type) {
params_.encodingDefinition = QNN_DEFINITION_DEFINED;
params_.quantizationEncoding = QNN_QUANTIZATION_ENCODING_BW_SCALE_OFFSET;
params_.bwScaleOffsetEncoding.bitwidth = 4;
params_.bwScaleOffsetEncoding.scale = scales[0];
if (ort_quant_params->zero_point != nullptr) {
ORT_RETURN_IF_NOT(zero_points.size() == 1, "Expected one zero-point value");
params_.bwScaleOffsetEncoding.offset = zero_points[0];
} else {
params_.bwScaleOffsetEncoding.offset = 0;
}
} else if (!is_per_tensor && is_int4_type) {
const auto* io_shape = io_def.node_arg.Shape();
ORT_RETURN_IF(io_shape == nullptr, "Input/output tensor proto must have a shape");
const int32_t io_rank = io_shape->dim_size();
constexpr int64_t DEFAULT_QDQ_AXIS = 1;
int64_t axis = ort_quant_params->axis.value_or(DEFAULT_QDQ_AXIS);
if (axis < 0) {
axis += io_rank;
}
ORT_RETURN_IF_NOT(axis >= 0 && axis < io_rank,
"Quantization axis must be within the range [0, rank - 1]");
const size_t num_elems = scales.size();
const bool no_zero_points = zero_points.empty();
ORT_RETURN_IF_NOT(num_elems > 1, "Expected more than one scale value");
ORT_RETURN_IF_NOT(no_zero_points || zero_points.size() == num_elems,
"Expected the same number of zero-points and scales for per-channel quantization");
params_.encodingDefinition = QNN_DEFINITION_DEFINED;
params_.quantizationEncoding = QNN_QUANTIZATION_ENCODING_BW_AXIS_SCALE_OFFSET;
params_.bwAxisScaleOffsetEncoding.axis = static_cast<int32_t>(*(ort_quant_params->axis));
params_.bwAxisScaleOffsetEncoding.bitwidth = 4;
params_.bwAxisScaleOffsetEncoding.numElements = static_cast<uint32_t>(num_elems);
const size_t num_scale_bytes = num_elems * sizeof(float);
const size_t num_zp_bytes = num_elems * sizeof(int32_t);
const size_t num_bytes = num_scale_bytes + num_zp_bytes;
constexpr std::uintptr_t align = alignof(float);
per_channel_data_ = std::make_unique<char[]>(num_bytes + align);
char* scales_begin = ALIGN_PTR_UP(per_channel_data_.get(), align, char*);
char* zps_begin = scales_begin + num_scale_bytes;
gsl::span<float> scales_span(reinterpret_cast<float*>(scales_begin), num_elems);
gsl::span<int32_t> zps_span(reinterpret_cast<int32_t*>(zps_begin), num_elems);
for (size_t i = 0; i < num_elems; i++) {
scales_span[i] = scales[i];
zps_span[i] = no_zero_points ? 0 : zero_points[i];
}
params_.bwAxisScaleOffsetEncoding.scales = scales_span.data();
params_.bwAxisScaleOffsetEncoding.offsets = zps_span.data();
} else if (!is_per_tensor && !is_int4_type) {
const auto* io_shape = io_def.node_arg.Shape();
ORT_RETURN_IF(io_shape == nullptr, "Input/output tensor proto must have a shape");
const int32_t io_rank = io_shape->dim_size();
@ -140,8 +243,11 @@ Status QnnQuantParamsWrapper::Init(const QnnModelWrapper& qnn_model_wrapper, con
ORT_RETURN_IF_NOT(no_zero_points || zero_points.size() == num_elems,
"Expected the same number of zero-points and scales for per-channel quantization");
scale_offset_data_ = std::make_unique<Qnn_ScaleOffset_t[]>(num_elems);
gsl::span<Qnn_ScaleOffset_t> data_span(scale_offset_data_.get(), num_elems);
const size_t num_bytes = num_elems * sizeof(Qnn_ScaleOffset_t);
constexpr std::uintptr_t align = alignof(Qnn_ScaleOffset_t);
per_channel_data_ = std::make_unique<char[]>(num_bytes + align);
Qnn_ScaleOffset_t* aligned_dst = ALIGN_PTR_UP(per_channel_data_.get(), align, Qnn_ScaleOffset_t*);
gsl::span<Qnn_ScaleOffset_t> data_span(aligned_dst, num_elems);
for (size_t i = 0; i < num_elems; i++) {
data_span[i].scale = scales[i];
@ -151,6 +257,8 @@ Status QnnQuantParamsWrapper::Init(const QnnModelWrapper& qnn_model_wrapper, con
params_.axisScaleOffsetEncoding.axis = static_cast<int32_t>(axis);
params_.axisScaleOffsetEncoding.numScaleOffsets = static_cast<uint32_t>(num_elems);
params_.axisScaleOffsetEncoding.scaleOffset = data_span.data();
} else {
return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Unexpected tensor kind for QuantParamsWrapper::Init()");
}
return Status::OK();

16
onnxruntime/core/providers/qnn/builder/qnn_quant_params_wrapper.h
View file

@ -48,17 +48,17 @@ class QnnQuantParamsWrapper {
(include_bw && params_.quantizationEncoding == QNN_QUANTIZATION_ENCODING_BW_SCALE_OFFSET));
}
bool IsPerChannel(bool include_bw = false) const {
bool IsPerChannel() const {
return params_.encodingDefinition == QNN_DEFINITION_DEFINED &&
(params_.quantizationEncoding == QNN_QUANTIZATION_ENCODING_AXIS_SCALE_OFFSET ||
(include_bw && params_.quantizationEncoding == QNN_QUANTIZATION_ENCODING_BW_AXIS_SCALE_OFFSET));
(params_.quantizationEncoding == QNN_QUANTIZATION_ENCODING_BW_AXIS_SCALE_OFFSET));
}
// Handle transposing of a per-channel quantized tensor. The quantization parameter's axis
// must be transposed using the inverse permutation of the Transpose.
template <typename IntType>
Status HandleTranspose(gsl::span<const IntType> perm) {
if (!IsPerChannel(true)) {
if (!IsPerChannel()) {
return Status::OK();
}
@ -82,7 +82,7 @@ class QnnQuantParamsWrapper {
template <typename IntType>
Status HandleUnsqueeze(gsl::span<const IntType> orig_shape,
gsl::span<const IntType> new_shape) {
if (!IsPerChannel(true)) {
if (!IsPerChannel()) {
return Status::OK();
}
@ -134,7 +134,13 @@ class QnnQuantParamsWrapper {
private:
Qnn_QuantizeParams_t params_;
std::unique_ptr<Qnn_ScaleOffset_t[]> scale_offset_data_; // Stores per-channel scales and offsets
// Stores arrays of per-channel scales and offsets. Fields in params_ point to this data.
//
// Use an opaque array of bytes because QNN uses different data layouts depending on the quantization encoding:
// - QNN_QUANTIZATION_ENCODING_AXIS_SCALE_OFFSET: array of scale/zp pairs [{scale0, zp0}, {scale1, zp1}, ...]
// - QNN_QUANTIZATION_ENCODING_BW_AXIS_SCALE_OFFSET: parallel arrays for scales and zps [scale0, ...] [zp0, zp1, ...]
std::unique_ptr<char[]> per_channel_data_;
};
} // namespace qnn

20
onnxruntime/core/providers/qnn/builder/qnn_utils.cc
View file

@ -43,6 +43,8 @@ size_t GetElementSizeByType(const Qnn_DataType_t& data_type) {
}
size_t GetElementSizeByType(ONNXTensorElementDataType elem_type) {
const static std::unordered_map<ONNXTensorElementDataType, size_t> elem_type_to_size = {
{ONNX_TENSOR_ELEMENT_DATA_TYPE_INT4, sizeof(Int4x2)},
{ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT4, sizeof(UInt4x2)},
{ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8, sizeof(int8_t)},
{ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16, sizeof(int16_t)},
{ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32, sizeof(int32_t)},
@ -162,6 +164,12 @@ std::ostream& operator<<(std::ostream& out, const Qnn_DataType_t& data_type) {
case QNN_DATATYPE_BOOL_8:
out << "QNN_DATATYPE_BOOL_8";
break;
case QNN_DATATYPE_SFIXED_POINT_4:
out << "QNN_DATATYPE_SFIXED_POINT_4";
break;
case QNN_DATATYPE_UFIXED_POINT_4:
out << "QNN_DATATYPE_UFIXED_POINT_4";
break;
default:
ORT_THROW("Unknown Qnn Data type");
}
@ -216,6 +224,10 @@ std::ostream& operator<<(std::ostream& out, const Qnn_QuantizeParams_t& quantize
if (quantize_params.quantizationEncoding == QNN_QUANTIZATION_ENCODING_SCALE_OFFSET) {
out << " scale=" << quantize_params.scaleOffsetEncoding.scale;
out << " offset=" << quantize_params.scaleOffsetEncoding.offset;
} else if (quantize_params.quantizationEncoding == QNN_QUANTIZATION_ENCODING_BW_SCALE_OFFSET) {
out << " bitwidth=" << quantize_params.bwScaleOffsetEncoding.bitwidth;
out << " scale=" << quantize_params.bwScaleOffsetEncoding.scale;
out << " offset=" << quantize_params.bwScaleOffsetEncoding.offset;
} else if (quantize_params.quantizationEncoding == QNN_QUANTIZATION_ENCODING_AXIS_SCALE_OFFSET) {
out << " axis=" << quantize_params.axisScaleOffsetEncoding.axis;
size_t num_elems = quantize_params.axisScaleOffsetEncoding.numScaleOffsets;
@ -292,7 +304,9 @@ std::ostream& operator<<(std::ostream& out, const Qnn_ClientBuffer_t& client_buf
T* data = reinterpret_cast<T*>(client_bufer.data);
out << " dataSize=" << client_bufer.dataSize;
uint32_t count = client_bufer.dataSize / sizeof(T);
count = count > 100 ? 100 : count; // limit to 100 data
const bool truncate = count > 100;
count = truncate ? 100 : count; // limit to 100 data
out << " clientBuf=(";
for (uint32_t i = 0; i < count; i++) {
if constexpr (sizeof(T) == 1) {
@ -301,7 +315,7 @@ std::ostream& operator<<(std::ostream& out, const Qnn_ClientBuffer_t& client_buf
out << data[i] << " ";
}
}
out << ")";
out << (truncate ? "..." : "") << ")";
return out;
}
@ -432,10 +446,12 @@ bool OnnxDataTypeToQnnDataType(const int32_t onnx_data_type, Qnn_DataType_t& qnn
};
const std::unordered_map<int32_t, Qnn_DataType_t> onnx_to_qnn_data_type_quantized = {
{ONNX_NAMESPACE::TensorProto_DataType_INT4, QNN_DATATYPE_SFIXED_POINT_8},
{ONNX_NAMESPACE::TensorProto_DataType_INT8, QNN_DATATYPE_SFIXED_POINT_8},
{ONNX_NAMESPACE::TensorProto_DataType_INT16, QNN_DATATYPE_SFIXED_POINT_16},
{ONNX_NAMESPACE::TensorProto_DataType_INT32, QNN_DATATYPE_SFIXED_POINT_32},
{ONNX_NAMESPACE::TensorProto_DataType_INT64, QNN_DATATYPE_INT_64},
{ONNX_NAMESPACE::TensorProto_DataType_UINT4, QNN_DATATYPE_UFIXED_POINT_8},
{ONNX_NAMESPACE::TensorProto_DataType_UINT8, QNN_DATATYPE_UFIXED_POINT_8},
{ONNX_NAMESPACE::TensorProto_DataType_UINT16, QNN_DATATYPE_UFIXED_POINT_16},
{ONNX_NAMESPACE::TensorProto_DataType_UINT32, QNN_DATATYPE_UFIXED_POINT_32},

12
onnxruntime/test/optimizer/graph_transform_test_builder.h
View file

@ -339,8 +339,10 @@ class ModelTestBuilder {
bool use_ms_domain = false) {
std::vector<NodeArg*> input_args;
input_args.push_back(input_arg);
input_args.push_back(Make1DInitializer<float>(input_scales));
input_args.push_back(Make1DInitializer<T>(input_zero_points));
std::vector<int64_t> qparams_shape = {static_cast<int64_t>(input_scales.size())};
input_args.push_back(MakeInitializer<float>(qparams_shape, input_scales));
input_args.push_back(MakeInitializer<T>(qparams_shape, input_zero_points));
std::string domain = use_ms_domain ? kMSDomain : "";
return AddNode("QuantizeLinear", input_args, {output_arg}, domain, attributes);
@ -415,8 +417,10 @@ class ModelTestBuilder {
bool use_ms_domain = false) {
std::vector<NodeArg*> input_args;
input_args.push_back(input_arg);
input_args.push_back(Make1DInitializer<float>(input_scales));
input_args.push_back(Make1DInitializer<T>(input_zero_points));
std::vector<int64_t> qparams_shape = {static_cast<int64_t>(input_scales.size())};
input_args.push_back(MakeInitializer<float>(qparams_shape, input_scales));
input_args.push_back(MakeInitializer<T>(qparams_shape, input_zero_points));
std::string domain = use_ms_domain ? kMSDomain : "";
return AddNode("DequantizeLinear", input_args, {output_arg}, domain, attributes);

81
onnxruntime/test/providers/qnn/conv_test.cc
View file

@ -182,7 +182,12 @@ static GetTestQDQModelFn<ActivationQType> BuildQDQPerChannelConvTestCase(const s
static_cast<size_t>(weight_quant_axis), true);
TensorShape weights_shape = weights_def.GetTensorShape();
std::vector<WeightQType> quantized_weights(weights_shape.Size());
std::vector<WeightQType> quantized_weights;
size_t num_weight_storage_elems = weights_shape.Size();
if constexpr (std::is_same_v<WeightQType, Int4x2> || std::is_same_v<WeightQType, UInt4x2>) {
num_weight_storage_elems = Int4x2::CalcNumInt4Pairs(weights_shape.Size());
}
quantized_weights.resize(num_weight_storage_elems);
QuantizeValues<float, WeightQType>(weights_def.GetRawData(), quantized_weights, weights_shape,
weight_scales, weight_zero_points, weight_quant_axis);
@ -727,6 +732,80 @@ TEST_F(QnnHTPBackendTests, ConvU8S8S32_PerChannel) {
13); // opset
}
// Test per-channel QDQ Conv with INT4 weights. in0: u16, in1 (weight): s4, in2 (bias): s32, out: u8
TEST_F(QnnHTPBackendTests, ConvU16S4S32_PerChannel) {
std::vector<int64_t> input_shape = {1, 2, 4, 4};
std::vector<int64_t> weight_shape = {3, 2, 2, 2};
std::vector<int64_t> bias_shape = {3};
TestInputDef<float> input_def(input_shape, false,
GetFloatDataInRange(0.0f, 1.0f, TensorShape(input_shape).Size()));
TestInputDef<float> weight_def(weight_shape, true,
GetFloatDataInRange(-1.0f, 5.0f, TensorShape(weight_shape).Size()));
TestInputDef<float> bias_def(bias_shape, true,
GetFloatDataInRange(-1.0f, 1.0f, TensorShape(bias_shape).Size()));
RunHTPConvOpPerChannelTest<uint8_t, Int4x2>("Conv",
input_def,
weight_def,
bias_def,
0, // weight quant axis
{1, 1}, // Strides
{0, 0, 0, 0}, // Pads
{1, 1}, // Dilations
1, // default group
"NOTSET",
ExpectedEPNodeAssignment::All,
false, // use_qdq_contrib_ops
21); // opset
}
// Test per-channel QDQ Conv with INT4 weights. in0: u16, in1 (weight): s4, in2 (bias): s32, out: u8
// TODO(adrianlizarraga): Investigate inaccuracy for QNN EP.
//
// Output values for all EPs:
// CPU EP (f32 model): 25.143 21.554 17.964 10.785 7.195 3.605 -3.574 -7.164 -10.753
// CPU EP (qdq model): 24.670 21.103 17.536 10.254 6.689 2.972 -4.161 -7.728 -10.700
// QNN EP (qdq model): 27.186 27.186 27.186 21.541 6.685 -8.022 -10.548 -10.548 -10.548
TEST_F(QnnHTPBackendTests, DISABLED_ConvU16S4S32_PerChannel_AccuracyIssue) {
std::vector<int64_t> input_shape = {1, 2, 4, 4};
std::vector<int64_t> weight_shape = {3, 2, 2, 2};
std::vector<int64_t> bias_shape = {3};
// Wrote out input data explicitly for easier reproduction.
// std::vector<float> input_data = GetFloatDataInRange(-10.0f, 10.0f, TensorShape(input_shape).Size());
std::vector<float> input_data = {-10.000f, -9.355f, -8.710f, -8.065f, -7.419f, -6.774f, -6.129f, -5.484f, -4.839f,
-4.194f, -3.548f, -2.903f, -2.258f, -1.613f, -0.968f, -0.323f, 0.323f, 0.968f,
1.613f, 2.258f, 2.903f, 3.548f, 4.194f, 4.839f, 5.484f, 6.129f, 6.774f,
7.419f, 8.065f, 8.710f, 9.355f, 10.000f};
// std::vector<float> weight_data = GetFloatDataInRange(-1.0f, 1.0f, TensorShape(weight_shape).Size());
std::vector<float> weight_data = {-1.000f, -0.913f, -0.826f, -0.739f, -0.652f, -0.565f, -0.478f, -0.391f, -0.304f,
-0.217f, -0.130f, -0.043f, 0.043f, 0.130f, 0.217f, 0.304f, 0.391f, 0.478f,
0.565f, 0.652f, 0.739f, 0.826f, 0.913f, 1.000f};
// std::vector<float> bias_data = GetFloatDataInRange(-1.0f, 1.0f, TensorShape(bias_shape).Size());
std::vector<float> bias_data = {-1.000f, 0.000f, 1.000f};
TestInputDef<float> input_def(input_shape, false, input_data);
TestInputDef<float> weight_def(weight_shape, true, weight_data);
TestInputDef<float> bias_def(bias_shape, true, bias_data);
RunHTPConvOpPerChannelTest<uint8_t, Int4x2>("Conv",
input_def,
weight_def,
bias_def,
0, // weight quant axis
{1, 1}, // Strides
{0, 0, 0, 0}, // Pads
{1, 1}, // Dilations
1, // default group
"NOTSET",
ExpectedEPNodeAssignment::All,
false, // use_qdq_contrib_ops
21); // opset
}
// Test per-channel QDQ Conv is rejected with weight axis != 0
TEST_F(QnnHTPBackendTests, Conv_PerChannel_UnsupportedAxis) {
std::vector<int64_t> input_shape = {1, 2, 4, 4};

45
onnxruntime/test/providers/qnn/qnn_basic_test.cc
View file

@ -236,6 +236,51 @@ TEST_F(QnnHTPBackendTests, TestConvWithExternalData) {
Ort::Session session(*ort_env, ort_model_path, so);
}
#if defined(__aarch64__) || defined(_M_ARM64) || defined(__linux__)
TEST_F(QnnHTPBackendTests, RunConvInt4Model) {
Ort::SessionOptions so;
so.AddConfigEntry(kOrtSessionOptionsDisableCPUEPFallback, "1"); // Disable fallback to the CPU EP.
so.SetGraphOptimizationLevel(ORT_ENABLE_ALL);
onnxruntime::ProviderOptions options;
#if defined(_WIN32)
options["backend_path"] = "QnnHtp.dll";
#else
options["backend_path"] = "libQnnHtp.so";
#endif
so.AppendExecutionProvider("QNN", options);
const ORTCHAR_T* ort_model_path = ORT_MODEL_FOLDER "conv.int4_weights.qdq.onnx";
Ort::Session session(*ort_env, ort_model_path, so);
TensorShape input_shape = {1, 3, 8, 8};
std::vector<float> input0_data(input_shape.Size(), 0.2f);
auto memory_info = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU);
std::vector<Ort::Value> ort_inputs;
std::vector<const char*> ort_input_names;
// Add input0
ort_inputs.emplace_back(Ort::Value::CreateTensor<float>(
memory_info, input0_data.data(), input0_data.size(), &input_shape[0], input_shape.NumDimensions()));
ort_input_names.push_back("input_0");
// Run session and get outputs
std::array<const char*, 1> output_names{"output_0"};
std::vector<Ort::Value> ort_outputs = session.Run(Ort::RunOptions{nullptr}, ort_input_names.data(), ort_inputs.data(),
ort_inputs.size(), output_names.data(), output_names.size());
// Check output shape.
Ort::Value& ort_output = ort_outputs[0];
auto typeshape = ort_output.GetTensorTypeAndShapeInfo();
std::vector<int64_t> output_shape = typeshape.GetShape();
EXPECT_THAT(output_shape, ::testing::ElementsAre(1, 5, 6, 6));
}
#endif // #if defined(__aarch64__) || defined(_M_ARM64) || defined(__linux__)
// Helper function that runs an ONNX model with a NHWC Resize operator to test that
// type/shape inference succeeds during layout transformation.
// Refer to onnxruntime/core/graph/contrib_ops/nhwc_inference_context.h.

106
onnxruntime/test/providers/qnn/qnn_test_utils.h
View file

@ -34,6 +34,15 @@ struct QuantParams {
QType zero_point;
static QuantParams<QType> Compute(float rmin, float rmax, bool symmetric = false) {
return Compute(
rmin,
rmax,
static_cast<float>(std::numeric_limits<QType>::min()),
static_cast<float>(std::numeric_limits<QType>::max()),
symmetric);
}
static QuantParams<QType> Compute(float rmin, float rmax, QType qmin, QType qmax, bool symmetric = false) {
// Ensure a minimum range of 0.0001 (required by QNN)
rmax = std::max(rmax, rmin + 0.0001f);
@ -41,27 +50,27 @@ struct QuantParams {
rmin = std::min(rmin, 0.0f);
rmax = std::max(rmax, 0.0f);
constexpr float qmin = static_cast<float>(std::numeric_limits<QType>::min());
constexpr float qmax = static_cast<float>(std::numeric_limits<QType>::max());
if (symmetric) {
const float abs_max = std::max(std::abs(rmin), std::abs(rmax));
rmax = abs_max;
rmin = -abs_max;
}
const float scale = (rmax - rmin) / (qmax - qmin);
float qmin_flt = static_cast<float>(qmin);
float qmax_flt = static_cast<float>(qmax);
const float scale = (rmax - rmin) / (qmax_flt - qmin_flt);
float initial_zero_point = 0.0f;
if (symmetric) {
// Symmetric uses same formula for zero-point as asymmetric, but we can cancel out terms for
// increased numerical accuracy.
initial_zero_point = (qmin + qmax) / 2.0f;
initial_zero_point = (qmin_flt + qmax_flt) / 2.0f;
} else {
initial_zero_point = qmin - (rmin / scale);
initial_zero_point = qmin_flt - (rmin / scale);
}
const QType zero_point = static_cast<QType>(RoundHalfToEven(std::max(qmin, std::min(qmax, initial_zero_point))));
const QType zero_point = static_cast<QType>(RoundHalfToEven(std::max(qmin_flt,
std::min(qmax_flt, initial_zero_point))));
return QuantParams<QType>{scale, zero_point};
}
@ -238,7 +247,7 @@ struct TestInputDef {
assert(which_type == 0);
const std::vector<T>& raw_data = std::get<RawData>(data_info_).data;
std::pair<T, T> init_range(std::numeric_limits<T>::max(), std::numeric_limits<T>::min());
std::pair<T, T> init_range(std::numeric_limits<T>::max(), std::numeric_limits<T>::lowest());
std::vector<std::pair<T, T>> per_axis_ranges(num_ranges, init_range);
TensorShape shape(shape_);
size_t num_blocks = shape.SizeToDimension(axis);
@ -292,6 +301,37 @@ static void GetTestInputQuantParamsPerChannel(const TestInputDef<float>& input_d
}
}
// Define functions to get the quantization parameters (i.e., scale/zp) for input data that will be quantized
// as int4 per-channel.
#define DEF_GET_INPUT_QPARAMS_PER_CHAN_INT4_FUNC(INT4x2_TYPE) \
template <> \
inline void GetTestInputQuantParamsPerChannel<INT4x2_TYPE>(const TestInputDef<float>& input_def, \
std::vector<float>& scales, \
std::vector<INT4x2_TYPE>& zero_points, \
size_t axis, bool symmetric) { \
using UnpackedType = typename INT4x2_TYPE::UnpackedType; \
const auto f32_ranges = input_def.GetRangePerChannel(axis); \
const size_t num_ranges = f32_ranges.size(); \
\
scales.resize(num_ranges); \
zero_points.resize(INT4x2_TYPE::CalcNumInt4Pairs(num_ranges)); \
\
for (size_t i = 0; i < num_ranges; i++) { \
const auto& range = f32_ranges[i]; \
QuantParams<UnpackedType> params = QuantParams<UnpackedType>::Compute(range.first, range.second, \
INT4x2_TYPE::min_val, \
INT4x2_TYPE::max_val, symmetric); \
scales[i] = params.scale; \
\
size_t r = i >> 1; \
size_t c = i & 0x1; \
zero_points[r].SetElem(c, params.zero_point); \
} \
}
DEF_GET_INPUT_QPARAMS_PER_CHAN_INT4_FUNC(Int4x2)
DEF_GET_INPUT_QPARAMS_PER_CHAN_INT4_FUNC(UInt4x2)
template <typename FloatType, typename QuantType>
static void QuantizeValues(gsl::span<const FloatType> input, gsl::span<QuantType> output, const TensorShape& shape,
gsl::span<const FloatType> scales, gsl::span<const QuantType> zero_points,
@ -332,6 +372,52 @@ static void QuantizeValues(gsl::span<const FloatType> input, gsl::span<QuantType
}
}
// Define functions to quantize input data to 4-bits. Quantization can be done per-tensor or per-channel.
#define DEF_QUANTIZE_VALUES_INT4_FUNC(INT4x2_TYPE, QUANT_FUNC) \
template <> \
inline void QuantizeValues<float, INT4x2_TYPE>(gsl::span<const float> input, \
gsl::span<INT4x2_TYPE> output, \
const TensorShape& shape, \
gsl::span<const float> scales, \
gsl::span<const INT4x2_TYPE> zero_points, \
std::optional<int64_t> axis) { \
using UnpackedType = typename INT4x2_TYPE::UnpackedType; \
const size_t input_rank = shape.NumDimensions(); \
const size_t num_int4_elems = static_cast<size_t>(shape.Size()); \
ORT_ENFORCE(input.size() == num_int4_elems); \
ORT_ENFORCE(output.size() == INT4x2_TYPE::CalcNumInt4Pairs(num_int4_elems)); \
\
size_t block_count = 1; \
size_t broadcast_dim = 1; \
size_t block_size = num_int4_elems; \
\
if (axis.has_value()) { \
size_t axis_no_neg = *axis < 0 ? static_cast<size_t>(*axis) + input_rank : static_cast<size_t>(*axis); \
block_count = shape.SizeToDimension(axis_no_neg); \
broadcast_dim = shape[axis_no_neg]; \
block_size = shape.SizeFromDimension(axis_no_neg + 1); \
} \
\
ORT_ENFORCE(scales.size() == broadcast_dim); \
ORT_ENFORCE(zero_points.empty() || zero_points.size() == INT4x2_TYPE::CalcNumInt4Pairs(broadcast_dim)); \
\
size_t i = 0; \
\
for (size_t n = 0; n < block_count; n++) { \
for (size_t bd = 0; bd < broadcast_dim; bd++) { \
size_t bd_i = bd >> 1; /* bd / 2 */ \
size_t bd_j = bd & 0x1; /* bd % 2 */ \
UnpackedType zp = !zero_points.empty() ? zero_points[bd_i].GetElem(bd_j) : 0; \
QUANT_FUNC(&input[i], output.data(), i, i + block_size, scales[bd], INT4x2_TYPE(zp, 0), nullptr); \
i += block_size; \
} \
} \
assert(i == (block_count * broadcast_dim * block_size)); \
}
DEF_QUANTIZE_VALUES_INT4_FUNC(Int4x2, ParQuantizeLinearStdS4)
DEF_QUANTIZE_VALUES_INT4_FUNC(UInt4x2, ParQuantizeLinearStdU4)
/**
* Inferences a given serialized model. Returns output values via an out-param.
*
@ -414,6 +500,10 @@ inline void TestQDQModelAccuracy(const GetTestModelFn& f32_model_fn, const GetTe
const std::unordered_map<std::string, int> domain_to_version = {{"", opset_version}, {kMSDomain, 1}};
auto& logging_manager = DefaultLoggingManager();
// Uncomment to dump LOGGER() output to stdout.
// logging_manager.RemoveSink(logging::SinkType::EtwSink);
logging_manager.SetDefaultLoggerSeverity(log_severity);
// Create float model and serialize it to a string.

BIN
onnxruntime/test/testdata/conv.int4_weights.qdq.onnx vendored Normal file
View file

Binary file not shown.

98
onnxruntime/test/testdata/make_conv_int4_weights_model.py vendored Normal file
View file

@ -0,0 +1,98 @@
import numpy as np
import onnx
from onnxruntime.quantization import CalibrationDataReader, QuantType, quantize
from onnxruntime.quantization.execution_providers.qnn import get_qnn_qdq_config
INPUT0_SHAPE = (1, 3, 8, 8)
INPUT0_NAME = "input_0"
def create_f32_model():
input_0 = onnx.helper.make_tensor_value_info(INPUT0_NAME, onnx.TensorProto.FLOAT, INPUT0_SHAPE)
output_0 = onnx.helper.make_tensor_value_info("output_0", onnx.TensorProto.FLOAT, None)
weight_data = [
[
[[-1.5, -1.0, -0.5], [-0.2, 0.0, 0.2], [0.5, 1.0, 1.5]], # range = 3.0, scale = 3.0/15, zp = 0
[[-1.5, -1.0, -0.5], [-0.2, 0.0, 0.2], [0.5, 1.0, 1.5]], # range = 3.0, scale = 3.0/15, zp = 0
[[-1.5, -1.0, -0.5], [-0.2, 0.0, 0.2], [0.5, 1.0, 1.5]], # range = 3.0, scale = 3.0/15, zp = 0
],
[
[[-1.0, -0.8, -0.6], [-0.1333, 0.0, -0.1333], [0.6, 0.8, 1.0]], # range = 2.0, scale = 2.0/15, zp = -3
[[-1.0, -0.8, -0.6], [-0.1333, 0.0, -0.1333], [0.6, 0.8, 1.0]], # range = 2.0, scale = 2.0/15, zp = -3
[[-1.0, -0.8, -0.6], [-0.1333, 0.0, -0.1333], [0.6, 0.8, 1.0]], # range = 2.0, scale = 2.0/15, zp = -3
],
[
[[-1.5, -1.0, -0.5], [-0.2, 0.0, 0.2], [0.5, 1.0, 1.5]], # range = 3.0, scale = 3.0/15, zp = 0
[[-1.5, -1.0, -0.5], [-0.2, 0.0, 0.2], [0.5, 1.0, 1.5]], # range = 3.0, scale = 3.0/15, zp = 0
[[-1.5, -1.0, -0.5], [-0.2, 0.0, 0.2], [0.5, 1.0, 1.5]], # range = 3.0, scale = 3.0/15, zp = 0
],
[
[[-1.0, -0.8, -0.6], [-0.1333, 0.0, -0.1333], [0.6, 0.8, 1.0]], # range = 2.0, scale = 2.0/15, zp = -3
[[-1.0, -0.8, -0.6], [-0.1333, 0.0, -0.1333], [0.6, 0.8, 1.0]], # range = 2.0, scale = 2.0/15, zp = -3
[[-1.0, -0.8, -0.6], [-0.1333, 0.0, -0.1333], [0.6, 0.8, 1.0]], # range = 2.0, scale = 2.0/15, zp = -3
],
[
[[-1.5, -1.0, -0.5], [-0.2, 0.0, 0.2], [0.5, 1.0, 1.5]], # range = 3.0, scale = 3.0/15, zp = 0
[[-1.5, -1.0, -0.5], [-0.2, 0.0, 0.2], [0.5, 1.0, 1.5]], # range = 3.0, scale = 3.0/15, zp = 0
[[-1.5, -1.0, -0.5], [-0.2, 0.0, 0.2], [0.5, 1.0, 1.5]], # range = 3.0, scale = 3.0/15, zp = 0
],
]
weight = onnx.numpy_helper.from_array(np.array(weight_data, dtype=np.float32), "weight")
bias_data = [-10.0, -8.0, 0.0, 8.0, 10.0]
bias = onnx.numpy_helper.from_array(np.array(bias_data, dtype=np.float32), "bias")
conv_node = onnx.helper.make_node("Conv", [INPUT0_NAME, "weight", "bias"], ["output_0"], name="Conv0")
graph = onnx.helper.make_graph(
[conv_node],
"Convf32",
[input_0],
[output_0],
initializer=[weight, bias],
)
opset_imports = [onnx.helper.make_opsetid("", 21)]
model = onnx.helper.make_model(graph, opset_imports=opset_imports)
model = onnx.shape_inference.infer_shapes(model)
onnx.checker.check_model(model, True)
return model
class DataReader(CalibrationDataReader):
def __init__(self):
self.enum_data = None
self.data_list = []
# Generate 10 random input values for calibration
for _ in range(10):
input_data = {INPUT0_NAME: np.random.random(INPUT0_SHAPE).astype(np.float32)}
self.data_list.append(input_data)
self.datasize = len(self.data_list)
def get_next(self):
if self.enum_data is None:
self.enum_data = iter(self.data_list)
return next(self.enum_data, None)
def rewind(self):
self.enum_data = None
def create_qdq_model(model_f32):
# Use tensor quantization overrides to quantize Conv's weight input to 4 bits on axis 0.
init_overrides = {"weight": [{"quant_type": QuantType.QInt4, "axis": 0, "symmetric": True}]}
qnn_config = get_qnn_qdq_config(
model_f32,
DataReader(),
init_overrides=init_overrides,
activation_type=QuantType.QUInt16,
weight_type=QuantType.QUInt8,
)
quantize(model_f32, "conv.int4_weights.qdq.onnx", qnn_config)
if __name__ == "__main__":
model_f32 = create_f32_model()
create_qdq_model(model_f32)