diff --git a/onnxruntime/core/providers/qnn/builder/opbuilder/conv_op_builder.cc b/onnxruntime/core/providers/qnn/builder/opbuilder/conv_op_builder.cc index 1713f201c9..5283e9a559 100644 --- a/onnxruntime/core/providers/qnn/builder/opbuilder/conv_op_builder.cc +++ b/onnxruntime/core/providers/qnn/builder/opbuilder/conv_op_builder.cc @@ -127,7 +127,8 @@ Status ConvOpBuilder::IsOpSupported(QnnModelWrapper& qnn_model_wrapper, int32_t elem_data_type = 0; ORT_RETURN_IF_ERROR(utils::GetOnnxTensorElemDataType(input_1.node_arg, elem_data_type)); - const bool is_signed_type = (elem_data_type == ONNX_NAMESPACE::TensorProto_DataType_INT8) || + const bool is_signed_type = (elem_data_type == ONNX_NAMESPACE::TensorProto_DataType_INT4) || + (elem_data_type == ONNX_NAMESPACE::TensorProto_DataType_INT8) || (elem_data_type == ONNX_NAMESPACE::TensorProto_DataType_INT16); ORT_RETURN_IF_NOT(is_signed_type, "Conv weights must be of a signed quantized type if quantized per-channel"); diff --git a/onnxruntime/core/providers/qnn/builder/qnn_model_wrapper.cc b/onnxruntime/core/providers/qnn/builder/qnn_model_wrapper.cc index 3a8a8af17b..f85cdc401a 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_model_wrapper.cc +++ b/onnxruntime/core/providers/qnn/builder/qnn_model_wrapper.cc @@ -5,6 +5,8 @@ #include #include #include +#include +#include #include "qnn_model_wrapper.h" #include "core/common/safeint.h" @@ -313,7 +315,8 @@ bool QnnModelWrapper::GetOnnxShape(const NodeArg& node_arg, std::vector& zero_points) const { + /*out*/ std::vector& zero_points, + /*out*/ int32_t& onnx_data_type) const { const auto& graph_initializers = GetInitializerTensors(); auto iter = graph_initializers.find(initializer_name); ORT_RETURN_IF(iter == graph_initializers.end(), "Unable to find initializer for zero-point(s): ", @@ -323,13 +326,14 @@ Status QnnModelWrapper::UnpackZeroPoints(const std::string& initializer_name, ORT_RETURN_IF_NOT(zp_tensor_proto->has_data_type(), "Expected zero-point initializer ", initializer_name.c_str(), " to have a proto data type."); - const int32_t onnx_data_type = zp_tensor_proto->data_type(); + onnx_data_type = zp_tensor_proto->data_type(); std::vector initializer_bytes; ORT_RETURN_IF_ERROR(UnpackInitializerData(*zp_tensor_proto, initializer_bytes)); switch (onnx_data_type) { // QNN use -offset for some reason + case ONNX_NAMESPACE::TensorProto_DataType_INT4: // INT4 zero-points are unpacked as 8-bit values for QNN case ONNX_NAMESPACE::TensorProto_DataType_INT8: { auto int8_span = ReinterpretAsSpan(gsl::make_span(initializer_bytes)); std::transform(int8_span.begin(), int8_span.end(), std::back_inserter(zero_points), @@ -338,6 +342,7 @@ Status QnnModelWrapper::UnpackZeroPoints(const std::string& initializer_name, }); break; } + case ONNX_NAMESPACE::TensorProto_DataType_UINT4: // UINT4 zero-points are unpacked as 8-bit values for QNN case ONNX_NAMESPACE::TensorProto_DataType_UINT8: { auto uint8_span = ReinterpretAsSpan(gsl::make_span(initializer_bytes)); std::transform(uint8_span.begin(), uint8_span.end(), std::back_inserter(zero_points), @@ -584,10 +589,36 @@ void QnnModelWrapper::GetGraphInputOutputTensorWrapper(const std::vector& unpacked_tensor) const { if (initializer.data_location() == onnx::TensorProto_DataLocation_EXTERNAL) { - return onnxruntime::utils::UnpackInitializerData(initializer, graph_viewer_.ModelPath(), unpacked_tensor); + ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(initializer, graph_viewer_.ModelPath(), + unpacked_tensor)); + } else { + ORT_RETURN_IF_ERROR(onnxruntime::utils::UnpackInitializerData(initializer, unpacked_tensor)); } - return onnxruntime::utils::UnpackInitializerData(initializer, unpacked_tensor); + int32_t onnx_data_type = initializer.data_type(); + + // If this is an int4, we need to unpack it because QNN treats int4 as a full int8. + if (onnx_data_type == ONNX_NAMESPACE::TensorProto_DataType_INT4) { + TensorShape shape = onnxruntime::utils::GetTensorShapeFromTensorProto(initializer); + const size_t num_elems = shape.Size(); + std::vector packed_int4_bytes = std::move(unpacked_tensor); + unpacked_tensor = std::vector(num_elems); + + auto dst = gsl::make_span(reinterpret_cast(unpacked_tensor.data()), unpacked_tensor.size()); + auto src = gsl::make_span(reinterpret_cast(packed_int4_bytes.data()), packed_int4_bytes.size()); + ORT_RETURN_IF_NOT(Int4x2::Unpack(dst, src), "Failed to unpack Tensor for QNN"); + } else if (onnx_data_type == ONNX_NAMESPACE::TensorProto_DataType_UINT4) { + TensorShape shape = onnxruntime::utils::GetTensorShapeFromTensorProto(initializer); + const size_t num_elems = shape.Size(); + std::vector packed_int4_bytes = std::move(unpacked_tensor); + unpacked_tensor = std::vector(num_elems); + + auto dst = gsl::make_span(reinterpret_cast(unpacked_tensor.data()), unpacked_tensor.size()); + auto src = gsl::make_span(reinterpret_cast(packed_int4_bytes.data()), packed_int4_bytes.size()); + ORT_RETURN_IF_NOT(UInt4x2::Unpack(dst, src), "Failed to unpack Tensor for QNN"); + } + + return Status::OK(); } } // namespace qnn diff --git a/onnxruntime/core/providers/qnn/builder/qnn_model_wrapper.h b/onnxruntime/core/providers/qnn/builder/qnn_model_wrapper.h index 0705a1d1b8..9ab122b7f8 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_model_wrapper.h +++ b/onnxruntime/core/providers/qnn/builder/qnn_model_wrapper.h @@ -216,7 +216,9 @@ class QnnModelWrapper { Status UnpackScales(const std::string& initializer_name, std::vector& scales) const; // Unpack zero-points from initializer and convert to int32_t (1 zero-point for per-tensor, > 1 for per-channel). - Status UnpackZeroPoints(const std::string& initializer_name, std::vector& zero_points) const; + Status UnpackZeroPoints(const std::string& initializer_name, + /*out*/ std::vector& zero_points, + /*out*/ int32_t& onnx_data_type) const; // Checks if a tensor in the ONNX graph is per-channel quantized. Status IsPerChannelQuantized(const onnxruntime::NodeUnitIODef& io_def, diff --git a/onnxruntime/core/providers/qnn/builder/qnn_quant_params_wrapper.cc b/onnxruntime/core/providers/qnn/builder/qnn_quant_params_wrapper.cc index 401d403c15..2d22c3c1b8 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_quant_params_wrapper.cc +++ b/onnxruntime/core/providers/qnn/builder/qnn_quant_params_wrapper.cc @@ -9,6 +9,9 @@ #include "QnnTypes.h" #include "core/providers/qnn/builder/qnn_model_wrapper.h" +#define ALIGN_PTR_UP(ptr, align, type) \ + reinterpret_cast((reinterpret_cast(ptr) + (align)-1) & ~((align)-1)) + namespace onnxruntime { namespace qnn { @@ -38,9 +41,10 @@ QnnQuantParamsWrapper QnnQuantParamsWrapper::Copy() const { return QnnQuantParamsWrapper(*this); } +// Initializes by copying from a Qnn_QuantizeParams_t. Status QnnQuantParamsWrapper::Init(const Qnn_QuantizeParams_t& params) { - if (scale_offset_data_) { - scale_offset_data_.reset(nullptr); + if (per_channel_data_) { + per_channel_data_.reset(nullptr); params_ = QNN_QUANTIZE_PARAMS_INIT; } @@ -51,6 +55,7 @@ Status QnnQuantParamsWrapper::Init(const Qnn_QuantizeParams_t& params) { switch (params.quantizationEncoding) { case QNN_QUANTIZATION_ENCODING_SCALE_OFFSET: + case QNN_QUANTIZATION_ENCODING_BW_SCALE_OFFSET: params_ = params; break; case QNN_QUANTIZATION_ENCODING_AXIS_SCALE_OFFSET: { @@ -63,15 +68,49 @@ Status QnnQuantParamsWrapper::Init(const Qnn_QuantizeParams_t& params) { const uint32_t num_elems = params.axisScaleOffsetEncoding.numScaleOffsets; if (num_elems > 0) { - scale_offset_data_ = std::make_unique(num_elems); - gsl::span src_span(params.axisScaleOffsetEncoding.scaleOffset, num_elems); - std::copy(src_span.begin(), src_span.end(), scale_offset_data_.get()); - params_.axisScaleOffsetEncoding.scaleOffset = scale_offset_data_.get(); + const size_t num_bytes = num_elems * sizeof(Qnn_ScaleOffset_t); + constexpr std::uintptr_t align = alignof(Qnn_ScaleOffset_t); + per_channel_data_ = std::make_unique(num_bytes + align); + Qnn_ScaleOffset_t* aligned_dst = ALIGN_PTR_UP(per_channel_data_.get(), align, Qnn_ScaleOffset_t*); + + std::memcpy(aligned_dst, params.axisScaleOffsetEncoding.scaleOffset, num_bytes); + params_.axisScaleOffsetEncoding.scaleOffset = aligned_dst; } else { params_.axisScaleOffsetEncoding.scaleOffset = nullptr; } break; } + case QNN_QUANTIZATION_ENCODING_BW_AXIS_SCALE_OFFSET: { + const uint32_t num_elems = params.bwAxisScaleOffsetEncoding.numElements; + + params_.encodingDefinition = params.encodingDefinition; + params_.quantizationEncoding = params.quantizationEncoding; + params_.bwAxisScaleOffsetEncoding.axis = params.bwAxisScaleOffsetEncoding.axis; + params_.bwAxisScaleOffsetEncoding.bitwidth = params.bwAxisScaleOffsetEncoding.bitwidth; + params_.bwAxisScaleOffsetEncoding.numElements = num_elems; + + // Deep copy the scales[] and offsets[] arrays + if (num_elems > 0) { + const size_t num_scale_bytes = num_elems * sizeof(float); + const size_t num_zp_bytes = num_elems * sizeof(int32_t); + const size_t num_bytes = num_scale_bytes + num_zp_bytes; + constexpr std::uintptr_t align = alignof(float); + static_assert(alignof(float) == alignof(int32_t)); + + per_channel_data_ = std::make_unique(num_bytes + align); + char* scales_begin = ALIGN_PTR_UP(per_channel_data_.get(), align, char*); + char* zps_begin = scales_begin + num_scale_bytes; + + std::memcpy(scales_begin, params.bwAxisScaleOffsetEncoding.scales, num_scale_bytes); + std::memcpy(zps_begin, params.bwAxisScaleOffsetEncoding.offsets, num_zp_bytes); + params_.bwAxisScaleOffsetEncoding.scales = reinterpret_cast(scales_begin); + params_.bwAxisScaleOffsetEncoding.offsets = reinterpret_cast(zps_begin); + } else { + params_.bwAxisScaleOffsetEncoding.scales = nullptr; + params_.bwAxisScaleOffsetEncoding.offsets = nullptr; + } + break; + } default: return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Unsupported QNN quantization encoding: ", params.quantizationEncoding); } @@ -79,11 +118,13 @@ Status QnnQuantParamsWrapper::Init(const Qnn_QuantizeParams_t& params) { return Status::OK(); } +// Initialize this object from a (potentially) quantized ONNX tensor. +// QnnModelWrapper provides utilities for unpacking scale and zero-point ONNX initializers. Status QnnQuantParamsWrapper::Init(const QnnModelWrapper& qnn_model_wrapper, const NodeUnitIODef& io_def) { const std::optional& ort_quant_params = io_def.quant_param; - if (scale_offset_data_) { - scale_offset_data_.reset(nullptr); + if (per_channel_data_) { + per_channel_data_.reset(nullptr); params_ = QNN_QUANTIZE_PARAMS_INIT; } @@ -98,17 +139,25 @@ Status QnnQuantParamsWrapper::Init(const QnnModelWrapper& qnn_model_wrapper, con ORT_RETURN_IF_ERROR(qnn_model_wrapper.UnpackScales(ort_quant_params->scale.Name(), scales)); + bool is_int4_type = false; + if (ort_quant_params->zero_point != nullptr) { - ORT_RETURN_IF_ERROR(qnn_model_wrapper.UnpackZeroPoints(ort_quant_params->zero_point->Name(), zero_points)); + int32_t onnx_tp_type = 0; + ORT_RETURN_IF_ERROR(qnn_model_wrapper.UnpackZeroPoints(ort_quant_params->zero_point->Name(), zero_points, + onnx_tp_type)); + + is_int4_type = (onnx_tp_type == ONNX_NAMESPACE::TensorProto_DataType_INT4) || + (onnx_tp_type == ONNX_NAMESPACE::TensorProto_DataType_UINT4); } const bool is_per_tensor = scales.size() == 1; - if (is_per_tensor) { + // QNN uses different structs to represent quantization parameters depending on + // - per-tensor vs per-channel + // - int4 vs not int4 + if (is_per_tensor && !is_int4_type) { params_.encodingDefinition = QNN_DEFINITION_DEFINED; params_.quantizationEncoding = QNN_QUANTIZATION_ENCODING_SCALE_OFFSET; - - // Parse scale & zero_point params_.scaleOffsetEncoding.scale = scales[0]; if (ort_quant_params->zero_point != nullptr) { @@ -117,8 +166,62 @@ Status QnnQuantParamsWrapper::Init(const QnnModelWrapper& qnn_model_wrapper, con } else { params_.scaleOffsetEncoding.offset = 0; } - } else { - // Per-channel quantization. + } else if (is_per_tensor && is_int4_type) { + params_.encodingDefinition = QNN_DEFINITION_DEFINED; + params_.quantizationEncoding = QNN_QUANTIZATION_ENCODING_BW_SCALE_OFFSET; + params_.bwScaleOffsetEncoding.bitwidth = 4; + params_.bwScaleOffsetEncoding.scale = scales[0]; + + if (ort_quant_params->zero_point != nullptr) { + ORT_RETURN_IF_NOT(zero_points.size() == 1, "Expected one zero-point value"); + params_.bwScaleOffsetEncoding.offset = zero_points[0]; + } else { + params_.bwScaleOffsetEncoding.offset = 0; + } + } else if (!is_per_tensor && is_int4_type) { + const auto* io_shape = io_def.node_arg.Shape(); + ORT_RETURN_IF(io_shape == nullptr, "Input/output tensor proto must have a shape"); + const int32_t io_rank = io_shape->dim_size(); + + constexpr int64_t DEFAULT_QDQ_AXIS = 1; + int64_t axis = ort_quant_params->axis.value_or(DEFAULT_QDQ_AXIS); + if (axis < 0) { + axis += io_rank; + } + ORT_RETURN_IF_NOT(axis >= 0 && axis < io_rank, + "Quantization axis must be within the range [0, rank - 1]"); + + const size_t num_elems = scales.size(); + const bool no_zero_points = zero_points.empty(); + ORT_RETURN_IF_NOT(num_elems > 1, "Expected more than one scale value"); + ORT_RETURN_IF_NOT(no_zero_points || zero_points.size() == num_elems, + "Expected the same number of zero-points and scales for per-channel quantization"); + + params_.encodingDefinition = QNN_DEFINITION_DEFINED; + params_.quantizationEncoding = QNN_QUANTIZATION_ENCODING_BW_AXIS_SCALE_OFFSET; + params_.bwAxisScaleOffsetEncoding.axis = static_cast(*(ort_quant_params->axis)); + params_.bwAxisScaleOffsetEncoding.bitwidth = 4; + params_.bwAxisScaleOffsetEncoding.numElements = static_cast(num_elems); + + const size_t num_scale_bytes = num_elems * sizeof(float); + const size_t num_zp_bytes = num_elems * sizeof(int32_t); + const size_t num_bytes = num_scale_bytes + num_zp_bytes; + constexpr std::uintptr_t align = alignof(float); + per_channel_data_ = std::make_unique(num_bytes + align); + + char* scales_begin = ALIGN_PTR_UP(per_channel_data_.get(), align, char*); + char* zps_begin = scales_begin + num_scale_bytes; + gsl::span scales_span(reinterpret_cast(scales_begin), num_elems); + gsl::span zps_span(reinterpret_cast(zps_begin), num_elems); + + for (size_t i = 0; i < num_elems; i++) { + scales_span[i] = scales[i]; + zps_span[i] = no_zero_points ? 0 : zero_points[i]; + } + + params_.bwAxisScaleOffsetEncoding.scales = scales_span.data(); + params_.bwAxisScaleOffsetEncoding.offsets = zps_span.data(); + } else if (!is_per_tensor && !is_int4_type) { const auto* io_shape = io_def.node_arg.Shape(); ORT_RETURN_IF(io_shape == nullptr, "Input/output tensor proto must have a shape"); const int32_t io_rank = io_shape->dim_size(); @@ -140,8 +243,11 @@ Status QnnQuantParamsWrapper::Init(const QnnModelWrapper& qnn_model_wrapper, con ORT_RETURN_IF_NOT(no_zero_points || zero_points.size() == num_elems, "Expected the same number of zero-points and scales for per-channel quantization"); - scale_offset_data_ = std::make_unique(num_elems); - gsl::span data_span(scale_offset_data_.get(), num_elems); + const size_t num_bytes = num_elems * sizeof(Qnn_ScaleOffset_t); + constexpr std::uintptr_t align = alignof(Qnn_ScaleOffset_t); + per_channel_data_ = std::make_unique(num_bytes + align); + Qnn_ScaleOffset_t* aligned_dst = ALIGN_PTR_UP(per_channel_data_.get(), align, Qnn_ScaleOffset_t*); + gsl::span data_span(aligned_dst, num_elems); for (size_t i = 0; i < num_elems; i++) { data_span[i].scale = scales[i]; @@ -151,6 +257,8 @@ Status QnnQuantParamsWrapper::Init(const QnnModelWrapper& qnn_model_wrapper, con params_.axisScaleOffsetEncoding.axis = static_cast(axis); params_.axisScaleOffsetEncoding.numScaleOffsets = static_cast(num_elems); params_.axisScaleOffsetEncoding.scaleOffset = data_span.data(); + } else { + return ORT_MAKE_STATUS(ONNXRUNTIME, FAIL, "Unexpected tensor kind for QuantParamsWrapper::Init()"); } return Status::OK(); diff --git a/onnxruntime/core/providers/qnn/builder/qnn_quant_params_wrapper.h b/onnxruntime/core/providers/qnn/builder/qnn_quant_params_wrapper.h index 8fa7d41410..d1f93e5a69 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_quant_params_wrapper.h +++ b/onnxruntime/core/providers/qnn/builder/qnn_quant_params_wrapper.h @@ -48,17 +48,17 @@ class QnnQuantParamsWrapper { (include_bw && params_.quantizationEncoding == QNN_QUANTIZATION_ENCODING_BW_SCALE_OFFSET)); } - bool IsPerChannel(bool include_bw = false) const { + bool IsPerChannel() const { return params_.encodingDefinition == QNN_DEFINITION_DEFINED && (params_.quantizationEncoding == QNN_QUANTIZATION_ENCODING_AXIS_SCALE_OFFSET || - (include_bw && params_.quantizationEncoding == QNN_QUANTIZATION_ENCODING_BW_AXIS_SCALE_OFFSET)); + (params_.quantizationEncoding == QNN_QUANTIZATION_ENCODING_BW_AXIS_SCALE_OFFSET)); } // Handle transposing of a per-channel quantized tensor. The quantization parameter's axis // must be transposed using the inverse permutation of the Transpose. template Status HandleTranspose(gsl::span perm) { - if (!IsPerChannel(true)) { + if (!IsPerChannel()) { return Status::OK(); } @@ -82,7 +82,7 @@ class QnnQuantParamsWrapper { template Status HandleUnsqueeze(gsl::span orig_shape, gsl::span new_shape) { - if (!IsPerChannel(true)) { + if (!IsPerChannel()) { return Status::OK(); } @@ -134,7 +134,13 @@ class QnnQuantParamsWrapper { private: Qnn_QuantizeParams_t params_; - std::unique_ptr scale_offset_data_; // Stores per-channel scales and offsets + + // Stores arrays of per-channel scales and offsets. Fields in params_ point to this data. + // + // Use an opaque array of bytes because QNN uses different data layouts depending on the quantization encoding: + // - QNN_QUANTIZATION_ENCODING_AXIS_SCALE_OFFSET: array of scale/zp pairs [{scale0, zp0}, {scale1, zp1}, ...] + // - QNN_QUANTIZATION_ENCODING_BW_AXIS_SCALE_OFFSET: parallel arrays for scales and zps [scale0, ...] [zp0, zp1, ...] + std::unique_ptr per_channel_data_; }; } // namespace qnn diff --git a/onnxruntime/core/providers/qnn/builder/qnn_utils.cc b/onnxruntime/core/providers/qnn/builder/qnn_utils.cc index 19362daee6..dc56bc2231 100644 --- a/onnxruntime/core/providers/qnn/builder/qnn_utils.cc +++ b/onnxruntime/core/providers/qnn/builder/qnn_utils.cc @@ -43,6 +43,8 @@ size_t GetElementSizeByType(const Qnn_DataType_t& data_type) { } size_t GetElementSizeByType(ONNXTensorElementDataType elem_type) { const static std::unordered_map elem_type_to_size = { + {ONNX_TENSOR_ELEMENT_DATA_TYPE_INT4, sizeof(Int4x2)}, + {ONNX_TENSOR_ELEMENT_DATA_TYPE_UINT4, sizeof(UInt4x2)}, {ONNX_TENSOR_ELEMENT_DATA_TYPE_INT8, sizeof(int8_t)}, {ONNX_TENSOR_ELEMENT_DATA_TYPE_INT16, sizeof(int16_t)}, {ONNX_TENSOR_ELEMENT_DATA_TYPE_INT32, sizeof(int32_t)}, @@ -162,6 +164,12 @@ std::ostream& operator<<(std::ostream& out, const Qnn_DataType_t& data_type) { case QNN_DATATYPE_BOOL_8: out << "QNN_DATATYPE_BOOL_8"; break; + case QNN_DATATYPE_SFIXED_POINT_4: + out << "QNN_DATATYPE_SFIXED_POINT_4"; + break; + case QNN_DATATYPE_UFIXED_POINT_4: + out << "QNN_DATATYPE_UFIXED_POINT_4"; + break; default: ORT_THROW("Unknown Qnn Data type"); } @@ -216,6 +224,10 @@ std::ostream& operator<<(std::ostream& out, const Qnn_QuantizeParams_t& quantize if (quantize_params.quantizationEncoding == QNN_QUANTIZATION_ENCODING_SCALE_OFFSET) { out << " scale=" << quantize_params.scaleOffsetEncoding.scale; out << " offset=" << quantize_params.scaleOffsetEncoding.offset; + } else if (quantize_params.quantizationEncoding == QNN_QUANTIZATION_ENCODING_BW_SCALE_OFFSET) { + out << " bitwidth=" << quantize_params.bwScaleOffsetEncoding.bitwidth; + out << " scale=" << quantize_params.bwScaleOffsetEncoding.scale; + out << " offset=" << quantize_params.bwScaleOffsetEncoding.offset; } else if (quantize_params.quantizationEncoding == QNN_QUANTIZATION_ENCODING_AXIS_SCALE_OFFSET) { out << " axis=" << quantize_params.axisScaleOffsetEncoding.axis; size_t num_elems = quantize_params.axisScaleOffsetEncoding.numScaleOffsets; @@ -292,7 +304,9 @@ std::ostream& operator<<(std::ostream& out, const Qnn_ClientBuffer_t& client_buf T* data = reinterpret_cast(client_bufer.data); out << " dataSize=" << client_bufer.dataSize; uint32_t count = client_bufer.dataSize / sizeof(T); - count = count > 100 ? 100 : count; // limit to 100 data + const bool truncate = count > 100; + + count = truncate ? 100 : count; // limit to 100 data out << " clientBuf=("; for (uint32_t i = 0; i < count; i++) { if constexpr (sizeof(T) == 1) { @@ -301,7 +315,7 @@ std::ostream& operator<<(std::ostream& out, const Qnn_ClientBuffer_t& client_buf out << data[i] << " "; } } - out << ")"; + out << (truncate ? "..." : "") << ")"; return out; } @@ -432,10 +446,12 @@ bool OnnxDataTypeToQnnDataType(const int32_t onnx_data_type, Qnn_DataType_t& qnn }; const std::unordered_map onnx_to_qnn_data_type_quantized = { + {ONNX_NAMESPACE::TensorProto_DataType_INT4, QNN_DATATYPE_SFIXED_POINT_8}, {ONNX_NAMESPACE::TensorProto_DataType_INT8, QNN_DATATYPE_SFIXED_POINT_8}, {ONNX_NAMESPACE::TensorProto_DataType_INT16, QNN_DATATYPE_SFIXED_POINT_16}, {ONNX_NAMESPACE::TensorProto_DataType_INT32, QNN_DATATYPE_SFIXED_POINT_32}, {ONNX_NAMESPACE::TensorProto_DataType_INT64, QNN_DATATYPE_INT_64}, + {ONNX_NAMESPACE::TensorProto_DataType_UINT4, QNN_DATATYPE_UFIXED_POINT_8}, {ONNX_NAMESPACE::TensorProto_DataType_UINT8, QNN_DATATYPE_UFIXED_POINT_8}, {ONNX_NAMESPACE::TensorProto_DataType_UINT16, QNN_DATATYPE_UFIXED_POINT_16}, {ONNX_NAMESPACE::TensorProto_DataType_UINT32, QNN_DATATYPE_UFIXED_POINT_32}, diff --git a/onnxruntime/test/optimizer/graph_transform_test_builder.h b/onnxruntime/test/optimizer/graph_transform_test_builder.h index 1e2d34e5ae..0282d09f34 100644 --- a/onnxruntime/test/optimizer/graph_transform_test_builder.h +++ b/onnxruntime/test/optimizer/graph_transform_test_builder.h @@ -339,8 +339,10 @@ class ModelTestBuilder { bool use_ms_domain = false) { std::vector input_args; input_args.push_back(input_arg); - input_args.push_back(Make1DInitializer(input_scales)); - input_args.push_back(Make1DInitializer(input_zero_points)); + + std::vector qparams_shape = {static_cast(input_scales.size())}; + input_args.push_back(MakeInitializer(qparams_shape, input_scales)); + input_args.push_back(MakeInitializer(qparams_shape, input_zero_points)); std::string domain = use_ms_domain ? kMSDomain : ""; return AddNode("QuantizeLinear", input_args, {output_arg}, domain, attributes); @@ -415,8 +417,10 @@ class ModelTestBuilder { bool use_ms_domain = false) { std::vector input_args; input_args.push_back(input_arg); - input_args.push_back(Make1DInitializer(input_scales)); - input_args.push_back(Make1DInitializer(input_zero_points)); + + std::vector qparams_shape = {static_cast(input_scales.size())}; + input_args.push_back(MakeInitializer(qparams_shape, input_scales)); + input_args.push_back(MakeInitializer(qparams_shape, input_zero_points)); std::string domain = use_ms_domain ? kMSDomain : ""; return AddNode("DequantizeLinear", input_args, {output_arg}, domain, attributes); diff --git a/onnxruntime/test/providers/qnn/conv_test.cc b/onnxruntime/test/providers/qnn/conv_test.cc index 5177a629ce..b07951d2a2 100644 --- a/onnxruntime/test/providers/qnn/conv_test.cc +++ b/onnxruntime/test/providers/qnn/conv_test.cc @@ -182,7 +182,12 @@ static GetTestQDQModelFn BuildQDQPerChannelConvTestCase(const s static_cast(weight_quant_axis), true); TensorShape weights_shape = weights_def.GetTensorShape(); - std::vector quantized_weights(weights_shape.Size()); + std::vector quantized_weights; + size_t num_weight_storage_elems = weights_shape.Size(); + if constexpr (std::is_same_v || std::is_same_v) { + num_weight_storage_elems = Int4x2::CalcNumInt4Pairs(weights_shape.Size()); + } + quantized_weights.resize(num_weight_storage_elems); QuantizeValues(weights_def.GetRawData(), quantized_weights, weights_shape, weight_scales, weight_zero_points, weight_quant_axis); @@ -727,6 +732,80 @@ TEST_F(QnnHTPBackendTests, ConvU8S8S32_PerChannel) { 13); // opset } +// Test per-channel QDQ Conv with INT4 weights. in0: u16, in1 (weight): s4, in2 (bias): s32, out: u8 +TEST_F(QnnHTPBackendTests, ConvU16S4S32_PerChannel) { + std::vector input_shape = {1, 2, 4, 4}; + std::vector weight_shape = {3, 2, 2, 2}; + std::vector bias_shape = {3}; + + TestInputDef input_def(input_shape, false, + GetFloatDataInRange(0.0f, 1.0f, TensorShape(input_shape).Size())); + TestInputDef weight_def(weight_shape, true, + GetFloatDataInRange(-1.0f, 5.0f, TensorShape(weight_shape).Size())); + TestInputDef bias_def(bias_shape, true, + GetFloatDataInRange(-1.0f, 1.0f, TensorShape(bias_shape).Size())); + + RunHTPConvOpPerChannelTest("Conv", + input_def, + weight_def, + bias_def, + 0, // weight quant axis + {1, 1}, // Strides + {0, 0, 0, 0}, // Pads + {1, 1}, // Dilations + 1, // default group + "NOTSET", + ExpectedEPNodeAssignment::All, + false, // use_qdq_contrib_ops + 21); // opset +} + +// Test per-channel QDQ Conv with INT4 weights. in0: u16, in1 (weight): s4, in2 (bias): s32, out: u8 +// TODO(adrianlizarraga): Investigate inaccuracy for QNN EP. +// +// Output values for all EPs: +// CPU EP (f32 model): 25.143 21.554 17.964 10.785 7.195 3.605 -3.574 -7.164 -10.753 +// CPU EP (qdq model): 24.670 21.103 17.536 10.254 6.689 2.972 -4.161 -7.728 -10.700 +// QNN EP (qdq model): 27.186 27.186 27.186 21.541 6.685 -8.022 -10.548 -10.548 -10.548 +TEST_F(QnnHTPBackendTests, DISABLED_ConvU16S4S32_PerChannel_AccuracyIssue) { + std::vector input_shape = {1, 2, 4, 4}; + std::vector weight_shape = {3, 2, 2, 2}; + std::vector bias_shape = {3}; + + // Wrote out input data explicitly for easier reproduction. + // std::vector input_data = GetFloatDataInRange(-10.0f, 10.0f, TensorShape(input_shape).Size()); + std::vector input_data = {-10.000f, -9.355f, -8.710f, -8.065f, -7.419f, -6.774f, -6.129f, -5.484f, -4.839f, + -4.194f, -3.548f, -2.903f, -2.258f, -1.613f, -0.968f, -0.323f, 0.323f, 0.968f, + 1.613f, 2.258f, 2.903f, 3.548f, 4.194f, 4.839f, 5.484f, 6.129f, 6.774f, + 7.419f, 8.065f, 8.710f, 9.355f, 10.000f}; + + // std::vector weight_data = GetFloatDataInRange(-1.0f, 1.0f, TensorShape(weight_shape).Size()); + std::vector weight_data = {-1.000f, -0.913f, -0.826f, -0.739f, -0.652f, -0.565f, -0.478f, -0.391f, -0.304f, + -0.217f, -0.130f, -0.043f, 0.043f, 0.130f, 0.217f, 0.304f, 0.391f, 0.478f, + 0.565f, 0.652f, 0.739f, 0.826f, 0.913f, 1.000f}; + + // std::vector bias_data = GetFloatDataInRange(-1.0f, 1.0f, TensorShape(bias_shape).Size()); + std::vector bias_data = {-1.000f, 0.000f, 1.000f}; + + TestInputDef input_def(input_shape, false, input_data); + TestInputDef weight_def(weight_shape, true, weight_data); + TestInputDef bias_def(bias_shape, true, bias_data); + + RunHTPConvOpPerChannelTest("Conv", + input_def, + weight_def, + bias_def, + 0, // weight quant axis + {1, 1}, // Strides + {0, 0, 0, 0}, // Pads + {1, 1}, // Dilations + 1, // default group + "NOTSET", + ExpectedEPNodeAssignment::All, + false, // use_qdq_contrib_ops + 21); // opset +} + // Test per-channel QDQ Conv is rejected with weight axis != 0 TEST_F(QnnHTPBackendTests, Conv_PerChannel_UnsupportedAxis) { std::vector input_shape = {1, 2, 4, 4}; diff --git a/onnxruntime/test/providers/qnn/qnn_basic_test.cc b/onnxruntime/test/providers/qnn/qnn_basic_test.cc index 6173f46839..9489d35475 100644 --- a/onnxruntime/test/providers/qnn/qnn_basic_test.cc +++ b/onnxruntime/test/providers/qnn/qnn_basic_test.cc @@ -236,6 +236,51 @@ TEST_F(QnnHTPBackendTests, TestConvWithExternalData) { Ort::Session session(*ort_env, ort_model_path, so); } +#if defined(__aarch64__) || defined(_M_ARM64) || defined(__linux__) +TEST_F(QnnHTPBackendTests, RunConvInt4Model) { + Ort::SessionOptions so; + + so.AddConfigEntry(kOrtSessionOptionsDisableCPUEPFallback, "1"); // Disable fallback to the CPU EP. + so.SetGraphOptimizationLevel(ORT_ENABLE_ALL); + onnxruntime::ProviderOptions options; + +#if defined(_WIN32) + options["backend_path"] = "QnnHtp.dll"; +#else + options["backend_path"] = "libQnnHtp.so"; +#endif + + so.AppendExecutionProvider("QNN", options); + + const ORTCHAR_T* ort_model_path = ORT_MODEL_FOLDER "conv.int4_weights.qdq.onnx"; + Ort::Session session(*ort_env, ort_model_path, so); + + TensorShape input_shape = {1, 3, 8, 8}; + std::vector input0_data(input_shape.Size(), 0.2f); + + auto memory_info = Ort::MemoryInfo::CreateCpu(OrtDeviceAllocator, OrtMemTypeCPU); + std::vector ort_inputs; + std::vector ort_input_names; + + // Add input0 + ort_inputs.emplace_back(Ort::Value::CreateTensor( + memory_info, input0_data.data(), input0_data.size(), &input_shape[0], input_shape.NumDimensions())); + ort_input_names.push_back("input_0"); + + // Run session and get outputs + std::array output_names{"output_0"}; + std::vector ort_outputs = session.Run(Ort::RunOptions{nullptr}, ort_input_names.data(), ort_inputs.data(), + ort_inputs.size(), output_names.data(), output_names.size()); + + // Check output shape. + Ort::Value& ort_output = ort_outputs[0]; + auto typeshape = ort_output.GetTensorTypeAndShapeInfo(); + std::vector output_shape = typeshape.GetShape(); + + EXPECT_THAT(output_shape, ::testing::ElementsAre(1, 5, 6, 6)); +} +#endif // #if defined(__aarch64__) || defined(_M_ARM64) || defined(__linux__) + // Helper function that runs an ONNX model with a NHWC Resize operator to test that // type/shape inference succeeds during layout transformation. // Refer to onnxruntime/core/graph/contrib_ops/nhwc_inference_context.h. diff --git a/onnxruntime/test/providers/qnn/qnn_test_utils.h b/onnxruntime/test/providers/qnn/qnn_test_utils.h index dd47e1df80..ad54e644af 100644 --- a/onnxruntime/test/providers/qnn/qnn_test_utils.h +++ b/onnxruntime/test/providers/qnn/qnn_test_utils.h @@ -34,6 +34,15 @@ struct QuantParams { QType zero_point; static QuantParams Compute(float rmin, float rmax, bool symmetric = false) { + return Compute( + rmin, + rmax, + static_cast(std::numeric_limits::min()), + static_cast(std::numeric_limits::max()), + symmetric); + } + + static QuantParams Compute(float rmin, float rmax, QType qmin, QType qmax, bool symmetric = false) { // Ensure a minimum range of 0.0001 (required by QNN) rmax = std::max(rmax, rmin + 0.0001f); @@ -41,27 +50,27 @@ struct QuantParams { rmin = std::min(rmin, 0.0f); rmax = std::max(rmax, 0.0f); - constexpr float qmin = static_cast(std::numeric_limits::min()); - constexpr float qmax = static_cast(std::numeric_limits::max()); - if (symmetric) { const float abs_max = std::max(std::abs(rmin), std::abs(rmax)); rmax = abs_max; rmin = -abs_max; } - const float scale = (rmax - rmin) / (qmax - qmin); + float qmin_flt = static_cast(qmin); + float qmax_flt = static_cast(qmax); + const float scale = (rmax - rmin) / (qmax_flt - qmin_flt); float initial_zero_point = 0.0f; if (symmetric) { // Symmetric uses same formula for zero-point as asymmetric, but we can cancel out terms for // increased numerical accuracy. - initial_zero_point = (qmin + qmax) / 2.0f; + initial_zero_point = (qmin_flt + qmax_flt) / 2.0f; } else { - initial_zero_point = qmin - (rmin / scale); + initial_zero_point = qmin_flt - (rmin / scale); } - const QType zero_point = static_cast(RoundHalfToEven(std::max(qmin, std::min(qmax, initial_zero_point)))); + const QType zero_point = static_cast(RoundHalfToEven(std::max(qmin_flt, + std::min(qmax_flt, initial_zero_point)))); return QuantParams{scale, zero_point}; } @@ -238,7 +247,7 @@ struct TestInputDef { assert(which_type == 0); const std::vector& raw_data = std::get(data_info_).data; - std::pair init_range(std::numeric_limits::max(), std::numeric_limits::min()); + std::pair init_range(std::numeric_limits::max(), std::numeric_limits::lowest()); std::vector> per_axis_ranges(num_ranges, init_range); TensorShape shape(shape_); size_t num_blocks = shape.SizeToDimension(axis); @@ -292,6 +301,37 @@ static void GetTestInputQuantParamsPerChannel(const TestInputDef& input_d } } +// Define functions to get the quantization parameters (i.e., scale/zp) for input data that will be quantized +// as int4 per-channel. +#define DEF_GET_INPUT_QPARAMS_PER_CHAN_INT4_FUNC(INT4x2_TYPE) \ + template <> \ + inline void GetTestInputQuantParamsPerChannel(const TestInputDef& input_def, \ + std::vector& scales, \ + std::vector& zero_points, \ + size_t axis, bool symmetric) { \ + using UnpackedType = typename INT4x2_TYPE::UnpackedType; \ + const auto f32_ranges = input_def.GetRangePerChannel(axis); \ + const size_t num_ranges = f32_ranges.size(); \ + \ + scales.resize(num_ranges); \ + zero_points.resize(INT4x2_TYPE::CalcNumInt4Pairs(num_ranges)); \ + \ + for (size_t i = 0; i < num_ranges; i++) { \ + const auto& range = f32_ranges[i]; \ + QuantParams params = QuantParams::Compute(range.first, range.second, \ + INT4x2_TYPE::min_val, \ + INT4x2_TYPE::max_val, symmetric); \ + scales[i] = params.scale; \ + \ + size_t r = i >> 1; \ + size_t c = i & 0x1; \ + zero_points[r].SetElem(c, params.zero_point); \ + } \ + } + +DEF_GET_INPUT_QPARAMS_PER_CHAN_INT4_FUNC(Int4x2) +DEF_GET_INPUT_QPARAMS_PER_CHAN_INT4_FUNC(UInt4x2) + template static void QuantizeValues(gsl::span input, gsl::span output, const TensorShape& shape, gsl::span scales, gsl::span zero_points, @@ -332,6 +372,52 @@ static void QuantizeValues(gsl::span input, gsl::span \ + inline void QuantizeValues(gsl::span input, \ + gsl::span output, \ + const TensorShape& shape, \ + gsl::span scales, \ + gsl::span zero_points, \ + std::optional axis) { \ + using UnpackedType = typename INT4x2_TYPE::UnpackedType; \ + const size_t input_rank = shape.NumDimensions(); \ + const size_t num_int4_elems = static_cast(shape.Size()); \ + ORT_ENFORCE(input.size() == num_int4_elems); \ + ORT_ENFORCE(output.size() == INT4x2_TYPE::CalcNumInt4Pairs(num_int4_elems)); \ + \ + size_t block_count = 1; \ + size_t broadcast_dim = 1; \ + size_t block_size = num_int4_elems; \ + \ + if (axis.has_value()) { \ + size_t axis_no_neg = *axis < 0 ? static_cast(*axis) + input_rank : static_cast(*axis); \ + block_count = shape.SizeToDimension(axis_no_neg); \ + broadcast_dim = shape[axis_no_neg]; \ + block_size = shape.SizeFromDimension(axis_no_neg + 1); \ + } \ + \ + ORT_ENFORCE(scales.size() == broadcast_dim); \ + ORT_ENFORCE(zero_points.empty() || zero_points.size() == INT4x2_TYPE::CalcNumInt4Pairs(broadcast_dim)); \ + \ + size_t i = 0; \ + \ + for (size_t n = 0; n < block_count; n++) { \ + for (size_t bd = 0; bd < broadcast_dim; bd++) { \ + size_t bd_i = bd >> 1; /* bd / 2 */ \ + size_t bd_j = bd & 0x1; /* bd % 2 */ \ + UnpackedType zp = !zero_points.empty() ? zero_points[bd_i].GetElem(bd_j) : 0; \ + QUANT_FUNC(&input[i], output.data(), i, i + block_size, scales[bd], INT4x2_TYPE(zp, 0), nullptr); \ + i += block_size; \ + } \ + } \ + assert(i == (block_count * broadcast_dim * block_size)); \ + } + +DEF_QUANTIZE_VALUES_INT4_FUNC(Int4x2, ParQuantizeLinearStdS4) +DEF_QUANTIZE_VALUES_INT4_FUNC(UInt4x2, ParQuantizeLinearStdU4) + /** * Inferences a given serialized model. Returns output values via an out-param. * @@ -414,6 +500,10 @@ inline void TestQDQModelAccuracy(const GetTestModelFn& f32_model_fn, const GetTe const std::unordered_map domain_to_version = {{"", opset_version}, {kMSDomain, 1}}; auto& logging_manager = DefaultLoggingManager(); + + // Uncomment to dump LOGGER() output to stdout. + // logging_manager.RemoveSink(logging::SinkType::EtwSink); + logging_manager.SetDefaultLoggerSeverity(log_severity); // Create float model and serialize it to a string. diff --git a/onnxruntime/test/testdata/conv.int4_weights.qdq.onnx b/onnxruntime/test/testdata/conv.int4_weights.qdq.onnx new file mode 100644 index 0000000000..56f965d0b4 Binary files /dev/null and b/onnxruntime/test/testdata/conv.int4_weights.qdq.onnx differ diff --git a/onnxruntime/test/testdata/make_conv_int4_weights_model.py b/onnxruntime/test/testdata/make_conv_int4_weights_model.py new file mode 100644 index 0000000000..004342b531 --- /dev/null +++ b/onnxruntime/test/testdata/make_conv_int4_weights_model.py @@ -0,0 +1,98 @@ +import numpy as np +import onnx + +from onnxruntime.quantization import CalibrationDataReader, QuantType, quantize +from onnxruntime.quantization.execution_providers.qnn import get_qnn_qdq_config + +INPUT0_SHAPE = (1, 3, 8, 8) +INPUT0_NAME = "input_0" + + +def create_f32_model(): + input_0 = onnx.helper.make_tensor_value_info(INPUT0_NAME, onnx.TensorProto.FLOAT, INPUT0_SHAPE) + output_0 = onnx.helper.make_tensor_value_info("output_0", onnx.TensorProto.FLOAT, None) + weight_data = [ + [ + [[-1.5, -1.0, -0.5], [-0.2, 0.0, 0.2], [0.5, 1.0, 1.5]], # range = 3.0, scale = 3.0/15, zp = 0 + [[-1.5, -1.0, -0.5], [-0.2, 0.0, 0.2], [0.5, 1.0, 1.5]], # range = 3.0, scale = 3.0/15, zp = 0 + [[-1.5, -1.0, -0.5], [-0.2, 0.0, 0.2], [0.5, 1.0, 1.5]], # range = 3.0, scale = 3.0/15, zp = 0 + ], + [ + [[-1.0, -0.8, -0.6], [-0.1333, 0.0, -0.1333], [0.6, 0.8, 1.0]], # range = 2.0, scale = 2.0/15, zp = -3 + [[-1.0, -0.8, -0.6], [-0.1333, 0.0, -0.1333], [0.6, 0.8, 1.0]], # range = 2.0, scale = 2.0/15, zp = -3 + [[-1.0, -0.8, -0.6], [-0.1333, 0.0, -0.1333], [0.6, 0.8, 1.0]], # range = 2.0, scale = 2.0/15, zp = -3 + ], + [ + [[-1.5, -1.0, -0.5], [-0.2, 0.0, 0.2], [0.5, 1.0, 1.5]], # range = 3.0, scale = 3.0/15, zp = 0 + [[-1.5, -1.0, -0.5], [-0.2, 0.0, 0.2], [0.5, 1.0, 1.5]], # range = 3.0, scale = 3.0/15, zp = 0 + [[-1.5, -1.0, -0.5], [-0.2, 0.0, 0.2], [0.5, 1.0, 1.5]], # range = 3.0, scale = 3.0/15, zp = 0 + ], + [ + [[-1.0, -0.8, -0.6], [-0.1333, 0.0, -0.1333], [0.6, 0.8, 1.0]], # range = 2.0, scale = 2.0/15, zp = -3 + [[-1.0, -0.8, -0.6], [-0.1333, 0.0, -0.1333], [0.6, 0.8, 1.0]], # range = 2.0, scale = 2.0/15, zp = -3 + [[-1.0, -0.8, -0.6], [-0.1333, 0.0, -0.1333], [0.6, 0.8, 1.0]], # range = 2.0, scale = 2.0/15, zp = -3 + ], + [ + [[-1.5, -1.0, -0.5], [-0.2, 0.0, 0.2], [0.5, 1.0, 1.5]], # range = 3.0, scale = 3.0/15, zp = 0 + [[-1.5, -1.0, -0.5], [-0.2, 0.0, 0.2], [0.5, 1.0, 1.5]], # range = 3.0, scale = 3.0/15, zp = 0 + [[-1.5, -1.0, -0.5], [-0.2, 0.0, 0.2], [0.5, 1.0, 1.5]], # range = 3.0, scale = 3.0/15, zp = 0 + ], + ] + weight = onnx.numpy_helper.from_array(np.array(weight_data, dtype=np.float32), "weight") + bias_data = [-10.0, -8.0, 0.0, 8.0, 10.0] + bias = onnx.numpy_helper.from_array(np.array(bias_data, dtype=np.float32), "bias") + + conv_node = onnx.helper.make_node("Conv", [INPUT0_NAME, "weight", "bias"], ["output_0"], name="Conv0") + graph = onnx.helper.make_graph( + [conv_node], + "Convf32", + [input_0], + [output_0], + initializer=[weight, bias], + ) + opset_imports = [onnx.helper.make_opsetid("", 21)] + model = onnx.helper.make_model(graph, opset_imports=opset_imports) + model = onnx.shape_inference.infer_shapes(model) + onnx.checker.check_model(model, True) + + return model + + +class DataReader(CalibrationDataReader): + def __init__(self): + self.enum_data = None + self.data_list = [] + + # Generate 10 random input values for calibration + for _ in range(10): + input_data = {INPUT0_NAME: np.random.random(INPUT0_SHAPE).astype(np.float32)} + self.data_list.append(input_data) + + self.datasize = len(self.data_list) + + def get_next(self): + if self.enum_data is None: + self.enum_data = iter(self.data_list) + return next(self.enum_data, None) + + def rewind(self): + self.enum_data = None + + +def create_qdq_model(model_f32): + # Use tensor quantization overrides to quantize Conv's weight input to 4 bits on axis 0. + init_overrides = {"weight": [{"quant_type": QuantType.QInt4, "axis": 0, "symmetric": True}]} + qnn_config = get_qnn_qdq_config( + model_f32, + DataReader(), + init_overrides=init_overrides, + activation_type=QuantType.QUInt16, + weight_type=QuantType.QUInt8, + ) + + quantize(model_f32, "conv.int4_weights.qdq.onnx", qnn_config) + + +if __name__ == "__main__": + model_f32 = create_f32_model() + create_qdq_model(model_f32)