From 73919a67563fa6b2bf33524be4770f9abb5db97d Mon Sep 17 00:00:00 2001 From: Chen Fu <1316708+chenfucn@users.noreply.github.com> Date: Thu, 28 Jul 2022 09:22:04 -0700 Subject: [PATCH] Convert DQ node with const weight tensor int8 to uint8 (#12331) Convert DQ node with const weight tensor int8 to uint8 This is a follow-up with #12088, where convert weight tensor from int8 to uint8. Here we do the same thing in DequantizeLinear node, so that we don't have to perform the same changes for every single future operator. --- .../core/optimizer/graph_transformer_utils.cc | 14 +- .../avx2_weight_s8_to_u8.cc | 125 +--------------- .../avx2_weight_s8_to_u8.h | 0 .../optimizer/qdq_transformer/qdq_s8_to_u8.cc | 141 ++++++++++-------- .../optimizer/qdq_transformer/qdq_s8_to_u8.h | 5 +- .../optimizer/qdq_transformer/s8_to_u8.cc | 56 +++++++ .../core/optimizer/qdq_transformer/s8_to_u8.h | 78 ++++++++++ .../optimizer/avx2_weight_s8_to_u8_test.cc | 2 +- .../test/optimizer/qdq_transformer_test.cc | 49 ++++++ 9 files changed, 281 insertions(+), 189 deletions(-) rename onnxruntime/core/optimizer/{ => qdq_transformer}/avx2_weight_s8_to_u8.cc (61%) rename onnxruntime/core/optimizer/{ => qdq_transformer}/avx2_weight_s8_to_u8.h (100%) create mode 100644 onnxruntime/core/optimizer/qdq_transformer/s8_to_u8.cc create mode 100644 onnxruntime/core/optimizer/qdq_transformer/s8_to_u8.h diff --git a/onnxruntime/core/optimizer/graph_transformer_utils.cc b/onnxruntime/core/optimizer/graph_transformer_utils.cc index 2af5cc6d9d..88158ca6d4 100644 --- a/onnxruntime/core/optimizer/graph_transformer_utils.cc +++ b/onnxruntime/core/optimizer/graph_transformer_utils.cc @@ -19,7 +19,7 @@ #include "core/mlas/inc/mlas.h" #include "core/optimizer/attention_fusion.h" #ifdef MLAS_TARGET_AMD64_IX86 -#include "core/optimizer/avx2_weight_s8_to_u8.h" +#include "core/optimizer/qdq_transformer/avx2_weight_s8_to_u8.h" #endif #include "core/optimizer/bias_dropout_fusion.h" #include "core/optimizer/bias_gelu_fusion.h" @@ -220,12 +220,18 @@ InlinedVector> GenerateTransformers( onnxruntime::kAclExecutionProvider, onnxruntime::kArmNNExecutionProvider}; +#ifdef MLAS_TARGET_AMD64_IX86 + const bool avx2_precision_mode = + session_options.config_options.GetConfigOrDefault(kOrtSessionOptionsAvx2PrecisionMode, "0") == "1" && MlasPlatformU8S8Overflow(); +#else + const bool avx2_precision_mode = false; +#endif if (!disable_quant_qdq) { // currently we don't support QDQS8ToU8Transformer in a minimal build and if supported, this needs to run in // Level 1 during export and not Level 2 at runtime as it would result in overlapping optimizations which // runtime optimization does not support, so add session config value here to force qdqisint8allowed to be true. if (!qdq_is_int8_allowed) { - transformers.emplace_back(std::make_unique(cpu_ep)); + transformers.emplace_back(std::make_unique(avx2_precision_mode, cpu_ep)); } transformers.emplace_back(std::make_unique(qdq_is_int8_allowed)); } @@ -263,10 +269,6 @@ InlinedVector> GenerateTransformers( } #ifdef MLAS_TARGET_AMD64_IX86 - const bool avx2_precision_mode = - session_options.config_options.GetConfigOrDefault(kOrtSessionOptionsAvx2PrecisionMode, "0") == "1" - && MlasPlatformU8S8Overflow(); - if (avx2_precision_mode) { transformers.emplace_back(std::make_unique(cpu_ep)); } diff --git a/onnxruntime/core/optimizer/avx2_weight_s8_to_u8.cc b/onnxruntime/core/optimizer/qdq_transformer/avx2_weight_s8_to_u8.cc similarity index 61% rename from onnxruntime/core/optimizer/avx2_weight_s8_to_u8.cc rename to onnxruntime/core/optimizer/qdq_transformer/avx2_weight_s8_to_u8.cc index 224cdc270f..1c87c8b074 100644 --- a/onnxruntime/core/optimizer/avx2_weight_s8_to_u8.cc +++ b/onnxruntime/core/optimizer/qdq_transformer/avx2_weight_s8_to_u8.cc @@ -3,129 +3,15 @@ #if (defined(_M_AMD64) && !defined(_M_ARM64EC)) || defined(_M_IX86) || defined(__x86_64__) || defined(__i386__) || !defined(DISABLE_CONTRIB_OPS) -#include "core/optimizer/avx2_weight_s8_to_u8.h" +#include "core/optimizer/qdq_transformer/avx2_weight_s8_to_u8.h" +#include "core/optimizer/qdq_transformer/s8_to_u8.h" #include #include "core/graph/graph_utils.h" -#include "core/optimizer/initializer.h" -#include "core/optimizer/utils.h" namespace onnxruntime { -/** - * @brief Convert the source int8_t TensorProto to a uint8_t one if the tensor - * contains values outside of [-64, 64] - * @param src The source tensor, must be type int8_t - * @param dst An empty tensor, will contain the converted tensor data - * @param graph Graph for generating tensor name or provide external - * data path - * @param force Perform conversion even when tensor values within [-64, 64] - * @return Whether the conversion happened. -*/ -static inline bool Int8TensorProto2Uint8( - const ONNX_NAMESPACE::TensorProto* src, - ONNX_NAMESPACE::TensorProto& dst, - Graph& graph, bool force = false) { - dst.set_data_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - - if (nullptr == src) { - uint8_t zero_val = 128; - dst.set_name(graph.GenerateNodeArgName("weight_zp_s8_2_u8")); - dst.set_raw_data(&zero_val, sizeof(uint8_t)); - return true; - } - - dst.set_name(src->name() + "_s8_2_u8"); - dst.mutable_dims()->CopyFrom(src->dims()); - - // TODO(fuchen): too many copies! - // - // Here we do two memory copies: Proto -> Initializer -> Proto. - // Ideally we only do 1 copy, just iterate the source data, and write directly - // to the dst raw buffer. - // Unfortunately iterating the source data is complicated, the data maybe in - // external file, a raw buffer, or a repeated field depending on the data - // type. UnpackTensor() already contains some of these logic and is closest - // to what we need. But it does not handle external data. Write our own code - // here means copy the logic of TensorProtoToTensor(), a violation of DRY - // principle. A better solution is to provide an efficient const iterator for - // TensorProto. This require coordination with onnx side. - - Initializer temp(*src, graph.ModelPath()); - int8_t* p = temp.data(); - bool should_convert = false; - for (int i = 0; i < temp.size(); i++) { - if (*p < -64 || *p > 64) { - should_convert = true; - } - *p ^= 0x80; - p++; - } - if (force || should_convert) { - dst.set_raw_data(temp.data(), temp.size()); - return true; - } - return false; -} - -/** - * @brief If the op_node has an uint8_t const weight tensor, convert it to int8_t - * @param graph - * @param op_node - * @param weights_idx - * @param weight_zp_idx - * @return true when conversion happened. -*/ -static bool ConvertS8WeightToU8(Graph& graph, Node& op_node, - size_t weights_idx, size_t weight_zp_idx) { - auto& input_defs = op_node.MutableInputDefs(); - if (input_defs.size() < weights_idx + 1) { - return false; - } - - // Weight tensor must be const int8_t - const ONNX_NAMESPACE::TensorProto* weight_tensor_proto = nullptr; - const auto* w_def = input_defs[weights_idx]; - if (!graph_utils::NodeArgIsConstant(graph, *w_def) || - !graph.GetInitializedTensor(w_def->Name(), weight_tensor_proto) || - weight_tensor_proto->data_type() != ONNX_NAMESPACE::TensorProto_DataType_INT8) { - return false; - } - ORT_ENFORCE(nullptr != weight_tensor_proto, - "Internal Error: weight tensor must be const int8 for Avx2WeightS8ToU8Transformer."); - - // Weight zero point must be either const int8_t or null tensor - const ONNX_NAMESPACE::TensorProto* weight_zp_tensor_proto = nullptr; - const auto* zp_def = input_defs.size() <= weight_zp_idx ? nullptr : input_defs[weight_zp_idx]; - if (nullptr != zp_def) { - if (!graph_utils::NodeArgIsConstant(graph, *zp_def) || - !graph.GetInitializedTensor(zp_def->Name(), weight_zp_tensor_proto) || - weight_zp_tensor_proto->data_type() != ONNX_NAMESPACE::TensorProto_DataType_INT8) { - return false; - } - ORT_ENFORCE(nullptr != weight_zp_tensor_proto, - "Internal Error: weight zero point must be const int8 for Avx2WeightS8ToU8Transformer."); - } - - // Convert weight tensor to uint8 - ONNX_NAMESPACE::TensorProto weights_proto_u8; - bool converted = Int8TensorProto2Uint8(weight_tensor_proto, weights_proto_u8, graph); - if (!converted) { - // The weights fits into S7, overflow is not a problem, no need to convert to U8 - return false; - } - input_defs[weights_idx] = &graph_utils::AddInitializer(graph, weights_proto_u8); - - // Convert weight zero point to uint8 - ONNX_NAMESPACE::TensorProto weight_zp_proto_u8; - Int8TensorProto2Uint8(weight_zp_tensor_proto, weight_zp_proto_u8, graph, true); - input_defs[weight_zp_idx] = &graph_utils::AddInitializer(graph, weight_zp_proto_u8); - - return true; -} - - struct OperatorWeightInfo { std::vector versions; const char* domain; @@ -141,6 +27,7 @@ static const std::unordered_map s8_overf {"MatMulInteger", {{10}, kOnnxDomain, 1, 3}}, {"QLinearMatMul", {{10}, kOnnxDomain, 3, 5}}, {"QLinearConv", {{10}, kOnnxDomain, 3, 5}}, + {"DequantizeLinear", {{10,13}, kMSDomain, 0, 2}}, // already covered in QDQS8ToU8Transformer but does not hurt /* {"ConvInteger", {10}, kOnnxDomain, 1, 3}, // ConvInteger does not support int8_t weight at all */ }; @@ -247,7 +134,7 @@ static bool TryConvertDynamicQuantizeLSTM(Node& op_node, Graph& graph) { input_defs[w_idx] = &graph_utils::AddInitializer(graph, weights_proto_u8); ONNX_NAMESPACE::TensorProto weight_zp_proto_u8; - Int8TensorProto2Uint8(weight_zp_tensor_proto, weight_zp_proto_u8, graph, true); + QDQ::Int8TensorProto2Uint8(weight_zp_tensor_proto, weight_zp_proto_u8, graph, true); input_defs[w_zp_idx] = &graph_utils::AddInitializer(graph, weight_zp_proto_u8); ONNX_NAMESPACE::TensorProto r_proto_u8; @@ -258,7 +145,7 @@ static bool TryConvertDynamicQuantizeLSTM(Node& op_node, Graph& graph) { input_defs[r_idx] = &graph_utils::AddInitializer(graph, r_proto_u8); ONNX_NAMESPACE::TensorProto r_zp_proto_u8; - Int8TensorProto2Uint8(r_zp_tensor_proto, r_zp_proto_u8, graph, true); + QDQ::Int8TensorProto2Uint8(r_zp_tensor_proto, r_zp_proto_u8, graph, true); input_defs[r_zp_idx] = &graph_utils::AddInitializer(graph, r_zp_proto_u8); return true; @@ -301,7 +188,7 @@ Status Avx2WeightS8ToU8Transformer::ApplyImpl(Graph& graph, bool& modified, int #endif MatchesOpSinceVersion(op_node, it->second.versions) && graph_utils::MatchesOpSetDomain(op_node, it->second.domain)) { - modified |= ConvertS8WeightToU8(graph, op_node, + modified |= QDQ::ConvertS8WeightToU8(graph, op_node, it->second.weights_idx, it->second.weight_zp_idx); continue; // finished with this op, next diff --git a/onnxruntime/core/optimizer/avx2_weight_s8_to_u8.h b/onnxruntime/core/optimizer/qdq_transformer/avx2_weight_s8_to_u8.h similarity index 100% rename from onnxruntime/core/optimizer/avx2_weight_s8_to_u8.h rename to onnxruntime/core/optimizer/qdq_transformer/avx2_weight_s8_to_u8.h diff --git a/onnxruntime/core/optimizer/qdq_transformer/qdq_s8_to_u8.cc b/onnxruntime/core/optimizer/qdq_transformer/qdq_s8_to_u8.cc index 97ce6e6f21..9ee4e9ab2b 100644 --- a/onnxruntime/core/optimizer/qdq_transformer/qdq_s8_to_u8.cc +++ b/onnxruntime/core/optimizer/qdq_transformer/qdq_s8_to_u8.cc @@ -6,10 +6,74 @@ #include "core/graph/graph_utils.h" #include "core/optimizer/initializer.h" #include "core/optimizer/qdq_transformer/qdq_util.h" +#include "core/optimizer/qdq_transformer/s8_to_u8.h" #include "core/optimizer/utils.h" namespace onnxruntime { +/** + * @brief Given a QuantizeLinear and DequantizeLinear pair with type int8_t, + * Convert them to uint8_t + * @param graph + * @param q_node + * @param dq_node + * @return whether conversion happened +*/ +static bool QDQ_S8_to_U8(Graph& graph, Node& q_node, Node& dq_node) { + auto& q_input_defs = q_node.MutableInputDefs(); + auto& dq_input_defs = dq_node.MutableInputDefs(); + + constexpr size_t input_cnt_required = 3; + if (q_input_defs.size() != input_cnt_required || + dq_input_defs.size() != input_cnt_required) { + return false; + } + + constexpr size_t zp_idx = 2; + const ONNX_NAMESPACE::TensorProto* q_zp_tensor_proto = nullptr; + const ONNX_NAMESPACE::TensorProto* dq_zp_tensor_proto = nullptr; + if (!graph_utils::NodeArgIsConstant(graph, *q_input_defs[zp_idx]) || + !graph_utils::NodeArgIsConstant(graph, *dq_input_defs[zp_idx]) || + !graph.GetInitializedTensor(q_input_defs[zp_idx]->Name(), q_zp_tensor_proto) || + !graph.GetInitializedTensor(dq_input_defs[zp_idx]->Name(), dq_zp_tensor_proto)) { + return false; + } + + //TODO(fuchen): need to augment this when we support per row quantization + using ONNX_TENSOR_ELEM_TYPE = ONNX_NAMESPACE::TensorProto::DataType; + Initializer q_zero_point(*q_zp_tensor_proto, graph.ModelPath()); + Initializer dq_zero_point(*dq_zp_tensor_proto, graph.ModelPath()); + if (q_zero_point.size() != 1 || + dq_zero_point.size() != 1 || + q_zero_point.data_type() != ONNX_TENSOR_ELEM_TYPE::TensorProto_DataType_INT8 || + dq_zero_point.data_type() != ONNX_TENSOR_ELEM_TYPE::TensorProto_DataType_INT8) { + return false; + } + + uint8_t q_zp_value = *q_zero_point.data() + 128; + uint8_t dq_zp_value = *dq_zero_point.data() + 128; + + if (q_zp_value != dq_zp_value) { + return false; // zero points for Q and DQ are expected to be same + } + + ONNX_NAMESPACE::TensorProto zp_tensor_proto_u8; + zp_tensor_proto_u8.set_data_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); + zp_tensor_proto_u8.set_name(graph.GenerateNodeArgName("qdq_s8_to_u8_zp_conversion")); + zp_tensor_proto_u8.set_raw_data(&q_zp_value, sizeof(uint8_t)); + NodeArg* zp_u8_arg = &graph_utils::AddInitializer(graph, zp_tensor_proto_u8); + + auto q_output_node_arg_name = graph.GenerateNodeArgName("qdq_s8_to_u8_quant"); + NodeArg* q_output_arg = &graph.GetOrCreateNodeArg(q_output_node_arg_name, nullptr); + + q_node.MutableOutputDefs()[0] = q_output_arg; + dq_input_defs[0] = q_output_arg; + q_input_defs[zp_idx] = zp_u8_arg; + dq_input_defs[zp_idx] = zp_u8_arg; + return true; +} + + // Convert QuantizeLinear and DequantizeLinear pair with type int8_t to type uint8_t Status QDQS8ToU8Transformer::ApplyImpl(Graph& graph, bool& modified, int graph_level, const logging::Logger& logger) const { @@ -21,73 +85,28 @@ Status QDQS8ToU8Transformer::ApplyImpl(Graph& graph, bool& modified, int graph_l if (q_node_ptr == nullptr) continue; // node removed as part of an earlier fusion - // recognize Q + DQ - Node& q_node = *q_node_ptr; - ORT_RETURN_IF_ERROR(Recurse(q_node, modified, graph_level, logger)); + Node& node = *q_node_ptr; + ORT_RETURN_IF_ERROR(Recurse(node, modified, graph_level, logger)); - if (!QDQ::MatchQNode(q_node) || - !graph_utils::IsSupportedProvider(q_node, GetCompatibleExecutionProviders()) || - !optimizer_utils::CheckOutputEdges(graph, q_node, 1)) { + if (!graph_utils::IsSupportedProvider(node, GetCompatibleExecutionProviders())) { + continue; + } + + // recognize Q + DQ pair + if (QDQ::MatchQNode(node) && + optimizer_utils::CheckOutputEdges(graph, node, 1)) { + Node& dq_node = *graph.GetNode(node.OutputNodesBegin()->Index()); + if (QDQ::MatchDQNode(dq_node)) { + modified |= QDQ_S8_to_U8(graph, node, dq_node); + } continue; } - Node& dq_node = *graph.GetNode(q_node.OutputNodesBegin()->Index()); - if (!QDQ::MatchDQNode(dq_node) || - !graph_utils::IsSupportedProvider(dq_node, GetCompatibleExecutionProviders())) { + // recognize lone DQ node + if (weights_to_u8_ && QDQ::MatchDQNode(node)) { + modified |= QDQ::ConvertS8WeightToU8(graph, node, 0, 2); continue; } - - auto& q_input_defs = q_node.MutableInputDefs(); - auto& dq_input_defs = dq_node.MutableInputDefs(); - - constexpr size_t input_cnt_required = 3; - if (q_input_defs.size() != input_cnt_required || - dq_input_defs.size() != input_cnt_required) { - continue; - } - - constexpr size_t zp_idx = 2; - const ONNX_NAMESPACE::TensorProto* q_zp_tensor_proto = nullptr; - const ONNX_NAMESPACE::TensorProto* dq_zp_tensor_proto = nullptr; - if (!graph_utils::NodeArgIsConstant(graph, *q_input_defs[zp_idx]) || - !graph_utils::NodeArgIsConstant(graph, *dq_input_defs[zp_idx]) || - !graph.GetInitializedTensor(q_input_defs[zp_idx]->Name(), q_zp_tensor_proto) || - !graph.GetInitializedTensor(dq_input_defs[zp_idx]->Name(), dq_zp_tensor_proto)) { - continue; - } - - using ONNX_TENSOR_ELEM_TYPE = ONNX_NAMESPACE::TensorProto::DataType; - Initializer q_zero_point(*q_zp_tensor_proto, graph.ModelPath()); - Initializer dq_zero_point(*dq_zp_tensor_proto, graph.ModelPath()); - if (q_zero_point.size() != 1 || - dq_zero_point.size() != 1 || - q_zero_point.data_type() != ONNX_TENSOR_ELEM_TYPE::TensorProto_DataType_INT8 || - dq_zero_point.data_type() != ONNX_TENSOR_ELEM_TYPE::TensorProto_DataType_INT8) { - continue; - } - - uint8_t q_zp_value = *q_zero_point.data() + 128; - uint8_t dq_zp_value = *dq_zero_point.data() + 128; - - if (q_zp_value != dq_zp_value) { - continue; // zero points for Q and DQ are expected to be same - } - - ONNX_NAMESPACE::TensorProto zp_tensor_proto_u8; - zp_tensor_proto_u8.set_data_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - zp_tensor_proto_u8.set_name(graph.GenerateNodeArgName("qdq_s8_to_u8_zp_conversion")); - zp_tensor_proto_u8.set_raw_data(&q_zp_value, sizeof(uint8_t)); - NodeArg* zp_u8_arg = &graph_utils::AddInitializer(graph, zp_tensor_proto_u8); - - auto q_output_node_arg_name = graph.GenerateNodeArgName("qdq_s8_to_u8_quant"); - NodeArg* q_output_arg = &graph.GetOrCreateNodeArg(q_output_node_arg_name, nullptr); - - q_node.MutableOutputDefs()[0] = q_output_arg; - dq_input_defs[0] = q_output_arg; - q_input_defs[zp_idx] = zp_u8_arg; - dq_input_defs[zp_idx] = zp_u8_arg; - - modified = true; } return Status::OK(); diff --git a/onnxruntime/core/optimizer/qdq_transformer/qdq_s8_to_u8.h b/onnxruntime/core/optimizer/qdq_transformer/qdq_s8_to_u8.h index d6e573175e..936cb36150 100644 --- a/onnxruntime/core/optimizer/qdq_transformer/qdq_s8_to_u8.h +++ b/onnxruntime/core/optimizer/qdq_transformer/qdq_s8_to_u8.h @@ -15,11 +15,12 @@ namespace onnxruntime { */ class QDQS8ToU8Transformer : public GraphTransformer { public: - QDQS8ToU8Transformer(const InlinedHashSet& compatible_execution_providers = {}) noexcept - : GraphTransformer("QDQS8ToU8Transformer", compatible_execution_providers) {} + QDQS8ToU8Transformer(bool weights_to_u8, const InlinedHashSet& compatible_execution_providers = {}) noexcept + : GraphTransformer("QDQS8ToU8Transformer", compatible_execution_providers), weights_to_u8_(weights_to_u8) {} private: Status ApplyImpl(Graph& graph, bool& modified, int graph_level, const logging::Logger& logger) const override; + bool weights_to_u8_; }; } // namespace onnxruntime diff --git a/onnxruntime/core/optimizer/qdq_transformer/s8_to_u8.cc b/onnxruntime/core/optimizer/qdq_transformer/s8_to_u8.cc new file mode 100644 index 0000000000..b4c1fd4d2e --- /dev/null +++ b/onnxruntime/core/optimizer/qdq_transformer/s8_to_u8.cc @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/optimizer/qdq_transformer/s8_to_u8.h" + +namespace onnxruntime::QDQ { + +bool ConvertS8WeightToU8(Graph& graph, Node& op_node, + size_t weights_idx, size_t weight_zp_idx) { + auto& input_defs = op_node.MutableInputDefs(); + if (input_defs.size() < weights_idx + 1) { + return false; + } + + // Weight tensor must be const int8_t + const ONNX_NAMESPACE::TensorProto* weight_tensor_proto = nullptr; + const auto* w_def = input_defs[weights_idx]; + if (!graph_utils::NodeArgIsConstant(graph, *w_def) || + !graph.GetInitializedTensor(w_def->Name(), weight_tensor_proto) || + weight_tensor_proto->data_type() != ONNX_NAMESPACE::TensorProto_DataType_INT8) { + return false; + } + ORT_ENFORCE(nullptr != weight_tensor_proto, + "Internal Error: weight tensor must be const int8 for Avx2WeightS8ToU8Transformer."); + + // Weight zero point must be either const int8_t or null tensor + const ONNX_NAMESPACE::TensorProto* weight_zp_tensor_proto = nullptr; + const auto* zp_def = input_defs.size() <= weight_zp_idx ? nullptr : input_defs[weight_zp_idx]; + if (nullptr != zp_def) { + if (!graph_utils::NodeArgIsConstant(graph, *zp_def) || + !graph.GetInitializedTensor(zp_def->Name(), weight_zp_tensor_proto) || + weight_zp_tensor_proto->data_type() != ONNX_NAMESPACE::TensorProto_DataType_INT8) { + return false; + } + ORT_ENFORCE(nullptr != weight_zp_tensor_proto, + "Internal Error: weight zero point must be const int8 for Avx2WeightS8ToU8Transformer."); + } + + // Convert weight tensor to uint8 + ONNX_NAMESPACE::TensorProto weights_proto_u8; + bool converted = Int8TensorProto2Uint8(weight_tensor_proto, weights_proto_u8, graph); + if (!converted) { + // The weights fits into S7, overflow is not a problem, no need to convert to U8 + return false; + } + input_defs[weights_idx] = &graph_utils::AddInitializer(graph, weights_proto_u8); + + // Convert weight zero point to uint8 + ONNX_NAMESPACE::TensorProto weight_zp_proto_u8; + Int8TensorProto2Uint8(weight_zp_tensor_proto, weight_zp_proto_u8, graph, true); + input_defs[weight_zp_idx] = &graph_utils::AddInitializer(graph, weight_zp_proto_u8); + + return true; +} + +} \ No newline at end of file diff --git a/onnxruntime/core/optimizer/qdq_transformer/s8_to_u8.h b/onnxruntime/core/optimizer/qdq_transformer/s8_to_u8.h new file mode 100644 index 0000000000..0279e248f4 --- /dev/null +++ b/onnxruntime/core/optimizer/qdq_transformer/s8_to_u8.h @@ -0,0 +1,78 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/graph/graph_utils.h" +#include "core/optimizer/initializer.h" + +namespace onnxruntime::QDQ { + +/** + * @brief Convert the source int8_t TensorProto to a uint8_t one if the tensor + * contains values outside of [-64, 64] + * @param src The source tensor, must be type int8_t + * @param dst An empty tensor, will contain the converted tensor data + * @param graph Graph for generating tensor name or provide external + * data path + * @param force Perform conversion even when tensor values within [-64, 64] + * @return Whether the conversion happened. +*/ +inline bool Int8TensorProto2Uint8( + const ONNX_NAMESPACE::TensorProto* src, + ONNX_NAMESPACE::TensorProto& dst, + Graph& graph, bool force = false) { + dst.set_data_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); + + if (nullptr == src) { + uint8_t zero_val = 128; + dst.set_name(graph.GenerateNodeArgName("weight_zp_s8_2_u8")); + dst.set_raw_data(&zero_val, sizeof(uint8_t)); + return true; + } + + dst.set_name(src->name() + "_s8_2_u8"); + dst.mutable_dims()->CopyFrom(src->dims()); + + // TODO(fuchen): too many copies! + // + // Here we do two memory copies: Proto -> Initializer -> Proto. + // Ideally we only do 1 copy, just iterate the source data, and write directly + // to the dst raw buffer. + // Unfortunately iterating the source data is complicated, the data maybe in + // external file, a raw buffer, or a repeated field depending on the data + // type. UnpackTensor() already contains some of these logic and is closest + // to what we need. But it does not handle external data. Write our own code + // here means copy the logic of TensorProtoToTensor(), a violation of DRY + // principle. A better solution is to provide an efficient const iterator for + // TensorProto. This require coordination with onnx side. + + Initializer temp(*src, graph.ModelPath()); + int8_t* p = temp.data(); + bool should_convert = false; + for (int i = 0; i < temp.size(); i++) { + if (*p < -64 || *p > 64) { + should_convert = true; + } + *p ^= 0x80; + p++; + } + if (force || should_convert) { + dst.set_raw_data(temp.data(), size_t(temp.size())); + return true; + } + return false; +} + +/** + * @brief If the op_node has an single int8_t const weight tensor, convert it to uint8_t + * @param graph + * @param op_node + * @param weights_idx input index of the weight tensor + * @param weight_zp_idx input index of the weight zero point tensor + * @return true when conversion happened. +*/ +extern bool ConvertS8WeightToU8(Graph& graph, Node& op_node, + size_t weights_idx, size_t weight_zp_idx); + +} \ No newline at end of file diff --git a/onnxruntime/test/optimizer/avx2_weight_s8_to_u8_test.cc b/onnxruntime/test/optimizer/avx2_weight_s8_to_u8_test.cc index 39709dd0e1..b25475e46c 100644 --- a/onnxruntime/test/optimizer/avx2_weight_s8_to_u8_test.cc +++ b/onnxruntime/test/optimizer/avx2_weight_s8_to_u8_test.cc @@ -1,4 +1,4 @@ -#include "core/optimizer/avx2_weight_s8_to_u8.h" +#include "core/optimizer/qdq_transformer/avx2_weight_s8_to_u8.h" #include #include diff --git a/onnxruntime/test/optimizer/qdq_transformer_test.cc b/onnxruntime/test/optimizer/qdq_transformer_test.cc index 79bb64b122..4131cbe497 100644 --- a/onnxruntime/test/optimizer/qdq_transformer_test.cc +++ b/onnxruntime/test/optimizer/qdq_transformer_test.cc @@ -217,6 +217,55 @@ TEST(QDQTransformerTests, ConvMaxPoolReshape_Int8) { test_case({1, 22, 11, 13, 15}, {30, 22, 5, 3, 3}); } +#if (defined(_M_AMD64) && !defined(_M_ARM64EC)) || defined(_M_IX86) || defined(__x86_64__) || defined(__i386__) || !defined(DISABLE_CONTRIB_OPS) + +TEST(QDQTransformerTests, DQ_S8_to_U8) { + const std::vector& input_shape = {19, 37}; + const std::vector& weights_shape = {37, 23}; + + auto build_test_case = [&](ModelTestBuilder& builder) { + auto* input1_arg = builder.MakeInput(input_shape, -1.f, 1.f); + + // Use full range weight values to expose avx2 u8s8 overflow problems + auto* weight = builder.MakeInitializer(weights_shape, -128, 127); + auto* output_arg = builder.MakeOutput(); + + // add QDQ activation + typedef std::numeric_limits Input1Limits; + auto* dq1_output = AddQDQNodePair(builder, input1_arg, .039f, (int8_t)((Input1Limits::max() + Input1Limits::min()) / 2 + 1)); + + // add DQ weight + auto* dq_w_output = builder.MakeIntermediate(); + builder.AddDequantizeLinearNode(weight, .003f, -10, dq_w_output); + + + builder.AddNode("MatMul", {dq1_output, dq_w_output}, {output_arg}); + }; + + auto check_graph = [&](InferenceSessionWrapper& session) { + auto op_to_count = CountOpsInGraph(session.GetGraph()); + EXPECT_EQ(op_to_count["com.microsoft.MatMulIntegerToFloat"], 1); + EXPECT_EQ(op_to_count["MatMul"], 0); + EXPECT_EQ(op_to_count["QuantizeLinear"], 1); + EXPECT_EQ(op_to_count["DequantizeLinear"], 0); + }; + + auto add_session_options = [&](SessionOptions& so) { + ASSERT_STATUS_OK(so.config_options.AddConfigEntry( + kOrtSessionOptionsAvx2PrecisionMode, "1")); + }; + + TransformerTester(build_test_case, + check_graph, + TransformerLevel::Level1, + TransformerLevel::Level2, + 12 /*opset_version*/, + 0.01 /*per_sample_tolerance*/, + 0.01 /*relative_per_sample_tolerance*/, + nullptr, add_session_options); +} +#endif // Only for X64 with contrib ops enabled + template void QDQTransformerAveragePoolTests() { auto test_case = [&](const std::vector& input_shape) {