diff --git a/onnxruntime/core/optimizer/graph_transformer_utils.cc b/onnxruntime/core/optimizer/graph_transformer_utils.cc index 2af5cc6d9d..88158ca6d4 100644 --- a/onnxruntime/core/optimizer/graph_transformer_utils.cc +++ b/onnxruntime/core/optimizer/graph_transformer_utils.cc @@ -19,7 +19,7 @@ #include "core/mlas/inc/mlas.h" #include "core/optimizer/attention_fusion.h" #ifdef MLAS_TARGET_AMD64_IX86 -#include "core/optimizer/avx2_weight_s8_to_u8.h" +#include "core/optimizer/qdq_transformer/avx2_weight_s8_to_u8.h" #endif #include "core/optimizer/bias_dropout_fusion.h" #include "core/optimizer/bias_gelu_fusion.h" @@ -220,12 +220,18 @@ InlinedVector> GenerateTransformers( onnxruntime::kAclExecutionProvider, onnxruntime::kArmNNExecutionProvider}; +#ifdef MLAS_TARGET_AMD64_IX86 + const bool avx2_precision_mode = + session_options.config_options.GetConfigOrDefault(kOrtSessionOptionsAvx2PrecisionMode, "0") == "1" && MlasPlatformU8S8Overflow(); +#else + const bool avx2_precision_mode = false; +#endif if (!disable_quant_qdq) { // currently we don't support QDQS8ToU8Transformer in a minimal build and if supported, this needs to run in // Level 1 during export and not Level 2 at runtime as it would result in overlapping optimizations which // runtime optimization does not support, so add session config value here to force qdqisint8allowed to be true. if (!qdq_is_int8_allowed) { - transformers.emplace_back(std::make_unique(cpu_ep)); + transformers.emplace_back(std::make_unique(avx2_precision_mode, cpu_ep)); } transformers.emplace_back(std::make_unique(qdq_is_int8_allowed)); } @@ -263,10 +269,6 @@ InlinedVector> GenerateTransformers( } #ifdef MLAS_TARGET_AMD64_IX86 - const bool avx2_precision_mode = - session_options.config_options.GetConfigOrDefault(kOrtSessionOptionsAvx2PrecisionMode, "0") == "1" - && MlasPlatformU8S8Overflow(); - if (avx2_precision_mode) { transformers.emplace_back(std::make_unique(cpu_ep)); } diff --git a/onnxruntime/core/optimizer/avx2_weight_s8_to_u8.cc b/onnxruntime/core/optimizer/qdq_transformer/avx2_weight_s8_to_u8.cc similarity index 61% rename from onnxruntime/core/optimizer/avx2_weight_s8_to_u8.cc rename to onnxruntime/core/optimizer/qdq_transformer/avx2_weight_s8_to_u8.cc index 224cdc270f..1c87c8b074 100644 --- a/onnxruntime/core/optimizer/avx2_weight_s8_to_u8.cc +++ b/onnxruntime/core/optimizer/qdq_transformer/avx2_weight_s8_to_u8.cc @@ -3,129 +3,15 @@ #if (defined(_M_AMD64) && !defined(_M_ARM64EC)) || defined(_M_IX86) || defined(__x86_64__) || defined(__i386__) || !defined(DISABLE_CONTRIB_OPS) -#include "core/optimizer/avx2_weight_s8_to_u8.h" +#include "core/optimizer/qdq_transformer/avx2_weight_s8_to_u8.h" +#include "core/optimizer/qdq_transformer/s8_to_u8.h" #include #include "core/graph/graph_utils.h" -#include "core/optimizer/initializer.h" -#include "core/optimizer/utils.h" namespace onnxruntime { -/** - * @brief Convert the source int8_t TensorProto to a uint8_t one if the tensor - * contains values outside of [-64, 64] - * @param src The source tensor, must be type int8_t - * @param dst An empty tensor, will contain the converted tensor data - * @param graph Graph for generating tensor name or provide external - * data path - * @param force Perform conversion even when tensor values within [-64, 64] - * @return Whether the conversion happened. -*/ -static inline bool Int8TensorProto2Uint8( - const ONNX_NAMESPACE::TensorProto* src, - ONNX_NAMESPACE::TensorProto& dst, - Graph& graph, bool force = false) { - dst.set_data_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - - if (nullptr == src) { - uint8_t zero_val = 128; - dst.set_name(graph.GenerateNodeArgName("weight_zp_s8_2_u8")); - dst.set_raw_data(&zero_val, sizeof(uint8_t)); - return true; - } - - dst.set_name(src->name() + "_s8_2_u8"); - dst.mutable_dims()->CopyFrom(src->dims()); - - // TODO(fuchen): too many copies! - // - // Here we do two memory copies: Proto -> Initializer -> Proto. - // Ideally we only do 1 copy, just iterate the source data, and write directly - // to the dst raw buffer. - // Unfortunately iterating the source data is complicated, the data maybe in - // external file, a raw buffer, or a repeated field depending on the data - // type. UnpackTensor() already contains some of these logic and is closest - // to what we need. But it does not handle external data. Write our own code - // here means copy the logic of TensorProtoToTensor(), a violation of DRY - // principle. A better solution is to provide an efficient const iterator for - // TensorProto. This require coordination with onnx side. - - Initializer temp(*src, graph.ModelPath()); - int8_t* p = temp.data(); - bool should_convert = false; - for (int i = 0; i < temp.size(); i++) { - if (*p < -64 || *p > 64) { - should_convert = true; - } - *p ^= 0x80; - p++; - } - if (force || should_convert) { - dst.set_raw_data(temp.data(), temp.size()); - return true; - } - return false; -} - -/** - * @brief If the op_node has an uint8_t const weight tensor, convert it to int8_t - * @param graph - * @param op_node - * @param weights_idx - * @param weight_zp_idx - * @return true when conversion happened. -*/ -static bool ConvertS8WeightToU8(Graph& graph, Node& op_node, - size_t weights_idx, size_t weight_zp_idx) { - auto& input_defs = op_node.MutableInputDefs(); - if (input_defs.size() < weights_idx + 1) { - return false; - } - - // Weight tensor must be const int8_t - const ONNX_NAMESPACE::TensorProto* weight_tensor_proto = nullptr; - const auto* w_def = input_defs[weights_idx]; - if (!graph_utils::NodeArgIsConstant(graph, *w_def) || - !graph.GetInitializedTensor(w_def->Name(), weight_tensor_proto) || - weight_tensor_proto->data_type() != ONNX_NAMESPACE::TensorProto_DataType_INT8) { - return false; - } - ORT_ENFORCE(nullptr != weight_tensor_proto, - "Internal Error: weight tensor must be const int8 for Avx2WeightS8ToU8Transformer."); - - // Weight zero point must be either const int8_t or null tensor - const ONNX_NAMESPACE::TensorProto* weight_zp_tensor_proto = nullptr; - const auto* zp_def = input_defs.size() <= weight_zp_idx ? nullptr : input_defs[weight_zp_idx]; - if (nullptr != zp_def) { - if (!graph_utils::NodeArgIsConstant(graph, *zp_def) || - !graph.GetInitializedTensor(zp_def->Name(), weight_zp_tensor_proto) || - weight_zp_tensor_proto->data_type() != ONNX_NAMESPACE::TensorProto_DataType_INT8) { - return false; - } - ORT_ENFORCE(nullptr != weight_zp_tensor_proto, - "Internal Error: weight zero point must be const int8 for Avx2WeightS8ToU8Transformer."); - } - - // Convert weight tensor to uint8 - ONNX_NAMESPACE::TensorProto weights_proto_u8; - bool converted = Int8TensorProto2Uint8(weight_tensor_proto, weights_proto_u8, graph); - if (!converted) { - // The weights fits into S7, overflow is not a problem, no need to convert to U8 - return false; - } - input_defs[weights_idx] = &graph_utils::AddInitializer(graph, weights_proto_u8); - - // Convert weight zero point to uint8 - ONNX_NAMESPACE::TensorProto weight_zp_proto_u8; - Int8TensorProto2Uint8(weight_zp_tensor_proto, weight_zp_proto_u8, graph, true); - input_defs[weight_zp_idx] = &graph_utils::AddInitializer(graph, weight_zp_proto_u8); - - return true; -} - - struct OperatorWeightInfo { std::vector versions; const char* domain; @@ -141,6 +27,7 @@ static const std::unordered_map s8_overf {"MatMulInteger", {{10}, kOnnxDomain, 1, 3}}, {"QLinearMatMul", {{10}, kOnnxDomain, 3, 5}}, {"QLinearConv", {{10}, kOnnxDomain, 3, 5}}, + {"DequantizeLinear", {{10,13}, kMSDomain, 0, 2}}, // already covered in QDQS8ToU8Transformer but does not hurt /* {"ConvInteger", {10}, kOnnxDomain, 1, 3}, // ConvInteger does not support int8_t weight at all */ }; @@ -247,7 +134,7 @@ static bool TryConvertDynamicQuantizeLSTM(Node& op_node, Graph& graph) { input_defs[w_idx] = &graph_utils::AddInitializer(graph, weights_proto_u8); ONNX_NAMESPACE::TensorProto weight_zp_proto_u8; - Int8TensorProto2Uint8(weight_zp_tensor_proto, weight_zp_proto_u8, graph, true); + QDQ::Int8TensorProto2Uint8(weight_zp_tensor_proto, weight_zp_proto_u8, graph, true); input_defs[w_zp_idx] = &graph_utils::AddInitializer(graph, weight_zp_proto_u8); ONNX_NAMESPACE::TensorProto r_proto_u8; @@ -258,7 +145,7 @@ static bool TryConvertDynamicQuantizeLSTM(Node& op_node, Graph& graph) { input_defs[r_idx] = &graph_utils::AddInitializer(graph, r_proto_u8); ONNX_NAMESPACE::TensorProto r_zp_proto_u8; - Int8TensorProto2Uint8(r_zp_tensor_proto, r_zp_proto_u8, graph, true); + QDQ::Int8TensorProto2Uint8(r_zp_tensor_proto, r_zp_proto_u8, graph, true); input_defs[r_zp_idx] = &graph_utils::AddInitializer(graph, r_zp_proto_u8); return true; @@ -301,7 +188,7 @@ Status Avx2WeightS8ToU8Transformer::ApplyImpl(Graph& graph, bool& modified, int #endif MatchesOpSinceVersion(op_node, it->second.versions) && graph_utils::MatchesOpSetDomain(op_node, it->second.domain)) { - modified |= ConvertS8WeightToU8(graph, op_node, + modified |= QDQ::ConvertS8WeightToU8(graph, op_node, it->second.weights_idx, it->second.weight_zp_idx); continue; // finished with this op, next diff --git a/onnxruntime/core/optimizer/avx2_weight_s8_to_u8.h b/onnxruntime/core/optimizer/qdq_transformer/avx2_weight_s8_to_u8.h similarity index 100% rename from onnxruntime/core/optimizer/avx2_weight_s8_to_u8.h rename to onnxruntime/core/optimizer/qdq_transformer/avx2_weight_s8_to_u8.h diff --git a/onnxruntime/core/optimizer/qdq_transformer/qdq_s8_to_u8.cc b/onnxruntime/core/optimizer/qdq_transformer/qdq_s8_to_u8.cc index 97ce6e6f21..9ee4e9ab2b 100644 --- a/onnxruntime/core/optimizer/qdq_transformer/qdq_s8_to_u8.cc +++ b/onnxruntime/core/optimizer/qdq_transformer/qdq_s8_to_u8.cc @@ -6,10 +6,74 @@ #include "core/graph/graph_utils.h" #include "core/optimizer/initializer.h" #include "core/optimizer/qdq_transformer/qdq_util.h" +#include "core/optimizer/qdq_transformer/s8_to_u8.h" #include "core/optimizer/utils.h" namespace onnxruntime { +/** + * @brief Given a QuantizeLinear and DequantizeLinear pair with type int8_t, + * Convert them to uint8_t + * @param graph + * @param q_node + * @param dq_node + * @return whether conversion happened +*/ +static bool QDQ_S8_to_U8(Graph& graph, Node& q_node, Node& dq_node) { + auto& q_input_defs = q_node.MutableInputDefs(); + auto& dq_input_defs = dq_node.MutableInputDefs(); + + constexpr size_t input_cnt_required = 3; + if (q_input_defs.size() != input_cnt_required || + dq_input_defs.size() != input_cnt_required) { + return false; + } + + constexpr size_t zp_idx = 2; + const ONNX_NAMESPACE::TensorProto* q_zp_tensor_proto = nullptr; + const ONNX_NAMESPACE::TensorProto* dq_zp_tensor_proto = nullptr; + if (!graph_utils::NodeArgIsConstant(graph, *q_input_defs[zp_idx]) || + !graph_utils::NodeArgIsConstant(graph, *dq_input_defs[zp_idx]) || + !graph.GetInitializedTensor(q_input_defs[zp_idx]->Name(), q_zp_tensor_proto) || + !graph.GetInitializedTensor(dq_input_defs[zp_idx]->Name(), dq_zp_tensor_proto)) { + return false; + } + + //TODO(fuchen): need to augment this when we support per row quantization + using ONNX_TENSOR_ELEM_TYPE = ONNX_NAMESPACE::TensorProto::DataType; + Initializer q_zero_point(*q_zp_tensor_proto, graph.ModelPath()); + Initializer dq_zero_point(*dq_zp_tensor_proto, graph.ModelPath()); + if (q_zero_point.size() != 1 || + dq_zero_point.size() != 1 || + q_zero_point.data_type() != ONNX_TENSOR_ELEM_TYPE::TensorProto_DataType_INT8 || + dq_zero_point.data_type() != ONNX_TENSOR_ELEM_TYPE::TensorProto_DataType_INT8) { + return false; + } + + uint8_t q_zp_value = *q_zero_point.data() + 128; + uint8_t dq_zp_value = *dq_zero_point.data() + 128; + + if (q_zp_value != dq_zp_value) { + return false; // zero points for Q and DQ are expected to be same + } + + ONNX_NAMESPACE::TensorProto zp_tensor_proto_u8; + zp_tensor_proto_u8.set_data_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); + zp_tensor_proto_u8.set_name(graph.GenerateNodeArgName("qdq_s8_to_u8_zp_conversion")); + zp_tensor_proto_u8.set_raw_data(&q_zp_value, sizeof(uint8_t)); + NodeArg* zp_u8_arg = &graph_utils::AddInitializer(graph, zp_tensor_proto_u8); + + auto q_output_node_arg_name = graph.GenerateNodeArgName("qdq_s8_to_u8_quant"); + NodeArg* q_output_arg = &graph.GetOrCreateNodeArg(q_output_node_arg_name, nullptr); + + q_node.MutableOutputDefs()[0] = q_output_arg; + dq_input_defs[0] = q_output_arg; + q_input_defs[zp_idx] = zp_u8_arg; + dq_input_defs[zp_idx] = zp_u8_arg; + return true; +} + + // Convert QuantizeLinear and DequantizeLinear pair with type int8_t to type uint8_t Status QDQS8ToU8Transformer::ApplyImpl(Graph& graph, bool& modified, int graph_level, const logging::Logger& logger) const { @@ -21,73 +85,28 @@ Status QDQS8ToU8Transformer::ApplyImpl(Graph& graph, bool& modified, int graph_l if (q_node_ptr == nullptr) continue; // node removed as part of an earlier fusion - // recognize Q + DQ - Node& q_node = *q_node_ptr; - ORT_RETURN_IF_ERROR(Recurse(q_node, modified, graph_level, logger)); + Node& node = *q_node_ptr; + ORT_RETURN_IF_ERROR(Recurse(node, modified, graph_level, logger)); - if (!QDQ::MatchQNode(q_node) || - !graph_utils::IsSupportedProvider(q_node, GetCompatibleExecutionProviders()) || - !optimizer_utils::CheckOutputEdges(graph, q_node, 1)) { + if (!graph_utils::IsSupportedProvider(node, GetCompatibleExecutionProviders())) { + continue; + } + + // recognize Q + DQ pair + if (QDQ::MatchQNode(node) && + optimizer_utils::CheckOutputEdges(graph, node, 1)) { + Node& dq_node = *graph.GetNode(node.OutputNodesBegin()->Index()); + if (QDQ::MatchDQNode(dq_node)) { + modified |= QDQ_S8_to_U8(graph, node, dq_node); + } continue; } - Node& dq_node = *graph.GetNode(q_node.OutputNodesBegin()->Index()); - if (!QDQ::MatchDQNode(dq_node) || - !graph_utils::IsSupportedProvider(dq_node, GetCompatibleExecutionProviders())) { + // recognize lone DQ node + if (weights_to_u8_ && QDQ::MatchDQNode(node)) { + modified |= QDQ::ConvertS8WeightToU8(graph, node, 0, 2); continue; } - - auto& q_input_defs = q_node.MutableInputDefs(); - auto& dq_input_defs = dq_node.MutableInputDefs(); - - constexpr size_t input_cnt_required = 3; - if (q_input_defs.size() != input_cnt_required || - dq_input_defs.size() != input_cnt_required) { - continue; - } - - constexpr size_t zp_idx = 2; - const ONNX_NAMESPACE::TensorProto* q_zp_tensor_proto = nullptr; - const ONNX_NAMESPACE::TensorProto* dq_zp_tensor_proto = nullptr; - if (!graph_utils::NodeArgIsConstant(graph, *q_input_defs[zp_idx]) || - !graph_utils::NodeArgIsConstant(graph, *dq_input_defs[zp_idx]) || - !graph.GetInitializedTensor(q_input_defs[zp_idx]->Name(), q_zp_tensor_proto) || - !graph.GetInitializedTensor(dq_input_defs[zp_idx]->Name(), dq_zp_tensor_proto)) { - continue; - } - - using ONNX_TENSOR_ELEM_TYPE = ONNX_NAMESPACE::TensorProto::DataType; - Initializer q_zero_point(*q_zp_tensor_proto, graph.ModelPath()); - Initializer dq_zero_point(*dq_zp_tensor_proto, graph.ModelPath()); - if (q_zero_point.size() != 1 || - dq_zero_point.size() != 1 || - q_zero_point.data_type() != ONNX_TENSOR_ELEM_TYPE::TensorProto_DataType_INT8 || - dq_zero_point.data_type() != ONNX_TENSOR_ELEM_TYPE::TensorProto_DataType_INT8) { - continue; - } - - uint8_t q_zp_value = *q_zero_point.data() + 128; - uint8_t dq_zp_value = *dq_zero_point.data() + 128; - - if (q_zp_value != dq_zp_value) { - continue; // zero points for Q and DQ are expected to be same - } - - ONNX_NAMESPACE::TensorProto zp_tensor_proto_u8; - zp_tensor_proto_u8.set_data_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); - zp_tensor_proto_u8.set_name(graph.GenerateNodeArgName("qdq_s8_to_u8_zp_conversion")); - zp_tensor_proto_u8.set_raw_data(&q_zp_value, sizeof(uint8_t)); - NodeArg* zp_u8_arg = &graph_utils::AddInitializer(graph, zp_tensor_proto_u8); - - auto q_output_node_arg_name = graph.GenerateNodeArgName("qdq_s8_to_u8_quant"); - NodeArg* q_output_arg = &graph.GetOrCreateNodeArg(q_output_node_arg_name, nullptr); - - q_node.MutableOutputDefs()[0] = q_output_arg; - dq_input_defs[0] = q_output_arg; - q_input_defs[zp_idx] = zp_u8_arg; - dq_input_defs[zp_idx] = zp_u8_arg; - - modified = true; } return Status::OK(); diff --git a/onnxruntime/core/optimizer/qdq_transformer/qdq_s8_to_u8.h b/onnxruntime/core/optimizer/qdq_transformer/qdq_s8_to_u8.h index d6e573175e..936cb36150 100644 --- a/onnxruntime/core/optimizer/qdq_transformer/qdq_s8_to_u8.h +++ b/onnxruntime/core/optimizer/qdq_transformer/qdq_s8_to_u8.h @@ -15,11 +15,12 @@ namespace onnxruntime { */ class QDQS8ToU8Transformer : public GraphTransformer { public: - QDQS8ToU8Transformer(const InlinedHashSet& compatible_execution_providers = {}) noexcept - : GraphTransformer("QDQS8ToU8Transformer", compatible_execution_providers) {} + QDQS8ToU8Transformer(bool weights_to_u8, const InlinedHashSet& compatible_execution_providers = {}) noexcept + : GraphTransformer("QDQS8ToU8Transformer", compatible_execution_providers), weights_to_u8_(weights_to_u8) {} private: Status ApplyImpl(Graph& graph, bool& modified, int graph_level, const logging::Logger& logger) const override; + bool weights_to_u8_; }; } // namespace onnxruntime diff --git a/onnxruntime/core/optimizer/qdq_transformer/s8_to_u8.cc b/onnxruntime/core/optimizer/qdq_transformer/s8_to_u8.cc new file mode 100644 index 0000000000..b4c1fd4d2e --- /dev/null +++ b/onnxruntime/core/optimizer/qdq_transformer/s8_to_u8.cc @@ -0,0 +1,56 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/optimizer/qdq_transformer/s8_to_u8.h" + +namespace onnxruntime::QDQ { + +bool ConvertS8WeightToU8(Graph& graph, Node& op_node, + size_t weights_idx, size_t weight_zp_idx) { + auto& input_defs = op_node.MutableInputDefs(); + if (input_defs.size() < weights_idx + 1) { + return false; + } + + // Weight tensor must be const int8_t + const ONNX_NAMESPACE::TensorProto* weight_tensor_proto = nullptr; + const auto* w_def = input_defs[weights_idx]; + if (!graph_utils::NodeArgIsConstant(graph, *w_def) || + !graph.GetInitializedTensor(w_def->Name(), weight_tensor_proto) || + weight_tensor_proto->data_type() != ONNX_NAMESPACE::TensorProto_DataType_INT8) { + return false; + } + ORT_ENFORCE(nullptr != weight_tensor_proto, + "Internal Error: weight tensor must be const int8 for Avx2WeightS8ToU8Transformer."); + + // Weight zero point must be either const int8_t or null tensor + const ONNX_NAMESPACE::TensorProto* weight_zp_tensor_proto = nullptr; + const auto* zp_def = input_defs.size() <= weight_zp_idx ? nullptr : input_defs[weight_zp_idx]; + if (nullptr != zp_def) { + if (!graph_utils::NodeArgIsConstant(graph, *zp_def) || + !graph.GetInitializedTensor(zp_def->Name(), weight_zp_tensor_proto) || + weight_zp_tensor_proto->data_type() != ONNX_NAMESPACE::TensorProto_DataType_INT8) { + return false; + } + ORT_ENFORCE(nullptr != weight_zp_tensor_proto, + "Internal Error: weight zero point must be const int8 for Avx2WeightS8ToU8Transformer."); + } + + // Convert weight tensor to uint8 + ONNX_NAMESPACE::TensorProto weights_proto_u8; + bool converted = Int8TensorProto2Uint8(weight_tensor_proto, weights_proto_u8, graph); + if (!converted) { + // The weights fits into S7, overflow is not a problem, no need to convert to U8 + return false; + } + input_defs[weights_idx] = &graph_utils::AddInitializer(graph, weights_proto_u8); + + // Convert weight zero point to uint8 + ONNX_NAMESPACE::TensorProto weight_zp_proto_u8; + Int8TensorProto2Uint8(weight_zp_tensor_proto, weight_zp_proto_u8, graph, true); + input_defs[weight_zp_idx] = &graph_utils::AddInitializer(graph, weight_zp_proto_u8); + + return true; +} + +} \ No newline at end of file diff --git a/onnxruntime/core/optimizer/qdq_transformer/s8_to_u8.h b/onnxruntime/core/optimizer/qdq_transformer/s8_to_u8.h new file mode 100644 index 0000000000..0279e248f4 --- /dev/null +++ b/onnxruntime/core/optimizer/qdq_transformer/s8_to_u8.h @@ -0,0 +1,78 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/graph/graph_utils.h" +#include "core/optimizer/initializer.h" + +namespace onnxruntime::QDQ { + +/** + * @brief Convert the source int8_t TensorProto to a uint8_t one if the tensor + * contains values outside of [-64, 64] + * @param src The source tensor, must be type int8_t + * @param dst An empty tensor, will contain the converted tensor data + * @param graph Graph for generating tensor name or provide external + * data path + * @param force Perform conversion even when tensor values within [-64, 64] + * @return Whether the conversion happened. +*/ +inline bool Int8TensorProto2Uint8( + const ONNX_NAMESPACE::TensorProto* src, + ONNX_NAMESPACE::TensorProto& dst, + Graph& graph, bool force = false) { + dst.set_data_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8); + + if (nullptr == src) { + uint8_t zero_val = 128; + dst.set_name(graph.GenerateNodeArgName("weight_zp_s8_2_u8")); + dst.set_raw_data(&zero_val, sizeof(uint8_t)); + return true; + } + + dst.set_name(src->name() + "_s8_2_u8"); + dst.mutable_dims()->CopyFrom(src->dims()); + + // TODO(fuchen): too many copies! + // + // Here we do two memory copies: Proto -> Initializer -> Proto. + // Ideally we only do 1 copy, just iterate the source data, and write directly + // to the dst raw buffer. + // Unfortunately iterating the source data is complicated, the data maybe in + // external file, a raw buffer, or a repeated field depending on the data + // type. UnpackTensor() already contains some of these logic and is closest + // to what we need. But it does not handle external data. Write our own code + // here means copy the logic of TensorProtoToTensor(), a violation of DRY + // principle. A better solution is to provide an efficient const iterator for + // TensorProto. This require coordination with onnx side. + + Initializer temp(*src, graph.ModelPath()); + int8_t* p = temp.data(); + bool should_convert = false; + for (int i = 0; i < temp.size(); i++) { + if (*p < -64 || *p > 64) { + should_convert = true; + } + *p ^= 0x80; + p++; + } + if (force || should_convert) { + dst.set_raw_data(temp.data(), size_t(temp.size())); + return true; + } + return false; +} + +/** + * @brief If the op_node has an single int8_t const weight tensor, convert it to uint8_t + * @param graph + * @param op_node + * @param weights_idx input index of the weight tensor + * @param weight_zp_idx input index of the weight zero point tensor + * @return true when conversion happened. +*/ +extern bool ConvertS8WeightToU8(Graph& graph, Node& op_node, + size_t weights_idx, size_t weight_zp_idx); + +} \ No newline at end of file diff --git a/onnxruntime/test/optimizer/avx2_weight_s8_to_u8_test.cc b/onnxruntime/test/optimizer/avx2_weight_s8_to_u8_test.cc index 39709dd0e1..b25475e46c 100644 --- a/onnxruntime/test/optimizer/avx2_weight_s8_to_u8_test.cc +++ b/onnxruntime/test/optimizer/avx2_weight_s8_to_u8_test.cc @@ -1,4 +1,4 @@ -#include "core/optimizer/avx2_weight_s8_to_u8.h" +#include "core/optimizer/qdq_transformer/avx2_weight_s8_to_u8.h" #include #include diff --git a/onnxruntime/test/optimizer/qdq_transformer_test.cc b/onnxruntime/test/optimizer/qdq_transformer_test.cc index 79bb64b122..4131cbe497 100644 --- a/onnxruntime/test/optimizer/qdq_transformer_test.cc +++ b/onnxruntime/test/optimizer/qdq_transformer_test.cc @@ -217,6 +217,55 @@ TEST(QDQTransformerTests, ConvMaxPoolReshape_Int8) { test_case({1, 22, 11, 13, 15}, {30, 22, 5, 3, 3}); } +#if (defined(_M_AMD64) && !defined(_M_ARM64EC)) || defined(_M_IX86) || defined(__x86_64__) || defined(__i386__) || !defined(DISABLE_CONTRIB_OPS) + +TEST(QDQTransformerTests, DQ_S8_to_U8) { + const std::vector& input_shape = {19, 37}; + const std::vector& weights_shape = {37, 23}; + + auto build_test_case = [&](ModelTestBuilder& builder) { + auto* input1_arg = builder.MakeInput(input_shape, -1.f, 1.f); + + // Use full range weight values to expose avx2 u8s8 overflow problems + auto* weight = builder.MakeInitializer(weights_shape, -128, 127); + auto* output_arg = builder.MakeOutput(); + + // add QDQ activation + typedef std::numeric_limits Input1Limits; + auto* dq1_output = AddQDQNodePair(builder, input1_arg, .039f, (int8_t)((Input1Limits::max() + Input1Limits::min()) / 2 + 1)); + + // add DQ weight + auto* dq_w_output = builder.MakeIntermediate(); + builder.AddDequantizeLinearNode(weight, .003f, -10, dq_w_output); + + + builder.AddNode("MatMul", {dq1_output, dq_w_output}, {output_arg}); + }; + + auto check_graph = [&](InferenceSessionWrapper& session) { + auto op_to_count = CountOpsInGraph(session.GetGraph()); + EXPECT_EQ(op_to_count["com.microsoft.MatMulIntegerToFloat"], 1); + EXPECT_EQ(op_to_count["MatMul"], 0); + EXPECT_EQ(op_to_count["QuantizeLinear"], 1); + EXPECT_EQ(op_to_count["DequantizeLinear"], 0); + }; + + auto add_session_options = [&](SessionOptions& so) { + ASSERT_STATUS_OK(so.config_options.AddConfigEntry( + kOrtSessionOptionsAvx2PrecisionMode, "1")); + }; + + TransformerTester(build_test_case, + check_graph, + TransformerLevel::Level1, + TransformerLevel::Level2, + 12 /*opset_version*/, + 0.01 /*per_sample_tolerance*/, + 0.01 /*relative_per_sample_tolerance*/, + nullptr, add_session_options); +} +#endif // Only for X64 with contrib ops enabled + template void QDQTransformerAveragePoolTests() { auto test_case = [&](const std::vector& input_shape) {