saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-29 20:14:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Convert DQ node with const weight tensor int8 to uint8 (#12331)

Browse source 
Convert DQ node with const weight tensor int8 to uint8

This is a follow-up with #12088, where convert weight tensor from int8 to uint8. Here we do the same thing in DequantizeLinear node, so that we don't have to perform the same changes for every single future operator.
This commit is contained in:
Chen Fu 2022-07-28 09:22:04 -07:00 committed by GitHub
parent e2423bb55c
commit 73919a6756
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
9 changed files with 281 additions and 189 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
onnxruntime/core/optimizer/graph_transformer_utils.cc
View file

@ -19,7 +19,7 @@
#include "core/mlas/inc/mlas.h"
#include "core/optimizer/attention_fusion.h"
#ifdef MLAS_TARGET_AMD64_IX86
#include "core/optimizer/avx2_weight_s8_to_u8.h"
#include "core/optimizer/qdq_transformer/avx2_weight_s8_to_u8.h"
#endif
#include "core/optimizer/bias_dropout_fusion.h"
#include "core/optimizer/bias_gelu_fusion.h"
@ -220,12 +220,18 @@ InlinedVector<std::unique_ptr<GraphTransformer>> GenerateTransformers(
onnxruntime::kAclExecutionProvider,
onnxruntime::kArmNNExecutionProvider};
#ifdef MLAS_TARGET_AMD64_IX86
const bool avx2_precision_mode =
session_options.config_options.GetConfigOrDefault(kOrtSessionOptionsAvx2PrecisionMode, "0") == "1" && MlasPlatformU8S8Overflow();
#else
const bool avx2_precision_mode = false;
#endif
if (!disable_quant_qdq) {
// currently we don't support QDQS8ToU8Transformer in a minimal build and if supported, this needs to run in
// Level 1 during export and not Level 2 at runtime as it would result in overlapping optimizations which
// runtime optimization does not support, so add session config value here to force qdqisint8allowed to be true.
if (!qdq_is_int8_allowed) {
transformers.emplace_back(std::make_unique<QDQS8ToU8Transformer>(cpu_ep));
transformers.emplace_back(std::make_unique<QDQS8ToU8Transformer>(avx2_precision_mode, cpu_ep));
}
transformers.emplace_back(std::make_unique<QDQSelectorActionTransformer>(qdq_is_int8_allowed));
}
@ -263,10 +269,6 @@ InlinedVector<std::unique_ptr<GraphTransformer>> GenerateTransformers(
}
#ifdef MLAS_TARGET_AMD64_IX86
const bool avx2_precision_mode =
session_options.config_options.GetConfigOrDefault(kOrtSessionOptionsAvx2PrecisionMode, "0") == "1"
&& MlasPlatformU8S8Overflow();
if (avx2_precision_mode) {
transformers.emplace_back(std::make_unique<Avx2WeightS8ToU8Transformer>(cpu_ep));
}

125
onnxruntime/core/optimizer/avx2_weight_s8_to_u8.cc → onnxruntime/core/optimizer/qdq_transformer/avx2_weight_s8_to_u8.cc
View file

@ -3,129 +3,15 @@
#if (defined(_M_AMD64) && !defined(_M_ARM64EC)) || defined(_M_IX86) || defined(__x86_64__) || defined(__i386__) || !defined(DISABLE_CONTRIB_OPS)
#include "core/optimizer/avx2_weight_s8_to_u8.h"
#include "core/optimizer/qdq_transformer/avx2_weight_s8_to_u8.h"
#include "core/optimizer/qdq_transformer/s8_to_u8.h"
#include <algorithm>
#include "core/graph/graph_utils.h"
#include "core/optimizer/initializer.h"
#include "core/optimizer/utils.h"
namespace onnxruntime {
/**
* @brief Convert the source int8_t TensorProto to a uint8_t one if the tensor
* contains values outside of [-64, 64]
* @param src The source tensor, must be type int8_t
* @param dst An empty tensor, will contain the converted tensor data
* @param graph Graph for generating tensor name or provide external
* data path
* @param force Perform conversion even when tensor values within [-64, 64]
* @return Whether the conversion happened.
*/
static inline bool Int8TensorProto2Uint8(
const ONNX_NAMESPACE::TensorProto* src,
ONNX_NAMESPACE::TensorProto& dst,
Graph& graph, bool force = false) {
dst.set_data_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8);
if (nullptr == src) {
uint8_t zero_val = 128;
dst.set_name(graph.GenerateNodeArgName("weight_zp_s8_2_u8"));
dst.set_raw_data(&zero_val, sizeof(uint8_t));
return true;
}
dst.set_name(src->name() + "_s8_2_u8");
dst.mutable_dims()->CopyFrom(src->dims());
// TODO(fuchen): too many copies!
//
// Here we do two memory copies: Proto -> Initializer -> Proto.
// Ideally we only do 1 copy, just iterate the source data, and write directly
// to the dst raw buffer.
// Unfortunately iterating the source data is complicated, the data maybe in
// external file, a raw buffer, or a repeated field depending on the data
// type. UnpackTensor() already contains some of these logic and is closest
// to what we need. But it does not handle external data. Write our own code
// here means copy the logic of TensorProtoToTensor(), a violation of DRY
// principle. A better solution is to provide an efficient const iterator for
// TensorProto. This require coordination with onnx side.
Initializer temp(*src, graph.ModelPath());
int8_t* p = temp.data<int8_t>();
bool should_convert = false;
for (int i = 0; i < temp.size(); i++) {
if (*p < -64 || *p > 64) {
should_convert = true;
}
*p ^= 0x80;
p++;
}
if (force || should_convert) {
dst.set_raw_data(temp.data<int8_t>(), temp.size());
return true;
}
return false;
}
/**
* @brief If the op_node has an uint8_t const weight tensor, convert it to int8_t
* @param graph
* @param op_node
* @param weights_idx
* @param weight_zp_idx
* @return true when conversion happened.
*/
static bool ConvertS8WeightToU8(Graph& graph, Node& op_node,
size_t weights_idx, size_t weight_zp_idx) {
auto& input_defs = op_node.MutableInputDefs();
if (input_defs.size() < weights_idx + 1) {
return false;
}
// Weight tensor must be const int8_t
const ONNX_NAMESPACE::TensorProto* weight_tensor_proto = nullptr;
const auto* w_def = input_defs[weights_idx];
if (!graph_utils::NodeArgIsConstant(graph, *w_def) ||
!graph.GetInitializedTensor(w_def->Name(), weight_tensor_proto) ||
weight_tensor_proto->data_type() != ONNX_NAMESPACE::TensorProto_DataType_INT8) {
return false;
}
ORT_ENFORCE(nullptr != weight_tensor_proto,
"Internal Error: weight tensor must be const int8 for Avx2WeightS8ToU8Transformer.");
// Weight zero point must be either const int8_t or null tensor
const ONNX_NAMESPACE::TensorProto* weight_zp_tensor_proto = nullptr;
const auto* zp_def = input_defs.size() <= weight_zp_idx ? nullptr : input_defs[weight_zp_idx];
if (nullptr != zp_def) {
if (!graph_utils::NodeArgIsConstant(graph, *zp_def) ||
!graph.GetInitializedTensor(zp_def->Name(), weight_zp_tensor_proto) ||
weight_zp_tensor_proto->data_type() != ONNX_NAMESPACE::TensorProto_DataType_INT8) {
return false;
}
ORT_ENFORCE(nullptr != weight_zp_tensor_proto,
"Internal Error: weight zero point must be const int8 for Avx2WeightS8ToU8Transformer.");
}
// Convert weight tensor to uint8
ONNX_NAMESPACE::TensorProto weights_proto_u8;
bool converted = Int8TensorProto2Uint8(weight_tensor_proto, weights_proto_u8, graph);
if (!converted) {
// The weights fits into S7, overflow is not a problem, no need to convert to U8
return false;
}
input_defs[weights_idx] = &graph_utils::AddInitializer(graph, weights_proto_u8);
// Convert weight zero point to uint8
ONNX_NAMESPACE::TensorProto weight_zp_proto_u8;
Int8TensorProto2Uint8(weight_zp_tensor_proto, weight_zp_proto_u8, graph, true);
input_defs[weight_zp_idx] = &graph_utils::AddInitializer(graph, weight_zp_proto_u8);
return true;
}
struct OperatorWeightInfo {
std::vector<ONNX_NAMESPACE::OperatorSetVersion> versions;
const char* domain;
@ -141,6 +27,7 @@ static const std::unordered_map<std::string, struct OperatorWeightInfo> s8_overf
{"MatMulInteger", {{10}, kOnnxDomain, 1, 3}},
{"QLinearMatMul", {{10}, kOnnxDomain, 3, 5}},
{"QLinearConv", {{10}, kOnnxDomain, 3, 5}},
{"DequantizeLinear", {{10,13}, kMSDomain, 0, 2}}, // already covered in QDQS8ToU8Transformer but does not hurt
/* {"ConvInteger", {10}, kOnnxDomain, 1, 3}, // ConvInteger does not support int8_t weight at all */
};
@ -247,7 +134,7 @@ static bool TryConvertDynamicQuantizeLSTM(Node& op_node, Graph& graph) {
input_defs[w_idx] = &graph_utils::AddInitializer(graph, weights_proto_u8);
ONNX_NAMESPACE::TensorProto weight_zp_proto_u8;
Int8TensorProto2Uint8(weight_zp_tensor_proto, weight_zp_proto_u8, graph, true);
QDQ::Int8TensorProto2Uint8(weight_zp_tensor_proto, weight_zp_proto_u8, graph, true);
input_defs[w_zp_idx] = &graph_utils::AddInitializer(graph, weight_zp_proto_u8);
ONNX_NAMESPACE::TensorProto r_proto_u8;
@ -258,7 +145,7 @@ static bool TryConvertDynamicQuantizeLSTM(Node& op_node, Graph& graph) {
input_defs[r_idx] = &graph_utils::AddInitializer(graph, r_proto_u8);
ONNX_NAMESPACE::TensorProto r_zp_proto_u8;
Int8TensorProto2Uint8(r_zp_tensor_proto, r_zp_proto_u8, graph, true);
QDQ::Int8TensorProto2Uint8(r_zp_tensor_proto, r_zp_proto_u8, graph, true);
input_defs[r_zp_idx] = &graph_utils::AddInitializer(graph, r_zp_proto_u8);
return true;
@ -301,7 +188,7 @@ Status Avx2WeightS8ToU8Transformer::ApplyImpl(Graph& graph, bool& modified, int
#endif
MatchesOpSinceVersion(op_node, it->second.versions) &&
graph_utils::MatchesOpSetDomain(op_node, it->second.domain)) {
modified |= ConvertS8WeightToU8(graph, op_node,
modified |= QDQ::ConvertS8WeightToU8(graph, op_node,
it->second.weights_idx,
it->second.weight_zp_idx);
continue; // finished with this op, next

0
onnxruntime/core/optimizer/avx2_weight_s8_to_u8.h → onnxruntime/core/optimizer/qdq_transformer/avx2_weight_s8_to_u8.h
View file

141
onnxruntime/core/optimizer/qdq_transformer/qdq_s8_to_u8.cc
View file

@ -6,10 +6,74 @@
#include "core/graph/graph_utils.h"
#include "core/optimizer/initializer.h"
#include "core/optimizer/qdq_transformer/qdq_util.h"
#include "core/optimizer/qdq_transformer/s8_to_u8.h"
#include "core/optimizer/utils.h"
namespace onnxruntime {
/**
* @brief Given a QuantizeLinear and DequantizeLinear pair with type int8_t,
* Convert them to uint8_t
* @param graph
* @param q_node
* @param dq_node
* @return whether conversion happened
*/
static bool QDQ_S8_to_U8(Graph& graph, Node& q_node, Node& dq_node) {
auto& q_input_defs = q_node.MutableInputDefs();
auto& dq_input_defs = dq_node.MutableInputDefs();
constexpr size_t input_cnt_required = 3;
if (q_input_defs.size() != input_cnt_required ||
dq_input_defs.size() != input_cnt_required) {
return false;
}
constexpr size_t zp_idx = 2;
const ONNX_NAMESPACE::TensorProto* q_zp_tensor_proto = nullptr;
const ONNX_NAMESPACE::TensorProto* dq_zp_tensor_proto = nullptr;
if (!graph_utils::NodeArgIsConstant(graph, *q_input_defs[zp_idx]) ||
!graph_utils::NodeArgIsConstant(graph, *dq_input_defs[zp_idx]) ||
!graph.GetInitializedTensor(q_input_defs[zp_idx]->Name(), q_zp_tensor_proto) ||
!graph.GetInitializedTensor(dq_input_defs[zp_idx]->Name(), dq_zp_tensor_proto)) {
return false;
}
//TODO(fuchen): need to augment this when we support per row quantization
using ONNX_TENSOR_ELEM_TYPE = ONNX_NAMESPACE::TensorProto::DataType;
Initializer q_zero_point(*q_zp_tensor_proto, graph.ModelPath());
Initializer dq_zero_point(*dq_zp_tensor_proto, graph.ModelPath());
if (q_zero_point.size() != 1 ||
dq_zero_point.size() != 1 ||
q_zero_point.data_type() != ONNX_TENSOR_ELEM_TYPE::TensorProto_DataType_INT8 ||
dq_zero_point.data_type() != ONNX_TENSOR_ELEM_TYPE::TensorProto_DataType_INT8) {
return false;
}
uint8_t q_zp_value = *q_zero_point.data<int8_t>() + 128;
uint8_t dq_zp_value = *dq_zero_point.data<int8_t>() + 128;
if (q_zp_value != dq_zp_value) {
return false; // zero points for Q and DQ are expected to be same
}
ONNX_NAMESPACE::TensorProto zp_tensor_proto_u8;
zp_tensor_proto_u8.set_data_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8);
zp_tensor_proto_u8.set_name(graph.GenerateNodeArgName("qdq_s8_to_u8_zp_conversion"));
zp_tensor_proto_u8.set_raw_data(&q_zp_value, sizeof(uint8_t));
NodeArg* zp_u8_arg = &graph_utils::AddInitializer(graph, zp_tensor_proto_u8);
auto q_output_node_arg_name = graph.GenerateNodeArgName("qdq_s8_to_u8_quant");
NodeArg* q_output_arg = &graph.GetOrCreateNodeArg(q_output_node_arg_name, nullptr);
q_node.MutableOutputDefs()[0] = q_output_arg;
dq_input_defs[0] = q_output_arg;
q_input_defs[zp_idx] = zp_u8_arg;
dq_input_defs[zp_idx] = zp_u8_arg;
return true;
}
// Convert QuantizeLinear and DequantizeLinear pair with type int8_t to type uint8_t
Status QDQS8ToU8Transformer::ApplyImpl(Graph& graph, bool& modified, int graph_level,
const logging::Logger& logger) const {
@ -21,73 +85,28 @@ Status QDQS8ToU8Transformer::ApplyImpl(Graph& graph, bool& modified, int graph_l
if (q_node_ptr == nullptr)
continue; // node removed as part of an earlier fusion
// recognize Q + DQ
Node& q_node = *q_node_ptr;
ORT_RETURN_IF_ERROR(Recurse(q_node, modified, graph_level, logger));
Node& node = *q_node_ptr;
ORT_RETURN_IF_ERROR(Recurse(node, modified, graph_level, logger));
if (!QDQ::MatchQNode(q_node) ||
!graph_utils::IsSupportedProvider(q_node, GetCompatibleExecutionProviders()) ||
!optimizer_utils::CheckOutputEdges(graph, q_node, 1)) {
if (!graph_utils::IsSupportedProvider(node, GetCompatibleExecutionProviders())) {
continue;
}
// recognize Q + DQ pair
if (QDQ::MatchQNode(node) &&
optimizer_utils::CheckOutputEdges(graph, node, 1)) {
Node& dq_node = *graph.GetNode(node.OutputNodesBegin()->Index());
if (QDQ::MatchDQNode(dq_node)) {
modified |= QDQ_S8_to_U8(graph, node, dq_node);
}
continue;
}
Node& dq_node = *graph.GetNode(q_node.OutputNodesBegin()->Index());
if (!QDQ::MatchDQNode(dq_node) ||
!graph_utils::IsSupportedProvider(dq_node, GetCompatibleExecutionProviders())) {
// recognize lone DQ node
if (weights_to_u8_ && QDQ::MatchDQNode(node)) {
modified |= QDQ::ConvertS8WeightToU8(graph, node, 0, 2);
continue;
}
auto& q_input_defs = q_node.MutableInputDefs();
auto& dq_input_defs = dq_node.MutableInputDefs();
constexpr size_t input_cnt_required = 3;
if (q_input_defs.size() != input_cnt_required ||
dq_input_defs.size() != input_cnt_required) {
continue;
}
constexpr size_t zp_idx = 2;
const ONNX_NAMESPACE::TensorProto* q_zp_tensor_proto = nullptr;
const ONNX_NAMESPACE::TensorProto* dq_zp_tensor_proto = nullptr;
if (!graph_utils::NodeArgIsConstant(graph, *q_input_defs[zp_idx]) ||
!graph_utils::NodeArgIsConstant(graph, *dq_input_defs[zp_idx]) ||
!graph.GetInitializedTensor(q_input_defs[zp_idx]->Name(), q_zp_tensor_proto) ||
!graph.GetInitializedTensor(dq_input_defs[zp_idx]->Name(), dq_zp_tensor_proto)) {
continue;
}
using ONNX_TENSOR_ELEM_TYPE = ONNX_NAMESPACE::TensorProto::DataType;
Initializer q_zero_point(*q_zp_tensor_proto, graph.ModelPath());
Initializer dq_zero_point(*dq_zp_tensor_proto, graph.ModelPath());
if (q_zero_point.size() != 1 ||
dq_zero_point.size() != 1 ||
q_zero_point.data_type() != ONNX_TENSOR_ELEM_TYPE::TensorProto_DataType_INT8 ||
dq_zero_point.data_type() != ONNX_TENSOR_ELEM_TYPE::TensorProto_DataType_INT8) {
continue;
}
uint8_t q_zp_value = *q_zero_point.data<int8_t>() + 128;
uint8_t dq_zp_value = *dq_zero_point.data<int8_t>() + 128;
if (q_zp_value != dq_zp_value) {
continue; // zero points for Q and DQ are expected to be same
}
ONNX_NAMESPACE::TensorProto zp_tensor_proto_u8;
zp_tensor_proto_u8.set_data_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8);
zp_tensor_proto_u8.set_name(graph.GenerateNodeArgName("qdq_s8_to_u8_zp_conversion"));
zp_tensor_proto_u8.set_raw_data(&q_zp_value, sizeof(uint8_t));
NodeArg* zp_u8_arg = &graph_utils::AddInitializer(graph, zp_tensor_proto_u8);
auto q_output_node_arg_name = graph.GenerateNodeArgName("qdq_s8_to_u8_quant");
NodeArg* q_output_arg = &graph.GetOrCreateNodeArg(q_output_node_arg_name, nullptr);
q_node.MutableOutputDefs()[0] = q_output_arg;
dq_input_defs[0] = q_output_arg;
q_input_defs[zp_idx] = zp_u8_arg;
dq_input_defs[zp_idx] = zp_u8_arg;
modified = true;
}
return Status::OK();

5
onnxruntime/core/optimizer/qdq_transformer/qdq_s8_to_u8.h
View file

@ -15,11 +15,12 @@ namespace onnxruntime {
*/
class QDQS8ToU8Transformer : public GraphTransformer {
public:
QDQS8ToU8Transformer(const InlinedHashSet<std::string_view>& compatible_execution_providers = {}) noexcept
: GraphTransformer("QDQS8ToU8Transformer", compatible_execution_providers) {}
QDQS8ToU8Transformer(bool weights_to_u8, const InlinedHashSet<std::string_view>& compatible_execution_providers = {}) noexcept
: GraphTransformer("QDQS8ToU8Transformer", compatible_execution_providers), weights_to_u8_(weights_to_u8) {}
private:
Status ApplyImpl(Graph& graph, bool& modified, int graph_level, const logging::Logger& logger) const override;
bool weights_to_u8_;
};
} // namespace onnxruntime

56
onnxruntime/core/optimizer/qdq_transformer/s8_to_u8.cc Normal file
View file

@ -0,0 +1,56 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include "core/optimizer/qdq_transformer/s8_to_u8.h"
namespace onnxruntime::QDQ {
bool ConvertS8WeightToU8(Graph& graph, Node& op_node,
size_t weights_idx, size_t weight_zp_idx) {
auto& input_defs = op_node.MutableInputDefs();
if (input_defs.size() < weights_idx + 1) {
return false;
}
// Weight tensor must be const int8_t
const ONNX_NAMESPACE::TensorProto* weight_tensor_proto = nullptr;
const auto* w_def = input_defs[weights_idx];
if (!graph_utils::NodeArgIsConstant(graph, *w_def) ||
!graph.GetInitializedTensor(w_def->Name(), weight_tensor_proto) ||
weight_tensor_proto->data_type() != ONNX_NAMESPACE::TensorProto_DataType_INT8) {
return false;
}
ORT_ENFORCE(nullptr != weight_tensor_proto,
"Internal Error: weight tensor must be const int8 for Avx2WeightS8ToU8Transformer.");
// Weight zero point must be either const int8_t or null tensor
const ONNX_NAMESPACE::TensorProto* weight_zp_tensor_proto = nullptr;
const auto* zp_def = input_defs.size() <= weight_zp_idx ? nullptr : input_defs[weight_zp_idx];
if (nullptr != zp_def) {
if (!graph_utils::NodeArgIsConstant(graph, *zp_def) ||
!graph.GetInitializedTensor(zp_def->Name(), weight_zp_tensor_proto) ||
weight_zp_tensor_proto->data_type() != ONNX_NAMESPACE::TensorProto_DataType_INT8) {
return false;
}
ORT_ENFORCE(nullptr != weight_zp_tensor_proto,
"Internal Error: weight zero point must be const int8 for Avx2WeightS8ToU8Transformer.");
}
// Convert weight tensor to uint8
ONNX_NAMESPACE::TensorProto weights_proto_u8;
bool converted = Int8TensorProto2Uint8(weight_tensor_proto, weights_proto_u8, graph);
if (!converted) {
// The weights fits into S7, overflow is not a problem, no need to convert to U8
return false;
}
input_defs[weights_idx] = &graph_utils::AddInitializer(graph, weights_proto_u8);
// Convert weight zero point to uint8
ONNX_NAMESPACE::TensorProto weight_zp_proto_u8;
Int8TensorProto2Uint8(weight_zp_tensor_proto, weight_zp_proto_u8, graph, true);
input_defs[weight_zp_idx] = &graph_utils::AddInitializer(graph, weight_zp_proto_u8);
return true;
}
}

78
onnxruntime/core/optimizer/qdq_transformer/s8_to_u8.h Normal file
View file

@ -0,0 +1,78 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#pragma once
#include "core/graph/graph_utils.h"
#include "core/optimizer/initializer.h"
namespace onnxruntime::QDQ {
/**
* @brief Convert the source int8_t TensorProto to a uint8_t one if the tensor
* contains values outside of [-64, 64]
* @param src The source tensor, must be type int8_t
* @param dst An empty tensor, will contain the converted tensor data
* @param graph Graph for generating tensor name or provide external
* data path
* @param force Perform conversion even when tensor values within [-64, 64]
* @return Whether the conversion happened.
*/
inline bool Int8TensorProto2Uint8(
const ONNX_NAMESPACE::TensorProto* src,
ONNX_NAMESPACE::TensorProto& dst,
Graph& graph, bool force = false) {
dst.set_data_type(ONNX_NAMESPACE::TensorProto_DataType_UINT8);
if (nullptr == src) {
uint8_t zero_val = 128;
dst.set_name(graph.GenerateNodeArgName("weight_zp_s8_2_u8"));
dst.set_raw_data(&zero_val, sizeof(uint8_t));
return true;
}
dst.set_name(src->name() + "_s8_2_u8");
dst.mutable_dims()->CopyFrom(src->dims());
// TODO(fuchen): too many copies!
//
// Here we do two memory copies: Proto -> Initializer -> Proto.
// Ideally we only do 1 copy, just iterate the source data, and write directly
// to the dst raw buffer.
// Unfortunately iterating the source data is complicated, the data maybe in
// external file, a raw buffer, or a repeated field depending on the data
// type. UnpackTensor() already contains some of these logic and is closest
// to what we need. But it does not handle external data. Write our own code
// here means copy the logic of TensorProtoToTensor(), a violation of DRY
// principle. A better solution is to provide an efficient const iterator for
// TensorProto. This require coordination with onnx side.
Initializer temp(*src, graph.ModelPath());
int8_t* p = temp.data<int8_t>();
bool should_convert = false;
for (int i = 0; i < temp.size(); i++) {
if (*p < -64 || *p > 64) {
should_convert = true;
}
*p ^= 0x80;
p++;
}
if (force || should_convert) {
dst.set_raw_data(temp.data<int8_t>(), size_t(temp.size()));
return true;
}
return false;
}
/**
* @brief If the op_node has an single int8_t const weight tensor, convert it to uint8_t
* @param graph
* @param op_node
* @param weights_idx input index of the weight tensor
* @param weight_zp_idx input index of the weight zero point tensor
* @return true when conversion happened.
*/
extern bool ConvertS8WeightToU8(Graph& graph, Node& op_node,
size_t weights_idx, size_t weight_zp_idx);
}

2
onnxruntime/test/optimizer/avx2_weight_s8_to_u8_test.cc
View file

@ -1,4 +1,4 @@
#include "core/optimizer/avx2_weight_s8_to_u8.h"
#include "core/optimizer/qdq_transformer/avx2_weight_s8_to_u8.h"
#include <functional>
#include <string>

49
onnxruntime/test/optimizer/qdq_transformer_test.cc
View file

@ -217,6 +217,55 @@ TEST(QDQTransformerTests, ConvMaxPoolReshape_Int8) {
test_case({1, 22, 11, 13, 15}, {30, 22, 5, 3, 3});
}
#if (defined(_M_AMD64) && !defined(_M_ARM64EC)) || defined(_M_IX86) || defined(__x86_64__) || defined(__i386__) || !defined(DISABLE_CONTRIB_OPS)
TEST(QDQTransformerTests, DQ_S8_to_U8) {
const std::vector<int64_t>& input_shape = {19, 37};
const std::vector<int64_t>& weights_shape = {37, 23};
auto build_test_case = [&](ModelTestBuilder& builder) {
auto* input1_arg = builder.MakeInput<float>(input_shape, -1.f, 1.f);
// Use full range weight values to expose avx2 u8s8 overflow problems
auto* weight = builder.MakeInitializer<int8_t>(weights_shape, -128, 127);
auto* output_arg = builder.MakeOutput();
// add QDQ activation
typedef std::numeric_limits<uint8_t> Input1Limits;
auto* dq1_output = AddQDQNodePair<int8_t>(builder, input1_arg, .039f, (int8_t)((Input1Limits::max() + Input1Limits::min()) / 2 + 1));
// add DQ weight
auto* dq_w_output = builder.MakeIntermediate();
builder.AddDequantizeLinearNode<int8_t>(weight, .003f, -10, dq_w_output);
builder.AddNode("MatMul", {dq1_output, dq_w_output}, {output_arg});
};
auto check_graph = [&](InferenceSessionWrapper& session) {
auto op_to_count = CountOpsInGraph(session.GetGraph());
EXPECT_EQ(op_to_count["com.microsoft.MatMulIntegerToFloat"], 1);
EXPECT_EQ(op_to_count["MatMul"], 0);
EXPECT_EQ(op_to_count["QuantizeLinear"], 1);
EXPECT_EQ(op_to_count["DequantizeLinear"], 0);
};
auto add_session_options = [&](SessionOptions& so) {
ASSERT_STATUS_OK(so.config_options.AddConfigEntry(
kOrtSessionOptionsAvx2PrecisionMode, "1"));
};
TransformerTester(build_test_case,
check_graph,
TransformerLevel::Level1,
TransformerLevel::Level2,
12 /*opset_version*/,
0.01 /*per_sample_tolerance*/,
0.01 /*relative_per_sample_tolerance*/,
nullptr, add_session_options);
}
#endif // Only for X64 with contrib ops enabled
template <typename InputType, typename OutputType>
void QDQTransformerAveragePoolTests() {
auto test_case = [&](const std::vector<int64_t>& input_shape) {