saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-30 20:18:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

add qdq support for LeakyRelu (#10077)

Browse source 
* add qdq support for LeakyRelu
This commit is contained in:
Yufeng Li 2022-01-03 14:48:49 -08:00 committed by GitHub
parent 1d3b34cc92
commit 230f323600
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
4 changed files with 98 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_selector_action_transformer.cc
View file

@ -45,20 +45,20 @@ void DropQDQNodesRules(SelectorsAndActions& qdq_selectors_and_actions) {
#endif
}
void UnaryOpQDQRules(SelectorsAndActions& qdq_selectors_and_actions, bool is_int8_allowed = false) {
void UnaryOpQDQRules(SelectorsAndActions& qdq_selectors_and_actions) {
// 3 nodes. DQ, target, Q
// Replace with internal QLinear version of operator. Delete all original nodes.
const std::string action_name{"1DQ"};
std::unique_ptr<Action> action = std::make_unique<QDQ::UnaryReplaceWithQLinear>(kMSDomain);
#if !defined(ORT_MINIMAL_BUILD)
std::unique_ptr<NodeSelector> selector = std::make_unique<QDQ::UnarySelector>(is_int8_allowed);
std::unique_ptr<NodeSelector> selector = std::make_unique<QDQ::UnarySelector>();
qdq_selectors_and_actions.RegisterSelectorAndAction(action_name,
SelectorAndAction::OpVersionsMap{{"AveragePool", {}}},
SelectorAndAction::OpVersionsMap{{"AveragePool", {}},
{"LeakyRelu", {}}},
std::move(selector),
std::move(action));
#else
ORT_UNUSED_PARAMETER(is_int8_allowed);
qdq_selectors_and_actions.RegisterAction(action_name, std::move(action));
#endif
}
@ -148,7 +148,7 @@ SelectorsAndActions CreateSelectorsAndActions(bool is_int8_allowed) {
SelectorsAndActions qdq_selectors_and_actions;
DropQDQNodesRules(qdq_selectors_and_actions);
UnaryOpQDQRules(qdq_selectors_and_actions, is_int8_allowed);
UnaryOpQDQRules(qdq_selectors_and_actions);
BinaryOpQDQRules(qdq_selectors_and_actions);
VariadicOpQDQRules(qdq_selectors_and_actions);
ConvQDQRules(qdq_selectors_and_actions, is_int8_allowed);

11
onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_selectors.cc
View file

@ -114,8 +114,7 @@ bool UnarySelector::Check(const GraphViewer& graph_viewer, const Node& node,
int32_t dt_input = dq_nodes[0]->InputDefs()[0]->TypeAsProto()->tensor_type().elem_type();
int32_t dt_output = q_nodes[0]->OutputDefs()[0]->TypeAsProto()->tensor_type().elem_type();
return dt_input == dt_output &&
(int8_allowed_ || dt_input == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_UINT8);
return dt_input == dt_output;
}
bool BinarySelector::Check(const GraphViewer& graph_viewer,
@ -173,8 +172,8 @@ bool ConvSelector::Check(const GraphViewer& graph_viewer,
return false;
}
if(dt_input == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT8) {
if(!int8_allowed_ || dt_weight != dt_input) {
if (dt_input == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT8) {
if (!int8_allowed_ || dt_weight != dt_input) {
return false;
}
}
@ -202,8 +201,8 @@ bool MatMulSelector::Check(const GraphViewer& graph_viewer,
int32_t dt_input = dq_nodes[0]->InputDefs()[0]->TypeAsProto()->tensor_type().elem_type();
int32_t dt_weight = dq_nodes[1]->InputDefs()[0]->TypeAsProto()->tensor_type().elem_type();
if(dt_input == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT8) {
if(!int8_allowed_ || dt_weight != dt_input) {
if (dt_input == ONNX_NAMESPACE::TensorProto_DataType::TensorProto_DataType_INT8) {
if (!int8_allowed_ || dt_weight != dt_input) {
return false;
}
}

5
onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_selectors.h
View file

@ -63,15 +63,10 @@ class DropDQDNodesSelector : public BaseSelector {
// single input. default is to only support uint8.
class UnarySelector : public BaseSelector {
public:
UnarySelector(bool int8_allowed = false) : int8_allowed_{int8_allowed} {}
private:
bool Check(const GraphViewer& graph_viewer, const Node& node,
const std::vector<const Node*>& dq_nodes,
const std::vector<const Node*>& q_nodes) const override;
bool int8_allowed_;
};
// 2 DQ nodes providing input -> node -> Q

108
onnxruntime/test/optimizer/qdq_transformer_test.cc
View file

@ -99,10 +99,10 @@ void QDQTransformerConvTests() {
auto check_conv_graph = [&](InferenceSessionWrapper& session) {
auto op_to_count = CountOpsInGraph(session.GetGraph());
if (std::is_same<InputType, OutputType>::value &&
std::is_same<BiasType, int32_t>::value &&
(std::is_same<InputType, uint8_t>::value ||
QDQIsInt8Allowed() && std::is_same<WeightType, int8_t>::value)) {
if constexpr (std::is_same<InputType, OutputType>::value &&
std::is_same<BiasType, int32_t>::value &&
(std::is_same<InputType, uint8_t>::value ||
QDQIsInt8Allowed() && std::is_same<WeightType, int8_t>::value)) {
EXPECT_EQ(op_to_count["QLinearConv"], 1);
EXPECT_EQ(op_to_count["QuantizeLinear"], 1);
EXPECT_EQ(op_to_count["DequantizeLinear"], 1);
@ -158,7 +158,7 @@ TEST(QDQTransformerTests, Conv_S8X8U8) {
TEST(QDQTransformerTests, Conv_S8X8S8) {
// input not uint8_t and output not uint8_t
QDQTransformerConvTests<int8_t, uint8_t, int32_t, int8_t>();
QDQTransformerConvTests<int8_t, int8_t, int32_t, int8_t>();
QDQTransformerConvTests<int8_t, int8_t, int32_t, int8_t>();
}
TEST(QDQTransformerTests, ConvMaxPoolReshape_UInt8) {
@ -248,7 +248,7 @@ TEST(QDQTransformerTests, ConvMaxPoolReshape_Int8) {
builder.AddNode("Reshape", {dq_reshape_output, reshape_shape}, {reshape_output});
// add Q
if (QDQIsInt8Allowed()) {
if constexpr (QDQIsInt8Allowed()) {
builder.AddQuantizeLinearNode<int8_t>(reshape_output, .0039f, 7, output_arg);
} else {
builder.AddQuantizeLinearNode<uint8_t>(reshape_output, .0039f, 135, output_arg);
@ -301,8 +301,7 @@ void QDQTransformerAveragePoolTests() {
auto check_binary_op_graph = [&](InferenceSessionWrapper& session) {
auto op_to_count = CountOpsInGraph(session.GetGraph());
if (std::is_same<InputType, OutputType>::value &&
(QDQIsInt8Allowed() || std::is_same<InputType, uint8_t>::value)) {
if constexpr (std::is_same<InputType, OutputType>::value) {
EXPECT_EQ(op_to_count["com.microsoft.QLinearAveragePool"], 1);
EXPECT_EQ(op_to_count["AveragePool"], 0);
EXPECT_EQ(op_to_count["QuantizeLinear"], 1);
@ -512,9 +511,9 @@ void QDQTransformerMatMulTests(bool has_output_q) {
auto check_binary_op_graph = [&](InferenceSessionWrapper& session) {
auto op_to_count = CountOpsInGraph(session.GetGraph());
if (has_output_q) {
if (std::is_same<Input1Type, OutputType>::value &&
(std::is_same<Input1Type, uint8_t>::value ||
QDQIsInt8Allowed() && std::is_same<Input2Type, int8_t>::value)) {
if constexpr (std::is_same<Input1Type, OutputType>::value &&
(std::is_same<Input1Type, uint8_t>::value ||
QDQIsInt8Allowed() && std::is_same<Input2Type, int8_t>::value)) {
EXPECT_EQ(op_to_count["QLinearMatMul"], 1);
EXPECT_EQ(op_to_count["MatMul"], 0);
EXPECT_EQ(op_to_count["QuantizeLinear"], 2);
@ -526,8 +525,8 @@ void QDQTransformerMatMulTests(bool has_output_q) {
EXPECT_EQ(op_to_count["DequantizeLinear"], 3);
}
} else {
if (std::is_same<Input1Type, uint8_t>::value ||
(QDQIsInt8Allowed() && std::is_same<Input2Type, int8_t>::value)) {
if constexpr (std::is_same<Input1Type, uint8_t>::value ||
(QDQIsInt8Allowed() && std::is_same<Input2Type, int8_t>::value)) {
EXPECT_EQ(op_to_count["com.microsoft.MatMulIntegerToFloat"], 1);
EXPECT_EQ(op_to_count["MatMul"], 0);
EXPECT_EQ(op_to_count["QuantizeLinear"], 2);
@ -1101,7 +1100,7 @@ TEST(QDQTransformerTests, ConvAveragePoolReshape_Int8) {
// add Q
auto* q_output = builder.MakeIntermediate();
if (QDQIsInt8Allowed()) {
if constexpr (QDQIsInt8Allowed()) {
builder.AddQuantizeLinearNode<int8_t>(reshape_output, .0035f, 7, q_output);
builder.AddDequantizeLinearNode<int8_t>(q_output, .0035f, 7, output_arg);
} else {
@ -1165,14 +1164,13 @@ TEST(QDQTransformerTests, ConvAveragePoolReshape_Int8_Fail) {
// add Q + DQ
auto* q_output = builder.MakeIntermediate();
if(QDQIsInt8Allowed()){
if constexpr (QDQIsInt8Allowed()) {
builder.AddQuantizeLinearNode<int8_t>(reshape_output, .0035f, 7, q_output);
builder.AddDequantizeLinearNode<int8_t>(q_output, .0035f, 7, output_arg);
} else {
builder.AddQuantizeLinearNode<uint8_t>(reshape_output, .0035f, 135, q_output);
builder.AddDequantizeLinearNode<uint8_t>(q_output, .0035f, 135, output_arg);
}
};
auto check_mp_reshape_graph = [&](InferenceSessionWrapper& session) {
@ -1199,6 +1197,76 @@ TEST(QDQTransformerTests, ConvAveragePoolReshape_Int8_Fail) {
test_case({1, 22, 11, 13, 15}, {30, 22, 5, 3, 3});
}
template <typename InputType, typename OutputType>
void QDQTransformerLeakyReluTests() {
auto test_case = [&](const std::vector<int64_t>& input_shape) {
auto build_test_case = [&](ModelTestBuilder& builder) {
auto* input_arg = builder.MakeInput<float>(input_shape, -1.f, 1.f);
auto* output_arg = builder.MakeOutput();
// add QDQ + LeakyRelu
auto* dq_output = AddQDQNodePair<InputType>(builder, input_arg, .0035f, 7);
auto* leakyrelu_output = builder.MakeIntermediate();
Node& leakyrelu_node = builder.AddNode("LeakyRelu", {dq_output}, {leakyrelu_output});
leakyrelu_node.AddAttribute("alpha", 0.2f);
// add QDQ output
auto* q_output = builder.MakeIntermediate();
builder.AddQuantizeLinearNode<OutputType>(leakyrelu_output,
.0038f,
std::numeric_limits<OutputType>::max() / 2,
q_output);
builder.AddDequantizeLinearNode<OutputType>(q_output,
.0039f,
std::numeric_limits<OutputType>::max() / 2,
output_arg);
};
auto check_binary_op_graph = [&](InferenceSessionWrapper& session) {
auto op_to_count = CountOpsInGraph(session.GetGraph());
if constexpr (std::is_same<InputType, OutputType>::value) {
EXPECT_EQ(op_to_count["com.microsoft.QLinearLeakyRelu"], 1);
EXPECT_EQ(op_to_count["LeakyRelu"], 0);
EXPECT_EQ(op_to_count["QuantizeLinear"], 1);
EXPECT_EQ(op_to_count["DequantizeLinear"], 1);
} else {
EXPECT_EQ(op_to_count["com.microsoft.QLinearLeakyRelu"], 0);
EXPECT_EQ(op_to_count["LeakyRelu"], 1);
EXPECT_EQ(op_to_count["QuantizeLinear"], 2);
EXPECT_EQ(op_to_count["DequantizeLinear"], 2);
}
};
TransformerTester(build_test_case,
check_binary_op_graph,
TransformerLevel::Level1,
TransformerLevel::Level2,
12 /*opset_version*/,
0.01 /*per_sample_tolerance*/,
0.01 /*relative_per_sample_tolerance*/,
std::make_unique<QDQSelectorActionTransformer>());
};
test_case({1, 12, 37});
test_case({1, 23, 13, 13});
test_case({1, 22, 11, 13, 15});
}
TEST(QDQTransformerTests, LeakyRelu_S8S8) {
QDQTransformerLeakyReluTests<int8_t, int8_t>();
}
TEST(QDQTransformerTests, LeakyRelu_U8U8) {
QDQTransformerLeakyReluTests<uint8_t, uint8_t>();
}
TEST(QDQTransformerTests, LeakyRelu_S8U8) {
QDQTransformerLeakyReluTests<int8_t, uint8_t>();
}
TEST(QDQTransformerTests, LeakyRelu_U8S8) {
QDQTransformerLeakyReluTests<uint8_t, int8_t>();
}
TEST(QDQTransformerTests, ConvTranspose_QBackward) {
auto test_case = [&](const std::vector<int64_t>& input_shape, const std::vector<int64_t>& weights_shape, const std::vector<int64_t>& perms) {
auto build_test_case = [&](ModelTestBuilder& builder) {
@ -1220,7 +1288,7 @@ TEST(QDQTransformerTests, ConvTranspose_QBackward) {
// add Q
auto* q_output = builder.MakeIntermediate();
if (QDQIsInt8Allowed()) {
if constexpr (QDQIsInt8Allowed()) {
builder.AddQuantizeLinearNode<int8_t>(transpose_output, .0035f, 7, q_output);
builder.AddDequantizeLinearNode<int8_t>(q_output, .0035f, 7, output_arg);
} else {
@ -1280,7 +1348,7 @@ TEST(QDQTransformerTests, QBackward_MutilpleSteps) {
// add Q + DQ
auto* q_output = builder.MakeIntermediate();
if (QDQIsInt8Allowed()) {
if constexpr (QDQIsInt8Allowed()) {
builder.AddQuantizeLinearNode<int8_t>(transpose_output, .0035f, 7, q_output);
builder.AddDequantizeLinearNode<int8_t>(q_output, .0035f, 7, output_arg);
} else {
@ -1331,7 +1399,7 @@ TEST(QDQTransformerTests, ConvTranspose_DQForward) {
// add Q
auto* q_output = builder.MakeIntermediate();
if (QDQIsInt8Allowed()) {
if constexpr (QDQIsInt8Allowed()) {
builder.AddQuantizeLinearNode<int8_t>(conv_output, .0035f, 7, q_output);
builder.AddDequantizeLinearNode<int8_t>(q_output, .0035f, 7, output_arg);
} else {
@ -1391,7 +1459,7 @@ TEST(QDQTransformerTests, DQForward_MutilpleSteps) {
// add Q + DQ
auto* q_output = builder.MakeIntermediate();
if (QDQIsInt8Allowed()) {
if constexpr (QDQIsInt8Allowed()) {
builder.AddQuantizeLinearNode<int8_t>(reshape_output, .0035f, 7, q_output);
builder.AddDequantizeLinearNode<int8_t>(q_output, .0035f, 7, output_arg);
} else {