saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-30 20:18:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

add qdq support of (un)squeeze and GlobalAveragePool (#10721)

Browse source
This commit is contained in:
Yufeng Li 2022-03-02 10:58:35 -08:00 committed by GitHub
parent 9ad95bf068
commit 7ab0c607b4
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 177 additions and 31 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
onnxruntime/core/optimizer/qdq_transformer/qdq_propagation.cc
View file

@ -18,7 +18,9 @@ namespace {
bool CanNodePropagate(const Node& node) {
return graph_utils::IsSupportedOptypeVersionAndDomain(node, "MaxPool", {12}) ||
graph_utils::IsSupportedOptypeVersionAndDomain(node, "Reshape", {5, 13, 14}) ||
graph_utils::IsSupportedOptypeVersionAndDomain(node, "Transpose", {1, 13});
graph_utils::IsSupportedOptypeVersionAndDomain(node, "Transpose", {1, 13}) ||
graph_utils::IsSupportedOptypeVersionAndDomain(node, "Squeeze", {1, 11, 13}) ||
graph_utils::IsSupportedOptypeVersionAndDomain(node, "Unsqueeze", {1, 11, 13});
}
// convert this: src_node -> dst_node

7
onnxruntime/core/optimizer/qdq_transformer/selectors_actions/qdq_selector_action_transformer.cc
View file

@ -37,7 +37,9 @@ void DropQDQNodesRules(SelectorActionRegistry& qdq_selector_action_registry) {
{"Reshape", {}},
{"Transpose", {}},
{"MaxPool", {12}},
{"Resize", {}}},
{"Resize", {}},
{"Squeeze", {}},
{"Unsqueeze", {}}},
std::move(selector),
std::move(action));
#else
@ -78,7 +80,8 @@ void UnaryOpQDQRules(SelectorActionRegistry& qdq_selector_action_registry) {
std::unique_ptr<NodeSelector> selector = std::make_unique<QDQ::UnarySelector>();
qdq_selector_action_registry.RegisterSelectorAndAction(action_name,
{{"AveragePool", {}},
{"LeakyRelu", {}}},
{"LeakyRelu", {}},
{"GlobalAveragePool", {}}},
std::move(selector),
std::move(action));
#else

40
onnxruntime/core/optimizer/qdq_transformer/selectors_actions/shared/utils.cc
View file

@ -27,20 +27,34 @@ void Selectors::RegisterSelector(const OpVersionsAndSelector::OpVersionsMap& ops
}
/* static methods to return different operator's OpVersionMap */
static const OpVersionsAndSelector::OpVersionsMap GetMiscOpVersionsMap() { return {{"Gather", {}},
{"Reshape", {}},
{"Transpose", {}},
{"MaxPool", {12}},
{"Resize", {}}}; }
static const OpVersionsAndSelector::OpVersionsMap GetMiscOpVersionsMap() {
return {{"Gather", {}},
{"Reshape", {}},
{"Transpose", {}},
{"MaxPool", {12}},
{"Resize", {}},
{"Squeeze", {}},
{"Unsqueeze", {}}};
}
static const OpVersionsAndSelector::OpVersionsMap GetUnaryOpVersionsMap() { return {{"AveragePool", {}},
{"Softmax", {}},
{"LeakyRelu", {}}}; }
static const OpVersionsAndSelector::OpVersionsMap GetBinaryOpVersionsMap() { return {{"Add", {}},
{"Mul", {}}}; }
static const OpVersionsAndSelector::OpVersionsMap GetVariadicOpVersionsMap() { return {{"Concat", {}}}; }
static const OpVersionsAndSelector::OpVersionsMap GetConvOpVersionsMap() { return {{"Conv", {}}}; }
static const OpVersionsAndSelector::OpVersionsMap GetMatMulOpVersionsMap() { return {{"MatMul", {}}}; }
static const OpVersionsAndSelector::OpVersionsMap GetUnaryOpVersionsMap() {
return {{"AveragePool", {}},
{"Softmax", {}},
{"LeakyRelu", {}}};
}
static const OpVersionsAndSelector::OpVersionsMap GetBinaryOpVersionsMap() {
return {{"Add", {}},
{"Mul", {}}};
}
static const OpVersionsAndSelector::OpVersionsMap GetVariadicOpVersionsMap() {
return {{"Concat", {}}};
}
static const OpVersionsAndSelector::OpVersionsMap GetConvOpVersionsMap() {
return {{"Conv", {}}};
}
static const OpVersionsAndSelector::OpVersionsMap GetMatMulOpVersionsMap() {
return {{"MatMul", {}}};
}
/* Selector rules registration related */
void RegisterMiscSelectors(Selectors& qdq_selectors) {

6
onnxruntime/python/tools/quantization/operators/qdq_base_operator.py
View file

@ -14,9 +14,9 @@ class QDQOperatorBase:
node = self.node
if self.disable_qdq_for_node_output:
nodes_to_iterate = node.input
tensors_to_quantize = node.input
else:
nodes_to_iterate = itertools.chain(node.input, node.output)
tensors_to_quantize = itertools.chain(node.input, node.output)
for tensor_name in nodes_to_iterate:
for tensor_name in tensors_to_quantize:
self.quantizer.quantize_tensor(tensor_name)

4
onnxruntime/python/tools/quantization/quantize.py
View file

@ -61,8 +61,8 @@ def check_static_quant_arguments(quant_format : QuantFormat,
raise ValueError("ONNXRuntime quantization doesn't support data format:"
"activation_type=QuantType.QInt8, weight_type = QuantType.QUInt8")
if activation_type == QuantType.QInt8 or \
weight_type == QuantType.QInt8 or \
if activation_type == QuantType.QInt8 and \
weight_type == QuantType.QInt8 and \
quant_format != QuantFormat.QDQ: \
logging.warning("Please use QuantFormat.QDQ for activation type QInt8 and weight type QInt8. "
"Or it will lead to bad performance on x64.")

42
onnxruntime/test/optimizer/qdq_test_utils.h
View file

@ -26,6 +26,16 @@ AddQDQNodePair(ModelTestBuilder& builder, NodeArg* q_input, float scale, T zp =
return dq_output;
}
template <typename T>
typename std::enable_if<IsTypeQuantLinearCompatible<T>::value, NodeArg*>::type
AddQDQNodePairWithOutputAsGraphOutput(ModelTestBuilder& builder, NodeArg* q_input, float scale, T zp = T()) {
auto* q_output = builder.MakeIntermediate();
auto* dq_output = builder.MakeOutput();
builder.AddQuantizeLinearNode<T>(q_input, scale, zp, q_output);
builder.AddDequantizeLinearNode<T>(q_output, scale, zp, dq_output);
return dq_output;
}
template <typename InputType, typename WeightType, typename BiasType, typename OutputType>
GetQDQTestCaseFn BuildQDQConvTestCase(const std::vector<int64_t>& input_shape, const std::vector<int64_t>& weights_shape) {
return [input_shape, weights_shape](ModelTestBuilder& builder) {
@ -122,6 +132,36 @@ GetQDQTestCaseFn BuildQDQAveragePoolTestCase(const std::vector<int64_t>& input_s
};
}
template <typename InputType, typename OutputType>
GetQDQTestCaseFn BuildQDQGlobalAveragePoolTestCase(const std::vector<int64_t>& input_shape) {
return [input_shape](ModelTestBuilder& builder) {
float dq_scale = 0.0035f;
float pool_output_scale = 0.0038f;
float q_scale = 0.0039f;
InputType dq_zp = 7;
InputType pool_output_zp = std::numeric_limits<OutputType>::max() / 2;
InputType q_zp = std::numeric_limits<OutputType>::max() / 2;
auto* input_arg = builder.MakeInput<float>(input_shape, -1.f, 1.f);
auto* output_arg = builder.MakeOutput();
// add QDQ + GlobalAveragePool
auto* dq_output = AddQDQNodePair<InputType>(builder, input_arg, dq_scale, dq_zp);
auto* globalaveragepool_output = builder.MakeIntermediate();
builder.AddNode("GlobalAveragePool", {dq_output}, {globalaveragepool_output});
// add QDQ output
auto* q_output = builder.MakeIntermediate();
builder.AddQuantizeLinearNode<OutputType>(globalaveragepool_output,
pool_output_scale,
pool_output_zp,
q_output);
builder.AddDequantizeLinearNode<OutputType>(q_output,
q_scale,
q_zp,
output_arg);
};
}
GetQDQTestCaseFn BuildQDQResizeTestCase(const std::vector<int64_t>& input_shape,
const std::vector<int64_t>& sizes_data,
const std::string& mode = "nearest",
@ -249,4 +289,4 @@ GetQDQTestCaseFn BuildQDQConcatTestCase(const std::vector<std::vector<int64_t>>&
bool has_output_int8 = false);
} // namespace test
} // namespace onnxruntime
} // namespace onnxruntime

105
onnxruntime/test/optimizer/qdq_transformer_test.cc
View file

@ -276,6 +276,55 @@ TEST(QDQTransformerTests, AveragePool_U8S8) {
QDQTransformerAveragePoolTests<uint8_t, int8_t>();
}
template <typename InputType, typename OutputType>
void QDQTransformerGlobalAveragePoolTests() {
auto test_case = [&](const std::vector<int64_t>& input_shape) {
auto check_graph = [&](InferenceSessionWrapper& session) {
auto op_to_count = CountOpsInGraph(session.GetGraph());
if constexpr (std::is_same<InputType, OutputType>::value) {
EXPECT_EQ(op_to_count["com.microsoft.QLinearGlobalAveragePool"], 1);
EXPECT_EQ(op_to_count["GlobalAveragePool"], 0);
EXPECT_EQ(op_to_count["QuantizeLinear"], 1);
EXPECT_EQ(op_to_count["DequantizeLinear"], 1);
} else {
EXPECT_EQ(op_to_count["com.microsoft.QLinearGlobalAveragePool"], 0);
EXPECT_EQ(op_to_count["GlobalAveragePool"], 1);
EXPECT_EQ(op_to_count["QuantizeLinear"], 2);
EXPECT_EQ(op_to_count["DequantizeLinear"], 2);
}
};
TransformerTester(BuildQDQGlobalAveragePoolTestCase<InputType, OutputType>(input_shape),
check_graph,
TransformerLevel::Level1,
TransformerLevel::Level2,
12 /*opset_version*/,
0.01 /*per_sample_tolerance*/,
0.01 /*relative_per_sample_tolerance*/,
std::make_unique<QDQSelectorActionTransformer>());
};
test_case({1, 12, 37});
test_case({1, 23, 13, 13});
test_case({1, 22, 11, 13, 15});
}
TEST(QDQTransformerTests, GlobalAveragePool_S8S8) {
QDQTransformerAveragePoolTests<int8_t, int8_t>();
}
TEST(QDQTransformerTests, GlobalAveragePool_U8U8) {
QDQTransformerAveragePoolTests<uint8_t, uint8_t>();
}
TEST(QDQTransformerTests, GlobalAveragePool_S8U8) {
QDQTransformerAveragePoolTests<int8_t, uint8_t>();
}
TEST(QDQTransformerTests, GlobalAveragePool_U8S8) {
QDQTransformerAveragePoolTests<uint8_t, int8_t>();
}
template <typename Input1Type, typename Input2Type, typename OutputType>
void QDQTransformerBinaryOpTests(const std::string& op_type) {
auto test_case = [&](const std::vector<int64_t>& input_shape) {
@ -798,7 +847,7 @@ TEST(QDQTransformerTests, Resize_No_Fusion) {
test_case({1, 8, 64, 64}, {4}, {1, 4, 128, 128}, 1);
}
TEST(QDQTransformerTests, ResizeReshape) {
TEST(QDQTransformerTests, ResizeReshapeSqueezeUnsqueeze) {
auto test_case = [&](const std::vector<int64_t>& input_shape,
const std::vector<int64_t>& sizes_shape) {
auto build_test_case = [&](ModelTestBuilder& builder) {
@ -808,7 +857,6 @@ TEST(QDQTransformerTests, ResizeReshape) {
auto* roi = builder.MakeInitializer<float>({0}, {});
auto* scales = builder.MakeInitializer<float>({0}, {});
auto* sizes = builder.MakeInitializer<int64_t>(sizes_shape, {1, 2, 52, 82});
auto* output_arg = builder.MakeOutput();
// add QDQ + Resize
auto* qdq_input = AddQDQNodePair<uint8_t>(builder, input_arg, .003f, 1);
@ -818,7 +866,23 @@ TEST(QDQTransformerTests, ResizeReshape) {
// add QDQ + Reshape
auto* qdq_resize_output = AddQDQNodePair<uint8_t>(builder, resize_output, .003f, 1);
auto* reshape_shape = builder.Make1DInitializer<int64_t>({1, 2, 52, 82});
builder.AddNode("Reshape", {qdq_resize_output, reshape_shape}, {output_arg});
auto* reshape_output = builder.MakeIntermediate();
builder.AddNode("Reshape", {qdq_resize_output, reshape_shape}, {reshape_output});
// add QDQ + Squeeze
auto* qdq_squeeze_output = AddQDQNodePair<uint8_t>(builder, reshape_output, .003f, 1);
auto* squeeze_axes = builder.Make1DInitializer<int64_t>({0});
auto* squeeze_output = builder.MakeIntermediate();
builder.AddNode("Squeeze", {qdq_squeeze_output, squeeze_axes}, {squeeze_output});
// add QDQ + Unsqueeze
auto* qdq_unsqueeze_output = AddQDQNodePair<uint8_t>(builder, squeeze_output, .003f, 1);
auto* unsqueeze_axes = builder.Make1DInitializer<int64_t>({0});
auto* unsqueeze_output = builder.MakeIntermediate();
builder.AddNode("Unsqueeze", {qdq_unsqueeze_output, unsqueeze_axes}, {unsqueeze_output});
// add QDQ
AddQDQNodePairWithOutputAsGraphOutput<uint8_t>(builder, unsqueeze_output, .003f, 1);
};
auto check_graph = [&](InferenceSessionWrapper& session) {
@ -831,7 +895,8 @@ TEST(QDQTransformerTests, ResizeReshape) {
TransformerTester(build_test_case, check_graph,
TransformerLevel::Level1,
TransformerLevel::Level2);
TransformerLevel::Level2,
13 /*opset_version*/);
};
test_case({1, 2, 26, 42}, {4});
@ -1404,13 +1469,23 @@ TEST(QDQTransformerTests, QBackward_MutilpleSteps) {
Node& transpose_node = builder.AddNode("Transpose", {reshape_output}, {transpose_output});
transpose_node.AddAttribute("perm", std::vector<int64_t>({1, 0}));
// add Unsqueeze
auto* unsqueeze_axes = builder.Make1DInitializer<int64_t>({0});
auto* unsqueeze_output = builder.MakeIntermediate();
builder.AddNode("Unsqueeze", {transpose_output, unsqueeze_axes}, {unsqueeze_output});
// add Squeeze
auto* squeeze_axes = builder.Make1DInitializer<int64_t>({0});
auto* squeeze_output = builder.MakeIntermediate();
builder.AddNode("Squeeze", {unsqueeze_output, squeeze_axes}, {squeeze_output});
// add Q + DQ
auto* q_output = builder.MakeIntermediate();
if constexpr (QDQIsInt8Allowed()) {
builder.AddQuantizeLinearNode<int8_t>(transpose_output, .0035f, 7, q_output);
builder.AddQuantizeLinearNode<int8_t>(squeeze_output, .0035f, 7, q_output);
builder.AddDequantizeLinearNode<int8_t>(q_output, .0035f, 7, output_arg);
} else {
builder.AddQuantizeLinearNode<uint8_t>(transpose_output, .0035f, 135, q_output);
builder.AddQuantizeLinearNode<uint8_t>(squeeze_output, .0035f, 135, q_output);
builder.AddDequantizeLinearNode<uint8_t>(q_output, .0035f, 135, output_arg);
}
};
@ -1428,7 +1503,8 @@ TEST(QDQTransformerTests, QBackward_MutilpleSteps) {
TransformerTester(build_test_case,
check_graph,
TransformerLevel::Level1,
TransformerLevel::Level2);
TransformerLevel::Level2,
13 /*opset_version*/);
};
test_case({1, 23, 13, 13}, {30, 23, 3, 3});
@ -1504,11 +1580,21 @@ TEST(QDQTransformerTests, DQForward_MutilpleSteps) {
std::vector<int64_t> kernel_shape(weights_shape.size() - 2, 3);
pool_node.AddAttribute("kernel_shape", kernel_shape);
// add Unsqueeze
auto* unsqueeze_axes = builder.Make1DInitializer<int64_t>({0});
auto* unsqueeze_output = builder.MakeIntermediate();
builder.AddNode("Unsqueeze", {maxpool_output, unsqueeze_axes}, {unsqueeze_output});
// add Squeeze
auto* squeeze_axes = builder.Make1DInitializer<int64_t>({0});
auto* squeeze_output = builder.MakeIntermediate();
builder.AddNode("Squeeze", {unsqueeze_output, squeeze_axes}, {squeeze_output});
// add Conv
auto* dq_w_output = builder.MakeIntermediate();
auto* conv_output = builder.MakeIntermediate();
builder.AddDequantizeLinearNode<int8_t>(weight, .003f, -10, dq_w_output);
builder.AddConvNode(maxpool_output, dq_w_output, conv_output);
builder.AddConvNode(squeeze_output, dq_w_output, conv_output);
// Reshape
auto* reshape_shape = builder.Make1DInitializer<int64_t>({-1, 0});
@ -1539,7 +1625,8 @@ TEST(QDQTransformerTests, DQForward_MutilpleSteps) {
TransformerTester(build_test_case,
check_graph,
TransformerLevel::Level1,
TransformerLevel::Level2);
TransformerLevel::Level2,
13 /*opset_version*/);
};
test_case({1, 13, 13, 23}, {30, 23, 3, 3}, {0, 3, 1, 2});