saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-26 19:52:38 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Add optional optimizer to remove leftover Q->DQ pairs after all other QDQ processing has completed (#10659)

Browse source 
Add an optimizer that can remove leftover Q->DQ pairs. Depending on the model this may help with performance and/or improve accuracy. Optional as it could make things worse so user needs to be aware of this and test what works best for their scenario. Enable with SessionOptions config param `session.enable_quant_qdq_cleanup`
This commit is contained in:
Scott McKay 2022-03-01 08:05:02 +10:00 committed by GitHub
parent e788cc2a23
commit 1f6d8248da
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
7 changed files with 287 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

9
include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h
View file

@ -47,6 +47,15 @@ static const char* const kOrtSessionOptionsConfigSetDenormalAsZero = "session.se
// Its default value is "0"
static const char* const kOrtSessionOptionsDisableQuantQDQ = "session.disable_quant_qdq";
// If set to "1", enables the removal of QuantizeLinear/DequantizeLinear node pairs once all QDQ handling has been
// completed. e.g. If after all QDQ handling has completed and we have -> FloatOp -> Q -> DQ -> FloatOp -> the
// Q -> DQ could potentially be removed. This will provide a performance benefit by avoiding going from float to
// 8-bit and back to float, but could impact accuracy. The impact on accuracy will be model specific and depend on
// other factors like whether the model was created using Quantization Aware Training or Post Training Quantization.
// As such, it's best to test to determine if enabling this works well for your scenario.
// The default value is "0"
static const char* const kOrtSessionOptionsEnableQuantQDQCleanup = "session.enable_quant_qdq_cleanup";
// Enable or disable gelu approximation in graph optimization. "0": disable; "1": enable. The default is "0".
// GeluApproximation has side effects which may change the inference results. It is disabled by default due to this.
static const char* const kOrtSessionOptionsEnableGeluApproximation = "optimization.enable_gelu_approximation";

6
onnxruntime/core/optimizer/graph_transformer_utils.cc
View file

@ -45,6 +45,7 @@
#include "core/optimizer/noop_elimination.h"
#include "core/optimizer/not_where_fusion.h"
#include "core/optimizer/qdq_transformer/clip_quantizelinear.h"
#include "core/optimizer/qdq_transformer/qdq_final_cleanup.h"
#include "core/optimizer/qdq_transformer/qdq_propagation.h"
#include "core/optimizer/qdq_transformer/qdq_s8_to_u8.h"
#include "core/optimizer/qdq_transformer/relu_quantizelinear.h"
@ -158,6 +159,8 @@ InlinedVector<std::unique_ptr<GraphTransformer>> GenerateTransformers(
InlinedVector<std::unique_ptr<GraphTransformer>> transformers;
const bool disable_quant_qdq =
session_options.config_options.GetConfigOrDefault(kOrtSessionOptionsDisableQuantQDQ, "0") == "1";
const bool enable_quant_qdq_cleanup =
session_options.config_options.GetConfigOrDefault(kOrtSessionOptionsEnableQuantQDQCleanup, "0") == "1";
#ifndef DISABLE_CONTRIB_OPS
const bool enable_gelu_approximation =
session_options.config_options.GetConfigOrDefault(kOrtSessionOptionsEnableGeluApproximation, "0") == "1";
@ -241,6 +244,9 @@ InlinedVector<std::unique_ptr<GraphTransformer>> GenerateTransformers(
}
#endif
if (enable_quant_qdq_cleanup) {
transformers.emplace_back(std::make_unique<QDQFinalCleanupTransformer>());
}
} break;
case TransformerLevel::Level3: {

110
onnxruntime/core/optimizer/qdq_transformer/qdq_final_cleanup.cc Normal file
View file

@ -0,0 +1,110 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include "core/optimizer/qdq_transformer/qdq_final_cleanup.h"
#include "core/graph/graph_utils.h"
#include "core/optimizer/initializer.h"
#include "core/optimizer/qdq_transformer/qdq_util.h"
#include "core/optimizer/selectors_actions/actions.h"
#include "core/optimizer/utils.h"
namespace onnxruntime {
Status QDQFinalCleanupTransformer::ApplyImpl(Graph& graph, bool& modified, int graph_level,
const logging::Logger& logger) const {
GraphViewer graph_viewer(graph);
const auto& node_topology_list = graph_viewer.GetNodesInTopologicalOrder();
for (auto node_index : node_topology_list) {
auto* node_ptr = graph.GetNode(node_index);
if (node_ptr == nullptr)
continue; // node was removed as part of an earlier optimization
Node& q_node = *node_ptr;
ORT_RETURN_IF_ERROR(Recurse(q_node, modified, graph_level, logger));
if (!QDQ::MatchQNode(q_node) ||
!graph_utils::IsSupportedProvider(q_node, GetCompatibleExecutionProviders()) ||
!optimizer_utils::CheckOutputEdges(graph, q_node, 1)) {
continue;
}
Node& dq_node = *graph.GetNode(q_node.OutputNodesBegin()->Index());
if (!QDQ::MatchDQNode(dq_node) ||
!graph_utils::IsSupportedProvider(dq_node, GetCompatibleExecutionProviders())) {
continue;
}
// we have Q -> DQ pair
// we support a DQ that produces a graph output if it has no output edges, or a DQ node with one output edge.
bool is_graph_output = graph.NodeProducesGraphOutput(dq_node);
auto edges_count = dq_node.GetOutputEdgesCount();
if ((is_graph_output && edges_count != 0) ||
(!is_graph_output && edges_count != 1)) {
continue;
}
// src node -> Q -> DQ -> downstream node or graph output
NodeIndex src_node_idx = 0;
int src_arg_idx = -1;
NodeIndex downstream_node_idx = 0;
int downstream_arg_idx = -1;
// input could be node or initializer/graph input so need to handle both.
// if it's a node we need to replace the edge, so need info on which output idx it was attached to on the src node.
const Node::EdgeEnd* input_edge = nullptr;
if (q_node.GetInputEdgesCount() == 1) {
input_edge = &*q_node.InputEdgesBegin();
src_node_idx = input_edge->GetNode().Index();
src_arg_idx = input_edge->GetSrcArgIndex();
// remove edge from src to Q. dest arg idx is 0 as Q only has one input
graph.RemoveEdge(src_node_idx, q_node.Index(), src_arg_idx, 0);
}
// remove edge between pair we're removing
// both DQ and Q are single input single output so src idx and dest idx must be 0
graph.RemoveEdge(q_node.Index(), dq_node.Index(), 0, 0);
if (!is_graph_output) {
// remove edge to downstream node
const Node::EdgeEnd& output_edge = *dq_node.OutputEdgesBegin();
downstream_node_idx = output_edge.GetNode().Index();
downstream_arg_idx = output_edge.GetDstArgIndex();
// source arg idx is 0 as Q/DQ only has one output
graph.RemoveEdge(dq_node.Index(), downstream_node_idx, 0, downstream_arg_idx);
// replace input on downstream node
Node& downstream_node = *graph.GetNode(downstream_node_idx);
downstream_node.MutableInputDefs()[downstream_arg_idx] = q_node.MutableInputDefs()[0];
// create edge between src_node (if available) and downstream node
if (input_edge) {
graph.AddEdge(src_node_idx, downstream_node_idx, src_arg_idx, downstream_arg_idx);
}
} else {
NodeArg* graph_output_nodearg = dq_node.MutableOutputDefs()[0];
if (src_arg_idx >= 0) {
// update the src node to produce the graph output that was being provided by the DQ node
Node& src_node = *graph.GetNode(src_node_idx);
src_node.MutableOutputDefs()[src_arg_idx] = graph_output_nodearg;
} else {
// add Identity node to connect the graph input or initializer to the graph output.
Node& id_node = graph.AddNode(graph.GenerateNodeName("QDQFinalCleanupTransformer"),
"Identity", "", {q_node.MutableInputDefs()[0]}, {graph_output_nodearg});
id_node.SetExecutionProviderType(dq_node.GetExecutionProviderType());
}
}
graph.RemoveNode(q_node.Index());
graph.RemoveNode(dq_node.Index());
modified = true;
}
return Status::OK();
}
} // namespace onnxruntime

41
onnxruntime/core/optimizer/qdq_transformer/qdq_final_cleanup.h Normal file
View file

@ -0,0 +1,41 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#pragma once
#include "core/common/common.h"
#include "core/optimizer/graph_transformer.h"
namespace onnxruntime {
/**
@Class QDQFinalCleanupTransformer
Remove any remaining back-to-back QuantizeLinear and DequantizeLinear pairs.
This is the final cleanup where no quantized operator was available and we're going to run the operators
using fp32.
e.g. if we have Op -> Q -> DQ -> Op2 and no more QDQ processing to run, the Q -> DQ can potentially be removed.
The impact on performance and accuracy of removing the pair will depend on the model.
If it was quantized with Quantization Aware Training (QAT) it may be better to pay the performance cost of keeping
the pair as the model was trained with a round-trip from float -> 8-bit -> float.
If the model was quantized with Post Training Quantization (PTQ) it is most likely better to remove the pair as the
loss of precision from the round trip of float -> 8-bit -> float was not present when the model was trained.
As we have no knowledge of how the model was quantized we require the user to specify an option to enable this
transformer.
*/
class QDQFinalCleanupTransformer : public GraphTransformer {
public:
QDQFinalCleanupTransformer(const InlinedHashSet<std::string_view>& compatible_execution_providers = {}) noexcept
: GraphTransformer("QDQFinalCleanupTransformer", compatible_execution_providers) {}
private:
Status ApplyImpl(Graph& graph, bool& modified, int graph_level, const logging::Logger& logger) const override;
};
} // namespace onnxruntime

6
onnxruntime/test/optimizer/graph_transform_test_builder.cc
View file

@ -24,7 +24,8 @@ void TransformerTester(const std::function<void(ModelTestBuilder& helper)>& buil
int opset_version,
double per_sample_tolerance,
double relative_per_sample_tolerance,
std::unique_ptr<GraphTransformer> transformer) {
std::unique_ptr<GraphTransformer> transformer,
const std::function<void(SessionOptions&)>* add_session_options) {
// Build the model for this test.
std::unordered_map<std::string, int> domain_to_version;
domain_to_version[kOnnxDomain] = opset_version;
@ -49,6 +50,9 @@ void TransformerTester(const std::function<void(ModelTestBuilder& helper)>& buil
session_options.optimized_model_filepath =
ToPathString("model" + std::to_string(static_cast<int>(level)) + ".onnx");
#endif
if (add_session_options) {
(*add_session_options)(session_options);
}
InferenceSessionWrapper session{session_options, GetEnvironment()};
ASSERT_STATUS_OK(session.Load(model_data.data(), static_cast<int>(model_data.size())));
if (transformer) {

3
onnxruntime/test/optimizer/graph_transform_test_builder.h
View file

@ -281,7 +281,8 @@ void TransformerTester(const std::function<void(ModelTestBuilder& helper)>& buil
int opset_version = 12,
double per_sample_tolerance = 0.0,
double relative_per_sample_tolerance = 0.0,
std::unique_ptr<GraphTransformer> transformer = nullptr);
std::unique_ptr<GraphTransformer> transformer = nullptr,
const std::function<void(SessionOptions&)>* add_session_options = nullptr);
} // namespace test
} // namespace onnxruntime

118
onnxruntime/test/optimizer/qdq_transformer_test.cc
View file

@ -5,10 +5,12 @@
#include "core/graph/model.h"
#include "core/graph/onnx_protobuf.h"
#include "core/mlas/inc/mlas.h"
#include "core/optimizer/qdq_transformer/qdq_final_cleanup.h"
#include "core/optimizer/qdq_transformer/selectors_actions/qdq_selectors.h"
#include "core/optimizer/qdq_transformer/selectors_actions/qdq_selector_action_transformer.h"
#include "core/optimizer/qdq_transformer/selectors_actions/shared/utils.h"
#include "core/providers/partitioning_utils.h"
#include "core/session/onnxruntime_session_options_config_keys.h"
#include "core/session/environment.h"
#include "core/session/inference_session.h"
@ -1757,9 +1759,9 @@ TEST(QDQTransformerTests, QDQPropagation_QBackward) {
// TODO re-enable tests after updating ONNX to get QuantizeLinear shape inference fix
// https://github.com/onnx/onnx/pull/3806
//test_case({1, 13, 13, 23}, 4, {0, 3, 1, 2}, false, false);
// test_case({1, 13, 13, 23}, 4, {0, 3, 1, 2}, false, false);
test_case({1, 13, 13, 23}, 4, {0, 3, 1, 2}, false, true);
//test_case({1, 13, 13, 23}, 4, {0, 3, 1, 2}, true, false);
// test_case({1, 13, 13, 23}, 4, {0, 3, 1, 2}, true, false);
test_case({1, 13, 13, 23}, 4, {0, 3, 1, 2}, true, true);
}
@ -1834,9 +1836,9 @@ TEST(QDQTransformerTests, QDQPropagation_DQForward) {
// TODO re-enable tests after updating ONNX to get QuantizeLinear shape inference fix
// https://github.com/onnx/onnx/pull/3806
//test_case({1, 13, 13, 23}, 4, {0, 3, 1, 2}, false, false);
// test_case({1, 13, 13, 23}, 4, {0, 3, 1, 2}, false, false);
test_case({1, 13, 13, 23}, 4, {0, 3, 1, 2}, false, true);
//test_case({1, 13, 13, 23}, 4, {0, 3, 1, 2}, true, false);
// test_case({1, 13, 13, 23}, 4, {0, 3, 1, 2}, true, false);
test_case({1, 13, 13, 23}, 4, {0, 3, 1, 2}, true, true);
}
@ -2140,5 +2142,113 @@ TEST(QDQTransformerTests, QDQ_Selector_Test) {
}
}
// test removal of Q->DQ pairs by QDQFinalCleanupTransformer
TEST(QDQTransformerTests, QDQFinalCleanupTransformer_Basic) {
auto test_case = [&](const std::vector<std::vector<int64_t>>& input_shapes,
bool block_removal_of_last_dq = false,
bool block_removal_of_first_dq = false) {
// create model with float input to multiple -> Q -> DQ -> Concat -> Q -> DQ -> output
// If we enable cleanup and don't run the QDQ transformer we should drop all the Q->DQ pairs
auto build_test_case = [&](ModelTestBuilder& builder) {
auto input_count = input_shapes.size();
std::vector<NodeArg*> input_args;
std::vector<NodeArg*> q_input_args;
for (size_t i = 0; i < input_count; i++) {
input_args.push_back(builder.MakeInput<float>(input_shapes[i], -1.f, 1.f));
q_input_args.push_back(AddQDQNodePair<uint8_t>(builder, input_args.back(), 0.05f, 128));
if (i == 0 && block_removal_of_first_dq) {
// add another edge to the DQ node
auto* output = builder.MakeOutput();
builder.AddNode("Identity", {q_input_args.back()}, {output});
}
}
auto* concat_output = builder.MakeIntermediate();
Node& concat_node = builder.AddNode("Concat", q_input_args, {concat_output});
concat_node.AddAttribute("axis", int64_t(1));
auto* q_concat_output = builder.MakeIntermediate();
builder.AddQuantizeLinearNode<uint8_t>(concat_output, 0.05f, 128, q_concat_output);
auto* output_arg = builder.MakeOutput();
Node& dq_node = builder.AddDequantizeLinearNode<uint8_t>(q_concat_output, 0.05f, 128, output_arg);
if (block_removal_of_last_dq) {
// add another edge to the DQ node
auto* output = builder.MakeOutput();
builder.AddNode("Identity", {dq_node.MutableOutputDefs()[0]}, {output});
}
};
// if we block removal of the DQ node the Q node in the pair will not be removed either
int expected_qdq_count = 0 + (block_removal_of_first_dq ? 1 : 0) + (block_removal_of_last_dq ? 1 : 0);
auto check_graph = [expected_qdq_count](InferenceSessionWrapper& session) {
auto op_to_count = CountOpsInGraph(session.GetGraph());
EXPECT_EQ(op_to_count["QuantizeLinear"], expected_qdq_count);
EXPECT_EQ(op_to_count["DequantizeLinear"], expected_qdq_count);
EXPECT_EQ(op_to_count["Concat"], 1);
};
std::function<void(SessionOptions&)> func = [](SessionOptions& so) {
ASSERT_STATUS_OK(so.config_options.AddConfigEntry(kOrtSessionOptionsEnableQuantQDQCleanup, "1"));
};
// we increase the tolerance as removing the QDQ nodes means there's no round-trip to 8-bit and back
// essentially rounding the input values.
TransformerTester(build_test_case,
check_graph,
TransformerLevel::Level1,
TransformerLevel::Level2,
12 /*opset_version*/,
0.025f /*per_sample_tolerance*/,
0.01f /*relative_per_sample_tolerance*/,
std::make_unique<QDQFinalCleanupTransformer>(),
&func);
};
test_case({{1, 2, 4}, {1, 3, 4}});
test_case({{1, 2, 4}, {1, 3, 4}}, true); // block removal of first dq
test_case({{1, 2, 4}, {1, 3, 4}}, false, true); // block removal of last dq
test_case({{1, 2, 4}, {1, 3, 4}}, true, true); // block removal of first and last dq
}
// test removal when we have graph input -> Q -> DQ -> graph output
TEST(QDQTransformerTests, QDQFinalCleanupTransformer_GraphInputToOutput) {
// create model with float input to -> Q -> DQ -> output
auto build_test_case = [&](ModelTestBuilder& builder) {
NodeArg* input = builder.MakeInput<float>({1, 2, 4}, -1.f, 1.f);
NodeArg* q_output = builder.MakeIntermediate();
builder.AddQuantizeLinearNode<uint8_t>(input, 0.05f, 128, q_output);
auto* output_arg = builder.MakeOutput();
builder.AddDequantizeLinearNode<uint8_t>(q_output, 0.05f, 128, output_arg);
};
// with the Q->DQ being dropped we should have inserted an Identity node
// to connect the graph input to the graph output
auto check_graph = [](InferenceSessionWrapper& session) {
auto op_to_count = CountOpsInGraph(session.GetGraph());
EXPECT_EQ(op_to_count["QuantizeLinear"], 0);
EXPECT_EQ(op_to_count["DequantizeLinear"], 0);
EXPECT_EQ(op_to_count["Identity"], 1);
};
std::function<void(SessionOptions&)> func = [](SessionOptions& so) {
ASSERT_STATUS_OK(so.config_options.AddConfigEntry(kOrtSessionOptionsEnableQuantQDQCleanup, "1"));
};
// we increase the tolerance as removing the QDQ nodes means there's no round-trip to 8-bit and back
// essentially rounding the input values.
TransformerTester(build_test_case,
check_graph,
TransformerLevel::Level1,
TransformerLevel::Level2,
12 /*opset_version*/,
0.025f /*per_sample_tolerance*/,
0.01f /*relative_per_sample_tolerance*/,
std::make_unique<QDQFinalCleanupTransformer>(),
&func);
}
} // namespace test
} // namespace onnxruntime