From 1f6d8248dadef0559e34730a3619f13ec4d30e6e Mon Sep 17 00:00:00 2001 From: Scott McKay Date: Tue, 1 Mar 2022 08:05:02 +1000 Subject: [PATCH] Add optional optimizer to remove leftover Q->DQ pairs after all other QDQ processing has completed (#10659) Add an optimizer that can remove leftover Q->DQ pairs. Depending on the model this may help with performance and/or improve accuracy. Optional as it could make things worse so user needs to be aware of this and test what works best for their scenario. Enable with SessionOptions config param `session.enable_quant_qdq_cleanup` --- .../onnxruntime_session_options_config_keys.h | 9 ++ .../core/optimizer/graph_transformer_utils.cc | 6 + .../qdq_transformer/qdq_final_cleanup.cc | 110 ++++++++++++++++ .../qdq_transformer/qdq_final_cleanup.h | 41 ++++++ .../optimizer/graph_transform_test_builder.cc | 6 +- .../optimizer/graph_transform_test_builder.h | 3 +- .../test/optimizer/qdq_transformer_test.cc | 118 +++++++++++++++++- 7 files changed, 287 insertions(+), 6 deletions(-) create mode 100644 onnxruntime/core/optimizer/qdq_transformer/qdq_final_cleanup.cc create mode 100644 onnxruntime/core/optimizer/qdq_transformer/qdq_final_cleanup.h diff --git a/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h b/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h index 55e328b1bb..82fe74cf76 100644 --- a/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h +++ b/include/onnxruntime/core/session/onnxruntime_session_options_config_keys.h @@ -47,6 +47,15 @@ static const char* const kOrtSessionOptionsConfigSetDenormalAsZero = "session.se // Its default value is "0" static const char* const kOrtSessionOptionsDisableQuantQDQ = "session.disable_quant_qdq"; +// If set to "1", enables the removal of QuantizeLinear/DequantizeLinear node pairs once all QDQ handling has been +// completed. e.g. If after all QDQ handling has completed and we have -> FloatOp -> Q -> DQ -> FloatOp -> the +// Q -> DQ could potentially be removed. This will provide a performance benefit by avoiding going from float to +// 8-bit and back to float, but could impact accuracy. The impact on accuracy will be model specific and depend on +// other factors like whether the model was created using Quantization Aware Training or Post Training Quantization. +// As such, it's best to test to determine if enabling this works well for your scenario. +// The default value is "0" +static const char* const kOrtSessionOptionsEnableQuantQDQCleanup = "session.enable_quant_qdq_cleanup"; + // Enable or disable gelu approximation in graph optimization. "0": disable; "1": enable. The default is "0". // GeluApproximation has side effects which may change the inference results. It is disabled by default due to this. static const char* const kOrtSessionOptionsEnableGeluApproximation = "optimization.enable_gelu_approximation"; diff --git a/onnxruntime/core/optimizer/graph_transformer_utils.cc b/onnxruntime/core/optimizer/graph_transformer_utils.cc index 276c0c629a..15ea8b4b64 100644 --- a/onnxruntime/core/optimizer/graph_transformer_utils.cc +++ b/onnxruntime/core/optimizer/graph_transformer_utils.cc @@ -45,6 +45,7 @@ #include "core/optimizer/noop_elimination.h" #include "core/optimizer/not_where_fusion.h" #include "core/optimizer/qdq_transformer/clip_quantizelinear.h" +#include "core/optimizer/qdq_transformer/qdq_final_cleanup.h" #include "core/optimizer/qdq_transformer/qdq_propagation.h" #include "core/optimizer/qdq_transformer/qdq_s8_to_u8.h" #include "core/optimizer/qdq_transformer/relu_quantizelinear.h" @@ -158,6 +159,8 @@ InlinedVector> GenerateTransformers( InlinedVector> transformers; const bool disable_quant_qdq = session_options.config_options.GetConfigOrDefault(kOrtSessionOptionsDisableQuantQDQ, "0") == "1"; + const bool enable_quant_qdq_cleanup = + session_options.config_options.GetConfigOrDefault(kOrtSessionOptionsEnableQuantQDQCleanup, "0") == "1"; #ifndef DISABLE_CONTRIB_OPS const bool enable_gelu_approximation = session_options.config_options.GetConfigOrDefault(kOrtSessionOptionsEnableGeluApproximation, "0") == "1"; @@ -241,6 +244,9 @@ InlinedVector> GenerateTransformers( } #endif + if (enable_quant_qdq_cleanup) { + transformers.emplace_back(std::make_unique()); + } } break; case TransformerLevel::Level3: { diff --git a/onnxruntime/core/optimizer/qdq_transformer/qdq_final_cleanup.cc b/onnxruntime/core/optimizer/qdq_transformer/qdq_final_cleanup.cc new file mode 100644 index 0000000000..4384de8eb4 --- /dev/null +++ b/onnxruntime/core/optimizer/qdq_transformer/qdq_final_cleanup.cc @@ -0,0 +1,110 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "core/optimizer/qdq_transformer/qdq_final_cleanup.h" + +#include "core/graph/graph_utils.h" +#include "core/optimizer/initializer.h" +#include "core/optimizer/qdq_transformer/qdq_util.h" +#include "core/optimizer/selectors_actions/actions.h" +#include "core/optimizer/utils.h" + +namespace onnxruntime { +Status QDQFinalCleanupTransformer::ApplyImpl(Graph& graph, bool& modified, int graph_level, + const logging::Logger& logger) const { + GraphViewer graph_viewer(graph); + const auto& node_topology_list = graph_viewer.GetNodesInTopologicalOrder(); + + for (auto node_index : node_topology_list) { + auto* node_ptr = graph.GetNode(node_index); + if (node_ptr == nullptr) + continue; // node was removed as part of an earlier optimization + + Node& q_node = *node_ptr; + ORT_RETURN_IF_ERROR(Recurse(q_node, modified, graph_level, logger)); + + if (!QDQ::MatchQNode(q_node) || + !graph_utils::IsSupportedProvider(q_node, GetCompatibleExecutionProviders()) || + !optimizer_utils::CheckOutputEdges(graph, q_node, 1)) { + continue; + } + + Node& dq_node = *graph.GetNode(q_node.OutputNodesBegin()->Index()); + if (!QDQ::MatchDQNode(dq_node) || + !graph_utils::IsSupportedProvider(dq_node, GetCompatibleExecutionProviders())) { + continue; + } + + // we have Q -> DQ pair + + // we support a DQ that produces a graph output if it has no output edges, or a DQ node with one output edge. + bool is_graph_output = graph.NodeProducesGraphOutput(dq_node); + auto edges_count = dq_node.GetOutputEdgesCount(); + + if ((is_graph_output && edges_count != 0) || + (!is_graph_output && edges_count != 1)) { + continue; + } + + // src node -> Q -> DQ -> downstream node or graph output + NodeIndex src_node_idx = 0; + int src_arg_idx = -1; + NodeIndex downstream_node_idx = 0; + int downstream_arg_idx = -1; + + // input could be node or initializer/graph input so need to handle both. + // if it's a node we need to replace the edge, so need info on which output idx it was attached to on the src node. + const Node::EdgeEnd* input_edge = nullptr; + if (q_node.GetInputEdgesCount() == 1) { + input_edge = &*q_node.InputEdgesBegin(); + src_node_idx = input_edge->GetNode().Index(); + src_arg_idx = input_edge->GetSrcArgIndex(); + // remove edge from src to Q. dest arg idx is 0 as Q only has one input + graph.RemoveEdge(src_node_idx, q_node.Index(), src_arg_idx, 0); + } + + // remove edge between pair we're removing + // both DQ and Q are single input single output so src idx and dest idx must be 0 + graph.RemoveEdge(q_node.Index(), dq_node.Index(), 0, 0); + + if (!is_graph_output) { + // remove edge to downstream node + const Node::EdgeEnd& output_edge = *dq_node.OutputEdgesBegin(); + downstream_node_idx = output_edge.GetNode().Index(); + downstream_arg_idx = output_edge.GetDstArgIndex(); + + // source arg idx is 0 as Q/DQ only has one output + graph.RemoveEdge(dq_node.Index(), downstream_node_idx, 0, downstream_arg_idx); + + // replace input on downstream node + Node& downstream_node = *graph.GetNode(downstream_node_idx); + downstream_node.MutableInputDefs()[downstream_arg_idx] = q_node.MutableInputDefs()[0]; + + // create edge between src_node (if available) and downstream node + if (input_edge) { + graph.AddEdge(src_node_idx, downstream_node_idx, src_arg_idx, downstream_arg_idx); + } + } else { + NodeArg* graph_output_nodearg = dq_node.MutableOutputDefs()[0]; + if (src_arg_idx >= 0) { + // update the src node to produce the graph output that was being provided by the DQ node + Node& src_node = *graph.GetNode(src_node_idx); + src_node.MutableOutputDefs()[src_arg_idx] = graph_output_nodearg; + } else { + // add Identity node to connect the graph input or initializer to the graph output. + Node& id_node = graph.AddNode(graph.GenerateNodeName("QDQFinalCleanupTransformer"), + "Identity", "", {q_node.MutableInputDefs()[0]}, {graph_output_nodearg}); + id_node.SetExecutionProviderType(dq_node.GetExecutionProviderType()); + } + } + + graph.RemoveNode(q_node.Index()); + graph.RemoveNode(dq_node.Index()); + + modified = true; + } + + return Status::OK(); +} + +} // namespace onnxruntime diff --git a/onnxruntime/core/optimizer/qdq_transformer/qdq_final_cleanup.h b/onnxruntime/core/optimizer/qdq_transformer/qdq_final_cleanup.h new file mode 100644 index 0000000000..b281299d1c --- /dev/null +++ b/onnxruntime/core/optimizer/qdq_transformer/qdq_final_cleanup.h @@ -0,0 +1,41 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/common/common.h" +#include "core/optimizer/graph_transformer.h" + +namespace onnxruntime { + +/** + @Class QDQFinalCleanupTransformer + + Remove any remaining back-to-back QuantizeLinear and DequantizeLinear pairs. + + This is the final cleanup where no quantized operator was available and we're going to run the operators + using fp32. + + e.g. if we have Op -> Q -> DQ -> Op2 and no more QDQ processing to run, the Q -> DQ can potentially be removed. + + The impact on performance and accuracy of removing the pair will depend on the model. + + If it was quantized with Quantization Aware Training (QAT) it may be better to pay the performance cost of keeping + the pair as the model was trained with a round-trip from float -> 8-bit -> float. + + If the model was quantized with Post Training Quantization (PTQ) it is most likely better to remove the pair as the + loss of precision from the round trip of float -> 8-bit -> float was not present when the model was trained. + + As we have no knowledge of how the model was quantized we require the user to specify an option to enable this + transformer. + */ +class QDQFinalCleanupTransformer : public GraphTransformer { + public: + QDQFinalCleanupTransformer(const InlinedHashSet& compatible_execution_providers = {}) noexcept + : GraphTransformer("QDQFinalCleanupTransformer", compatible_execution_providers) {} + + private: + Status ApplyImpl(Graph& graph, bool& modified, int graph_level, const logging::Logger& logger) const override; +}; + +} // namespace onnxruntime diff --git a/onnxruntime/test/optimizer/graph_transform_test_builder.cc b/onnxruntime/test/optimizer/graph_transform_test_builder.cc index 6312547d8a..38615d053d 100644 --- a/onnxruntime/test/optimizer/graph_transform_test_builder.cc +++ b/onnxruntime/test/optimizer/graph_transform_test_builder.cc @@ -24,7 +24,8 @@ void TransformerTester(const std::function& buil int opset_version, double per_sample_tolerance, double relative_per_sample_tolerance, - std::unique_ptr transformer) { + std::unique_ptr transformer, + const std::function* add_session_options) { // Build the model for this test. std::unordered_map domain_to_version; domain_to_version[kOnnxDomain] = opset_version; @@ -49,6 +50,9 @@ void TransformerTester(const std::function& buil session_options.optimized_model_filepath = ToPathString("model" + std::to_string(static_cast(level)) + ".onnx"); #endif + if (add_session_options) { + (*add_session_options)(session_options); + } InferenceSessionWrapper session{session_options, GetEnvironment()}; ASSERT_STATUS_OK(session.Load(model_data.data(), static_cast(model_data.size()))); if (transformer) { diff --git a/onnxruntime/test/optimizer/graph_transform_test_builder.h b/onnxruntime/test/optimizer/graph_transform_test_builder.h index 13936b7f2e..a0e8e39995 100644 --- a/onnxruntime/test/optimizer/graph_transform_test_builder.h +++ b/onnxruntime/test/optimizer/graph_transform_test_builder.h @@ -281,7 +281,8 @@ void TransformerTester(const std::function& buil int opset_version = 12, double per_sample_tolerance = 0.0, double relative_per_sample_tolerance = 0.0, - std::unique_ptr transformer = nullptr); + std::unique_ptr transformer = nullptr, + const std::function* add_session_options = nullptr); } // namespace test } // namespace onnxruntime diff --git a/onnxruntime/test/optimizer/qdq_transformer_test.cc b/onnxruntime/test/optimizer/qdq_transformer_test.cc index ca59772275..c9f6805f32 100644 --- a/onnxruntime/test/optimizer/qdq_transformer_test.cc +++ b/onnxruntime/test/optimizer/qdq_transformer_test.cc @@ -5,10 +5,12 @@ #include "core/graph/model.h" #include "core/graph/onnx_protobuf.h" #include "core/mlas/inc/mlas.h" +#include "core/optimizer/qdq_transformer/qdq_final_cleanup.h" #include "core/optimizer/qdq_transformer/selectors_actions/qdq_selectors.h" #include "core/optimizer/qdq_transformer/selectors_actions/qdq_selector_action_transformer.h" #include "core/optimizer/qdq_transformer/selectors_actions/shared/utils.h" #include "core/providers/partitioning_utils.h" +#include "core/session/onnxruntime_session_options_config_keys.h" #include "core/session/environment.h" #include "core/session/inference_session.h" @@ -1757,9 +1759,9 @@ TEST(QDQTransformerTests, QDQPropagation_QBackward) { // TODO re-enable tests after updating ONNX to get QuantizeLinear shape inference fix // https://github.com/onnx/onnx/pull/3806 - //test_case({1, 13, 13, 23}, 4, {0, 3, 1, 2}, false, false); + // test_case({1, 13, 13, 23}, 4, {0, 3, 1, 2}, false, false); test_case({1, 13, 13, 23}, 4, {0, 3, 1, 2}, false, true); - //test_case({1, 13, 13, 23}, 4, {0, 3, 1, 2}, true, false); + // test_case({1, 13, 13, 23}, 4, {0, 3, 1, 2}, true, false); test_case({1, 13, 13, 23}, 4, {0, 3, 1, 2}, true, true); } @@ -1834,9 +1836,9 @@ TEST(QDQTransformerTests, QDQPropagation_DQForward) { // TODO re-enable tests after updating ONNX to get QuantizeLinear shape inference fix // https://github.com/onnx/onnx/pull/3806 - //test_case({1, 13, 13, 23}, 4, {0, 3, 1, 2}, false, false); + // test_case({1, 13, 13, 23}, 4, {0, 3, 1, 2}, false, false); test_case({1, 13, 13, 23}, 4, {0, 3, 1, 2}, false, true); - //test_case({1, 13, 13, 23}, 4, {0, 3, 1, 2}, true, false); + // test_case({1, 13, 13, 23}, 4, {0, 3, 1, 2}, true, false); test_case({1, 13, 13, 23}, 4, {0, 3, 1, 2}, true, true); } @@ -2140,5 +2142,113 @@ TEST(QDQTransformerTests, QDQ_Selector_Test) { } } +// test removal of Q->DQ pairs by QDQFinalCleanupTransformer +TEST(QDQTransformerTests, QDQFinalCleanupTransformer_Basic) { + auto test_case = [&](const std::vector>& input_shapes, + bool block_removal_of_last_dq = false, + bool block_removal_of_first_dq = false) { + // create model with float input to multiple -> Q -> DQ -> Concat -> Q -> DQ -> output + // If we enable cleanup and don't run the QDQ transformer we should drop all the Q->DQ pairs + auto build_test_case = [&](ModelTestBuilder& builder) { + auto input_count = input_shapes.size(); + std::vector input_args; + std::vector q_input_args; + for (size_t i = 0; i < input_count; i++) { + input_args.push_back(builder.MakeInput(input_shapes[i], -1.f, 1.f)); + q_input_args.push_back(AddQDQNodePair(builder, input_args.back(), 0.05f, 128)); + + if (i == 0 && block_removal_of_first_dq) { + // add another edge to the DQ node + auto* output = builder.MakeOutput(); + builder.AddNode("Identity", {q_input_args.back()}, {output}); + } + } + auto* concat_output = builder.MakeIntermediate(); + Node& concat_node = builder.AddNode("Concat", q_input_args, {concat_output}); + concat_node.AddAttribute("axis", int64_t(1)); + + auto* q_concat_output = builder.MakeIntermediate(); + builder.AddQuantizeLinearNode(concat_output, 0.05f, 128, q_concat_output); + + auto* output_arg = builder.MakeOutput(); + Node& dq_node = builder.AddDequantizeLinearNode(q_concat_output, 0.05f, 128, output_arg); + + if (block_removal_of_last_dq) { + // add another edge to the DQ node + auto* output = builder.MakeOutput(); + builder.AddNode("Identity", {dq_node.MutableOutputDefs()[0]}, {output}); + } + }; + + // if we block removal of the DQ node the Q node in the pair will not be removed either + int expected_qdq_count = 0 + (block_removal_of_first_dq ? 1 : 0) + (block_removal_of_last_dq ? 1 : 0); + + auto check_graph = [expected_qdq_count](InferenceSessionWrapper& session) { + auto op_to_count = CountOpsInGraph(session.GetGraph()); + EXPECT_EQ(op_to_count["QuantizeLinear"], expected_qdq_count); + EXPECT_EQ(op_to_count["DequantizeLinear"], expected_qdq_count); + EXPECT_EQ(op_to_count["Concat"], 1); + }; + + std::function func = [](SessionOptions& so) { + ASSERT_STATUS_OK(so.config_options.AddConfigEntry(kOrtSessionOptionsEnableQuantQDQCleanup, "1")); + }; + + // we increase the tolerance as removing the QDQ nodes means there's no round-trip to 8-bit and back + // essentially rounding the input values. + TransformerTester(build_test_case, + check_graph, + TransformerLevel::Level1, + TransformerLevel::Level2, + 12 /*opset_version*/, + 0.025f /*per_sample_tolerance*/, + 0.01f /*relative_per_sample_tolerance*/, + std::make_unique(), + &func); + }; + + test_case({{1, 2, 4}, {1, 3, 4}}); + test_case({{1, 2, 4}, {1, 3, 4}}, true); // block removal of first dq + test_case({{1, 2, 4}, {1, 3, 4}}, false, true); // block removal of last dq + test_case({{1, 2, 4}, {1, 3, 4}}, true, true); // block removal of first and last dq +} + +// test removal when we have graph input -> Q -> DQ -> graph output +TEST(QDQTransformerTests, QDQFinalCleanupTransformer_GraphInputToOutput) { + // create model with float input to -> Q -> DQ -> output + auto build_test_case = [&](ModelTestBuilder& builder) { + NodeArg* input = builder.MakeInput({1, 2, 4}, -1.f, 1.f); + NodeArg* q_output = builder.MakeIntermediate(); + builder.AddQuantizeLinearNode(input, 0.05f, 128, q_output); + auto* output_arg = builder.MakeOutput(); + builder.AddDequantizeLinearNode(q_output, 0.05f, 128, output_arg); + }; + + // with the Q->DQ being dropped we should have inserted an Identity node + // to connect the graph input to the graph output + auto check_graph = [](InferenceSessionWrapper& session) { + auto op_to_count = CountOpsInGraph(session.GetGraph()); + EXPECT_EQ(op_to_count["QuantizeLinear"], 0); + EXPECT_EQ(op_to_count["DequantizeLinear"], 0); + EXPECT_EQ(op_to_count["Identity"], 1); + }; + + std::function func = [](SessionOptions& so) { + ASSERT_STATUS_OK(so.config_options.AddConfigEntry(kOrtSessionOptionsEnableQuantQDQCleanup, "1")); + }; + + // we increase the tolerance as removing the QDQ nodes means there's no round-trip to 8-bit and back + // essentially rounding the input values. + TransformerTester(build_test_case, + check_graph, + TransformerLevel::Level1, + TransformerLevel::Level2, + 12 /*opset_version*/, + 0.025f /*per_sample_tolerance*/, + 0.01f /*relative_per_sample_tolerance*/, + std::make_unique(), + &func); +} + } // namespace test } // namespace onnxruntime