From 6808bfefff03c2390d59c77eef9d196833821e7e Mon Sep 17 00:00:00 2001 From: Yufeng Li Date: Fri, 20 Nov 2020 11:30:49 -0800 Subject: [PATCH] Extend MatMulInteger fusion (#5871) Extend MatMulInteger fusion to MatMulIntegerToFloat or DynamicQuantizeMatMul to support scenario: Matrix B, B scale, B zero point are non constant --- onnxruntime/core/graph/graph_utils.cc | 4 +- onnxruntime/core/graph/graph_utils.h | 4 +- .../dynamic_quantize_matmul_fusion.cc | 211 ++++-------------- .../core/optimizer/graph_transformer_utils.cc | 2 + .../core/optimizer/matmul_integer_to_float.cc | 165 ++++++++++++++ .../core/optimizer/matmul_integer_to_float.h | 23 ++ .../test/optimizer/graph_transform_test.cc | 6 + .../fusion/dynamic_quantize_matmul.fused.onnx | Bin 435 -> 0 bytes .../fusion/dynamic_quantize_matmul.onnx | 18 +- .../fusion/dynamic_quantize_matmul.py | 23 +- .../dynamic_quantize_matmul_b_no_zp_bias.onnx | Bin 554 -> 0 bytes .../fusion/dynamic_quantize_matmul_bias.onnx | Bin 574 -> 586 bytes .../dynamic_quantize_matmul_bias_ND.onnx | Bin 580 -> 586 bytes .../dynamic_quantize_matmul_bias_b_no_zp.onnx | Bin 554 -> 560 bytes .../fusion/matmul_integer_to_float.onnx | Bin 1510 -> 1520 bytes .../fusion/matmul_integer_to_float.py | 7 +- 16 files changed, 271 insertions(+), 192 deletions(-) create mode 100644 onnxruntime/core/optimizer/matmul_integer_to_float.cc create mode 100644 onnxruntime/core/optimizer/matmul_integer_to_float.h delete mode 100644 onnxruntime/test/testdata/transform/fusion/dynamic_quantize_matmul.fused.onnx delete mode 100644 onnxruntime/test/testdata/transform/fusion/dynamic_quantize_matmul_b_no_zp_bias.onnx diff --git a/onnxruntime/core/graph/graph_utils.cc b/onnxruntime/core/graph/graph_utils.cc index b5f47a7b87..f7a694d9f0 100644 --- a/onnxruntime/core/graph/graph_utils.cc +++ b/onnxruntime/core/graph/graph_utils.cc @@ -547,7 +547,7 @@ bool AllNodeInputsAreConstant(const Graph& graph, const Node& node, InitializedT return true; } -const Node* FirstChildByType(Node& node, const std::string& child_type) { +const Node* FirstChildByType(const Node& node, const std::string& child_type) { for (auto it = node.OutputNodesBegin(); it != node.OutputNodesEnd(); ++it) { if ((*it).OpType().compare(child_type) == 0) { return &(*it); @@ -556,7 +556,7 @@ const Node* FirstChildByType(Node& node, const std::string& child_type) { return nullptr; } -const Node* FirstParentByType(Node& node, const std::string& parent_type) { +const Node* FirstParentByType(const Node& node, const std::string& parent_type) { for (auto it = node.InputNodesBegin(); it != node.InputNodesEnd(); ++it) { if ((*it).OpType().compare(parent_type) == 0) { return &(*it); diff --git a/onnxruntime/core/graph/graph_utils.h b/onnxruntime/core/graph/graph_utils.h index 267b8a683b..f0fbc98cc0 100644 --- a/onnxruntime/core/graph/graph_utils.h +++ b/onnxruntime/core/graph/graph_utils.h @@ -96,9 +96,9 @@ bool GetRepeatedNodeAttributeValues(const Node& node, } /** Find the first child of the specified op type. */ -const Node* FirstChildByType(Node& node, const std::string& child_type); +const Node* FirstChildByType(const Node& node, const std::string& child_type); /** Find the first parent of the specified op type. */ -const Node* FirstParentByType(Node& node, const std::string& parent_type); +const Node* FirstParentByType(const Node& node, const std::string& parent_type); /** Tests if we can remove a node and merge its input edge (if any) with its output edges. Conditions: diff --git a/onnxruntime/core/optimizer/dynamic_quantize_matmul_fusion.cc b/onnxruntime/core/optimizer/dynamic_quantize_matmul_fusion.cc index 404b2c13cd..11db736f2e 100644 --- a/onnxruntime/core/optimizer/dynamic_quantize_matmul_fusion.cc +++ b/onnxruntime/core/optimizer/dynamic_quantize_matmul_fusion.cc @@ -12,65 +12,21 @@ using namespace ONNX_NAMESPACE; using namespace ::onnxruntime::common; namespace onnxruntime { -// Check if bias is a 1-D tensor, or N-D tensor with the prior N-1 dimension equal to 1. -// And its last dimension equal to MatMul's last dimension -static bool CheckBiasShape(const TensorShapeProto* bias_shape, const TensorShapeProto* matmul_shape) { - if (nullptr == matmul_shape || matmul_shape->dim_size() <= 1 || - nullptr == bias_shape || bias_shape->dim_size() < 1) { - return false; - } - - // First N-1 dimension must equal to 1 - for (int i = 0; i < bias_shape->dim_size() - 1; i++) { - if (bias_shape->dim(i).dim_value() != 1) { - return false; - } - } - - int64_t bias_last_dim = bias_shape->dim(bias_shape->dim_size() - 1).dim_value(); - int64_t matmul_last_dim = matmul_shape->dim(matmul_shape->dim_size() - 1).dim_value(); - return bias_last_dim == matmul_last_dim && bias_last_dim > 0; -} - /** DynamicQuantizeMatMulFusion will fuse subgraph like below into DynamicQuantizeMatMul: - (input) - | - v -DynamicQuantizeLinear --------+ - | | - v v -MatMulInteger (B const) Mul (B const) (input) - | | | - v v v - Cast ------------------>Mul ----> DynamicQuantizeMatMul - | | - v v - Add (B const, Optional) (output) - | - v - (output) - -It also fuses subgraph like below into MatMulIntegerToFloat: - input input - | | - v v - +----------------------------DynamicQuantizeLinear------------------------+ DynamicQuantizeLinear - | | | | - | +----------------+--------------+ | +---------+----------+ - | | | | | | - V v v v V v - MatMulInteger(B const) Mul(B const) MatMulInteger (B const) Mul (B const) ---> MatMulIntegerToFloat MatMulIntegerToFloat - | | | | | | - v v v v v v - Cast ---------------->Mul Cast ---------------->Mul (output1) ----------(output2) - | | - v v - Add (B const, Optional) Add (B const, Optional) - | | - v v - (output1) (output2) - + (input) + | + v + DynamicQuantizeLinear B,B_Scale,B_Zero Bias (input) B,B_Scale,B_Zero Bias + | | | | | | | | + | | | | | | | | + A| A_Scale| A_Zero | | | | | | + | | | | | | | | + v v v | | | | | + MatMulIntegerToFloat <-------------+---------------+ ----> DynamicQuantizeMatMul<-------------+---------------+ + | | + v v + (output) (output) */ Status DynamicQuantizeMatMulFusion::ApplyImpl(Graph& graph, bool& modified, int graph_level, const logging::Logger& logger) const { GraphViewer graph_viewer(graph); @@ -82,151 +38,64 @@ Status DynamicQuantizeMatMulFusion::ApplyImpl(Graph& graph, bool& modified, int if (nullptr == node_ptr) continue; // node was removed - auto& mul_node = *node_ptr; + auto& matmul_integer_to_float_node = *node_ptr; - ORT_RETURN_IF_ERROR(Recurse(mul_node, modified, graph_level, logger)); + ORT_RETURN_IF_ERROR(Recurse(matmul_integer_to_float_node, modified, graph_level, logger)); - if (!graph_utils::IsSupportedOptypeVersionAndDomain(mul_node, "Mul", {7, 13}) || - !graph_utils::IsSupportedProvider(mul_node, GetCompatibleExecutionProviders())) { + if (!graph_utils::IsSupportedOptypeVersionAndDomain(matmul_integer_to_float_node, "MatMulIntegerToFloat", {1}, kMSDomain) || + !graph_utils::IsSupportedProvider(matmul_integer_to_float_node, GetCompatibleExecutionProviders())) { continue; } - // Left Parents path - std::vector left_parent_path{ - {0, 0, "Cast", {6, 9, 13}, kOnnxDomain}, - {0, 0, "MatMulInteger", {10}, kOnnxDomain}, - {0, 0, "DynamicQuantizeLinear", {11}, kOnnxDomain}}; - - std::vector right_parent_path{ - {0, 1, "Mul", {7, 13}, kOnnxDomain}, - {1, 0, "DynamicQuantizeLinear", {11}, kOnnxDomain}}; - - std::vector> left_nodes; - std::vector> right_nodes; - if (!graph_utils::FindPath(graph, mul_node, true, left_parent_path, left_nodes, logger) || - !graph_utils::FindPath(graph, mul_node, true, right_parent_path, right_nodes, logger)) { + const Node* p_dynamic_quant_linear = graph_utils::GetInputNode(matmul_integer_to_float_node, 0 /*arg_index*/); + if (p_dynamic_quant_linear == nullptr) { continue; } - Node& cast_node = left_nodes[0]; - Node& matmulinteger_node = left_nodes[1]; - Node& dql_node_left = left_nodes[2]; - - Node& mul_node_right = right_nodes[0]; - Node& dql_node_right = right_nodes[1]; - - // Check if left DynamicQuantizeLinear is same as right DynamicQuantizeLinear - if (dql_node_left.Index() != dql_node_right.Index()) { + Node& dynamic_quant_linear = *graph.GetNode(p_dynamic_quant_linear->Index()); + if (!optimizer_utils::CheckOutputEdges(graph, dynamic_quant_linear, 3)) { continue; } - // Check Nodes' Edges count and Nodes' outputs are not in Graph output - if (!optimizer_utils::CheckOutputEdges(graph, cast_node, 1) || - !optimizer_utils::CheckOutputEdges(graph, matmulinteger_node, 1) || - !optimizer_utils::CheckOutputEdges(graph, mul_node_right, 1)) { - continue; - } - - const NodeArg& matmulinteger_B = *(matmulinteger_node.InputDefs()[1]); - if (!graph_utils::IsConstantInitializer(graph, matmulinteger_B.Name(), true)) { - continue; - } - - const NodeArg& mul_right_B = *(mul_node_right.InputDefs()[1]); - if (!graph_utils::IsConstantInitializer(graph, mul_right_B.Name(), true)) { - continue; - } - - // Find bias node - Node* add_node = nullptr; - // const Node* add_node = FindBiasNode(graph, mul_node, ; - if (optimizer_utils::CheckOutputEdges(graph, mul_node, 1)) { - const Node* tmp_add_node = graph_utils::FirstChildByType(mul_node, "Add"); - if (nullptr != tmp_add_node) { - const NodeArg& tmp_add_node_B = *(tmp_add_node->InputDefs()[1]); - if (graph_utils::IsConstantInitializer(graph, tmp_add_node_B.Name(), true) && - CheckBiasShape(tmp_add_node_B.Shape(), matmulinteger_B.Shape())) { - add_node = graph.GetNode(tmp_add_node->Index()); - } - } - } - - // DynamicQuantizeLinear outputs are only used by one MatMulInteger, - // thus it can fused into DynamicQuantizeMatMul NodeArg optional_node_arg("", nullptr); - std::vector input_defs; std::string op_type_to_fuse = "DynamicQuantizeMatMul"; - if (optimizer_utils::CheckOutputEdges(graph, dql_node_left, 3)) { - input_defs.push_back(dql_node_left.MutableInputDefs()[0]); - input_defs.push_back(matmulinteger_node.MutableInputDefs()[1]); - input_defs.push_back(mul_node_right.MutableInputDefs()[1]); - input_defs.push_back(&optional_node_arg); + std::vector input_defs{ + dynamic_quant_linear.MutableInputDefs()[0], + matmul_integer_to_float_node.MutableInputDefs()[1], + matmul_integer_to_float_node.MutableInputDefs()[3], + &optional_node_arg, + &optional_node_arg}; - if (matmulinteger_node.InputDefs().size() == 4) { - const NodeArg& matmulinteger_B_zp = *(matmulinteger_node.InputDefs()[3]); - if (!graph_utils::IsConstantInitializer(graph, matmulinteger_B_zp.Name(), true)) { - continue; - } - input_defs[3] = matmulinteger_node.MutableInputDefs()[3]; - } + if (matmul_integer_to_float_node.InputDefs().size() >= 6) { + input_defs[3] = matmul_integer_to_float_node.MutableInputDefs()[5]; - nodes_to_remove.push_back(dql_node_left); - } else { - op_type_to_fuse = "MatMulIntegerToFloat"; - - input_defs.push_back(matmulinteger_node.MutableInputDefs()[0]); - input_defs.push_back(matmulinteger_node.MutableInputDefs()[1]); - input_defs.push_back(mul_node_right.MutableInputDefs()[0]); - input_defs.push_back(mul_node_right.MutableInputDefs()[1]); - input_defs.push_back(&optional_node_arg); - input_defs.push_back(&optional_node_arg); - - if (matmulinteger_node.InputDefs().size() >= 3) { - // Add zero point of A - input_defs[4] = matmulinteger_node.MutableInputDefs()[2]; - - // Add zero point of B - if (matmulinteger_node.InputDefs().size() == 4) { - const NodeArg& matmulinteger_B_zp = *(matmulinteger_node.InputDefs()[3]); - if (!graph_utils::IsConstantInitializer(graph, matmulinteger_B_zp.Name(), true)) { - continue; - } - input_defs[5] = matmulinteger_node.MutableInputDefs()[3]; - } + if (matmul_integer_to_float_node.InputDefs().size() >= 7) { + input_defs[4] = matmul_integer_to_float_node.MutableInputDefs()[6]; } } - if (add_node != nullptr) { - input_defs.push_back(add_node->MutableInputDefs()[1]); - } - - Node* fused_node = &graph.AddNode(graph.GenerateNodeName(op_type_to_fuse), + Node* fused_node = &graph.AddNode(matmul_integer_to_float_node.Name(), op_type_to_fuse, "", input_defs, - add_node != nullptr ? add_node->MutableOutputDefs() : mul_node.MutableOutputDefs(), + matmul_integer_to_float_node.MutableOutputDefs(), nullptr, kMSDomain); - // Assign provider to this new node. Provider should be same as the provider for old node. - ORT_ENFORCE(nullptr != fused_node); - fused_node->SetExecutionProviderType(mul_node.GetExecutionProviderType()); - nodes_to_remove.push_back(matmulinteger_node); - nodes_to_remove.push_back(cast_node); - nodes_to_remove.push_back(mul_node_right); - nodes_to_remove.push_back(mul_node); - if (add_node != nullptr) { - nodes_to_remove.push_back(*add_node); - } + // Assign provider to this new node. Provider should be same as the provider for old node. + fused_node->SetExecutionProviderType(matmul_integer_to_float_node.GetExecutionProviderType()); + + nodes_to_remove.push_back(dynamic_quant_linear); + nodes_to_remove.push_back(matmul_integer_to_float_node); } + modified = !nodes_to_remove.empty(); + for (const auto& node : nodes_to_remove) { graph_utils::RemoveNodeOutputEdges(graph, node); graph.RemoveNode(node.get().Index()); } - modified = true; - return Status::OK(); } } // namespace onnxruntime diff --git a/onnxruntime/core/optimizer/graph_transformer_utils.cc b/onnxruntime/core/optimizer/graph_transformer_utils.cc index 68e8b7ef0f..4c48d69088 100644 --- a/onnxruntime/core/optimizer/graph_transformer_utils.cc +++ b/onnxruntime/core/optimizer/graph_transformer_utils.cc @@ -26,6 +26,7 @@ #include "core/optimizer/identity_elimination.h" #include "core/optimizer/layer_norm_fusion.h" #include "core/optimizer/matmul_add_fusion.h" +#include "core/optimizer/matmul_integer_to_float.h" #include "core/optimizer/matmul_scale_fusion.h" #include "core/optimizer/nchwc_transformer.h" #include "core/optimizer/nhwc_transformer.h" @@ -135,6 +136,7 @@ std::vector> GenerateTransformers(TransformerL #ifndef DISABLE_CONTRIB_OPS transformers.emplace_back(onnxruntime::make_unique(cpu_execution_providers)); + transformers.emplace_back(onnxruntime::make_unique(cpu_execution_providers)); transformers.emplace_back(onnxruntime::make_unique(cpu_execution_providers)); std::unordered_set cpu_acl_execution_providers = {onnxruntime::kCpuExecutionProvider, onnxruntime::kAclExecutionProvider}; diff --git a/onnxruntime/core/optimizer/matmul_integer_to_float.cc b/onnxruntime/core/optimizer/matmul_integer_to_float.cc new file mode 100644 index 0000000000..630455c621 --- /dev/null +++ b/onnxruntime/core/optimizer/matmul_integer_to_float.cc @@ -0,0 +1,165 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "matmul_integer_to_float.h" + +#include "core/graph/graph_utils.h" +#include "core/optimizer/initializer.h" +#include "core/optimizer/utils.h" + +using namespace ONNX_NAMESPACE; +using namespace ::onnxruntime::common; +namespace onnxruntime { + +// Check if bias is a 1-D tensor, or N-D tensor with the prior N-1 dimension equal to 1. +// And its last dimension equal to MatMul's last dimension +static bool CheckBiasShape(const TensorShapeProto* bias_shape) { + if (nullptr == bias_shape || bias_shape->dim_size() < 1) { + return false; + } + + // First N-1 dimension must equal to 1 + for (int i = 0; i < bias_shape->dim_size() - 1; i++) { + if (bias_shape->dim(i).dim_value() != 1) { + return false; + } + } + + int64_t bias_last_dim = bias_shape->dim(bias_shape->dim_size() - 1).dim_value(); + + // Don't allow last dimension to be 1, to be on the safe side + return bias_last_dim > 1; +} + +/** +MatMulIntegerToFloatFusion will fuse subgraph like below into MatMulIntegerToFloat: + + A A_Zero B B_Zero A_Scale) B_Scale Bias (Const, Optional) + \ | | / \ / | + \ | | / \ / | + \ | | / \ / | + MatMulInteger Mul | (A, B, A_Scale, B_Scale, A_Zero, B_Zero, Bias) + | | | | + v v | v + Cast ------------------>Mul | ----> MatMulIntegerToFloat + | | | + v | v + Add <-------------+ (output) + | + v + (output) + + */ +Status MatMulIntegerToFloatFusion::ApplyImpl(Graph& graph, bool& modified, int graph_level, const logging::Logger& logger) const { + GraphViewer graph_viewer(graph); + const auto& node_topology_list = graph_viewer.GetNodesInTopologicalOrder(); + std::vector> nodes_to_remove; + + for (auto node_index : node_topology_list) { + auto* node_ptr = graph.GetNode(node_index); + if (nullptr == node_ptr) + continue; // node was removed + + auto& mul_node = *node_ptr; + + ORT_RETURN_IF_ERROR(Recurse(mul_node, modified, graph_level, logger)); + + if (!graph_utils::IsSupportedOptypeVersionAndDomain(mul_node, "Mul", {7, 13}) || + !graph_utils::IsSupportedProvider(mul_node, GetCompatibleExecutionProviders())) { + continue; + } + + const Node* p_cast_node = graph_utils::FirstParentByType(mul_node, "Cast"); + if (p_cast_node == nullptr) { + continue; + } + + const Node* p_matmulinteger_node = graph_utils::FirstParentByType(*p_cast_node, "MatMulInteger"); + if (p_matmulinteger_node == nullptr) { + continue; + } + + const Node* p_mul_node_right = graph_utils::FirstParentByType(mul_node, "Mul"); + if (p_mul_node_right == nullptr) { + continue; + } + + Node& cast_node = *graph.GetNode(p_cast_node->Index()); + Node& matmulinteger_node = *graph.GetNode(p_matmulinteger_node->Index()); + Node& mul_node_right = *graph.GetNode(p_mul_node_right->Index()); + + // Check Nodes' Edges count and Nodes' outputs are not in Graph output + if (!optimizer_utils::CheckOutputEdges(graph, cast_node, 1) || + !optimizer_utils::CheckOutputEdges(graph, matmulinteger_node, 1) || + !optimizer_utils::CheckOutputEdges(graph, mul_node_right, 1)) { + continue; + } + + // Find bias node + Node* p_add_node = nullptr; + if (optimizer_utils::CheckOutputEdges(graph, mul_node, 1)) { + const Node* tmp_add_node = graph_utils::FirstChildByType(mul_node, "Add"); + if (nullptr != tmp_add_node) { + const NodeArg& tmp_add_node_B = *(tmp_add_node->InputDefs()[1]); + if (graph_utils::IsConstantInitializer(graph, tmp_add_node_B.Name(), true) && + CheckBiasShape(tmp_add_node_B.Shape())) { + p_add_node = graph.GetNode(tmp_add_node->Index()); + } + } + } + + // DynamicQuantizeLinear outputs are only used by one MatMulInteger, + // thus it can fused into DynamicQuantizeMatMul + NodeArg optional_node_arg("", nullptr); + std::vector input_defs{ + matmulinteger_node.MutableInputDefs()[0], + matmulinteger_node.MutableInputDefs()[1], + mul_node_right.MutableInputDefs()[0], + mul_node_right.MutableInputDefs()[1], + &optional_node_arg, + &optional_node_arg}; + + if (p_matmulinteger_node->InputDefs().size() >= 3) { + // Add zero point of A + input_defs[4] = matmulinteger_node.MutableInputDefs()[2]; + + // Add zero point of B + if (p_matmulinteger_node->InputDefs().size() >= 4) { + input_defs[5] = matmulinteger_node.MutableInputDefs()[3]; + } + } + + if (p_add_node != nullptr) { + input_defs.push_back(p_add_node->MutableInputDefs()[1]); + } + + std::string op_type = "MatMulIntegerToFloat"; + Node& fused_node = graph.AddNode(matmulinteger_node.Name(), + op_type, + "", + input_defs, + p_add_node != nullptr ? p_add_node->MutableOutputDefs() : mul_node.MutableOutputDefs(), + nullptr, + kMSDomain); + // Assign provider to this new node. Provider should be same as the provider for old node. + fused_node.SetExecutionProviderType(mul_node.GetExecutionProviderType()); + + nodes_to_remove.push_back(matmulinteger_node); + nodes_to_remove.push_back(cast_node); + nodes_to_remove.push_back(mul_node_right); + nodes_to_remove.push_back(mul_node); + if (p_add_node != nullptr) { + nodes_to_remove.push_back(*p_add_node); + } + } + + modified = !nodes_to_remove.empty(); + + for (const auto& node : nodes_to_remove) { + graph_utils::RemoveNodeOutputEdges(graph, node); + graph.RemoveNode(node.get().Index()); + } + + return Status::OK(); +} +} // namespace onnxruntime diff --git a/onnxruntime/core/optimizer/matmul_integer_to_float.h b/onnxruntime/core/optimizer/matmul_integer_to_float.h new file mode 100644 index 0000000000..b1c87c5916 --- /dev/null +++ b/onnxruntime/core/optimizer/matmul_integer_to_float.h @@ -0,0 +1,23 @@ +// Copyright (c) Microsoft Corporation. All rights reserved. +// Licensed under the MIT License. + +#pragma once + +#include "core/optimizer/graph_transformer.h" + +namespace onnxruntime { + +/** +@Class MatMulIntegerToFloatFusion +Fuse MatMulInteger and corresponding cast and mul to MatMulIntegerToFloat +*/ +class MatMulIntegerToFloatFusion : public GraphTransformer { + public: + MatMulIntegerToFloatFusion(const std::unordered_set& compatible_execution_providers = {}) noexcept + : GraphTransformer("MatMulIntegerToFloatFusion", compatible_execution_providers) { + } + + Status ApplyImpl(Graph& graph, bool& modified, int graph_level, const logging::Logger& logger) const override; +}; + +} // namespace onnxruntime diff --git a/onnxruntime/test/optimizer/graph_transform_test.cc b/onnxruntime/test/optimizer/graph_transform_test.cc index 40b1cd8abb..63bad72be4 100644 --- a/onnxruntime/test/optimizer/graph_transform_test.cc +++ b/onnxruntime/test/optimizer/graph_transform_test.cc @@ -38,6 +38,7 @@ #include "core/optimizer/initializer.h" #include "core/optimizer/layer_norm_fusion.h" #include "core/optimizer/matmul_add_fusion.h" +#include "core/optimizer/matmul_integer_to_float.h" #include "core/optimizer/matmul_scale_fusion.h" #include "core/optimizer/matmul_transpose_fusion.h" #include "core/optimizer/relu_clip_fusion.h" @@ -2980,6 +2981,7 @@ TEST_F(GraphTransformationTests, DynamicQuantizeMatMulTest) { Graph& graph = p_model->MainGraph(); onnxruntime::GraphTransformerManager graph_transformation_mgr{5}; + graph_transformation_mgr.Register(onnxruntime::make_unique(), TransformerLevel::Level2); graph_transformation_mgr.Register(onnxruntime::make_unique(), TransformerLevel::Level2); auto ret = graph_transformation_mgr.ApplyTransformers(graph, TransformerLevel::Level2, *logger_); ASSERT_TRUE(ret.IsOK()); @@ -2999,6 +3001,7 @@ TEST_F(GraphTransformationTests, DynamicQuantizeMatMulTest_With_Bias) { Graph& graph = p_model->MainGraph(); onnxruntime::GraphTransformerManager graph_transformation_mgr{5}; + graph_transformation_mgr.Register(onnxruntime::make_unique(), TransformerLevel::Level2); graph_transformation_mgr.Register(onnxruntime::make_unique(), TransformerLevel::Level2); auto ret = graph_transformation_mgr.ApplyTransformers(graph, TransformerLevel::Level2, *logger_); ASSERT_TRUE(ret.IsOK()); @@ -3018,6 +3021,7 @@ TEST_F(GraphTransformationTests, DynamicQuantizeMatMulTest_With_ND_bias) { Graph& graph = p_model->MainGraph(); onnxruntime::GraphTransformerManager graph_transformation_mgr{5}; + graph_transformation_mgr.Register(onnxruntime::make_unique(), TransformerLevel::Level2); graph_transformation_mgr.Register(onnxruntime::make_unique(), TransformerLevel::Level2); auto ret = graph_transformation_mgr.ApplyTransformers(graph, TransformerLevel::Level2, *logger_); ASSERT_TRUE(ret.IsOK()); @@ -3038,6 +3042,7 @@ TEST_F(GraphTransformationTests, DynamicQuantizeMatMulTest_With_Bias_No_B_ZP) { Graph& graph = p_model->MainGraph(); onnxruntime::GraphTransformerManager graph_transformation_mgr{5}; + graph_transformation_mgr.Register(onnxruntime::make_unique(), TransformerLevel::Level2); graph_transformation_mgr.Register(onnxruntime::make_unique(), TransformerLevel::Level2); auto ret = graph_transformation_mgr.ApplyTransformers(graph, TransformerLevel::Level2, *logger_); ASSERT_TRUE(ret.IsOK()); @@ -3057,6 +3062,7 @@ TEST_F(GraphTransformationTests, MatMulIntegerToFloatTest) { Graph& graph = p_model->MainGraph(); onnxruntime::GraphTransformerManager graph_transformation_mgr{5}; + graph_transformation_mgr.Register(onnxruntime::make_unique(), TransformerLevel::Level2); graph_transformation_mgr.Register(onnxruntime::make_unique(), TransformerLevel::Level2); auto ret = graph_transformation_mgr.ApplyTransformers(graph, TransformerLevel::Level2, *logger_); ASSERT_TRUE(ret.IsOK()); diff --git a/onnxruntime/test/testdata/transform/fusion/dynamic_quantize_matmul.fused.onnx b/onnxruntime/test/testdata/transform/fusion/dynamic_quantize_matmul.fused.onnx deleted file mode 100644 index 077168c0dbe77074820c9e09b0b9d377d54022c5..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 435 zcmZvYu};G<6h*O{rq*kz+yH7tDuFs?$iUbI$zr4fj4Y9pI$$Yrf@4$KiG`2g2l);x zluAw2!NYs^`0CzA{N)dZZv=g*YHMJaChwLjBX?pAgEVO}UI_5gWLF1%WsPI5uO7CF zmoj^5!4)?vTP#km@50gEd z-yaZmm^B(X0e>O(A2pg#wD9u$_nz%LJfFQx+mgW;Mg#=}oSl0(ok3geNa-<5hinQL zos_Z4-ZmLU>;}TFP!_r1#%gyw-8f(jz-3%1^%<81#q1L3em0s*CDp5=<}llDYb`#c N*u)3w1iCk(=npX?fcF3Z diff --git a/onnxruntime/test/testdata/transform/fusion/dynamic_quantize_matmul.onnx b/onnxruntime/test/testdata/transform/fusion/dynamic_quantize_matmul.onnx index 7f0dfcedf0..ff5ac7b26f 100644 --- a/onnxruntime/test/testdata/transform/fusion/dynamic_quantize_matmul.onnx +++ b/onnxruntime/test/testdata/transform/fusion/dynamic_quantize_matmul.onnx @@ -1,4 +1,4 @@ -:ì +:ø Q input a_quantizeda_scalea_zpDynamicQuantizeLinear"DynamicQuantizeLinear Y @@ -17,11 +17,23 @@ A matmul_output_float multiplieroutput -mul_bottom"MulDynamicQuantizeLinear_fusion**B b_quantized*"ffæ?Bb_scale* *€Bb_zpZ +mul_bottom"MulDynamicQuantizeLinear_fusionZ input   -b +Z + b_quantized +  + +Z +b_scale + + +Z +b_zp + + +b output   diff --git a/onnxruntime/test/testdata/transform/fusion/dynamic_quantize_matmul.py b/onnxruntime/test/testdata/transform/fusion/dynamic_quantize_matmul.py index a1d68664a7..cec03dcbdf 100644 --- a/onnxruntime/test/testdata/transform/fusion/dynamic_quantize_matmul.py +++ b/onnxruntime/test/testdata/transform/fusion/dynamic_quantize_matmul.py @@ -20,15 +20,7 @@ def GenerateModel(model_name, b_has_zp = True, has_bias = False, bias_ND = False if has_bias: nodes.extend([helper.make_node("Add", [mul_output, "bias"], ["output"], "bias_add")]) - initializers = [ # initializers - helper.make_tensor('b_quantized', TensorProto.UINT8, [2,3], [2, 4, 5, 6, 7, 8]), - helper.make_tensor('b_scale', TensorProto.FLOAT, [], [1.8]), - ] - - if b_has_zp: - initializers.extend([ # initializers - helper.make_tensor('b_zp', TensorProto.UINT8, [], [128]), - ]) + initializers = [] if has_bias: if bias_ND: @@ -40,12 +32,19 @@ def GenerateModel(model_name, b_has_zp = True, has_bias = False, bias_ND = False helper.make_tensor('bias', TensorProto.FLOAT, [3], [3.0, 4.0, 5.0]), ]) + inputs = [ # inputs + helper.make_tensor_value_info('input', TensorProto.FLOAT, [3, 2]), + helper.make_tensor_value_info('b_quantized', TensorProto.UINT8, [2,3]), + helper.make_tensor_value_info('b_scale', TensorProto.FLOAT, [1]), + ] + + if b_has_zp: + inputs.extend([helper.make_tensor_value_info('b_zp', TensorProto.UINT8, [1])]) + graph = helper.make_graph( nodes, "DynamicQuantizeLinear_fusion", #name - [ # inputs - helper.make_tensor_value_info('input', TensorProto.FLOAT, [3, 2]), - ], + inputs, [ # outputs helper.make_tensor_value_info('output', TensorProto.FLOAT, [3, 3]), ], diff --git a/onnxruntime/test/testdata/transform/fusion/dynamic_quantize_matmul_b_no_zp_bias.onnx b/onnxruntime/test/testdata/transform/fusion/dynamic_quantize_matmul_b_no_zp_bias.onnx deleted file mode 100644 index 9724c06a949252161f2ed4cc70a1c53ff22b4154..0000000000000000000000000000000000000000 GIT binary patch literal 0 HcmV?d00001 literal 554 zcmZvZzfQw25XKY7ZQN@?#RyRaCU0b5fUN_xfdz>U6&s7?G)b|PI3;lgI`Jsw<#-O< zHWg{r;Vj)B{l4$c#2x-%m_kdcMWtz1@XtypEthdbUBSzlNMnixUoO0h=PxBPIh)o4 zuTsULV8_ew22B=p!+$Vrc=S9II;&EiSK3H(sr3DwcQO%rQl&3S$Md*g`zhSQ&J(y{ zZ4L}W%S9^V!t1Pgg`9tA<`@c$;h4ywa+2mkdw3>F&G3_X{SF|V5A2nV;NcLlnTCcc z1tV2&;jQaD%(c$5t%VzCv;l`wl(fBF*(2+Wizs5wIEv`?(F60ODrK(xD}spAM}8aO zR@)^(^P>J)-)1;TejkI~UHpNWwpq`zMkCAmHnAE78~OLk-EW^lkJyxe0h>VK0BUD+ KzvBd`-}?h}nWz;2 diff --git a/onnxruntime/test/testdata/transform/fusion/dynamic_quantize_matmul_bias.onnx b/onnxruntime/test/testdata/transform/fusion/dynamic_quantize_matmul_bias.onnx index aa20374fb4ffa4dc73e00fbced268fc200081a23..85052ee1eb70ac615964e037841697836696950f 100644 GIT binary patch delta 96 zcmdnTa*Bn8gWc-jMwT^MKDd~tGOPO1QBnJRgo)WGA delta 83 zcmX@bvX6y@gWYQDMwT^MKDd~tGOPO1MKDd~tGOPO1gWc-+Mix^RQEA`A65rAs&%Bb<^wgq|e7BtZ#FEJYESZ!4vP=w!lIP-0 ziZ3ip%qz*PN==D365`|H;b0Qt;9}xn0%GPU5iYi*_^JY+3>Oy%P=*C4$`~ci#hDae SoSc}G3YB1lNKEcyjQ{{TQy%mH delta 88 zcmeys{fwK1gWc-xMix^RaarHQ65rAs&%Bb<^wc8Pijvg4;>`TK$$l)EYDQXo0!&&= s4UA4~N%2($@kUz00*p#5X=%^woj8-?i<1*`Qsa##&tzFYc?N3)0LZT!rT_o{ diff --git a/onnxruntime/test/testdata/transform/fusion/matmul_integer_to_float.py b/onnxruntime/test/testdata/transform/fusion/matmul_integer_to_float.py index a509f10af6..1a270043ba 100644 --- a/onnxruntime/test/testdata/transform/fusion/matmul_integer_to_float.py +++ b/onnxruntime/test/testdata/transform/fusion/matmul_integer_to_float.py @@ -32,7 +32,6 @@ def GenerateModel(model_name): initializers = [] initializers.extend(MakeInitializer("_1")) - initializers.extend(MakeInitializer("_2")) initializers.extend(MakeInitializer("_3")) initializers.extend([ @@ -43,8 +42,12 @@ def GenerateModel(model_name): graph = helper.make_graph( nodes, "MatMulIntegerToFloat_fusion", #name - [ # inputs + [ # inputs helper.make_tensor_value_info('input', TensorProto.FLOAT, [3, 2]), + # matrix b corresponding inputs for subgraph 2 + helper.make_tensor_value_info('b_quantized_2', TensorProto.UINT8, [2, 3]), + helper.make_tensor_value_info('b_zp_2', TensorProto.UINT8, [1]), + helper.make_tensor_value_info('b_scale_2', TensorProto.FLOAT, [1]), ], [ # outputs helper.make_tensor_value_info('output_1', TensorProto.FLOAT, [3, 3]),