saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-29 20:14:01 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Extend MatMulInteger fusion (#5871)

Browse source 
Extend MatMulInteger fusion to MatMulIntegerToFloat or DynamicQuantizeMatMul to support scenario: Matrix B, B scale, B zero point are non constant
This commit is contained in:
Yufeng Li 2020-11-20 11:30:49 -08:00 committed by GitHub
parent 44313970d3
commit 6808bfefff
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
16 changed files with 271 additions and 192 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

4
onnxruntime/core/graph/graph_utils.cc
View file

@ -547,7 +547,7 @@ bool AllNodeInputsAreConstant(const Graph& graph, const Node& node, InitializedT
return true;
}
const Node* FirstChildByType(Node& node, const std::string& child_type) {
const Node* FirstChildByType(const Node& node, const std::string& child_type) {
for (auto it = node.OutputNodesBegin(); it != node.OutputNodesEnd(); ++it) {
if ((*it).OpType().compare(child_type) == 0) {
return &(*it);
@ -556,7 +556,7 @@ const Node* FirstChildByType(Node& node, const std::string& child_type) {
return nullptr;
}
const Node* FirstParentByType(Node& node, const std::string& parent_type) {
const Node* FirstParentByType(const Node& node, const std::string& parent_type) {
for (auto it = node.InputNodesBegin(); it != node.InputNodesEnd(); ++it) {
if ((*it).OpType().compare(parent_type) == 0) {
return &(*it);

4
onnxruntime/core/graph/graph_utils.h
View file

@ -96,9 +96,9 @@ bool GetRepeatedNodeAttributeValues(const Node& node,
}
/** Find the first child of the specified op type. */
const Node* FirstChildByType(Node& node, const std::string& child_type);
const Node* FirstChildByType(const Node& node, const std::string& child_type);
/** Find the first parent of the specified op type. */
const Node* FirstParentByType(Node& node, const std::string& parent_type);
const Node* FirstParentByType(const Node& node, const std::string& parent_type);
/** Tests if we can remove a node and merge its input edge (if any) with its output edges.
Conditions:

211
onnxruntime/core/optimizer/dynamic_quantize_matmul_fusion.cc
View file

@ -12,65 +12,21 @@ using namespace ONNX_NAMESPACE;
using namespace ::onnxruntime::common;
namespace onnxruntime {
// Check if bias is a 1-D tensor, or N-D tensor with the prior N-1 dimension equal to 1.
// And its last dimension equal to MatMul's last dimension
static bool CheckBiasShape(const TensorShapeProto* bias_shape, const TensorShapeProto* matmul_shape) {
if (nullptr == matmul_shape || matmul_shape->dim_size() <= 1 ||
nullptr == bias_shape || bias_shape->dim_size() < 1) {
return false;
}
// First N-1 dimension must equal to 1
for (int i = 0; i < bias_shape->dim_size() - 1; i++) {
if (bias_shape->dim(i).dim_value() != 1) {
return false;
}
}
int64_t bias_last_dim = bias_shape->dim(bias_shape->dim_size() - 1).dim_value();
int64_t matmul_last_dim = matmul_shape->dim(matmul_shape->dim_size() - 1).dim_value();
return bias_last_dim == matmul_last_dim && bias_last_dim > 0;
}
/**
DynamicQuantizeMatMulFusion will fuse subgraph like below into DynamicQuantizeMatMul:
(input)
|
v
DynamicQuantizeLinear --------+
| |
v v
MatMulInteger (B const) Mul (B const) (input)
| | |
v v v
Cast ------------------>Mul ----> DynamicQuantizeMatMul
| |
v v
Add (B const, Optional) (output)
|
v
(output)
It also fuses subgraph like below into MatMulIntegerToFloat:
input input
| |
v v
+----------------------------DynamicQuantizeLinear------------------------+ DynamicQuantizeLinear
| | | |
| +----------------+--------------+ | +---------+----------+
| | | | | |
V v v v V v
MatMulInteger(B const) Mul(B const) MatMulInteger (B const) Mul (B const) ---> MatMulIntegerToFloat MatMulIntegerToFloat
| | | | | |
v v v v v v
Cast ---------------->Mul Cast ---------------->Mul (output1) ----------(output2)
| |
v v
Add (B const, Optional) Add (B const, Optional)
| |
v v
(output1) (output2)
(input)
|
v
DynamicQuantizeLinear B,B_Scale,B_Zero Bias (input) B,B_Scale,B_Zero Bias
| | | | | | | |
| | | | | | | |
A| A_Scale| A_Zero | | | | | |
| | | | | | | |
v v v | | | | |
MatMulIntegerToFloat <-------------+---------------+ ----> DynamicQuantizeMatMul<-------------+---------------+
| |
v v
(output) (output)
*/
Status DynamicQuantizeMatMulFusion::ApplyImpl(Graph& graph, bool& modified, int graph_level, const logging::Logger& logger) const {
GraphViewer graph_viewer(graph);
@ -82,151 +38,64 @@ Status DynamicQuantizeMatMulFusion::ApplyImpl(Graph& graph, bool& modified, int
if (nullptr == node_ptr)
continue; // node was removed
auto& mul_node = *node_ptr;
auto& matmul_integer_to_float_node = *node_ptr;
ORT_RETURN_IF_ERROR(Recurse(mul_node, modified, graph_level, logger));
ORT_RETURN_IF_ERROR(Recurse(matmul_integer_to_float_node, modified, graph_level, logger));
if (!graph_utils::IsSupportedOptypeVersionAndDomain(mul_node, "Mul", {7, 13}) ||
!graph_utils::IsSupportedProvider(mul_node, GetCompatibleExecutionProviders())) {
if (!graph_utils::IsSupportedOptypeVersionAndDomain(matmul_integer_to_float_node, "MatMulIntegerToFloat", {1}, kMSDomain) ||
!graph_utils::IsSupportedProvider(matmul_integer_to_float_node, GetCompatibleExecutionProviders())) {
continue;
}
// Left Parents path
std::vector<graph_utils::EdgeEndToMatch> left_parent_path{
{0, 0, "Cast", {6, 9, 13}, kOnnxDomain},
{0, 0, "MatMulInteger", {10}, kOnnxDomain},
{0, 0, "DynamicQuantizeLinear", {11}, kOnnxDomain}};
std::vector<graph_utils::EdgeEndToMatch> right_parent_path{
{0, 1, "Mul", {7, 13}, kOnnxDomain},
{1, 0, "DynamicQuantizeLinear", {11}, kOnnxDomain}};
std::vector<std::reference_wrapper<Node>> left_nodes;
std::vector<std::reference_wrapper<Node>> right_nodes;
if (!graph_utils::FindPath(graph, mul_node, true, left_parent_path, left_nodes, logger) ||
!graph_utils::FindPath(graph, mul_node, true, right_parent_path, right_nodes, logger)) {
const Node* p_dynamic_quant_linear = graph_utils::GetInputNode(matmul_integer_to_float_node, 0 /*arg_index*/);
if (p_dynamic_quant_linear == nullptr) {
continue;
}
Node& cast_node = left_nodes[0];
Node& matmulinteger_node = left_nodes[1];
Node& dql_node_left = left_nodes[2];
Node& mul_node_right = right_nodes[0];
Node& dql_node_right = right_nodes[1];
// Check if left DynamicQuantizeLinear is same as right DynamicQuantizeLinear
if (dql_node_left.Index() != dql_node_right.Index()) {
Node& dynamic_quant_linear = *graph.GetNode(p_dynamic_quant_linear->Index());
if (!optimizer_utils::CheckOutputEdges(graph, dynamic_quant_linear, 3)) {
continue;
}
// Check Nodes' Edges count and Nodes' outputs are not in Graph output
if (!optimizer_utils::CheckOutputEdges(graph, cast_node, 1) ||
!optimizer_utils::CheckOutputEdges(graph, matmulinteger_node, 1) ||
!optimizer_utils::CheckOutputEdges(graph, mul_node_right, 1)) {
continue;
}
const NodeArg& matmulinteger_B = *(matmulinteger_node.InputDefs()[1]);
if (!graph_utils::IsConstantInitializer(graph, matmulinteger_B.Name(), true)) {
continue;
}
const NodeArg& mul_right_B = *(mul_node_right.InputDefs()[1]);
if (!graph_utils::IsConstantInitializer(graph, mul_right_B.Name(), true)) {
continue;
}
// Find bias node
Node* add_node = nullptr;
// const Node* add_node = FindBiasNode(graph, mul_node, ;
if (optimizer_utils::CheckOutputEdges(graph, mul_node, 1)) {
const Node* tmp_add_node = graph_utils::FirstChildByType(mul_node, "Add");
if (nullptr != tmp_add_node) {
const NodeArg& tmp_add_node_B = *(tmp_add_node->InputDefs()[1]);
if (graph_utils::IsConstantInitializer(graph, tmp_add_node_B.Name(), true) &&
CheckBiasShape(tmp_add_node_B.Shape(), matmulinteger_B.Shape())) {
add_node = graph.GetNode(tmp_add_node->Index());
}
}
}
// DynamicQuantizeLinear outputs are only used by one MatMulInteger,
// thus it can fused into DynamicQuantizeMatMul
NodeArg optional_node_arg("", nullptr);
std::vector<NodeArg*> input_defs;
std::string op_type_to_fuse = "DynamicQuantizeMatMul";
if (optimizer_utils::CheckOutputEdges(graph, dql_node_left, 3)) {
input_defs.push_back(dql_node_left.MutableInputDefs()[0]);
input_defs.push_back(matmulinteger_node.MutableInputDefs()[1]);
input_defs.push_back(mul_node_right.MutableInputDefs()[1]);
input_defs.push_back(&optional_node_arg);
std::vector<NodeArg*> input_defs{
dynamic_quant_linear.MutableInputDefs()[0],
matmul_integer_to_float_node.MutableInputDefs()[1],
matmul_integer_to_float_node.MutableInputDefs()[3],
&optional_node_arg,
&optional_node_arg};
if (matmulinteger_node.InputDefs().size() == 4) {
const NodeArg& matmulinteger_B_zp = *(matmulinteger_node.InputDefs()[3]);
if (!graph_utils::IsConstantInitializer(graph, matmulinteger_B_zp.Name(), true)) {
continue;
}
input_defs[3] = matmulinteger_node.MutableInputDefs()[3];
}
if (matmul_integer_to_float_node.InputDefs().size() >= 6) {
input_defs[3] = matmul_integer_to_float_node.MutableInputDefs()[5];
nodes_to_remove.push_back(dql_node_left);
} else {
op_type_to_fuse = "MatMulIntegerToFloat";
input_defs.push_back(matmulinteger_node.MutableInputDefs()[0]);
input_defs.push_back(matmulinteger_node.MutableInputDefs()[1]);
input_defs.push_back(mul_node_right.MutableInputDefs()[0]);
input_defs.push_back(mul_node_right.MutableInputDefs()[1]);
input_defs.push_back(&optional_node_arg);
input_defs.push_back(&optional_node_arg);
if (matmulinteger_node.InputDefs().size() >= 3) {
// Add zero point of A
input_defs[4] = matmulinteger_node.MutableInputDefs()[2];
// Add zero point of B
if (matmulinteger_node.InputDefs().size() == 4) {
const NodeArg& matmulinteger_B_zp = *(matmulinteger_node.InputDefs()[3]);
if (!graph_utils::IsConstantInitializer(graph, matmulinteger_B_zp.Name(), true)) {
continue;
}
input_defs[5] = matmulinteger_node.MutableInputDefs()[3];
}
if (matmul_integer_to_float_node.InputDefs().size() >= 7) {
input_defs[4] = matmul_integer_to_float_node.MutableInputDefs()[6];
}
}
if (add_node != nullptr) {
input_defs.push_back(add_node->MutableInputDefs()[1]);
}
Node* fused_node = &graph.AddNode(graph.GenerateNodeName(op_type_to_fuse),
Node* fused_node = &graph.AddNode(matmul_integer_to_float_node.Name(),
op_type_to_fuse,
"",
input_defs,
add_node != nullptr ? add_node->MutableOutputDefs() : mul_node.MutableOutputDefs(),
matmul_integer_to_float_node.MutableOutputDefs(),
nullptr,
kMSDomain);
// Assign provider to this new node. Provider should be same as the provider for old node.
ORT_ENFORCE(nullptr != fused_node);
fused_node->SetExecutionProviderType(mul_node.GetExecutionProviderType());
nodes_to_remove.push_back(matmulinteger_node);
nodes_to_remove.push_back(cast_node);
nodes_to_remove.push_back(mul_node_right);
nodes_to_remove.push_back(mul_node);
if (add_node != nullptr) {
nodes_to_remove.push_back(*add_node);
}
// Assign provider to this new node. Provider should be same as the provider for old node.
fused_node->SetExecutionProviderType(matmul_integer_to_float_node.GetExecutionProviderType());
nodes_to_remove.push_back(dynamic_quant_linear);
nodes_to_remove.push_back(matmul_integer_to_float_node);
}
modified = !nodes_to_remove.empty();
for (const auto& node : nodes_to_remove) {
graph_utils::RemoveNodeOutputEdges(graph, node);
graph.RemoveNode(node.get().Index());
}
modified = true;
return Status::OK();
}
} // namespace onnxruntime

2
onnxruntime/core/optimizer/graph_transformer_utils.cc
View file

@ -26,6 +26,7 @@
#include "core/optimizer/identity_elimination.h"
#include "core/optimizer/layer_norm_fusion.h"
#include "core/optimizer/matmul_add_fusion.h"
#include "core/optimizer/matmul_integer_to_float.h"
#include "core/optimizer/matmul_scale_fusion.h"
#include "core/optimizer/nchwc_transformer.h"
#include "core/optimizer/nhwc_transformer.h"
@ -135,6 +136,7 @@ std::vector<std::unique_ptr<GraphTransformer>> GenerateTransformers(TransformerL
#ifndef DISABLE_CONTRIB_OPS
transformers.emplace_back(onnxruntime::make_unique<GemmActivationFusion>(cpu_execution_providers));
transformers.emplace_back(onnxruntime::make_unique<MatMulIntegerToFloatFusion>(cpu_execution_providers));
transformers.emplace_back(onnxruntime::make_unique<DynamicQuantizeMatMulFusion>(cpu_execution_providers));
std::unordered_set<std::string> cpu_acl_execution_providers = {onnxruntime::kCpuExecutionProvider, onnxruntime::kAclExecutionProvider};

165
onnxruntime/core/optimizer/matmul_integer_to_float.cc Normal file
View file

@ -0,0 +1,165 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#include "matmul_integer_to_float.h"
#include "core/graph/graph_utils.h"
#include "core/optimizer/initializer.h"
#include "core/optimizer/utils.h"
using namespace ONNX_NAMESPACE;
using namespace ::onnxruntime::common;
namespace onnxruntime {
// Check if bias is a 1-D tensor, or N-D tensor with the prior N-1 dimension equal to 1.
// And its last dimension equal to MatMul's last dimension
static bool CheckBiasShape(const TensorShapeProto* bias_shape) {
if (nullptr == bias_shape || bias_shape->dim_size() < 1) {
return false;
}
// First N-1 dimension must equal to 1
for (int i = 0; i < bias_shape->dim_size() - 1; i++) {
if (bias_shape->dim(i).dim_value() != 1) {
return false;
}
}
int64_t bias_last_dim = bias_shape->dim(bias_shape->dim_size() - 1).dim_value();
// Don't allow last dimension to be 1, to be on the safe side
return bias_last_dim > 1;
}
/**
MatMulIntegerToFloatFusion will fuse subgraph like below into MatMulIntegerToFloat:
A A_Zero B B_Zero A_Scale) B_Scale Bias (Const, Optional)
\ | | / \ / |
\ | | / \ / |
\ | | / \ / |
MatMulInteger Mul | (A, B, A_Scale, B_Scale, A_Zero, B_Zero, Bias)
| | | |
v v | v
Cast ------------------>Mul | ----> MatMulIntegerToFloat
| | |
v | v
Add <-------------+ (output)
|
v
(output)
*/
Status MatMulIntegerToFloatFusion::ApplyImpl(Graph& graph, bool& modified, int graph_level, const logging::Logger& logger) const {
GraphViewer graph_viewer(graph);
const auto& node_topology_list = graph_viewer.GetNodesInTopologicalOrder();
std::vector<std::reference_wrapper<Node>> nodes_to_remove;
for (auto node_index : node_topology_list) {
auto* node_ptr = graph.GetNode(node_index);
if (nullptr == node_ptr)
continue; // node was removed
auto& mul_node = *node_ptr;
ORT_RETURN_IF_ERROR(Recurse(mul_node, modified, graph_level, logger));
if (!graph_utils::IsSupportedOptypeVersionAndDomain(mul_node, "Mul", {7, 13}) ||
!graph_utils::IsSupportedProvider(mul_node, GetCompatibleExecutionProviders())) {
continue;
}
const Node* p_cast_node = graph_utils::FirstParentByType(mul_node, "Cast");
if (p_cast_node == nullptr) {
continue;
}
const Node* p_matmulinteger_node = graph_utils::FirstParentByType(*p_cast_node, "MatMulInteger");
if (p_matmulinteger_node == nullptr) {
continue;
}
const Node* p_mul_node_right = graph_utils::FirstParentByType(mul_node, "Mul");
if (p_mul_node_right == nullptr) {
continue;
}
Node& cast_node = *graph.GetNode(p_cast_node->Index());
Node& matmulinteger_node = *graph.GetNode(p_matmulinteger_node->Index());
Node& mul_node_right = *graph.GetNode(p_mul_node_right->Index());
// Check Nodes' Edges count and Nodes' outputs are not in Graph output
if (!optimizer_utils::CheckOutputEdges(graph, cast_node, 1) ||
!optimizer_utils::CheckOutputEdges(graph, matmulinteger_node, 1) ||
!optimizer_utils::CheckOutputEdges(graph, mul_node_right, 1)) {
continue;
}
// Find bias node
Node* p_add_node = nullptr;
if (optimizer_utils::CheckOutputEdges(graph, mul_node, 1)) {
const Node* tmp_add_node = graph_utils::FirstChildByType(mul_node, "Add");
if (nullptr != tmp_add_node) {
const NodeArg& tmp_add_node_B = *(tmp_add_node->InputDefs()[1]);
if (graph_utils::IsConstantInitializer(graph, tmp_add_node_B.Name(), true) &&
CheckBiasShape(tmp_add_node_B.Shape())) {
p_add_node = graph.GetNode(tmp_add_node->Index());
}
}
}
// DynamicQuantizeLinear outputs are only used by one MatMulInteger,
// thus it can fused into DynamicQuantizeMatMul
NodeArg optional_node_arg("", nullptr);
std::vector<NodeArg*> input_defs{
matmulinteger_node.MutableInputDefs()[0],
matmulinteger_node.MutableInputDefs()[1],
mul_node_right.MutableInputDefs()[0],
mul_node_right.MutableInputDefs()[1],
&optional_node_arg,
&optional_node_arg};
if (p_matmulinteger_node->InputDefs().size() >= 3) {
// Add zero point of A
input_defs[4] = matmulinteger_node.MutableInputDefs()[2];
// Add zero point of B
if (p_matmulinteger_node->InputDefs().size() >= 4) {
input_defs[5] = matmulinteger_node.MutableInputDefs()[3];
}
}
if (p_add_node != nullptr) {
input_defs.push_back(p_add_node->MutableInputDefs()[1]);
}
std::string op_type = "MatMulIntegerToFloat";
Node& fused_node = graph.AddNode(matmulinteger_node.Name(),
op_type,
"",
input_defs,
p_add_node != nullptr ? p_add_node->MutableOutputDefs() : mul_node.MutableOutputDefs(),
nullptr,
kMSDomain);
// Assign provider to this new node. Provider should be same as the provider for old node.
fused_node.SetExecutionProviderType(mul_node.GetExecutionProviderType());
nodes_to_remove.push_back(matmulinteger_node);
nodes_to_remove.push_back(cast_node);
nodes_to_remove.push_back(mul_node_right);
nodes_to_remove.push_back(mul_node);
if (p_add_node != nullptr) {
nodes_to_remove.push_back(*p_add_node);
}
}
modified = !nodes_to_remove.empty();
for (const auto& node : nodes_to_remove) {
graph_utils::RemoveNodeOutputEdges(graph, node);
graph.RemoveNode(node.get().Index());
}
return Status::OK();
}
} // namespace onnxruntime

23
onnxruntime/core/optimizer/matmul_integer_to_float.h Normal file
View file

@ -0,0 +1,23 @@
// Copyright (c) Microsoft Corporation. All rights reserved.
// Licensed under the MIT License.
#pragma once
#include "core/optimizer/graph_transformer.h"
namespace onnxruntime {
/**
@Class MatMulIntegerToFloatFusion
Fuse MatMulInteger and corresponding cast and mul to MatMulIntegerToFloat
*/
class MatMulIntegerToFloatFusion : public GraphTransformer {
public:
MatMulIntegerToFloatFusion(const std::unordered_set<std::string>& compatible_execution_providers = {}) noexcept
: GraphTransformer("MatMulIntegerToFloatFusion", compatible_execution_providers) {
}
Status ApplyImpl(Graph& graph, bool& modified, int graph_level, const logging::Logger& logger) const override;
};
} // namespace onnxruntime

6
onnxruntime/test/optimizer/graph_transform_test.cc
View file

@ -38,6 +38,7 @@
#include "core/optimizer/initializer.h"
#include "core/optimizer/layer_norm_fusion.h"
#include "core/optimizer/matmul_add_fusion.h"
#include "core/optimizer/matmul_integer_to_float.h"
#include "core/optimizer/matmul_scale_fusion.h"
#include "core/optimizer/matmul_transpose_fusion.h"
#include "core/optimizer/relu_clip_fusion.h"
@ -2980,6 +2981,7 @@ TEST_F(GraphTransformationTests, DynamicQuantizeMatMulTest) {
Graph& graph = p_model->MainGraph();
onnxruntime::GraphTransformerManager graph_transformation_mgr{5};
graph_transformation_mgr.Register(onnxruntime::make_unique<MatMulIntegerToFloatFusion>(), TransformerLevel::Level2);
graph_transformation_mgr.Register(onnxruntime::make_unique<DynamicQuantizeMatMulFusion>(), TransformerLevel::Level2);
auto ret = graph_transformation_mgr.ApplyTransformers(graph, TransformerLevel::Level2, *logger_);
ASSERT_TRUE(ret.IsOK());
@ -2999,6 +3001,7 @@ TEST_F(GraphTransformationTests, DynamicQuantizeMatMulTest_With_Bias) {
Graph& graph = p_model->MainGraph();
onnxruntime::GraphTransformerManager graph_transformation_mgr{5};
graph_transformation_mgr.Register(onnxruntime::make_unique<MatMulIntegerToFloatFusion>(), TransformerLevel::Level2);
graph_transformation_mgr.Register(onnxruntime::make_unique<DynamicQuantizeMatMulFusion>(), TransformerLevel::Level2);
auto ret = graph_transformation_mgr.ApplyTransformers(graph, TransformerLevel::Level2, *logger_);
ASSERT_TRUE(ret.IsOK());
@ -3018,6 +3021,7 @@ TEST_F(GraphTransformationTests, DynamicQuantizeMatMulTest_With_ND_bias) {
Graph& graph = p_model->MainGraph();
onnxruntime::GraphTransformerManager graph_transformation_mgr{5};
graph_transformation_mgr.Register(onnxruntime::make_unique<MatMulIntegerToFloatFusion>(), TransformerLevel::Level2);
graph_transformation_mgr.Register(onnxruntime::make_unique<DynamicQuantizeMatMulFusion>(), TransformerLevel::Level2);
auto ret = graph_transformation_mgr.ApplyTransformers(graph, TransformerLevel::Level2, *logger_);
ASSERT_TRUE(ret.IsOK());
@ -3038,6 +3042,7 @@ TEST_F(GraphTransformationTests, DynamicQuantizeMatMulTest_With_Bias_No_B_ZP) {
Graph& graph = p_model->MainGraph();
onnxruntime::GraphTransformerManager graph_transformation_mgr{5};
graph_transformation_mgr.Register(onnxruntime::make_unique<MatMulIntegerToFloatFusion>(), TransformerLevel::Level2);
graph_transformation_mgr.Register(onnxruntime::make_unique<DynamicQuantizeMatMulFusion>(), TransformerLevel::Level2);
auto ret = graph_transformation_mgr.ApplyTransformers(graph, TransformerLevel::Level2, *logger_);
ASSERT_TRUE(ret.IsOK());
@ -3057,6 +3062,7 @@ TEST_F(GraphTransformationTests, MatMulIntegerToFloatTest) {
Graph& graph = p_model->MainGraph();
onnxruntime::GraphTransformerManager graph_transformation_mgr{5};
graph_transformation_mgr.Register(onnxruntime::make_unique<MatMulIntegerToFloatFusion>(), TransformerLevel::Level2);
graph_transformation_mgr.Register(onnxruntime::make_unique<DynamicQuantizeMatMulFusion>(), TransformerLevel::Level2);
auto ret = graph_transformation_mgr.ApplyTransformers(graph, TransformerLevel::Level2, *logger_);
ASSERT_TRUE(ret.IsOK());

BIN
onnxruntime/test/testdata/transform/fusion/dynamic_quantize_matmul.fused.onnx vendored
View file

Binary file not shown.

18
onnxruntime/test/testdata/transform/fusion/dynamic_quantize_matmul.onnx vendored
View file

@ -1,4 +1,4 @@
:ì
:ø
Q
input a_quantizeda_scalea_zpDynamicQuantizeLinear"DynamicQuantizeLinear
Y
@ -17,11 +17,23 @@ A
matmul_output_float
multiplieroutput
mul_bottom"MulDynamicQuantizeLinear_fusion**B b_quantized*"ffæ?Bb_scale* *€Bb_zpZ
mul_bottom"MulDynamicQuantizeLinear_fusionZ
input


b
Z
b_quantized


Z
b_scale

Z
b_zp

b
output



23
onnxruntime/test/testdata/transform/fusion/dynamic_quantize_matmul.py vendored
View file

@ -20,15 +20,7 @@ def GenerateModel(model_name, b_has_zp = True, has_bias = False, bias_ND = False
if has_bias:
nodes.extend([helper.make_node("Add", [mul_output, "bias"], ["output"], "bias_add")])
initializers = [ # initializers
helper.make_tensor('b_quantized', TensorProto.UINT8, [2,3], [2, 4, 5, 6, 7, 8]),
helper.make_tensor('b_scale', TensorProto.FLOAT, [], [1.8]),
]
if b_has_zp:
initializers.extend([ # initializers
helper.make_tensor('b_zp', TensorProto.UINT8, [], [128]),
])
initializers = []
if has_bias:
if bias_ND:
@ -40,12 +32,19 @@ def GenerateModel(model_name, b_has_zp = True, has_bias = False, bias_ND = False
helper.make_tensor('bias', TensorProto.FLOAT, [3], [3.0, 4.0, 5.0]),
])
inputs = [ # inputs
helper.make_tensor_value_info('input', TensorProto.FLOAT, [3, 2]),
helper.make_tensor_value_info('b_quantized', TensorProto.UINT8, [2,3]),
helper.make_tensor_value_info('b_scale', TensorProto.FLOAT, [1]),
]
if b_has_zp:
inputs.extend([helper.make_tensor_value_info('b_zp', TensorProto.UINT8, [1])])
graph = helper.make_graph(
nodes,
"DynamicQuantizeLinear_fusion", #name
[ # inputs
helper.make_tensor_value_info('input', TensorProto.FLOAT, [3, 2]),
],
inputs,
[ # outputs
helper.make_tensor_value_info('output', TensorProto.FLOAT, [3, 3]),
],

BIN
onnxruntime/test/testdata/transform/fusion/dynamic_quantize_matmul_b_no_zp_bias.onnx vendored
View file

Binary file not shown.

BIN
onnxruntime/test/testdata/transform/fusion/dynamic_quantize_matmul_bias.onnx vendored
View file

Binary file not shown.

BIN
onnxruntime/test/testdata/transform/fusion/dynamic_quantize_matmul_bias_ND.onnx vendored
View file

Binary file not shown.

BIN
onnxruntime/test/testdata/transform/fusion/dynamic_quantize_matmul_bias_b_no_zp.onnx vendored
View file

Binary file not shown.

BIN
onnxruntime/test/testdata/transform/fusion/matmul_integer_to_float.onnx vendored
View file

Binary file not shown.

7
onnxruntime/test/testdata/transform/fusion/matmul_integer_to_float.py vendored
View file

@ -32,7 +32,6 @@ def GenerateModel(model_name):
initializers = []
initializers.extend(MakeInitializer("_1"))
initializers.extend(MakeInitializer("_2"))
initializers.extend(MakeInitializer("_3"))
initializers.extend([
@ -43,8 +42,12 @@ def GenerateModel(model_name):
graph = helper.make_graph(
nodes,
"MatMulIntegerToFloat_fusion", #name
[ # inputs
[ # inputs
helper.make_tensor_value_info('input', TensorProto.FLOAT, [3, 2]),
# matrix b corresponding inputs for subgraph 2
helper.make_tensor_value_info('b_quantized_2', TensorProto.UINT8, [2, 3]),
helper.make_tensor_value_info('b_zp_2', TensorProto.UINT8, [1]),
helper.make_tensor_value_info('b_scale_2', TensorProto.FLOAT, [1]),
],
[ # outputs
helper.make_tensor_value_info('output_1', TensorProto.FLOAT, [3, 3]),