From 48eebed869d4bf9712549f84e8396ca73223da4c Mon Sep 17 00:00:00 2001 From: satyajandhyala Date: Tue, 9 Mar 2021 08:54:56 -0800 Subject: [PATCH] Interchange Cast and Transpose operations to facilitate Transpose-MatMul fusion (#6924) * Added support to interchange Cast and Transpose operations. * Added ONNX models for the Transpose-Cast-MatMul fusion testcases. * Added python code to generate the ONNX models required for testing Transpose+Cast+Matmul fusion to Cast+FusedMatMul. * Added diagram of the Transpose+MatMul fusion documentation --- .../core/optimizer/matmul_transpose_fusion.cc | 184 ++++++++++++++++++ .../test/optimizer/graph_transform_test.cc | 22 +++ .../transpose_cast_matmul_4d_fusion0.onnx | Bin 0 -> 415 bytes .../transpose_cast_matmul_4d_fusion1.onnx | Bin 0 -> 415 bytes .../transform/fusion/transpose_matmul_gen.py | 46 +++++ 5 files changed, 252 insertions(+) create mode 100644 onnxruntime/test/testdata/transform/fusion/transpose_cast_matmul_4d_fusion0.onnx create mode 100644 onnxruntime/test/testdata/transform/fusion/transpose_cast_matmul_4d_fusion1.onnx diff --git a/onnxruntime/core/optimizer/matmul_transpose_fusion.cc b/onnxruntime/core/optimizer/matmul_transpose_fusion.cc index acd43b1df3..bdffc7701d 100644 --- a/onnxruntime/core/optimizer/matmul_transpose_fusion.cc +++ b/onnxruntime/core/optimizer/matmul_transpose_fusion.cc @@ -86,6 +86,177 @@ static size_t UpdateConsumerCount(Graph& graph, NodeArg* target, std::unordered_ } } +/* GetTransposeNodeFromCast: Interchange Cast and Transpose nodes in the graph and return Transpose node if possible +* Requirements to interchange Cast and Transpose nodes changing the order of the operations. +* 1. Both Cast and Transpose are one-output nodes (assuming both have one-input only) +* 2. Transpose only feeds the Cast node (and no other node) +* 3. Cast only feeds the MalMul node (and no other node) +* +* Transform the following pattern +* | +* _____|______ +* |Transpose | +* |__________| +* | +* | +* _____V______ +* | Cast | +* |__________| +* | +* V +* +* to +* | +* _____|______ +* | Cast | +* |__________| +* | +* | +* _____V______ +* | Transpose| +* |__________| +* | +* V +*/ +static Node* GetTransposeNodeFromCast(Graph& graph, Node* cast) { + + ORT_ENFORCE(cast != nullptr); + auto transpose = GetTransposeNodeFromOutput(graph, *cast->MutableInputDefs()[0]); + if (transpose == nullptr || cast->GetOutputEdgesCount() != 1 || transpose->GetOutputEdgesCount() != 1) { + return nullptr; + } + NodeArg* cast_output = cast->MutableOutputDefs()[0]; + NodeArg* transpose_input = transpose->MutableInputDefs()[0]; + + // Create a new NodeArg to feed the output from the new Cast to the new Transpose. + // The shape of the new NodeArg is same as the original input to Transport but type + // should match that of the output from the original Cast. + + auto new_cast_output_type_proto = *transpose_input->TypeAsProto(); + const ONNX_NAMESPACE::TensorProto_DataType element_type = + static_cast(cast_output->TypeAsProto()->tensor_type().elem_type()); + new_cast_output_type_proto.mutable_tensor_type()->set_elem_type(element_type); + auto& new_cast_output = graph.GetOrCreateNodeArg(cast_output->Name() + "_transformed", &new_cast_output_type_proto); + + const std::vector new_cast_input_defs {transpose_input}; + const std::vector new_cast_output_defs {&new_cast_output}; + const std::vector new_transpose_input_defs = {&new_cast_output}; + const std::vector new_transpose_output_defs = {cast_output}; + + (void) graph.AddNode(graph.GenerateNodeName(cast->Name() + "_transformed"), + cast->OpType(), + "Created a new Cast node to interchange Cast and Transpose nodes", + new_cast_input_defs, + new_cast_output_defs, + &cast->GetAttributes(), + cast->Domain()); + + Node& new_transpose = graph.AddNode(graph.GenerateNodeName(transpose->Name() + "_transformed"), + transpose->OpType(), + "Created a new Transpose node to interchange Cast and Transpose nodes", + new_transpose_input_defs, + new_transpose_output_defs, + &transpose->GetAttributes(), + transpose->Domain()); + + graph_utils::RemoveNodeOutputEdges(graph, *cast); + graph_utils::RemoveNodeOutputEdges(graph, *transpose); + graph.RemoveNode(cast->Index()); + graph.RemoveNode(transpose->Index()); + return &new_transpose; +} + +/********************************************************************************************* + +Case I: The followin is a scenario where Transpose output feeds MatMul. The Transpose input can be either on the left or right. + The input graph + __________ __________ + | input0 | | input1 | + |________| |________| + | | + | | + | | + _____V______ | + |Transpose | | + |__________| | + | | + | | + |______________ _____________| + | | + | | + | | + __V___________V__ + | MatMul | + |_______________| + | + V + is transformed to the following + + __________ __________ + | input0 | | input1 | + |________| |________| + | | + | | + | | + |_____________ _____________| + | | + | | + | | + __V___________V__ + | FusedMatMul | + |_______________| + | + V + +Case II: The output of Tanspose feeds Cast and the output from the Cast feeds MatMul + The input graph + __________ __________ + | input0 | | input1 | + |________| |________| + | | + | | + _____V______ | + |Transpose | | + |__________| | + | | + | | + _____V______ | + | Cast | | + |__________| | + | | + |______________ _____________| + | | + | | + | | + __V___________V__ + | MatMul | + |_______________| + | + V + is transformed to the following + + __________ __________ + | input0 | | input1 | + |________| |________| + | | + | | + | | + _____V______ | + | Cast | | + |__________| | + | | + |______________ _____________| + | | + | | + | | + __V___________V__ + | FusedMatMul | + |_______________| + | + V + +********************************************************************************************************************/ + Status MatmulTransposeFusion::ApplyImpl(Graph& graph, bool& modified, int graph_level, const logging::Logger& logger) const { GraphViewer graph_viewer(graph); const auto& node_topology_list = graph_viewer.GetNodesInTopologicalOrder(); @@ -109,6 +280,19 @@ Status MatmulTransposeFusion::ApplyImpl(Graph& graph, bool& modified, int graph_ NodeArg* right_input = node.MutableInputDefs()[1]; auto right = GetTransposeNodeFromOutput(graph, *right_input); + if (!left && !right) { + Node* left_node = graph.GetMutableProducerNode(left_input->Name()); + if (left_node && left_node->OpType() == "Cast") { + left = GetTransposeNodeFromCast(graph, left_node); + } + if (!left) { + Node* right_node = graph.GetMutableProducerNode(right_input->Name()); + if (right_node && right_node->OpType() == "Cast") { + right = GetTransposeNodeFromCast(graph, right_node); + } + } + } + if (!left && !right) { continue; } diff --git a/onnxruntime/test/optimizer/graph_transform_test.cc b/onnxruntime/test/optimizer/graph_transform_test.cc index 87d2ec6d87..1c88e15e6e 100644 --- a/onnxruntime/test/optimizer/graph_transform_test.cc +++ b/onnxruntime/test/optimizer/graph_transform_test.cc @@ -844,6 +844,28 @@ TEST_F(GraphTransformationTests, TransposeMatmulFusion) { ASSERT_TRUE(op_to_count["com.microsoft.FusedMatMul"] == 1); } +TEST_F(GraphTransformationTests, TransposeCastMatmulFusion) { + const std::vector model_uris = { + MODEL_FOLDER "fusion/transpose_cast_matmul_4d_fusion0.onnx", // Test fusion from the right input + MODEL_FOLDER "fusion/transpose_cast_matmul_4d_fusion1.onnx" // Test fusion from the left input + }; + for (const auto& model_uri : model_uris) { + std::shared_ptr p_model; + ASSERT_TRUE(Model::Load(model_uri, p_model, nullptr, *logger_).IsOK()); + Graph& graph = p_model->MainGraph(); + + onnxruntime::GraphTransformerManager graph_transformation_mgr{5}; + graph_transformation_mgr.Register(onnxruntime::make_unique(), TransformerLevel::Level1); + auto ret = graph_transformation_mgr.ApplyTransformers(graph, TransformerLevel::Level1, *logger_); + ASSERT_TRUE(ret.IsOK()); + std::map op_to_count = CountOpsInGraph(graph); + ASSERT_TRUE(op_to_count["Transpose"] == 0); + ASSERT_TRUE(op_to_count["MatMul"] == 0); + ASSERT_TRUE(op_to_count["Cast"] == 2); + ASSERT_TRUE(op_to_count["com.microsoft.FusedMatMul"] == 1); + } +} + TEST_F(GraphTransformationTests, TransposeMatmulFusionOnTwoTranspose) { auto model_uri = MODEL_FOLDER "fusion/transpose_matmul_4d_fusion_2_transpose.onnx"; std::shared_ptr p_model; diff --git a/onnxruntime/test/testdata/transform/fusion/transpose_cast_matmul_4d_fusion0.onnx b/onnxruntime/test/testdata/transform/fusion/transpose_cast_matmul_4d_fusion0.onnx new file mode 100644 index 0000000000000000000000000000000000000000..8d5c73c82fc9957d9049111e9ebe23d30738aa12 GIT binary patch literal 415 zcmd;J7ZS+N%d03V%`3^wP1P+)EiSQo&%~w0#h#g0P+AgiD8!eXSX`2t5)b7ou{Z4U{-TU}9Q=Tr35tMY#?P4vY@W4onLe z*|{9Ju;>wzL^21(dIL=BZMkr(=0bD45J!G#3CLRpN^HJ~CBCIOLc(z8K?Fj8ArPgE z@R9_V7zd+}5El~%GY~UzF$*zzg9u-QB10iuiu{sPxZqZTRdJyjjAIxsjeIxsshEnsBla^S+EM@SOMoaDsf5+vhToPm5TPA;aBdt{E}3-;8ucFaiJRJ3o^>jiG_