diff --git a/onnxruntime/core/providers/dnnl/subgraph/dnnl_matmul_integer.cc b/onnxruntime/core/providers/dnnl/subgraph/dnnl_matmul_integer.cc index cf7e0bdfd4..7c92243f98 100644 --- a/onnxruntime/core/providers/dnnl/subgraph/dnnl_matmul_integer.cc +++ b/onnxruntime/core/providers/dnnl/subgraph/dnnl_matmul_integer.cc @@ -4,6 +4,11 @@ #include "dnnl_matmul_integer.h" #include "dnnl_subgraph.h" #include "dnnl_subgraph_primitive.h" +#include "dnnl_util.h" + +#include +#include +#include namespace onnxruntime { namespace ort_dnnl { @@ -11,8 +16,28 @@ namespace ort_dnnl { DnnlMatMulInteger::DnnlMatMulInteger() {} void DnnlMatMulInteger::CreatePrimitive(DnnlSubgraphPrimitive& sp, DnnlNode& node) { + std::unordered_set binary_ops = {"Add", "Div", "Mul", "Sub"}; + std::unordered_set elementwise_ops = {"Abs", "Elu", "Exp", "LeakyRelu", "Log", "Relu", + "Round", "Sigmoid", "Softplus", "Sqrt", "Tanh"}; auto eng = sp.GetEngine(); + bool has_postop_fusion = false; + std::vector post_ops; + + if (node.OpType() == "MatMulIntegerPostOps") { + has_postop_fusion = true; + post_ops = node.GetPostOps(); + + int binary_count = 0; + // Check we have enough inputs for MatMul and the binary post ops + for (size_t i = 0; i < post_ops.size(); ++i) { + if (binary_ops.count(post_ops[i]) != 0) { + assert(node.Input(IN_BINARY_0 + binary_count).Exists()); + binary_count++; + } + } + } + auto src_dims = sp.GetMemory(node.Input(IN_A)).get_desc().dims(); auto weights_dims = sp.GetMemory(node.Input(IN_B)).get_desc().dims(); @@ -52,6 +77,67 @@ void DnnlMatMulInteger::CreatePrimitive(DnnlSubgraphPrimitive& sp, DnnlNode& nod matmul_attr.set_zero_points(DNNL_ARG_WEIGHTS, /* mask */ 0, {DNNL_RUNTIME_S32_VAL}); } + /* + create a post op binary with possible unsqueezing in order to make sure onednn properly broadcast + current limitation + 1. is no unsqueeze for matmul output as it is not exposed due to post op fusion + 2. the third input has to be reordered to plain format + (eg, no memory format propagation if the third input is internal to subgraph) + 3. adding 1s to front (unsqueeze/expand) in logical dims would possibly fail if + physical layout is not plain format + */ + if (has_postop_fusion) { + int binary_count = 0; + dnnl::post_ops ops; + for (size_t i = 0; i < post_ops.size(); ++i) { + dnnl::algorithm algo = dnnl_util::OrtOperatorToDnnlAlgorithm(post_ops[i]); + // Handle Binary post ops including the input memory + if (binary_ops.count(post_ops[i]) != 0) { + auto ori_binary_md = sp.GetMemory(node.Input(IN_BINARY_0 + binary_count).Name()).get_desc(); + auto ori_binary_dims = ori_binary_md.dims(); + auto binary_mem_dims = ori_binary_dims; + if (ori_binary_dims.size() != output_shape.size()) { + if (ori_binary_dims.size() > output_shape.size()) { + ORT_THROW("add fusion with matmul output broadcasting by unsqueezing is not supported"); + } + // expand the input (from the binary op) if needed to support broadcasting + while (binary_mem_dims.size() < output_shape.size()) { + binary_mem_dims.insert(binary_mem_dims.begin(), 1); + } + } + + // expand the dims by 1s (should always be possible) + // will throw exception if not possible + auto binary_md = ori_binary_md.reshape(binary_mem_dims); + // Possible improvment: use format any to choose the best layout + ops.append_binary(algo, binary_md); + binary_count++; + // Handle Elementwise post ops. Some of these require obtaining an 'alpha' attribute + } else if (elementwise_ops.count(post_ops[i]) != 0) { + float post_op_alpha = 0.0; + switch (algo) { + case dnnl::algorithm::eltwise_relu: { + // Need to check operator since both Relu and LeakyRelu are covered by algorithm::eltwise_relu + if (post_ops[i] == "LeakyRelu") { + post_op_alpha = GetFloatAttr(node, "alpha", /*default_alpha*/ 0.01f); + } else { + post_op_alpha = 0.0; + } + break; + } + case dnnl::algorithm::eltwise_elu: { + post_op_alpha = GetFloatAttr(node, "alpha", /*default_alpha*/ 1.0f); + break; + } + default: + post_op_alpha = 0.0; + } + ops.append_eltwise(1.0f, algo, post_op_alpha, 0.0f); + } + } + matmul_attr.set_post_ops(ops); + } + auto matmul_d = dnnl::matmul::desc(src_md, weights_md, dst_md); auto matmul_pd = dnnl::matmul::primitive_desc(matmul_d, matmul_attr, eng); @@ -79,10 +165,33 @@ void DnnlMatMulInteger::CreatePrimitive(DnnlSubgraphPrimitive& sp, DnnlNode& nod mem_map[DNNL_ARG_ATTR_ZERO_POINTS | DNNL_ARG_WEIGHTS] = zp_B_mem_s32; } - sp.AddPrimitive(matmul_prim, mem_map); + if (has_postop_fusion) { + // add to memory map for extra binary inputs + int binary_count = 0; + for (size_t i = 0; i < post_ops.size(); ++i) { + if (binary_ops.count(post_ops[i]) != 0) { + dnnl::algorithm algo; + dnnl::memory::desc binary_mem_desc; + matmul_pd.get_primitive_attr().get_post_ops().get_params_binary(static_cast(i), algo, binary_mem_desc); + auto binary_post_op_mem = sp.GetMemoryAndReshape(node.Input(IN_BINARY_0 + binary_count), binary_mem_desc, eng); + mem_map[DNNL_ARG_ATTR_MULTIPLE_POST_OP(static_cast(i)) | DNNL_ARG_SRC_1] = binary_post_op_mem; + binary_count++; + } + } + } + + sp.AddPrimitive(matmul_prim, mem_map /*, {DNNL_ARG_SRC, DNNL_ARG_WEIGHTS, DNNL_ARG_DST}*/); sp.SetMemory(node.Output(OUT_Y), matmul_dst_mem); } +float DnnlMatMulInteger::GetFloatAttr(DnnlNode& node, std::string attr_name, float default_value) { + auto attr = node.Attributes().find(attr_name); + if (attr != node.Attributes().end()) { + return attr->second().f(); + } + return default_value; +} + } // namespace ort_dnnl } // namespace onnxruntime diff --git a/onnxruntime/core/providers/dnnl/subgraph/dnnl_matmul_integer.h b/onnxruntime/core/providers/dnnl/subgraph/dnnl_matmul_integer.h index 7f8239c35a..90f49a136d 100644 --- a/onnxruntime/core/providers/dnnl/subgraph/dnnl_matmul_integer.h +++ b/onnxruntime/core/providers/dnnl/subgraph/dnnl_matmul_integer.h @@ -5,6 +5,8 @@ #include "dnnl_subgraph.h" #include "dnnl_subgraph_primitive.h" +#include + namespace onnxruntime { namespace ort_dnnl { @@ -14,13 +16,17 @@ class DnnlMatMulInteger { IN_A = 0, IN_B = 1, IN_A_ZERO_POINT = 2, - IN_B_ZERO_POINT = 3 + IN_B_ZERO_POINT = 3, + IN_BINARY_0 = 4 // the first binary input due to matmul + binary fusion }; enum OutputTensors : int { OUT_Y = 0 }; DnnlMatMulInteger(); void CreatePrimitive(DnnlSubgraphPrimitive& sp, DnnlNode& node); + + private: + float GetFloatAttr(DnnlNode& node, std::string attr_name, float default_value); }; } // namespace ort_dnnl diff --git a/onnxruntime/core/providers/dnnl/subgraph/dnnl_subgraph_primitive.cc b/onnxruntime/core/providers/dnnl/subgraph/dnnl_subgraph_primitive.cc index e27b61f766..4c85fa9790 100644 --- a/onnxruntime/core/providers/dnnl/subgraph/dnnl_subgraph_primitive.cc +++ b/onnxruntime/core/providers/dnnl/subgraph/dnnl_subgraph_primitive.cc @@ -200,7 +200,7 @@ void DnnlSubgraphPrimitive::AddKernels() { // https://github.com/microsoft/onnxruntime/blob/master/docs/ContribOperators.md#com.microsoft.FusedMatMul } else if (node.OpType() == "MatMul" || node.OpType() == "MatMulPostOps" || node.OpType() == "FusedMatMul") { DnnlMatMul().CreatePrimitive(*this, node); - } else if (node.OpType() == "MatMulInteger") { + } else if (node.OpType() == "MatMulInteger" || node.OpType() == "MatMulIntegerPostOps") { DnnlMatMulInteger().CreatePrimitive(*this, node); } else if (pool_ops.count(node.OpType())) { DnnlPool().CreatePrimitive(*this, node); diff --git a/onnxruntime/core/providers/dnnl/subgraph/dnnl_subgraph_transformer.cc b/onnxruntime/core/providers/dnnl/subgraph/dnnl_subgraph_transformer.cc index 8394a0e05d..328fe29f93 100644 --- a/onnxruntime/core/providers/dnnl/subgraph/dnnl_subgraph_transformer.cc +++ b/onnxruntime/core/providers/dnnl/subgraph/dnnl_subgraph_transformer.cc @@ -12,6 +12,7 @@ #include #include #include +#include namespace onnxruntime { namespace ort_dnnl { @@ -23,6 +24,7 @@ void DnnlGraphTransformer::Apply(DnnlSubgraph& subgraph, const onnxruntime::Grap Gelu(subgraph, onnx_subgraph_viewer); FastGelu(subgraph, onnx_subgraph_viewer); RemoveMatMulIntegerZP(subgraph, onnx_subgraph_viewer); + MatMulIntegerBinaryEltwise(subgraph); } //resolve a fusion by replacing old_indices nodes with a new_node @@ -687,17 +689,12 @@ void DnnlGraphTransformer::ConvRelu(DnnlSubgraph& subgraph) { } void DnnlGraphTransformer::MatMulBinaryEltwise(DnnlSubgraph& subgraph) { - std::unordered_set binary_ops = {"Add", "Div", "Mul" , "Sub"}; - std::unordered_set elementwise_ops = {"Abs", "Elu", "Exp", "LeakyRelu", "Log", "Relu", - "Round", "Sigmoid", "Softplus", "Sqrt", "Tanh"}; - static int fused_index = 0; size_t max_index = subgraph.GetMaxNodeIndex(); for (size_t index = 0; index < max_index; index++) { std::vector matmul_binary_eltwize_indices = {}; auto dnnl_node = subgraph.GetDnnlNode(index); auto attr_node = dnnl_node; - bool attribute_flag = false; if (dnnl_node == nullptr || dnnl_node->OpType() != "MatMul") { continue; @@ -709,50 +706,11 @@ void DnnlGraphTransformer::MatMulBinaryEltwise(DnnlSubgraph& subgraph) { auto fused_node_inputs = dnnl_node->Inputs(); matmul_binary_eltwize_indices.push_back(dnnl_node->Index()); - // Upto 32 post-ops are supported. Since the initial MatMul is not a - // post op we check for MAX_POST_OP_COUNT + 1 - const size_t MAX_POST_OP_COUNT = 32; - - while (matmul_binary_eltwize_indices.size() < MAX_POST_OP_COUNT + 1) { - if (!IsNodeFusable(subgraph, dnnl_node)) { - break; - } - - auto next_dnnl_node = dnnl_node->Output(0).GetConsumers()[0].GetNode(); - if (next_dnnl_node == nullptr) { - break; - } - - auto next_type = next_dnnl_node->OpType(); - bool is_binary_op = !(binary_ops.count(next_type) == 0); - bool is_eltwise_op = !(elementwise_ops.count(next_type) == 0); - if (!is_binary_op && !is_eltwise_op) { - break; - } - - if (is_binary_op) { - if (dnnl_node->Output(0).Name() == next_dnnl_node->Inputs()[0]->Name()) { - fused_node_inputs.push_back(next_dnnl_node->Inputs()[1]); - } else { - if (next_dnnl_node->OpType() == "Div" || next_dnnl_node->OpType() == "Sub") { - break; - } - fused_node_inputs.push_back(next_dnnl_node->Inputs()[0]); - } - } else if (is_eltwise_op) { - // We only support a single node with an "alpha" attribute. If we see Elu or - // LeakyRelu set attr_node and attribute_flag. If the attribute_flag is set break - // out of the while loop looking for additional post ops. Since the next op cannot - // be part of the postop fusion. - if (next_dnnl_node->OpType() == "Elu" || next_dnnl_node->OpType() == "LeakyRelu") { - if (attribute_flag) break; - attr_node = next_dnnl_node; - attribute_flag = true; - } - } - dnnl_node = next_dnnl_node; - matmul_binary_eltwize_indices.push_back(dnnl_node->Index()); - } + dnnl_node = FuseBinaryEltwisePostOps(subgraph, + dnnl_node, + matmul_binary_eltwize_indices, + fused_node_inputs, + attr_node); if (!(matmul_binary_eltwize_indices.size() > 1)) { matmul_binary_eltwize_indices.clear(); @@ -845,5 +803,137 @@ void DnnlGraphTransformer::RemoveMatMulIntegerZP(DnnlSubgraph& subgraph, const o } } +void DnnlGraphTransformer::MatMulIntegerBinaryEltwise(DnnlSubgraph& subgraph) { + static int fused_index = 0; + size_t max_index = subgraph.GetMaxNodeIndex(); + for (size_t index = 0; index < max_index; index++) { + std::vector matmul_binary_eltwize_indices = {}; + auto dnnl_node = subgraph.GetDnnlNode(index); + auto attr_node = dnnl_node; + + if (dnnl_node == nullptr || dnnl_node->OpType() != "MatMulInteger") { + continue; + } + + if (!IsNodeFusable(subgraph, dnnl_node)) { + continue; + } + + auto fused_node_inputs = dnnl_node->Inputs(); + matmul_binary_eltwize_indices.push_back(dnnl_node->Index()); + + auto next_dnnl_node = dnnl_node->Output(0).GetConsumers()[0].GetNode(); + if (next_dnnl_node == nullptr) { + continue; + } + if (next_dnnl_node->OpType() != "Cast") { + continue; + } + + if (!IsNodeFusable(subgraph, next_dnnl_node)) { + continue; + } + + dnnl_node = next_dnnl_node; + matmul_binary_eltwize_indices.push_back(dnnl_node->Index()); + + dnnl_node = FuseBinaryEltwisePostOps(subgraph, + dnnl_node, + matmul_binary_eltwize_indices, + fused_node_inputs, + attr_node); + + if (!(matmul_binary_eltwize_indices.size() > 1)) { + matmul_binary_eltwize_indices.clear(); + continue; + } + + //construct new node + auto fused_node = std::make_unique(); + fused_node->Name() = "MatMulIntegerPostOps_fusion" + std::to_string(fused_index++); + std::string fused_node_name = "MatMulIntegerPostOps"; + for (size_t i : matmul_binary_eltwize_indices) { + if (subgraph.GetDnnlNode(i)->OpType() != "MatMulInteger" && subgraph.GetDnnlNode(i)->OpType() != "Cast") { + fused_node->AppendPostOp(subgraph.GetDnnlNode(i)->OpType()); + } + } + fused_node->OpType() = fused_node_name; + fused_node->Inputs() = fused_node_inputs; + fused_node->Outputs() = {dnnl_node->Outputs()[0]}; + + fused_node->Attributes().insert(attr_node->Attributes()); + + if (debug_log_) { + std::stringstream ss; + for (size_t i : matmul_binary_eltwize_indices) { + ss << subgraph.GetDnnlNode(i)->OpType() << "[" << subgraph.GetDnnlNode(i)->Name() << "] "; + } + LOGS_DEFAULT(ERROR) << fused_node->OpType() << "[" << fused_node->Name() << "] fusion of " << ss.str(); + } + // insert new node, remove original nodes, connect new edges + ResolveFusion(subgraph, matmul_binary_eltwize_indices, std::move(fused_node)); + } +} + +DnnlNode* DnnlGraphTransformer::FuseBinaryEltwisePostOps(DnnlSubgraph& subgraph, + DnnlNode* node, + std::vector& indices, + std::vector& fused_node_inputs, + DnnlNode*& attr_node) { + std::unordered_set binary_ops = {"Add", "Div", "Mul", "Sub"}; + std::unordered_set elementwise_ops = {"Abs", "Elu", "Exp", "LeakyRelu", "Log", "Relu", + "Round", "Sigmoid", "Softplus", "Sqrt", "Tanh"}; + // Upto 32 post-ops are supported by oneDNN framework. + const size_t MAX_POST_OP_COUNT = 32; + bool attribute_flag = false; + auto dnnl_node = node; + size_t post_op_count = 0; + while (post_op_count < MAX_POST_OP_COUNT) { + if (!IsNodeFusable(subgraph, dnnl_node)) { + break; + } + + auto next_dnnl_node = dnnl_node->Output(0).GetConsumers()[0].GetNode(); + if (next_dnnl_node == nullptr) { + break; + } + + auto next_type = next_dnnl_node->OpType(); + bool is_binary_op = !(binary_ops.count(next_type) == 0); + bool is_eltwise_op = !(elementwise_ops.count(next_type) == 0); + if (!is_binary_op && !is_eltwise_op) { + break; + } + + if (is_binary_op) { + if (dnnl_node->Output(0).Name() == next_dnnl_node->Inputs()[0]->Name()) { + fused_node_inputs.push_back(next_dnnl_node->Inputs()[1]); + } else { + // oneDNN can only fuse binary post op for the second input. Due to the fact + // that division and subraction are not associative we can not fuse them if the + // input for that node is the first input. + if (next_dnnl_node->OpType() == "Div" || next_dnnl_node->OpType() == "Sub") { + break; + } + fused_node_inputs.push_back(next_dnnl_node->Inputs()[0]); + } + } else if (is_eltwise_op) { + // We only support a single node with an "alpha" attribute. If we see Elu or + // LeakyRelu set attr_node and attribute_flag. If the attribute_flag is set break + // out of the while loop looking for additional post ops. Since the next op cannot + // be part of the postop fusion. + if (next_dnnl_node->OpType() == "Elu" || next_dnnl_node->OpType() == "LeakyRelu") { + if (attribute_flag) break; + attr_node = next_dnnl_node; + attribute_flag = true; + } + } + dnnl_node = next_dnnl_node; + indices.push_back(dnnl_node->Index()); + post_op_count++; + } + return dnnl_node; +} + } // namespace ort_dnnl } // namespace onnxruntime diff --git a/onnxruntime/core/providers/dnnl/subgraph/dnnl_subgraph_transformer.h b/onnxruntime/core/providers/dnnl/subgraph/dnnl_subgraph_transformer.h index e81c447344..56163025a1 100644 --- a/onnxruntime/core/providers/dnnl/subgraph/dnnl_subgraph_transformer.h +++ b/onnxruntime/core/providers/dnnl/subgraph/dnnl_subgraph_transformer.h @@ -29,8 +29,18 @@ class DnnlGraphTransformer { bool IsInitilizedWithExpectedValue(const onnxruntime::GraphViewer& onnx_subgraph_viewer, DnnlTensor& input_arg, float expected_value); void ConvRelu(DnnlSubgraph& subgraph); void MatMulBinaryEltwise(DnnlSubgraph& subgraph); - void MatMulAdd(DnnlSubgraph& subgraph); void RemoveMatMulIntegerZP(DnnlSubgraph& subgraph, const onnxruntime::GraphViewer& onnx_subgraph_viewer); + void MatMulIntegerBinaryEltwise(DnnlSubgraph& subgraph); + // Function used to identify and fuse post ops + // + // @param[in] subgraph the DnnlSubgrapy that we are searching for possible fusions + // @param[in] node is the first node to check if it contains a binary or an elementwise op + // @param[in/out] indicies list of all the indicies for the nodes that will be fused + // @param[in/out] fused_node_inputs list of all the inputs that will be part of the fused node + // @param[in/out] attr_node this node contains the attributes that will be passed onto the final fused node + // + // @return a pointer to the node after the last identified binary/elementwise fusion + DnnlNode* FuseBinaryEltwisePostOps(DnnlSubgraph& subgraph, DnnlNode* node, std::vector& indices, std::vector& fused_node_inputs, DnnlNode*& attr_node); // This function checks a few things // - the node in question has a single output // - The output of the node is only consumed by a one other node diff --git a/onnxruntime/test/providers/dnnl/transformer/matmulinteger_post_op_transform_test.cc b/onnxruntime/test/providers/dnnl/transformer/matmulinteger_post_op_transform_test.cc new file mode 100644 index 0000000000..361f433cba --- /dev/null +++ b/onnxruntime/test/providers/dnnl/transformer/matmulinteger_post_op_transform_test.cc @@ -0,0 +1,1388 @@ +// Copyright (c) Intel Corporation. All rights reserved. +// Licensed under the MIT License. + +#include "gtest/gtest.h" +#include "gmock/gmock.h" +#include "core/framework/session_state.h" +#include "core/providers/cpu/controlflow/if.h" +#include "test/providers/provider_test_utils.h" +#include "core/session/inference_session.h" + +#include "test/util/include/default_providers.h" + +/* +* The tests validate that if a fusion occures the expected output matches +* the output of each graph if they had not be done separatly. +* +* Unfortantly there is no hook to actually check that the fussion occurred +* other than inspecting debug logs. +* +* The tests use patterns that we have seen in actual models during testing. +* +* A few tests are there simply to validate the limits of the MatMulInteger + +* post op fusion. The max number of ops fusable are 32 post ops so we exced +* that number and make sure the generated fusion is not a broken graph. +* +* A current implementation limitation is that we can only support a single instance +* ops that use the 'alpha' attribute. We purposly test models that have more than +* one instance of LeakRelu or Elu to make sure the graph generated is not broken. +* +* Most numbers for the tests were randomly generated and calculated using +* python numpy library. +* +* // fusions seen in most bert models (bert_base_int8, DistilBert_int8, MobileBert_int8) +* MatMulInteger_Cast +* MatMulInteger_Cast_Mul +* MatMulInteger_Cast_Mul_Add +* MatMulInteger_Cast_Mul_Add_Add +* MatMulInteger_Cast_Mul_Add_Mul_Add +* MatMulInteger_Cast_Mul_Add_Add_Mul_Add +* MatMulInteger_Cast_Mul_Add_Relu +* +* // testing other possible combinations that are not seen in models +* // Non-associative ops +* // not all layouts can be fused +* matmul_div_add_0 +* matmul_div_add_1 +* matmul_div_sub_0 +* matmul_div_sub_1 +* +* // Max number of post ops supported by OneDNN is 32. +* // Test that the post-op fusion does not fail when that value is exceded +* matmul_36_post_ops +* +* // test fusion of remaining eltwise ops +* matmul_add_abs_mul +* matmul_add_exp_mul +* matmul_add_abs_log_mul +* matmul_add_round_mul +* matmul_add_softplus_mul +* matmul_add_abs_sqrt_mul +* matmul_add_tanh_mul +* +* //element wise functions that take alpha attribute +* matmul_add_leakyrelu_mul +* matmul_add_leakyrelu_mul_leakyrelu +* matmul_add_elu_mul_leakyrelu +*/ +namespace onnxruntime { +namespace test { +// Although these tests should not fail when run on other EPs there +// is not much gained by running these on other EPs +#ifdef USE_DNNL +class Dnnl_MatMulInteger_to_float_PostOpTester : public OpTester { + public: + explicit Dnnl_MatMulInteger_to_float_PostOpTester(int opset_version = 10) + : OpTester("MatMulInteger", opset_version) { + } + + protected: + void AddNodes(onnxruntime::Graph& graph, + std::vector& graph_input_defs, + std::vector& graph_output_defs, + std::vector>& /*add_attribute_funcs*/) override { + ASSERT_EQ(graph_input_defs.size(), 4u); + ASSERT_EQ(graph_output_defs.size(), 1u); + + NodeArg* a = graph_input_defs[0]; + NodeArg* b = graph_input_defs[1]; + NodeArg* a_zp = graph_input_defs[2]; + NodeArg* b_zp = graph_input_defs[3]; + + NodeArg* y = graph_output_defs[0]; + + // internal NodeArgs + // The output from MatMulInteger is int32 but this is only an internal Node + // so we make the int32 TypeProto manually to match the same shape as the + // output shape. + const onnx::TensorShapeProto* output_shape = y->Shape(); + onnx::TypeProto output_int32; + output_int32.mutable_tensor_type()->set_elem_type(onnx::TensorProto_DataType_INT32); + for (int i = 0; i < output_shape->dim_size(); ++i) { + auto dim = output_int32.mutable_tensor_type()->mutable_shape()->add_dim(); + *dim = output_shape->dim(i); + } + auto& matmul_out = graph.GetOrCreateNodeArg("matmul_out", &output_int32); + + graph.AddNode("matmul1", "MatMulInteger", "", {a, b, a_zp, b_zp}, {&matmul_out}); + auto& cast_node = graph.AddNode("cast1", "Cast", "", {&matmul_out}, {y}); + cast_node.AddAttribute("to", int64_t{ONNX_NAMESPACE::TensorProto_DataType_FLOAT}); + } +}; + +TEST(DnnlMatMulIntegerFusion, MatMulInteger_Cast_to_float) { + Dnnl_MatMulInteger_to_float_PostOpTester test; + test.AddInput("a", {2, 4}, + {-3, 7, 5, -6, + 4, -5, 8, 7}); + test.AddInput("b", {4, 4}, + {5, -3, 7, 8, + -6, -8, -3, 6, + 7, 9, 9, -5, + 8, 7, -6, 7}); + test.AddInput("a_zp", {}, {5}); + test.AddInput("b_zp", {}, {5}); + test.AddOutput("Y", {2, 4}, + {-55.0f, 16.0f, 89.0f, -44.0f, + 122.0f, 154.0f, 68.0f, -39.0f}); + + test.Run(); +} + +class Dnnl_MatMulInteger_Mul_PostOpTester : public OpTester { + public: + explicit Dnnl_MatMulInteger_Mul_PostOpTester(int opset_version = 10) + : OpTester("MatMulInteger", opset_version) { + } + + protected: + void AddNodes(onnxruntime::Graph& graph, + std::vector& graph_input_defs, + std::vector& graph_output_defs, + std::vector>& /*add_attribute_funcs*/) override { + ASSERT_EQ(graph_input_defs.size(), 5u); + ASSERT_EQ(graph_output_defs.size(), 1u); + + NodeArg* a = graph_input_defs[0]; + NodeArg* b = graph_input_defs[1]; + NodeArg* a_zp = graph_input_defs[2]; + NodeArg* b_zp = graph_input_defs[3]; + + NodeArg* c1 = graph_input_defs[4]; + NodeArg* y = graph_output_defs[0]; + + // internal NodeArgs + // The output from MatMulInteger is int32 but this is only an internal Node + // so we make the int32 TypeProto manually to match the same shape as the + // output shape. + const onnx::TensorShapeProto* output_shape = y->Shape(); + onnx::TypeProto output_int32; + output_int32.mutable_tensor_type()->set_elem_type(onnx::TensorProto_DataType_INT32); + for (int i = 0; i < output_shape->dim_size(); ++i) { + auto dim = output_int32.mutable_tensor_type()->mutable_shape()->add_dim(); + *dim = output_shape->dim(i); + } + auto& matmul_out = graph.GetOrCreateNodeArg("matmul_out", &output_int32); + auto& cast_out = graph.GetOrCreateNodeArg("cast_out", y->TypeAsProto()); + + + graph.AddNode("matmul1", "MatMulInteger", "", {a, b, a_zp, b_zp}, {&matmul_out}); + auto& cast_node = graph.AddNode("cast1", "Cast", "", {&matmul_out}, {&cast_out}); + cast_node.AddAttribute("to", int64_t{ONNX_NAMESPACE::TensorProto_DataType_FLOAT}); + graph.AddNode("mul1", "Mul", "", {&cast_out, c1}, {y}); + } +}; + +TEST(DnnlMatMulIntegerFusion, MatMulInteger_Cast_Mul) { + Dnnl_MatMulInteger_Mul_PostOpTester test; + test.AddInput("a", {2, 4}, + {-3, 7, 5, -6, + 4, -5, 8, 7}); + test.AddInput("b", {4, 4}, + {5, -3, 7, 8, + -6, -8, -3, 6, + 7, 9, 9, -5, + 8, 7, -6, 7}); + test.AddInput("a_zp", {}, {5}); + test.AddInput("b_zp", {}, {5}); + test.AddInput("c1", {2, 4}, + {0.5f, 1.5f, 2.0f, 2.5f, + -0.5f, -1.5f, -2.0f, -2.5f}); + test.AddOutput("Y", {2, 4}, + {-27.5f, 24.0f, 178.0f, -110.0f, + -61.0f, -231.0f, -136.0f, 97.5f}); + + test.Run(); +} + +// MatMulInteger_Cast_Mul_Add +class Dnnl_MatMulInteger_Cast_Mul_Add_PostOpTester : public OpTester { + public: + explicit Dnnl_MatMulInteger_Cast_Mul_Add_PostOpTester(int opset_version = 10) + : OpTester("MatMulInteger", opset_version) { + } + + protected: + void AddNodes(onnxruntime::Graph& graph, + std::vector& graph_input_defs, + std::vector& graph_output_defs, + std::vector>& /*add_attribute_funcs*/) override { + ASSERT_EQ(graph_input_defs.size(), 6u); + ASSERT_EQ(graph_output_defs.size(), 1u); + + NodeArg* a = graph_input_defs[0]; + NodeArg* b = graph_input_defs[1]; + NodeArg* a_zp = graph_input_defs[2]; + NodeArg* b_zp = graph_input_defs[3]; + NodeArg* c1 = graph_input_defs[4]; + NodeArg* c2 = graph_input_defs[5]; + + NodeArg* y = graph_output_defs[0]; + + // internal NodeArgs + // The output from MatMulInteger is int32 but this is only an internal Node + // so we make the int32 TypeProto manually to match the same shape as the + // output shape. + const onnx::TensorShapeProto* output_shape = y->Shape(); + onnx::TypeProto output_int32; + output_int32.mutable_tensor_type()->set_elem_type(onnx::TensorProto_DataType_INT32); + for (int i = 0; i < output_shape->dim_size(); ++i) { + auto dim = output_int32.mutable_tensor_type()->mutable_shape()->add_dim(); + *dim = output_shape->dim(i); + } + auto& matmul_out = graph.GetOrCreateNodeArg("matmul_out", &output_int32); + auto& cast_out = graph.GetOrCreateNodeArg("cast_out", y->TypeAsProto()); + auto& mul1_out = graph.GetOrCreateNodeArg("mul1_out", c2->TypeAsProto()); + + graph.AddNode("matmul1", "MatMulInteger", "", {a, b, a_zp, b_zp}, {&matmul_out}); + auto& cast_node = graph.AddNode("cast1", "Cast", "", {&matmul_out}, {&cast_out}); + cast_node.AddAttribute("to", int64_t{ONNX_NAMESPACE::TensorProto_DataType_FLOAT}); + graph.AddNode("mul1", "Mul", "", {&cast_out, c1}, {&mul1_out}); + graph.AddNode("add1", "Add", "", {&mul1_out, c2}, {y}); + } +}; + +TEST(DnnlMatMulIntegerFusion, MatMulInteger_Cast_Mul_Add) { + Dnnl_MatMulInteger_Cast_Mul_Add_PostOpTester test; + test.AddInput("a", {2, 4}, + {-3, 7, 5, -6, + 4, -5, 8, 7}); + test.AddInput("b", {4, 4}, + {5, -3, 7, 8, + -6, -8, -3, 6, + 7, 9, 9, -5, + 8, 7, -6, 7}); + test.AddInput("a_zp", {}, {5}); + test.AddInput("b_zp", {}, {5}); + test.AddInput("c1", {2, 4}, + {0.5f, 1.5f, 2.0f, 2.5f, + -0.5f, -1.5f, -2.0f, -2.5f}); + test.AddInput("c2", {2, 4}, + {0.2f, 1.4f, 2.6f, 2.8f, + -0.2f, -1.4f, -2.6f, -2.8f}); + test.AddOutput("Y", {2, 4}, + {-27.3f, 25.4f, 180.6f, -107.2f, + -61.2f, -232.4f, -138.6f, 94.7f}); + + test.Run(); +} +// MatMulInteger_Cast_Mul_Add_Add +// MatMulInteger_Cast_Mul_Add_Mul_Add +// MatMulInteger_Cast_Mul_Add_Add_Mul_Add +// MatMulInteger_Cast_Mul_Add_Relu +class Dnnl_MatMulInteger_Cast_Mul_Add_Relu_PostOpTester : public OpTester { + public: + explicit Dnnl_MatMulInteger_Cast_Mul_Add_Relu_PostOpTester(int opset_version = 10) + : OpTester("MatMulInteger", opset_version) { + } + + protected: + void AddNodes(onnxruntime::Graph& graph, + std::vector& graph_input_defs, + std::vector& graph_output_defs, + std::vector>& /*add_attribute_funcs*/) override { + ASSERT_EQ(graph_input_defs.size(), 6u); + ASSERT_EQ(graph_output_defs.size(), 1u); + + NodeArg* a = graph_input_defs[0]; + NodeArg* b = graph_input_defs[1]; + NodeArg* a_zp = graph_input_defs[2]; + NodeArg* b_zp = graph_input_defs[3]; + NodeArg* c1 = graph_input_defs[4]; + NodeArg* c2 = graph_input_defs[5]; + + NodeArg* y = graph_output_defs[0]; + + // internal NodeArgs + // The output from MatMulInteger is int32 but this is only an internal Node + // so we make the int32 TypeProto manually to match the same shape as the + // output shape. + const onnx::TensorShapeProto* output_shape = y->Shape(); + onnx::TypeProto output_int32; + output_int32.mutable_tensor_type()->set_elem_type(onnx::TensorProto_DataType_INT32); + for (int i = 0; i < output_shape->dim_size(); ++i) { + auto dim = output_int32.mutable_tensor_type()->mutable_shape()->add_dim(); + *dim = output_shape->dim(i); + } + auto& matmul_out = graph.GetOrCreateNodeArg("matmul_out", &output_int32); + auto& cast_out = graph.GetOrCreateNodeArg("cast_out", y->TypeAsProto()); + auto& mul1_out = graph.GetOrCreateNodeArg("mul1_out", c2->TypeAsProto()); + auto& add1_out = graph.GetOrCreateNodeArg("add1_out", c2->TypeAsProto()); + + graph.AddNode("matmul1", "MatMulInteger", "", {a, b, a_zp, b_zp}, {&matmul_out}); + auto& cast_node = graph.AddNode("cast1", "Cast", "", {&matmul_out}, {&cast_out}); + cast_node.AddAttribute("to", int64_t{ONNX_NAMESPACE::TensorProto_DataType_FLOAT}); + graph.AddNode("mul1", "Mul", "", {&cast_out, c1}, {&mul1_out}); + graph.AddNode("add1", "Add", "", {&mul1_out, c2}, {&add1_out}); + graph.AddNode("relu1", "Relu", "", {&add1_out}, {y}); + } +}; + +TEST(DnnlMatMulIntegerFusion, MatMulInteger_Cast_Mul_Add_Relu) { + Dnnl_MatMulInteger_Cast_Mul_Add_Relu_PostOpTester test; + test.AddInput("a", {2, 4}, + {-3, 7, 5, -6, + 4, -5, 8, 7}); + test.AddInput("b", {4, 4}, + {5, -3, 7, 8, + -6, -8, -3, 6, + 7, 9, 9, -5, + 8, 7, -6, 7}); + test.AddInput("a_zp", {}, {5}); + test.AddInput("b_zp", {}, {5}); + test.AddInput("c1", {2, 4}, + {0.5f, 1.5f, 2.0f, 2.5f, + -0.5f, -1.5f, -2.0f, -2.5f}); + test.AddInput("c2", {2, 4}, + {0.2f, 1.4f, 2.6f, 2.8f, + -0.2f, -1.4f, -2.6f, -2.8f}); + test.AddOutput("Y", {2, 4}, + {0.0f, 25.4f, 180.6f, 0.0f, + 0.0f, 0.0f, 0.0f, 94.7f}); + + test.Run(); +} + +class Dnnl_MatMulInteger_Cast_Div1_PostOpTester : public OpTester { + public: + explicit Dnnl_MatMulInteger_Cast_Div1_PostOpTester(int opset_version = 10) + : OpTester("MatMulInteger", opset_version) { + } + + protected: + void AddNodes(onnxruntime::Graph& graph, + std::vector& graph_input_defs, + std::vector& graph_output_defs, + std::vector>& /*add_attribute_funcs*/) override { + ASSERT_EQ(graph_input_defs.size(), 5u); + ASSERT_EQ(graph_output_defs.size(), 1u); + + NodeArg* a = graph_input_defs[0]; + NodeArg* b = graph_input_defs[1]; + NodeArg* a_zp = graph_input_defs[2]; + NodeArg* b_zp = graph_input_defs[3]; + + NodeArg* c1 = graph_input_defs[4]; + NodeArg* y = graph_output_defs[0]; + + const onnx::TensorShapeProto* output_shape = y->Shape(); + onnx::TypeProto output_int32; + output_int32.mutable_tensor_type()->set_elem_type(onnx::TensorProto_DataType_INT32); + for (int i = 0; i < output_shape->dim_size(); ++i) { + auto dim = output_int32.mutable_tensor_type()->mutable_shape()->add_dim(); + *dim = output_shape->dim(i); + } + auto& matmul_out = graph.GetOrCreateNodeArg("matmul_out", &output_int32); + auto& cast_out = graph.GetOrCreateNodeArg("cast_out", y->TypeAsProto()); + + graph.AddNode("matmul1", "MatMulInteger", "", {a, b, a_zp, b_zp}, {&matmul_out}); + auto& cast_node = graph.AddNode("cast1", "Cast", "", {&matmul_out}, {&cast_out}); + cast_node.AddAttribute("to", int64_t{ONNX_NAMESPACE::TensorProto_DataType_FLOAT}); + graph.AddNode("div1", "Div", "", {&cast_out, c1}, {y}); + } +}; + +TEST(DnnlMatMulIntegerFusion, MatMulInteger_Cast_Div1) { + Dnnl_MatMulInteger_Cast_Div1_PostOpTester test; + test.AddInput("a", {2, 4}, + {-3, 7, 5, -6, + 4, -5, 8, 7}); + test.AddInput("b", {4, 4}, + {5, -3, 7, 8, + -6, -8, -3, 6, + 7, 9, 9, -5, + 8, 7, -6, 7}); + test.AddInput("a_zp", {}, {5}); + test.AddInput("b_zp", {}, {5}); + test.AddInput("c1", {2, 4}, + {0.5f, 1.5f, 2.0f, 2.5f, + -0.5f, -1.5f, -2.0f, -2.5f}); + test.AddOutput("Y", {2, 4}, + {-110.0f, 10.666667f, 44.5f, -17.6f, + -244.0f, -102.666664f, -34.0f, 15.6f}); + + test.Run(); +} + +class Dnnl_MatMulInteger_Cast_Div2_PostOpTester : public OpTester { + public: + explicit Dnnl_MatMulInteger_Cast_Div2_PostOpTester(int opset_version = 10) + : OpTester("MatMulInteger", opset_version) { + } + + protected: + void AddNodes(onnxruntime::Graph& graph, + std::vector& graph_input_defs, + std::vector& graph_output_defs, + std::vector>& /*add_attribute_funcs*/) override { + ASSERT_EQ(graph_input_defs.size(), 5u); + ASSERT_EQ(graph_output_defs.size(), 1u); + + NodeArg* a = graph_input_defs[0]; + NodeArg* b = graph_input_defs[1]; + NodeArg* a_zp = graph_input_defs[2]; + NodeArg* b_zp = graph_input_defs[3]; + + NodeArg* c1 = graph_input_defs[4]; + NodeArg* y = graph_output_defs[0]; + + const onnx::TensorShapeProto* output_shape = y->Shape(); + onnx::TypeProto output_int32; + output_int32.mutable_tensor_type()->set_elem_type(onnx::TensorProto_DataType_INT32); + for (int i = 0; i < output_shape->dim_size(); ++i) { + auto dim = output_int32.mutable_tensor_type()->mutable_shape()->add_dim(); + *dim = output_shape->dim(i); + } + auto& matmul_out = graph.GetOrCreateNodeArg("matmul_out", &output_int32); + auto& cast_out = graph.GetOrCreateNodeArg("cast_out", y->TypeAsProto()); + + graph.AddNode("matmul1", "MatMulInteger", "", {a, b, a_zp, b_zp}, {&matmul_out}); + auto& cast_node = graph.AddNode("cast1", "Cast", "", {&matmul_out}, {&cast_out}); + cast_node.AddAttribute("to", int64_t{ONNX_NAMESPACE::TensorProto_DataType_FLOAT}); + graph.AddNode("div1", "Div", "", {c1, &cast_out}, {y}); + } +}; + +TEST(DnnlMatMulIntegerFusion, MatMulInteger_Cast_Div2) { + Dnnl_MatMulInteger_Cast_Div2_PostOpTester test; + test.AddInput("a", {2, 4}, + {-3, 7, 5, -6, + 4, -5, 8, 7}); + test.AddInput("b", {4, 4}, + {5, -3, 7, 8, + -6, -8, -3, 6, + 7, 9, 9, -5, + 8, 7, -6, 7}); + test.AddInput("a_zp", {}, {5}); + test.AddInput("b_zp", {}, {5}); + test.AddInput("c1", {2, 4}, + {0.5f, 1.5f, 2.0f, 2.5f, + -0.5f, -1.5f, -2.0f, -2.5f}); + test.AddOutput("Y", {2, 4}, + {-0.009090909f, 0.09375f, 0.02247191f, -0.056818184f, + -0.0040983604f, -0.0097402595f, -0.029411765f, 0.06410257f}); + + test.Run(); +} + +class Dnnl_MatMulInteger_Cast_Sub1_PostOpTester : public OpTester { + public: + explicit Dnnl_MatMulInteger_Cast_Sub1_PostOpTester(int opset_version = 10) + : OpTester("MatMulInteger", opset_version) { + } + + protected: + void AddNodes(onnxruntime::Graph& graph, + std::vector& graph_input_defs, + std::vector& graph_output_defs, + std::vector>& /*add_attribute_funcs*/) override { + ASSERT_EQ(graph_input_defs.size(), 5u); + ASSERT_EQ(graph_output_defs.size(), 1u); + + NodeArg* a = graph_input_defs[0]; + NodeArg* b = graph_input_defs[1]; + NodeArg* a_zp = graph_input_defs[2]; + NodeArg* b_zp = graph_input_defs[3]; + + NodeArg* c1 = graph_input_defs[4]; + NodeArg* y = graph_output_defs[0]; + + const onnx::TensorShapeProto* output_shape = y->Shape(); + onnx::TypeProto output_int32; + output_int32.mutable_tensor_type()->set_elem_type(onnx::TensorProto_DataType_INT32); + for (int i = 0; i < output_shape->dim_size(); ++i) { + auto dim = output_int32.mutable_tensor_type()->mutable_shape()->add_dim(); + *dim = output_shape->dim(i); + } + auto& matmul_out = graph.GetOrCreateNodeArg("matmul_out", &output_int32); + auto& cast_out = graph.GetOrCreateNodeArg("cast_out", y->TypeAsProto()); + + graph.AddNode("matmul1", "MatMulInteger", "", {a, b, a_zp, b_zp}, {&matmul_out}); + auto& cast_node = graph.AddNode("cast1", "Cast", "", {&matmul_out}, {&cast_out}); + cast_node.AddAttribute("to", int64_t{ONNX_NAMESPACE::TensorProto_DataType_FLOAT}); + graph.AddNode("div1", "Sub", "", {&cast_out, c1}, {y}); + } +}; + +TEST(DnnlMatMulIntegerFusion, MatMulInteger_Cast_Sub1) { + Dnnl_MatMulInteger_Cast_Sub1_PostOpTester test; + test.AddInput("a", {2, 4}, + {-3, 7, 5, -6, + 4, -5, 8, 7}); + test.AddInput("b", {4, 4}, + {5, -3, 7, 8, + -6, -8, -3, 6, + 7, 9, 9, -5, + 8, 7, -6, 7}); + test.AddInput("a_zp", {}, {5}); + test.AddInput("b_zp", {}, {5}); + test.AddInput("c1", {2, 4}, + {0.5f, 1.5f, 2.0f, 2.5f, + -0.5f, -1.5f, -2.0f, -2.5f}); + test.AddOutput("Y", {2, 4}, + {-55.5f, 14.5f, 87.0f, -46.5f, + 122.5f, 155.5f, 70.0f, -36.5f}); + + test.Run(); +} + +// Fusion not possible with this but it should still run and pass +class Dnnl_MatMulInteger_Cast_Sub2_PostOpTester : public OpTester { + public: + explicit Dnnl_MatMulInteger_Cast_Sub2_PostOpTester(int opset_version = 10) + : OpTester("MatMulInteger", opset_version) { + } + + protected: + void AddNodes(onnxruntime::Graph& graph, + std::vector& graph_input_defs, + std::vector& graph_output_defs, + std::vector>& /*add_attribute_funcs*/) override { + ASSERT_EQ(graph_input_defs.size(), 5u); + ASSERT_EQ(graph_output_defs.size(), 1u); + + NodeArg* a = graph_input_defs[0]; + NodeArg* b = graph_input_defs[1]; + NodeArg* a_zp = graph_input_defs[2]; + NodeArg* b_zp = graph_input_defs[3]; + + NodeArg* c1 = graph_input_defs[4]; + NodeArg* y = graph_output_defs[0]; + + const onnx::TensorShapeProto* output_shape = y->Shape(); + onnx::TypeProto output_int32; + output_int32.mutable_tensor_type()->set_elem_type(onnx::TensorProto_DataType_INT32); + for (int i = 0; i < output_shape->dim_size(); ++i) { + auto dim = output_int32.mutable_tensor_type()->mutable_shape()->add_dim(); + *dim = output_shape->dim(i); + } + auto& matmul_out = graph.GetOrCreateNodeArg("matmul_out", &output_int32); + auto& cast_out = graph.GetOrCreateNodeArg("cast_out", y->TypeAsProto()); + + graph.AddNode("matmul1", "MatMulInteger", "", {a, b, a_zp, b_zp}, {&matmul_out}); + auto& cast_node = graph.AddNode("cast1", "Cast", "", {&matmul_out}, {&cast_out}); + cast_node.AddAttribute("to", int64_t{ONNX_NAMESPACE::TensorProto_DataType_FLOAT}); + graph.AddNode("div1", "Sub", "", {c1, &cast_out}, {y}); + } +}; + +TEST(DnnlMatMulIntegerFusion, MatMulInteger_Cast_Sub2) { + Dnnl_MatMulInteger_Cast_Sub2_PostOpTester test; + test.AddInput("a", {2, 4}, + {-3, 7, 5, -6, + 4, -5, 8, 7}); + test.AddInput("b", {4, 4}, + {5, -3, 7, 8, + -6, -8, -3, 6, + 7, 9, 9, -5, + 8, 7, -6, 7}); + test.AddInput("a_zp", {}, {5}); + test.AddInput("b_zp", {}, {5}); + test.AddInput("c1", {2, 4}, + {0.5f, 1.5f, 2.0f, 2.5f, + -0.5f, -1.5f, -2.0f, -2.5f}); + test.AddOutput("Y", {2, 4}, + {55.5f, -14.5f, -87.0f, 46.5f, + -122.5f, -155.5f, -70.0f, 36.5f}); + + test.Run(); +} + +class Dnnl_MatMulInteger_Cast_Abs_PostOpTester : public OpTester { + public: + explicit Dnnl_MatMulInteger_Cast_Abs_PostOpTester(int opset_version = 10) + : OpTester("MatMulInteger", opset_version) { + } + + protected: + void AddNodes(onnxruntime::Graph& graph, + std::vector& graph_input_defs, + std::vector& graph_output_defs, + std::vector>& /*add_attribute_funcs*/) override { + ASSERT_EQ(graph_input_defs.size(), 4u); + ASSERT_EQ(graph_output_defs.size(), 1u); + + NodeArg* a = graph_input_defs[0]; + NodeArg* b = graph_input_defs[1]; + NodeArg* a_zp = graph_input_defs[2]; + NodeArg* b_zp = graph_input_defs[3]; + + NodeArg* y = graph_output_defs[0]; + + const onnx::TensorShapeProto* output_shape = y->Shape(); + onnx::TypeProto output_int32; + output_int32.mutable_tensor_type()->set_elem_type(onnx::TensorProto_DataType_INT32); + for (int i = 0; i < output_shape->dim_size(); ++i) { + auto dim = output_int32.mutable_tensor_type()->mutable_shape()->add_dim(); + *dim = output_shape->dim(i); + } + auto& matmul_out = graph.GetOrCreateNodeArg("matmul_out", &output_int32); + auto& cast_out = graph.GetOrCreateNodeArg("cast_out", y->TypeAsProto()); + + graph.AddNode("matmul1", "MatMulInteger", "", {a, b, a_zp, b_zp}, {&matmul_out}); + auto& cast_node = graph.AddNode("cast1", "Cast", "", {&matmul_out}, {&cast_out}); + cast_node.AddAttribute("to", int64_t{ONNX_NAMESPACE::TensorProto_DataType_FLOAT}); + graph.AddNode("abs1", "Abs", "", {&cast_out}, {y}); + } +}; + +TEST(DnnlMatMulIntegerFusion, MatMulInteger_Cast_Abs) { + Dnnl_MatMulInteger_Cast_Abs_PostOpTester test; + test.AddInput("a", {2, 4}, + {-3, 7, 5, -6, + 4, -5, 8, 7}); + test.AddInput("b", {4, 4}, + {5, -3, 7, 8, + -6, -8, -3, 6, + 7, 9, 9, -5, + 8, 7, -6, 7}); + test.AddInput("a_zp", {}, {5}); + test.AddInput("b_zp", {}, {5}); + test.AddOutput("Y", {2, 4}, + {55.0f, 16.0f, 89.0f, 44.0f, + 122.0f, 154.0f, 68.0f, 39.0f}); + + test.Run(); +} + +class Dnnl_MatMulInteger_Cast_Elu_PostOpTester : public OpTester { + public: + explicit Dnnl_MatMulInteger_Cast_Elu_PostOpTester(int opset_version = 10) + : OpTester("MatMulInteger", opset_version) { + } + + protected: + void AddNodes(onnxruntime::Graph& graph, + std::vector& graph_input_defs, + std::vector& graph_output_defs, + std::vector>& /*add_attribute_funcs*/) override { + ASSERT_EQ(graph_input_defs.size(), 4u); + ASSERT_EQ(graph_output_defs.size(), 1u); + + NodeArg* a = graph_input_defs[0]; + NodeArg* b = graph_input_defs[1]; + NodeArg* a_zp = graph_input_defs[2]; + NodeArg* b_zp = graph_input_defs[3]; + + NodeArg* y = graph_output_defs[0]; + + const onnx::TensorShapeProto* output_shape = y->Shape(); + onnx::TypeProto output_int32; + output_int32.mutable_tensor_type()->set_elem_type(onnx::TensorProto_DataType_INT32); + for (int i = 0; i < output_shape->dim_size(); ++i) { + auto dim = output_int32.mutable_tensor_type()->mutable_shape()->add_dim(); + *dim = output_shape->dim(i); + } + auto& matmul_out = graph.GetOrCreateNodeArg("matmul_out", &output_int32); + auto& cast_out = graph.GetOrCreateNodeArg("cast_out", y->TypeAsProto()); + + graph.AddNode("matmul1", "MatMulInteger", "", {a, b, a_zp, b_zp}, {&matmul_out}); + auto& cast_node = graph.AddNode("cast1", "Cast", "", {&matmul_out}, {&cast_out}); + cast_node.AddAttribute("to", int64_t{ONNX_NAMESPACE::TensorProto_DataType_FLOAT}); + graph.AddNode("abs1", "Elu", "", {&cast_out}, {y}).AddAttribute("alpha", 1.5f); + } +}; + +TEST(DnnlMatMulIntegerFusion, MatMulInteger_Cast_Elu) { + Dnnl_MatMulInteger_Cast_Elu_PostOpTester test; + test.AddInput("a", {2, 4}, + {-3, 7, 5, -6, + 4, -5, 8, 7}); + test.AddInput("b", {4, 4}, + {5, -3, 7, 8, + -6, -8, -3, 6, + 7, 9, 9, -5, + 8, 7, -6, 7}); + test.AddInput("a_zp", {}, {5}); + test.AddInput("b_zp", {}, {5}); + test.AddOutput("Y", {2, 4}, + {-1.5f, 16.0f, 89.0f, -1.5f, + 122.0f, 154.0f, 68.0f, -1.5f}); + + test.Run(); +} + +class Dnnl_MatMulInteger_Cast_Mul_Exp_PostOpTester : public OpTester { + public: + explicit Dnnl_MatMulInteger_Cast_Mul_Exp_PostOpTester(int opset_version = 10) + : OpTester("MatMulInteger", opset_version) { + } + + protected: + void AddNodes(onnxruntime::Graph& graph, + std::vector& graph_input_defs, + std::vector& graph_output_defs, + std::vector>& /*add_attribute_funcs*/) override { + ASSERT_EQ(graph_input_defs.size(), 5u); + ASSERT_EQ(graph_output_defs.size(), 1u); + + NodeArg* a = graph_input_defs[0]; + NodeArg* b = graph_input_defs[1]; + NodeArg* a_zp = graph_input_defs[2]; + NodeArg* b_zp = graph_input_defs[3]; + NodeArg* c1 = graph_input_defs[4]; + + NodeArg* y = graph_output_defs[0]; + + const onnx::TensorShapeProto* output_shape = y->Shape(); + onnx::TypeProto output_int32; + output_int32.mutable_tensor_type()->set_elem_type(onnx::TensorProto_DataType_INT32); + for (int i = 0; i < output_shape->dim_size(); ++i) { + auto dim = output_int32.mutable_tensor_type()->mutable_shape()->add_dim(); + *dim = output_shape->dim(i); + } + auto& matmul_out = graph.GetOrCreateNodeArg("matmul_out", &output_int32); + auto& cast_out = graph.GetOrCreateNodeArg("cast_out", y->TypeAsProto()); + auto& mul1_out = graph.GetOrCreateNodeArg("mul1_out", y->TypeAsProto()); + + graph.AddNode("matmul1", "MatMulInteger", "", {a, b, a_zp, b_zp}, {&matmul_out}); + auto& cast_node = graph.AddNode("cast1", "Cast", "", {&matmul_out}, {&cast_out}); + cast_node.AddAttribute("to", int64_t{ONNX_NAMESPACE::TensorProto_DataType_FLOAT}); + graph.AddNode("mul1", "Mul", "", {&cast_out, c1}, {&mul1_out}); + graph.AddNode("exp1", "Exp", "", {&mul1_out}, {y}); + } +}; + +TEST(DnnlMatMulIntegerFusion, MatMulInteger_Cast_Mul_Exp) { + Dnnl_MatMulInteger_Cast_Mul_Exp_PostOpTester test; + test.AddInput("a", {2, 4}, + {-3, 7, 5, -6, + 4, -5, 8, 7}); + test.AddInput("b", {4, 4}, + {5, -3, 7, 8, + -6, -8, -3, 6, + 7, 9, 9, -5, + 8, 7, -6, 7}); + test.AddInput("a_zp", {}, {5}); + test.AddInput("b_zp", {}, {5}); + test.AddInput("c1", {}, {0.01f}); + test.AddOutput("Y", {2, 4}, + {0.5769498103804866f, 1.1735108709918103f, + 2.4351296512898744f, 0.6440364210831414f, + 3.3871877336213347f, 4.664590270988126f, + 1.9738777322304477f, 0.6770568744981647f}); + + test.Run(); +} + +class Dnnl_MatMulInteger_Cast_LeakyRelu_PostOpTester : public OpTester { + public: + explicit Dnnl_MatMulInteger_Cast_LeakyRelu_PostOpTester(int opset_version = 10) + : OpTester("MatMulInteger", opset_version) { + } + + protected: + void AddNodes(onnxruntime::Graph& graph, + std::vector& graph_input_defs, + std::vector& graph_output_defs, + std::vector>& /*add_attribute_funcs*/) override { + ASSERT_EQ(graph_input_defs.size(), 4u); + ASSERT_EQ(graph_output_defs.size(), 1u); + + NodeArg* a = graph_input_defs[0]; + NodeArg* b = graph_input_defs[1]; + NodeArg* a_zp = graph_input_defs[2]; + NodeArg* b_zp = graph_input_defs[3]; + + NodeArg* y = graph_output_defs[0]; + + const onnx::TensorShapeProto* output_shape = y->Shape(); + onnx::TypeProto output_int32; + output_int32.mutable_tensor_type()->set_elem_type(onnx::TensorProto_DataType_INT32); + for (int i = 0; i < output_shape->dim_size(); ++i) { + auto dim = output_int32.mutable_tensor_type()->mutable_shape()->add_dim(); + *dim = output_shape->dim(i); + } + auto& matmul_out = graph.GetOrCreateNodeArg("matmul_out", &output_int32); + auto& cast_out = graph.GetOrCreateNodeArg("cast_out", y->TypeAsProto()); + + graph.AddNode("matmul1", "MatMulInteger", "", {a, b, a_zp, b_zp}, {&matmul_out}); + auto& cast_node = graph.AddNode("cast1", "Cast", "", {&matmul_out}, {&cast_out}); + cast_node.AddAttribute("to", int64_t{ONNX_NAMESPACE::TensorProto_DataType_FLOAT}); + graph.AddNode("leakyrelu1", "LeakyRelu", "", {&cast_out}, {y}).AddAttribute("alpha", 0.015f); + } +}; + +TEST(DnnlMatMulIntegerFusion, MatMulInteger_Cast_LeakyRelu) { + Dnnl_MatMulInteger_Cast_LeakyRelu_PostOpTester test; + test.AddInput("a", {2, 4}, + {-3, 7, 5, -6, + 4, -5, 8, 7}); + test.AddInput("b", {4, 4}, + {5, -3, 7, 8, + -6, -8, -3, 6, + 7, 9, 9, -5, + 8, 7, -6, 7}); + test.AddInput("a_zp", {}, {5}); + test.AddInput("b_zp", {}, {5}); + test.AddOutput("Y", {2, 4}, + {-0.825f, 16.0f, 89.0f, -0.65999997f, + 122.0f, 154.0f, 68.0f, -0.585f}); + + test.Run(); +} + +class Dnnl_MatMulInteger_Cast_Log_PostOpTester : public OpTester { + public: + explicit Dnnl_MatMulInteger_Cast_Log_PostOpTester(int opset_version = 10) + : OpTester("MatMulInteger", opset_version) { + } + + protected: + void AddNodes(onnxruntime::Graph& graph, + std::vector& graph_input_defs, + std::vector& graph_output_defs, + std::vector>& /*add_attribute_funcs*/) override { + ASSERT_EQ(graph_input_defs.size(), 4u); + ASSERT_EQ(graph_output_defs.size(), 1u); + + NodeArg* a = graph_input_defs[0]; + NodeArg* b = graph_input_defs[1]; + NodeArg* a_zp = graph_input_defs[2]; + NodeArg* b_zp = graph_input_defs[3]; + + NodeArg* y = graph_output_defs[0]; + + const onnx::TensorShapeProto* output_shape = y->Shape(); + onnx::TypeProto output_int32; + output_int32.mutable_tensor_type()->set_elem_type(onnx::TensorProto_DataType_INT32); + for (int i = 0; i < output_shape->dim_size(); ++i) { + auto dim = output_int32.mutable_tensor_type()->mutable_shape()->add_dim(); + *dim = output_shape->dim(i); + } + auto& matmul_out = graph.GetOrCreateNodeArg("matmul_out", &output_int32); + auto& cast_out = graph.GetOrCreateNodeArg("cast_out", y->TypeAsProto()); + auto& abs_out = graph.GetOrCreateNodeArg("abs_out", y->TypeAsProto()); + + graph.AddNode("matmul1", "MatMulInteger", "", {a, b, a_zp, b_zp}, {&matmul_out}); + auto& cast_node = graph.AddNode("cast1", "Cast", "", {&matmul_out}, {&cast_out}); + cast_node.AddAttribute("to", int64_t{ONNX_NAMESPACE::TensorProto_DataType_FLOAT}); + graph.AddNode("abs1", "Abs", "", {&cast_out}, {&abs_out}); + graph.AddNode("log1", "Log", "", {&abs_out}, {y}); + } +}; + +TEST(DnnlMatMulIntegerFusion, MatMulInteger_Cast_Abs_Log) { + Dnnl_MatMulInteger_Cast_Log_PostOpTester test; + test.AddInput("a", {2, 4}, + {-3, 7, 5, -6, + 4, -5, 8, 7}); + test.AddInput("b", {4, 4}, + {5, -3, 7, 8, + -6, -8, -3, 6, + 7, 9, 9, -5, + 8, 7, -6, 7}); + test.AddInput("a_zp", {}, {5}); + test.AddInput("b_zp", {}, {5}); + test.AddOutput("Y", {2, 4}, + {4.0073333f, 2.7725887f, 4.4886365f, 3.7841897f, + 4.804021f, 5.0369525f, 4.2195077f, 3.6635616f}); + + test.Run(); +} + +class Dnnl_MatMulInteger_Add_Round_PostOpTester : public OpTester { + public: + explicit Dnnl_MatMulInteger_Add_Round_PostOpTester(int opset_version = 11) + : OpTester("MatMulInteger", opset_version) { + } + + protected: + void AddNodes(onnxruntime::Graph& graph, + std::vector& graph_input_defs, + std::vector& graph_output_defs, + std::vector>& /*add_attribute_funcs*/) override { + ASSERT_EQ(graph_input_defs.size(), 5u); + ASSERT_EQ(graph_output_defs.size(), 1u); + + NodeArg* a = graph_input_defs[0]; + NodeArg* b = graph_input_defs[1]; + NodeArg* a_zp = graph_input_defs[2]; + NodeArg* b_zp = graph_input_defs[3]; + NodeArg* c1 = graph_input_defs[4]; + + NodeArg* y = graph_output_defs[0]; + + const onnx::TensorShapeProto* output_shape = y->Shape(); + onnx::TypeProto output_int32; + output_int32.mutable_tensor_type()->set_elem_type(onnx::TensorProto_DataType_INT32); + for (int i = 0; i < output_shape->dim_size(); ++i) { + auto dim = output_int32.mutable_tensor_type()->mutable_shape()->add_dim(); + *dim = output_shape->dim(i); + } + auto& matmul_out = graph.GetOrCreateNodeArg("matmul_out", &output_int32); + auto& cast_out = graph.GetOrCreateNodeArg("cast_out", y->TypeAsProto()); + auto& add1_out = graph.GetOrCreateNodeArg("add1_out", y->TypeAsProto()); + + graph.AddNode("matmul1", "MatMulInteger", "", {a, b, a_zp, b_zp}, {&matmul_out}); + auto& cast_node = graph.AddNode("cast1", "Cast", "", {&matmul_out}, {&cast_out}); + cast_node.AddAttribute("to", int64_t{ONNX_NAMESPACE::TensorProto_DataType_FLOAT}); + graph.AddNode("add", "Add", "", {&cast_out, c1}, {&add1_out}); + graph.AddNode("round1", "Round", "", {&add1_out}, {y}); + } +}; + +TEST(DnnlMatMulIntegerFusion, MatMulInteger_Cast_Add_Round) { + Dnnl_MatMulInteger_Add_Round_PostOpTester test; + test.AddInput("a", {2, 4}, + {-3, 7, 5, -6, + 4, -5, 8, 7}); + test.AddInput("b", {4, 4}, + {5, -3, 7, 8, + -6, -8, -3, 6, + 7, 9, 9, -5, + 8, 7, -6, 7}); + test.AddInput("a_zp", {}, {5}); + test.AddInput("b_zp", {}, {5}); + test.AddInput("c1", {2, 4}, + {0.4f, 0.6f, 0.4f, 0.6f, + 0.4f, 0.6f, 0.5f, 0.5f}); + test.AddOutput("Y", {2, 4}, + {-55.0f, 17.0f, 89.0f, -43.0f, + 122.0f, 155.0f, 68.0f, -38.0f}); + + test.Run(); +} + +class Dnnl_MatMulInteger_Cast_Sigmoid_PostOpTester : public OpTester { + public: + explicit Dnnl_MatMulInteger_Cast_Sigmoid_PostOpTester(int opset_version = 13) + : OpTester("MatMulInteger", opset_version) { + } + + protected: + void AddNodes(onnxruntime::Graph& graph, + std::vector& graph_input_defs, + std::vector& graph_output_defs, + std::vector>& /*add_attribute_funcs*/) override { + ASSERT_EQ(graph_input_defs.size(), 4u); + ASSERT_EQ(graph_output_defs.size(), 1u); + + NodeArg* a = graph_input_defs[0]; + NodeArg* b = graph_input_defs[1]; + NodeArg* a_zp = graph_input_defs[2]; + NodeArg* b_zp = graph_input_defs[3]; + + NodeArg* y = graph_output_defs[0]; + + const onnx::TensorShapeProto* output_shape = y->Shape(); + onnx::TypeProto output_int32; + output_int32.mutable_tensor_type()->set_elem_type(onnx::TensorProto_DataType_INT32); + for (int i = 0; i < output_shape->dim_size(); ++i) { + auto dim = output_int32.mutable_tensor_type()->mutable_shape()->add_dim(); + *dim = output_shape->dim(i); + } + auto& matmul_out = graph.GetOrCreateNodeArg("matmul_out", &output_int32); + auto& cast_out = graph.GetOrCreateNodeArg("cast_out", y->TypeAsProto()); + + graph.AddNode("matmul1", "MatMulInteger", "", {a, b, a_zp, b_zp}, {&matmul_out}); + auto& cast_node = graph.AddNode("cast1", "Cast", "", {&matmul_out}, {&cast_out}); + cast_node.AddAttribute("to", int64_t{ONNX_NAMESPACE::TensorProto_DataType_FLOAT}); + graph.AddNode("sigmoid1", "Sigmoid", "", {&cast_out}, {y}); + } +}; + +TEST(DnnlMatMulIntegerFusion, MatMulInteger_Cast_Sigmoid) { + Dnnl_MatMulInteger_Cast_Sigmoid_PostOpTester test; + test.AddInput("a", {2, 4}, + {-3, 7, 5, -6, + 4, -5, 8, 7}); + test.AddInput("b", {4, 4}, + {5, -3, 7, 8, + -6, -8, -3, 6, + 7, 9, 9, -5, + 8, 7, -6, 7}); + test.AddInput("a_zp", {}, {5}); + test.AddInput("b_zp", {}, {5}); + test.AddOutput("Y", {2, 4}, + {1.2995814e-24f, 0.9999999f, 1.0f, 7.781132e-20f, + 1.0f, 1.0f, 1.0f, 1.1548225e-17f}); + + test.Run(); +} + +class Dnnl_MatMulInteger_Mul_Softplus_PostOpTester : public OpTester { + public: + explicit Dnnl_MatMulInteger_Mul_Softplus_PostOpTester(int opset_version = 10) + : OpTester("MatMulInteger", opset_version) { + } + + protected: + void AddNodes(onnxruntime::Graph& graph, + std::vector& graph_input_defs, + std::vector& graph_output_defs, + std::vector>& /*add_attribute_funcs*/) override { + ASSERT_EQ(graph_input_defs.size(), 5u); + ASSERT_EQ(graph_output_defs.size(), 1u); + + NodeArg* a = graph_input_defs[0]; + NodeArg* b = graph_input_defs[1]; + NodeArg* a_zp = graph_input_defs[2]; + NodeArg* b_zp = graph_input_defs[3]; + NodeArg* c1 = graph_input_defs[4]; + + NodeArg* y = graph_output_defs[0]; + + const onnx::TensorShapeProto* output_shape = y->Shape(); + onnx::TypeProto output_int32; + output_int32.mutable_tensor_type()->set_elem_type(onnx::TensorProto_DataType_INT32); + for (int i = 0; i < output_shape->dim_size(); ++i) { + auto dim = output_int32.mutable_tensor_type()->mutable_shape()->add_dim(); + *dim = output_shape->dim(i); + } + auto& matmul_out = graph.GetOrCreateNodeArg("matmul_out", &output_int32); + auto& cast_out = graph.GetOrCreateNodeArg("cast_out", y->TypeAsProto()); + auto& mul1_out = graph.GetOrCreateNodeArg("add1_out", y->TypeAsProto()); + + graph.AddNode("matmul1", "MatMulInteger", "", {a, b, a_zp, b_zp}, {&matmul_out}); + auto& cast_node = graph.AddNode("cast1", "Cast", "", {&matmul_out}, {&cast_out}); + cast_node.AddAttribute("to", int64_t{ONNX_NAMESPACE::TensorProto_DataType_FLOAT}); + graph.AddNode("mul1", "Mul", "", {&cast_out, c1}, {&mul1_out}); + graph.AddNode("softplus1", "Softplus", "", {&mul1_out}, {y}); + } +}; + +TEST(DnnlMatMulIntegerFusion, MatMulInteger_Cast_Mul_Softplus) { + Dnnl_MatMulInteger_Mul_Softplus_PostOpTester test; + test.AddInput("a", {2, 4}, + {-3, 7, 5, -6, + 4, -5, 8, 7}); + test.AddInput("b", {4, 4}, + {5, -3, 7, 8, + -6, -8, -3, 6, + 7, 9, 9, -5, + 8, 7, -6, 7}); + test.AddInput("a_zp", {}, {5}); + test.AddInput("b_zp", {}, {5}); + test.AddInput("c1", {}, + {0.01f}); + test.AddOutput("Y", {2, 4}, + {0.4554924814633376f, 0.7763437730407398f, + 1.2340546691512106f, 0.4971544503321099f, + 1.4786884144349526f, 1.7342345654720792f, + 1.0898667349636622f, 0.5170403966954268f}); + + test.Run(); +} + +class Dnnl_MatMulInteger_Cast_Abs_Sqrt_PostOpTester : public OpTester { + public: + explicit Dnnl_MatMulInteger_Cast_Abs_Sqrt_PostOpTester(int opset_version = 10) + : OpTester("MatMulInteger", opset_version) { + } + + protected: + void AddNodes(onnxruntime::Graph& graph, + std::vector& graph_input_defs, + std::vector& graph_output_defs, + std::vector>& /*add_attribute_funcs*/) override { + ASSERT_EQ(graph_input_defs.size(), 4u); + ASSERT_EQ(graph_output_defs.size(), 1u); + + NodeArg* a = graph_input_defs[0]; + NodeArg* b = graph_input_defs[1]; + NodeArg* a_zp = graph_input_defs[2]; + NodeArg* b_zp = graph_input_defs[3]; + + NodeArg* y = graph_output_defs[0]; + + const onnx::TensorShapeProto* output_shape = y->Shape(); + onnx::TypeProto output_int32; + output_int32.mutable_tensor_type()->set_elem_type(onnx::TensorProto_DataType_INT32); + for (int i = 0; i < output_shape->dim_size(); ++i) { + auto dim = output_int32.mutable_tensor_type()->mutable_shape()->add_dim(); + *dim = output_shape->dim(i); + } + auto& matmul_out = graph.GetOrCreateNodeArg("matmul_out", &output_int32); + auto& cast_out = graph.GetOrCreateNodeArg("cast_out", y->TypeAsProto()); + auto& abs_out = graph.GetOrCreateNodeArg("abs_out", y->TypeAsProto()); + + graph.AddNode("matmul1", "MatMulInteger", "", {a, b, a_zp, b_zp}, {&matmul_out}); + auto& cast_node = graph.AddNode("cast1", "Cast", "", {&matmul_out}, {&cast_out}); + cast_node.AddAttribute("to", int64_t{ONNX_NAMESPACE::TensorProto_DataType_FLOAT}); + graph.AddNode("abs1", "Abs", "", {&cast_out}, {&abs_out}); + graph.AddNode("sqrt1", "Sqrt", "", {&abs_out}, {y}); + } +}; + +TEST(DnnlMatMulIntegerFusion, MatMulInteger_Cast_Abs_Sqrt) { + Dnnl_MatMulInteger_Cast_Abs_Sqrt_PostOpTester test; + test.AddInput("a", {2, 4}, + {-3, 7, 5, -6, + 4, -5, 8, 7}); + test.AddInput("b", {4, 4}, + {5, -3, 7, 8, + -6, -8, -3, 6, + 7, 9, 9, -5, + 8, 7, -6, 7}); + test.AddInput("a_zp", {}, {5}); + test.AddInput("b_zp", {}, {5}); + test.AddOutput("Y", {2, 4}, + {7.4161983f, 4.0f, 9.433981f, 6.6332498f, + 11.045361f, 12.409674f, 8.246211f, 6.244998f}); + + test.Run(); +} + +class Dnnl_MatMulInteger_Mul_Tanh_PostOpTester : public OpTester { + public: + explicit Dnnl_MatMulInteger_Mul_Tanh_PostOpTester(int opset_version = 10) + : OpTester("MatMulInteger", opset_version) { + } + + protected: + void AddNodes(onnxruntime::Graph& graph, + std::vector& graph_input_defs, + std::vector& graph_output_defs, + std::vector>& /*add_attribute_funcs*/) override { + ASSERT_EQ(graph_input_defs.size(), 5u); + ASSERT_EQ(graph_output_defs.size(), 1u); + + NodeArg* a = graph_input_defs[0]; + NodeArg* b = graph_input_defs[1]; + NodeArg* a_zp = graph_input_defs[2]; + NodeArg* b_zp = graph_input_defs[3]; + NodeArg* c1 = graph_input_defs[4]; + + NodeArg* y = graph_output_defs[0]; + + const onnx::TensorShapeProto* output_shape = y->Shape(); + onnx::TypeProto output_int32; + output_int32.mutable_tensor_type()->set_elem_type(onnx::TensorProto_DataType_INT32); + for (int i = 0; i < output_shape->dim_size(); ++i) { + auto dim = output_int32.mutable_tensor_type()->mutable_shape()->add_dim(); + *dim = output_shape->dim(i); + } + auto& matmul_out = graph.GetOrCreateNodeArg("matmul_out", &output_int32); + auto& cast_out = graph.GetOrCreateNodeArg("cast_out", y->TypeAsProto()); + auto& mul1_out = graph.GetOrCreateNodeArg("add1_out", y->TypeAsProto()); + + graph.AddNode("matmul1", "MatMulInteger", "", {a, b, a_zp, b_zp}, {&matmul_out}); + auto& cast_node = graph.AddNode("cast1", "Cast", "", {&matmul_out}, {&cast_out}); + cast_node.AddAttribute("to", int64_t{ONNX_NAMESPACE::TensorProto_DataType_FLOAT}); + graph.AddNode("mul1", "Mul", "", {&cast_out, c1}, {&mul1_out}); + graph.AddNode("tanh1", "Tanh", "", {&mul1_out}, {y}); + } +}; + +TEST(DnnlMatMulIntegerFusion, MatMulInteger_Cast_Mul_Tanh) { + Dnnl_MatMulInteger_Mul_Tanh_PostOpTester test; + test.AddInput("a", {2, 4}, + {-3, 7, 5, -6, + 4, -5, 8, 7}); + test.AddInput("b", {4, 4}, + {5, -3, 7, 8, + -6, -8, -3, 6, + 7, 9, 9, -5, + 8, 7, -6, 7}); + test.AddInput("a_zp", {}, {5}); + test.AddInput("b_zp", {}, {5}); + test.AddInput("c1", {}, {0.01f}); + test.AddOutput("Y", {2, 4}, + {-0.5005202111902353f, 0.1586485042974989f, + 0.7113937318189626f, -0.41364444218713514f, + 0.8396541756543753f, 0.9121203692077173f, + 0.5915193954318164f, -0.3713602278765077f}); + + test.Run(); +} + +class Dnnl_MatMulInteger_36_ops_PostOpTester : public OpTester { + public: + explicit Dnnl_MatMulInteger_36_ops_PostOpTester(int opset_version = 10) + : OpTester("MatMulInteger", opset_version) { + } + + protected: + void AddNodes(onnxruntime::Graph& graph, + std::vector& graph_input_defs, + std::vector& graph_output_defs, + std::vector>& /*add_attribute_funcs*/) override { + ASSERT_EQ(graph_input_defs.size(), 5u); + ASSERT_EQ(graph_output_defs.size(), 1u); + + NodeArg* a = graph_input_defs[0]; + NodeArg* b = graph_input_defs[1]; + NodeArg* a_zp = graph_input_defs[2]; + NodeArg* b_zp = graph_input_defs[3]; + NodeArg* c1 = graph_input_defs[4]; + + NodeArg* y = graph_output_defs[0]; + + const onnx::TensorShapeProto* output_shape = y->Shape(); + onnx::TypeProto output_int32; + output_int32.mutable_tensor_type()->set_elem_type(onnx::TensorProto_DataType_INT32); + for (int i = 0; i < output_shape->dim_size(); ++i) { + auto dim = output_int32.mutable_tensor_type()->mutable_shape()->add_dim(); + *dim = output_shape->dim(i); + } + auto& matmul_out = graph.GetOrCreateNodeArg("matmul_out", &output_int32); + auto& cast_out = graph.GetOrCreateNodeArg("cast_out", y->TypeAsProto()); + + auto& m1_out = graph.GetOrCreateNodeArg("mul1_out", y->TypeAsProto()); + auto& a1_out = graph.GetOrCreateNodeArg("add1_out", y->TypeAsProto()); + auto& s1_out = graph.GetOrCreateNodeArg("sub1_out", y->TypeAsProto()); + auto& d1_out = graph.GetOrCreateNodeArg("div1_out", y->TypeAsProto()); + + auto& m2_out = graph.GetOrCreateNodeArg("mul2_out", y->TypeAsProto()); + auto& a2_out = graph.GetOrCreateNodeArg("add2_out", y->TypeAsProto()); + auto& s2_out = graph.GetOrCreateNodeArg("sub2_out", y->TypeAsProto()); + auto& d2_out = graph.GetOrCreateNodeArg("div2_out", y->TypeAsProto()); + + auto& m3_out = graph.GetOrCreateNodeArg("mul3_out", y->TypeAsProto()); + auto& a3_out = graph.GetOrCreateNodeArg("add3_out", y->TypeAsProto()); + auto& s3_out = graph.GetOrCreateNodeArg("sub3_out", y->TypeAsProto()); + auto& d3_out = graph.GetOrCreateNodeArg("div3_out", y->TypeAsProto()); + + auto& m4_out = graph.GetOrCreateNodeArg("mul4_out", y->TypeAsProto()); + auto& a4_out = graph.GetOrCreateNodeArg("add4_out", y->TypeAsProto()); + auto& s4_out = graph.GetOrCreateNodeArg("sub4_out", y->TypeAsProto()); + auto& d4_out = graph.GetOrCreateNodeArg("div4_out", y->TypeAsProto()); + + auto& m5_out = graph.GetOrCreateNodeArg("mul5_out", y->TypeAsProto()); + auto& a5_out = graph.GetOrCreateNodeArg("add5_out", y->TypeAsProto()); + auto& s5_out = graph.GetOrCreateNodeArg("sub5_out", y->TypeAsProto()); + auto& d5_out = graph.GetOrCreateNodeArg("div5_out", y->TypeAsProto()); + + auto& m6_out = graph.GetOrCreateNodeArg("mul6_out", y->TypeAsProto()); + auto& a6_out = graph.GetOrCreateNodeArg("add6_out", y->TypeAsProto()); + auto& s6_out = graph.GetOrCreateNodeArg("sub6_out", y->TypeAsProto()); + auto& d6_out = graph.GetOrCreateNodeArg("div6_out", y->TypeAsProto()); + + auto& m7_out = graph.GetOrCreateNodeArg("mul7_out", y->TypeAsProto()); + auto& a7_out = graph.GetOrCreateNodeArg("add7_out", y->TypeAsProto()); + auto& s7_out = graph.GetOrCreateNodeArg("sub7_out", y->TypeAsProto()); + auto& d7_out = graph.GetOrCreateNodeArg("div7_out", y->TypeAsProto()); + + auto& m8_out = graph.GetOrCreateNodeArg("mul8_out", y->TypeAsProto()); + auto& a8_out = graph.GetOrCreateNodeArg("add8_out", y->TypeAsProto()); + auto& s8_out = graph.GetOrCreateNodeArg("sub8_out", y->TypeAsProto()); + auto& d8_out = graph.GetOrCreateNodeArg("div8_out", y->TypeAsProto()); + + auto& m9_out = graph.GetOrCreateNodeArg("mul9_out", y->TypeAsProto()); + auto& a9_out = graph.GetOrCreateNodeArg("add9_out", y->TypeAsProto()); + auto& s9_out = graph.GetOrCreateNodeArg("sub9_out", y->TypeAsProto()); + + graph.AddNode("matmul1", "MatMulInteger", "", {a, b, a_zp, b_zp}, {&matmul_out}); + auto& cast_node = graph.AddNode("cast1", "Cast", "", {&matmul_out}, {&cast_out}); + cast_node.AddAttribute("to", int64_t{ONNX_NAMESPACE::TensorProto_DataType_FLOAT}); + graph.AddNode("mul1", "Mul", "", {&cast_out, c1}, {&m1_out}); + graph.AddNode("add1", "Add", "", {&m1_out, c1}, {&a1_out}); + graph.AddNode("sub1", "Sub", "", {&a1_out, c1}, {&s1_out}); + graph.AddNode("div1", "Div", "", {&s1_out, c1}, {&d1_out}); + graph.AddNode("mul2", "Mul", "", {&d1_out, c1}, {&m2_out}); + graph.AddNode("add2", "Add", "", {&m2_out, c1}, {&a2_out}); + graph.AddNode("sub2", "Sub", "", {&a2_out, c1}, {&s2_out}); + graph.AddNode("div2", "Div", "", {&s2_out, c1}, {&d2_out}); + graph.AddNode("mul3", "Mul", "", {&d2_out, c1}, {&m3_out}); + graph.AddNode("add3", "Add", "", {&m3_out, c1}, {&a3_out}); + graph.AddNode("sub3", "Sub", "", {&a3_out, c1}, {&s3_out}); + graph.AddNode("div3", "Div", "", {&s3_out, c1}, {&d3_out}); + graph.AddNode("mul4", "Mul", "", {&d3_out, c1}, {&m4_out}); + graph.AddNode("add4", "Add", "", {&m4_out, c1}, {&a4_out}); + graph.AddNode("sub4", "Sub", "", {&a4_out, c1}, {&s4_out}); + graph.AddNode("div4", "Div", "", {&s4_out, c1}, {&d4_out}); + graph.AddNode("mul5", "Mul", "", {&d4_out, c1}, {&m5_out}); + graph.AddNode("add5", "Add", "", {&m5_out, c1}, {&a5_out}); + graph.AddNode("sub5", "Sub", "", {&a5_out, c1}, {&s5_out}); + graph.AddNode("div5", "Div", "", {&s5_out, c1}, {&d5_out}); + graph.AddNode("mul6", "Mul", "", {&d5_out, c1}, {&m6_out}); + graph.AddNode("add6", "Add", "", {&m6_out, c1}, {&a6_out}); + graph.AddNode("sub6", "Sub", "", {&a6_out, c1}, {&s6_out}); + graph.AddNode("div6", "Div", "", {&s6_out, c1}, {&d6_out}); + graph.AddNode("mul7", "Mul", "", {&d6_out, c1}, {&m7_out}); + graph.AddNode("add7", "Add", "", {&m7_out, c1}, {&a7_out}); + graph.AddNode("sub7", "Sub", "", {&a7_out, c1}, {&s7_out}); + graph.AddNode("div7", "Div", "", {&s7_out, c1}, {&d7_out}); + graph.AddNode("mul8", "Mul", "", {&d7_out, c1}, {&m8_out}); + graph.AddNode("add8", "Add", "", {&m8_out, c1}, {&a8_out}); + graph.AddNode("sub8", "Sub", "", {&a8_out, c1}, {&s8_out}); + graph.AddNode("div8", "Div", "", {&s8_out, c1}, {&d8_out}); + graph.AddNode("mul9", "Mul", "", {&d8_out, c1}, {&m9_out}); + graph.AddNode("add9", "Add", "", {&m9_out, c1}, {&a9_out}); + graph.AddNode("sub9", "Sub", "", {&a9_out, c1}, {&s9_out}); + graph.AddNode("div9", "Div", "", {&s9_out, c1}, {y}); + } +}; + +TEST(DnnlMatMulIntegerFusion, MatMulInteger_36_ops) { + Dnnl_MatMulInteger_36_ops_PostOpTester test; + test.AddInput("a", {2, 4}, + {-3, 7, 5, -6, + 4, -5, 8, 7}); + test.AddInput("b", {4, 4}, + {5, -3, 7, 8, + -6, -8, -3, 6, + 7, 9, 9, -5, + 8, 7, -6, 7}); + test.AddInput("a_zp", {}, {5}); + test.AddInput("b_zp", {}, {5}); + test.AddInput("c1", {}, {2.5f}); + test.AddOutput("Y", {2, 4}, + {-55.0f, 16.0f, 89.0f, -44.0f, + 122.0f, 154.0f, 68.0f, -39.0f}); + + test.Run(); +} + +class Dnnl_MatMulInteger_Cast_Elu_LeakyRelu_PostOpTester : public OpTester { + public: + explicit Dnnl_MatMulInteger_Cast_Elu_LeakyRelu_PostOpTester(int opset_version = 10) + : OpTester("MatMulInteger", opset_version) { + } + + protected: + void AddNodes(onnxruntime::Graph& graph, + std::vector& graph_input_defs, + std::vector& graph_output_defs, + std::vector>& /*add_attribute_funcs*/) override { + ASSERT_EQ(graph_input_defs.size(), 4u); + ASSERT_EQ(graph_output_defs.size(), 1u); + + NodeArg* a = graph_input_defs[0]; + NodeArg* b = graph_input_defs[1]; + NodeArg* a_zp = graph_input_defs[2]; + NodeArg* b_zp = graph_input_defs[3]; + + NodeArg* y = graph_output_defs[0]; + + const onnx::TensorShapeProto* output_shape = y->Shape(); + onnx::TypeProto output_int32; + output_int32.mutable_tensor_type()->set_elem_type(onnx::TensorProto_DataType_INT32); + for (int i = 0; i < output_shape->dim_size(); ++i) { + auto dim = output_int32.mutable_tensor_type()->mutable_shape()->add_dim(); + *dim = output_shape->dim(i); + } + auto& matmul_out = graph.GetOrCreateNodeArg("matmul_out", &output_int32); + auto& cast_out = graph.GetOrCreateNodeArg("cast_out", y->TypeAsProto()); + auto& elu_out = graph.GetOrCreateNodeArg("elu_out", y->TypeAsProto()); + + graph.AddNode("matmul1", "MatMulInteger", "", {a, b, a_zp, b_zp}, {&matmul_out}); + auto& cast_node = graph.AddNode("cast1", "Cast", "", {&matmul_out}, {&cast_out}); + cast_node.AddAttribute("to", int64_t{ONNX_NAMESPACE::TensorProto_DataType_FLOAT}); + graph.AddNode("elu1", "Elu", "", {&cast_out}, {&elu_out}).AddAttribute("alpha", 1.5f); + graph.AddNode("leakyrelu1", "LeakyRelu", "", {&elu_out}, {y}).AddAttribute("alpha", 0.1f); + } +}; + +TEST(DnnlMatMulIntegerFusion, MatMulInteger_Cast_Elu_LeakyRelu) { + Dnnl_MatMulInteger_Cast_Elu_LeakyRelu_PostOpTester test; + test.AddInput("a", {2, 4}, + {-3, 7, 5, -6, + 4, -5, 8, 7}); + test.AddInput("b", {4, 4}, + {5, -3, 7, 8, + -6, -8, -3, 6, + 7, 9, 9, -5, + 8, 7, -6, 7}); + test.AddInput("a_zp", {}, {5}); + test.AddInput("b_zp", {}, {5}); + test.AddOutput("Y", {2, 4}, + {-0.15f, 16.0f, 89.0f, -0.15f, + 122.0f, 154.0f, 68.0f, -0.15f}); + + test.Run(); +} +#endif // USE_DNNL +} // namespace test +} // namespace onnxruntime