diff --git a/onnxruntime/core/optimizer/matmul_add_fusion.cc b/onnxruntime/core/optimizer/matmul_add_fusion.cc
index 2241244f6c..194ddc8beb 100644
--- a/onnxruntime/core/optimizer/matmul_add_fusion.cc
+++ b/onnxruntime/core/optimizer/matmul_add_fusion.cc
@@ -64,15 +64,6 @@ Status MatMulAddFusion::ApplyImpl(Graph& graph, bool& modified, int graph_level,
       continue;
     }
 
-    if (1 == matmul_a_shape->dim_size() && 2 == matmul_b_shape->dim_size()) {
-      // MatMul has shape [K] * [K, N], reset it to [1, K] * [K, N], so that it can work for Gemm
-      auto mutable_matmul_a_shape = const_cast<ONNX_NAMESPACE::TensorShapeProto*>(matmul_a_shape);
-      auto dim_0 = mutable_matmul_a_shape->mutable_dim(0);
-      auto dim_1 = (const_cast<ONNX_NAMESPACE::TensorShapeProto*>(matmul_a_shape))->add_dim();
-      (*dim_1) = (*dim_0);
-      dim_0->set_dim_value(1);
-    }
-
     if (2 != matmul_a_shape->dim_size() || 2 != matmul_b_shape->dim_size()) {
       // Gemm only support Matrix
       continue;
diff --git a/onnxruntime/test/optimizer/graph_transform_test.cc b/onnxruntime/test/optimizer/graph_transform_test.cc
index 93d40cbd9a..17c478eab5 100644
--- a/onnxruntime/test/optimizer/graph_transform_test.cc
+++ b/onnxruntime/test/optimizer/graph_transform_test.cc
@@ -542,6 +542,27 @@ TEST(GraphTransformationTests, MatMulAddFusion_three_input) {
   ASSERT_TRUE(op_to_count["Gemm"] == 1);
 }
 
+// Matmul+Add with shape [k]*[k,N]+[N], won't do the fusion
+// We can do the fusion by changing shape to [1,k]*[k,N]+[1,N], then add a reshape [1,N]=>[N]
+// This will bring extra cost. And there's only very limited gain to fuse Matmul+Add to Gemm
+// Since the basic implementation is almost same
+TEST(GraphTransformationTests, MatMulAddFusion_negitive_case) {
+  auto model_uri = MODEL_FOLDER "matmul_add_fusion/3Input/neg_model.onnx";
+
+  std::shared_ptr<Model> p_model;
+  ASSERT_TRUE(Model::Load(model_uri, p_model, nullptr, DefaultLoggingManager().DefaultLogger()).IsOK());
+  Graph& graph = p_model->MainGraph();
+
+  onnxruntime::GraphTransformerManager graph_transformation_mgr{5};
+  graph_transformation_mgr.Register(onnxruntime::make_unique<MatMulAddFusion>(), TransformerLevel::Level1);
+  ASSERT_TRUE(graph_transformation_mgr.ApplyTransformers(graph, TransformerLevel::Level1, DefaultLoggingManager().DefaultLogger()).IsOK());
+
+  std::map<std::string, int> op_to_count = CountOpsInGraph(graph);
+  ASSERT_TRUE(op_to_count["MatMul"] == 1);
+  ASSERT_TRUE(op_to_count["Add"] == 1);
+  ASSERT_TRUE(op_to_count["Gemm"] == 0);
+}
+
 #ifndef DISABLE_CONTRIB_OPS
 TEST(GraphTransformationTests, Gemm_Relu_three_input) {
   auto model_uri = MODEL_FOLDER "matmul_add_fusion/3Input/gemm_relu.onnx";
diff --git a/onnxruntime/test/testdata/transform/matmul_add_fusion/3Input/neg_model.onnx b/onnxruntime/test/testdata/transform/matmul_add_fusion/3Input/neg_model.onnx
new file mode 100644
index 0000000000..fa8c1dccba
Binary files /dev/null and b/onnxruntime/test/testdata/transform/matmul_add_fusion/3Input/neg_model.onnx differ