diff --git a/docs/ONNX_Runtime_Graph_Optimizations.md b/docs/ONNX_Runtime_Graph_Optimizations.md
index 1002b41d68..9b23d30fed 100644
--- a/docs/ONNX_Runtime_Graph_Optimizations.md
+++ b/docs/ONNX_Runtime_Graph_Optimizations.md
@@ -55,6 +55,8 @@ These optimizations include complex node fusions. They are run after graph parti
 
 To optimize inference performance of BERT model, approximation is used in GELU approximation and Attention fusion for cuda execution provider. There might be slight difference in result. The impact on accuracy could be neglected based on our evaluation: F1 score for a BERT model on SQuAD v1.1 is almost same (87.05 vs 87.03).
 
+GELU approximation is disabled by default.
+
 ### Layout Optimizations
 
 These optimizations change the data layout for applicable nodes to achieve higher performance improvements. They are run after graph partitioning and are only applied to nodes assigned to CPU execution provider. Available layout optimizations are as follows:
diff --git a/onnxruntime/core/optimizer/graph_transformer_utils.cc b/onnxruntime/core/optimizer/graph_transformer_utils.cc
index 823e15e0f1..2b779d96ec 100644
--- a/onnxruntime/core/optimizer/graph_transformer_utils.cc
+++ b/onnxruntime/core/optimizer/graph_transformer_utils.cc
@@ -136,8 +136,6 @@ std::vector<std::unique_ptr<GraphTransformer>> GenerateTransformers(TransformerL
       transformers.emplace_back(onnxruntime::make_unique<BiasGelu>(cpu_cuda_execution_providers));
       transformers.emplace_back(onnxruntime::make_unique<SkipLayerNormFusion>(cpu_cuda_execution_providers));
 
-      std::unordered_set<std::string> cuda_execution_providers = {onnxruntime::kCudaExecutionProvider};
-      transformers.emplace_back(onnxruntime::make_unique<GeluApproximation>(cuda_execution_providers));
       transformers.emplace_back(onnxruntime::make_unique<FastGeluFusion>(cpu_cuda_execution_providers));
 #endif
     } break;
@@ -164,6 +162,16 @@ std::vector<std::unique_ptr<GraphTransformer>> GenerateTransformers(TransformerL
     }
     return transformers;
   }
+
+  // Some transformers have side-effect like result is not exactly same.
+  // These transformers could only be enabled by custom transformer list.
+#ifndef DISABLE_CONTRIB_OPS
+  if (level == TransformerLevel::Level2) {
+      std::unordered_set<std::string> cuda_execution_providers = {onnxruntime::kCudaExecutionProvider};
+      transformers.emplace_back(onnxruntime::make_unique<GeluApproximation>(cuda_execution_providers));
+  }
+#endif
+
   std::vector<std::unique_ptr<GraphTransformer>> filtered_list;
   // If the rule-based transformer is not empty, it should be included in the custom transformer list below.
   if (rule_transformer != nullptr) {
diff --git a/onnxruntime/test/optimizer/graph_transform_utils_test.cc b/onnxruntime/test/optimizer/graph_transform_utils_test.cc
index 4351fd873a..ff2e17d6c3 100644
--- a/onnxruntime/test/optimizer/graph_transform_utils_test.cc
+++ b/onnxruntime/test/optimizer/graph_transform_utils_test.cc
@@ -60,5 +60,25 @@ TEST(GraphTransformerUtilsTests, TestGenerateGraphTransformers) {
 #endif
 }
 
+TEST(GraphTransformerUtilsTests, TestCustomOnlyTransformers) {
+  // Transformers that are disabled by default. They can only be enabled by custom list.
+  std::string l2_transformer = "GeluApproximation";
+
+  std::vector<std::string> default_list = {};
+  auto default_transformers = optimizer_utils::GenerateTransformers(TransformerLevel::Level2, {}, default_list);
+  for (auto& transformer : default_transformers) {
+    ASSERT_TRUE(transformer->Name() != l2_transformer);
+  }
+
+  std::vector<std::string> custom_list = {l2_transformer};
+  auto custom_transformers = optimizer_utils::GenerateTransformers(TransformerLevel::Level2, {}, custom_list);
+#ifndef DISABLE_CONTRIB_OPS
+  ASSERT_TRUE(custom_transformers.size() == 1);
+  ASSERT_TRUE(custom_transformers[0]->Name() == l2_transformer);
+#else
+  ASSERT_TRUE(custom_transformers.size() == 0);
+#endif
+}
+
 }  // namespace test
 }  // namespace onnxruntime