diff --git a/cmake/CMakeLists.txt b/cmake/CMakeLists.txt
index f153955bb2..9a668c7f8a 100644
--- a/cmake/CMakeLists.txt
+++ b/cmake/CMakeLists.txt
@@ -513,6 +513,7 @@ include(onnxruntime_framework.cmake)
 include(onnxruntime_util.cmake)
 #The next file will define 'onnxruntime_libs' cmake var
 include(onnxruntime_providers.cmake)
+include(onnxruntime_optimizer.cmake)
 include(onnxruntime_session.cmake)
 include(onnxruntime_mlas.cmake)
 
diff --git a/cmake/onnxruntime.cmake b/cmake/onnxruntime.cmake
index 6e9a9c485e..31579d9375 100644
--- a/cmake/onnxruntime.cmake
+++ b/cmake/onnxruntime.cmake
@@ -49,6 +49,7 @@ target_link_libraries(onnxruntime PRIVATE
     ${onnxruntime_libs}
     ${PROVIDERS_CUDA}
     ${PROVIDERS_MKLDNN}
+    onnxruntime_optimizer
     onnxruntime_providers    
     onnxruntime_util
     ${onnxruntime_tvm_libs}
diff --git a/cmake/onnxruntime_optimizer.cmake b/cmake/onnxruntime_optimizer.cmake
new file mode 100644
index 0000000000..4bf0af2873
--- /dev/null
+++ b/cmake/onnxruntime_optimizer.cmake
@@ -0,0 +1,16 @@
+# Copyright (c) Microsoft Corporation. All rights reserved.
+# Licensed under the MIT License.
+
+file(GLOB onnxruntime_optimizer_srcs
+    "${ONNXRUNTIME_ROOT}/core/optimizer/*.h"
+    "${ONNXRUNTIME_ROOT}/core/optimizer/*.cc"
+    )
+
+source_group(TREE ${REPO_ROOT} FILES ${onnxruntime_optimizer_srcs})
+
+add_library(onnxruntime_optimizer ${onnxruntime_optimizer_srcs})
+install(DIRECTORY ${PROJECT_SOURCE_DIR}/../include/onnxruntime/core/optimizer  DESTINATION ${CMAKE_INSTALL_INCLUDEDIR}/onnxruntime/core)
+onnxruntime_add_include_to_target(onnxruntime_optimizer onnxruntime_common onnxruntime_framework gsl onnx onnx_proto protobuf::libprotobuf)
+target_include_directories(onnxruntime_optimizer PRIVATE ${ONNXRUNTIME_ROOT})
+add_dependencies(onnxruntime_optimizer ${onnxruntime_EXTERNAL_DEPENDENCIES})
+set_target_properties(onnxruntime_optimizer PROPERTIES FOLDER "ONNXRuntime")
diff --git a/cmake/onnxruntime_python.cmake b/cmake/onnxruntime_python.cmake
index fb00e827b6..c7df6a7de3 100644
--- a/cmake/onnxruntime_python.cmake
+++ b/cmake/onnxruntime_python.cmake
@@ -58,6 +58,7 @@ set(onnxruntime_pybind11_state_libs
     ${onnxruntime_libs}
     ${PROVIDERS_CUDA}
     ${PROVIDERS_MKLDNN}
+    onnxruntime_optimizer
     onnxruntime_providers
     onnxruntime_util
     ${onnxruntime_tvm_libs}
diff --git a/cmake/onnxruntime_unittests.cmake b/cmake/onnxruntime_unittests.cmake
index 21ef942142..a8a327edb6 100644
--- a/cmake/onnxruntime_unittests.cmake
+++ b/cmake/onnxruntime_unittests.cmake
@@ -135,6 +135,7 @@ set(onnxruntime_test_ir_libs
 
 set(onnxruntime_test_framework_libs
   onnxruntime_test_utils_for_framework
+  onnxruntime_optimizer
   onnxruntime_framework
   onnxruntime_util
   onnxruntime_graph
@@ -175,6 +176,7 @@ set(ONNXRUNTIME_TEST_LIBS
     ${onnxruntime_libs}
     ${PROVIDERS_CUDA}
     ${PROVIDERS_MKLDNN}
+    onnxruntime_optimizer
     onnxruntime_providers
     onnxruntime_util
     ${onnxruntime_tvm_libs}
diff --git a/include/onnxruntime/core/graph/graph_transformer.h b/include/onnxruntime/core/optimizer/graph_transformer.h
similarity index 99%
rename from include/onnxruntime/core/graph/graph_transformer.h
rename to include/onnxruntime/core/optimizer/graph_transformer.h
index 48c6a5c830..b019833b09 100644
--- a/include/onnxruntime/core/graph/graph_transformer.h
+++ b/include/onnxruntime/core/optimizer/graph_transformer.h
@@ -5,7 +5,7 @@
 
 #include "core/common/common.h"
 #include "core/graph/graph_viewer.h"
-#include "core/graph/rewrite_rule.h"
+#include "core/optimizer/rewrite_rule.h"
 
 namespace onnxruntime {
 
diff --git a/include/onnxruntime/core/graph/rewrite_rule.h b/include/onnxruntime/core/optimizer/rewrite_rule.h
similarity index 100%
rename from include/onnxruntime/core/graph/rewrite_rule.h
rename to include/onnxruntime/core/optimizer/rewrite_rule.h
diff --git a/onnxruntime/core/framework/session_state_initializer.cc b/onnxruntime/core/framework/session_state_initializer.cc
index de951e5576..84e95ec302 100644
--- a/onnxruntime/core/framework/session_state_initializer.cc
+++ b/onnxruntime/core/framework/session_state_initializer.cc
@@ -10,11 +10,7 @@
 #include "core/common/logging/logging.h"
 
 #include "core/graph/graph_viewer.h"
-#include "core/graph/graph_transformer.h"
-#include "core/graph/graph_transformer_mgr.h"
-
 #include "core/framework/graph_partitioner.h"
-#include "core/framework/insert_cast_transformer.h"
 #include "core/framework/ml_value.h"
 #include "core/framework/ml_value_patterns_planner.h"
 #include "core/framework/mlvalue_name_idx_map.h"
@@ -22,7 +18,6 @@
 #include "core/framework/session_state.h"
 #include "core/framework/tensorutils.h"
 #include "core/framework/tensorprotoutils.h"
-#include "core/framework/transformer_memcpy.h"
 #include "core/framework/utils.h"
 
 namespace onnxruntime {
diff --git a/onnxruntime/core/graph/conv_activation_fusion.cc b/onnxruntime/core/optimizer/conv_activation_fusion.cc
similarity index 97%
rename from onnxruntime/core/graph/conv_activation_fusion.cc
rename to onnxruntime/core/optimizer/conv_activation_fusion.cc
index a9f869347b..ac546a0570 100644
--- a/onnxruntime/core/graph/conv_activation_fusion.cc
+++ b/onnxruntime/core/optimizer/conv_activation_fusion.cc
@@ -1,10 +1,10 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
-#include "core/graph/initializer.h"
-#include "core/graph/conv_activation_fusion.h"
-#include "core/graph/graph_utils.h"
 #include <deque>
+#include "core/graph/graph_utils.h"
+#include "core/optimizer/initializer.h"
+#include "core/optimizer/conv_activation_fusion.h"
 
 using namespace onnx;
 using namespace ::onnxruntime::common;
diff --git a/onnxruntime/core/graph/conv_activation_fusion.h b/onnxruntime/core/optimizer/conv_activation_fusion.h
similarity index 91%
rename from onnxruntime/core/graph/conv_activation_fusion.h
rename to onnxruntime/core/optimizer/conv_activation_fusion.h
index 220d91cdce..6ae03fd929 100644
--- a/onnxruntime/core/graph/conv_activation_fusion.h
+++ b/onnxruntime/core/optimizer/conv_activation_fusion.h
@@ -3,7 +3,7 @@
 
 #pragma once
 
-#include "core/graph/graph_transformer.h"
+#include "core/optimizer/graph_transformer.h"
 
 namespace onnxruntime {
 
diff --git a/onnxruntime/core/graph/conv_add_fusion.cc b/onnxruntime/core/optimizer/conv_add_fusion.cc
similarity index 98%
rename from onnxruntime/core/graph/conv_add_fusion.cc
rename to onnxruntime/core/optimizer/conv_add_fusion.cc
index 7ec4363de3..0fe70a6636 100644
--- a/onnxruntime/core/graph/conv_add_fusion.cc
+++ b/onnxruntime/core/optimizer/conv_add_fusion.cc
@@ -1,9 +1,9 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
-#include "core/graph/initializer.h"
 #include "core/graph/graph_utils.h"
-#include "core/graph/conv_add_fusion.h"
+#include "core/optimizer/initializer.h"
+#include "core/optimizer/conv_add_fusion.h"
 
 using namespace onnx;
 using namespace ::onnxruntime::common;
diff --git a/onnxruntime/core/graph/conv_add_fusion.h b/onnxruntime/core/optimizer/conv_add_fusion.h
similarity index 90%
rename from onnxruntime/core/graph/conv_add_fusion.h
rename to onnxruntime/core/optimizer/conv_add_fusion.h
index 09fc473fff..649ae8c203 100644
--- a/onnxruntime/core/graph/conv_add_fusion.h
+++ b/onnxruntime/core/optimizer/conv_add_fusion.h
@@ -3,7 +3,7 @@
 
 #pragma once
 
-#include "core/graph/graph_transformer.h"
+#include "core/optimizer/graph_transformer.h"
 
 namespace onnxruntime {
 
diff --git a/onnxruntime/core/graph/conv_bn_fusion.cc b/onnxruntime/core/optimizer/conv_bn_fusion.cc
similarity index 98%
rename from onnxruntime/core/graph/conv_bn_fusion.cc
rename to onnxruntime/core/optimizer/conv_bn_fusion.cc
index 3016bd750a..58d4e71cd8 100644
--- a/onnxruntime/core/graph/conv_bn_fusion.cc
+++ b/onnxruntime/core/optimizer/conv_bn_fusion.cc
@@ -1,9 +1,9 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
-#include "core/graph/initializer.h"
 #include "core/graph/graph_utils.h"
-#include "core/graph/conv_bn_fusion.h"
+#include "core/optimizer/initializer.h"
+#include "core/optimizer/conv_bn_fusion.h"
 
 using namespace onnx;
 using namespace ::onnxruntime::common;
diff --git a/onnxruntime/core/graph/conv_bn_fusion.h b/onnxruntime/core/optimizer/conv_bn_fusion.h
similarity index 90%
rename from onnxruntime/core/graph/conv_bn_fusion.h
rename to onnxruntime/core/optimizer/conv_bn_fusion.h
index 9de6818e39..b331e6a080 100644
--- a/onnxruntime/core/graph/conv_bn_fusion.h
+++ b/onnxruntime/core/optimizer/conv_bn_fusion.h
@@ -3,7 +3,7 @@
 
 #pragma once
 
-#include "core/graph/graph_transformer.h"
+#include "core/optimizer/graph_transformer.h"
 
 namespace onnxruntime {
 
diff --git a/onnxruntime/core/graph/conv_mul_fusion.cc b/onnxruntime/core/optimizer/conv_mul_fusion.cc
similarity index 98%
rename from onnxruntime/core/graph/conv_mul_fusion.cc
rename to onnxruntime/core/optimizer/conv_mul_fusion.cc
index c0de9dd33c..0e04ae0a73 100644
--- a/onnxruntime/core/graph/conv_mul_fusion.cc
+++ b/onnxruntime/core/optimizer/conv_mul_fusion.cc
@@ -1,9 +1,9 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
-#include "core/graph/initializer.h"
 #include "core/graph/graph_utils.h"
-#include "core/graph/conv_mul_fusion.h"
+#include "core/optimizer/initializer.h"
+#include "core/optimizer/conv_mul_fusion.h"
 
 using namespace onnx;
 using namespace ::onnxruntime::common;
diff --git a/onnxruntime/core/graph/conv_mul_fusion.h b/onnxruntime/core/optimizer/conv_mul_fusion.h
similarity index 90%
rename from onnxruntime/core/graph/conv_mul_fusion.h
rename to onnxruntime/core/optimizer/conv_mul_fusion.h
index 8402e5a0b1..d0f06b7bfe 100644
--- a/onnxruntime/core/graph/conv_mul_fusion.h
+++ b/onnxruntime/core/optimizer/conv_mul_fusion.h
@@ -2,7 +2,7 @@
 // Licensed under the MIT License.
 
 #pragma once
-#include "core/graph/graph_transformer.h"
+#include "core/optimizer/graph_transformer.h"
 
 namespace onnxruntime {
 
diff --git a/onnxruntime/core/graph/gemm_activation_fusion.cc b/onnxruntime/core/optimizer/gemm_activation_fusion.cc
similarity index 97%
rename from onnxruntime/core/graph/gemm_activation_fusion.cc
rename to onnxruntime/core/optimizer/gemm_activation_fusion.cc
index 1e7246f79b..a054d46740 100644
--- a/onnxruntime/core/graph/gemm_activation_fusion.cc
+++ b/onnxruntime/core/optimizer/gemm_activation_fusion.cc
@@ -1,8 +1,8 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
-#include "core/graph/initializer.h"
-#include "core/graph/gemm_activation_fusion.h"
+#include "core/optimizer/initializer.h"
+#include "core/optimizer/gemm_activation_fusion.h"
 #include "core/graph/graph_utils.h"
 #include <deque>
 
diff --git a/onnxruntime/core/graph/gemm_activation_fusion.h b/onnxruntime/core/optimizer/gemm_activation_fusion.h
similarity index 91%
rename from onnxruntime/core/graph/gemm_activation_fusion.h
rename to onnxruntime/core/optimizer/gemm_activation_fusion.h
index 2c0c40ac05..8c65c93ab4 100644
--- a/onnxruntime/core/graph/gemm_activation_fusion.h
+++ b/onnxruntime/core/optimizer/gemm_activation_fusion.h
@@ -3,7 +3,7 @@
 
 #pragma once
 
-#include "core/graph/graph_transformer.h"
+#include "core/optimizer/graph_transformer.h"
 
 namespace onnxruntime {
 
diff --git a/onnxruntime/core/graph/graph_transformer.cc b/onnxruntime/core/optimizer/graph_transformer.cc
similarity index 97%
rename from onnxruntime/core/graph/graph_transformer.cc
rename to onnxruntime/core/optimizer/graph_transformer.cc
index 5e1ebc43f2..8d4e112511 100644
--- a/onnxruntime/core/graph/graph_transformer.cc
+++ b/onnxruntime/core/optimizer/graph_transformer.cc
@@ -1,7 +1,7 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
-#include "core/graph/graph_transformer.h"
+#include "core/optimizer/graph_transformer.h"
 
 using namespace ::onnxruntime::common;
 
diff --git a/onnxruntime/core/graph/graph_transformer_mgr.cc b/onnxruntime/core/optimizer/graph_transformer_mgr.cc
similarity index 92%
rename from onnxruntime/core/graph/graph_transformer_mgr.cc
rename to onnxruntime/core/optimizer/graph_transformer_mgr.cc
index 2275c3e13f..9c4d0840e4 100644
--- a/onnxruntime/core/graph/graph_transformer_mgr.cc
+++ b/onnxruntime/core/optimizer/graph_transformer_mgr.cc
@@ -1,7 +1,7 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
-#include "core/graph/graph_transformer_mgr.h"
+#include "core/optimizer/graph_transformer_mgr.h"
 using namespace onnxruntime;
 using namespace ::onnxruntime::common;
 
diff --git a/onnxruntime/core/graph/graph_transformer_mgr.h b/onnxruntime/core/optimizer/graph_transformer_mgr.h
similarity index 95%
rename from onnxruntime/core/graph/graph_transformer_mgr.h
rename to onnxruntime/core/optimizer/graph_transformer_mgr.h
index dc3e264b69..d221638eaf 100644
--- a/onnxruntime/core/graph/graph_transformer_mgr.h
+++ b/onnxruntime/core/optimizer/graph_transformer_mgr.h
@@ -3,7 +3,7 @@
 
 #pragma once
 
-#include "core/graph/graph_transformer.h"
+#include "core/optimizer/graph_transformer.h"
 
 namespace onnxruntime {
 // Manages a list of graph transformers. It is initialized with a list of graph
diff --git a/onnxruntime/core/graph/identity_elimination.cc b/onnxruntime/core/optimizer/identity_elimination.cc
similarity index 72%
rename from onnxruntime/core/graph/identity_elimination.cc
rename to onnxruntime/core/optimizer/identity_elimination.cc
index 3a045b6e0c..2b1c168da2 100644
--- a/onnxruntime/core/graph/identity_elimination.cc
+++ b/onnxruntime/core/optimizer/identity_elimination.cc
@@ -1,8 +1,12 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
-#include "core/graph/identity_elimination.h"
+#include "core/common/logging/logging.h"
+#include "core/graph/graph_viewer.h"
+#include "core/graph/op.h"
 #include "core/graph/graph_utils.h"
+#include "core/optimizer/rewrite_rule.h"
+#include "core/optimizer/identity_elimination.h"
 
 namespace onnxruntime {
 
diff --git a/onnxruntime/core/graph/identity_elimination.h b/onnxruntime/core/optimizer/identity_elimination.h
similarity index 92%
rename from onnxruntime/core/graph/identity_elimination.h
rename to onnxruntime/core/optimizer/identity_elimination.h
index 2f7d4402b3..55862dd2ad 100644
--- a/onnxruntime/core/graph/identity_elimination.h
+++ b/onnxruntime/core/optimizer/identity_elimination.h
@@ -3,7 +3,7 @@
 
 #pragma once
 
-#include "core/graph/rewrite_rule.h"
+#include "core/optimizer/rewrite_rule.h"
 
 namespace onnxruntime {
 
diff --git a/onnxruntime/core/graph/initializer.h b/onnxruntime/core/optimizer/initializer.h
similarity index 100%
rename from onnxruntime/core/graph/initializer.h
rename to onnxruntime/core/optimizer/initializer.h
diff --git a/onnxruntime/core/framework/insert_cast_transformer.cc b/onnxruntime/core/optimizer/insert_cast_transformer.cc
similarity index 99%
rename from onnxruntime/core/framework/insert_cast_transformer.cc
rename to onnxruntime/core/optimizer/insert_cast_transformer.cc
index 1248cfdc35..6c36f62015 100644
--- a/onnxruntime/core/framework/insert_cast_transformer.cc
+++ b/onnxruntime/core/optimizer/insert_cast_transformer.cc
@@ -1,7 +1,7 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
-#include "core/framework/insert_cast_transformer.h"
+#include "core/optimizer/insert_cast_transformer.h"
 #include "core/framework/data_types.h"
 
 using namespace ONNX_NAMESPACE;
diff --git a/onnxruntime/core/framework/insert_cast_transformer.h b/onnxruntime/core/optimizer/insert_cast_transformer.h
similarity index 97%
rename from onnxruntime/core/framework/insert_cast_transformer.h
rename to onnxruntime/core/optimizer/insert_cast_transformer.h
index cb6ca54673..a8d27223da 100644
--- a/onnxruntime/core/framework/insert_cast_transformer.h
+++ b/onnxruntime/core/optimizer/insert_cast_transformer.h
@@ -2,10 +2,10 @@
 // Licensed under the MIT License.
 
 #pragma once
-#include "core/graph/graph_viewer.h"
-#include "core/graph/graph_transformer.h"
 #include "core/common/common.h"
+#include "core/graph/graph_viewer.h"
 #include "core/framework/op_kernel.h"
+#include "core/optimizer/graph_transformer.h"
 
 namespace onnxruntime {
 class InsertCastTransformer : public onnxruntime::GraphTransformer {
diff --git a/onnxruntime/core/graph/matmul_add_fusion.cc b/onnxruntime/core/optimizer/matmul_add_fusion.cc
similarity index 97%
rename from onnxruntime/core/graph/matmul_add_fusion.cc
rename to onnxruntime/core/optimizer/matmul_add_fusion.cc
index 729b9ff2a3..90b6df2769 100644
--- a/onnxruntime/core/graph/matmul_add_fusion.cc
+++ b/onnxruntime/core/optimizer/matmul_add_fusion.cc
@@ -1,8 +1,8 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
-#include "core/graph/initializer.h"
-#include "core/graph/matmul_add_fusion.h"
+#include "core/optimizer/initializer.h"
+#include "core/optimizer/matmul_add_fusion.h"
 #include "core/graph/graph_utils.h"
 #include <deque>
 
diff --git a/onnxruntime/core/graph/matmul_add_fusion.h b/onnxruntime/core/optimizer/matmul_add_fusion.h
similarity index 90%
rename from onnxruntime/core/graph/matmul_add_fusion.h
rename to onnxruntime/core/optimizer/matmul_add_fusion.h
index 74b43da5c5..efc9026bdd 100644
--- a/onnxruntime/core/graph/matmul_add_fusion.h
+++ b/onnxruntime/core/optimizer/matmul_add_fusion.h
@@ -3,7 +3,7 @@
 
 #pragma once
 
-#include "core/graph/graph_transformer.h"
+#include "core/optimizer/graph_transformer.h"
 
 namespace onnxruntime {
 
diff --git a/onnxruntime/core/graph/slice_elimination.cc b/onnxruntime/core/optimizer/slice_elimination.cc
similarity index 97%
rename from onnxruntime/core/graph/slice_elimination.cc
rename to onnxruntime/core/optimizer/slice_elimination.cc
index b45accf6aa..13148a85ee 100644
--- a/onnxruntime/core/graph/slice_elimination.cc
+++ b/onnxruntime/core/optimizer/slice_elimination.cc
@@ -1,7 +1,7 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
-#include "core/graph/slice_elimination.h"
+#include "core/optimizer/slice_elimination.h"
 #include "core/graph/graph.h"
 #include "core/graph/graph_utils.h"
 #include "core/graph/op.h"
diff --git a/onnxruntime/core/graph/slice_elimination.h b/onnxruntime/core/optimizer/slice_elimination.h
similarity index 93%
rename from onnxruntime/core/graph/slice_elimination.h
rename to onnxruntime/core/optimizer/slice_elimination.h
index 66fc3e94a7..d429af6a62 100644
--- a/onnxruntime/core/graph/slice_elimination.h
+++ b/onnxruntime/core/optimizer/slice_elimination.h
@@ -3,7 +3,7 @@
 
 #pragma once
 
-#include "core/graph/rewrite_rule.h"
+#include "core/optimizer/rewrite_rule.h"
 
 namespace onnxruntime {
 
diff --git a/onnxruntime/core/framework/transformer_memcpy.cc b/onnxruntime/core/optimizer/transformer_memcpy.cc
similarity index 100%
rename from onnxruntime/core/framework/transformer_memcpy.cc
rename to onnxruntime/core/optimizer/transformer_memcpy.cc
diff --git a/onnxruntime/core/framework/transformer_memcpy.h b/onnxruntime/core/optimizer/transformer_memcpy.h
similarity index 95%
rename from onnxruntime/core/framework/transformer_memcpy.h
rename to onnxruntime/core/optimizer/transformer_memcpy.h
index 9f53af95e1..815660602e 100644
--- a/onnxruntime/core/framework/transformer_memcpy.h
+++ b/onnxruntime/core/optimizer/transformer_memcpy.h
@@ -6,7 +6,7 @@
 #include "core/common/common.h"
 #include "core/framework/op_kernel.h"
 #include "core/framework/kernel_registry_manager.h"
-#include "core/graph/graph_transformer.h"
+#include "core/optimizer/graph_transformer.h"
 
 namespace onnxruntime {
 
diff --git a/onnxruntime/core/graph/unsqueeze_elimination.cc b/onnxruntime/core/optimizer/unsqueeze_elimination.cc
similarity index 98%
rename from onnxruntime/core/graph/unsqueeze_elimination.cc
rename to onnxruntime/core/optimizer/unsqueeze_elimination.cc
index 6c9b0a86bc..bbfe45f80b 100644
--- a/onnxruntime/core/graph/unsqueeze_elimination.cc
+++ b/onnxruntime/core/optimizer/unsqueeze_elimination.cc
@@ -1,7 +1,7 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
-#include "core/graph/unsqueeze_elimination.h"
+#include "core/optimizer/unsqueeze_elimination.h"
 
 using namespace onnx;
 using namespace ::onnxruntime::common;
diff --git a/onnxruntime/core/graph/unsqueeze_elimination.h b/onnxruntime/core/optimizer/unsqueeze_elimination.h
similarity index 91%
rename from onnxruntime/core/graph/unsqueeze_elimination.h
rename to onnxruntime/core/optimizer/unsqueeze_elimination.h
index d57d583007..74ca280985 100644
--- a/onnxruntime/core/graph/unsqueeze_elimination.h
+++ b/onnxruntime/core/optimizer/unsqueeze_elimination.h
@@ -3,7 +3,7 @@
 
 #pragma once
 
-#include "core/graph/graph_transformer.h"
+#include "core/optimizer/graph_transformer.h"
 
 namespace onnxruntime {
 
diff --git a/onnxruntime/core/providers/cpu/cpu_execution_provider.h b/onnxruntime/core/providers/cpu/cpu_execution_provider.h
index 408c4f801c..b8580a3d89 100644
--- a/onnxruntime/core/providers/cpu/cpu_execution_provider.h
+++ b/onnxruntime/core/providers/cpu/cpu_execution_provider.h
@@ -5,7 +5,6 @@
 
 #include "core/framework/allocatormgr.h"
 #include "core/framework/execution_provider.h"
-#include "core/graph/graph_transformer.h"
 #include "core/graph/constants.h"
 
 namespace onnxruntime {
diff --git a/onnxruntime/core/providers/cuda/cuda_execution_provider.cc b/onnxruntime/core/providers/cuda/cuda_execution_provider.cc
index 1b31c132c0..db8a5c88d3 100644
--- a/onnxruntime/core/providers/cuda/cuda_execution_provider.cc
+++ b/onnxruntime/core/providers/cuda/cuda_execution_provider.cc
@@ -3,7 +3,6 @@
 
 #include "cuda_common.h"
 #include "cuda_execution_provider.h"
-#include "core/framework/transformer_memcpy.h"
 #include "core/framework/memcpy.h"
 #include "cuda_fence.h"
 #include "cuda_allocator.h"
diff --git a/onnxruntime/core/providers/cuda/cuda_execution_provider.h b/onnxruntime/core/providers/cuda/cuda_execution_provider.h
index 6e4faf644c..60bd5bc9cd 100644
--- a/onnxruntime/core/providers/cuda/cuda_execution_provider.h
+++ b/onnxruntime/core/providers/cuda/cuda_execution_provider.h
@@ -4,7 +4,7 @@
 #pragma once
 #include "cuda_pch.h"
 #include "core/platform/ort_mutex.h"
-#include "core/graph/graph_transformer.h"
+#include "core/graph/constants.h"
 #include "core/framework/allocatormgr.h"
 #include "core/framework/execution_provider.h"
 #include "shared_inc/cuda_utils.h"
diff --git a/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.cc b/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.cc
index 673aa19e41..5862e21e5f 100644
--- a/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.cc
+++ b/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.cc
@@ -4,7 +4,6 @@
 #include "mkldnn_allocator.h"
 #include "mkldnn_execution_provider.h"
 #include "core/framework/allocator.h"
-#include "core/framework/transformer_memcpy.h"
 #include "core/framework/memcpy.h"
 #include "core/framework/kernel_registry.h"
 #include "mkldnn_fwd.h"
diff --git a/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.h b/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.h
index 2727dff6c0..a85446ea1b 100644
--- a/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.h
+++ b/onnxruntime/core/providers/mkldnn/mkldnn_execution_provider.h
@@ -9,9 +9,9 @@
 #include <memory.h>
 
 #include "core/platform/ort_mutex.h"
+#include "core/graph/constants.h"
 #include "core/framework/allocatormgr.h"
 #include "core/framework/execution_provider.h"
-#include "core/graph/graph_transformer.h"
 
 namespace mkldnn {
 struct memory;
diff --git a/onnxruntime/core/session/inference_session.cc b/onnxruntime/core/session/inference_session.cc
index b8b9328e5a..93d98b8aad 100644
--- a/onnxruntime/core/session/inference_session.cc
+++ b/onnxruntime/core/session/inference_session.cc
@@ -16,8 +16,6 @@
 #include "core/common/logging/logging.h"
 #include "core/common/task_thread_pool.h"
 #include "core/graph/graph_viewer.h"
-#include "core/graph/graph_transformer.h"
-#include "core/graph/graph_transformer_mgr.h"
 #include "core/graph/graph_utils.h"
 #include "core/graph/model.h"
 #include "core/framework/allocatormgr.h"
@@ -25,7 +23,6 @@
 #include "core/framework/environment.h"
 #include "core/framework/execution_frame.h"
 #include "core/framework/graph_partitioner.h"
-#include "core/framework/insert_cast_transformer.h"
 #include "core/framework/kernel_def_builder.h"
 #include "core/framework/kernel_registry.h"
 #include "core/framework/ml_value_patterns_planner.h"
@@ -37,8 +34,11 @@
 #include "core/framework/session_state_initializer.h"
 #include "core/framework/tensorprotoutils.h"
 #include "core/framework/tensorutils.h"
-#include "core/framework/transformer_memcpy.h"
 #include "core/framework/utils.h"
+#include "core/optimizer/graph_transformer.h"
+#include "core/optimizer/graph_transformer_mgr.h"
+#include "core/optimizer/insert_cast_transformer.h"
+#include "core/optimizer/transformer_memcpy.h"
 #include "core/platform/notification.h"
 #include "core/providers/cpu/cpu_execution_provider.h"
 #include "core/session/CustomOpsLoader.h"
diff --git a/onnxruntime/test/framework/insert_cast_transformer_test.cc b/onnxruntime/test/framework/insert_cast_transformer_test.cc
index 2ddbf796be..a4a0ffebdf 100644
--- a/onnxruntime/test/framework/insert_cast_transformer_test.cc
+++ b/onnxruntime/test/framework/insert_cast_transformer_test.cc
@@ -3,7 +3,7 @@
 
 #include "core/framework/allocatormgr.h"
 #include "core/framework/allocator.h"
-#include "core/framework/insert_cast_transformer.h"
+#include "core/optimizer/insert_cast_transformer.h"
 #include "core/graph/model.h"
 #include "gtest/gtest.h"
 #include "test_utils.h"
diff --git a/onnxruntime/test/framework/memcpy_transformer_test.cc b/onnxruntime/test/framework/memcpy_transformer_test.cc
index 8ca68c6c12..59f97ffb11 100644
--- a/onnxruntime/test/framework/memcpy_transformer_test.cc
+++ b/onnxruntime/test/framework/memcpy_transformer_test.cc
@@ -2,8 +2,9 @@
 // Licensed under the MIT License.
 
 #include <iterator>
-#include "core/framework/transformer_memcpy.h"
+
 #include "core/framework/execution_providers.h"
+#include "core/optimizer/transformer_memcpy.h"
 #include "core/graph/model.h"
 #include "gtest/gtest.h"
 #include "test_utils.h"
diff --git a/onnxruntime/test/ir/graph_transform_test.cc b/onnxruntime/test/ir/graph_transform_test.cc
index 042e9b7c85..d5671b621b 100644
--- a/onnxruntime/test/ir/graph_transform_test.cc
+++ b/onnxruntime/test/ir/graph_transform_test.cc
@@ -4,19 +4,18 @@
 #include "core/session/inference_session.h"
 #include "core/graph/graph_viewer.h"
 #include "core/graph/model.h"
-#include "core/graph/graph_transformer.h"
-#include "core/graph/graph_transformer_mgr.h"
-#include "core/graph/identity_elimination.h"
-#include "core/graph/slice_elimination.h"
-#include "core/graph/unsqueeze_elimination.h"
-#include "core/graph/conv_bn_fusion.h"
-#include "core/graph/conv_mul_fusion.h"
-#include "core/graph/conv_add_fusion.h"
-#include "core/graph/conv_activation_fusion.h"
-#include "core/graph/matmul_add_fusion.h"
-#include "core/graph/gemm_activation_fusion.h"
+#include "core/optimizer/graph_transformer.h"
+#include "core/optimizer/graph_transformer_mgr.h"
+#include "core/optimizer/identity_elimination.h"
+#include "core/optimizer/slice_elimination.h"
+#include "core/optimizer/unsqueeze_elimination.h"
+#include "core/optimizer/conv_bn_fusion.h"
+#include "core/optimizer/conv_mul_fusion.h"
+#include "core/optimizer/conv_add_fusion.h"
+#include "core/optimizer/conv_activation_fusion.h"
+#include "core/optimizer/matmul_add_fusion.h"
+#include "core/optimizer/gemm_activation_fusion.h"
 #include "core/platform/env.h"
-
 #include "test/capturing_sink.h"
 #include "test/test_environment.h"
 #include "gtest/gtest.h"