diff --git a/onnxruntime/core/optimizer/compute_optimizer/compute_optimizer.cc b/onnxruntime/core/optimizer/compute_optimizer/upstream_gather.cc
similarity index 99%
rename from onnxruntime/core/optimizer/compute_optimizer/compute_optimizer.cc
rename to onnxruntime/core/optimizer/compute_optimizer/upstream_gather.cc
index 5831576d7b..ec43c4da3c 100644
--- a/onnxruntime/core/optimizer/compute_optimizer/compute_optimizer.cc
+++ b/onnxruntime/core/optimizer/compute_optimizer/upstream_gather.cc
@@ -1,8 +1,6 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
-// TODO(pengwa): rename to upstream_gather.cc later, keep original name for now to make diff easier to review.
-
 #ifdef ENABLE_TRAINING_CORE
 
 #include <onnx/defs/attr_proto_util.h>
@@ -10,10 +8,11 @@
 #include "core/optimizer/initializer.h"
 #include "core/optimizer/utils.h"
 #include "core/optimizer/compute_optimizer/upstream_gather_actors.h"
-#include "core/optimizer/compute_optimizer/compute_optimizer.h"
+#include "core/optimizer/compute_optimizer/upstream_gather.h"
 #include "core/optimizer/compute_optimizer/upstream_transformer_base.h"
 
 using namespace onnxruntime::optimizer::compute_optimizer;
+
 namespace onnxruntime {
 
 UpStreamGatherGraphTransformer::UpStreamGatherGraphTransformer(
diff --git a/onnxruntime/core/optimizer/compute_optimizer/compute_optimizer.h b/onnxruntime/core/optimizer/compute_optimizer/upstream_gather.h
similarity index 98%
rename from onnxruntime/core/optimizer/compute_optimizer/compute_optimizer.h
rename to onnxruntime/core/optimizer/compute_optimizer/upstream_gather.h
index b929ac3a90..5317174643 100644
--- a/onnxruntime/core/optimizer/compute_optimizer/compute_optimizer.h
+++ b/onnxruntime/core/optimizer/compute_optimizer/upstream_gather.h
@@ -1,8 +1,6 @@
 // Copyright (c) Microsoft Corporation. All rights reserved.
 // Licensed under the MIT License.
 
-// TODO(pengwa): rename to upstream_gather.h later, keep original name for now to make diff easier to review.
-
 // The optimization here ideally is applicable to both training and inferencing,
 // while so far we mainly validate on training during cooking the optimization.
 #ifdef ENABLE_TRAINING_CORE
diff --git a/onnxruntime/test/optimizer/compute_optimizer_test.cc b/onnxruntime/test/optimizer/compute_optimizer_test.cc
index 05fcebce52..5e4f2f0255 100644
--- a/onnxruntime/test/optimizer/compute_optimizer_test.cc
+++ b/onnxruntime/test/optimizer/compute_optimizer_test.cc
@@ -23,7 +23,7 @@
 #include "core/graph/model.h"
 
 #include "core/optimizer/common_subexpression_elimination.h"
-#include "core/optimizer/compute_optimizer/compute_optimizer.h"
+#include "core/optimizer/compute_optimizer/upstream_gather.h"
 #include "core/optimizer/utils.h"
 #include "core/platform/env.h"
 #include "core/session/inference_session.h"
diff --git a/orttraining/orttraining/core/optimizer/graph_transformer_utils.cc b/orttraining/orttraining/core/optimizer/graph_transformer_utils.cc
index 1326a7c61a..9906b2e3fd 100644
--- a/orttraining/orttraining/core/optimizer/graph_transformer_utils.cc
+++ b/orttraining/orttraining/core/optimizer/graph_transformer_utils.cc
@@ -10,7 +10,7 @@
 #include "core/optimizer/bias_softmax_fusion.h"
 #include "core/optimizer/cast_elimination.h"
 #include "core/optimizer/common_subexpression_elimination.h"
-#include "core/optimizer/compute_optimizer/compute_optimizer.h"
+#include "core/optimizer/compute_optimizer/upstream_gather.h"
 #include "core/optimizer/concat_slice_elimination.h"
 #include "core/optimizer/constant_folding.h"
 #include "core/optimizer/conv_activation_fusion.h"