diff --git a/orttraining/orttraining/test/gradient/gradient_ops_test.cc b/orttraining/orttraining/test/gradient/gradient_ops_test.cc
index 75d5d38f4d..13dfb56ebf 100644
--- a/orttraining/orttraining/test/gradient/gradient_ops_test.cc
+++ b/orttraining/orttraining/test/gradient/gradient_ops_test.cc
@@ -2156,7 +2156,8 @@ TEST(GradientUtilsTest, InPlaceAccumulatorV2Overwrite) {
   test.Run();
 }
 
-#if defined(USE_CUDA) || defined(USE_ROCM)
+#if defined(USE_CUDA)
+// TODO: Add rocm kernel defs
 TEST(GradientUtilsTest, InPlaceAccumulatorV2_GPU) {
   OpTester test("InPlaceAccumulatorV2", 1, onnxruntime::kMSDomain);
 
@@ -2174,7 +2175,7 @@ TEST(GradientUtilsTest, InPlaceAccumulatorV2_Float16) {
   std::vector<float> old_sum = {1.0f, 2.0f, 3.0f};
   std::vector<float> value = {4.0f, 5.0f, 6.0f};
   std::vector<float> new_sum = {4.0f, 5.0f, 6.0f};
- 
+
   std::vector<MLFloat16> value_half(3);
   ConvertFloatToMLFloat16(value.data(), value_half.data(), 3);
 
@@ -2187,7 +2188,9 @@ TEST(GradientUtilsTest, InPlaceAccumulatorV2_Float16) {
   // Didn't implement mixed precision InPlaceAccumulatorV2 in CPU
   test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCpuExecutionProvider});
 }
+#endif
 
+#if defined(USE_CUDA) || defined(USE_ROCM)
 TEST(GradientUtilsTest, InPlaceAccumulatorFloat16) {
   OpTester test("InPlaceAccumulator", 1, onnxruntime::kMSDomain);