diff --git a/orttraining/orttraining/core/graph/gradient_builder_base.h b/orttraining/orttraining/core/graph/gradient_builder_base.h
index 67f32fe3ec..2d8a87f6d4 100644
--- a/orttraining/orttraining/core/graph/gradient_builder_base.h
+++ b/orttraining/orttraining/core/graph/gradient_builder_base.h
@@ -269,6 +269,10 @@ class GradientBuilderBase {
       return ConstantScalarNode(BFloat16(value), {1}, arg_name);
     }
 
+    if (elem_type == ONNX_NAMESPACE::TensorProto_DataType_DOUBLE) {
+      return ConstantScalarNode(double(value), {1}, arg_name);
+    }
+
 #if !defined(DISABLE_FLOAT8_TYPES)
 
     if (elem_type == ONNX_NAMESPACE::TensorProto_DataType_FLOAT8E4M3FN) {
@@ -289,6 +293,9 @@ class GradientBuilderBase {
 
 #endif
 
+    ORT_ENFORCE(elem_type == ONNX_NAMESPACE::TensorProto_DataType_FLOAT,
+                "Unsupported element type for constant node: ", elem_type);
+
     return ConstantScalarNode(value, {1}, arg_name);
   }