diff --git a/orttraining/orttraining/test/gradient/gradient_ops_test.cc b/orttraining/orttraining/test/gradient/gradient_ops_test.cc
index c4040454df..e0043eb281 100644
--- a/orttraining/orttraining/test/gradient/gradient_ops_test.cc
+++ b/orttraining/orttraining/test/gradient/gradient_ops_test.cc
@@ -601,7 +601,20 @@ TEST(GradientCheckerTest, ReduceLogSumExpGrad) {
 }
 
 TEST(GradientCheckerTest, ReluGrad) {
-  UnaryOpGradientTest("Relu");
+  TensorShape shape({2, 3, 4});
+  float max_error;
+  float error_tolerance = 1e-3f;
+  GradientChecker<float, float, float> gradient_checker;
+  OpDef op_def{"Relu"};
+
+  // Exclude input data at 0, since Relu is not smooth at 0
+  std::function<float(float)> transformer = [](float x) { return x > 0 ? x + 0.2f : x - 0.2f; };
+  TensorInfo x_info(shape, true, &transformer);
+  TensorInfo y_info(shape);
+
+  gradient_checker.ComputeGradientError(op_def, {x_info}, {y_info}, &max_error);
+
+  EXPECT_IS_TINIER_THAN(max_error, error_tolerance);
 }
 
 #ifndef USE_CUDA
diff --git a/orttraining/orttraining/test/training_ops/cuda/layer_norm_test.cc b/orttraining/orttraining/test/training_ops/cuda/layer_norm_test.cc
index fe5b5ff202..4e14b20d8d 100644
--- a/orttraining/orttraining/test/training_ops/cuda/layer_norm_test.cc
+++ b/orttraining/orttraining/test/training_ops/cuda/layer_norm_test.cc
@@ -106,7 +106,7 @@ TEST(CudaKernelTest, LayerNormGrad_SmallSizeTensor_IntermediateAxis) {
 
 TEST(CudaKernelTest, LayerNormGrad_MidSizeTensor) {
   const std::vector<int64_t> X_dims{8, 80, 768};
-  TestLayerNormGrad(X_dims, LAYER_NORM_GRAD_OP);
+  TestLayerNormGrad(X_dims, LAYER_NORM_GRAD_OP, 1, 5e-3);
 }
 
 TEST(CudaKernelTest, LayerNormGrad_LargeSizeTensor) {
@@ -139,7 +139,7 @@ static void TestInvertibleLayerNormGrad(
     const std::vector<int64_t>& x_dims,
     int64_t axis = -1,
     double error_tolerance = 1e-4,
-    bool test_fp16=false) {
+    bool test_fp16 = false) {
   const std::vector<int64_t>& n_x_m_dims = x_dims;
   std::vector<int64_t> n_dims, m_dims;
   ASSERT_TRUE(SplitDims(n_x_m_dims, axis, n_dims, m_dims).IsOK());
@@ -244,7 +244,7 @@ TEST(CudaKernelTest, InvertibleLayerNormGrad_SmallSizeTensor_IntermediateAxis) {
 
 TEST(CudaKernelTest, InvertibleLayerNormGrad_MidSizeTensor) {
   const std::vector<int64_t> X_dims{8, 80, 768};
-  TestInvertibleLayerNormGrad(X_dims);
+  TestInvertibleLayerNormGrad(X_dims, 1, 5e-3);
 }
 
 TEST(CudaKernelTest, InvertibleLayerNormGrad_LargeSizeTensor) {