From ac6ceffb2c13f1525fa2a52f61c3554cdea20fc0 Mon Sep 17 00:00:00 2001
From: Wei-Sheng Chin <wschin@outlook.com>
Date: Fri, 14 Apr 2023 18:44:51 -0700
Subject: [PATCH] Force using fixed random seeds for flaky tests (#15515)

Some gradient-related tests fail frequently due to their math
properties. This PR fixes their random seed so that it's possible to
debug in the future.

Fixed
[AB#14605](https://aiinfra.visualstudio.com/6a833879-cd9b-44a4-a9de-adc2d818f13c/_workitems/edit/14605),
[AB#14604](https://aiinfra.visualstudio.com/6a833879-cd9b-44a4-a9de-adc2d818f13c/_workitems/edit/14604)
---
 orttraining/orttraining/test/gradient/gradient_checker.cc  | 7 ++++---
 orttraining/orttraining/test/gradient/gradient_ops_test.cc | 2 +-
 .../orttraining/test/training_ops/cuda/softmax_test.cc     | 6 ++++--
 3 files changed, 9 insertions(+), 6 deletions(-)

diff --git a/orttraining/orttraining/test/gradient/gradient_checker.cc b/orttraining/orttraining/test/gradient/gradient_checker.cc
index 78c80b4cab..4adf31679d 100644
--- a/orttraining/orttraining/test/gradient/gradient_checker.cc
+++ b/orttraining/orttraining/test/gradient/gradient_checker.cc
@@ -465,9 +465,10 @@ inline Status GradientChecker<X_T, Y_T, JAC_T>::ComputeGradientError(
     bool check_not_have_shape_inferencing /* = false*/,
     std::vector<std::unique_ptr<IExecutionProvider>>* execution_providers /* = nullptr */) {
   // TODO: Consider varying mean and variance
-  float scale = 5.f;
-  float mean = 0.f;
-  const auto seed = GetTestRandomSeed();
+  constexpr float scale = 5.f;
+  constexpr float mean = 0.f;
+  constexpr int seed = 5566;
+  // Use fixed random since numerically compute gradient is not stable.
   std::default_random_engine generator{gsl::narrow_cast<decltype(generator)::result_type>(seed)};
   std::normal_distribution<X_T> distribution{mean, scale};
 
diff --git a/orttraining/orttraining/test/gradient/gradient_ops_test.cc b/orttraining/orttraining/test/gradient/gradient_ops_test.cc
index de849c5526..79d1e045f0 100644
--- a/orttraining/orttraining/test/gradient/gradient_ops_test.cc
+++ b/orttraining/orttraining/test/gradient/gradient_ops_test.cc
@@ -1385,7 +1385,7 @@ TEST(GradientCheckerTest, UnsqueezeGrad) {
 
 // TODO: Reshape missing
 
-TEST(GradientCheckerTest, DISABLED_BatchNormalizationGrad) {
+TEST(GradientCheckerTest, BatchNormalizationGrad) {
   float max_error;
   GradientChecker<float, float, float> gradient_checker;
   OpDef op_def{"BatchNormInternal", kMSDomain, 1};
diff --git a/orttraining/orttraining/test/training_ops/cuda/softmax_test.cc b/orttraining/orttraining/test/training_ops/cuda/softmax_test.cc
index 1eadc5e155..564a164973 100644
--- a/orttraining/orttraining/test/training_ops/cuda/softmax_test.cc
+++ b/orttraining/orttraining/test/training_ops/cuda/softmax_test.cc
@@ -24,8 +24,10 @@ static void TestSoftmax(const std::vector<int64_t>& X_dims,
   CompareOpTester test(op);
   test.AddAttribute<int64_t>("axis", axis);
 
+  // Use fixed random seed because those tests are not stable.
+  // It's impossible to debug if the test fails randomly.
+  RandomValueGenerator random{5566};
   // create rand inputs
-  RandomValueGenerator random{};
   std::vector<T> X_data = random.Uniform<T>(X_dims, -10.0f, 10.0f);
   test.AddInput<T>("X", X_dims, X_data);
 
@@ -156,7 +158,7 @@ static void TestSoftmaxGrad(const std::vector<int64_t>& dY_dims,
   test.AddAttribute<int64_t>("axis", axis);
 
   // create rand inputs
-  RandomValueGenerator random{};
+  RandomValueGenerator random{5566};
   std::vector<T> dY_data = random.Uniform<T>(dY_dims, -1.0f, 1.0f);
   // Add 1e-2 for numerical stability to prevent zero probability.
   std::vector<T> Y_data = random.Uniform<T>(Y_dims, -1.02f, 1.02f);