Loosen tolerance of CudaKernelTest.ReduceSum_MidTensor, allow test random seed to be regenerated within a test run. (#5675)

onnxruntime/core/platform/env_var_utils.h

@@ -0,0 +1,44 @@
+// Copyright (c) Microsoft Corporation. All rights reserved.
+// Licensed under the MIT License.
+
+#pragma once
+
+#include <sstream>
+
+#include "core/common/common.h"
+#include "core/common/optional.h"
+#include "core/platform/env.h"
+
+namespace onnxruntime {
+/**
+ * Parses an environment variable value if available (defined and not empty).
+ */
+template <typename T>
+optional<T> ParseEnvironmentVariable(const std::string& name) {
+  const std::string value_str = Env::Default().GetEnvironmentVar(name);
+  if (value_str.empty()) {
+    return {};
+  }
+
+  std::istringstream is{value_str};
+  T parsed_value;
+  ORT_ENFORCE(
+      is >> std::noskipws >> parsed_value && is.eof(),
+      "Failed to parse environment variable - name: \"", name, "\", value: \"", value_str, "\"");
+
+  return parsed_value;
+}
+
+/**
+ * Parses an environment variable value or returns the given default if unavailable.
+ */
+template <typename T>
+T ParseEnvironmentVariable(const std::string& name, const T& default_value) {
+  const auto parsed = ParseEnvironmentVariable<T>(name);
+  if (parsed.has_value()) {
+    return parsed.value();
+  }
+
+  return default_value;
+}
+}  // namespace onnxruntime

onnxruntime/test/common/tensor_op_test_utils.cc

@@ -2,13 +2,15 @@
 // Licensed under the MIT License.
 
 #include "test/common/tensor_op_test_utils.h"
+#include "test/util/include/test_random_seed.h"
 
 namespace onnxruntime {
 namespace test {
 
-RandomValueGenerator::RandomValueGenerator()
-    : random_seed_{GetTestRandomSeed()},
-      generator_{static_cast<decltype(generator_)::result_type>(random_seed_)},
+RandomValueGenerator::RandomValueGenerator(optional<RandomSeedType> seed)
+    : random_seed_{
+          seed.has_value() ? seed.value() : static_cast<RandomSeedType>(GetTestRandomSeed())},
+      generator_{random_seed_},
       output_trace_{__FILE__, __LINE__, "ORT test random seed: " + std::to_string(random_seed_)} {
 }
 

onnxruntime/test/common/tensor_op_test_utils.h

@@ -9,8 +9,8 @@
 #include "gtest/gtest.h"
 
 #include "core/common/common.h"
+#include "core/common/optional.h"
 #include "core/util/math.h"
-#include "test/util/include/test_random_seed.h"
 
 namespace onnxruntime {
 namespace test {
@@ -26,7 +26,14 @@ inline int64_t SizeFromDims(const std::vector<int64_t>& dims) {
 
 class RandomValueGenerator {
  public:
-  RandomValueGenerator();
+  using RandomEngine = std::default_random_engine;
+  using RandomSeedType = RandomEngine::result_type;
+
+  explicit RandomValueGenerator(optional<RandomSeedType> seed = {});
+
+  RandomSeedType GetRandomSeed() const {
+    return random_seed_;
+  }
 
   // Random values generated are in the range [min, max).
   template <typename TFloat>
@@ -112,7 +119,7 @@ class RandomValueGenerator {
 
  private:
   const RandomSeedType random_seed_;
-  std::default_random_engine generator_;
+  RandomEngine generator_;
   // while this instance is in scope, output some context information on test failure like the random seed value
   const ::testing::ScopedTrace output_trace_;
 };

onnxruntime/test/util/include/test_random_seed.h

@@ -8,27 +8,23 @@
 namespace onnxruntime {
 namespace test {
 
+// These variables control the behavior of GetTestRandomSeed().
+namespace test_random_seed_env_vars {
+// Specifies a fixed seed value to return.
+// If set, this has the highest precedence.
+constexpr const char* kValue = "ORT_TEST_RANDOM_SEED_VALUE";
+// If set (and not using a fixed value), specifies that a new seed value is returned each time.
+// The default behavior is to return the same cached seed value per process.
+// This is useful when repeatedly running flaky tests to reproduce errors.
+constexpr const char* kDoNotCache = "ORT_TEST_RANDOM_SEED_DO_NOT_CACHE";
+}  // namespace test_random_seed_env_vars
+
 using RandomSeedType = uint32_t;
 
-// Possible improvement:
-// We could make this a bit nicer by setting the seed with a GTest
-// ::testing::Environment and registering that as a global environment.
-// That way we could get a different generated seed on each test run when using
-// --gtest_repeat.
-// That was the initial approach, but there were some issues with the Mac CI
-// build in onnxruntime_shared_lib_test.
-
 /**
- * Gets the test random seed value which does not change during the test run.
- * The random seed value is obtained as follows, in order:
- * 1. environment variable ORT_TEST_RANDOM_SEED, if available and valid
- * 2. generated from current time
+ * Gets a test random seed value.
  */
 RandomSeedType GetTestRandomSeed();
 
-inline const char* GetTestRandomSeedEnvironmentVariableName() {
-  return "ORT_TEST_RANDOM_SEED";
-}
-
 }  // namespace test
 }  // namespace onnxruntime

onnxruntime/test/util/test_random_seed.cc

@@ -4,43 +4,35 @@
 #include "test/util/include/test_random_seed.h"
 
 #include <chrono>
-#include <iostream>
-#include <sstream>
 
-#include "core/platform/env.h"
+#include "core/platform/env_var_utils.h"
 
 namespace onnxruntime {
 namespace test {
 
-namespace {
-RandomSeedType LoadRandomSeed() {
-  // parse from environment variable
-  {
-    const std::string value_str = Env::Default().GetEnvironmentVar(
-        GetTestRandomSeedEnvironmentVariableName());
-
-    if (!value_str.empty()) {
-      std::istringstream is{value_str};
-      RandomSeedType parsed_value;
-      if (is >> std::noskipws >> parsed_value && is.eof()) {
-        return parsed_value;
-      } else {
-        std::cerr << GetTestRandomSeedEnvironmentVariableName()
-                  << " was set but not able to be parsed: \""
-                  << value_str << "\"\n";
-      }
-    }
+RandomSeedType GetTestRandomSeed() {
+  static const auto fixed_random_seed =
+      ParseEnvironmentVariable<RandomSeedType>(test_random_seed_env_vars::kValue);
+  if (fixed_random_seed.has_value()) {
+    // use fixed value
+    return fixed_random_seed.value();
   }
 
-  // generate from time
-  return static_cast<RandomSeedType>(
-      std::chrono::steady_clock::now().time_since_epoch().count());
-}
-}  // namespace
+  auto generate_from_time = []() {
+    return static_cast<RandomSeedType>(
+        std::chrono::steady_clock::now().time_since_epoch().count());
+  };
 
-RandomSeedType GetTestRandomSeed() {
-  static const RandomSeedType test_random_seed = LoadRandomSeed();
-  return test_random_seed;
+  static const auto use_cached =
+      !ParseEnvironmentVariable<bool>(test_random_seed_env_vars::kDoNotCache, false);
+  if (use_cached) {
+    // initially generate from current time
+    static const auto static_random_seed = generate_from_time();
+    return static_random_seed;
+  }
+
+  // generate from current time
+  return generate_from_time();
 }
 
 }  // namespace test

orttraining/orttraining/test/training_ops/cpu/tensor/gather_grad_op_test.cc

@@ -8,6 +8,7 @@
 #include "test/common/cuda_op_test_utils.h"
 #include "test/common/tensor_op_test_utils.h"
 #include "test/providers/provider_test_utils.h"
+#include "test/util/include/test_random_seed.h"
 
 namespace onnxruntime {
 namespace test {
@@ -46,6 +47,7 @@ void ConfigureGatherGradRandomDataOpTester(
     int64_t axis,
     const TensorShape& X_shape,
     const TensorShape& indices_shape,
+    optional<RandomValueGenerator::RandomSeedType> random_seed,
     OpTester& test) {
   ASSERT_LE(0, axis);
   ASSERT_LT(static_cast<size_t>(axis), X_shape.NumDimensions());
@@ -58,7 +60,7 @@ void ConfigureGatherGradRandomDataOpTester(
     return TensorShape(dY_dims);
   }();
 
-  RandomValueGenerator random{};
+  RandomValueGenerator random{random_seed};
   const auto grad = random.Uniform<T>(dY_shape.GetDims(), T{1}, T{10});
   const auto indices = random.Uniform<int64_t>(indices_shape.GetDims(), 0, X_shape[axis]);
   const auto output = CalculateOutput(axis, X_shape, grad, indices);
@@ -71,6 +73,15 @@ void ConfigureGatherGradRandomDataOpTester(
   test.AddOutput<T>("output", X_shape.GetDims(), output);
 }
 
+template <typename T>
+void ConfigureGatherGradRandomDataOpTester(
+    int64_t axis,
+    const TensorShape& X_shape,
+    const TensorShape& indices_shape,
+    OpTester& test) {
+  ConfigureGatherGradRandomDataOpTester<T>(axis, X_shape, indices_shape, {}, test);
+}
+
 template <typename T>
 void RunGatherGradTestWithRandomData(
     int64_t axis,
@@ -165,10 +176,11 @@ void RunGatherGradConsistentOutputTest(
     int64_t axis,
     const TensorShape& X_shape,
     const TensorShape& indices_shape) {
+  const auto random_seed = static_cast<RandomValueGenerator::RandomSeedType>(GetTestRandomSeed());
   std::map<std::string, std::vector<std::vector<float>>> provider_outputs;
   for (int i = 0; i < 2; ++i) {
     OpTester test("GatherGrad", 1, kMSDomain);
-    ConfigureGatherGradRandomDataOpTester<float>(axis, X_shape, indices_shape, test);
+    ConfigureGatherGradRandomDataOpTester<float>(axis, X_shape, indices_shape, random_seed, test);
 
     auto output_handler =
         [&provider_outputs](const std::vector<OrtValue>& fetches, const std::string& provider_type) {

orttraining/orttraining/test/training_ops/cuda/reduce_sum_test.cc

@@ -62,7 +62,8 @@ TEST(CudaKernelTest, ReduceSum_MidTensor) {
   std::vector<int64_t> Y_dims{3072};
   std::vector<int64_t> axes{0, 1};
   bool keepdims = false;
-  TestReduceSum(X_dims, Y_dims, axes, keepdims);
+  double per_sample_tolerance = 4e-4;
+  TestReduceSum(X_dims, Y_dims, axes, keepdims, per_sample_tolerance);
 }
 
 TEST(CudaKernelTest, ReduceSum_LargeTensor) {