saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-05-24 22:17:32 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

Force using fixed random seeds for flaky tests (#15515)

Browse source 
Some gradient-related tests fail frequently due to their math
properties. This PR fixes their random seed so that it's possible to
debug in the future.

Fixed
[AB#14605](https://aiinfra.visualstudio.com/6a833879-cd9b-44a4-a9de-adc2d818f13c/_workitems/edit/14605),
[AB#14604](https://aiinfra.visualstudio.com/6a833879-cd9b-44a4-a9de-adc2d818f13c/_workitems/edit/14604)
This commit is contained in:
Wei-Sheng Chin 2023-04-14 18:44:51 -07:00 committed by GitHub
parent 5ebe700a9b
commit ac6ceffb2c
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 9 additions and 6 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

7
orttraining/orttraining/test/gradient/gradient_checker.cc
View file

@ -465,9 +465,10 @@ inline Status GradientChecker<X_T, Y_T, JAC_T>::ComputeGradientError(
bool check_not_have_shape_inferencing /* = false*/,
std::vector<std::unique_ptr<IExecutionProvider>>* execution_providers /* = nullptr */) {
// TODO: Consider varying mean and variance
float scale = 5.f;
float mean = 0.f;
const auto seed = GetTestRandomSeed();
constexpr float scale = 5.f;
constexpr float mean = 0.f;
constexpr int seed = 5566;
// Use fixed random since numerically compute gradient is not stable.
std::default_random_engine generator{gsl::narrow_cast<decltype(generator)::result_type>(seed)};
std::normal_distribution<X_T> distribution{mean, scale};

2
orttraining/orttraining/test/gradient/gradient_ops_test.cc
View file

@ -1385,7 +1385,7 @@ TEST(GradientCheckerTest, UnsqueezeGrad) {
// TODO: Reshape missing
TEST(GradientCheckerTest, DISABLED_BatchNormalizationGrad) {
TEST(GradientCheckerTest, BatchNormalizationGrad) {
float max_error;
GradientChecker<float, float, float> gradient_checker;
OpDef op_def{"BatchNormInternal", kMSDomain, 1};

6
orttraining/orttraining/test/training_ops/cuda/softmax_test.cc
View file

@ -24,8 +24,10 @@ static void TestSoftmax(const std::vector<int64_t>& X_dims,
CompareOpTester test(op);
test.AddAttribute<int64_t>("axis", axis);
// Use fixed random seed because those tests are not stable.
// It's impossible to debug if the test fails randomly.
RandomValueGenerator random{5566};
// create rand inputs
RandomValueGenerator random{};
std::vector<T> X_data = random.Uniform<T>(X_dims, -10.0f, 10.0f);
test.AddInput<T>("X", X_dims, X_data);
@ -156,7 +158,7 @@ static void TestSoftmaxGrad(const std::vector<int64_t>& dY_dims,
test.AddAttribute<int64_t>("axis", axis);
// create rand inputs
RandomValueGenerator random{};
RandomValueGenerator random{5566};
std::vector<T> dY_data = random.Uniform<T>(dY_dims, -1.0f, 1.0f);
// Add 1e-2 for numerical stability to prevent zero probability.
std::vector<T> Y_data = random.Uniform<T>(Y_dims, -1.02f, 1.02f);