From 047975e4040ef4d7f0d2d5e5b8fec66141f0a8f6 Mon Sep 17 00:00:00 2001 From: edgchen1 <18449977+edgchen1@users.noreply.github.com> Date: Fri, 1 May 2020 09:18:26 -0700 Subject: [PATCH] Address flaky test ReduceApiTest.Sum. (#3716) Increase test comparison tolerance. Add output of random seed value for easier debugging later. Unify RandomValueGenerator::Uniform() to consistently use [min, max) interval. --- .../test/common/tensor_op_test_utils.h | 49 +++++--- .../cpu/reduction/reduction_ops_test.cc | 108 +++++++++--------- .../cpu/tensor/gather_grad_op_test.cc | 6 +- .../training_ops/cuda/cross_entropy_test.cc | 8 +- 4 files changed, 88 insertions(+), 83 deletions(-) diff --git a/onnxruntime/test/common/tensor_op_test_utils.h b/onnxruntime/test/common/tensor_op_test_utils.h index b925b1a446..e6b97e27ac 100644 --- a/onnxruntime/test/common/tensor_op_test_utils.h +++ b/onnxruntime/test/common/tensor_op_test_utils.h @@ -4,9 +4,11 @@ #pragma once #include +#include #include "gtest/gtest.h" +#include "core/common/common.h" #include "core/util/math.h" #include "test/providers/provider_test_utils.h" #include "test/util/include/test_random_seed.h" @@ -14,38 +16,50 @@ namespace onnxruntime { namespace test { +namespace detail { +inline int64_t SizeFromDims(const std::vector& dims) { + const int64_t size = std::accumulate( + dims.cbegin(), dims.cend(), static_cast(1), std::multiplies{}); + ORT_ENFORCE(size >= 0); + return size; +} +} // namespace detail + class RandomValueGenerator { public: RandomValueGenerator(); - // Random values generated are in the range [a, b). - template - inline std::vector Uniform(const std::vector& dims, float min, float max) { - int64_t size = std::accumulate(dims.cbegin(), dims.cend(), static_cast(1), std::multiplies{}); - std::vector val(size); - std::uniform_real_distribution distribution(min, max); + // Random values generated are in the range [min, max). + template + typename std::enable_if< + std::is_floating_point::value, + std::vector>::type + Uniform(const std::vector& dims, TFloat min, TFloat max) { + std::vector val(detail::SizeFromDims(dims)); + std::uniform_real_distribution distribution(min, max); for (size_t i = 0; i < val.size(); ++i) { - val[i] = T(distribution(generator_)); + val[i] = distribution(generator_); } return val; } - // Random values generated are in the range [a, b]. - template - inline std::vector Uniform(const std::vector& dims, int64_t min, int64_t max) { - int64_t size = std::accumulate(dims.cbegin(), dims.cend(), static_cast(1), std::multiplies{}); - std::vector val(size); - std::uniform_int_distribution distribution(min, max); + // Random values generated are in the range [min, max). + template + typename std::enable_if< + std::is_integral::value, + std::vector>::type + Uniform(const std::vector& dims, TInt min, TInt max) { + std::vector val(detail::SizeFromDims(dims)); + std::uniform_int_distribution distribution(min, max - 1); for (size_t i = 0; i < val.size(); ++i) { - val[i] = T(distribution(generator_)); + val[i] = distribution(generator_); } return val; } template inline std::vector OneHot(const std::vector& dims, int64_t stride) { - int64_t size = std::accumulate(dims.cbegin(), dims.cend(), static_cast(1), std::multiplies{}); - std::vector val(size, T(0)); + std::vector val(detail::SizeFromDims(dims), T(0)); std::uniform_int_distribution distribution(0, stride - 1); for (size_t offset = 0; offset < val.size(); offset += stride) { size_t rand_index = static_cast(distribution(generator_)); @@ -63,8 +77,7 @@ class RandomValueGenerator { template inline std::vector FillZeros(const std::vector& dims) { - int64_t size = std::accumulate(dims.cbegin(), dims.cend(), static_cast(1), std::multiplies{}); - std::vector val(size, T(0)); + std::vector val(detail::SizeFromDims(dims), T(0)); return val; } diff --git a/onnxruntime/test/providers/cpu/reduction/reduction_ops_test.cc b/onnxruntime/test/providers/cpu/reduction/reduction_ops_test.cc index 2d6cbeefb5..e1bac6ddab 100644 --- a/onnxruntime/test/providers/cpu/reduction/reduction_ops_test.cc +++ b/onnxruntime/test/providers/cpu/reduction/reduction_ops_test.cc @@ -5,6 +5,7 @@ #include #include #include "gtest/gtest.h" +#include "test/common/tensor_op_test_utils.h" #include "test/providers/provider_test_utils.h" #include "test/providers/cpu/reduction/reduction_test_cases.h" #ifdef USE_CUDA @@ -37,11 +38,11 @@ void TestReduceOp(const std::string& op, test.AddAttribute("keepdims", keepdims); test.AddInput("data", input_dims, data); test.AddOutput("reduced", expected_dims, expected_data); - #if defined(OPENVINO_CONFIG_GPU_FP32) - test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaExecutionProvider, kOpenVINOExecutionProvider, kTensorrtExecutionProvider}); //TensorRT,OpenVINO: result differs - #else - test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaExecutionProvider, kTensorrtExecutionProvider}); //TensorRT: result differs - #endif +#if defined(OPENVINO_CONFIG_GPU_FP32) + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaExecutionProvider, kOpenVINOExecutionProvider, kTensorrtExecutionProvider}); //TensorRT,OpenVINO: result differs +#else + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kCudaExecutionProvider, kTensorrtExecutionProvider}); //TensorRT: result differs +#endif } //TODO:investigate why it is so slow. It need 12 seconds on an Azure Standard F48s_v2 (48 vcpus, 96 GiB memory) @@ -175,7 +176,6 @@ TEST(ReductionOpTest, ReduceL10DTensor) { } #endif // !(defined USE_TENSORRT) && !(defined USE_TVM) - TEST(ReductionOpTest, ReduceL2_default_axes_keepdims) { OpTester test("ReduceL2"); test.AddAttribute("keepdims", (int64_t)1); @@ -425,7 +425,6 @@ TEST(ReductionOpTest, ReduceLogSumExp_do_not_keepdims_2) { {1.0f, 2.0f, 3.0f}); test.AddOutput("reduced", {}, {3.40760596f}); test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: full reduce without keepDimensions is not supported with explicit batch - } TEST(ReductionOpTest, ReduceLogSumExp_keepdims) { @@ -596,13 +595,12 @@ TEST(ReductionOpTest, ReduceMax_int32) { 11, 12}); test.AddOutput("reduced", {3, 1, 1}, {4, 8, 12}); - - #if defined (OPENVINO_CONFIG_GPU_FP32) - test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider, kOpenVINOExecutionProvider}); // OpenVINO: Disabled temporarily - #else - test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: axis must be 0 - #endif - } +#if defined(OPENVINO_CONFIG_GPU_FP32) + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider, kOpenVINOExecutionProvider}); // OpenVINO: Disabled temporarily +#else + test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: axis must be 0 +#endif +} TEST(ReductionOpTest, ReduceMax_int64) { OpTester test("ReduceMax"); @@ -626,14 +624,14 @@ TEST(ReductionOpTest, ReduceMax_int8) { test.AddAttribute("axes", std::vector{1, 2}); test.AddAttribute("keepdims", (int64_t)1); test.AddInput("data", {3, 2, 2}, - {1, 2, - 3, 4, + {1, 2, + 3, 4, - 5, 6, - 7, 8, + 5, 6, + 7, 8, - 9, 10, - 11, 12}); + 9, 10, + 11, 12}); test.AddOutput("reduced", {3, 1, 1}, {4, 8, 12}); test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: axis must be 0 } @@ -643,14 +641,14 @@ TEST(ReductionOpTest, ReduceMax_uint8) { test.AddAttribute("axes", std::vector{1, 2}); test.AddAttribute("keepdims", (int64_t)1); test.AddInput("data", {3, 2, 2}, - {1, 2, - 3, 4, + {1, 2, + 3, 4, - 5, 6, - 7, 8, + 5, 6, + 7, 8, - 9, 10, - 11, 12}); + 9, 10, + 11, 12}); test.AddOutput("reduced", {3, 1, 1}, {4, 8, 12}); test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); //TensorRT: axis must be 0 } @@ -908,14 +906,14 @@ TEST(ReductionOpTest, ReduceMin_int8) { test.AddAttribute("axes", std::vector{0, 2}); test.AddAttribute("keepdims", (int64_t)1); test.AddInput("data", {3, 2, 2}, - {1, 2, - 3, 4, + {1, 2, + 3, 4, - 5, 6, - 7, 8, + 5, 6, + 7, 8, - 9, 10, - 11, 12}); + 9, 10, + 11, 12}); test.AddOutput("reduced", {1, 2, 1}, {1, 3}); test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } @@ -925,19 +923,18 @@ TEST(ReductionOpTest, ReduceMin_uint8) { test.AddAttribute("axes", std::vector{0, 2}); test.AddAttribute("keepdims", (int64_t)1); test.AddInput("data", {3, 2, 2}, - {1, 2, - 3, 4, + {1, 2, + 3, 4, - 5, 6, - 7, 8, + 5, 6, + 7, 8, - 9, 10, - 11, 12}); + 9, 10, + 11, 12}); test.AddOutput("reduced", {1, 2, 1}, {1, 3}); test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider}); } - #if !(defined USE_TENSORRT) && !(defined USE_TVM) TEST(ReductionOpTest, ReduceMin0DTensor) { OpTester test("ReduceMin"); @@ -1056,7 +1053,7 @@ TEST(ReductionOpTest, ReduceSumHalfHalf) { test.AddInput("data", {3, 2, 2}, data_half); test.AddOutput("reduced", {2}, result_half); test.Run(); -} +} void test_half_reduce_sum( int64_t m, int64_t n) { @@ -1791,7 +1788,6 @@ TEST(ReductionOpTest, ArgMin_do_not_keepdims_2_select_last) { test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider, kNGraphExecutionProvider}); } - TEST(ReductionOpTest, ArgMin_int32) { OpTester test("ArgMin"); test.AddAttribute("axis", (int64_t)0); @@ -1834,23 +1830,20 @@ TEST(ReductionOpTest, ArgMin_int32_select_last) { #ifdef USE_CUDA -void test_reduce_apis(size_t size) { +void test_reduce_apis(int64_t size, float relative_error_tolerance = 1e-4f) { float output_sum = 0; float output_square_sum = 0; float output_mean = 0; float expected_output_sum = 0; float expected_output_square_sum = 0; float expected_output_mean = 0; - const std::vector shape = {static_cast(size)}; - std::random_device random_device; - std::mt19937 random_engine(random_device()); - std::uniform_real_distribution dist(0.1f, 1.0f); - std::vector input(size); - for (size_t i = 0; i < size; ++i) { - input[i] = dist(random_engine); - expected_output_sum += input[i]; - expected_output_square_sum += input[i] * input[i]; - expected_output_mean += input[i] / float(size); + const std::vector shape = {size}; + RandomValueGenerator random_value_generator{}; + const auto input = random_value_generator.Uniform(shape, 0.1f, 1.0f); + for (const auto input_value : input) { + expected_output_sum += input_value; + expected_output_square_sum += input_value * input_value; + expected_output_mean += input_value / float(size); } const int buffer_size_in_byte = onnxruntime::cuda::compute_reduction_buffer_size( static_cast(sizeof(float)), static_cast(size)); @@ -1892,9 +1885,10 @@ void test_reduce_apis(size_t size) { cudaFree(device_output_square_sum); cudaFree(device_output_mean); - EXPECT_TRUE(std::abs(output_sum - expected_output_sum) / expected_output_sum < 1e-4f); - EXPECT_TRUE(std::abs(output_square_sum - expected_output_square_sum) / expected_output_square_sum < 1e-4); - EXPECT_TRUE(std::abs(output_mean - expected_output_mean) / expected_output_mean < 1e-4f); + EXPECT_LT(std::abs(output_sum - expected_output_sum) / expected_output_sum, relative_error_tolerance); + EXPECT_LT(std::abs(output_square_sum - expected_output_square_sum) / expected_output_square_sum, + relative_error_tolerance); + EXPECT_LT(std::abs(output_mean - expected_output_mean) / expected_output_mean, relative_error_tolerance); } TEST(ReduceApiTest, Sum) { @@ -1903,7 +1897,7 @@ TEST(ReduceApiTest, Sum) { test_reduce_apis(123); test_reduce_apis(1128); test_reduce_apis(5566); - test_reduce_apis(941736); + test_reduce_apis(941736, 2e-4f); } #endif @@ -1935,7 +1929,7 @@ TEST(ReductionOpTest, ReduceDimWithZero) { : OpTester::ExpectResult::kExpectFailure; // exclude OpenVINO, NGraph and TensorRT as this isn't handled by those EPs - tester.Run(expect, error_msg, {kTensorrtExecutionProvider, kNGraphExecutionProvider,kOpenVINOExecutionProvider, kNupharExecutionProvider}); + tester.Run(expect, error_msg, {kTensorrtExecutionProvider, kNGraphExecutionProvider, kOpenVINOExecutionProvider, kNupharExecutionProvider}); }; // reduce on all axes keeping dims. should allow the 0 to be the reduced value diff --git a/orttraining/orttraining/test/training_ops/cpu/tensor/gather_grad_op_test.cc b/orttraining/orttraining/test/training_ops/cpu/tensor/gather_grad_op_test.cc index d91dc4154a..d34752d26c 100644 --- a/orttraining/orttraining/test/training_ops/cpu/tensor/gather_grad_op_test.cc +++ b/orttraining/orttraining/test/training_ops/cpu/tensor/gather_grad_op_test.cc @@ -105,8 +105,7 @@ TEST(GatherGradOpTest, Gather_axis1_float_impl2) { int64_t output_shape = 4; RandomValueGenerator random{}; std::vector grad(random.Uniform({axis_0 * axis_1 * axis_2}, 1.0f, 1.0f)); - std::vector indices(random.Uniform({axis_1 * axis_2}, static_cast(0), - static_cast(2))); + std::vector indices(random.Uniform({axis_1 * axis_2}, 0, 3)); std::vector shape{axis_0, output_shape}; std::vector output(axis_0 * output_shape); @@ -133,8 +132,7 @@ TEST(GatherGradOpTest, Gather_axis0_float_impl2) { int64_t output_shape = 4; RandomValueGenerator random{}; std::vector grad(random.Uniform({axis_1 * axis_2 * output_shape}, 1.0f, 1.0f)); - std::vector indices(random.Uniform({axis_1 * axis_2}, static_cast(0), - static_cast(2))); + std::vector indices(random.Uniform({axis_1 * axis_2}, 0, 3)); std::vector shape{axis_0, output_shape}; std::vector output(axis_0 * output_shape); diff --git a/orttraining/orttraining/test/training_ops/cuda/cross_entropy_test.cc b/orttraining/orttraining/test/training_ops/cuda/cross_entropy_test.cc index 7d195bb424..4701456d8b 100644 --- a/orttraining/orttraining/test/training_ops/cuda/cross_entropy_test.cc +++ b/orttraining/orttraining/test/training_ops/cuda/cross_entropy_test.cc @@ -154,7 +154,7 @@ static void TestSparseSoftmaxCrossEntropy(const std::vector* X_dims, // create rand inputs RandomValueGenerator random{}; std::vector X_data = random.Uniform(*X_dims, -200.0f, 200.0f); - std::vector index_data = random.Uniform(*index_dims, 0.0f, static_cast(X_dims->back())); + std::vector index_data = random.Uniform(*index_dims, 0, X_dims->back()); test.AddInput("X", *X_dims, X_data); test.AddInput("index", *index_dims, index_data); @@ -233,7 +233,7 @@ static void TestSparseSoftmaxCrossEntropyGrad(const std::vector& dY_dim RandomValueGenerator random{}; std::vector dY_data = random.Uniform(dY_dims, -10.0f, 10.0f); std::vector log_prob_data = random.Uniform(log_prob_dims, -10.0f, 10.0f); - std::vector index_data = random.Uniform(index_dims, 0.0f, static_cast(dX_dims.back())); + std::vector index_data = random.Uniform(index_dims, 0, dX_dims.back()); test.AddInput("dY", dY_dims, dY_data); test.AddInput("log_prob", log_prob_dims, log_prob_data); @@ -287,7 +287,7 @@ static void TestSoftmaxCrossEntropyLoss(const std::vector* X_dims, // create rand inputs RandomValueGenerator random{}; std::vector X_data = random.Uniform(*X_dims, -200.0f, 200.0f); - std::vector index_data = random.Uniform(*index_dims, 0, ((*X_dims)[1]) - 1); + std::vector index_data = random.Uniform(*index_dims, 0, (*X_dims)[1]); //Add one data point that has ignore_index. if (index_data.size() > 0) { index_data[0] = ignore_index; @@ -394,7 +394,7 @@ static void TestSoftmaxCrossEntropyLossGrad(const std::vector& dY_dims, RandomValueGenerator random{}; std::vector dY_data = random.Uniform(dY_dims, -10.0f, 10.0f); std::vector log_prob_data = random.Uniform(log_prob_dims, -10.0f, 10.0f); - std::vector index_data = random.Uniform(index_dims, 0, dX_dims[1] - 1); + std::vector index_data = random.Uniform(index_dims, 0, dX_dims[1]); //Add one data point that has ignore_index. if (index_data.size() > 0) { index_data[0] = ignore_index;