saymrwulf/onnxruntime
1
0
Fork 0
mirror of https://github.com/saymrwulf/onnxruntime.git synced 2026-07-05 04:17:53 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

softmax perf improvement pr1 - add more softmax related test (#15176)

Browse source 
1. add fp16 test
2. add test for shape is not power of two.
This commit is contained in:
zhijiang 2023-04-11 17:02:40 +08:00 committed by GitHub
parent ef42fd09fb
commit 29c74d3c43
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
3 changed files with 173 additions and 37 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

14
onnxruntime/test/common/tensor_op_test_utils.h
View file

@ -65,6 +65,20 @@ class RandomValueGenerator {
return val;
}
// Random values generated are in the range [min, max).
template <typename TFloat16>
typename std::enable_if<
std::is_same_v<TFloat16, MLFloat16>,
std::vector<TFloat16>>::type
Uniform(gsl::span<const int64_t> dims, float min, float max) {
std::vector<TFloat16> val(detail::SizeFromDims(dims));
std::uniform_real_distribution<float> distribution(min, max);
for (size_t i = 0; i < val.size(); ++i) {
val[i] = TFloat16(math::floatToHalf(distribution(generator_)));
}
return val;
}
// Random values generated are in the range [min, max).
template <typename TInt>
typename std::enable_if<

188
orttraining/orttraining/test/training_ops/cuda/softmax_test.cc
View file

@ -12,24 +12,25 @@ constexpr const char* kGpuExecutionProvider = kCudaExecutionProvider;
constexpr const char* kGpuExecutionProvider = kRocmExecutionProvider;
#endif
template <typename T>
static void TestSoftmax(const std::vector<int64_t>& X_dims,
const std::vector<int64_t>& Y_dims,
int axis = 1,
bool is_log_softmax=false,
double per_sample_tolerance = 1e-4,
double relative_per_sample_tolerance = 1e-4) {
const char* op = is_log_softmax? "LogSoftmax" : "Softmax";
CompareOpTester test(op);
test.AddAttribute<int64_t>("axis", axis);
// create rand inputs
RandomValueGenerator random{};
std::vector<float> X_data = random.Uniform<float>(X_dims, -10.0f, 10.0f);
test.AddInput<float>("X", X_dims, X_data);
std::vector<T> X_data = random.Uniform<T>(X_dims, -10.0f, 10.0f);
test.AddInput<T>("X", X_dims, X_data);
std::vector<float> Y_data = FillZeros<float>(Y_dims);
test.AddOutput<float>("Y", Y_dims, Y_data);
std::vector<T> Y_data = FillZeros<T>(Y_dims);
test.AddOutput<T>("Y", Y_dims, Y_data);
test.CompareWithCPU(kGpuExecutionProvider, per_sample_tolerance, relative_per_sample_tolerance);
}
@ -39,63 +40,116 @@ static void TestSoftmax(const std::vector<int64_t>& X_dims,
TEST(CudaKernelTest, Softmax_SmallTensor_LastAxis) {
std::vector<int64_t> X_dims{4, 2, 128};
std::vector<int64_t> Y_dims{4, 2, 128};
TestSoftmax(X_dims, Y_dims, 2, false);
TestSoftmax<float>(X_dims, Y_dims, 2, false);
}
TEST(CudaKernelTest, Softmax_SmallTensor_AllAxis) {
std::vector<int64_t> X_dims{4, 2, 128};
std::vector<int64_t> Y_dims{4, 2, 128};
TestSoftmax(X_dims, Y_dims, 0, false);
TestSoftmax(X_dims, Y_dims, 1, false);
TestSoftmax<float>(X_dims, Y_dims, 0, false);
TestSoftmax<float>(X_dims, Y_dims, 1, false);
}
// large tensor to check cuda DNN softmax forward
TEST(CudaKernelTest, Softmax_LargeTensor_LastAxis) {
std::vector<int64_t> X_dims{8, 16, 2048};
std::vector<int64_t> Y_dims{8, 16, 2048};
TestSoftmax(X_dims, Y_dims, 2, false);
TestSoftmax<float>(X_dims, Y_dims, 2, false);
}
TEST(CudaKernelTest, Softmax_LargeTensor_LastAxis_Float16) {
std::vector<int64_t> X_dims{8, 16, 2048};
std::vector<int64_t> Y_dims{8, 16, 2048};
TestSoftmax<MLFloat16>(X_dims, Y_dims, 2, false, 1e-3, 1e-3);
}
TEST(CudaKernelTest, Softmax_LargeTensor_LastAxis_Float16_NoPowerOfTwo) {
std::vector<int64_t> X_dims{8, 16, 1500};
std::vector<int64_t> Y_dims{8, 16, 1500};
TestSoftmax<MLFloat16>(X_dims, Y_dims, 2, false, 1e-3, 1e-3);
}
TEST(CudaKernelTest, Softmax_LargeTensor_AllAxis) {
std::vector<int64_t> X_dims{8, 16, 512};
std::vector<int64_t> Y_dims{8, 16, 512};
TestSoftmax(X_dims, Y_dims, 0, false);
TestSoftmax(X_dims, Y_dims, 1, false);
TestSoftmax<float>(X_dims, Y_dims, 0, false);
TestSoftmax<float>(X_dims, Y_dims, 1, false);
}
TEST(CudaKernelTest, Softmax_LargeTensor_AllAxis_Float16) {
std::vector<int64_t> X_dims{8, 16, 512};
std::vector<int64_t> Y_dims{8, 16, 512};
TestSoftmax<MLFloat16>(X_dims, Y_dims, 0, false, 1e-3, 1e-3);
TestSoftmax<MLFloat16>(X_dims, Y_dims, 1, false, 1e-3, 1e-3);
}
TEST(CudaKernelTest, Softmax_LargeTensor_AllAxis_Float16_NoPowerOfTwo) {
std::vector<int64_t> X_dims{8, 16, 1500};
std::vector<int64_t> Y_dims{8, 16, 1500};
TestSoftmax<MLFloat16>(X_dims, Y_dims, 0, false, 1e-3, 1e-3);
TestSoftmax<MLFloat16>(X_dims, Y_dims, 1, false, 1e-3, 1e-3);
}
TEST(CudaKernelTest, LogSoftmax_SmallTensor_LastAxis) {
std::vector<int64_t> X_dims{4, 2, 128};
std::vector<int64_t> Y_dims{4, 2, 128};
TestSoftmax(X_dims, Y_dims, 2, true);
TestSoftmax<float>(X_dims, Y_dims, 2, true);
}
TEST(CudaKernelTest, LogSoftmax_SmallTensor_AllAxis) {
std::vector<int64_t> X_dims{4, 2, 128};
std::vector<int64_t> Y_dims{4, 2, 128};
TestSoftmax(X_dims, Y_dims, 0, true);
TestSoftmax(X_dims, Y_dims, 1, true);
TestSoftmax<float>(X_dims, Y_dims, 0, true);
TestSoftmax<float>(X_dims, Y_dims, 1, true);
}
TEST(CudaKernelTest, LogSoftmax_LargeTensor_LastAxis) {
std::vector<int64_t> X_dims{8, 16, 2048};
std::vector<int64_t> Y_dims{8, 16, 2048};
TestSoftmax(X_dims, Y_dims, 2, true);
TestSoftmax<float>(X_dims, Y_dims, 2, true);
}
TEST(CudaKernelTest, LogSoftmax_LargeTensor_LastAxis_Float16) {
std::vector<int64_t> X_dims{8, 16, 2048};
std::vector<int64_t> Y_dims{8, 16, 2048};
TestSoftmax<MLFloat16>(X_dims, Y_dims, 2, true, 1e-3, 1e-3);
}
TEST(CudaKernelTest, LogSoftmax_LargeTensor_LastAxis_Float16_NoPowerOfTwo) {
std::vector<int64_t> X_dims{8, 16, 1500};
std::vector<int64_t> Y_dims{8, 16, 1500};
TestSoftmax<MLFloat16>(X_dims, Y_dims, 2, true, 1e-3, 1e-3);
}
TEST(CudaKernelTest, LogSoftmax_LargeTensor_AllAxis) {
std::vector<int64_t> X_dims{8, 16, 512};
std::vector<int64_t> Y_dims{8, 16, 512};
TestSoftmax(X_dims, Y_dims, 0, true);
TestSoftmax(X_dims, Y_dims, 1, true);
TestSoftmax<float>(X_dims, Y_dims, 0, true);
TestSoftmax<float>(X_dims, Y_dims, 1, true);
}
TEST(CudaKernelTest, LogSoftmax_LargeTensor_AllAxis_Float16) {
std::vector<int64_t> X_dims{8, 16, 512};
std::vector<int64_t> Y_dims{8, 16, 512};
TestSoftmax<MLFloat16>(X_dims, Y_dims, 0, true, 1e-3, 1e-3);
TestSoftmax<MLFloat16>(X_dims, Y_dims, 1, true, 1e-3, 1e-3);
}
TEST(CudaKernelTest, LogSoftmax_LargeTensor_AllAxis_Float16_NoPowerOfTwo) {
std::vector<int64_t> X_dims{8, 16, 1500};
std::vector<int64_t> Y_dims{8, 16, 1500};
TestSoftmax<MLFloat16>(X_dims, Y_dims, 0, true, 1e-3, 1e-3);
TestSoftmax<MLFloat16>(X_dims, Y_dims, 1, true, 1e-3, 1e-3);
}
template <typename T>
static void TestSoftmaxGrad(const std::vector<int64_t>& dY_dims,
const std::vector<int64_t>& Y_dims,
const std::vector<int64_t>& dX_dims,
int axis = 1,
bool is_log_softmax = false,
double per_sample_tolerance = 1e-4,
double relative_per_sample_tolerance = 1e-4) {
double relative_per_sample_tolerance = 1e-4) {
const char* op = is_log_softmax? "LogSoftmaxGrad" : "SoftmaxGrad";
CompareOpTester test(op, 1, kMSDomain);
@ -103,15 +157,15 @@ static void TestSoftmaxGrad(const std::vector<int64_t>& dY_dims,
// create rand inputs
RandomValueGenerator random{};
std::vector<float> dY_data = random.Uniform<float>(dY_dims, 0.0f, 1.0f);
std::vector<T> dY_data = random.Uniform<T>(dY_dims, -1.0f, 1.0f);
// Add 1e-2 for numerical stability to prevent zero probability.
std::vector<float> Y_data = random.Uniform<float>(Y_dims, 0.02f, 1.02f);
std::vector<T> Y_data = random.Uniform<T>(Y_dims, -1.02f, 1.02f);
test.AddInput<float>("dY", dY_dims, dY_data);
test.AddInput<float>("Y", Y_dims, Y_data);
test.AddInput<T>("dY", dY_dims, dY_data);
test.AddInput<T>("Y", Y_dims, Y_data);
std::vector<float> dX_data = FillZeros<float>(dX_dims);
test.AddOutput<float>("dX", dX_dims, dX_data);
std::vector<T> dX_data = FillZeros<T>(dX_dims);
test.AddOutput<T>("dX", dX_dims, dX_data);
test.CompareWithCPU(kGpuExecutionProvider, per_sample_tolerance, relative_per_sample_tolerance);
}
@ -121,15 +175,15 @@ TEST(CudaKernelTest, SoftmaxGrad_SmallTensor_LastAxis) {
std::vector<int64_t> dY_dims{4, 2, 128};
std::vector<int64_t> Y_dims{4, 2, 128};
std::vector<int64_t> dX_dims{4, 2, 128};
TestSoftmaxGrad(dY_dims, Y_dims, dX_dims, 2);
TestSoftmaxGrad<float>(dY_dims, Y_dims, dX_dims, 2);
}
TEST(CudaKernelTest, SoftmaxGrad_SmallTensor_AllAxis) {
std::vector<int64_t> dY_dims{4, 2, 128};
std::vector<int64_t> Y_dims{4, 2, 128};
std::vector<int64_t> dX_dims{4, 2, 128};
TestSoftmaxGrad(dY_dims, Y_dims, dX_dims, 0);
TestSoftmaxGrad(dY_dims, Y_dims, dX_dims, 1);
TestSoftmaxGrad<float>(dY_dims, Y_dims, dX_dims, 0);
TestSoftmaxGrad<float>(dY_dims, Y_dims, dX_dims, 1);
}
// large tensor to check cuda DNN softmax backward
@ -137,7 +191,21 @@ TEST(CudaKernelTest, SoftmaxGrad_LargeTensor_LastAxis) {
std::vector<int64_t> dY_dims{8, 16, 2048};
std::vector<int64_t> Y_dims{8, 16, 2048};
std::vector<int64_t> dX_dims{8, 16, 2048};
TestSoftmaxGrad(dY_dims, Y_dims, dX_dims, 2);
TestSoftmaxGrad<float>(dY_dims, Y_dims, dX_dims, 2);
}
TEST(CudaKernelTest, SoftmaxGrad_LargeTensor_LastAxis_Float16) {
std::vector<int64_t> dY_dims{8, 16, 2048};
std::vector<int64_t> Y_dims{8, 16, 2048};
std::vector<int64_t> dX_dims{8, 16, 2048};
TestSoftmaxGrad<MLFloat16>(dY_dims, Y_dims, dX_dims, 2, false, 1e-3, 1e-3);
}
TEST(CudaKernelTest, SoftmaxGrad_LargeTensor_LastAxis_Float16_NoPowerOfTwo) {
std::vector<int64_t> dY_dims{8, 16, 1500};
std::vector<int64_t> Y_dims{8, 16, 1500};
std::vector<int64_t> dX_dims{8, 16, 1500};
TestSoftmaxGrad<MLFloat16>(dY_dims, Y_dims, dX_dims, 2, false, 1e-3, 1e-3);
}
// large tensor to check cuda DNN softmax backward
@ -145,38 +213,84 @@ TEST(CudaKernelTest, SoftmaxGrad_LargeTensor_AllAxis) {
std::vector<int64_t> dY_dims{8, 16, 512};
std::vector<int64_t> Y_dims{8, 16, 512};
std::vector<int64_t> dX_dims{8, 16, 512};
TestSoftmaxGrad(dY_dims, Y_dims, dX_dims, 0);
TestSoftmaxGrad(dY_dims, Y_dims, dX_dims, 1);
TestSoftmaxGrad<float>(dY_dims, Y_dims, dX_dims, 0);
TestSoftmaxGrad<float>(dY_dims, Y_dims, dX_dims, 1);
}
TEST(CudaKernelTest, SoftmaxGrad_LargeTensor_AllAxis_Float16) {
std::vector<int64_t> dY_dims{8, 16, 512};
std::vector<int64_t> Y_dims{8, 16, 512};
std::vector<int64_t> dX_dims{8, 16, 512};
TestSoftmaxGrad<MLFloat16>(dY_dims, Y_dims, dX_dims, 0, false, 1e-3, 1e-3);
TestSoftmaxGrad<MLFloat16>(dY_dims, Y_dims, dX_dims, 1, false, 1e-3, 1e-3);
}
TEST(CudaKernelTest, SoftmaxGrad_LargeTensor_AllAxis_Float16_NoPowerOfTwo) {
std::vector<int64_t> dY_dims{8, 16, 1500};
std::vector<int64_t> Y_dims{8, 16, 1500};
std::vector<int64_t> dX_dims{8, 16, 1500};
TestSoftmaxGrad<MLFloat16>(dY_dims, Y_dims, dX_dims, 0, false, 1e-3, 1e-3);
TestSoftmaxGrad<MLFloat16>(dY_dims, Y_dims, dX_dims, 1, false, 1e-3, 1e-3);
}
TEST(CudaKernelTest, LogSoftmaxGrad_SmallTensor_LastAxis) {
std::vector<int64_t> dY_dims{4, 2, 128};
std::vector<int64_t> Y_dims{4, 2, 128};
std::vector<int64_t> dX_dims{4, 2, 128};
TestSoftmaxGrad(dY_dims, Y_dims, dX_dims, 2, true);
TestSoftmaxGrad<float>(dY_dims, Y_dims, dX_dims, 2, true);
}
TEST(CudaKernelTest, LogSoftmaxGrad_SmallTensor_AllAxis) {
std::vector<int64_t> dY_dims{4, 2, 128};
std::vector<int64_t> Y_dims{4, 2, 128};
std::vector<int64_t> dX_dims{4, 2, 128};
TestSoftmaxGrad(dY_dims, Y_dims, dX_dims, 0, true);
TestSoftmaxGrad(dY_dims, Y_dims, dX_dims, 1, true);
TestSoftmaxGrad<float>(dY_dims, Y_dims, dX_dims, 0, true);
TestSoftmaxGrad<float>(dY_dims, Y_dims, dX_dims, 1, true);
}
TEST(CudaKernelTest, LogSoftmaxGrad_LargeTensor_LastAxis) {
std::vector<int64_t> dY_dims{8, 16, 2048};
std::vector<int64_t> Y_dims{8, 16, 2048};
std::vector<int64_t> dX_dims{8, 16, 2048};
TestSoftmaxGrad(dY_dims, Y_dims, dX_dims, 2, true);
TestSoftmaxGrad<float>(dY_dims, Y_dims, dX_dims, 2, true);
}
TEST(CudaKernelTest, LogSoftmaxGrad_LargeTensor_LastAxis_Float16) {
std::vector<int64_t> dY_dims{8, 16, 2048};
std::vector<int64_t> Y_dims{8, 16, 2048};
std::vector<int64_t> dX_dims{8, 16, 2048};
TestSoftmaxGrad<MLFloat16>(dY_dims, Y_dims, dX_dims, 2, true, 1e-3, 1e-3);
}
TEST(CudaKernelTest, LogSoftmaxGrad_LargeTensor_LastAxis_Float16_NoPowerOfTwo) {
std::vector<int64_t> dY_dims{8, 16, 1500};
std::vector<int64_t> Y_dims{8, 16, 1500};
std::vector<int64_t> dX_dims{8, 16, 1500};
TestSoftmaxGrad<MLFloat16>(dY_dims, Y_dims, dX_dims, 2, true, 1e-3, 1e-3);
}
TEST(CudaKernelTest, LogSoftmaxGrad_LargeTensor_AllAxis) {
std::vector<int64_t> dY_dims{8, 16, 512};
std::vector<int64_t> Y_dims{8, 16, 512};
std::vector<int64_t> dX_dims{8, 16, 512};
TestSoftmaxGrad(dY_dims, Y_dims, dX_dims, 0, true);
TestSoftmaxGrad(dY_dims, Y_dims, dX_dims, 1, true);
TestSoftmaxGrad<float>(dY_dims, Y_dims, dX_dims, 0, true, 1e-3, 1e-3);
TestSoftmaxGrad<float>(dY_dims, Y_dims, dX_dims, 1, true, 1e-3, 1e-3);
}
TEST(CudaKernelTest, LogSoftmaxGrad_LargeTensor_AllAxis_Float16) {
std::vector<int64_t> dY_dims{8, 16, 512};
std::vector<int64_t> Y_dims{8, 16, 512};
std::vector<int64_t> dX_dims{8, 16, 512};
TestSoftmaxGrad<MLFloat16>(dY_dims, Y_dims, dX_dims, 0, true, 1e-3, 1e-3);
TestSoftmaxGrad<MLFloat16>(dY_dims, Y_dims, dX_dims, 1, true, 1e-3, 1e-3);
}
TEST(CudaKernelTest, LogSoftmaxGrad_LargeTensor_AllAxis_Float16_NoPowerOfTwo) {
std::vector<int64_t> dY_dims{8, 16, 1500};
std::vector<int64_t> Y_dims{8, 16, 1500};
std::vector<int64_t> dX_dims{8, 16, 1500};
TestSoftmaxGrad<MLFloat16>(dY_dims, Y_dims, dX_dims, 0, true, 1e-3, 1e-3);
TestSoftmaxGrad<MLFloat16>(dY_dims, Y_dims, dX_dims, 1, true, 1e-3, 1e-3);
}
static void TestSoftmaxGrad_13(const std::vector<int64_t>& dY_dims,
@ -185,7 +299,7 @@ static void TestSoftmaxGrad_13(const std::vector<int64_t>& dY_dims,
int axis = 1,
bool is_log_softmax = false,
double per_sample_tolerance = 1e-4,
double relative_per_sample_tolerance = 1e-4) {
double relative_per_sample_tolerance = 1e-4) {
const char* op = is_log_softmax? "LogSoftmaxGrad_13" : "SoftmaxGrad_13";
CompareOpTester test(op, 1, kMSDomain);

8
tools/ci_build/github/pai/pai-excluded-tests.txt
View file

@ -1,6 +1,14 @@
CudaKernelTest.NegativeLogLikelihoodLoss_TinySizeTensor
CudaKernelTest.NegativeLogLikelihoodLoss_SmallSizeTensor
CudaKernelTest.NegativeLogLikelihoodLoss_MediumSizeTensor
CudaKernelTest.SoftmaxGrad_LargeTensor_LastAxis_Float16
CudaKernelTest.SoftmaxGrad_LargeTensor_LastAxis_Float16_NoPowerOfTwo
CudaKernelTest.SoftmaxGrad_LargeTensor_AllAxis_Float16
CudaKernelTest.SoftmaxGrad_LargeTensor_AllAxis_Float16_NoPowerOfTwo
CudaKernelTest.LogSoftmaxGrad_LargeTensor_LastAxis_Float16
CudaKernelTest.LogSoftmaxGrad_LargeTensor_LastAxis_Float16_NoPowerOfTwo
CudaKernelTest.LogSoftmaxGrad_LargeTensor_AllAxis_Float16
CudaKernelTest.LogSoftmaxGrad_LargeTensor_AllAxis_Float16_NoPowerOfTwo
ReductionOpTest.ReductionVariationTest
ReductionOpTest.ReduceLogSumExp_default_axes_keepdims_double
ReductionOpTest.ReduceLogSumExp_default_axes_do_not_keep_dims_double