From 240f31ef6eb93d3ce672c1c6ec84cf92fdafaed1 Mon Sep 17 00:00:00 2001
From: Guoyu Wang <62914304+gwang-msft@users.noreply.github.com>
Date: Sun, 27 Feb 2022 15:27:07 -0800
Subject: [PATCH] fix softplus (#10576)

---
 .../providers/cpu/activation/activations.h    |  2 +-
 .../cpu/activation/activation_op_test.cc      | 36 +++++++++----------
 2 files changed, 19 insertions(+), 19 deletions(-)

diff --git a/onnxruntime/core/providers/cpu/activation/activations.h b/onnxruntime/core/providers/cpu/activation/activations.h
index 9b5fd84929..c6b8b5ae99 100644
--- a/onnxruntime/core/providers/cpu/activation/activations.h
+++ b/onnxruntime/core/providers/cpu/activation/activations.h
@@ -98,7 +98,7 @@ struct Softplus : public ElementWiseRangedTransform<T> {
     T* output_ptr = this->output + first;
     ConstEigenVectorArrayMap<T> xm(this->input + first, len);
     EigenVectorArrayMap<T> ym(output_ptr, len);
-    ym = (xm > 0).select(xm + ((-xm).exp() + 1.0f).log(), ((xm).exp() + 1.0f).log());
+    ym = (xm > 0).select(xm + ((-xm).exp()).log1p(), ((xm).exp()).log1p());
   }
 };
 
diff --git a/onnxruntime/test/providers/cpu/activation/activation_op_test.cc b/onnxruntime/test/providers/cpu/activation/activation_op_test.cc
index 82e9690718..c378132266 100644
--- a/onnxruntime/test/providers/cpu/activation/activation_op_test.cc
+++ b/onnxruntime/test/providers/cpu/activation/activation_op_test.cc
@@ -132,11 +132,11 @@ TEST_F(ActivationOpTest, Sigmoid_fp16) {
 #endif
   OpTester test("Sigmoid", 14);
 
-  auto formula = [](float x) { 
-                   auto y = 1.f / (1.f + std::exp(-std::abs(x)));  // safe sigmoid
-                   y = x > 0 ? y : 1 - y;
-                   return y; 
-                 };
+  auto formula = [](float x) {
+    auto y = 1.f / (1.f + std::exp(-std::abs(x)));  // safe sigmoid
+    y = x > 0 ? y : 1 - y;
+    return y;
+  };
 
   std::vector<float> X = input_values.front();
   std::vector<float> Y;
@@ -151,7 +151,7 @@ TEST_F(ActivationOpTest, Sigmoid_fp16) {
 
   test.AddInput<MLFloat16>("X", dims, f_X);
   test.AddOutput<MLFloat16>("Y", dims, f_Y);
-  test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});  
+  test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
 }
 
 TEST_F(ActivationOpTest, Tanh_fp16) {
@@ -179,7 +179,7 @@ TEST_F(ActivationOpTest, Tanh_fp16) {
 
   test.AddInput<MLFloat16>("X", dims, f_X);
   test.AddOutput<MLFloat16>("Y", dims, f_Y);
-  test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});  
+  test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
 }
 
 TEST_F(ActivationOpTest, Relu_fp16) {
@@ -207,7 +207,7 @@ TEST_F(ActivationOpTest, Relu_fp16) {
 
   test.AddInput<MLFloat16>("X", dims, f_X);
   test.AddOutput<MLFloat16>("Y", dims, f_Y);
-  test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});  
+  test.Run(OpTester::ExpectResult::kExpectSuccess, "", {kTensorrtExecutionProvider});
 }
 #endif
 
@@ -222,11 +222,11 @@ TEST_F(ActivationOpTest, Sigmoid_bfloat16) {
 #endif
   OpTester test("Sigmoid", 14);
 
-  auto formula = [](float x) { 
-                   auto y = 1.f / (1.f + std::exp(-std::abs(x)));  // safe sigmoid
-                   y = x > 0 ? y : 1 - y;
-                   return y; 
-                 };
+  auto formula = [](float x) {
+    auto y = 1.f / (1.f + std::exp(-std::abs(x)));  // safe sigmoid
+    y = x > 0 ? y : 1 - y;
+    return y;
+  };
 
   std::vector<float> X = input_values.front();
   std::vector<float> Y;
@@ -244,7 +244,7 @@ TEST_F(ActivationOpTest, Sigmoid_bfloat16) {
   execution_providers.push_back(DefaultCudaExecutionProvider());
 #elif USE_ROCM
   execution_providers.push_back(DefaultRocmExecutionProvider());
-#endif 
+#endif
   test.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers);
 }
 
@@ -276,7 +276,7 @@ TEST_F(ActivationOpTest, Tanh_bfloat16) {
   execution_providers.push_back(DefaultCudaExecutionProvider());
 #elif USE_ROCM
   execution_providers.push_back(DefaultRocmExecutionProvider());
-#endif 
+#endif
   test.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers);
 }
 
@@ -308,7 +308,7 @@ TEST_F(ActivationOpTest, Relu_bfloat16) {
   execution_providers.push_back(DefaultCudaExecutionProvider());
 #elif USE_ROCM
   execution_providers.push_back(DefaultRocmExecutionProvider());
-#endif 
+#endif
   test.Run(OpTester::ExpectResult::kExpectSuccess, "", {}, nullptr, &execution_providers);
 }
 #endif
@@ -431,9 +431,9 @@ TEST_F(ActivationOpTest, Softplus) {
                           input_values,
                           [](float x) {
                             if (x > 0)
-                              return x + logf(expf(-x) + 1);
+                              return x + log1pf(expf(-x));
                             else
-                              return logf(expf(x) + 1);
+                              return log1pf(expf(x));
                           });
 }