diff --git a/src/transformers/activations.py b/src/transformers/activations.py
index 759b2fa80..86882bb72 100644
--- a/src/transformers/activations.py
+++ b/src/transformers/activations.py
@@ -47,7 +47,7 @@ ACT2FN = {
     "relu": F.relu,
     "swish": swish,
     "gelu": gelu,
-    "tanh": F.tanh,
+    "tanh": torch.tanh,
     "gelu_new": gelu_new,
 }