diff --git a/src/transformers/utils/hub.py b/src/transformers/utils/hub.py
index a4065dbf8..8a75e4c2e 100644
--- a/src/transformers/utils/hub.py
+++ b/src/transformers/utils/hub.py
@@ -660,7 +660,14 @@ def has_file(
             proxies=proxies,
             timeout=10,
         )
-    except OfflineModeIsEnabled:
+    except (requests.exceptions.SSLError, requests.exceptions.ProxyError):
+        # Actually raise for those subclasses of ConnectionError
+        raise
+    except (
+        requests.exceptions.ConnectionError,
+        requests.exceptions.Timeout,
+        OfflineModeIsEnabled,
+    ):
         return has_file_in_cache
 
     try: