diff --git a/src/transformers/__init__.py b/src/transformers/__init__.py
index f3b15757d..1462a61c4 100755
--- a/src/transformers/__init__.py
+++ b/src/transformers/__init__.py
@@ -652,10 +652,8 @@ if is_torch_available():
             "IBertForQuestionAnswering",
             "IBertForSequenceClassification",
             "IBertForTokenClassification",
-            "IBertLayer",
             "IBertModel",
             "IBertPreTrainedModel",
-            "load_tf_weights_in_ibert",
         ]
     )
     _import_structure["models.layoutlm"].extend(
diff --git a/src/transformers/models/ibert/__init__.py b/src/transformers/models/ibert/__init__.py
index c3215a88a..af1df0b80 100644
--- a/src/transformers/models/ibert/__init__.py
+++ b/src/transformers/models/ibert/__init__.py
@@ -18,7 +18,7 @@
 
 from typing import TYPE_CHECKING
 
-from ...file_utils import _BaseLazyModule, is_tokenizers_available, is_torch_available
+from ...file_utils import _BaseLazyModule, is_torch_available
 
 
 _import_structure = {
@@ -28,6 +28,7 @@ _import_structure = {
 if is_torch_available():
     _import_structure["modeling_ibert"] = [
         "IBERT_PRETRAINED_MODEL_ARCHIVE_LIST",
+        "IBertPreTrainedModel",
         "IBertForMaskedLM",
         "IBertForMultipleChoice",
         "IBertForQuestionAnswering",
@@ -48,6 +49,7 @@ if TYPE_CHECKING:
             IBertForSequenceClassification,
             IBertForTokenClassification,
             IBertModel,
+            IBertPreTrainedModel,
         )
 
 else:
diff --git a/tests/test_file_utils.py b/tests/test_file_utils.py
index c7192267f..63f665647 100644
--- a/tests/test_file_utils.py
+++ b/tests/test_file_utils.py
@@ -15,6 +15,9 @@
 import unittest
 
 import requests
+
+# Try to import everything from transformers to ensure every object can be loaded.
+from transformers import *  # noqa F406
 from transformers.file_utils import CONFIG_NAME, WEIGHTS_NAME, filename_to_url, get_from_cache, hf_bucket_url
 from transformers.testing_utils import DUMMY_UNKWOWN_IDENTIFIER