diff --git a/.circleci/config.yml b/.circleci/config.yml
index 2f8b3f83a..d33e5b223 100644
--- a/.circleci/config.yml
+++ b/.circleci/config.yml
@@ -277,7 +277,7 @@ jobs:
                       - v0.4-tf-{{ checksum "setup.py" }}
                       - v0.4-{{ checksum "setup.py" }}
             - run: pip install --upgrade pip
-            - run: pip install .[sklearn,tf-cpu,testing,sentencepiece,tf-speech]
+            - run: pip install .[sklearn,tf-cpu,testing,sentencepiece,tf-speech,vision]
             - save_cache:
                   key: v0.4-tf-{{ checksum "setup.py" }}
                   paths:
@@ -310,7 +310,7 @@ jobs:
                       - v0.4-tf-{{ checksum "setup.py" }}
                       - v0.4-{{ checksum "setup.py" }}
             - run: pip install --upgrade pip
-            - run: pip install .[sklearn,tf-cpu,testing,sentencepiece,tf-speech]
+            - run: pip install .[sklearn,tf-cpu,testing,sentencepiece,tf-speech,vision]
             - save_cache:
                   key: v0.4-tf-{{ checksum "setup.py" }}
                   paths:
diff --git a/.github/workflows/self-scheduled.yml b/.github/workflows/self-scheduled.yml
index 0027e1399..1d17d96ce 100644
--- a/.github/workflows/self-scheduled.yml
+++ b/.github/workflows/self-scheduled.yml
@@ -143,7 +143,7 @@ jobs:
         run: |
           apt -y update && apt install -y libsndfile1-dev git
           pip install --upgrade pip
-          pip install .[sklearn,testing,onnx,sentencepiece,tf-speech]
+          pip install .[sklearn,testing,onnx,sentencepiece,tf-speech,vision]
 
       - name: Are GPUs recognized by our DL frameworks
         run: |
@@ -293,7 +293,7 @@ jobs:
         run: |
           apt -y update && apt install -y libsndfile1-dev git
           pip install --upgrade pip
-          pip install .[sklearn,testing,onnx,sentencepiece,tf-speech]
+          pip install .[sklearn,testing,onnx,sentencepiece,tf-speech,vision]
 
       - name: Are GPUs recognized by our DL frameworks
         run: |
diff --git a/tests/test_modeling_beit.py b/tests/test_modeling_beit.py
index 9ead09a7d..db8bd8c6d 100644
--- a/tests/test_modeling_beit.py
+++ b/tests/test_modeling_beit.py
@@ -414,6 +414,7 @@ def prepare_img():
     return image
 
 
+@require_torch
 @require_vision
 class BeitModelIntegrationTest(unittest.TestCase):
     @cached_property
diff --git a/tests/test_modeling_deit.py b/tests/test_modeling_deit.py
index 222f2afbe..925dbc6b0 100644
--- a/tests/test_modeling_deit.py
+++ b/tests/test_modeling_deit.py
@@ -391,6 +391,7 @@ def prepare_img():
     return image
 
 
+@require_torch
 @require_vision
 class DeiTModelIntegrationTest(unittest.TestCase):
     @cached_property
diff --git a/tests/test_modeling_tf_vit.py b/tests/test_modeling_tf_vit.py
index eb342aa68..ea493fc59 100644
--- a/tests/test_modeling_tf_vit.py
+++ b/tests/test_modeling_tf_vit.py
@@ -353,7 +353,7 @@ class TFViTModelTest(TFModelTesterMixin, unittest.TestCase):
     @slow
     def test_model_from_pretrained(self):
 
-        model = TFViTModel.from_pretrained("google/vit-base-patch16-224", from_pt=True)
+        model = TFViTModel.from_pretrained("google/vit-base-patch16-224")
         self.assertIsNotNone(model)
 
 
@@ -363,6 +363,7 @@ def prepare_img():
     return image
 
 
+@require_tf
 @require_vision
 class TFViTModelIntegrationTest(unittest.TestCase):
     @cached_property
diff --git a/tests/test_modeling_vit.py b/tests/test_modeling_vit.py
index 6073bf239..c24ae535a 100644
--- a/tests/test_modeling_vit.py
+++ b/tests/test_modeling_vit.py
@@ -331,6 +331,7 @@ def prepare_img():
     return image
 
 
+@require_torch
 @require_vision
 class ViTModelIntegrationTest(unittest.TestCase):
     @cached_property