diff --git a/tests/test_modeling_common.py b/tests/test_modeling_common.py
index 6d0306d74..18d96d3d0 100755
--- a/tests/test_modeling_common.py
+++ b/tests/test_modeling_common.py
@@ -822,7 +822,7 @@ class ModelTesterMixin:
             with self.assertRaises(AssertionError):
                 # generating multiple sequences when no beam search generation
                 # is not allowed as it would always generate the same sequences
-                model.generate(input_ids, do_sample=False, num_return_sequences=2)
+                model.generate(input_ids, do_sample=False, num_beams=1, num_return_sequences=2)
 
             # num_return_sequences > 1, sample
             self._check_generated_ids(model.generate(input_ids, do_sample=True, num_return_sequences=2))