From c4cbed8081914aa0081101c7e339c4c3b7ce317f Mon Sep 17 00:00:00 2001
From: Pedro Cuenca <pedro@huggingface.co>
Date: Thu, 6 Feb 2025 10:41:05 +0100
Subject: [PATCH] Processor: prevent duplicated tokens

When using text-only LLMs, the chat template is expected to take care of
adding the required special tokens, such as bos. Hence, tokenization
must not include special tokens.

The same contract should be honored for multimodal processors.
---
 src/transformers/processing_utils.py | 1 +
 1 file changed, 1 insertion(+)

diff --git a/src/transformers/processing_utils.py b/src/transformers/processing_utils.py
index b94230c7d..17b07274a 100644
--- a/src/transformers/processing_utils.py
+++ b/src/transformers/processing_utils.py
@@ -1246,6 +1246,7 @@ class ProcessorMixin(PushToHubMixin):
                 text=prompt,
                 images=images if images else None,
                 videos=videos if videos else None,
+                add_special_tokens=False,
                 **kwargs,
             )
             if return_dict: