From b040e6efc19f62d332fdbf00c28603d85d1d2981 Mon Sep 17 00:00:00 2001 From: Andrea Bacciu Date: Thu, 25 Feb 2021 17:18:33 +0100 Subject: [PATCH] Fix None in add_token_positions - issue #10210 (#10374) * Fix None in add_token_positions - issue #10210 Fix None in add_token_positions related to the issue #10210 * add_token_positions fix None values in end_positions vector add_token_positions fix None in end_positions vector as proposed by @joeddav --- docs/source/custom_datasets.rst | 1 + 1 file changed, 1 insertion(+) diff --git a/docs/source/custom_datasets.rst b/docs/source/custom_datasets.rst index 06a85010d..931b43533 100644 --- a/docs/source/custom_datasets.rst +++ b/docs/source/custom_datasets.rst @@ -563,6 +563,7 @@ we can use the built in :func:`~transformers.BatchEncoding.char_to_token` method # if start position is None, the answer passage has been truncated if start_positions[-1] is None: start_positions[-1] = tokenizer.model_max_length + if end_positions[-1] is None: end_positions[-1] = tokenizer.model_max_length encodings.update({'start_positions': start_positions, 'end_positions': end_positions})