saymrwulf/transformers
1
0
Fork 0
mirror of https://github.com/saymrwulf/transformers.git synced 2026-05-14 20:58:08 +00:00
Code Issues Projects Releases Packages Wiki Activity Actions

fix: use correct var names for check_tokenizers script (#33702)

Browse source
This commit is contained in:
Nicola De Angeli 2024-09-26 17:24:46 +02:00 committed by GitHub
parent 46841d3eb2
commit c716fc0e48
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
1 changed files with 1 additions and 1 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

2
scripts/check_tokenizers.py
View file

@ -88,7 +88,7 @@ def check_details(line, spm_ids, tok_ids, slow, fast):
if tok_ids[first + k : first + k + min_width] == spm_ids[first + i : first + i + min_width]
]
for j in possible_matches:
if check_diff(spm_ids[first : first + i], tok_ids[first : first + j], sp, tok) and check_details(
if check_diff(spm_ids[first : first + i], tok_ids[first : first + j], slow, fast) and check_details(
line,
spm_ids[first + i : last],
tok_ids[first + j : last],