biondizzle/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Remove all_special_tokens_extended from tokenizer code (#29686)

Browse Source 
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2025-11-28 20:26:51 +00:00
committed by GitHub
parent 8d9338fae4
commit fecae12cd7
6 changed files with 40 additions and 65 deletions
Download Patch File Download Diff File Expand all files Collapse all files

1
tests/tokenization/test_cached_tokenizer.py
View File

@@ -31,7 +31,6 @@ def _check_consistency(target: AnyTokenizer, expected: AnyTokenizer):
# Cached attributes
assert target.all_special_ids == expected.all_special_ids
assert target.all_special_tokens == expected.all_special_tokens
assert target.all_special_tokens_extended == expected.all_special_tokens_extended
assert target.get_vocab() == expected.get_vocab()
assert len(target) == len(expected)