biondizzle/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[Bugfix] Fix edge-case crash when using chat with the Mistral Tekken Tokenizer (#10051)

Browse Source 
Signed-off-by: Travis Johnson <tsjohnso@us.ibm.com>
This commit is contained in:
Travis Johnson
2024-11-05 21:28:29 -07:00
committed by GitHub
parent ffc0f2b47a
commit 2bcbae704c
2 changed files with 12 additions and 5 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
vllm/transformers_utils/tokenizers/mistral.py
View File

@@ -254,7 +254,7 @@ class MistralTokenizer:
skip_special_tokens: bool = True) -> str:
assert (
skip_special_tokens
), "Skipping special tokens is not supported for Mistral tokenizers."
), "skip_special_tokens=False is not supported for Mistral tokenizers."
if isinstance(ids, int):
ids = [ids]
@@ -268,12 +268,16 @@ class MistralTokenizer:
# TODO(Patrick) - potentially allow special tokens to not be skipped
assert (
skip_special_tokens
), "Skipping special tokens is not supported for Mistral tokenizers."
), "skip_special_tokens=False is not supported for Mistral tokenizers."
assert isinstance(self.tokenizer,
(Tekkenizer, SentencePieceTokenizer)), type(
self.tokenizer)
if isinstance(self.tokenizer, Tekkenizer):
# skip special tokens
ids = [i for i in ids if i > self.tokenizer.num_special_tokens]
tokens = [self.tokenizer.id_to_piece(id) for id in ids]
if any("<EFBFBD>" in t for t in tokens):