[Bugfix] Fix DeepSeek v3.2 tokenizer outputting None issue (#33832)
Signed-off-by: wzhao18 <wzhao18.sz@gmail.com>
This commit is contained in:
@@ -157,6 +157,10 @@ def detokenize_incrementally(
|
||||
)
|
||||
if isinstance(new_tokens, str):
|
||||
new_tokens = [new_tokens]
|
||||
else:
|
||||
# This is required to guard against out-of-vocab prompt token ids
|
||||
# (for example when using dummy weights)
|
||||
_replace_none_with_empty(new_tokens) # type: ignore[arg-type]
|
||||
else:
|
||||
new_tokens = [""]
|
||||
output_tokens = prev_tokens + new_tokens
|
||||
|
||||
Reference in New Issue
Block a user