[Bugfix] Fix DeepSeek v3.2 tokenizer outputting None issue (#33832)

Signed-off-by: wzhao18 <wzhao18.sz@gmail.com>
This commit is contained in:
Wei Zhao
2026-02-05 18:50:49 -05:00
committed by GitHub
parent d5c4800112
commit 91a07ff618

View File

@@ -157,6 +157,10 @@ def detokenize_incrementally(
)
if isinstance(new_tokens, str):
new_tokens = [new_tokens]
else:
# This is required to guard against out-of-vocab prompt token ids
# (for example when using dummy weights)
_replace_none_with_empty(new_tokens) # type: ignore[arg-type]
else:
new_tokens = [""]
output_tokens = prev_tokens + new_tokens