From 91a07ff6187e7308794b2a4863ab9e1f821ed464 Mon Sep 17 00:00:00 2001 From: Wei Zhao <51183510+wzhao18@users.noreply.github.com> Date: Thu, 5 Feb 2026 18:50:49 -0500 Subject: [PATCH] [Bugfix] Fix DeepSeek v3.2 tokenizer outputting None issue (#33832) Signed-off-by: wzhao18 --- vllm/tokenizers/detokenizer_utils.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/vllm/tokenizers/detokenizer_utils.py b/vllm/tokenizers/detokenizer_utils.py index e586a5d46..8e73d5dc5 100644 --- a/vllm/tokenizers/detokenizer_utils.py +++ b/vllm/tokenizers/detokenizer_utils.py @@ -157,6 +157,10 @@ def detokenize_incrementally( ) if isinstance(new_tokens, str): new_tokens = [new_tokens] + else: + # This is required to guard against out-of-vocab prompt token ids + # (for example when using dummy weights) + _replace_none_with_empty(new_tokens) # type: ignore[arg-type] else: new_tokens = [""] output_tokens = prev_tokens + new_tokens