diff --git a/vllm/tokenizers/detokenizer_utils.py b/vllm/tokenizers/detokenizer_utils.py
index e586a5d46..8e73d5dc5 100644
--- a/vllm/tokenizers/detokenizer_utils.py
+++ b/vllm/tokenizers/detokenizer_utils.py
@@ -157,6 +157,10 @@ def detokenize_incrementally(
         )
         if isinstance(new_tokens, str):
             new_tokens = [new_tokens]
+        else:
+            # This is required to guard against out-of-vocab prompt token ids
+            # (for example when using dummy weights)
+            _replace_none_with_empty(new_tokens)  # type: ignore[arg-type]
     else:
         new_tokens = [""]
     output_tokens = prev_tokens + new_tokens