Fix handling of special tokens in decoding. (#418)

2023-07-12 23:14:56 +08:00
parent 51be365143
commit c6dfc3cdbe
2 changed files with 6 additions and 3 deletions
--- a/vllm/engine/llm_engine.py
+++ b/vllm/engine/llm_engine.py
@@ -276,8 +276,9 @@ class LLMEngine:
                    seq.get_last_token_id(),
                    skip_special_tokens=True,
                )
-                seq.output_tokens.append(new_token)
-                seq.output_text = new_output_text
+                if new_token is not None:
+                    seq.output_tokens.append(new_token)
+                    seq.output_text = new_output_text

    def _stop_sequences(self, seq_groups: List[SequenceGroup]) -> None:
        """Stop the finished sequences."""