biondizzle/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Add support for spaces_between_special_tokens

Browse Source
This commit is contained in:
Dan Lord
2023-10-30 16:52:56 -07:00
committed by GitHub
parent 79a30912b8
commit 7013a80170
5 changed files with 28 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
vllm/engine/llm_engine.py
View File

@@ -632,8 +632,7 @@ class LLMEngine:
f"CPU KV cache usage: {cpu_cache_usage * 100:.1f}%")
self.last_logging_time = now
def _decode_sequence(self, seq: Sequence,
sampling_params: SamplingParams) -> None:
def _decode_sequence(self, seq: Sequence, prms: SamplingParams) -> None:
"""Decodes the new token for a sequence."""
(new_tokens, new_output_text, prefix_offset,
read_offset) = detokenize_incrementally(
@@ -642,7 +641,8 @@ class LLMEngine:
prev_tokens=seq.tokens,
prefix_offset=seq.prefix_offset,
read_offset=seq.read_offset,
skip_special_tokens=sampling_params.skip_special_tokens,
skip_special_tokens=prms.skip_special_tokens,
spaces_between_special_tokens=prms.spaces_between_special_tokens,
)
if seq.tokens is None:
seq.tokens = new_tokens