[Bugfix] EAGLE output norm bug (#14464)

Signed-off-by: Bryan Lu <yuzhelu@amazon.com>
This commit is contained in:
Bryan Lu
2025-03-14 23:50:33 -07:00
committed by GitHub
parent ee3778d5fc
commit 9ed6ee92d6
8 changed files with 152 additions and 35 deletions

View File

@@ -853,6 +853,10 @@ class LLMEngine:
self.generation_config_fields, seq.eos_token_id)
# Create the sequence group.
draft_size = 1
if self.vllm_config.speculative_config is not None:
draft_size = \
self.vllm_config.speculative_config.num_speculative_tokens + 1
seq_group = SequenceGroup(
request_id=request_id,
seqs=[seq],
@@ -862,7 +866,8 @@ class LLMEngine:
trace_headers=trace_headers,
prompt_adapter_request=prompt_adapter_request,
encoder_seq=encoder_seq,
priority=priority)
priority=priority,
draft_size=draft_size)
return seq_group