[Bugfix] EAGLE output norm bug (#14464)
Signed-off-by: Bryan Lu <yuzhelu@amazon.com>
This commit is contained in:
@@ -853,6 +853,10 @@ class LLMEngine:
|
||||
self.generation_config_fields, seq.eos_token_id)
|
||||
|
||||
# Create the sequence group.
|
||||
draft_size = 1
|
||||
if self.vllm_config.speculative_config is not None:
|
||||
draft_size = \
|
||||
self.vllm_config.speculative_config.num_speculative_tokens + 1
|
||||
seq_group = SequenceGroup(
|
||||
request_id=request_id,
|
||||
seqs=[seq],
|
||||
@@ -862,7 +866,8 @@ class LLMEngine:
|
||||
trace_headers=trace_headers,
|
||||
prompt_adapter_request=prompt_adapter_request,
|
||||
encoder_seq=encoder_seq,
|
||||
priority=priority)
|
||||
priority=priority,
|
||||
draft_size=draft_size)
|
||||
|
||||
return seq_group
|
||||
|
||||
|
||||
Reference in New Issue
Block a user