[Bugfix][Frontend] Reject guided decoding in multistep mode (#9892)

Signed-off-by: Joe Runde <Joseph.Runde@ibm.com>
This commit is contained in:
Joe Runde
2024-10-31 19:09:46 -06:00
committed by GitHub
parent b63c64d95b
commit 031a7995f3
4 changed files with 30 additions and 3 deletions

View File

@@ -829,6 +829,13 @@ class LLMEngine:
raise ValueError(f"Got priority {priority} but "
"Priority scheduling is not enabled.")
if isinstance(params, SamplingParams) \
and (params.guided_decoding or params.logits_processors) \
and self.scheduler_config.num_scheduler_steps > 1:
raise ValueError(
"Guided decoding and logits processors are not supported "
"in multi-step decoding")
if arrival_time is None:
arrival_time = time.time()