[BugFix] Fix Eagle IndexError: list index out of range for even num_speculative_tokens (#29102)

Signed-off-by: Lucas Wilkinson <lwilkins@redhat.com>
This commit is contained in:
Lucas Wilkinson
2025-11-21 19:47:05 -05:00
committed by GitHub
parent e9af6ba62a
commit 30d6466238
3 changed files with 37 additions and 20 deletions

View File

@@ -748,6 +748,14 @@ class VllmRunner:
# being captured which can trigger edge cases that we don't handle yet.
kwargs["compilation_config"] = {"cudagraph_capture_sizes": [4]}
# Make sure we have atleast one cudagraph large enough for a single decode.
if (speculative_config := kwargs.get("speculative_config")) and (
num_speculative_tokens := speculative_config["num_speculative_tokens"]
):
kwargs["compilation_config"]["cudagraph_capture_sizes"].append(
num_speculative_tokens + 1
)
with init_ctx:
self.llm = LLM(
model=model_name,