[V1][Spec Decode] Ngram Spec Decode (#12193)

Signed-off-by: LiuXiaoxuanPKU <lilyliupku@gmail.com>
2025-02-15 18:05:11 -08:00
parent 367cb8ce8c
commit 80f63a3966
21 changed files with 1023 additions and 82 deletions
--- a/vllm/v1/sample/metadata.py
+++ b/vllm/v1/sample/metadata.py
@@ -12,6 +12,8 @@ class SamplingMetadata:
    temperature: torch.Tensor
    all_greedy: bool
    all_random: bool
+    rejection_sampling: bool
+    spec_token_ids: List[List[int]]

    top_p: torch.Tensor
    top_k: torch.Tensor