[V1] Support per-request seed (#9945)

Signed-off-by: Nick Hill <nickhill@us.ibm.com>
2024-11-03 17:14:17 +00:00
parent 3bb4befea7
commit 1f1b6d6eda
3 changed files with 41 additions and 48 deletions
--- a/vllm/v1/sample/metadata.py
+++ b/vllm/v1/sample/metadata.py
@@ -1,5 +1,5 @@
 from dataclasses import dataclass
-from typing import List, Optional
+from typing import Dict

 import torch

@@ -16,7 +16,6 @@ class SamplingMetadata:
    no_top_p: bool
    no_top_k: bool

-    generators: List[Optional[torch.Generator]]
-    no_generator: bool
+    generators: Dict[int, torch.Generator]

    max_num_logprobs: int