Support logit_bias in v1 Sampler (#13079)

2025-02-14 04:34:59 -08:00
parent 085b7b2d6c
commit 6224a9f620
6 changed files with 200 additions and 101 deletions
--- a/vllm/v1/sample/metadata.py
+++ b/vllm/v1/sample/metadata.py
@@ -32,3 +32,5 @@ class SamplingMetadata:
    output_token_ids: List[List[int]]
    min_tokens: List[int]
    stop_token_ids: List[Set[int]]
+
+    logit_bias: List[Optional[Dict[int, float]]]