[v1] Support allowed_token_ids in v1 Sampler (#13210)
Signed-off-by: Lu Fang <lufang@fb.com>
This commit is contained in:
@@ -37,3 +37,7 @@ class SamplingMetadata:
|
||||
min_tokens: Dict[int, Tuple[int, Set[int]]]
|
||||
|
||||
logit_bias: List[Optional[Dict[int, float]]]
|
||||
|
||||
# `allowed_token_ids_mask` is a 2D bool tensor of shape (max batch size,
|
||||
# vocab size).
|
||||
allowed_token_ids_mask: Optional[torch.Tensor]
|
||||
|
||||
Reference in New Issue
Block a user