[V1][Sampler] Avoid an operation during temperature application (#13587)

2025-02-20 22:05:56 -08:00
parent a30c093502
commit 31aa045c11
4 changed files with 18 additions and 10 deletions
--- a/vllm/v1/sample/metadata.py
+++ b/vllm/v1/sample/metadata.py
@@ -9,7 +9,7 @@ import torch
@dataclass
 class SamplingMetadata:

-    temperature: torch.Tensor
+    temperature: Optional[torch.Tensor]
    all_greedy: bool
    all_random: bool