[Bugfix] use right truncation for non-generative tasks (#12050)

Signed-off-by: Joe Runde <Joseph.Runde@ibm.com>
2025-01-15 09:31:01 -07:00
parent 57e729e874
commit edce722eaa
3 changed files with 13 additions and 1 deletions
--- a/vllm/config.py
+++ b/vllm/config.py
@@ -357,6 +357,10 @@ class ModelConfig:
        supported_tasks, task = self._resolve_task(task, self.hf_config)
        self.supported_tasks = supported_tasks
        self.task: Final = task
+        if self.task in ("draft", "generate"):
+            self.truncation_side = "left"
+        else:
+            self.truncation_side = "right"

        self.pooler_config = self._init_pooler_config(override_pooler_config)
        self.logits_processor_pattern = logits_processor_pattern