[Misc] Clean up input processing (#17582)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
2025-05-02 23:11:53 +08:00
parent 3a500cd0b6
commit cb234955df
9 changed files with 359 additions and 285 deletions
--- a/vllm/engine/async_llm_engine.py
+++ b/vllm/engine/async_llm_engine.py
@@ -497,10 +497,6 @@ class _AsyncLLMEngine(LLMEngine):
            prompt["prompt_token_ids"] = [0
                                          ] * prompt["prompt_embeds"].shape[-2]

-        if self.tokenizer is not None:
-            tokenizer = await self.get_tokenizer_async(lora_request)
-            self._validate_token_prompt(prompt, tokenizer=tokenizer)
-
        processed_inputs = await self.input_preprocessor.preprocess_async(
            prompt,
            lora_request=lora_request,