[Frontend] Consolidate tokenizer init code (#26276)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>

vllm/entrypoints/llm.py

@@ -74,7 +74,6 @@ from vllm.transformers_utils.tokenizer import (
     AnyTokenizer,
     MistralTokenizer,
     get_cached_tokenizer,
-    init_tokenizer_from_configs,
 )
 from vllm.usage.usage_lib import UsageContext
 from vllm.utils import Counter, Device, as_iter, is_list_of
@@ -367,11 +366,8 @@ class LLM:
     def _get_processor(self) -> Processor:
         if not hasattr(self, "_processor"):
             vllm_config = self.llm_engine.vllm_config
-            if self.model_config.skip_tokenizer_init:
-                tokenizer = None
-            else:
-                tokenizer = init_tokenizer_from_configs(self.model_config)
-            self._processor = Processor(vllm_config, tokenizer)
+            self._processor = Processor(vllm_config)
+
         return self._processor
 
     def get_default_sampling_params(self) -> SamplingParams: