[Core] Remove tokenizer group in vLLM (#24078)

Signed-off-by: Zhuohan Li <zhuohan123@gmail.com>

vllm/transformers_utils/tokenizer.py

@@ -12,6 +12,7 @@ from typing import TYPE_CHECKING, Any, Optional, Union
 import huggingface_hub
 from transformers import (AutoTokenizer, PreTrainedTokenizer,
                           PreTrainedTokenizerFast)
+from typing_extensions import assert_never
 
 from vllm import envs
 from vllm.logger import init_logger
@@ -19,7 +20,6 @@ from vllm.transformers_utils.config import (
     get_sentence_transformer_tokenizer_config)
 from vllm.transformers_utils.tokenizers import MistralTokenizer
 from vllm.transformers_utils.utils import check_gguf_file
-from vllm.utils import make_async
 
 if TYPE_CHECKING:
     from vllm.config import ModelConfig
@@ -274,20 +274,19 @@ def cached_tokenizer_from_config(
     )
 
 
-def get_lora_tokenizer(lora_request: LoRARequest, *args,
-                       **kwargs) -> Optional[AnyTokenizer]:
-    if lora_request is None:
-        return None
-    try:
-        tokenizer = get_tokenizer(lora_request.lora_path, *args, **kwargs)
-    except Exception as e:
-        # No tokenizer was found in the LoRA folder,
-        # use base model tokenizer
-        logger.warning(
-            "No tokenizer found in %s, using base model tokenizer instead. "
-            "(Exception: %s)", lora_request.lora_path, e)
-        tokenizer = None
-    return tokenizer
+def init_tokenizer_from_configs(model_config: ModelConfig):
+    runner_type = model_config.runner_type
+    if runner_type == "generate" or runner_type == "draft":
+        truncation_side = "left"
+    elif runner_type == "pooling":
+        truncation_side = "right"
+    else:
+        assert_never(runner_type)
 
-
-get_lora_tokenizer_async = make_async(get_lora_tokenizer)
+    return get_tokenizer(
+        model_config.tokenizer,
+        tokenizer_mode=model_config.tokenizer_mode,
+        trust_remote_code=model_config.trust_remote_code,
+        revision=model_config.tokenizer_revision,
+        truncation_side=truncation_side,
+    )