[Misc] Refactor tokenizer interface (#29693)

Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
2025-11-29 20:02:21 +08:00
parent f223ed4181
commit 34a984274e
119 changed files with 752 additions and 821 deletions
--- a/tools/pre_commit/mypy.py
+++ b/tools/pre_commit/mypy.py
@@ -35,6 +35,7 @@ FILES = [
    "vllm/multimodal",
    "vllm/platforms",
    "vllm/plugins",
+    "vllm/tokenizers",
    "vllm/transformers_utils",
    "vllm/triton_utils",
    "vllm/usage",