[Core] Remove tokenizer group in vLLM (#24078)

Signed-off-by: Zhuohan Li <zhuohan123@gmail.com>
2025-09-17 01:42:59 -07:00
parent c15309a730
commit 6c47f6bfa4
49 changed files with 276 additions and 934 deletions
--- a/vllm/transformers_utils/tokenizers/mistral.py
+++ b/vllm/transformers_utils/tokenizers/mistral.py
@@ -327,6 +327,10 @@ class MistralTokenizer(TokenizerBase):
    def max_token_id(self) -> int:
        return self._max_token_id

+    @property
+    def truncation_side(self) -> str:
+        raise NotImplementedError()
+
    def __len__(self) -> int:
        return self.vocab_size