[RFC][vllm-API] Support tokenizer registry for customized tokenizer in vLLM (#12518)

Signed-off-by: Keyun Tong <tongkeyun@gmail.com>
2025-02-11 20:25:58 -08:00
parent 72c2b68dc9
commit 3ee696a63d
11 changed files with 343 additions and 41 deletions
--- a/vllm/logits_process.py
+++ b/vllm/logits_process.py
@@ -31,7 +31,7 @@ def get_bad_words_logits_processors(

            if isinstance(tokenizer, MistralTokenizer):
                # Mistral tokenizers should not add special tokens
-                prompt_token_ids = tokenizer.encode(prompt=prompt)
+                prompt_token_ids = tokenizer.encode(text=prompt)
            else:
                prompt_token_ids = tokenizer.encode(text=prompt,
                                                    add_special_tokens=False)