biondizzle/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Improve Mistral format checks. (#33253)

Browse Source 
Signed-off-by: Julien Denize <julien.denize@mistral.ai>
Signed-off-by: juliendenize <julien.denize@mistral.ai>
Co-authored-by: Michael Goin <mgoin64@gmail.com>
This commit is contained in:
Julien Denize
2026-01-30 15:23:33 +01:00
committed by GitHub
parent a11bc12d53
commit ae5b7aff2b
8 changed files with 193 additions and 24 deletions
Download Patch File Download Diff File Expand all files Collapse all files

36
vllm/tokenizers/registry.py
View File

@@ -1,6 +1,5 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import importlib.util
from dataclasses import dataclass, field
from functools import lru_cache
from pathlib import Path
@@ -18,7 +17,10 @@ from vllm.transformers_utils.gguf_utils import (
is_remote_gguf,
split_remote_gguf,
)
from vllm.transformers_utils.repo_utils import list_filtered_repo_files
from vllm.transformers_utils.repo_utils import (
any_pattern_in_repo_files,
is_mistral_model_repo,
)
from vllm.utils.import_utils import resolve_obj_by_qualname
from .protocol import TokenizerLike
@@ -142,26 +144,26 @@ def resolve_tokenizer_args(
kwargs["use_fast"] = False
# Try to use official Mistral tokenizer if possible
if tokenizer_mode == "auto" and importlib.util.find_spec("mistral_common"):
allow_patterns = ["tekken.json", "tokenizer.model.v*"]
files_list = list_filtered_repo_files(
if (
tokenizer_mode == "auto"
and is_mistral_model_repo(
model_name_or_path=str(tokenizer_name), revision=revision
)
and any_pattern_in_repo_files(
model_name_or_path=str(tokenizer_name),
allow_patterns=allow_patterns,
allow_patterns=["tekken.json", "tokenizer.model.v*"],
revision=revision,
)
if len(files_list) > 0:
tokenizer_mode = "mistral"
):
tokenizer_mode = "mistral"
# Try to use Grok2 tiktoken tokenizer if possible
if tokenizer_mode == "auto":
allow_patterns = ["tokenizer.tok.json"]
files_list = list_filtered_repo_files(
model_name_or_path=str(tokenizer_name),
allow_patterns=allow_patterns,
revision=revision,
)
if len(files_list) > 0:
tokenizer_mode = "grok2"
if tokenizer_mode == "auto" and any_pattern_in_repo_files(
model_name_or_path=str(tokenizer_name),
allow_patterns=["tokenizer.tok.json"],
revision=revision,
):
tokenizer_mode = "grok2"
# Fallback to HF tokenizer
if tokenizer_mode == "auto":