[Model] Add Grok-2 (#31847)

Signed-off-by: dangoldbj <dangoldbj23@gmail.com>

vllm/tokenizers/registry.py

@@ -31,6 +31,7 @@ logger = init_logger(__name__)
 
 _VLLM_TOKENIZERS = {
     "deepseek_v32": ("deepseek_v32", "DeepseekV32Tokenizer"),
+    "grok2": ("grok2", "Grok2Tokenizer"),
     "hf": ("hf", "CachedHfTokenizer"),
     "mistral": ("mistral", "MistralTokenizer"),
 }
@@ -151,6 +152,17 @@ def resolve_tokenizer_args(
         if len(files_list) > 0:
             tokenizer_mode = "mistral"
 
+    # Try to use Grok2 tiktoken tokenizer if possible
+    if tokenizer_mode == "auto":
+        allow_patterns = ["tokenizer.tok.json"]
+        files_list = list_filtered_repo_files(
+            model_name_or_path=str(tokenizer_name),
+            allow_patterns=allow_patterns,
+            revision=revision,
+        )
+        if len(files_list) > 0:
+            tokenizer_mode = "grok2"
+
     # Fallback to HF tokenizer
     if tokenizer_mode == "auto":
         tokenizer_mode = "hf"