[Core] Remove tokenizer group in vLLM (#24078)

Signed-off-by: Zhuohan Li <zhuohan123@gmail.com>

tests/tokenization/test_detokenize.py

@@ -11,7 +11,7 @@ from transformers import (AutoTokenizer, PreTrainedTokenizer,
 from vllm.inputs import token_inputs
 from vllm.sequence import Logprob, SamplingParams, Sequence, SequenceGroup
 from vllm.transformers_utils.detokenizer import Detokenizer
-from vllm.transformers_utils.tokenizer_group import TokenizerGroup
+from vllm.transformers_utils.tokenizer import get_tokenizer
 from vllm.transformers_utils.tokenizers.mistral import MistralTokenizer
 from vllm.v1.engine import EngineCoreRequest
 from vllm.v1.engine.detokenizer import (FastIncrementalDetokenizer,
@@ -221,17 +221,14 @@ def test_oov_decode(tokenizer, fast):
 
 @pytest.fixture
 def detokenizer(tokenizer_name: str) -> Detokenizer:
-    tokenizer_group = TokenizerGroup(
-        tokenizer_id=tokenizer_name,
-        enable_lora=False,
-        max_num_seqs=100,
-        max_input_length=None,
+    tokenizer = get_tokenizer(
+        tokenizer_name,
         tokenizer_mode="mistral" if "mistral" in tokenizer_name else "auto",
         trust_remote_code=False,
         revision=None,
     )
 
-    return Detokenizer(tokenizer_group)
+    return Detokenizer(tokenizer)
 
 
 @pytest.fixture(name="complete_sequence_token_ids")
@@ -312,8 +309,7 @@ def test_decode_prompt_logprobs(complete_sequence: str,
     # don't support that.
     if complete_sequence not in SPECIAL_TOKS_TRUTH:
         skip_special_tokens = True
-    elif not isinstance(detokenizer.tokenizer_group.get_lora_tokenizer(None),
-                        MistralTokenizer):
+    elif not isinstance(detokenizer.tokenizer, MistralTokenizer):
         skip_special_tokens = False
     else:
         pytest.skip("MistralTokenizers don't support "
@@ -339,7 +335,7 @@ def test_decode_prompt_logprobs(complete_sequence: str,
 
     # decoded_prompt_logprobs doesn't contain the first token.
     token_ids = complete_sequence_token_ids
-    tokenizer = detokenizer.get_tokenizer_for_seq(seq)
+    tokenizer = detokenizer.tokenizer
     text_full = tokenizer.decode(token_ids,
                                  skip_special_tokens=skip_special_tokens)
     text_first = tokenizer.decode(token_ids[0],