biondizzle/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[Chore] Move tokenizer initialization methods (#29793)

Browse Source 
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2025-12-02 13:33:37 +08:00
committed by GitHub
parent e2fbfc955e
commit 653591d5e7
51 changed files with 150 additions and 129 deletions
Download Patch File Download Diff File Expand all files Collapse all files

2
tests/entrypoints/openai/test_chat_template.py
View File

@@ -6,7 +6,7 @@ import pytest
from vllm.config import ModelConfig
from vllm.entrypoints.chat_utils import apply_hf_chat_template, load_chat_template
from vllm.entrypoints.openai.protocol import ChatCompletionRequest
from vllm.transformers_utils.tokenizer import get_tokenizer
from vllm.tokenizers import get_tokenizer
from ...models.registry import HF_EXAMPLE_MODELS
from ...utils import VLLM_PATH

2
tests/entrypoints/openai/test_lora_resolvers.py
View File

@@ -14,7 +14,7 @@ from vllm.entrypoints.openai.serving_completion import OpenAIServingCompletion
from vllm.entrypoints.openai.serving_models import BaseModelPath, OpenAIServingModels
from vllm.lora.request import LoRARequest
from vllm.lora.resolver import LoRAResolver, LoRAResolverRegistry
from vllm.transformers_utils.tokenizer import get_tokenizer
from vllm.tokenizers import get_tokenizer
from vllm.v1.engine.async_llm import AsyncLLM
MODEL_NAME = "openai-community/gpt2"

2
tests/entrypoints/openai/test_return_token_ids.py
View File

@@ -3,7 +3,7 @@
import pytest
from vllm.transformers_utils.tokenizer import get_tokenizer
from vllm.tokenizers import get_tokenizer
from ...utils import RemoteOpenAIServer

2
tests/entrypoints/openai/test_return_tokens_as_ids.py
View File

@@ -7,7 +7,7 @@
import pytest
from vllm.transformers_utils.tokenizer import get_tokenizer
from vllm.tokenizers import get_tokenizer
from ...utils import RemoteOpenAIServer

2
tests/entrypoints/openai/test_serving_chat.py
View File

@@ -14,7 +14,7 @@ from vllm.config.multimodal import MultiModalConfig
from vllm.entrypoints.openai.protocol import ChatCompletionRequest
from vllm.entrypoints.openai.serving_chat import OpenAIServingChat
from vllm.entrypoints.openai.serving_models import BaseModelPath, OpenAIServingModels
from vllm.transformers_utils.tokenizer import get_tokenizer
from vllm.tokenizers import get_tokenizer
from vllm.v1.engine.async_llm import AsyncLLM
from ...utils import RemoteOpenAIServer

2
tests/entrypoints/openai/test_token_in_token_out.py
View File

@@ -7,7 +7,7 @@ import tempfile
import pytest
from vllm.model_executor.model_loader.weight_utils import download_weights_from_hf
from vllm.transformers_utils.tokenizer import get_tokenizer
from vllm.tokenizers import get_tokenizer
from ...utils import RemoteOpenAIServer

2
tests/entrypoints/openai/test_tokenization.py
View File

@@ -5,7 +5,7 @@ import pytest
import pytest_asyncio
import requests
from vllm.transformers_utils.tokenizer import get_tokenizer
from vllm.tokenizers import get_tokenizer
from ...utils import RemoteOpenAIServer

2
tests/entrypoints/openai/tool_parsers/test_hermes_tool_parser.py
View File

@@ -271,7 +271,7 @@ async def test_streaming_product_tool_call():
@pytest.fixture
def qwen_tokenizer() -> TokenizerLike:
from vllm.transformers_utils.tokenizer import get_tokenizer
from vllm.tokenizers import get_tokenizer
return get_tokenizer("Qwen/Qwen3-32B")