2025-10-27 11:06:25 -04:00
|
|
|
# SPDX-License-Identifier: Apache-2.0
|
|
|
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
|
|
|
|
|
|
|
|
import pytest
|
|
|
|
|
from transformers import AutoTokenizer
|
|
|
|
|
|
2025-11-29 20:02:21 +08:00
|
|
|
from vllm.tokenizers import TokenizerLike
|
2025-10-27 11:06:25 -04:00
|
|
|
|
|
|
|
|
|
|
|
|
|
@pytest.fixture(scope="function")
|
2025-11-29 20:02:21 +08:00
|
|
|
def default_tokenizer() -> TokenizerLike:
|
2025-10-27 11:06:25 -04:00
|
|
|
return AutoTokenizer.from_pretrained("gpt2")
|
