biondizzle/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[V1] Make AsyncLLMEngine v1-v0 opaque (#11383)

Browse Source 
Signed-off-by: Ricky Xu <xuchen727@hotmail.com>
This commit is contained in:
Ricky Xu
2024-12-20 23:14:08 -08:00
committed by GitHub
parent 51ff216d85
commit 584f0ae40d
3 changed files with 9 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
vllm/entrypoints/openai/api_server.py
View File

@@ -27,6 +27,7 @@ from typing_extensions import assert_never
import vllm.envs as envs
from vllm.config import ModelConfig
from vllm.engine.arg_utils import AsyncEngineArgs
from vllm.engine.async_llm_engine import AsyncLLMEngine # type: ignore
from vllm.engine.multiprocessing.client import MQLLMEngineClient
from vllm.engine.multiprocessing.engine import run_mp_engine
from vllm.engine.protocol import EngineClient
@@ -66,11 +67,6 @@ from vllm.utils import (FlexibleArgumentParser, get_open_zmq_ipc_path,
is_valid_ipv6_address)
from vllm.version import __version__ as VLLM_VERSION
if envs.VLLM_USE_V1:
from vllm.v1.engine.async_llm import AsyncLLMEngine # type: ignore
else:
from vllm.engine.async_llm_engine import AsyncLLMEngine # type: ignore
TIMEOUT_KEEP_ALIVE = 5 # seconds
prometheus_multiproc_dir: tempfile.TemporaryDirectory