biondizzle/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Convert formatting to use ruff instead of yapf + isort (#26247)

Browse Source 
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
Harry Mellor
2025-10-05 15:06:22 +01:00
committed by GitHub
parent 17edd8a807
commit d6953beb91
1508 changed files with 115244 additions and 94146 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
vllm/entrypoints/cli/__init__.py
View File

@@ -2,11 +2,10 @@
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
from vllm.entrypoints.cli.benchmark.latency import BenchmarkLatencySubcommand
from vllm.entrypoints.cli.benchmark.serve import BenchmarkServingSubcommand
from vllm.entrypoints.cli.benchmark.throughput import (
BenchmarkThroughputSubcommand)
from vllm.entrypoints.cli.benchmark.throughput import BenchmarkThroughputSubcommand
__all__: list[str] = [
"BenchmarkLatencySubcommand",
"BenchmarkServingSubcommand",
"BenchmarkThroughputSubcommand",
]
]

2
vllm/entrypoints/cli/benchmark/base.py
View File

@@ -6,7 +6,7 @@ from vllm.entrypoints.cli.types import CLISubcommand
class BenchmarkSubcommandBase(CLISubcommand):
""" The base class of subcommands for vllm bench. """
"""The base class of subcommands for vllm bench."""
help: str

2
vllm/entrypoints/cli/benchmark/latency.py
View File

@@ -7,7 +7,7 @@ from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
class BenchmarkLatencySubcommand(BenchmarkSubcommandBase):
""" The `latency` subcommand for vllm bench. """
"""The `latency` subcommand for vllm bench."""
name = "latency"
help = "Benchmark the latency of a single batch of requests."

15
vllm/entrypoints/cli/benchmark/main.py
View File

@@ -15,7 +15,7 @@ if typing.TYPE_CHECKING:
class BenchmarkSubcommand(CLISubcommand):
""" The `bench` subcommand for the vLLM CLI. """
"""The `bench` subcommand for the vLLM CLI."""
name = "bench"
help = "vLLM bench subcommand."
@@ -28,14 +28,14 @@ class BenchmarkSubcommand(CLISubcommand):
pass
def subparser_init(
self,
subparsers: argparse._SubParsersAction) -> FlexibleArgumentParser:
self, subparsers: argparse._SubParsersAction
) -> FlexibleArgumentParser:
bench_parser = subparsers.add_parser(
self.name,
description=self.help,
usage=f"vllm {self.name} <bench_type> [options]")
bench_subparsers = bench_parser.add_subparsers(required=True,
dest="bench_type")
usage=f"vllm {self.name} <bench_type> [options]",
)
bench_subparsers = bench_parser.add_subparsers(required=True, dest="bench_type")
for cmd_cls in BenchmarkSubcommandBase.__subclasses__():
cmd_subparser = bench_subparsers.add_parser(
@@ -47,7 +47,8 @@ class BenchmarkSubcommand(CLISubcommand):
cmd_subparser.set_defaults(dispatch_function=cmd_cls.cmd)
cmd_cls.add_cli_args(cmd_subparser)
cmd_subparser.epilog = VLLM_SUBCMD_PARSER_EPILOG.format(
subcmd=f"{self.name} {cmd_cls.name}")
subcmd=f"{self.name} {cmd_cls.name}"
)
return bench_parser

2
vllm/entrypoints/cli/benchmark/serve.py
View File

@@ -7,7 +7,7 @@ from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
class BenchmarkServingSubcommand(BenchmarkSubcommandBase):
""" The `serve` subcommand for vllm bench. """
"""The `serve` subcommand for vllm bench."""
name = "serve"
help = "Benchmark the online serving throughput."

2
vllm/entrypoints/cli/benchmark/throughput.py
View File

@@ -7,7 +7,7 @@ from vllm.entrypoints.cli.benchmark.base import BenchmarkSubcommandBase
class BenchmarkThroughputSubcommand(BenchmarkSubcommandBase):
""" The `throughput` subcommand for vllm bench. """
"""The `throughput` subcommand for vllm bench."""
name = "throughput"
help = "Benchmark offline inference throughput."

10
vllm/entrypoints/cli/collect_env.py
View File

@@ -14,7 +14,8 @@ if typing.TYPE_CHECKING:
class CollectEnvSubcommand(CLISubcommand):
"""The `collect-env` subcommand for the vLLM CLI. """
"""The `collect-env` subcommand for the vLLM CLI."""
name = "collect-env"
@staticmethod
@@ -23,13 +24,14 @@ class CollectEnvSubcommand(CLISubcommand):
collect_env_main()
def subparser_init(
self,
subparsers: argparse._SubParsersAction) -> FlexibleArgumentParser:
self, subparsers: argparse._SubParsersAction
) -> FlexibleArgumentParser:
return subparsers.add_parser(
"collect-env",
help="Start collecting environment information.",
description="Start collecting environment information.",
usage="vllm collect-env")
usage="vllm collect-env",
)
def cmd_init() -> list[CLISubcommand]:

16
vllm/entrypoints/cli/main.py
View File

@@ -1,9 +1,10 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
'''The CLI entrypoints of vLLM
"""The CLI entrypoints of vLLM
Note that all future modules must be lazily loaded within main
to avoid certain eager import breakage.'''
to avoid certain eager import breakage."""
from __future__ import annotations
import importlib.metadata
@@ -33,18 +34,17 @@ def main():
epilog=VLLM_SUBCMD_PARSER_EPILOG.format(subcmd="[subcommand]"),
)
parser.add_argument(
'-v',
'--version',
action='version',
version=importlib.metadata.version('vllm'),
"-v",
"--version",
action="version",
version=importlib.metadata.version("vllm"),
)
subparsers = parser.add_subparsers(required=False, dest="subparser")
cmds = {}
for cmd_module in CMD_MODULES:
new_cmds = cmd_module.cmd_init()
for cmd in new_cmds:
cmd.subparser_init(subparsers).set_defaults(
dispatch_function=cmd.cmd)
cmd.subparser_init(subparsers).set_defaults(dispatch_function=cmd.cmd)
cmds[cmd.name] = cmd
args = parser.parse_args()
if args.subparser in cmds:

105
vllm/entrypoints/cli/openai.py
View File

@@ -19,7 +19,6 @@ if TYPE_CHECKING:
def _register_signal_handlers():
def signal_handler(sig, frame):
sys.exit(0)
@@ -80,26 +79,29 @@ def chat(system_prompt: str | None, model_name: str, client: OpenAI) -> None:
break
conversation.append({"role": "user", "content": input_message})
stream = client.chat.completions.create(model=model_name,
messages=conversation,
stream=True)
stream = client.chat.completions.create(
model=model_name, messages=conversation, stream=True
)
output = _print_chat_stream(stream)
conversation.append({"role": "assistant", "content": output})
def _add_query_options(
parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
def _add_query_options(parser: FlexibleArgumentParser) -> FlexibleArgumentParser:
parser.add_argument(
"--url",
type=str,
default="http://localhost:8000/v1",
help="url of the running OpenAI-Compatible RESTful API server")
help="url of the running OpenAI-Compatible RESTful API server",
)
parser.add_argument(
"--model-name",
type=str,
default=None,
help=("The model name used in prompt completion, default to "
"the first model in list models API call."))
help=(
"The model name used in prompt completion, default to "
"the first model in list models API call."
),
)
parser.add_argument(
"--api-key",
type=str,
@@ -107,12 +109,14 @@ def _add_query_options(
help=(
"API key for OpenAI services. If provided, this api key "
"will overwrite the api key obtained through environment variables."
))
),
)
return parser
class ChatCommand(CLISubcommand):
"""The `chat` subcommand for the vLLM CLI. """
"""The `chat` subcommand for the vLLM CLI."""
name = "chat"
@staticmethod
@@ -127,9 +131,9 @@ class ChatCommand(CLISubcommand):
if args.quick:
conversation.append({"role": "user", "content": args.quick})
stream = client.chat.completions.create(model=model_name,
messages=conversation,
stream=True)
stream = client.chat.completions.create(
model=model_name, messages=conversation, stream=True
)
output = _print_chat_stream(stream)
conversation.append({"role": "assistant", "content": output})
return
@@ -142,9 +146,9 @@ class ChatCommand(CLISubcommand):
break
conversation.append({"role": "user", "content": input_message})
stream = client.chat.completions.create(model=model_name,
messages=conversation,
stream=True)
stream = client.chat.completions.create(
model=model_name, messages=conversation, stream=True
)
output = _print_chat_stream(stream)
conversation.append({"role": "assistant", "content": output})
@@ -156,39 +160,45 @@ class ChatCommand(CLISubcommand):
"--system-prompt",
type=str,
default=None,
help=("The system prompt to be added to the chat template, "
"used for models that support system prompts."))
parser.add_argument("-q",
"--quick",
type=str,
metavar="MESSAGE",
help=("Send a single prompt as MESSAGE "
"and print the response, then exit."))
help=(
"The system prompt to be added to the chat template, "
"used for models that support system prompts."
),
)
parser.add_argument(
"-q",
"--quick",
type=str,
metavar="MESSAGE",
help=("Send a single prompt as MESSAGE and print the response, then exit."),
)
return parser
def subparser_init(
self,
subparsers: argparse._SubParsersAction) -> FlexibleArgumentParser:
self, subparsers: argparse._SubParsersAction
) -> FlexibleArgumentParser:
parser = subparsers.add_parser(
"chat",
help="Generate chat completions via the running API server.",
description="Generate chat completions via the running API server.",
usage="vllm chat [options]")
usage="vllm chat [options]",
)
return ChatCommand.add_cli_args(parser)
class CompleteCommand(CLISubcommand):
"""The `complete` subcommand for the vLLM CLI. """
name = 'complete'
"""The `complete` subcommand for the vLLM CLI."""
name = "complete"
@staticmethod
def cmd(args: argparse.Namespace) -> None:
model_name, client = _interactive_cli(args)
if args.quick:
stream = client.completions.create(model=model_name,
prompt=args.quick,
stream=True)
stream = client.completions.create(
model=model_name, prompt=args.quick, stream=True
)
_print_completion_stream(stream)
return
@@ -198,9 +208,9 @@ class CompleteCommand(CLISubcommand):
input_prompt = input("> ")
except EOFError:
break
stream = client.completions.create(model=model_name,
prompt=input_prompt,
stream=True)
stream = client.completions.create(
model=model_name, prompt=input_prompt, stream=True
)
_print_completion_stream(stream)
@staticmethod
@@ -212,20 +222,25 @@ class CompleteCommand(CLISubcommand):
"--quick",
type=str,
metavar="PROMPT",
help=
"Send a single prompt and print the completion output, then exit.")
help="Send a single prompt and print the completion output, then exit.",
)
return parser
def subparser_init(
self,
subparsers: argparse._SubParsersAction) -> FlexibleArgumentParser:
self, subparsers: argparse._SubParsersAction
) -> FlexibleArgumentParser:
parser = subparsers.add_parser(
"complete",
help=("Generate text completions based on the given prompt "
"via the running API server."),
description=("Generate text completions based on the given prompt "
"via the running API server."),
usage="vllm complete [options]")
help=(
"Generate text completions based on the given prompt "
"via the running API server."
),
description=(
"Generate text completions based on the given prompt "
"via the running API server."
),
usage="vllm complete [options]",
)
return CompleteCommand.add_cli_args(parser)

19
vllm/entrypoints/cli/run_batch.py
View File

@@ -20,14 +20,16 @@ logger = init_logger(__name__)
class RunBatchSubcommand(CLISubcommand):
"""The `run-batch` subcommand for vLLM CLI."""
name = "run-batch"
@staticmethod
def cmd(args: argparse.Namespace) -> None:
from vllm.entrypoints.openai.run_batch import main as run_batch_main
logger.info("vLLM batch processing API version %s",
importlib.metadata.version("vllm"))
logger.info(
"vLLM batch processing API version %s", importlib.metadata.version("vllm")
)
logger.info("args: %s", args)
# Start the Prometheus metrics server.
@@ -44,8 +46,8 @@ class RunBatchSubcommand(CLISubcommand):
asyncio.run(run_batch_main(args))
def subparser_init(
self,
subparsers: argparse._SubParsersAction) -> FlexibleArgumentParser:
self, subparsers: argparse._SubParsersAction
) -> FlexibleArgumentParser:
from vllm.entrypoints.openai.run_batch import make_arg_parser
run_batch_parser = subparsers.add_parser(
@@ -53,13 +55,12 @@ class RunBatchSubcommand(CLISubcommand):
help="Run batch prompts and write results to file.",
description=(
"Run batch prompts using vLLM's OpenAI-compatible API.\n"
"Supports local or HTTP input/output files."),
usage=
"vllm run-batch -i INPUT.jsonl -o OUTPUT.jsonl --model <model>",
"Supports local or HTTP input/output files."
),
usage="vllm run-batch -i INPUT.jsonl -o OUTPUT.jsonl --model <model>",
)
run_batch_parser = make_arg_parser(run_batch_parser)
run_batch_parser.epilog = VLLM_SUBCMD_PARSER_EPILOG.format(
subcmd=self.name)
run_batch_parser.epilog = VLLM_SUBCMD_PARSER_EPILOG.format(subcmd=self.name)
return run_batch_parser

102
vllm/entrypoints/cli/serve.py
View File

@@ -10,21 +10,26 @@ import uvloop
import vllm
import vllm.envs as envs
from vllm.entrypoints.cli.types import CLISubcommand
from vllm.entrypoints.openai.api_server import (run_server, run_server_worker,
setup_server)
from vllm.entrypoints.openai.cli_args import (make_arg_parser,
validate_parsed_serve_args)
from vllm.entrypoints.openai.api_server import (
run_server,
run_server_worker,
setup_server,
)
from vllm.entrypoints.openai.cli_args import make_arg_parser, validate_parsed_serve_args
from vllm.entrypoints.utils import VLLM_SUBCMD_PARSER_EPILOG
from vllm.logger import init_logger
from vllm.usage.usage_lib import UsageContext
from vllm.utils import (FlexibleArgumentParser, decorate_logs, get_tcp_uri,
set_process_title)
from vllm.utils import (
FlexibleArgumentParser,
decorate_logs,
get_tcp_uri,
set_process_title,
)
from vllm.v1.engine.core import EngineCoreProc
from vllm.v1.engine.utils import CoreEngineProcManager, launch_core_engines
from vllm.v1.executor.abstract import Executor
from vllm.v1.metrics.prometheus import setup_multiprocess_prometheus
from vllm.v1.utils import (APIServerProcessManager,
wait_for_completion_or_failure)
from vllm.v1.utils import APIServerProcessManager, wait_for_completion_or_failure
logger = init_logger(__name__)
@@ -38,13 +43,14 @@ Search by using: `--help=<ConfigGroup>` to explore options by section (e.g.,
class ServeSubcommand(CLISubcommand):
"""The `serve` subcommand for the vLLM CLI. """
"""The `serve` subcommand for the vLLM CLI."""
name = "serve"
@staticmethod
def cmd(args: argparse.Namespace) -> None:
# If model is specified in CLI (as positional arg), it takes precedence
if hasattr(args, 'model_tag') and args.model_tag is not None:
if hasattr(args, "model_tag") and args.model_tag is not None:
args.model = args.model_tag
if args.headless or args.api_server_count < 1:
@@ -60,16 +66,14 @@ class ServeSubcommand(CLISubcommand):
validate_parsed_serve_args(args)
def subparser_init(
self,
subparsers: argparse._SubParsersAction) -> FlexibleArgumentParser:
self, subparsers: argparse._SubParsersAction
) -> FlexibleArgumentParser:
serve_parser = subparsers.add_parser(
self.name,
description=DESCRIPTION,
usage="vllm serve [model_tag] [options]")
self.name, description=DESCRIPTION, usage="vllm serve [model_tag] [options]"
)
serve_parser = make_arg_parser(serve_parser)
serve_parser.epilog = VLLM_SUBCMD_PARSER_EPILOG.format(
subcmd=self.name)
serve_parser.epilog = VLLM_SUBCMD_PARSER_EPILOG.format(subcmd=self.name)
return serve_parser
@@ -78,29 +82,27 @@ def cmd_init() -> list[CLISubcommand]:
def run_headless(args: argparse.Namespace):
if args.api_server_count > 1:
raise ValueError("api_server_count can't be set in headless mode")
# Create the EngineConfig.
engine_args = vllm.AsyncEngineArgs.from_cli_args(args)
usage_context = UsageContext.OPENAI_API_SERVER
vllm_config = engine_args.create_engine_config(usage_context=usage_context,
headless=True)
vllm_config = engine_args.create_engine_config(
usage_context=usage_context, headless=True
)
if not envs.VLLM_USE_V1:
raise ValueError("Headless mode is only supported for V1")
if engine_args.data_parallel_hybrid_lb:
raise ValueError("data_parallel_hybrid_lb is not applicable in "
"headless mode")
raise ValueError("data_parallel_hybrid_lb is not applicable in headless mode")
parallel_config = vllm_config.parallel_config
local_engine_count = parallel_config.data_parallel_size_local
if local_engine_count <= 0:
raise ValueError("data_parallel_size_local must be > 0 in "
"headless mode")
raise ValueError("data_parallel_size_local must be > 0 in headless mode")
host = parallel_config.data_parallel_master_ip
port = engine_args.data_parallel_rpc_port # add to config too
@@ -116,7 +118,10 @@ def run_headless(args: argparse.Namespace):
logger.info(
"Launching %d data parallel engine(s) in headless mode, "
"with head node address %s.", local_engine_count, handshake_address)
"with head node address %s.",
local_engine_count,
handshake_address,
)
# Create the engines.
engine_manager = CoreEngineProcManager(
@@ -139,7 +144,6 @@ def run_headless(args: argparse.Namespace):
def run_multi_api_server(args: argparse.Namespace):
assert not args.headless
num_api_servers: int = args.api_server_count
assert num_api_servers > 0
@@ -161,8 +165,10 @@ def run_multi_api_server(args: argparse.Namespace):
raise ValueError("api_server_count > 1 is only supported for V1")
if envs.VLLM_ALLOW_RUNTIME_LORA_UPDATING:
raise ValueError("VLLM_ALLOW_RUNTIME_LORA_UPDATING cannot be used "
"with api_server_count > 1")
raise ValueError(
"VLLM_ALLOW_RUNTIME_LORA_UPDATING cannot be used "
"with api_server_count > 1"
)
executor_class = Executor.get_class(vllm_config)
log_stats = not engine_args.disable_log_stats
@@ -175,10 +181,9 @@ def run_multi_api_server(args: argparse.Namespace):
api_server_manager: Optional[APIServerProcessManager] = None
with launch_core_engines(vllm_config, executor_class, log_stats,
num_api_servers) as (local_engine_manager,
coordinator, addresses):
with launch_core_engines(
vllm_config, executor_class, log_stats, num_api_servers
) as (local_engine_manager, coordinator, addresses):
# Construct common args for the APIServerProcessManager up-front.
api_server_manager_kwargs = dict(
target_server_fn=run_api_server_worker_proc,
@@ -189,7 +194,9 @@ def run_multi_api_server(args: argparse.Namespace):
input_addresses=addresses.inputs,
output_addresses=addresses.outputs,
stats_update_address=coordinator.get_stats_publish_address()
if coordinator else None)
if coordinator
else None,
)
# For dp ranks > 0 in external/hybrid DP LB modes, we must delay the
# start of the API servers until the local engine is started
@@ -198,27 +205,26 @@ def run_multi_api_server(args: argparse.Namespace):
# via the handshake with the local engine.
if dp_rank == 0 or not (external_dp_lb or hybrid_dp_lb):
# Start API servers using the manager.
api_server_manager = APIServerProcessManager(
**api_server_manager_kwargs)
api_server_manager = APIServerProcessManager(**api_server_manager_kwargs)
# Start API servers now if they weren't already started.
if api_server_manager is None:
api_server_manager_kwargs["stats_update_address"] = (
addresses.frontend_stats_publish_address)
api_server_manager = APIServerProcessManager(
**api_server_manager_kwargs)
addresses.frontend_stats_publish_address
)
api_server_manager = APIServerProcessManager(**api_server_manager_kwargs)
# Wait for API servers
wait_for_completion_or_failure(api_server_manager=api_server_manager,
engine_manager=local_engine_manager,
coordinator=coordinator)
wait_for_completion_or_failure(
api_server_manager=api_server_manager,
engine_manager=local_engine_manager,
coordinator=coordinator,
)
def run_api_server_worker_proc(listen_address,
sock,
args,
client_config=None,
**uvicorn_kwargs) -> None:
def run_api_server_worker_proc(
listen_address, sock, args, client_config=None, **uvicorn_kwargs
) -> None:
"""Entrypoint for individual API server worker processes."""
client_config = client_config or {}
server_index = client_config.get("client_index", 0)
@@ -228,5 +234,5 @@ def run_api_server_worker_proc(listen_address,
decorate_logs()
uvloop.run(
run_server_worker(listen_address, sock, args, client_config,
**uvicorn_kwargs))
run_server_worker(listen_address, sock, args, client_config, **uvicorn_kwargs)
)

4
vllm/entrypoints/cli/types.py
View File

@@ -24,6 +24,6 @@ class CLISubcommand:
pass
def subparser_init(
self,
subparsers: argparse._SubParsersAction) -> FlexibleArgumentParser:
self, subparsers: argparse._SubParsersAction
) -> FlexibleArgumentParser:
raise NotImplementedError("Subclasses should implement this method")