biondizzle/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[Core] Support multiple tasks per model (#20771)

Browse Source 
Signed-off-by: NickLucche <nlucches@redhat.com>
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Nicolò Lucchesi
2025-07-13 04:40:11 +02:00
committed by GitHub
parent c1acd6d7d4
commit 020f58abcd
8 changed files with 279 additions and 148 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
vllm/entrypoints/openai/run_batch.py
View File

@@ -348,7 +348,7 @@ async def main(args):
chat_template=None,
chat_template_content_format="auto",
enable_prompt_tokens_details=args.enable_prompt_tokens_details,
) if model_config.runner_type == "generate" else None
) if "generate" in model_config.supported_tasks else None
openai_serving_embedding = OpenAIServingEmbedding(
engine,
model_config,
@@ -356,17 +356,19 @@ async def main(args):
request_logger=request_logger,
chat_template=None,
chat_template_content_format="auto",
) if model_config.task == "embed" else None
) if "embed" in model_config.supported_tasks else None
enable_serving_reranking = (model_config.task == "classify" and getattr(
model_config.hf_config, "num_labels", 0) == 1)
enable_serving_reranking = ("classify" in model_config.supported_tasks
and getattr(model_config.hf_config,
"num_labels", 0) == 1)
openai_serving_scores = (ServingScores(
openai_serving_scores = ServingScores(
engine,
model_config,
openai_serving_models,
request_logger=request_logger,
) if (model_config.task == "embed" or enable_serving_reranking) else None)
) if ("embed" in model_config.supported_tasks
or enable_serving_reranking) else None
tracker = BatchProgressTracker()
logger.info("Reading batch from %s...", args.input_file)