[Frontend] Delegate preprocessing to OpenAIServingRender (#36483)
Signed-off-by: Sage Ahrac <sagiahrak@gmail.com>
This commit is contained in:
@@ -508,11 +508,25 @@ async def test_header_dp_rank_argument():
|
||||
base_model_paths=BASE_MODEL_PATHS,
|
||||
)
|
||||
|
||||
# Create render serving instance (required by OpenAIServingChat)
|
||||
from vllm.entrypoints.serve.render.serving import OpenAIServingRender
|
||||
|
||||
serving_render = OpenAIServingRender(
|
||||
model_config=engine.model_config,
|
||||
renderer=engine.renderer,
|
||||
io_processor=engine.io_processor,
|
||||
model_registry=models.registry,
|
||||
request_logger=None,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
)
|
||||
|
||||
# Create serving chat instance
|
||||
serving_chat = OpenAIServingChat(
|
||||
engine_client=engine,
|
||||
models=models,
|
||||
response_role="assistant",
|
||||
openai_serving_render=serving_render,
|
||||
chat_template=None,
|
||||
chat_template_content_format="auto",
|
||||
request_logger=None,
|
||||
|
||||
Reference in New Issue
Block a user