[Frontend] Kill the server on engine death (#6594)

Signed-off-by: Joe Runde <joe@joerun.de> Signed-off-by: Joe Runde <Joseph.Runde@ibm.com>
2024-08-08 10:47:48 -06:00
parent 5fb4a3f678
commit 21b9c49aa3
8 changed files with 136 additions and 14 deletions
--- a/vllm/entrypoints/openai/api_server.py
+++ b/vllm/entrypoints/openai/api_server.py
@@ -357,6 +357,7 @@ async def run_server(args, **uvicorn_kwargs) -> None:

        shutdown_task = await serve_http(
            app,
+            engine=async_engine_client,
            host=args.host,
            port=args.port,
            log_level=args.uvicorn_log_level,