Start background task in AsyncLLMEngine.generate (#988)

Co-authored-by: Zhuohan Li <zhuohan123@gmail.com>
2023-09-08 00:03:39 -07:00
parent 4b5bcf8906
commit 080438477f
4 changed files with 16 additions and 24 deletions
--- a/tests/async_engine/api_server_async_engine.py
+++ b/tests/async_engine/api_server_async_engine.py
@@ -40,8 +40,7 @@ if __name__ == "__main__":
    args = parser.parse_args()

    engine_args = AsyncEngineArgs.from_cli_args(args)
-    engine = AsyncLLMEngineWithStats.from_engine_args(engine_args,
-                                                      start_engine_loop=False)
+    engine = AsyncLLMEngineWithStats.from_engine_args(engine_args)
    vllm.entrypoints.api_server.engine = engine
    uvicorn.run(
        app,