Non-streaming simple fastapi server (#144)

2023-06-11 01:43:07 +08:00
parent 4298374265
commit 5020e1e80c
3 changed files with 61 additions and 20 deletions
--- a/cacheflow/entrypoints/openai/openai_frontend.py
+++ b/cacheflow/entrypoints/openai/openai_frontend.py
@@ -233,7 +233,7 @@ async def create_completion(raw_request: Request):
    async for res in result_generator:
        if await raw_request.is_disconnected():
            # Abort the request if the client disconnects.
-            await server.abort(request_id)
+            await abort_request()
            return create_error_response(HTTPStatus.BAD_REQUEST,
                                         "Client disconnected")
        final_res = res