Fix an endless loop issue when engine_step throws a RuntimeError (#339)

2023-07-04 06:22:28 +08:00
parent 42e0c1df78
commit 7717d0838b
1 changed files with 5 additions and 1 deletions
--- a/vllm/engine/async_llm_engine.py
+++ b/vllm/engine/async_llm_engine.py
@@ -144,7 +144,11 @@ class AsyncLLMEngine:

            # Kick the engine if the engine is not running.
            if not self.is_engine_running:
-                await self.engine_step(request_id)
+                try:
+                    await self.engine_step(request_id)
+                except RuntimeError as e:
+                    await self.abort(request_id)
+                    raise e

            # Wait for new output. The group_event will be set in engine_step
            # when there is new output available for the sequence group.