[Core] Add fault tolerance for RayTokenizerGroupPool (#5748)

2024-06-25 10:15:10 -07:00
parent 7b99314301
commit 67882dbb44
5 changed files with 194 additions and 23 deletions
--- a/vllm/engine/async_llm_engine.py
+++ b/vllm/engine/async_llm_engine.py
@@ -310,6 +310,8 @@ class _AsyncLLMEngine(LLMEngine):
        )

    async def check_health_async(self) -> None:
+        if self.tokenizer:
+            self.tokenizer.check_health()
        self.model_executor.check_health()