Add distributed model executor abstraction (#3191)

2024-03-11 11:03:45 -07:00
parent 657061fdce
commit 4c922709b6
13 changed files with 817 additions and 508 deletions
--- a/docs/source/dev/engine/llm_engine.rst
+++ b/docs/source/dev/engine/llm_engine.rst
@@ -2,5 +2,5 @@ LLMEngine
 =================================

 .. autoclass:: vllm.engine.llm_engine.LLMEngine
-    :members: add_request, abort_request, step, _init_cache
+    :members: add_request, abort_request, step
    :show-inheritance: