[Core] Support multi-node inference(eager and cuda graph) (#3686)

2024-03-29 06:01:55 +08:00
parent a4075cba4d
commit 515386ef3c
7 changed files with 25 additions and 22 deletions
--- a/vllm/executor/ray_gpu_executor.py
+++ b/vllm/executor/ray_gpu_executor.py
@@ -188,8 +188,6 @@ class RayGPUExecutor(ExecutorBase):
            is_driver_worker=True,
        )

-        # FIXME(woosuk): We are not properly initializing pynccl when
-        # we have multiple nodes.
        self._run_workers("init_device")
        self._run_workers(
            "load_model",