[Core] Support multi-node inference(eager and cuda graph) (#3686)

vllm/test_utils.py

@@ -8,6 +8,7 @@ from vllm.worker.worker import init_distributed_environment
 def init_test_distributed_environment(
     pipeline_parallel_size: int,
     tensor_parallel_size: int,
+    local_rank: int,
     rank: int,
     distributed_init_port: str,
 ) -> None:
@@ -16,7 +17,10 @@ def init_test_distributed_environment(
                                      worker_use_ray=True)
     distributed_init_method = f"tcp://localhost:{distributed_init_port}"
     init_distributed_environment(
-        parallel_config, rank, distributed_init_method=distributed_init_method)
+        parallel_config,
+        local_rank,
+        rank,
+        distributed_init_method=distributed_init_method)
 
 
 def multi_process_tensor_parallel(