From 1b44aaf4e3559e4e321f32715b08f1aa7e4f3d50 Mon Sep 17 00:00:00 2001 From: youkaichao Date: Mon, 17 Jun 2024 14:35:04 -0700 Subject: [PATCH] [bugfix][distributed] fix 16 gpus local rank arrangement (#5604) --- vllm/executor/ray_gpu_executor.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/vllm/executor/ray_gpu_executor.py b/vllm/executor/ray_gpu_executor.py index 89d1c4ac7..843332e5e 100644 --- a/vllm/executor/ray_gpu_executor.py +++ b/vllm/executor/ray_gpu_executor.py @@ -137,6 +137,12 @@ class RayGPUExecutor(DistributedGPUExecutor): for i, (node_id, gpu_ids) in enumerate(worker_node_and_gpu_ids): node_workers[node_id].append(i) + # `gpu_ids` can be a list of strings or integers. + # convert them to integers for consistency. + # NOTE: gpu_ids can be larger than 9 (e.g. 16 GPUs), + # string sorting is not sufficient. + # see https://github.com/vllm-project/vllm/issues/5590 + gpu_ids = [int(x) for x in gpu_ids] node_gpus[node_id].extend(gpu_ids) for node_id, gpu_ids in node_gpus.items(): node_gpus[node_id] = sorted(gpu_ids)