biondizzle/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[misc][distributed] add VLLM_SKIP_P2P_CHECK flag (#8911)

Browse Source
This commit is contained in:
youkaichao
2024-09-27 14:27:56 -07:00
committed by GitHub
parent c2ec430ab5
commit 18e60d7d13
2 changed files with 12 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
vllm/distributed/device_communicators/custom_all_reduce.py
View File

@@ -28,6 +28,10 @@ def _can_p2p(rank: int, world_size: int) -> bool:
for i in range(world_size): for i in range(world_size):
if i == rank: if i == rank:
continue continue
if envs.VLLM_SKIP_P2P_CHECK:
logger.info(
"Skipping P2P check and trusting the driver's P2P report.")
return torch.cuda.can_device_access_peer(rank, i)
if not gpu_p2p_access_check(rank, i): if not gpu_p2p_access_check(rank, i):
return False return False
return True return True

8
vllm/envs.py
View File

@@ -63,6 +63,7 @@ if TYPE_CHECKING:
VLLM_USE_TRITON_AWQ: bool = False VLLM_USE_TRITON_AWQ: bool = False
VLLM_ALLOW_RUNTIME_LORA_UPDATING: bool = False VLLM_ALLOW_RUNTIME_LORA_UPDATING: bool = False
VLLM_ALLOW_DEPRECATED_BEAM_SEARCH: bool = False VLLM_ALLOW_DEPRECATED_BEAM_SEARCH: bool = False
VLLM_SKIP_P2P_CHECK: bool = False
def get_default_cache_root(): def get_default_cache_root():
@@ -423,6 +424,13 @@ environment_variables: Dict[str, Callable[[], Any]] = {
lambda: lambda:
(os.environ.get("VLLM_ALLOW_RUNTIME_LORA_UPDATING", "0").strip().lower() in (os.environ.get("VLLM_ALLOW_RUNTIME_LORA_UPDATING", "0").strip().lower() in
("1", "true")), ("1", "true")),
# By default, vLLM will check the peer-to-peer capability itself,
# in case of broken drivers. See https://github.com/vllm-project/vllm/blob/a9b15c606fea67a072416ea0ea115261a2756058/vllm/distributed/device_communicators/custom_all_reduce_utils.py#L101-L108 for details. # noqa
# If this env var is set to 1, vLLM will skip the peer-to-peer check,
# and trust the driver's peer-to-peer capability report.
"VLLM_SKIP_P2P_CHECK":
lambda: os.getenv("VLLM_SKIP_P2P_CHECK", "0") == "1",
} }
# end-env-vars-definition # end-env-vars-definition