From 96fc09503a2dddde52cc0513bff9c08ebe664cbd Mon Sep 17 00:00:00 2001 From: Hanjie Qiu <50634613+hjjq@users.noreply.github.com> Date: Mon, 2 Mar 2026 18:57:38 -0500 Subject: [PATCH] [All Reduce] Change default backend of Flashinfer All Reduce to trtllm (#35793) Signed-off-by: hjjq --- vllm/envs.py | 7 +++++-- 1 file changed, 5 insertions(+), 2 deletions(-) diff --git a/vllm/envs.py b/vllm/envs.py index 864ea6649..cfbf56ee1 100755 --- a/vllm/envs.py +++ b/vllm/envs.py @@ -168,7 +168,7 @@ if TYPE_CHECKING: VLLM_FLASHINFER_MOE_BACKEND: Literal["throughput", "latency", "masked_gemm"] = ( "latency" ) - VLLM_FLASHINFER_ALLREDUCE_BACKEND: Literal["auto", "trtllm", "mnnvl"] = "auto" + VLLM_FLASHINFER_ALLREDUCE_BACKEND: Literal["auto", "trtllm", "mnnvl"] = "trtllm" VLLM_FLASHINFER_WORKSPACE_BUFFER_SIZE: int = 394 * 1024 * 1024 VLLM_XGRAMMAR_CACHE_MB: int = 0 VLLM_MSGPACK_ZERO_COPY_THRESHOLD: int = 256 @@ -1297,9 +1297,12 @@ environment_variables: dict[str, Callable[[], Any]] = { # Flashinfer fused allreduce backend. # "auto" will default to "mnnvl", which performs mostly same/better than "trtllm". # But "mnnvl" backend does not support fuse with quantization. + # TODO: Default is "trtllm" right now because "mnnvl" has issues with cudagraph: + # https://github.com/vllm-project/vllm/issues/35772 + # Should switch back to "auto" if the issue is resolved. "VLLM_FLASHINFER_ALLREDUCE_BACKEND": env_with_choices( "VLLM_FLASHINFER_ALLREDUCE_BACKEND", - "auto", + "trtllm", ["auto", "trtllm", "mnnvl"], ), # Control the workspace buffer size for the FlashInfer backend.