From 6d17988b51b72a37dc87ec228f436c80689afae5 Mon Sep 17 00:00:00 2001 From: biondizzle Date: Sat, 16 May 2026 04:04:40 +0000 Subject: [PATCH] =?UTF-8?q?fix:=20L1=20gate/up=20split=20=E2=80=94=20inter?= =?UTF-8?q?mediate=5Fsize=20is=20per-projection,=20not=20fused?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit intermediate_size=3072 is the size of gate OR up, not gate+up. Split L1 output at intermediate_size, not intermediate_size//2. gate = l1_out[:, :3072], up = l1_out[:, 3072:] --- vllm/nvfp4_cutedsl.py | 5 ++--- 1 file changed, 2 insertions(+), 3 deletions(-) diff --git a/vllm/nvfp4_cutedsl.py b/vllm/nvfp4_cutedsl.py index f2ce0299..5b29c07f 100644 --- a/vllm/nvfp4_cutedsl.py +++ b/vllm/nvfp4_cutedsl.py @@ -165,9 +165,8 @@ class CuTeDSLMoERunner: # ════════════════════════════════════════════════════════════ # SiLU(gate) * up (BF16) # ════════════════════════════════════════════════════════════ - half = self.intermediate_size // 2 - gate = l1_out[:, :half] - up = l1_out[:, half:] + gate = l1_out[:, :self.intermediate_size] + up = l1_out[:, self.intermediate_size:] activated = torch.nn.functional.silu(gate) * up # ════════════════════════════════════════════════════════════