From af6583eb19a65f0717d8bf4793df12452c67b663 Mon Sep 17 00:00:00 2001 From: biondizzle Date: Fri, 15 May 2026 00:51:50 +0000 Subject: [PATCH] =?UTF-8?q?fix:=20unpack=20uint8=20NVFP4=E2=86=92bf16=20fo?= =?UTF-8?q?r=20non-stacked=20params=20(weights=5Fproj)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit indexer.weights_proj is uint8 [64,3584] in checkpoint but bf16 [64,7168] in model. The uint8→bf16 unpack logic only ran in the stacked_params loop, so non-stacked NVFP4 params hit a size mismatch assertion. --- vllm/patches/deepseek_v4.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/vllm/patches/deepseek_v4.py b/vllm/patches/deepseek_v4.py index 94c183db..9b2b9ce1 100644 --- a/vllm/patches/deepseek_v4.py +++ b/vllm/patches/deepseek_v4.py @@ -1534,6 +1534,27 @@ class DeepseekV4Model(nn.Module): loaded_params.add(is_name) continue + # Handle uint8 NVFP4 packed → bf16 unpack for non-stacked + # params (e.g. indexer.weights_proj). Checkpoint stores + # NVFP4 as uint8 (2 values/byte), but model param is bf16. + if (loaded_weight.dtype == torch.uint8 + and param.data.dtype != torch.uint8 + and loaded_weight.shape[-1] * 2 == param.data.shape[-1]): + FP4_LUT = torch.tensor([ + 0.0, 0.5, 1.0, 1.5, 2.0, 3.0, 4.0, 6.0, + -0.0, -0.5, -1.0, -1.5, -2.0, -3.0, -4.0, -6.0, + ], dtype=torch.float32, device=loaded_weight.device) + lower = FP4_LUT[(loaded_weight & 0x0F).long()] + upper = FP4_LUT[((loaded_weight >> 4) & 0x0F).long()] + out = torch.empty( + *loaded_weight.shape[:-1], + loaded_weight.shape[-1] * 2, + dtype=torch.float32, device=loaded_weight.device, + ) + out[..., 0::2] = lower + out[..., 1::2] = upper + loaded_weight = out.to(torch.bfloat16) + weight_loader = getattr( param, "weight_loader", default_weight_loader )