From b0b5113467bc96ee3f9a6f4f6ceff2fabd48d441 Mon Sep 17 00:00:00 2001 From: biondizzle Date: Tue, 19 May 2026 03:20:41 +0000 Subject: [PATCH] =?UTF-8?q?Fix=20weight=20mapper:=20compressor=20=E2=86=92?= =?UTF-8?q?=20attn.compressor=20(not=20mla=5Fattn),=20quant=20weights=5Fpr?= =?UTF-8?q?oj?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - The compressor is on attn.compressor (not attn.mla_attn.compressor) - weights_proj in indexer is NVFP4-quantized in our checkpoint --- vllm/patches/deepseek_v4.py | 2 +- vllm/patches/deepseek_v4_attention.py | 4 +++- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/vllm/patches/deepseek_v4.py b/vllm/patches/deepseek_v4.py index c5f93db7..6c99cdef 100644 --- a/vllm/patches/deepseek_v4.py +++ b/vllm/patches/deepseek_v4.py @@ -1707,7 +1707,7 @@ def _make_deepseek_v4_nvfp4_weights_mapper() -> WeightsMapper: "compressor.kv_norm.": "compressor.norm.", "compressor.position_bias": "compressor.ape", # Attention compressor (after indexer renames) - ".self_attn.compressor.": ".attn.mla_attn.compressor.", + ".self_attn.compressor.": ".attn.compressor.", # Attention projections (specific before .self_attn. → .attn.) ".self_attn.q_a_proj.": ".attn.wq_a.", ".self_attn.kv_proj.": ".attn.wkv.", diff --git a/vllm/patches/deepseek_v4_attention.py b/vllm/patches/deepseek_v4_attention.py index 7ed9d59f..db461b45 100644 --- a/vllm/patches/deepseek_v4_attention.py +++ b/vllm/patches/deepseek_v4_attention.py @@ -1143,11 +1143,13 @@ class DeepseekV4Indexer(nn.Module): quant_config=quant_config, prefix=f"{prefix}.wq_b", ) + # weights_proj is NVFP4-quantized in the modelopt checkpoint. + # Upstream uses quant_config=None for the MXFP4 checkpoint. self.weights_proj = ReplicatedLinear( hidden_size, self.n_head, bias=False, - quant_config=None, + quant_config=quant_config, prefix=f"{prefix}.weights_proj", ) self.softmax_scale = self.head_dim**-0.5