Fix post-quant hook: register on inner model, fix module refs
vLLM V1 calls DeepseekV4Model.forward() directly, not DeepseekV4ForCausalLM.forward(). Hook on the outer model never fires. Moved hook to self.model (inner) and fixed module.model.layers → module.layers.
This commit is contained in:
@@ -2438,7 +2438,7 @@ class DeepseekV4ForCausalLM(nn.Module):
|
||||
|
||||
E2M1_LUT = torch.tensor([0, 0.5, 1, 1.5, 2, 3, 4, 6], dtype=torch.bfloat16)
|
||||
fixed = 0
|
||||
for layer_idx, layer in enumerate(module.model.layers):
|
||||
for layer_idx, layer in enumerate(module.layers):
|
||||
attn = layer.attn
|
||||
for proj_name in ["fused_wqa_wkv", "wq_b", "wo_b"]:
|
||||
if not hasattr(attn, proj_name):
|
||||
@@ -2447,7 +2447,7 @@ class DeepseekV4ForCausalLM(nn.Module):
|
||||
if not hasattr(mod, "weight") or mod.weight.dtype not in (torch.uint8, torch.int8):
|
||||
continue
|
||||
# Dequantize to BF16
|
||||
module.model._dequant_nvfp4_to_bf16(mod, E2M1_LUT)
|
||||
module._dequant_nvfp4_to_bf16(mod, E2M1_LUT)
|
||||
# Replace quant method (quant method already ran, won't overwrite again)
|
||||
mod.quant_method = UnquantizedLinearMethod()
|
||||
# Clean up NVFP4 attributes that might confuse forward
|
||||
@@ -2468,6 +2468,6 @@ class DeepseekV4ForCausalLM(nn.Module):
|
||||
del module._post_quant_fix_handle
|
||||
print(" [CLAWMINE] Post-quant-init fix done ✓", flush=True)
|
||||
|
||||
handle = self.register_forward_pre_hook(_fix_attn_nvfp4)
|
||||
handle = self.model.register_forward_pre_hook(_fix_attn_nvfp4)
|
||||
self._post_quant_fix_handle = handle
|
||||
|
||||
|
||||
Reference in New Issue
Block a user