From acc20dffd74a2cba1a54abc90b238291a0f815c1 Mon Sep 17 00:00:00 2001 From: biondizzle Date: Sun, 31 May 2026 22:03:55 +0000 Subject: [PATCH] CRITICAL FIX: don't fold input_scale into NVFP4 weight dequant input_scale is the activation quantization scale (for FP8 inputs). Since we use BF16 activations, the weight dequant is simply: lut[weight] * weight_scale * weight_scale_2 Folding input_scale in produced weights ~4000x too small, causing all attention and FFN outputs to be effectively zero. --- single_shot_inference.py | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/single_shot_inference.py b/single_shot_inference.py index c85465a5..20e8a762 100644 --- a/single_shot_inference.py +++ b/single_shot_inference.py @@ -84,7 +84,9 @@ def dequant_nvfp4(weight, weight_scale, weight_scale_2=None, input_scale=None): w = torch.stack([lo_f, hi_f], -1).reshape(O, I) s = weight_scale.float().repeat_interleave(16, 1) if weight_scale_2 is not None: s = s * weight_scale_2.float() - if input_scale is not None: s = s * input_scale.float() + # NOTE: input_scale is intentionally NOT used. It's the activation + # quantization scale (for FP8 inputs). Since we use BF16 activations, + # the weight dequant is: lut[weight] * weight_scale * weight_scale_2. return (w * s).bfloat16() def nvfp4_linear(x, weight, weight_scale, weight_scale_2=None, input_scale=None):