diff --git a/vllm/cutedsl_quant_method.py b/vllm/cutedsl_quant_method.py index a858144e..6e032a0e 100644 --- a/vllm/cutedsl_quant_method.py +++ b/vllm/cutedsl_quant_method.py @@ -98,7 +98,11 @@ class CuTeDSLNvfp4Method(LinearMethodBase): layer._cutedsl_runner_id = register_runner(runner) layer._cutedsl_out_features = out_features - # Warmup: compute activation global scale from sample data + # Warmup: compute activation global scale from sample data. + # The checkpoint's input_scale is a calibration-time value that does NOT + # match what quantize_activation_nvfp4 expects at runtime. Using it + # produces garbage output (empty EOS tokens). The correct approach is + # a warmup forward pass that measures the actual activation distribution. with torch.no_grad(): sample = torch.randn(min(8, 256), in_features, dtype=torch.bfloat16, device=device) * 2.0 diff --git a/vllm/kernels/linear/nvfp4/cutedsl.py b/vllm/kernels/linear/nvfp4/cutedsl.py index 55f58b61..060a6f99 100644 --- a/vllm/kernels/linear/nvfp4/cutedsl.py +++ b/vllm/kernels/linear/nvfp4/cutedsl.py @@ -80,16 +80,17 @@ class CuTeDSLNvFp4LinearKernel(NvFp4LinearKernel): runner.gs = [gs] runner.finalize_weights() - activation_global_scale = 1.0 / 2688.0 - if hasattr(layer, 'input_global_scale_inv') and layer.input_global_scale_inv is not None: - inv = layer.input_global_scale_inv.data.item() - if inv != 0: - # input_global_scale_inv = 1.0 / input_global_scale - # input_global_scale = 1.0 / nvfp4_global_scale (the dequant scale) - # So input_global_scale_inv = nvfp4_global_scale = amax / (6.0 * 448.0) - # This is exactly what quantize_activation_nvfp4 expects. - activation_global_scale = inv - runner._activation_global_scale = activation_global_scale + # Warmup: compute activation global scale from sample data. + # The checkpoint's input_scale is a calibration-time value that does NOT + # match what quantize_activation_nvfp4 expects at runtime. Using it + # produces garbage output (empty EOS tokens). The correct approach is + # a warmup forward pass that measures the actual activation distribution. + with torch.no_grad(): + sample = torch.randn( + min(8, 256), in_features, + dtype=torch.bfloat16, device=str(device), + ) * 2.0 + runner.compute_activation_global_scale(sample) # Register the runner and store the ID (not the runner itself) layer._cutedsl_runner_id = register_runner(runner)