Fix MoE non-fused L2 runtime gsa + update test harness for extra args

This commit is contained in:
2026-06-01 15:03:54 +00:00
parent a48d6e14ae
commit 038fe81c68

View File

@@ -651,7 +651,11 @@ class Nvfp4MoE:
gate_silu = gate_silu.clamp(max=self._swiglu_limit)
up = up.clamp(min=-self._swiglu_limit, max=self._swiglu_limit)
activated = gate_silu * up
# Compute runtime gsa for L2 from activated output (non-fused path)
if not self._fused_swiglu and getattr(self, '_use_runtime_gsa', False):
amax_l2 = activated.float().abs().max().clamp(min=1e-8).item()
self._l2_activation_global_scale = amax_l2 / (6.0 * 448.0)
# === L2: down ===
# Quantize activated (per-token) using GPU-only kernel, scatter into padded FP4 buffer.
# For fused_swiglu path, slot_l2_x_fp4/sf already set by deinterleave_quantize_nvfp4_cuda.