DEBUG: check intermediate magnitude before SE L2

2026-06-04 01:30:29 +00:00
parent 5303d6a82f
commit 26ecf96328
1 changed files with 2 additions and 0 deletions
--- a/dsv4/layers/shared_expert.py
+++ b/dsv4/layers/shared_expert.py
@@ -443,6 +443,8 @@ class Nvfp4SharedExpert:
                gate = gate.clamp(max=self.swiglu_limit)
                up = up.clamp(min=-self.swiglu_limit, max=self.swiglu_limit)
            intermediate = torch.nn.functional.silu(gate) * up
+        # DEBUG: check intermediate before L2
+        print(f"  SE intermediate: shape={tuple(intermediate.shape)} |max|={intermediate.abs().max().item():.6f} nan={torch.isnan(intermediate).any().item()}", flush=True)

        output = self._run_l2(intermediate)
        return output