Add NaN debug checks after L1 and L2 GEMM

This commit is contained in:
2026-05-17 22:02:24 +00:00
parent e65f2b2ba2
commit d332f4f900

View File

@@ -452,6 +452,11 @@ class CuTeDSLMoERunner:
global_scale_a=l1_gsa, global_scale_b=self._l1_gsb,
)
# DEBUG: Check for NaN after L1 GEMM
if torch.isnan(l1_out).any():
nan_count = torch.isnan(l1_out).sum().item()
print(f"[CLAWMINE] NaN in L1 output! {nan_count}/{l1_out.numel()} values are NaN")
# Extract real token outputs from padded GEMM output
l1_out_real = l1_out[padded_dst]
@@ -488,6 +493,11 @@ class CuTeDSLMoERunner:
global_scale_a=l2_gsa, global_scale_b=self._l2_gsb,
)
# DEBUG: Check for NaN after L2 GEMM
if torch.isnan(l2_out).any():
nan_count = torch.isnan(l2_out).sum().item()
print(f"[CLAWMINE] NaN in L2 output! {nan_count}/{l2_out.numel()} values are NaN")
l2_out_real = l2_out[padded_dst]
# === Scatter -> final output ===