Fix NaN check: use os.environ gate instead of is_current_stream_capturing

torch.cuda.is_current_stream_capturing() returns bool, which breaks
Dynamo FX tracing (non-Tensor output). Switch to env var gate:
CLAWMINE_NAN_CHECK=1 enables NaN/Inf detection.

Dynamo evaluates os.environ at trace time — if the env var is not set,
the entire NaN check block is compiled away. Set it before first
inference to get NaN detection during prefill only.
This commit is contained in:
2026-05-18 02:20:14 +00:00
parent 0c02d84514
commit b8df4a8cc5

View File

@@ -1,6 +1,7 @@
# SPDX-License-Identifier: Apache-2.0
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
import os
import typing
from collections.abc import Callable, Iterable
from itertools import islice
@@ -1315,18 +1316,21 @@ class DeepseekV4Model(nn.Module):
positions,
input_ids,
)
# NaN detection (prefill only — no CPU-GPU syncs during cudagraph)
if not torch.cuda.is_current_stream_capturing():
if layer_idx % 10 == 0:
print(f"[CLAWMINE] Layer {layer_idx}: amax={hidden_states.amax().item():.4f} mean={hidden_states.mean().item():.6f}")
if torch.isnan(hidden_states).any():
nan_pct = torch.isnan(hidden_states).float().mean().item() * 100
print(f"[CLAWMINE] NaN after layer {layer_idx}! {nan_pct:.2f}% NaN, amax={hidden_states.amax().item():.4f}")
break
if torch.isinf(hidden_states).any():
inf_pct = torch.isinf(hidden_states).float().mean().item() * 100
print(f"[CLAWMINE] Inf after layer {layer_idx}! {inf_pct:.2f}% Inf, amax={hidden_states.amax().item():.4f}")
break
# NaN detection — only during prefill. Disabled via env var during cudagraph.
# os.environ is evaluated at trace time by Dynamo, so the entire
# NaN check block is skipped during compilation.
if os.environ.get('CLAWMINE_NAN_CHECK', '0') == '1':
with torch.no_grad():
if torch.isnan(hidden_states).any():
nan_pct = torch.isnan(hidden_states).float().mean().item() * 100
print(f"[CLAWMINE] NaN after layer {layer_idx}! {nan_pct:.2f}% NaN, amax={hidden_states.amax().item():.4f}")
break
if torch.isinf(hidden_states).any():
inf_pct = torch.isinf(hidden_states).float().mean().item() * 100
print(f"[CLAWMINE] Inf after layer {layer_idx}! {inf_pct:.2f}% Inf, amax={hidden_states.amax().item():.4f}")
break
if layer_idx % 10 == 0:
print(f"[CLAWMINE] Layer {layer_idx}: amax={hidden_states.amax().item():.4f} mean={hidden_states.mean().item():.6f}")
# Stash pre-hc_head residual for the MTP draft (captured copy_).
num_tokens = hidden_states.shape[0]