From 41ef0ebd0f2e914b78428023d634046f7366b0b8 Mon Sep 17 00:00:00 2001 From: biondizzle Date: Sun, 31 May 2026 19:17:24 +0000 Subject: [PATCH] Add KV cache length diagnostic during decode --- single_shot_inference.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/single_shot_inference.py b/single_shot_inference.py index 58a971b5..7b5bef25 100644 --- a/single_shot_inference.py +++ b/single_shot_inference.py @@ -64,7 +64,8 @@ NUM_GPUS = 8 SKIP_ROUTED_MOE = _args.skip_moe # If True, only use shared expert (debug) INVERSE_ROPE = not _args.no_inverse_rope # If False, skip inverse RoPE on attention output (diagnostic) SKIP_MHC = _args.skip_mhc # If True, bypass mHC and use simple residual connections (diagnostic) -MHC_DIAG = True # If True, print per-layer mHC diagnostics (B_l row/col sums, C_l values) +MHC_DIAG = False # If True, print per-layer mHC diagnostics (B_l row/col sums, C_l values) +GROWTH_DIAG = True # If True, print per-layer residual growth analysis # When True: applies inverse RoPE at query position → converts absolute→relative # When False: leaves relative position encoding intact for output projection # DSV4 partial RoPE only affects last 64/512 dims; first 448 are always un-RoPE'd @@ -1055,6 +1056,11 @@ def main(): decode_pos = len(all_tokens) - 1 positions = torch.tensor([decode_pos], dtype=torch.long, device='cuda:0') + # Diag: KV cache length + if step < 3 or step % 10 == 0: + k0, _ = kv_caches[0].get() + print(f" KV: {k0.shape[1]} entries, pos={decode_pos}", flush=True) + emb = embed(tid) # (1, H) on gpu0 X = mHCBlock.init_state(emb, n_hc) # (1, n_hc, H)