"""
D1: Verify per-row LSE correctness.
"""
import torch, math
import cutlass.cute as cute
import cutlass.torch as ct
import cuda.bindings.driver as cuda
from dsv4.kernels.attention.fmha import FmhaKernel


def test_lse(hd=64):
    m = 128
    s_k = 128
    torch.manual_seed(42)

    q = torch.randn(m, hd, 1, dtype=torch.bfloat16, device='cuda')
    k = torch.randn(s_k, hd, 1, dtype=torch.bfloat16, device='cuda')
    v = torch.randn(s_k, hd, dtype=torch.bfloat16, device='cuda')

    # FP32 reference LSE
    qf = q[:, :, 0].float()
    kf = k[:, :, 0].float()
    scale = 1.0 / math.sqrt(hd)
    attn = qf @ kf.T * scale
    attn_max = attn.max(dim=-1)[0]  # (m,)
    attn_exp = torch.exp(attn - attn_max.unsqueeze(-1))
    attn_sum = attn_exp.sum(dim=-1)  # (m,)
    ref_lse = torch.log(attn_sum) + attn_max  # (m,) natural log domain

    # Our kernel LSE: lse = ln(row_sum) + row_max * ln(2)
    # row_max is in scale_log2 domain: max(S * scale * log2(e))
    # So row_max * ln(2) converts back to natural domain.
    # Thus lse = ln(row_sum) + row_max * ln(2)
    # But row_max = max(S * scale * log2(e)) = max(S * scale) * log2(e)
    # So row_max * ln(2) = max(S * scale) * log2(e) * ln(2) = max(S * scale) 
    # Therefore lse = ln(row_sum) + max(S * scale)
    # And our ref: lse = ln(sum(exp(S * scale - max))) + max = ln(sum) + max
    # These should be the same.

    kernel = FmhaKernel(head_dim=hd, s_k=s_k, use_smem_p=False, normalize=False)
    pv_n_tile = kernel.pv_n_tile

    stream = cuda.CUstream(torch.cuda.current_stream().cuda_stream)

    v_tile = v[:, 0:pv_n_tile].contiguous()
    v_kernel = v_tile.unsqueeze(-1)
    c_tile = torch.zeros(m, pv_n_tile, 1, dtype=torch.bfloat16, device='cuda')
    lse_tensor = torch.zeros(m, dtype=torch.float32, device='cuda')

    mQ = ct.from_dlpack(q).mark_layout_dynamic(leading_dim=ct.get_leading_dim(q))
    mK = ct.from_dlpack(k).mark_layout_dynamic(leading_dim=ct.get_leading_dim(k))
    mV = ct.from_dlpack(v_kernel).mark_layout_dynamic(leading_dim=ct.get_leading_dim(v_kernel))
    mC = ct.from_dlpack(c_tile).mark_layout_dynamic(leading_dim=ct.get_leading_dim(c_tile))
    mLSE = ct.from_dlpack(lse_tensor).mark_layout_dynamic(leading_dim=ct.get_leading_dim(lse_tensor))

    print(f'  Compiling...', flush=True)
    compiled = cute.compile(kernel, mQ, mK, mV, mC, stream, mLSE)
    compiled(mQ, mK, mV, mC, stream, mLSE)
    torch.cuda.synchronize()

    kernel_lse = lse_tensor.cpu()
    ref_lse_cpu = ref_lse.cpu()

    # Compare
    lse_err = (kernel_lse - ref_lse_cpu).abs()
    print(f'  LSE max err: {lse_err.max().item():.6f}')
    print(f'  LSE mean err: {lse_err.mean().item():.6f}')
    print(f'  Kernel LSE[:8]: {kernel_lse[:8].tolist()}')
    print(f'  Ref LSE[:8]:    {ref_lse_cpu[:8].tolist()}')

    # Check O too
    ref_unnorm = attn_exp @ v.float()
    out = c_tile[:, :, 0].float()
    cos = torch.nn.functional.cosine_similarity(
        out.flatten().unsqueeze(0), ref_unnorm.flatten().unsqueeze(0)
    ).item()
    print(f'  O cos_unnorm: {cos:.6f}')


if __name__ == '__main__':
    test_lse()