diff --git a/dsv4/kernels/attention/fmha.py b/dsv4/kernels/attention/fmha.py index a3aa5eb5..952bd54d 100644 --- a/dsv4/kernels/attention/fmha.py +++ b/dsv4/kernels/attention/fmha.py @@ -273,10 +273,8 @@ class FmhaKernel: thr_smem_copy = tiled_smem_copy.get_slice(sfw_idx) # Debug sP shape print(f"[SMEM-P PROPER] sP shape: {cute.shape(sP)} rank: {len(cute.shape(sP))}") - # Try grouping ALL modes (0-4) to flatten - sP_flat = cute.group_modes(sP, 0, 4) - print(f"[SMEM-P PROPER] sP_flat shape after group_modes(0,4): {cute.shape(sP_flat)}") - tSMEM_CPYsP = thr_smem_copy.partition_D(sP_flat) # destination (SMEM) + # Don't flatten - keep original sP + tSMEM_CPYsP = thr_smem_copy.partition_D(sP) # destination (SMEM) print(f"[SMEM-P PROPER] After partition_D: tSMEM_CPYsP layout: {tSMEM_CPYsP.layout}") print(f"[SMEM-P PROPER] After partition_D: tSMEM_CPYsP shape: {cute.shape(tSMEM_CPYsP)} rank: {len(cute.shape(tSMEM_CPYsP))}") @@ -359,10 +357,7 @@ class FmhaKernel: # Partition source with QK layout print(f"[SMEM-P PROPER] Before partition_S: rP_bf16_qk shape: {cute.shape(rP_bf16_qk)} layout: {rP_bf16_qk.layout}") - # Try flattening source too (rP_bf16_qk has 3 modes: group_modes(0,3)) - rP_bf16_qk_flat = cute.group_modes(rP_bf16_qk, 0, 3) - print(f"[SMEM-P PROPER] rP_bf16_qk_flat shape after group_modes(0,3): {cute.shape(rP_bf16_qk_flat)}") - tSMEM_CPYrP_qk = thr_smem_copy.partition_S(rP_bf16_qk_flat) + tSMEM_CPYrP_qk = thr_smem_copy.partition_S(rP_bf16_qk) print(f"[SMEM-P PROPER] After partition_S: tSMEM_CPYrP_qk layout: {tSMEM_CPYrP_qk.layout}") print(f"[SMEM-P PROPER] After partition_S: tSMEM_CPYrP_qk shape: {cute.shape(tSMEM_CPYrP_qk)} rank: {len(cute.shape(tSMEM_CPYrP_qk))}") @@ -372,12 +367,20 @@ class FmhaKernel: print(f"[SMEM-P PROPER] tSMEM_CPYrP_qk shape: {cute.shape(tSMEM_CPYrP_qk)} rank: {len(cute.shape(tSMEM_CPYrP_qk))}") print(f"[SMEM-P PROPER] tSMEM_CPYsP shape: {cute.shape(tSMEM_CPYsP)} rank: {len(cute.shape(tSMEM_CPYsP))}") - # Attempt copy with correct layout - try: - cute.copy(tiled_smem_copy, tSMEM_CPYrP_qk, tSMEM_CPYsP) - print(f"[SMEM-P PROPER] Copy succeeded with QK C-fragment layout") - except Exception as e: - print(f"[SMEM-P PROPER] Copy failed: {e}") + # Manual copy instead of cute.copy (helpers are a trap) + # Get sizes and iterate + src_size = cute.size(tSMEM_CPYrP_qk) + dst_size = cute.size(tSMEM_CPYsP) + print(f"[SMEM-P PROPER] Manual copy: src_size={src_size}, dst_size={dst_size}") + + if src_size == dst_size: + # Same number of elements - copy element by element + for j in cutlass.range(src_size, vectorize=True): + val = tSMEM_CPYrP_qk[j] + tSMEM_CPYsP[j] = val + print(f"[SMEM-P PROPER] Manual copy completed (src_size==dst_size)") + else: + print(f"[SMEM-P PROPER] Manual copy: size mismatch, using fallback stub") # Fallback to stub for now for j in cutlass.range(cute.size(sP), vectorize=True): sP[j] = BFloat16(0.0)