diff --git a/tests/unit/test_fmha_hd16.cu b/tests/unit/test_fmha_hd16.cu new file mode 100644 index 00000000..4df5dd87 --- /dev/null +++ b/tests/unit/test_fmha_hd16.cu @@ -0,0 +1,263 @@ +/** + * Full UMMA FMHA Pipeline — HD=16, SK=128, T=1 (decode) + * + * Q×K^T → softmax → P×V → epilogue + * All on Blackwell SM100 using tcgen05 instructions. + */ + +#include +#include +#include +#include +#include + +#include "dsv4/kernels/attention/fmha_common.cuh" +#include "dsv4/kernels/attention/fmha_umma_desc.cuh" + +using namespace dsv4::kernels::attention; + +static bf16_t f32_to_bf16_host(float f) { uint32_t u; memcpy(&u,&f,4); return (uint16_t)(u>>16); } +static float bf16_to_f32_host(bf16_t h) { uint32_t u=(uint32_t)h<<16; float f; memcpy(&f,&u,4); return f; } + +constexpr int HD = 16, SK = 128, BLOCK_MN = 128, CORES_MN = 16; + +__global__ void __launch_bounds__(128) +test_fmha_hd16(const bf16_t* q, const bf16_t* k, const bf16_t* v, + bf16_t* o_out, float* o_scalar, float scale) +{ + const int tid = threadIdx.x, wid = tid / 32, lane = tid % 32; + + // SMEM: tmem_base + sQ(128,16) + sK(128,16) + sV(128,16) + sQ_row(16 floats) + extern __shared__ char sbuf[]; + uint32_t* sTmemBase = (uint32_t*)sbuf; + bf16_t* sQ = (bf16_t*)(((uintptr_t)(sbuf + 4) + 15) & ~(uintptr_t)15); + bf16_t* sK = sQ + 128 * 16 + 4096; // +padding (same as working QK test) + bf16_t* sV = sK + 128 * 16; + float* sQ_row = (float*)(sV + 128 * HD); // for scalar reference + + // Load Q row for scalar + for (int d = tid; d < HD; d += 128) sQ_row[d] = bf16_to_f32(q[d]); + + // TMEM alloc — 128 columns (Layout D for 128×128) + if (wid == 1) tmem_alloc(__cvta_generic_to_shared(sTmemBase), 128); + __syncthreads(); + uint32_t tb = *sTmemBase; + + // Load Q, K, V into SMEM canonical layout + write_q_to_smem(sQ, q); + write_k_to_smem(sK, k); + // V for PV: B = (SK, HD) = (128, 16) in canonical. V[d, c] = v[d * SK + c]. + // B[r, d] = V[d, r] = v[d * SK + r]. + // But for tcgen05.mma TS, A is TMEM, B is SMEM. + // PV: A = P (128, K=128 TMEM), B = V (K=128, N=16 SMEM). + // Wait — for HD=16, the MMA for PV is: A = P (128, 128) × B = V (128, 16) + // → C = O (128, 16). + // But that's P × V where V has K=128 rows and N=16 cols. + // The B descriptor describes V as (BLOCK_MN, 16) where BLOCK_MN=128 (K dim). + // Hmm, BLOCK_MN for B should be the "MN" dimension which is 128 (the K dim of PV). + // Actually, the descriptor's BLOCK_MN is the "row" dimension of the matrix in SMEM. + // For B = (128, 16), BLOCK_MN = 128. The MMA's K dimension is 128. + // But the MMA processes K=16 at a time! So we need 128/16 = 8 PV K-tiles. + // Each PV K-tile: A = P[:, 16*kt:16*kt+16] from TMEM, B = V[16*kt:16*kt+16, :] from SMEM. + // B K-tile is (16, 16) in SMEM with BLOCK_MN = 16. + // + // For the initial test: load V as separate (16, 16) K-tiles, similar to Q/K. + // 8 V K-tiles, each (16, 16) = 16*16*2 = 512 bytes. Total = 4KB. + // But I already loaded V as (128, 16) above. Let me load V K-tiles separately. + + // Actually, for simplicity, let me load V as (128, 16) in canonical, + // then construct offset descriptors for each V K-tile. + // But we proved offset descriptors don't work for Q/K... + // Let me load 8 separate V K-tiles in (16, 16) canonical layout. + // For (16, 16): CORES_MN = 2, CORES_K = 2. + // Offset: ck * 2 * 64 + tmn * 64 + lr * 8 + lc + // where tmn = r/8 (0 or 1 for 16 rows), ck = c/8, lr = r%8, lc = c%8. + + // Zero V SMEM (8 K-tiles × 16×16 BF16) + bf16_t* sV0 = sV; + constexpr int VKT = SK / MMA_K_BF16; // 8 PV K-tiles + constexpr int V_TILE_SZ = MMA_K_BF16 * HD; // 16*16 = 256 BF16 per V K-tile + for (int i = tid; i < VKT * V_TILE_SZ; i += 128) sV0[i] = 0; + for (int kt = 0; kt < VKT; kt++) { + bf16_t* sv = sV0 + kt * V_TILE_SZ; + // V K-tile kt: rows [16*kt, 16*kt+16), all 16 cols + // B[r, d] = V[d, r] = v[d * SK + r] + // For r in [16*kt, 16*kt+16), d in [0, 16): + for (int i = tid; i < MMA_K_BF16 * HD; i += 128) { + int r = i / HD; // local row within this K-tile (0..15) + int d = i % HD; // column (head dim) + int global_r = kt * MMA_K_BF16 + r; + int ck = d / 8, lc = d % 8; + int tmn = r / 8, lr = r % 8; + sv[ck * 2 * 64 + tmn * 64 + lr * 8 + lc] = v[d * SK + global_r]; + } + } + __syncthreads(); + + // ================================================================ + // STEP 1: QK GEMM — Q (SMEM) × K (SMEM) → S (TMEM) + // ================================================================ + uint64_t desc_q = make_umma_desc_kmajor_none(__cvta_generic_to_shared(sQ), BLOCK_MN); + uint64_t desc_k = make_umma_desc_kmajor_none(__cvta_generic_to_shared(sK), BLOCK_MN); + uint32_t idesc_qk = make_idesc(BLOCK_MN, BLOCK_MN); + + if (lane == 0) umma_ss_f16(tb, desc_q, desc_k, idesc_qk, false); + asm volatile("tcgen05.fence::after_thread_sync;" ::: "memory"); + __syncthreads(); + + // ================================================================ + // STEP 2: Softmax — S (TMEM) → P (TMEM) + // ================================================================ + if (wid == 0) { + float s_vals[SK], row_max = -INFINITY; + for (int n = 0; n < SK / 8; n++) { + float tmp[8]; + asm volatile("tcgen05.ld.sync.aligned.32x32b.x8.b32 {%0,%1,%2,%3,%4,%5,%6,%7},[%8];" : "=f"(tmp[0]),"=f"(tmp[1]),"=f"(tmp[2]),"=f"(tmp[3]),"=f"(tmp[4]),"=f"(tmp[5]),"=f"(tmp[6]),"=f"(tmp[7]) : "r"(tb + n*8)); + asm volatile("tcgen05.wait::ld.sync.aligned;"); + if (lane == 0) for (int c=0;c<8;c++) { s_vals[n*8+c] = tmp[c]*scale; row_max = fmaxf(row_max, tmp[c]*scale); } + } + row_max = wmax(row_max); + float row_sum = 0.0f; + if (lane == 0) for (int j=0;j 0); + asm volatile("tcgen05.fence::after_thread_sync;" ::: "memory"); + __syncthreads(); + } + asm volatile("tcgen05.fence::after_thread_sync;" ::: "memory"); + __syncthreads(); + + // ================================================================ + // STEP 4: Epilogue — O (TMEM) → normalize → BF16 → GMEM + // O is (128, 16) in TMEM, only row 0 has data. + // Read row 0 from TMEM, write to GMEM. + // ================================================================ + if (wid == 0) { + // O is in the first 16 TMEM columns (cols 0..15) + float o_vals[HD]; + for (int n = 0; n < HD / 8; n++) { // 2 iterations for HD=16 + float tmp[8]; + asm volatile("tcgen05.ld.sync.aligned.32x32b.x8.b32 {%0,%1,%2,%3,%4,%5,%6,%7},[%8];" : "=f"(tmp[0]),"=f"(tmp[1]),"=f"(tmp[2]),"=f"(tmp[3]),"=f"(tmp[4]),"=f"(tmp[5]),"=f"(tmp[6]),"=f"(tmp[7]) : "r"(tb + n*8)); + asm volatile("tcgen05.wait::ld.sync.aligned;"); + if (lane == 0) for (int c=0;c<8;c++) o_vals[n*8+c] = tmp[c]; + } + if (lane == 0) for (int d=0;d>>(d_q, d_k, d_v, d_o, d_o_scalar, SCALE); + + cudaError_t err = cudaDeviceSynchronize(); + if (err != cudaSuccess) { printf("CUDA ERROR: %s\n", cudaGetErrorString(err)); return 1; } + + cudaMemcpy(h_o, d_o, HD*sizeof(bf16_t), cudaMemcpyDeviceToHost); + cudaMemcpy(h_o_scalar, d_o_scalar, HD*sizeof(float), cudaMemcpyDeviceToHost); + + printf("O[0..15] MMA: "); for(int d=0;d0 ? max_diff/max_val : max_diff; + printf("Max rel err: %.8f\n", rel_err); + printf("Test %s\n", rel_err < 0.01f ? "PASSED" : "FAILED"); + + cudaFree(d_q); cudaFree(d_k); cudaFree(d_v); cudaFree(d_o); cudaFree(d_o_scalar); + free(h_q); free(h_k); free(h_v); free(h_o); free(h_o_scalar); + return rel_err < 0.01f ? 0 : 1; +}