diff --git a/tests/unit/test_fmha_hd16.cu b/tests/unit/test_fmha_hd16.cu index 6a23a8b0..b08e21cd 100644 --- a/tests/unit/test_fmha_hd16.cu +++ b/tests/unit/test_fmha_hd16.cu @@ -1,8 +1,6 @@ /** - * Full UMMA FMHA Pipeline — HD=16, SK=128, T=1 (decode) - * - * Q×K^T → softmax → P×V → epilogue - * All on Blackwell SM100 using tcgen05 instructions. + * UMMA FMHA — QK + Softmax only, HD=16 + * Stripped down: no V, no PV. Just verify QK→softmax pipeline. */ #include @@ -19,95 +17,41 @@ using namespace dsv4::kernels::attention; static bf16_t f32_to_bf16_host(float f) { uint32_t u; memcpy(&u,&f,4); return (uint16_t)(u>>16); } static float bf16_to_f32_host(bf16_t h) { uint32_t u=(uint32_t)h<<16; float f; memcpy(&f,&u,4); return f; } -constexpr int HD = 16, SK = 128, BLOCK_MN = 128, CORES_MN = 16; +constexpr int HD = 16, SK = 128, BLOCK_MN = 128; __global__ void __launch_bounds__(128) -test_fmha_hd16(const bf16_t* q, const bf16_t* k, const bf16_t* v, - bf16_t* o_out, float* o_scalar, float scale) +test_fmha_softmax_hd16(const bf16_t* q, const bf16_t* k, + bf16_t* p_out, float* p_scalar, float scale) { const int tid = threadIdx.x, wid = tid / 32, lane = tid % 32; - // SMEM: tmem_base + sQ(128,16) + sK(128,16) + sV(128,16) + sQ_row(16 floats) extern __shared__ char sbuf[]; uint32_t* sTmemBase = (uint32_t*)sbuf; bf16_t* sQ = (bf16_t*)(((uintptr_t)(sbuf + 4) + 15) & ~(uintptr_t)15); - bf16_t* sK = sQ + 128 * 16 + 4096; // +padding (same as working QK test) - bf16_t* sV = sK + 128 * 16; - float* sQ_row = (float*)(sV + 128 * HD); // for scalar reference + bf16_t* sK = sQ + 128 * 16 + 4096; // Same padding as working QK test + float* sQ_row = (float*)(sK + 128 * 16); - // Load Q row for scalar for (int d = tid; d < HD; d += 128) sQ_row[d] = bf16_to_f32(q[d]); - // TMEM alloc — 128 columns (Layout D for 128×128) if (wid == 1) tmem_alloc(__cvta_generic_to_shared(sTmemBase), 128); __syncthreads(); uint32_t tb = *sTmemBase; - // Load Q, K, V into SMEM canonical layout write_q_to_smem(sQ, q); write_k_to_smem(sK, k); - // V for PV: B = (SK, HD) = (128, 16) in canonical. V[d, c] = v[d * SK + c]. - // B[r, d] = V[d, r] = v[d * SK + r]. - // But for tcgen05.mma TS, A is TMEM, B is SMEM. - // PV: A = P (128, K=128 TMEM), B = V (K=128, N=16 SMEM). - // Wait — for HD=16, the MMA for PV is: A = P (128, 128) × B = V (128, 16) - // → C = O (128, 16). - // But that's P × V where V has K=128 rows and N=16 cols. - // The B descriptor describes V as (BLOCK_MN, 16) where BLOCK_MN=128 (K dim). - // Hmm, BLOCK_MN for B should be the "MN" dimension which is 128 (the K dim of PV). - // Actually, the descriptor's BLOCK_MN is the "row" dimension of the matrix in SMEM. - // For B = (128, 16), BLOCK_MN = 128. The MMA's K dimension is 128. - // But the MMA processes K=16 at a time! So we need 128/16 = 8 PV K-tiles. - // Each PV K-tile: A = P[:, 16*kt:16*kt+16] from TMEM, B = V[16*kt:16*kt+16, :] from SMEM. - // B K-tile is (16, 16) in SMEM with BLOCK_MN = 16. - // - // For the initial test: load V as separate (16, 16) K-tiles, similar to Q/K. - // 8 V K-tiles, each (16, 16) = 16*16*2 = 512 bytes. Total = 4KB. - // But I already loaded V as (128, 16) above. Let me load V K-tiles separately. - - // Actually, for simplicity, let me load V as (128, 16) in canonical, - // then construct offset descriptors for each V K-tile. - // But we proved offset descriptors don't work for Q/K... - // Let me load 8 separate V K-tiles in (16, 16) canonical layout. - // For (16, 16): CORES_MN = 2, CORES_K = 2. - // Offset: ck * 2 * 64 + tmn * 64 + lr * 8 + lc - // where tmn = r/8 (0 or 1 for 16 rows), ck = c/8, lr = r%8, lc = c%8. - - // Zero V SMEM (8 K-tiles × 16×16 BF16) - bf16_t* sV0 = sV; - constexpr int VKT = SK / MMA_K_BF16; // 8 PV K-tiles - constexpr int V_TILE_SZ = MMA_K_BF16 * HD; // 16*16 = 256 BF16 per V K-tile - for (int i = tid; i < VKT * V_TILE_SZ; i += 128) sV0[i] = 0; - for (int kt = 0; kt < VKT; kt++) { - bf16_t* sv = sV0 + kt * V_TILE_SZ; - // V K-tile kt: rows [16*kt, 16*kt+16), all 16 cols - // B[r, d] = V[d, r] = v[d * SK + r] - // For r in [16*kt, 16*kt+16), d in [0, 16): - for (int i = tid; i < MMA_K_BF16 * HD; i += 128) { - int r = i / HD; // local row within this K-tile (0..15) - int d = i % HD; // column (head dim) - int global_r = kt * MMA_K_BF16 + r; - int ck = d / 8, lc = d % 8; - int tmn = r / 8, lr = r % 8; - sv[ck * 2 * 64 + tmn * 64 + lr * 8 + lc] = v[d * SK + global_r]; - } - } + bf16_t* sQ_pad = sQ + 128 * 16; + for (int i = tid; i < 4096; i += 128) sQ_pad[i] = 0; __syncthreads(); - // ================================================================ - // STEP 1: QK GEMM — Q (SMEM) × K (SMEM) → S (TMEM) - // ================================================================ + // QK GEMM uint64_t desc_q = make_umma_desc_kmajor_none(__cvta_generic_to_shared(sQ), BLOCK_MN); uint64_t desc_k = make_umma_desc_kmajor_none(__cvta_generic_to_shared(sK), BLOCK_MN); - uint32_t idesc_qk = make_idesc(BLOCK_MN, BLOCK_MN); - - if (lane == 0) umma_ss_f16(tb, desc_q, desc_k, idesc_qk, false); + uint32_t idesc = make_idesc(BLOCK_MN, BLOCK_MN); + if (lane == 0) umma_ss_f16(tb, desc_q, desc_k, idesc, false); asm volatile("tcgen05.fence::after_thread_sync;" ::: "memory"); __syncthreads(); - // ================================================================ - // STEP 2: Softmax — S (TMEM) → P (TMEM) - // ================================================================ + // Softmax if (wid == 0) { float s_vals[SK], row_max = -INFINITY; for (int n = 0; n < SK / 8; n++) { @@ -122,7 +66,7 @@ test_fmha_hd16(const bf16_t* q, const bf16_t* k, const bf16_t* v, row_sum = wsum(row_sum); if (lane == 0) for (int j=0;j>>(d_q, d_k, d_v, d_o, d_o_scalar, SCALE); + int smem = (4 + 16 + 128*16*2 + 4096*2 + 128*16*2 + 16*4 + 256 + 127) & ~127; + test_fmha_softmax_hd16<<<1, 128, smem>>>(d_q, d_k, d_p, d_p_scalar, SCALE); cudaError_t err = cudaDeviceSynchronize(); if (err != cudaSuccess) { printf("CUDA ERROR: %s\n", cudaGetErrorString(err)); return 1; } - cudaMemcpy(h_o, d_o, HD*sizeof(bf16_t), cudaMemcpyDeviceToHost); - cudaMemcpy(h_o_scalar, d_o_scalar, HD*sizeof(float), cudaMemcpyDeviceToHost); + cudaMemcpy(h_p, d_p, SK*sizeof(bf16_t), cudaMemcpyDeviceToHost); + cudaMemcpy(h_p_scalar, d_p_scalar, SK*sizeof(float), cudaMemcpyDeviceToHost); - printf("O[0..15] MMA: "); for(int d=0;d0 ? max_diff/max_val : max_diff; - printf("Max rel err: %.8f\n", rel_err); - printf("Test %s\n", rel_err < 0.01f ? "PASSED" : "FAILED"); + float p_sum = 0.0f; + for (int j=0;j