FMHA SM100: Add SMEM descriptor construction for tcgen05.mma

This commit is contained in:
2026-05-28 05:08:25 +00:00
parent 3eb432d064
commit 41e59a2423

View File

@@ -93,6 +93,58 @@ constexpr int TOTAL_SMEM(int hd) {
return (sQ + sK + sV + sC) * sizeof(__nv_bfloat16);
}
// =====================================================================
// SMEM Descriptor construction for tcgen05.mma
// =====================================================================
// The SMEM descriptor is a 64-bit value encoding the SMEM address,
// layout, and dimensions for the MMA operation.
// See cute/arch/mma_sm100_desc.hpp for the bitfield format.
enum class SmemSwizzle : uint8_t {
NONE = 0,
SWIZZLE_128B_BASE32B = 1,
SWIZZLE_128B = 2,
SWIZZLE_64B = 4,
};
/**
* Construct a UMMA SMEM descriptor for BF16 K-major layout.
*
* For Q (A-matrix): K-major, shape (TILE_M, HEAD_DIM), ld = HEAD_DIM * sizeof(bf16)
* For K (B-matrix): K-major, shape (SMEM_TILE_K, HEAD_DIM), ld = HEAD_DIM * sizeof(bf16)
*
* @param smem_ptr SMEM pointer (must be 16B-aligned)
* @param ld_bytes Leading dimension in bytes (row stride)
* @param swizzle Swizzle mode
*/
__device__ __forceinline__ uint64_t make_umma_smem_desc_bf16(
const void* smem_ptr,
uint32_t ld_bytes,
SmemSwizzle swizzle = SmemSwizzle::SWIZZLE_128B
) {
uint64_t desc = 0;
// start_address: bits [0,14), shifted left by 4 (16B-aligned)
uint32_t addr = static_cast<uint32_t>(__cvta_generic_to_shared(smem_ptr));
uint16_t start_addr = static_cast<uint16_t>(addr >> 4);
// leading_byte_offset: bits [16,30), shifted left by 4
uint16_t leading_offset = static_cast<uint16_t>(ld_bytes >> 4);
// stride_byte_offset: bits [32,46), shifted left by 4
// For 2D matrices, stride = ld * num_rows (for TMA, this is the 2nd dimension stride)
// But for MMA SMEM descriptors, stride is the offset between tiles (0 for single-tile)
uint16_t stride_offset = 0;
// Pack into 64-bit descriptor
desc = static_cast<uint64_t>(start_addr) |
(static_cast<uint64_t>(leading_offset) << 16) |
(static_cast<uint64_t>(stride_offset) << 32) |
(static_cast<uint64_t>(static_cast<uint8_t>(swizzle)) << 61);
return desc;
}
// =====================================================================
// Device helpers
// =====================================================================