fix: _fold_global_scale — remove broken logical_widths branch
The logical_widths branch took expert 0 and 1's global scales and applied them to ALL experts. For L1 with logical_widths=[3072,3072], every expert got expert-0's scale on its gate half and expert-1's scale on its up half. All other experts' global scales were discarded. The else branch correctly broadcasts each expert's own (E,1) global scale across (E, N, K//16). Removed the dead logical_widths code.
This commit is contained in:
@@ -22,25 +22,19 @@ import torch
|
||||
|
||||
def _fold_global_scale(
|
||||
weight_scale: torch.Tensor, # (E, N, K//16) float8_e4m3fn
|
||||
weight_scale_2: torch.Tensor, # (E, num_logical) or (E, 1) or scalar float32
|
||||
logical_widths: list[int] = None,
|
||||
weight_scale_2: torch.Tensor, # (E, 1) or (E,) or scalar float32
|
||||
) -> torch.Tensor:
|
||||
"""Fold global scale into block scales: UE4M3 * FP32 → float32."""
|
||||
"""Fold global scale into block scales: UE4M3 * FP32 → float32.
|
||||
|
||||
Each expert has its own global scale. Broadcasts (E,1,1) → (E, N, K//16).
|
||||
"""
|
||||
sf_f32 = weight_scale.to(torch.float32)
|
||||
gs = weight_scale_2.to(torch.float32)
|
||||
|
||||
if gs.numel() == 1:
|
||||
sf_f32 = sf_f32 * gs
|
||||
elif gs.numel() > 1 and logical_widths is not None:
|
||||
expanded = []
|
||||
for i, w in enumerate(logical_widths):
|
||||
if i < len(gs.flatten()):
|
||||
expanded.append(gs.flatten()[i].expand(w))
|
||||
if expanded:
|
||||
global_scale = torch.cat(expanded).unsqueeze(1)
|
||||
sf_f32 = sf_f32 * global_scale
|
||||
else:
|
||||
# Per-expert or per-row global scale — broadcast multiply
|
||||
# Per-expert global scale — broadcast multiply
|
||||
while gs.dim() < sf_f32.dim():
|
||||
gs = gs.unsqueeze(-1)
|
||||
sf_f32 = sf_f32 * gs.expand_as(sf_f32)
|
||||
@@ -82,9 +76,12 @@ def transform_nvfp4_weights_for_mega_moe(
|
||||
l2_weight, l2_weight_scale = l2_tuple
|
||||
|
||||
# Fold global scales into block scales
|
||||
# Both L1 and L2 use per-expert global scales (shape (E,1) or (E,)).
|
||||
# The logical_widths branch was wrong: it treated gs as per-projection
|
||||
# scalars and only used experts 0 and 1's scales for ALL experts.
|
||||
# The else branch correctly broadcasts each expert's own global scale.
|
||||
if l1_weight_scale_2 is not None:
|
||||
l1_sf_folded = _fold_global_scale(l1_weight_scale, l1_weight_scale_2,
|
||||
logical_widths=[3072, 3072])
|
||||
l1_sf_folded = _fold_global_scale(l1_weight_scale, l1_weight_scale_2)
|
||||
else:
|
||||
l1_sf_folded = l1_weight_scale.to(torch.float32)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user