biondizzle
7a3b81e833
- scripts/upcast_to_bf16.py: Converts mixed-precision V4 Pro to pure BF16 by upcasting all FP8 tensors (float8_e8m0fnu etc.) to bfloat16. Needed because modelopt PTQ calibration crashes on Blackwell with FP8 kernels (DeepGEMM unsupported, Triton finegrained-fp8 has K mismatches). - patches/patch_finegrained_fp8_blackwell.py: Patches transformers to reject DeepGEMM on SM100+ (Blackwell), letting it fall back to Triton. Note: the Triton fallback also fails during modelopt calibration on quantized weights, so upcasting to BF16 is the working solution.
67 lines
2.4 KiB
Python
67 lines
2.4 KiB
Python
#!/usr/bin/env python3
|
|
"""
|
|
Patch transformers' finegrained_fp8.py to reject DeepGEMM on Blackwell (SM100+).
|
|
|
|
DeepGEMM only supports Hopper (SM90). On Blackwell GPUs, _load_deepgemm_kernel()
|
|
passes the SM90 check but then fails trying to download/load the kernel from HF Hub
|
|
(rate limits, missing builds). This patch adds a check for SM100+ that raises
|
|
ImportError, which the existing try/except in w8a8_fp8_matmul catches, falling
|
|
back to the Triton finegrained-fp8 kernel.
|
|
|
|
Also needed because the Triton finegrained-fp8 matmul has shape mismatches during
|
|
modelopt calibration (K mismatch on quantized expert weights). The real fix is to
|
|
upcast the model to BF16 first (see scripts/upcast_to_bf16.py).
|
|
|
|
Usage:
|
|
python3 patch_finegrained_fp8_blackwell.py [path_to_finegrained_fp8.py]
|
|
|
|
If no path given, auto-detects from the installed transformers package.
|
|
"""
|
|
|
|
import sys
|
|
import os
|
|
|
|
|
|
def patch(fp8_file: str):
|
|
with open(fp8_file) as f:
|
|
content = f.read()
|
|
|
|
old = """ # DeepGEMM requires Hopper (SM90) or newer for FP8 WGMMA instructions
|
|
major = torch.cuda.get_device_capability()[0]
|
|
if major < 9:
|
|
raise ImportError(
|
|
f"DeepGEMM requires a Hopper (SM90+) or newer GPU, but the current device "
|
|
f"has compute capability {major}.x. Use a different `experts_implementation`."
|
|
)"""
|
|
|
|
new = """ # DeepGEMM requires Hopper (SM90) specifically - not yet supported on Blackwell (SM100+)
|
|
major = torch.cuda.get_device_capability()[0]
|
|
if major < 9:
|
|
raise ImportError(
|
|
f"DeepGEMM requires a Hopper (SM90+) or newer GPU, but the current device "
|
|
f"has compute capability {major}.x. Use a different `experts_implementation`."
|
|
)
|
|
if major >= 10:
|
|
raise ImportError(
|
|
f"DeepGEMM is not yet supported on Blackwell (SM100+). "
|
|
f"Use a different `experts_implementation`."
|
|
)"""
|
|
|
|
if old in content:
|
|
content = content.replace(old, new)
|
|
with open(fp8_file, "w") as f:
|
|
f.write(content)
|
|
print(f"PATCHED: {fp8_file} — DeepGEMM now rejected on Blackwell (SM100+)")
|
|
else:
|
|
print("Patch target not found (may already be patched or different version)")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
if len(sys.argv) > 1:
|
|
fp8_file = sys.argv[1]
|
|
else:
|
|
import transformers.integrations.finegrained_fp8 as fp8
|
|
import inspect
|
|
fp8_file = inspect.getfile(fp8)
|
|
patch(fp8_file)
|