biondizzle/deepseek-v4-quant
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

nvfp4_full: drop calib to 128, gpu_max_mem to 0.7 for VRAM headroom

Browse Source
This commit is contained in:
biondizzle
2026-05-08 06:24:45 +00:00
parent d0fc5338fe
commit 3d38e1d5cd
1 changed files with 2 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

4
scripts/model_opt_nvfp4_full.py
View File

@@ -32,13 +32,13 @@ import os
MODEL = "/root/nvidia-meeting/DeepSeek-V4-Pro-BF16" # Dequantized BF16 (from scripts/dequant_fp8_to_bf16.py)
QUANT = "nvfp4"
TP = 8
CALIB = 256
CALIB = 128
KV_CACHE_QUANT = "fp8_cast"
# 3TB BF16 model can't fit on 8×B200 VRAM (~1.4TB total)
# Use seq_device_map: loads model into CPU RAM, moves layers to GPU for forward passes
# 2.8TB RAM is enough for the 3TB model (with memory-mapped loading)
EXTRA_FLAGS = "--trust_remote_code --use_seq_device_map --gpu_max_mem_percentage 0.9"
EXTRA_FLAGS = "--trust_remote_code --use_seq_device_map --gpu_max_mem_percentage 0.7"
# Output dir follows modelopt convention: <model>_<quant>_kv_<kv_quant>
# We override the model name to make the strategy clear