diff --git a/scripts/model_opt_nvfp4_full.py b/scripts/model_opt_nvfp4_full.py index 564183f..0dc622f 100644 --- a/scripts/model_opt_nvfp4_full.py +++ b/scripts/model_opt_nvfp4_full.py @@ -32,13 +32,13 @@ import os MODEL = "/root/nvidia-meeting/DeepSeek-V4-Pro-BF16" # Dequantized BF16 (from scripts/dequant_fp8_to_bf16.py) QUANT = "nvfp4" TP = 8 -CALIB = 256 +CALIB = 128 KV_CACHE_QUANT = "fp8_cast" # 3TB BF16 model can't fit on 8×B200 VRAM (~1.4TB total) # Use seq_device_map: loads model into CPU RAM, moves layers to GPU for forward passes # 2.8TB RAM is enough for the 3TB model (with memory-mapped loading) -EXTRA_FLAGS = "--trust_remote_code --use_seq_device_map --gpu_max_mem_percentage 0.9" +EXTRA_FLAGS = "--trust_remote_code --use_seq_device_map --gpu_max_mem_percentage 0.7" # Output dir follows modelopt convention: __kv_ # We override the model name to make the strategy clear