biondizzle/deepseek-v4-quant
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Lower GPU max_memory to 100GiB, add CPU-only fallback for low_memory_mode

Browse Source
This commit is contained in:
biondizzle
2026-05-07 02:49:24 +00:00
parent 717151b98c
commit b8bdd00d19
1 changed files with 4 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
quantize_modelopt.py
View File

@@ -102,8 +102,11 @@ def main():
model_kwargs["device_map"] = "auto"
model_kwargs["offload_folder"] = "offload"
model_kwargs["offload_state_dict"] = True
model_kwargs["max_memory"] = {i: "160GiB" for i in range(8)}
model_kwargs["max_memory"] = {i: "100GiB" for i in range(8)}
model_kwargs["max_memory"]["cpu"] = "2500GiB"
elif args.low_memory_mode:
# Load entirely on CPU, modelopt will handle placement
model_kwargs["device_map"] = {"": "cpu"}
model = AutoModelForCausalLM.from_pretrained(args.model, **model_kwargs)