From 0d74b97fb214db4829f17f359b27cf615a9b62e5 Mon Sep 17 00:00:00 2001
From: biondizzle <biondizzle@gmail.com>
Date: Sun, 10 May 2026 08:23:11 +0000
Subject: [PATCH] Config patches doc + compress_ratios runtime patch in serve
 script

---
 README.md             | 12 ++++++++++++
 scripts/serve_vllm.py | 41 +++++++++++++++++++++++++++++++++++++++++
 2 files changed, 53 insertions(+)

diff --git a/README.md b/README.md
index 15a460d..76eb34e 100644
--- a/README.md
+++ b/README.md
@@ -14,6 +14,18 @@ Full NVFP4 quantization of DeepSeek V4 Pro on a single B200 node (8× B200, 2.7T
 - **Calibrated state:** 721.4GB (insurance, can re-export with `--export-only`)
 - A few experts (11, 83, 100, 112, 254) had uncalibrated amax — weight-derived fallback used (normal for sparse MoE with 256 experts)
 
+## ⚠️ Model Config Patches (post-export)
+
+modelopt 0.45.0.dev64's export doesn't fully match what vllm 0.20.2 expects. These changes were made to `DeepSeek-V4-Pro-NVFP4/config.json` and `hf_quant_config.json` after export:
+
+| Field | modelopt export | vllm expects | Fix |
+|-------|----------------|-------------|-----|
+| `compress_ratios` | Missing (transformers 5.8.0 uses `compress_rates` dict) | List of 61 ints, indexed by layer_id | Copied from BF16 source model's config.json |
+| `quantization_config.scale_fmt` | Missing | `"ue8m0"` string | Added to config.json |
+| `hf_quant_config.scale_fmt` | Missing | `"ue8m0"` string | Added to hf_quant_config.json |
+
+The `compress_rates` dict (`{'compressed_sparse_attention': 4, 'heavily_compressed_attention': 128}`) is the new transformers 5.8.0 format. vllm still expects the old per-layer list. The serve script (`serve_vllm.py`) also monkey-patches `DeepseekV4Config.__init__` to auto-convert when loading.
+
 ## Architecture
 
 We call modelopt's `hf_ptq.main()` directly — the same entry point the shell script uses. We don't rewrite the pipeline. We just:
diff --git a/scripts/serve_vllm.py b/scripts/serve_vllm.py
index c7e3cdd..bf84426 100644
--- a/scripts/serve_vllm.py
+++ b/scripts/serve_vllm.py
@@ -14,6 +14,47 @@ Or in the background:
 import subprocess
 import sys
 
+# ── Patch: Add compress_ratios to DeepseekV4Config ──────────────────────────
+# transformers 5.8.0 renamed compress_ratios → compress_rates (dict format).
+# vllm 0.20.2 still expects compress_ratios as a list indexed by layer_id.
+# We patch the Config class to expose compress_ratios as a property that
+# converts the new dict format back to the list format vllm expects.
+import transformers
+try:
+    from transformers import DeepseekV4Config
+
+    _orig_init = DeepseekV4Config.__init__
+
+    def _patched_init(self, *args, **kwargs):
+        _orig_init(self, *args, **kwargs)
+        # If compress_ratios already exists as a list, leave it alone
+        if hasattr(self, 'compress_ratios') and isinstance(self.compress_ratios, list):
+            return
+        # Convert compress_rates dict → compress_ratios list
+        if hasattr(self, 'compress_rates') and isinstance(self.compress_rates, dict):
+            rates = self.compress_rates
+            # Build per-layer list from the dict schema
+            # V4 pattern: layers 0-1=128, then alternating 4/128, last=0
+            n_layers = getattr(self, 'num_hidden_layers', 61)
+            cr = rates.get('compressed_sparse_attention', 4)
+            hr = rates.get('heavily_compressed_attention', 128)
+            ratios = []
+            for i in range(n_layers):
+                if i < 2:
+                    ratios.append(hr)
+                elif i == n_layers - 1:
+                    ratios.append(0)
+                else:
+                    ratios.append(cr if i % 2 == 0 else hr)
+            self.compress_ratios = ratios
+        elif hasattr(self, 'compress_rates') and isinstance(self.compress_rates, list):
+            self.compress_ratios = self.compress_rates
+
+    DeepseekV4Config.__init__ = _patched_init
+    print("✓ Patched DeepseekV4Config.__init__ to add compress_ratios")
+except ImportError:
+    print("⚠ DeepseekV4Config not found, skipping compress_ratios patch")
+
 MODEL = "/root/nvidia-meeting/DeepSeek-V4-Pro-NVFP4"
 
 # These flags are critical for V4 — do not change without understanding why: