biondizzle/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Update modelopt KV cache quantization resolution to new scheme (#31895)

Browse Source 
Signed-off-by: Roi Koren <roik@nvidia.com>
This commit is contained in:
roikoren755
2026-01-10 06:54:13 +02:00
committed by GitHub
parent 583a90e005
commit 0c9614876e
1 changed files with 21 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

23
vllm/utils/torch_utils.py
View File

@@ -219,9 +219,28 @@ def get_kv_cache_quant_algo_string(quant_cfg: dict[str, Any]) -> str | None:
if quant_method.startswith("modelopt"):
quantization_inner = quant_cfg.get("quantization", quant_cfg)
# Check if quant config is specified and use kv cache quant algo
kv_algo = quantization_inner.get("kv_cache_quant_algo") or quant_cfg.get(
"kv_cache_quant_algo"
kv_algo = (
quantization_inner.get("kv_cache_scheme")
or quant_cfg.get("kv_cache_scheme")
or quantization_inner.get("kv_cache_quant_algo")
or quant_cfg.get("kv_cache_quant_algo")
)
if isinstance(kv_algo, dict):
if (
kv_algo.get("dynamic") is False
and kv_algo.get("num_bits") == 8
and kv_algo.get("type") == "float"
):
kv_algo = "fp8"
else:
# Unknown/unsupported format - return "auto" as safe fallback
logger.warning(
"WARNING: Unknown kv_cache_quant_algo '%s' in model "
"config. Supported values: %s. Falling back to 'auto'.",
f"{kv_algo}",
list(MODELOPT_TO_VLLM_KV_CACHE_DTYPE_MAP.keys()),
)
return "auto"
if isinstance(kv_algo, str):
kv_algo_lower = kv_algo.lower()