biondizzle/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[Misc] Modify BNB parameter name (#9997)

Browse Source 
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
Jee Jee Li
2024-11-06 03:40:08 +08:00
committed by GitHub
parent d2e80332a7
commit b9c64c0ca7
3 changed files with 11 additions and 14 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
vllm/model_executor/model_loader/loader.py
View File

@@ -892,7 +892,7 @@ class BitsAndBytesModelLoader(BaseModelLoader):
if not weight_name.lower().endswith(".scb"):
continue
weight_key = weight_name.lower().replace(".scb", ".qweight")
weight_key = weight_name.lower().replace(".scb", ".weight")
quant_state_dict[weight_key] = weight_tensor
for weight_name, weight_tensor in self._hf_weight_iter(
@@ -901,11 +901,9 @@ class BitsAndBytesModelLoader(BaseModelLoader):
if self._is_8bit_weight_name(weight_name):
continue
qweight_name = weight_name.replace(".weight", ".qweight")
if qweight_name in quant_state_dict:
if weight_name in quant_state_dict:
set_weight_attrs(weight_tensor, {"load_in_8bit": True})
yield qweight_name, weight_tensor
yield weight_name, weight_tensor
else:
yield weight_name, weight_tensor
@@ -950,9 +948,8 @@ class BitsAndBytesModelLoader(BaseModelLoader):
(f"{weight_name}.quant_state.bitsandbytes__fp4" \
in temp_state_dict):
quant_state = _parse_quant_state(weight_name, temp_state_dict)
weight_name = weight_name.replace(".weight", ".qweight")
quant_state_dict[weight_name] = quant_state
yield weight_name.replace(".weight", ".qweight"), weight_tensor
yield weight_name, weight_tensor
else:
yield weight_name, weight_tensor
@@ -967,7 +964,6 @@ class BitsAndBytesModelLoader(BaseModelLoader):
if any(target_module in weight_name for target_module in
self.target_modules) and weight_name.endswith(".weight"):
weight_name = weight_name.replace(".weight", ".qweight")
# Without sharding
if any(
weight_name.startswith(module)
@@ -1093,7 +1089,7 @@ class BitsAndBytesModelLoader(BaseModelLoader):
# Some models, such as MiniCPM V2.5/2.6, contain both
# module names 'kv_proj' and 'qkv_proj'. To prevent 'kv_proj'
# from being incorrectly identified as being present in
# 'vpm.encoder.layers.0.self_attn.qkv_proj.qweight
# 'vpm.encoder.layers.0.self_attn.qkv_proj.weight
if shard_pos > 0 and quant_param_name[shard_pos - 1] == ".":
shard_index = index
quant_param_name = quant_param_name.replace(