biondizzle/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[Misc] Remove qlora_adapter_name_or_path (#17699)

Browse Source 
Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
This commit is contained in:
Jee Jee Li
2025-05-07 14:10:37 +08:00
committed by GitHub
parent f80ae5bdcf
commit ba7703e659
3 changed files with 50 additions and 63 deletions
Download Patch File Download Diff File Expand all files Collapse all files

22
vllm/model_executor/model_loader/weight_utils.py
View File

@@ -162,23 +162,15 @@ def get_quant_config(model_config: ModelConfig,
None)
if hf_quant_config is not None:
return quant_cls.from_config(hf_quant_config)
# In case of bitsandbytes/QLoRA, get quant config from the adapter model.
# Inflight BNB quantization
if model_config.quantization == "bitsandbytes":
if (not load_config.model_loader_extra_config
or "qlora_adapter_name_or_path"
not in load_config.model_loader_extra_config):
return quant_cls.from_config({"adapter_name_or_path": ""})
model_name_or_path = load_config.model_loader_extra_config[
"qlora_adapter_name_or_path"]
else:
model_name_or_path = model_config.model
is_local = os.path.isdir(model_name_or_path)
return quant_cls.from_config({})
is_local = os.path.isdir(model_config.model)
if not is_local:
# Download the config files.
with get_lock(model_name_or_path, load_config.download_dir):
with get_lock(model_config.model, load_config.download_dir):
hf_folder = snapshot_download(
model_name_or_path,
model_config.model,
revision=model_config.revision,
allow_patterns="*.json",
cache_dir=load_config.download_dir,
@@ -186,7 +178,7 @@ def get_quant_config(model_config: ModelConfig,
tqdm_class=DisabledTqdm,
)
else:
hf_folder = model_name_or_path
hf_folder = model_config.model
possible_config_filenames = quant_cls.get_config_filenames()
@@ -213,7 +205,7 @@ def get_quant_config(model_config: ModelConfig,
config = json.load(f)
if model_config.quantization == "bitsandbytes":
config["adapter_name_or_path"] = model_name_or_path
config["adapter_name_or_path"] = model_config.model
elif model_config.quantization == "modelopt":
if config["producer"]["name"] == "modelopt":
return quant_cls.from_config(config)