biondizzle/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[BugFix][TritonMLA] Process weights after model loading for GGUF (#14555)

Browse Source 
Signed-off-by: TianyuanWu <Tianyuan.Wu@amd.com>
This commit is contained in:
TY-AMD
2025-03-13 11:14:36 +08:00
committed by GitHub
parent a94a699c3f
commit 128bf75283
1 changed files with 4 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
vllm/model_executor/model_loader/loader.py
View File

@@ -1330,11 +1330,14 @@ class GGUFModelLoader(BaseModelLoader):
local_model_path, gguf_weights_map): local_model_path, gguf_weights_map):
model_config.hf_config.update({"tie_word_embeddings": True}) model_config.hf_config.update({"tie_word_embeddings": True})
target_device = torch.device(device_config.device)
with set_default_torch_dtype(model_config.dtype): with set_default_torch_dtype(model_config.dtype):
with torch.device(device_config.device): with target_device:
model = _initialize_model(vllm_config=vllm_config) model = _initialize_model(vllm_config=vllm_config)
model.load_weights( model.load_weights(
self._get_weights_iterator(local_model_path, gguf_weights_map)) self._get_weights_iterator(local_model_path, gguf_weights_map))
_process_weights_after_loading(model, model_config, target_device)
return model return model