[Misc] Qwen2.5 VL support LoRA (#13261)

2025-02-20 10:37:55 +08:00
parent 473f51cfd9
commit 512368e34a
4 changed files with 129 additions and 62 deletions
--- a/vllm/model_executor/models/qwen2_5_vl.py
+++ b/vllm/model_executor/models/qwen2_5_vl.py
@@ -734,16 +734,17 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module, SupportsMultiModal,
            "up_proj",
        ],
    }
-    # LoRA specific attributes, TODO: double check
+    # LoRA specific attributes
    supported_lora_modules = [
+        # language model
        "qkv_proj",
        "o_proj",
        "gate_up_proj",
-        "down_proj",
-        "gate_proj"
-        "up_proj",
+        "down_proj",  # Same name with vision encoder
        # vision tower
        "qkv",
+        "gate_proj",
+        "up_proj",
        "attn.proj",  # Distinguish patch_embed.proj
        "fc1",
        "fc2",
@@ -751,6 +752,7 @@ class Qwen2_5_VLForConditionalGeneration(nn.Module, SupportsMultiModal,
        "mlp.0",
        "mlp.2"
    ]
+
    embedding_modules = {}
    embedding_padding_modules = []