[Bugfix] Fix Phi3.5 mini and MoE LoRA inference (#8571)

2024-09-19 17:54:02 -07:00
parent de6f90a13d
commit 18ae428a0d
3 changed files with 22 additions and 1 deletions
--- a/vllm/model_executor/models/phimoe.py
+++ b/vllm/model_executor/models/phimoe.py
@@ -491,6 +491,10 @@ class PhiMoEForCausalLM(nn.Module, SupportsLoRA):
        "o_proj",
        "embed_tokens",
        "lm_head",
+        "w1",
+        "w2",
+        "w3",
+        "gate",
    ]
    embedding_modules = {
        "embed_tokens": "input_embeddings",