[Bugfix]Fix Phi-3 BNB online quantization (#10417)

Signed-off-by: Jee Jee Li <pandaleefree@gmail.com>
2024-11-19 11:21:42 +08:00
parent 284203f171
commit 7eb719df13
2 changed files with 19 additions and 3 deletions
--- a/vllm/model_executor/models/phi3.py
+++ b/vllm/model_executor/models/phi3.py
@@ -14,3 +14,13 @@ class Phi3ForCausalLM(LlamaForCausalLM):
            "gate_up_proj",
        ],
    }
+
+    # BitandBytes specific attributes
+    default_bitsandbytes_target_modules = [
+        ".gate_up_proj.",
+        ".down_proj.",
+        ".qkv_proj.",
+        ".o_proj.",
+    ]
+    # Initialize an empty dict when there is no stacked parameter mapping.
+    bitsandbytes_stacked_params_mapping = {}