Add AWQ support for all models (#1714)

2023-11-18 17:56:47 -08:00
parent e946260cf3
commit 8d17774f92
13 changed files with 90 additions and 17 deletions
--- a/vllm/model_executor/models/mpt.py
+++ b/vllm/model_executor/models/mpt.py
@@ -130,7 +130,8 @@ class MPTMLP(nn.Module):
            bias=not config.no_bias,
            linear_method=linear_method,
        )
-        self.act = get_act_fn("gelu")
+        quant_config = getattr(linear_method, "quant_config", None)
+        self.act = get_act_fn("gelu", quant_config, intermediate_size)
        self.down_proj = RowParallelLinear(
            intermediate_size,
            hidden_size,