Add AWQ support for all models (#1714)

2023-11-18 17:56:47 -08:00
parent e946260cf3
commit 8d17774f92
13 changed files with 90 additions and 17 deletions
--- a/vllm/model_executor/models/gpt_bigcode.py
+++ b/vllm/model_executor/models/gpt_bigcode.py
@@ -137,7 +137,9 @@ class GPTBigMLP(nn.Module):
            bias=True,
            linear_method=linear_method,
        )
-        self.act = get_act_fn(config.activation_function)
+        quant_config = getattr(linear_method, "quant_config", None)
+        self.act = get_act_fn(config.activation_function, quant_config,
+                              intermediate_size)

    def forward(self, hidden_states: torch.Tensor) -> torch.Tensor:
        hidden_states, _ = self.c_fc(hidden_states)