Add AWQ support for all models (#1714)

2023-11-18 17:56:47 -08:00
parent e946260cf3
commit 8d17774f92
13 changed files with 90 additions and 17 deletions
--- a/vllm/model_executor/layers/quantization/base_config.py
+++ b/vllm/model_executor/layers/quantization/base_config.py
@@ -54,3 +54,11 @@ class QuantizationConfig(ABC):
    def get_linear_method(self) -> LinearMethodBase:
        """Get the linear method to use for the quantized linear layer."""
        raise NotImplementedError
+
+    @abstractmethod
+    def get_scaled_act_names(self) -> List[str]:
+        """Returns the activation function names that should be post-scaled.
+
+        For now, this is only used by AWQ.
+        """
+        raise NotImplementedError