[ Misc ] non-uniform quantization via compressed-tensors for Llama (#6515)

2024-07-18 22:39:18 -04:00
parent d4201e06d5
commit dbe5588554
11 changed files with 301 additions and 91 deletions
--- a/vllm/model_executor/layers/fused_moe/layer.py
+++ b/vllm/model_executor/layers/fused_moe/layer.py
@@ -158,6 +158,7 @@ class FusedMoE(torch.nn.Module):
        topk_group: Optional[int] = None,
        quant_config: Optional[QuantizationConfig] = None,
        tp_size: Optional[int] = None,
+        prefix: str = "",
    ):
        super().__init__()