[ Misc ] fbgemm checkpoints (#6559)

2024-07-20 12:36:57 -04:00
parent 9042d68362
commit 683e3cb9c4
24 changed files with 234 additions and 47 deletions
--- a/vllm/model_executor/layers/quantization/bitsandbytes.py
+++ b/vllm/model_executor/layers/quantization/bitsandbytes.py
@@ -60,9 +60,8 @@ class BitsAndBytesConfig(QuantizationConfig):
            target_modules = cls.get_from_keys(config, ["target_modules"])
        return cls(adapter_name, target_modules)

-    def get_quant_method(
-            self,
-            layer: torch.nn.Module) -> Optional["BitsAndBytesLinearMethod"]:
+    def get_quant_method(self, layer: torch.nn.Module,
+                         prefix: str) -> Optional["BitsAndBytesLinearMethod"]:
        if isinstance(layer, LinearBase):
            return BitsAndBytesLinearMethod(self)
        return None