Add GPTQ support (#916)

2023-12-15 19:04:22 +08:00
parent c06170cc8e
commit 0fbfc4b81b
35 changed files with 1782 additions and 82 deletions
--- a/vllm/engine/arg_utils.py
+++ b/vllm/engine/arg_utils.py
@@ -179,7 +179,7 @@ class EngineArgs:
        parser.add_argument('--quantization',
                            '-q',
                            type=str,
-                            choices=['awq', 'squeezellm', None],
+                            choices=['awq', 'gptq', 'squeezellm', None],
                            default=None,
                            help='Method used to quantize the weights')
        return parser