Enable CUDA graph for GPTQ & SqueezeLLM (#2318)

2024-01-03 09:52:29 -08:00
parent 9140561059
commit 6ef00b03a2
3 changed files with 15 additions and 13 deletions
--- a/vllm/config.py
+++ b/vllm/config.py
@@ -181,12 +181,6 @@ class ModelConfig:
            self.max_context_len_to_capture = self.max_model_len
        self.max_context_len_to_capture = min(self.max_context_len_to_capture,
                                              self.max_model_len)
-        if (self.quantization in ["gptq", "squeezellm"]
-                and not self.enforce_eager):
-            # Related issue: https://github.com/vllm-project/vllm/issues/2147
-            logger.warning(f"{self.quantization} does not support CUDA graph "
-                           "yet. Disabling CUDA graph.")
-            self.enforce_eager = True

    def verify_with_parallel_config(
        self,