[v1] fix use compile sizes (#11000)

Signed-off-by: youkaichao <youkaichao@gmail.com>
2024-12-09 13:47:24 -08:00
parent cbcbdb1ceb
commit 1a2f8fb828
2 changed files with 4 additions and 0 deletions
--- a/vllm/config.py
+++ b/vllm/config.py
@@ -2522,6 +2522,7 @@ class VllmConfig:
            self.compilation_config.custom_ops = ["none"]
            self.compilation_config.use_cudagraph = True
            self.compilation_config.use_inductor = True
+            self.compilation_config.cudagraph_num_of_warmups = 1
            self.compilation_config.pass_config.enable_fusion = False
            self.compilation_config.pass_config.enable_reshape = False
            self.compilation_config.level = CompilationLevel.PIECEWISE