biondizzle/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[misc] add torch.compile compatibility check (#10618)

Browse Source 
Signed-off-by: youkaichao <youkaichao@gmail.com>
This commit is contained in:
youkaichao
2024-11-24 23:40:08 -08:00
committed by GitHub
parent 65813781a2
commit 25d806e953
3 changed files with 22 additions and 1 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
vllm/config.py
View File

@@ -2394,6 +2394,20 @@ class VllmConfig:
self.compilation_config.pass_config.enable_reshape = False
self.compilation_config.level = CompilationLevel.PIECEWISE
if self.cache_config is not None and \
self.cache_config.cpu_offload_gb > 0 and \
self.compilation_config.level != CompilationLevel.NO_COMPILATION:
logger.warning(
"CPU offload is not supported with `torch.compile` yet."
" Disabling `torch.compile`.")
self.compilation_config.level = CompilationLevel.NO_COMPILATION
if self.lora_config is not None and self.compilation_config.level !=\
CompilationLevel.NO_COMPILATION:
logger.warning("LoRA is not supported with `torch.compile` yet. "
"Disabling `torch.compile`.")
self.compilation_config.level = CompilationLevel.NO_COMPILATION
current_platform.check_and_update_config(self)
def __str__(self):