[misc] add torch.compile compatibility check (#10618)

Signed-off-by: youkaichao <youkaichao@gmail.com>
This commit is contained in:
youkaichao
2024-11-24 23:40:08 -08:00
committed by GitHub
parent 65813781a2
commit 25d806e953
3 changed files with 22 additions and 1 deletions

View File

@@ -197,6 +197,13 @@ class EngineArgs:
if not self.tokenizer:
self.tokenizer = self.model
# support `EngineArgs(compilation_config={...})`
# without having to manually construct a
# CompilationConfig object
if isinstance(self.compilation_config, (int, dict)):
self.compilation_config = CompilationConfig.from_cli(
json.dumps(self.compilation_config))
# Setup plugins
from vllm.plugins import load_general_plugins
load_general_plugins()