biondizzle/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[Bugfix] Fix using -O[0,3] with LLM entrypoint (#10677)

Browse Source 
Signed-off-by: mgoin <michael@neuralmagic.com>
This commit is contained in:
Michael Goin
2024-11-26 13:44:01 -05:00
committed by GitHub
parent f5792c7c4a
commit 9a99273b48
2 changed files with 12 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

5
vllm/engine/arg_utils.py
View File

@@ -206,7 +206,10 @@ class EngineArgs:
# support `EngineArgs(compilation_config={...})` # support `EngineArgs(compilation_config={...})`
# without having to manually construct a # without having to manually construct a
# CompilationConfig object # CompilationConfig object
if isinstance(self.compilation_config, (int, dict)): if isinstance(self.compilation_config, (int)):
self.compilation_config = CompilationConfig.from_cli(
str(self.compilation_config))
elif isinstance(self.compilation_config, (dict)):
self.compilation_config = CompilationConfig.from_cli( self.compilation_config = CompilationConfig.from_cli(
json.dumps(self.compilation_config)) json.dumps(self.compilation_config))

10
vllm/entrypoints/llm.py
View File

@@ -185,8 +185,14 @@ class LLM:
kwargs["disable_log_stats"] = True kwargs["disable_log_stats"] = True
if compilation_config is not None: if compilation_config is not None:
compilation_config_instance = CompilationConfig.from_cli( if isinstance(compilation_config, (int)):
json.dumps(compilation_config)) compilation_config_instance = CompilationConfig.from_cli(
str(compilation_config))
elif isinstance(compilation_config, (dict)):
compilation_config_instance = CompilationConfig.from_cli(
json.dumps(compilation_config))
else:
compilation_config_instance = compilation_config
else: else:
compilation_config_instance = None compilation_config_instance = None