[Bugfix] OpenAI entrypoint limits logprobs while ignoring server defined --max-logprobs (#5312)

Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>
2024-06-11 05:30:31 +03:00
parent a008629807
commit 351d5e7b82
4 changed files with 12 additions and 9 deletions
--- a/vllm/engine/arg_utils.py
+++ b/vllm/engine/arg_utils.py
@@ -48,7 +48,7 @@ class EngineArgs:
    gpu_memory_utilization: float = 0.90
    max_num_batched_tokens: Optional[int] = None
    max_num_seqs: int = 256
-    max_logprobs: int = 5  # OpenAI default value
+    max_logprobs: int = 20  # Default value for OpenAI Chat Completions API
    disable_log_stats: bool = False
    revision: Optional[str] = None
    code_revision: Optional[str] = None