[Frontend] Add --logprobs argument to benchmark_serving.py (#8191)

2024-09-06 12:01:14 -04:00
parent baa5467547
commit e5cab71531
3 changed files with 19 additions and 1 deletions
--- a/tests/multi_step/test_correctness_llm.py
+++ b/tests/multi_step/test_correctness_llm.py
@@ -57,7 +57,7 @@ def test_multi_step_llm(
                           GPU -> CPU output transfer
      num_prompts: number of example prompts under test
      num_logprobs: corresponds to the `logprobs` argument to the OpenAI
-                    completions endpoint; `None` -> no logprobs
+                    completions endpoint; `None` -> 1 logprob returned.
    """

    prompts = example_prompts