biondizzle/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[Misc] Include matched stop string/token in responses (#2976)

Browse Source 
Co-authored-by: Sahil Suneja <sahilsuneja@gmail.com>
This commit is contained in:
Nick Hill
2024-03-25 17:31:32 -07:00
committed by GitHub
parent 3a243095e5
commit dfeb2ecc3a
7 changed files with 97 additions and 7 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
vllm/entrypoints/openai/serving_completion.py
View File

@@ -266,6 +266,7 @@ class OpenAIServingCompletion(OpenAIServing):
previous_texts[i] = output.text
previous_num_tokens[i] = len(output.token_ids)
finish_reason = output.finish_reason
stop_reason = output.stop_reason
if output.finish_reason is not None: # return final usage
prompt_tokens = len(res.prompt_token_ids)
completion_tokens = len(output.token_ids)
@@ -286,6 +287,7 @@ class OpenAIServingCompletion(OpenAIServing):
text=delta_text,
logprobs=logprobs,
finish_reason=finish_reason,
stop_reason=stop_reason,
)
],
usage=final_usage,
@@ -342,6 +344,7 @@ class OpenAIServingCompletion(OpenAIServing):
text=output_text,
logprobs=logprobs,
finish_reason=output.finish_reason,
stop_reason=output.stop_reason,
)
choices.append(choice_data)