[Frontend] Added support for HF's new continue_final_message parameter (#8942)

2024-09-29 20:59:47 +03:00
parent 1fb9c1b0bf
commit 6c9ba48fde
7 changed files with 102 additions and 28 deletions
--- a/vllm/entrypoints/openai/serving_tokenization.py
+++ b/vllm/entrypoints/openai/serving_tokenization.py
@@ -87,6 +87,7 @@ class OpenAIServingTokenization(OpenAIServing):
                    messages=request.messages,
                    chat_template=self.chat_template,
                    add_generation_prompt=request.add_generation_prompt,
+                    continue_final_message=request.continue_final_message,
                )
            else:
                prompt = apply_hf_chat_template(
@@ -94,6 +95,7 @@ class OpenAIServingTokenization(OpenAIServing):
                    conversation=conversation,
                    chat_template=self.chat_template,
                    add_generation_prompt=request.add_generation_prompt,
+                    continue_final_message=request.continue_final_message,
                )
        else:
            prompt = request.prompt