[V1][Perf] Faster incremental detokenization (#15137)

Signed-off-by: Nick Hill <nhill@redhat.com>
This commit is contained in:
Nick Hill
2025-04-17 07:45:24 -07:00
committed by GitHub
parent 7c02d6a137
commit 05fcd1b430
7 changed files with 317 additions and 145 deletions

View File

@@ -70,6 +70,15 @@ class MistralToolParser(ToolParser):
"Mistral Tool Parser could not locate the tool call token in "
"the tokenizer!")
def adjust_request(
self, request: ChatCompletionRequest) -> ChatCompletionRequest:
if request.tools and request.tool_choice != 'none':
# do not skip special tokens because mistral uses the special
# tokens to indicate the start and end of the tool calls
# information.
request.skip_special_tokens = False
return request
def extract_tool_calls(
self,
model_output: str,