[Feature] OpenAI-Compatible Tools API + Streaming for Hermes & Mistral models (#5649)

Co-authored-by: constellate <constellate@1-ai-appserver-staging.codereach.com>
Co-authored-by: Kyle Mistele <kyle@constellate.ai>
This commit is contained in:
Kyle Mistele
2024-09-04 15:18:13 -05:00
committed by GitHub
parent 561d6f8077
commit e02ce498be
26 changed files with 2591 additions and 86 deletions

View File

@@ -43,7 +43,11 @@ class OpenAIServingTokenization(OpenAIServing):
request_logger=request_logger)
# If this is None we use the tokenizer's default chat template
self.chat_template = load_chat_template(chat_template)
# the list of commonly-used chat template names for HF named templates
hf_chat_templates: List[str] = ['default', 'tool_use']
self.chat_template = chat_template \
if chat_template in hf_chat_templates \
else load_chat_template(chat_template)
async def create_tokenize(
self,