Add guided decoding for OpenAI API server (#2819)

Co-authored-by: br3no <breno@veltefaria.de>
Co-authored-by: simon-mo <simon.mo@hey.com>
This commit is contained in:
felixzhu555
2024-02-29 14:13:08 -08:00
committed by GitHub
parent 29a8d6a554
commit 703e42ee4b
9 changed files with 597 additions and 1 deletions

View File

@@ -333,6 +333,9 @@ class AsyncLLMEngine:
return (self.background_loop is not None
and not self.background_loop.done())
def get_tokenizer(self):
return self.engine.tokenizer.tokenizer
def start_background_loop(self) -> None:
"""Start the background loop."""
if self.is_running: