[Misc] Include matched stop string/token in responses (#2976)

Co-authored-by: Sahil Suneja <sahilsuneja@gmail.com>
This commit is contained in:
Nick Hill
2024-03-25 17:31:32 -07:00
committed by GitHub
parent 3a243095e5
commit dfeb2ecc3a
7 changed files with 97 additions and 7 deletions

View File

@@ -183,6 +183,7 @@ class Sequence:
# Initialize the logical token blocks with the prompt token ids.
self._append_tokens_to_blocks(prompt_token_ids)
self.status = SequenceStatus.WAITING
self.stop_reason: Union[int, str, None] = None
# Used for incremental detokenization
self.prefix_offset = 0