Implement presence and frequency penalties (#95)

2023-05-10 23:39:12 -07:00
parent 9f88db35da
commit 55f8b0a5de
9 changed files with 215 additions and 82 deletions
--- a/cacheflow/frontend/fastapi_frontend.py
+++ b/cacheflow/frontend/fastapi_frontend.py
@@ -96,7 +96,7 @@ class FastAPIServer:
        seqs: List[Sequence] = []
        for _ in range(sampling_params.n):
            seq_id = next(self.seq_counter)
-            seq = Sequence(seq_id, token_ids, block_size=self.block_size)
+            seq = Sequence(seq_id, prompt, token_ids, block_size=self.block_size)
            seqs.append(seq)

        arrival_time = time.time()