biondizzle/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

OpenAI Compatible Frontend (#116)

Browse Source
This commit is contained in:
Zhuohan Li
2023-05-23 21:39:50 -07:00
committed by GitHub
parent e86717833d
commit 057daef778
20 changed files with 644 additions and 169 deletions
Download Patch File Download Diff File Expand all files Collapse all files

6
cacheflow/core/block_manager.py
View File

@@ -148,7 +148,7 @@ class BlockSpaceManager:
# the sequences in the same group.
blocks: Set[PhysicalTokenBlock] = set()
for seq in seq_group.get_seqs():
if seq.status == SequenceStatus.FINISHED:
if SequenceStatus.is_finished(seq.status):
continue
block_table = self.block_tables[seq.seq_id]
for block in block_table:
@@ -169,7 +169,7 @@ class BlockSpaceManager:
# CPU block -> GPU block.
mapping: Dict[PhysicalTokenBlock, PhysicalTokenBlock] = {}
for seq in seq_group.get_seqs():
if seq.status == SequenceStatus.FINISHED:
if SequenceStatus.is_finished(seq.status):
continue
new_block_table: BlockTable = []
block_table = self.block_tables[seq.seq_id]
@@ -200,7 +200,7 @@ class BlockSpaceManager:
# GPU block -> CPU block.
mapping: Dict[PhysicalTokenBlock, PhysicalTokenBlock] = {}
for seq in seq_group.get_seqs():
if seq.status == SequenceStatus.FINISHED:
if SequenceStatus.is_finished(seq.status):
continue
new_block_table: BlockTable = []
block_table = self.block_tables[seq.seq_id]