biondizzle/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[FIX] Fix styles in automatic prefix caching & add a automatic prefix caching benchmark (#3158)

Browse Source
This commit is contained in:
Zhuohan Li
2024-03-03 14:37:18 -08:00
committed by GitHub
parent d65fac2738
commit 996d095c54
4 changed files with 69 additions and 18 deletions
Download Patch File Download Diff File Expand all files Collapse all files

8
vllm/sequence.py
View File

@@ -160,10 +160,10 @@ class Sequence:
def lora_int_id(self) -> int:
return self.lora_request.lora_int_id if self.lora_request else 0
# TODO The current hashing function is O(L^2). We should optimize this in
# the future.
def hash_of_block(self, logical_idx: int) -> int:
# Compute the number of tokens in the sequence
# TODO: The current hashing function is O(L^2). We should optimize
# this in the future.
num_tokens = self.num_hashed_tokens_of_block(logical_idx)
return hash(tuple(self.data.get_token_ids()[0:num_tokens]))
@@ -308,10 +308,6 @@ class SequenceGroup:
# We use the prompt of an arbitrary sequence.
return next(iter(self.seqs_dict.values())).data.prompt_token_ids
@property
def block_size(self) -> int:
return next(iter(self.seqs_dict.values())).block_size
@property
def lora_int_id(self) -> int:
return self.lora_request.lora_int_id if self.lora_request else 0