biondizzle/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[BugFix] Fix input positions for long context with sliding window (#2088)

Browse Source
This commit is contained in:
Woosuk Kwon
2023-12-13 12:28:13 -08:00
committed by GitHub
parent 096827c284
commit f1c8520146
5 changed files with 75 additions and 17 deletions
Download Patch File Download Diff File Expand all files Collapse all files

12
vllm/worker/model_runner.py
View File

@@ -134,14 +134,14 @@ class ModelRunner:
generation_token = seq_data.get_last_token_id()
input_tokens.append([generation_token])
context_len = seq_data.get_len()
if self.sliding_window is not None:
context_len = min(context_len, self.sliding_window)
context_lens.append(context_len)
position = context_len - 1
seq_len = seq_data.get_len()
position = seq_len - 1
input_positions.append([position])
context_len = seq_len if self.sliding_window is None else min(
seq_len, self.sliding_window)
context_lens.append(context_len)
block_table = seq_group_metadata.block_tables[seq_id]
block_number = block_table[position // self.block_size]
block_offset = position % self.block_size