biondizzle/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[Bugfix] Fix error handling of unsupported sliding window (#11213)

Browse Source 
Signed-off-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Cyrus Leung
2024-12-16 01:59:42 +08:00
committed by GitHub
parent b10609e6a1
commit 96d673e0f8
1 changed files with 8 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files

14
vllm/model_executor/models/llama.py
View File

@@ -169,13 +169,15 @@ class LlamaAttention(nn.Module):
) )
if hasattr(config, "interleaved_sliding_window"): if hasattr(config, "interleaved_sliding_window"):
if isinstance(config.interleaved_sliding_window, int): interleaved_sliding_window = config.interleaved_sliding_window
sliding_window = config.interleaved_sliding_window if isinstance(interleaved_sliding_window, int):
elif isinstance(config.interleaved_sliding_window, list): sliding_window = interleaved_sliding_window
sw_idx = layer_idx % len(config.interleaved_sliding_window) elif isinstance(interleaved_sliding_window, list):
sliding_window = config.interleaved_sliding_window[sw_idx] sw_idx = layer_idx % len(interleaved_sliding_window)
sliding_window = interleaved_sliding_window[sw_idx]
else: else:
raise ValueError(f"{type(sliding_window)} is not supported.") raise ValueError(
f"{type(interleaved_sliding_window)} is not supported.")
else: else:
sliding_window = None sliding_window = None