biondizzle/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[Bugfix] Initialize attention bias on the same device as Query/Key/Value for QwenVL Series (#14031)

Browse Source
This commit is contained in:
Yang Liu
2025-02-28 23:36:08 +08:00
committed by GitHub
parent f7bee5c815
commit 9b61dd41e7
2 changed files with 4 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
vllm/model_executor/models/qwen2_5_vl.py
View File

@@ -323,7 +323,8 @@ class Qwen2_5_VisionAttention(nn.Module):
seqlens = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist() seqlens = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist()
attn_bias = BlockDiagonalMask.from_seqlens(q_seqlen=seqlens, attn_bias = BlockDiagonalMask.from_seqlens(q_seqlen=seqlens,
kv_seqlen=None) kv_seqlen=None,
device=q.device)
context_layer = xops.memory_efficient_attention_forward( context_layer = xops.memory_efficient_attention_forward(
q, k, v, attn_bias=attn_bias, p=0, scale=None) q, k, v, attn_bias=attn_bias, p=0, scale=None)

3
vllm/model_executor/models/qwen2_vl.py
View File

@@ -367,7 +367,8 @@ class Qwen2VisionAttention(nn.Module):
seqlens = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist() seqlens = (cu_seqlens[1:] - cu_seqlens[:-1]).tolist()
attn_bias = BlockDiagonalMask.from_seqlens(q_seqlen=seqlens, attn_bias = BlockDiagonalMask.from_seqlens(q_seqlen=seqlens,
kv_seqlen=None) kv_seqlen=None,
device=q.device)
context_layer = xops.memory_efficient_attention_forward( context_layer = xops.memory_efficient_attention_forward(
q, k, v, attn_bias=attn_bias, p=0, scale=None) q, k, v, attn_bias=attn_bias, p=0, scale=None)