biondizzle/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Apply torchfix (#15532)

Browse Source 
Signed-off-by: cyy <cyyever@outlook.com>
This commit is contained in:
cyyever
2025-03-26 20:09:06 +08:00
committed by GitHub
parent cf5c8f1686
commit 1aa162e030
5 changed files with 15 additions and 11 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
vllm/model_executor/models/phi4mm_utils.py
View File

@@ -1766,9 +1766,12 @@ class MultiHeadedAttention(nn.Module):
if mask.dtype != q.dtype:
attn_mask = attn_mask.to(q.dtype)
with torch.backends.cuda.sdp_kernel(enable_flash=True,
enable_math=True,
enable_mem_efficient=True):
with torch.nn.attention.sdpa_kernel([
torch.nn.attention.SDPBackend.FLASH_ATTENTION,
torch.nn.attention.SDPBackend.EFFICIENT_ATTENTION,
torch.nn.attention.SDPBackend.MATH,
torch.nn.attention.SDPBackend.CUDNN_ATTENTION,
]):
x = torch.nn.functional.scaled_dot_product_attention(
q,
k,