[Flashinfer][gpt-oss] Support FP8-qkv Flashinfer TRTLLM Sinks Attention (#25674)

Signed-off-by: elvischenv <219235043+elvischenv@users.noreply.github.com>
2025-10-10 04:13:39 +08:00
parent a462331e36
commit 44f633dba1
2 changed files with 76 additions and 46 deletions
--- a/vllm/utils/flashinfer.py
+++ b/vllm/utils/flashinfer.py
@@ -269,11 +269,6 @@ def use_trtllm_attention(

    # Must use TRTLLM attention if query is FP8 quantized
    if q_dtype == current_platform.fp8_dtype():
-        if has_sinks:
-            raise RuntimeError(
-                "TRTLLM FP8-qkv kernel is not supported for attention sinks. "
-                "Use kv_cache_dtype=auto for now."
-            )
        logger.info_once("Using TRTLLM attention (query is quantized).")
        return True