[torch.compile] Hide KV cache behind torch.compile boundary (#11677)

Signed-off-by: Chen Zhang <zhangch99@outlook.com>
2025-01-10 13:14:42 +08:00
parent 3de2b1eafb
commit cf5f000d21
18 changed files with 198 additions and 44 deletions
--- a/vllm/config.py
+++ b/vllm/config.py
@@ -2780,7 +2780,6 @@ class CompilationConfig(BaseModel):
    compilation_time: float = PrivateAttr

    # Per-model forward context
-    # Mainly used to store attention cls
    # Map from layer name to the attention cls
    static_forward_context: Dict[str, Any] = PrivateAttr