[Model] PP support for Mamba-like models (#10992)

Signed-off-by: mzusman <mor.zusmann@gmail.com>
2024-12-11 04:53:37 +02:00
parent d5c5154fcf
commit ffa48c9146
11 changed files with 227 additions and 79 deletions
--- a/vllm/utils.py
+++ b/vllm/utils.py
@@ -170,6 +170,11 @@ class Device(enum.Enum):
    CPU = enum.auto()


+class LayerBlockType(enum.Enum):
+    attention = "attention"
+    mamba = "mamba"
+
+
 class Counter:

    def __init__(self, start: int = 0) -> None: