[Core] Support loading GGUF model (#5191)

Co-authored-by: Michael Goin <michael@neuralmagic.com>
2024-08-06 07:54:23 +08:00
parent ef527be06c
commit 360bd67cf0
29 changed files with 4970 additions and 21 deletions
--- a/vllm/config.py
+++ b/vllm/config.py
@@ -582,6 +582,7 @@ class LoadFormat(str, enum.Enum):
    DUMMY = "dummy"
    TENSORIZER = "tensorizer"
    SHARDED_STATE = "sharded_state"
+    GGUF = "gguf"
    BITSANDBYTES = "bitsandbytes"