Refactor Worker & InputMetadata (#1843)

2023-11-29 22:16:37 -08:00
parent c782195662
commit 27feead2f8
27 changed files with 668 additions and 443 deletions
--- a/vllm/config.py
+++ b/vllm/config.py
@@ -161,6 +161,12 @@ class ModelConfig:
                "must be divisible by pipeline parallel size "
                f"({pipeline_parallel_size}).")

+    def get_sliding_window(self) -> Optional[int]:
+        return getattr(self.hf_config, "sliding_window", None)
+
+    def get_vocab_size(self) -> int:
+        return self.hf_config.vocab_size
+
    def get_hidden_size(self) -> int:
        return self.hf_config.hidden_size