[Model] VLM2Vec, the first multimodal embedding model in vLLM (#9303)

2024-10-16 14:31:00 +08:00
parent 7e7eae338d
commit 7abba39ee6
16 changed files with 465 additions and 261 deletions
--- a/vllm/model_executor/models/registry.py
+++ b/vllm/model_executor/models/registry.py
@@ -86,9 +86,12 @@ _TEXT_GENERATION_MODELS = {
 }

 _EMBEDDING_MODELS = {
-    "MistralModel": ("llama_embedding", "LlamaEmbeddingModel"),
+    # [Text-only]
+    "Gemma2Model": ("gemma2", "Gemma2EmbeddingModel"),
+    "MistralModel": ("llama", "LlamaEmbeddingModel"),
    "Qwen2ForRewardModel": ("qwen2_rm", "Qwen2ForRewardModel"),
-    "Gemma2Model": ("gemma2_embedding", "Gemma2EmbeddingModel"),
+    # [Multimodal]
+    "Phi3VForCausalLM": ("phi3v", "Phi3VForCausalLM"),
 }

 _MULTIMODAL_MODELS = {