[Core][VLM] Support image embeddings as input (#6613)

2024-08-12 01:16:06 -07:00
parent ec2affa8ae
commit e6e42e4b17
13 changed files with 517 additions and 138 deletions
--- a/vllm/model_executor/models/clip.py
+++ b/vllm/model_executor/models/clip.py
@@ -88,7 +88,13 @@ def input_processor_for_clip(
    tokenizer = cached_get_tokenizer(model_config.tokenizer)

    if image_feature_size_override is None:
-        image_feature_size = get_clip_image_feature_size(hf_config)
+        image_data = multi_modal_data["image"]
+        if isinstance(image_data, Image.Image):
+            image_feature_size = get_clip_image_feature_size(hf_config)
+        elif isinstance(image_data, torch.Tensor):
+            image_feature_size = image_data.shape[0]
+        else:
+            raise TypeError(f"Invalid image type: {type(image_data)}")
    else:
        image_feature_size = image_feature_size_override