[Model] Multi-input support for LLaVA (#8238)

2024-09-07 10:57:24 +08:00
parent 41e95c5247
commit 2f707fcb35
10 changed files with 176 additions and 45 deletions
--- a/vllm/model_executor/models/clip.py
+++ b/vllm/model_executor/models/clip.py
@@ -105,7 +105,7 @@ def input_processor_for_clip(
        if isinstance(image_data, Image.Image):
            image_feature_size = get_clip_image_feature_size(hf_config)
        elif isinstance(image_data, torch.Tensor):
-            image_feature_size = image_data.shape[0]
+            num_images, image_feature_size, hidden_size = image_data.shape
        else:
            raise TypeError(f"Invalid image type: {type(image_data)}")
    else: