biondizzle/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[Bugfix]Disable the post_norm layer of the vision encoder for LLaVA models (#9653)

Browse Source
This commit is contained in:
litianjian
2024-10-24 22:52:07 +08:00
committed by GitHub
parent b979143d5b
commit f58454968f
4 changed files with 8 additions and 4 deletions
Download Patch File Download Diff File Expand all files Collapse all files

3
vllm/model_executor/models/llava_onevision.py
View File

@@ -400,7 +400,8 @@ class LlavaOnevisionForConditionalGeneration(nn.Module, SupportsMultiModal,
self.multimodal_config = multimodal_config
# Initialize the vision tower only up to the required feature layer
self.vision_tower = init_vision_tower_for_llava(config, quant_config)
self.vision_tower = init_vision_tower_for_llava(
config, quant_config, require_post_norm=False)
self.multi_modal_projector = LlavaOnevisionMultiModalProjector(config)
self.language_model = init_vllm_registered_model(
config.text_config, cache_config, quant_config)