biondizzle/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[Model] Enable LoRA support for internvl2 (#32397)

Browse Source 
Signed-off-by: Matteo Fari <matteofari06@gmail.com>
This commit is contained in:
Matteo Fari
2026-01-23 18:39:01 +01:00
committed by GitHub
parent bbbd696af9
commit fec9da0af4
1 changed files with 16 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

19
vllm/model_executor/models/internvl.py
View File

@@ -1086,9 +1086,8 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA)
image_size = config.force_image_size or config.vision_config.image_size
patch_size = config.vision_config.patch_size
self.patch_size = patch_size
self.num_image_token = int(
(image_size // patch_size) ** 2 * (config.downsample_ratio**2)
)
self.patch_tokens = (image_size // patch_size) ** 2
self.num_image_token = int(self.patch_tokens * (config.downsample_ratio**2))
self.downsample_ratio = config.downsample_ratio
self.ps_version = config.ps_version
@@ -1430,3 +1429,17 @@ class InternVLChatModel(nn.Module, SupportsMultiModal, SupportsPP, SupportsLoRA)
connector="mlp1",
tower_model="vision_model",
)
def get_num_mm_encoder_tokens(self, num_image_tokens: int) -> int:
if num_image_tokens <= 0 or self.num_image_token <= 0:
return 0
num_patches = num_image_tokens // self.num_image_token
return num_patches * (self.patch_tokens + 1)
def get_num_mm_connector_tokens(self, num_vision_tokens: int) -> int:
if num_vision_tokens <= 0 or self.num_image_token <= 0:
return 0
num_patches = num_vision_tokens // (self.patch_tokens + 1)
return num_patches * self.num_image_token