[MODEL] Qwen Multimodal Support (Qwen-VL / Qwen-VL-Chat) (#8029)
Signed-off-by: Alex-Brooks <Alex.Brooks@ibm.com> Co-authored-by: DarkLight1337 <tlleungac@connect.ust.hk>
This commit is contained in:
Alex Brooks
@@ -26,11 +26,9 @@ import re
|
||||
from array import array
|
||||
from functools import partial
|
||||
from typing import (Any, Callable, Iterable, List, Mapping, Optional, Tuple,
|
||||
TypedDict, Union)
|
||||
TypedDict)
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
import torch.nn.functional as F
|
||||
import torch.types
|
||||
from PIL import Image
|
||||
from torch import nn
|
||||
@@ -44,6 +42,8 @@ from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.linear import ReplicatedLinear
|
||||
from vllm.model_executor.layers.logits_processor import LogitsProcessor
|
||||
from vllm.model_executor.layers.quantization import QuantizationConfig
|
||||
from vllm.model_executor.layers.resampler import (Resampler2,
|
||||
get_2d_sincos_pos_embed)
|
||||
from vllm.model_executor.layers.sampler import Sampler, SamplerOutput
|
||||
from vllm.model_executor.layers.vocab_parallel_embedding import ParallelLMHead
|
||||
from vllm.model_executor.model_loader.utils import set_default_torch_dtype
|
||||
@@ -98,101 +98,6 @@ MiniCPMVImageInputs = MiniCPMVImagePixelInputs
|
||||
DEFAULT_LN = partial(nn.LayerNorm, eps=1e-6)
|
||||
|
||||
|
||||
def get_abs_pos(abs_pos: torch.Tensor, tgt_size: torch.Tensor):
|
||||
# abs_pos: L, C
|
||||
# tgt_size: (H, W)
|
||||
# return: M, C
|
||||
src_size = int(math.sqrt(abs_pos.size(0)))
|
||||
# tgt_size = int(math.sqrt(tgt_size))
|
||||
dtype = abs_pos.dtype
|
||||
|
||||
return (F.interpolate(
|
||||
abs_pos.float().reshape(1, src_size, src_size, -1).permute(0, 3, 1, 2),
|
||||
size=(tgt_size[0], tgt_size[1]),
|
||||
mode="bicubic",
|
||||
align_corners=False,
|
||||
).permute(0, 2, 3, 1).flatten(0, 2).to(dtype=dtype))
|
||||
|
||||
|
||||
# https://github.com/facebookresearch/mae/blob/efb2a8062c206524e35e47d04501ed4f544c0ae8/util/pos_embed.py#L20
|
||||
def get_2d_sincos_pos_embed(
|
||||
embed_dim: int,
|
||||
grid_size: Union[int, Tuple[int, int]],
|
||||
cls_token: bool = False,
|
||||
version: Tuple[int, int] = (2, 0),
|
||||
):
|
||||
"""
|
||||
grid_size: int of the grid height and width
|
||||
return:
|
||||
pos_embed: [grid_size*grid_size, embed_dim] or
|
||||
[1+grid_size*grid_size, embed_dim] (w/ or w/o cls_token)
|
||||
"""
|
||||
if isinstance(grid_size, int):
|
||||
grid_h_size, grid_w_size = grid_size, grid_size
|
||||
else:
|
||||
grid_h_size, grid_w_size = grid_size[0], grid_size[1]
|
||||
|
||||
grid_h = np.arange(grid_h_size, dtype=np.float32)
|
||||
grid_w = np.arange(grid_w_size, dtype=np.float32)
|
||||
grid = np.meshgrid(grid_w, grid_h) # here w goes first
|
||||
grid = np.stack(grid, axis=0)
|
||||
|
||||
if version == (2, 0):
|
||||
grid = grid.reshape([2, 1, grid_h_size, grid_w_size])
|
||||
pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid, version)
|
||||
if cls_token:
|
||||
pos_embed = np.concatenate([np.zeros([1, embed_dim]), pos_embed],
|
||||
axis=0)
|
||||
else:
|
||||
pos_embed = get_2d_sincos_pos_embed_from_grid(embed_dim, grid, version)
|
||||
return pos_embed
|
||||
|
||||
|
||||
def get_2d_sincos_pos_embed_from_grid(embed_dim: int,
|
||||
grid: np.ndarray,
|
||||
version: Tuple[int, int] = (2, 0)):
|
||||
assert embed_dim % 2 == 0
|
||||
|
||||
# use half of dimensions to encode grid_h
|
||||
emb_h = get_1d_sincos_pos_embed_from_grid(
|
||||
embed_dim // 2, grid[0], version) # (H*W, D/2) or (H, W, D/2)
|
||||
emb_w = get_1d_sincos_pos_embed_from_grid(
|
||||
embed_dim // 2, grid[1], version) # (H*W, D/2) or (H, W, D/2)
|
||||
|
||||
if version == (2, 0):
|
||||
emb = np.concatenate([emb_h, emb_w], axis=1) # (H*W, D)
|
||||
else:
|
||||
emb = np.concatenate([emb_h, emb_w], axis=-1) # (H, W, D)
|
||||
return emb
|
||||
|
||||
|
||||
def get_1d_sincos_pos_embed_from_grid(embed_dim: int,
|
||||
pos: np.ndarray,
|
||||
version: Tuple[int, int] = (2, 0)):
|
||||
"""
|
||||
embed_dim: output dimension for each position
|
||||
pos: a list of positions to be encoded: size (M,) / (H, W)
|
||||
out: (M, D) / (H, W, D)
|
||||
"""
|
||||
assert embed_dim % 2 == 0
|
||||
omega = np.arange(embed_dim // 2, dtype=np.float32)
|
||||
omega /= embed_dim / 2.0
|
||||
omega = 1.0 / 10000**omega # (D/2,)
|
||||
|
||||
if version == (2, 0):
|
||||
pos = pos.reshape(-1) # (M,)
|
||||
out = np.einsum("m,d->md", pos, omega) # (M, D/2), outer product
|
||||
emb_sin = np.sin(out) # (M, D/2)
|
||||
emb_cos = np.cos(out) # (M, D/2)
|
||||
emb = np.concatenate([emb_sin, emb_cos], axis=1) # (M, D)
|
||||
else:
|
||||
out = np.einsum("hw,d->hwd", pos, omega) # (H, W, D/2), outer product
|
||||
emb_sin = np.sin(out) # (H, W, D/2)
|
||||
emb_cos = np.cos(out) # (H, W, D/2)
|
||||
emb = np.concatenate([emb_sin, emb_cos], axis=-1) # (H, W, D)
|
||||
return emb
|
||||
|
||||
|
||||
class BaseResampler(nn.Module):
|
||||
"""
|
||||
A 2D perceiver-resampler network with one cross attention layers by
|
||||
@@ -245,62 +150,6 @@ class BaseResampler(nn.Module):
|
||||
return query.unsqueeze(1).repeat(1, N, 1)
|
||||
|
||||
|
||||
class Resampler2(BaseResampler):
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
grid_size: int,
|
||||
embed_dim: int,
|
||||
num_heads: int,
|
||||
kv_dim: Optional[int] = None,
|
||||
norm_layer: Callable[[int], nn.LayerNorm] = DEFAULT_LN,
|
||||
adaptive: bool = False,
|
||||
) -> None:
|
||||
super().__init__(grid_size**2, embed_dim, num_heads, kv_dim,
|
||||
norm_layer)
|
||||
|
||||
self.adaptive = adaptive
|
||||
pos_embed_arr = get_2d_sincos_pos_embed(embed_dim,
|
||||
grid_size,
|
||||
version=(2, 0))
|
||||
self.pos_embed = nn.Parameter(
|
||||
torch.from_numpy(pos_embed_arr).float()).requires_grad_(False)
|
||||
|
||||
self.apply(self._init_weights)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
x: torch.Tensor,
|
||||
tgt_sizes: torch.Tensor,
|
||||
attn_mask: Optional[torch.Tensor] = None,
|
||||
):
|
||||
if self.adaptive:
|
||||
pos_embed_arr = get_2d_sincos_pos_embed(self.embed_dim,
|
||||
tgt_sizes,
|
||||
version=(2, 0))
|
||||
pos_embed = torch.from_numpy(pos_embed_arr).to(device=x.device,
|
||||
dtype=x.dtype)
|
||||
else:
|
||||
pos_embed = get_abs_pos(self.pos_embed, tgt_sizes)
|
||||
|
||||
x, _ = self.kv_proj(x)
|
||||
x = self.ln_kv(x).permute(1, 0, 2)
|
||||
|
||||
N = x.shape[1]
|
||||
q = self.ln_q(self.query)
|
||||
out = self.attn(
|
||||
self._repeat(q, N) + self.pos_embed.unsqueeze(1),
|
||||
x + pos_embed.unsqueeze(1),
|
||||
x,
|
||||
attn_mask=attn_mask,
|
||||
)[0]
|
||||
x = out.permute(1, 0, 2)
|
||||
|
||||
x = self.ln_post(x)
|
||||
x = x @ self.proj
|
||||
return x
|
||||
|
||||
|
||||
class Resampler2_5(BaseResampler):
|
||||
|
||||
def __init__(
|
||||
@@ -782,7 +631,8 @@ class MiniCPMV2_0(MiniCPMVBaseModel):
|
||||
num_heads=embed_dim // 128,
|
||||
grid_size=int(math.sqrt(self.config.query_num)),
|
||||
kv_dim=vision_dim,
|
||||
adaptive=True,
|
||||
adaptive=False,
|
||||
do_post_projection=True,
|
||||
)
|
||||
|
||||
return resampler
|
||||
|
||||
Reference in New Issue
Block a user