diff --git a/vllm/model_executor/models/isaac.py b/vllm/model_executor/models/isaac.py index 8e03e29a7..e29646182 100644 --- a/vllm/model_executor/models/isaac.py +++ b/vllm/model_executor/models/isaac.py @@ -334,15 +334,14 @@ class IsaacProcessingInfo(BaseProcessingInfo): return IsaacConfig() def get_image_processor(self, **kwargs) -> IsaacImageProcessor: - return IsaacImageProcessor(kwargs) + return IsaacImageProcessor(**kwargs) def get_hf_processor(self, **kwargs) -> IsaacProcessor: hf_config = self.get_hf_config() - return self.ctx.init_processor( - IsaacProcessor, + return IsaacProcessor( tokenizer=self.get_tokenizer(), - image_processor=self.get_image_processor(), + image_processor=self.get_image_processor(**kwargs), image_token=hf_config.vision_token, ) diff --git a/vllm/model_executor/models/kimi_k25.py b/vllm/model_executor/models/kimi_k25.py index 4b2b6a4b6..10d21aab0 100644 --- a/vllm/model_executor/models/kimi_k25.py +++ b/vllm/model_executor/models/kimi_k25.py @@ -104,19 +104,25 @@ class KimiK25ProcessingInfo(BaseProcessingInfo): def __init__(self, ctx: InputProcessingContext) -> None: super().__init__(ctx) - self.hf_config = self.get_hf_config() - self.media_token_id = self.hf_config.media_placeholder_token_id - media_processor = cached_get_image_processor( + + self.hf_config = hf_config = self.get_hf_config() + + tokenizer = self.get_tokenizer() + image_processor = cached_get_image_processor( self.ctx.model_config.model, trust_remote_code=self.ctx.model_config.trust_remote_code, ) - self.media_processor = media_processor + + self.media_token_id = media_token_id = hf_config.media_placeholder_token_id + self.media_token = tokenizer.decode(media_token_id) + + self.image_processor = image_processor self.hf_processor = KimiK25Processor( - media_processor=self.media_processor, - tokenizer=self.get_tokenizer(), - media_token_id=self.media_token_id, + tokenizer=tokenizer, + image_processor=image_processor, + media_token_id=media_token_id, ) - self.media_tokens_calculator = self.media_processor.media_tokens_calculator + self.media_tokens_calculator = image_processor.media_tokens_calculator def get_hf_processor(self): return self.hf_processor @@ -132,20 +138,15 @@ class KimiK25ProcessingInfo(BaseProcessingInfo): class KimiK25DummyInputsBuilder(BaseDummyInputsBuilder[KimiK25ProcessingInfo]): """Builds dummy inputs for Kimi-K2.5 model profiling.""" - def __init__(self, info: KimiK25ProcessingInfo) -> None: - super().__init__(info) - self.media_token_id = self.info.media_token_id - self.frame_per_chunk = self.info.media_processor.num_frames_per_chunk - def get_dummy_text(self, mm_counts: Mapping[str, int]) -> str: num_media = mm_counts.get("vision_chunk", 0) - return "<|media_pad|>" * num_media + return self.info.media_token * num_media def get_dummy_mm_items(self): dummy_videos = self._get_dummy_images( height=MaxImageTokenMeta.height, width=MaxImageTokenMeta.width, - num_images=self.frame_per_chunk, + num_images=self.info.image_processor.num_frames_per_chunk, ) video_chunk_dummy_item = VisionChunkVideo( @@ -236,9 +237,6 @@ class KimiK25MultiModalProcessor(BaseMultiModalProcessor[KimiK25ProcessingInfo]) ), ] - def split_video_chunks(self, video): - return self.info.media_processor.split_video_chunks(video) - @MULTIMODAL_REGISTRY.register_processor( KimiK25MultiModalProcessor, diff --git a/vllm/transformers_utils/processors/isaac.py b/vllm/transformers_utils/processors/isaac.py index 986b70840..1464afc66 100644 --- a/vllm/transformers_utils/processors/isaac.py +++ b/vllm/transformers_utils/processors/isaac.py @@ -6,12 +6,14 @@ import math from typing import Any import numpy as np -import PIL.Image import torch import torch.nn.functional as F +from PIL import Image from transformers import BatchFeature, ProcessorMixin, TensorType from typing_extensions import TypedDict, Unpack +from vllm.tokenizers.hf import HfTokenizer + MAX_PIXELS = 60_000_000 # 60-megapixel ceiling ≈ 8200 × 7300 px # Vision preprocessing constants @@ -39,7 +41,7 @@ def _make_writeable(arr: np.ndarray) -> np.ndarray: return arr.copy() -def extract_image_pil(image: PIL.Image.Image) -> torch.Tensor | None: +def extract_image_pil(image: Image.Image) -> torch.Tensor: if image.width * image.height > MAX_PIXELS: raise ValueError( f"Image (w={image.width}, h={image.height}) > MAX=`{MAX_PIXELS}`" @@ -314,31 +316,30 @@ class IsaacImageProcessorKwargs(TypedDict, total=False): class IsaacImageProcessor: - patch_size = 16 - max_num_patches = 6144 - min_num_patches = 256 - pixel_shuffle_scale = 2 - valid_kwargs = IsaacImageProcessorKwargs model_input_names = ["pixel_values", "image_grid_thw"] - def __init__(self, kwargs): - self.patch_size = kwargs.pop("patch_size", self.patch_size) - self.vision_max_num_patches = kwargs.pop( - "vision_max_num_patches", self.max_num_patches - ) - self.vision_min_num_patches = kwargs.pop( - "vision_min_num_patches", self.min_num_patches - ) - self.pixel_shuffle_scale = kwargs.pop("pixel_shuffle_scale", 2) - - def preprocess( + def __init__( self, - images: list[torch.Tensor], - return_tensors: str | TensorType | None, + patch_size: int = 16, + vision_max_num_patches: int = 6144, + vision_min_num_patches: int = 256, + pixel_shuffle_scale: int = 2, + ) -> None: + self.patch_size = patch_size + self.vision_max_num_patches = vision_max_num_patches + self.vision_min_num_patches = vision_min_num_patches + self.pixel_shuffle_scale = pixel_shuffle_scale + + def __call__( + self, + images: Image.Image | list[Image.Image], + return_tensors: str | TensorType | None = None, **kwargs: Unpack[IsaacImageProcessorKwargs], ) -> BatchFeature: """Preprocess images into format compatible with vLLM input processing.""" + if not isinstance(images, list): + images = [images] all_pixel_values: list[torch.Tensor] = [] all_image_grids: list[torch.Tensor] = [] @@ -388,23 +389,40 @@ class IsaacImageProcessor: class IsaacProcessor(ProcessorMixin): attributes = ["image_processor", "tokenizer"] - def __init__(self, image_processor=None, tokenizer=None, **kwargs): - self.image_token = kwargs.pop("image_token", "") + def __init__( + self, + image_processor: IsaacImageProcessor, + tokenizer: HfTokenizer, + image_token: str = "", + ): self.image_processor = image_processor self.tokenizer = tokenizer - def __call__(self, text=None, images=None, **kwargs) -> BatchFeature: - result = {} + self.image_token = image_token + def __call__( + self, + text: str | list[str] | None = None, + images: Image.Image | list[Image.Image] | None = None, + return_tensors: str | TensorType | None = None, + **kwargs, + ) -> BatchFeature: if images is not None: - image_inputs = self.image_processor.preprocess(images, **kwargs) + image_inputs = self.image_processor( + images, + return_tensors=return_tensors, + **kwargs, + ) image_grid_thw = image_inputs["image_grid_thw"] - result.update(image_inputs) + else: + image_inputs = {} + image_grid_thw = [] - if text is not None: - if not isinstance(text, list): - text = [text] + if text is not None: + if not isinstance(text, list): + text = [text] + if image_inputs: text = text.copy() # below lines change text in-place merge_length = self.image_processor.pixel_shuffle_scale**2 index = 0 @@ -417,10 +435,14 @@ class IsaacProcessor(ProcessorMixin): index += 1 text[i] = text[i].replace("<|placeholder|>", "<|image_pad|>") - if text is not None: - result.update(self.tokenizer(text, **kwargs)) + text_inputs = self.tokenizer(text, return_tensors=return_tensors) + else: + text_inputs = {} - return BatchFeature(result) + return BatchFeature( + data={**text_inputs, **image_inputs}, + tensor_type=return_tensors, + ) def apply_chat_template( self, diff --git a/vllm/transformers_utils/processors/kimi_k25.py b/vllm/transformers_utils/processors/kimi_k25.py index 06147f211..edee9734c 100644 --- a/vllm/transformers_utils/processors/kimi_k25.py +++ b/vllm/transformers_utils/processors/kimi_k25.py @@ -1,38 +1,41 @@ # SPDX-License-Identifier: Apache-2.0 # SPDX-FileCopyrightText: Copyright contributors to the vLLM project -import torch -from transformers import BatchFeature +from transformers import BaseImageProcessor, BatchFeature, TensorType from transformers.processing_utils import ProcessorMixin from vllm.multimodal.inputs import VisionChunk +from vllm.tokenizers.hf import HfTokenizer class KimiK25Processor(ProcessorMixin): - attributes = ["tokenizer"] - tokenizer_class = "AutoTokenizer" + attributes = ["image_processor", "tokenizer"] def __init__( - self, media_processor=None, tokenizer=None, media_token_id: int | None = None - ): - super().__init__(tokenizer) - self.media_processor = media_processor + self, + image_processor: BaseImageProcessor, + tokenizer: HfTokenizer, + media_token_id: int, + ) -> None: + self.image_processor = image_processor + self.tokenizer = tokenizer + self.media_token_id = media_token_id - assert self.media_token_id is not None def __call__( self, + text: str | list[str] | None = None, vision_chunks: list[VisionChunk] | None = None, - *, - text: list[int] | str, + return_tensors: str | TensorType | None = None, **kwargs, ) -> BatchFeature: """ Args: - vision_chunks: List of VisionChunk items to be processed. - For image: VisionChunkImage with type='image', image=PIL.Image - For video_chunk: VisionChunkVideo with type='video_chunk', - video_chunk=list[PIL.Image] - text: The token ids to be fed to a model (required). + text: The text to be field to the model. + vision_chunks: List of `VisionChunk` items to be processed. + For image: `VisionChunkImage` with + `type='image', image=PIL.Image` + For video_chunk: `VisionChunkVideo` with + `type='video_chunk', video_chunk=list[PIL.Image]` Returns: [`BatchFeature`]: A [`BatchFeature`] with the following fields: @@ -42,31 +45,44 @@ class KimiK25Processor(ProcessorMixin): - **grid_thws** -- list of image 3D grid in LLM. Returned when `vision_chunks` is not `None`. """ - mm_inputs = {} - input_ids = self.tokenizer.encode(text) if isinstance(text, str) else text if vision_chunks is not None: - assert isinstance(vision_chunks, list) - mm_inputs = self.media_processor.preprocess(vision_chunks) + mm_inputs = self.image_processor.preprocess( + vision_chunks, + return_tensors=return_tensors, + ) + else: + mm_inputs = {} - num_tokens_per_chunk = [ - self.media_processor.media_tokens_calculator(chunk) - for chunk in vision_chunks - ] + if text is not None: + if not isinstance(text, list): + text = [text] - new_input_ids = [] - for token in input_ids: - if token == self.media_token_id: - new_input_ids.extend( - [self.media_token_id] * num_tokens_per_chunk.pop(0) - ) - else: - new_input_ids.append(token) - input_ids = new_input_ids + text_inputs = self.tokenizer(text) + + # Note: Modify in-place + input_ids: list[list[int]] = text_inputs["input_ids"] # type: ignore + + if vision_chunks is not None: + num_tokens_per_chunk = [ + self.image_processor.media_tokens_calculator(chunk) + for chunk in vision_chunks + ] + + for i in range(len(input_ids)): + new_input_ids = [] + for token in input_ids[i]: + if token == self.media_token_id: + new_input_ids.extend( + [self.media_token_id] * num_tokens_per_chunk.pop(0) + ) + else: + new_input_ids.append(token) + + input_ids[i] = new_input_ids + else: + text_inputs = {} - # XXX: _apply_hf_processor_text_mm will call tolist() on input_ids return BatchFeature( - data={ - "input_ids": torch.tensor([input_ids]), - **mm_inputs, - } + data={**text_inputs, **mm_inputs}, + tensor_type=return_tensors, ) diff --git a/vllm/transformers_utils/processors/step3_vl.py b/vllm/transformers_utils/processors/step3_vl.py index 66cf10e39..71540f433 100644 --- a/vllm/transformers_utils/processors/step3_vl.py +++ b/vllm/transformers_utils/processors/step3_vl.py @@ -286,11 +286,9 @@ class Step3VLImageProcessor: def __call__( self, - images: Image.Image | list[Image.Image] | None = None, + images: Image.Image | list[Image.Image], return_tensors: str | TensorType | None = None, ) -> BatchFeature: - if images is None: - images = [] if not isinstance(images, list): images = [images]