biondizzle/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

PaliGemma 2 support (#11142)

Browse Source
This commit is contained in:
Jani Monoses
2024-12-13 09:40:07 +02:00
committed by GitHub
parent be39e3cd18
commit 7cd7409142
3 changed files with 25 additions and 3 deletions
Download Patch File Download Diff File Expand all files Collapse all files

11
vllm/model_executor/models/paligemma.py
View File

@@ -105,6 +105,11 @@ def input_processor_for_paligemma(ctx: InputContext,
orig_prompt_ids.remove(hf_config.image_token_index)
new_prompt = f"{image_token_str_pad}{bos_token}{orig_prompt}\n"
# The PaliGemma 2 tokenizer does not include a starting BOS token
if orig_prompt_ids[0] != hf_config.bos_token_id:
orig_prompt_ids = [hf_config.bos_token_id] + orig_prompt_ids
new_token_ids = image_token_ids_pad + orig_prompt_ids + [108] #newline
# NOTE: Create a defensive copy of the original inputs
@@ -149,7 +154,11 @@ class PaliGemmaForConditionalGeneration(nn.Module, SupportsMultiModal,
projection_dim=config.vision_config.projection_dim)
self.quant_config = quant_config
config.text_config.architectures = ["GemmaForCausalLM"]
if config.text_config.model_type == "gemma":
config.text_config.architectures = ["GemmaForCausalLM"]
else:
config.text_config.architectures = ["Gemma2ForCausalLM"]
self.language_model = init_vllm_registered_model(
vllm_config=vllm_config,
hf_config=config.text_config,