Convert formatting to use ruff instead of yapf + isort (#26247)
Signed-off-by: Harry Mellor <19981378+hmellor@users.noreply.github.com>
This commit is contained in:
@@ -10,36 +10,34 @@ from transformers import BatchFeature
|
||||
from vllm.config import ModelConfig, VllmConfig
|
||||
from vllm.inputs import TokensPrompt
|
||||
from vllm.logger import init_logger
|
||||
from vllm.model_executor.layers.linear import (ColumnParallelLinear,
|
||||
RowParallelLinear)
|
||||
from vllm.model_executor.layers.linear import ColumnParallelLinear, RowParallelLinear
|
||||
from vllm.model_executor.layers.pooler import DispatchPooler, Pooler
|
||||
from vllm.multimodal import MULTIMODAL_REGISTRY
|
||||
from vllm.sequence import IntermediateTensors
|
||||
|
||||
from .interfaces import (SupportsCrossEncoding, SupportsMultiModal,
|
||||
SupportsScoreTemplate)
|
||||
from .qwen2_vl import (Qwen2VLDummyInputsBuilder,
|
||||
Qwen2VLForConditionalGeneration,
|
||||
Qwen2VLMultiModalProcessor, Qwen2VLProcessingInfo)
|
||||
from .interfaces import SupportsCrossEncoding, SupportsMultiModal, SupportsScoreTemplate
|
||||
from .qwen2_vl import (
|
||||
Qwen2VLDummyInputsBuilder,
|
||||
Qwen2VLForConditionalGeneration,
|
||||
Qwen2VLMultiModalProcessor,
|
||||
Qwen2VLProcessingInfo,
|
||||
)
|
||||
from .utils import AutoWeightsLoader, WeightsMapper, maybe_prefix
|
||||
|
||||
logger = init_logger(__name__)
|
||||
|
||||
|
||||
class JinaVLScorer(nn.Module):
|
||||
|
||||
def __init__(self, model_config: "ModelConfig"):
|
||||
super().__init__()
|
||||
config = model_config.hf_config
|
||||
head_dtype = model_config.head_dtype
|
||||
self.dense = ColumnParallelLinear(config.hidden_size,
|
||||
config.hidden_size,
|
||||
params_dtype=head_dtype,
|
||||
bias=True)
|
||||
self.out_proj = RowParallelLinear(config.hidden_size,
|
||||
config.num_labels,
|
||||
params_dtype=head_dtype,
|
||||
bias=True)
|
||||
self.dense = ColumnParallelLinear(
|
||||
config.hidden_size, config.hidden_size, params_dtype=head_dtype, bias=True
|
||||
)
|
||||
self.out_proj = RowParallelLinear(
|
||||
config.hidden_size, config.num_labels, params_dtype=head_dtype, bias=True
|
||||
)
|
||||
|
||||
def forward(self, x, **kwargs):
|
||||
x, _ = self.dense(x)
|
||||
@@ -49,7 +47,6 @@ class JinaVLScorer(nn.Module):
|
||||
|
||||
|
||||
class JinaVLMultiModalProcessor(Qwen2VLMultiModalProcessor):
|
||||
|
||||
def _call_hf_processor(
|
||||
self,
|
||||
prompt: str,
|
||||
@@ -57,25 +54,26 @@ class JinaVLMultiModalProcessor(Qwen2VLMultiModalProcessor):
|
||||
mm_kwargs: Mapping[str, object],
|
||||
tok_kwargs: Mapping[str, object],
|
||||
) -> BatchFeature:
|
||||
|
||||
# NOTE: We should reverse the order of the mm_data because the
|
||||
# query prompt is placed after the document prompt in the score
|
||||
# template for JinaVLForRanking model, but in mm_data they are
|
||||
# stored in the opposite order (query first, then document).
|
||||
for _, value in mm_data.items():
|
||||
value.reverse()
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs,
|
||||
tok_kwargs)
|
||||
return super()._call_hf_processor(prompt, mm_data, mm_kwargs, tok_kwargs)
|
||||
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(JinaVLMultiModalProcessor,
|
||||
info=Qwen2VLProcessingInfo,
|
||||
dummy_inputs=Qwen2VLDummyInputsBuilder)
|
||||
class JinaVLForSequenceClassification(Qwen2VLForConditionalGeneration,
|
||||
SupportsCrossEncoding,
|
||||
SupportsMultiModal,
|
||||
SupportsScoreTemplate):
|
||||
|
||||
@MULTIMODAL_REGISTRY.register_processor(
|
||||
JinaVLMultiModalProcessor,
|
||||
info=Qwen2VLProcessingInfo,
|
||||
dummy_inputs=Qwen2VLDummyInputsBuilder,
|
||||
)
|
||||
class JinaVLForSequenceClassification(
|
||||
Qwen2VLForConditionalGeneration,
|
||||
SupportsCrossEncoding,
|
||||
SupportsMultiModal,
|
||||
SupportsScoreTemplate,
|
||||
):
|
||||
is_pooling_model = True
|
||||
weight_mapper = WeightsMapper(
|
||||
orig_to_new_prefix={
|
||||
@@ -87,23 +85,24 @@ class JinaVLForSequenceClassification(Qwen2VLForConditionalGeneration,
|
||||
# mapping for original checkpoint
|
||||
"lm_head.": "language_model.lm_head.",
|
||||
"model.": "language_model.model.",
|
||||
})
|
||||
}
|
||||
)
|
||||
|
||||
def __init__(self, *, vllm_config: VllmConfig, prefix: str = ""):
|
||||
super().__init__(vllm_config=vllm_config,
|
||||
prefix=maybe_prefix(prefix, "qwen2_vl"))
|
||||
super().__init__(
|
||||
vllm_config=vllm_config, prefix=maybe_prefix(prefix, "qwen2_vl")
|
||||
)
|
||||
pooler_config = vllm_config.model_config.pooler_config
|
||||
assert pooler_config is not None
|
||||
|
||||
self.score = JinaVLScorer(vllm_config.model_config)
|
||||
self.pooler = DispatchPooler({
|
||||
"encode":
|
||||
Pooler.for_encode(pooler_config),
|
||||
"classify":
|
||||
Pooler.for_classify(pooler_config, classifier=self.score),
|
||||
"score":
|
||||
Pooler.for_classify(pooler_config, classifier=self.score),
|
||||
})
|
||||
self.pooler = DispatchPooler(
|
||||
{
|
||||
"encode": Pooler.for_encode(pooler_config),
|
||||
"classify": Pooler.for_classify(pooler_config, classifier=self.score),
|
||||
"score": Pooler.for_classify(pooler_config, classifier=self.score),
|
||||
}
|
||||
)
|
||||
|
||||
@classmethod
|
||||
def get_placeholder_str(cls, modality: str, i: int) -> Optional[str]:
|
||||
@@ -118,9 +117,8 @@ class JinaVLForSequenceClassification(Qwen2VLForConditionalGeneration,
|
||||
|
||||
@classmethod
|
||||
def post_process_tokens(cls, prompt: TokensPrompt) -> None:
|
||||
|
||||
# add score target token at the end of prompt tokens
|
||||
prompt['prompt_token_ids'].append(100)
|
||||
prompt["prompt_token_ids"].append(100)
|
||||
|
||||
def forward(
|
||||
self,
|
||||
|
||||
Reference in New Issue
Block a user