biondizzle/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

Support Cross encoder models (#10400)

Browse Source 
Signed-off-by: Max de Bayser <maxdebayser@gmail.com>
Signed-off-by: Max de Bayser <mbayser@br.ibm.com>
Signed-off-by: Flavia Beo <flavia.beo@ibm.com>
Co-authored-by: Flavia Beo <flavia.beo@ibm.com>
This commit is contained in:
Maximilien de Bayser
2024-11-24 23:56:20 -03:00
committed by GitHub
parent 49628fe13e
commit 214efc2c3c
28 changed files with 1370 additions and 62 deletions
Download Patch File Download Diff File Expand all files Collapse all files

9
vllm/sequence.py
View File

@@ -449,6 +449,10 @@ class Sequence:
def prompt_embeds(self) -> Optional[torch.Tensor]:
return self.inputs.prompt_embeds
@property
def token_type_ids(self) -> List[int]:
return self.inputs.token_type_ids
@property
def multi_modal_data(self) -> "MultiModalDataDict":
return self.inputs.multi_modal_data
@@ -687,6 +691,10 @@ class SequenceGroup:
return (self.encoder_seq.prompt_token_ids
if self.encoder_seq is not None else None)
@property
def token_type_ids(self) -> Optional[List[int]]:
return self.first_seq.token_type_ids
@property
def multi_modal_data(self) -> MultiModalDataDict:
return self.first_seq.multi_modal_data
@@ -909,6 +917,7 @@ class SequenceGroupMetadata(
default_factory=lambda: SequenceGroupState())
# "MultiModalDataDict" types. We have to use Any due to msgspec
# doesn't allow to have union of 2 different dicts.
token_type_ids: Optional[List[int]] = None
multi_modal_data: Optional[Any] = None
multi_modal_placeholders: Optional[MultiModalPlaceholderDict] = None
mm_processor_kwargs: Optional[Dict[str, Any]] = None