[Model] Initialize support for InternVL2 series models (#6514)

Co-authored-by: Roger Wang <ywang@roblox.com>
This commit is contained in:
Isotr0py
2024-07-29 18:16:30 +08:00
committed by GitHub
parent 3eeb148f46
commit 7cbd9ec7a9
14 changed files with 1042 additions and 6 deletions

View File

@@ -219,14 +219,22 @@ class InternLM2Model(nn.Module):
])
self.norm = RMSNorm(config.hidden_size, eps=config.rms_norm_eps)
def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.tok_embeddings(input_ids)
def forward(
self,
input_ids: torch.Tensor,
positions: torch.Tensor,
kv_caches: List[torch.Tensor],
attn_metadata: AttentionMetadata,
intermediate_tensors: IntermediateTensors = None,
inputs_embeds: Optional[torch.Tensor] = None,
) -> torch.Tensor:
hidden_states = self.tok_embeddings(input_ids)
if inputs_embeds is not None:
hidden_states = inputs_embeds
else:
hidden_states = self.tok_embeddings(input_ids)
residual = None
for i in range(len(self.layers)):
layer = self.layers[i]