[model] Add support for openPangu7B-VL (#32449)
Signed-off-by: hujiaxin <524446785@qq.com> Signed-off-by: Emilie1001 <79921183+Emilie1001@users.noreply.github.com> Co-authored-by: Emilie1001 <79921183+Emilie1001@users.noreply.github.com>
This commit is contained in:
@@ -537,10 +537,16 @@ class OpenPanguEmbeddedAttention(nn.Module):
|
||||
if is_gguf and config.model_type == "PanguEmbedded":
|
||||
is_neox_style = False
|
||||
|
||||
rope_parameters = config.rope_parameters or {}
|
||||
if rope_parameters is not None and rope_parameters.get(
|
||||
"mrope_interleaved", False
|
||||
):
|
||||
rope_parameters["rope_type"] = "openpangu"
|
||||
|
||||
self.rotary_emb = get_rope(
|
||||
self.head_dim,
|
||||
max_position=self.max_position_embeddings,
|
||||
rope_parameters=config.rope_parameters,
|
||||
rope_parameters=rope_parameters,
|
||||
is_neox_style=is_neox_style,
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user