biondizzle/vllm
1
0
Fork 0
Code Issues Pull Requests Actions 2 Packages Projects Releases Wiki Activity

[Bugfix] Correct LayerNorm epsilon parameter in modernbert.py (#27008)

Browse Source 
Signed-off-by: bogdanm <152898065+bogdan01m@users.noreply.github.com>
Co-authored-by: gemini-code-assist[bot] <176961590+gemini-code-assist[bot]@users.noreply.github.com>
This commit is contained in:
bogdanm
2025-10-16 17:27:44 +05:00
committed by GitHub
parent 00417f4e44
commit dcbb3f1871
1 changed files with 5 additions and 2 deletions
Download Patch File Download Diff File Expand all files Collapse all files

7
vllm/model_executor/models/modernbert.py
View File

@@ -39,9 +39,12 @@ class ModernBertEmbeddings(nn.Module):
self.tok_embeddings = VocabParallelEmbedding( self.tok_embeddings = VocabParallelEmbedding(
config.vocab_size, config.hidden_size config.vocab_size, config.hidden_size
) )
self.norm = nn.LayerNorm( eps = (
config.hidden_size, eps=config.layer_norm_eps, bias=config.norm_bias getattr(config, "norm_eps", None)
or getattr(config, "layer_norm_eps", None)
or 1e-5
) )
self.norm = nn.LayerNorm(config.hidden_size, eps=eps, bias=config.norm_bias)
def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor: def get_input_embeddings(self, input_ids: torch.Tensor) -> torch.Tensor:
return self.tok_embeddings(input_ids) return self.tok_embeddings(input_ids)