Revert "Refactor llama family models (#2637)" (#2851)

This reverts commit 5c976a7e1a.
This commit is contained in:
Philipp Moritz
2024-02-13 09:24:59 -08:00
committed by GitHub
parent 5c976a7e1a
commit ea356004d4
17 changed files with 2720 additions and 236 deletions

View File

@@ -7,31 +7,6 @@ import torch.nn as nn
from vllm._C import ops
class LayerNorm(nn.LayerNorm):
def __init__(
self,
hidden_size: int,
eps: float = 1e-6,
) -> None:
super().__init__(hidden_size, eps=eps)
def forward(
self,
x: torch.Tensor,
residual: Optional[torch.Tensor] = None,
) -> Union[torch.Tensor, Tuple[torch.Tensor, torch.Tensor]]:
"""normalization."""
if residual is not None:
x = x + residual
residual = x
x = super().forward(x)
if residual is None:
return x
else:
return x, residual
class RMSNorm(nn.Module):
"""Root mean square normalization.