74 lines
2.7 KiB
Python
Executable File
74 lines
2.7 KiB
Python
Executable File
# SPDX-License-Identifier: Apache-2.0
|
|
# SPDX-FileCopyrightText: Copyright contributors to the vLLM project
|
|
from transformers import PretrainedConfig
|
|
|
|
|
|
class DeepseekV4Config(PretrainedConfig):
|
|
model_type = "deepseek_v4"
|
|
|
|
def __init__(
|
|
self,
|
|
vocab_size: int = 129024,
|
|
hidden_size: int = 7168,
|
|
intermediate_size: int = 18432,
|
|
moe_intermediate_size: int = 1408,
|
|
num_hidden_layers: int = 61,
|
|
num_attention_heads: int = 128,
|
|
num_key_value_heads: int = 128,
|
|
n_routed_experts: int = 256,
|
|
n_shared_experts: int = 1,
|
|
num_experts_per_tok: int = 8,
|
|
head_dim: int = 256,
|
|
q_lora_rank: int = 1536,
|
|
o_lora_rank: int = 1536,
|
|
qk_rope_head_dim: int = 64,
|
|
o_groups: int = 128,
|
|
rms_norm_eps: float = 1e-6,
|
|
rope_theta: float = 10000.0,
|
|
rope_scaling: dict | None = None,
|
|
max_position_embeddings: int = 163840,
|
|
hidden_act: str = "silu",
|
|
norm_topk_prob: bool = True,
|
|
compress_ratios: list | None = None,
|
|
compress_rope_theta: float = 10000.0,
|
|
num_hash_layers: int = 0,
|
|
hc_mult: float = 1.0,
|
|
hc_eps: float = 1e-6,
|
|
hc_sinkhorn_iters: int = 8,
|
|
swiglu_limit: int = 0,
|
|
index_topk: int = 0,
|
|
num_nextn_predict_layers: int = 0,
|
|
**kwargs,
|
|
):
|
|
self.vocab_size = vocab_size
|
|
self.hidden_size = hidden_size
|
|
self.intermediate_size = intermediate_size
|
|
self.moe_intermediate_size = moe_intermediate_size
|
|
self.num_hidden_layers = num_hidden_layers
|
|
self.num_attention_heads = num_attention_heads
|
|
self.num_key_value_heads = num_key_value_heads
|
|
self.n_routed_experts = n_routed_experts
|
|
self.n_shared_experts = n_shared_experts
|
|
self.num_experts_per_tok = num_experts_per_tok
|
|
self.head_dim = head_dim
|
|
self.q_lora_rank = q_lora_rank
|
|
self.o_lora_rank = o_lora_rank
|
|
self.qk_rope_head_dim = qk_rope_head_dim
|
|
self.o_groups = o_groups
|
|
self.rms_norm_eps = rms_norm_eps
|
|
self.rope_theta = rope_theta
|
|
self.rope_scaling = rope_scaling
|
|
self.max_position_embeddings = max_position_embeddings
|
|
self.hidden_act = hidden_act
|
|
self.norm_topk_prob = norm_topk_prob
|
|
self.compress_ratios = compress_ratios or [1] * num_hidden_layers
|
|
self.compress_rope_theta = compress_rope_theta
|
|
self.num_hash_layers = num_hash_layers
|
|
self.hc_mult = hc_mult
|
|
self.hc_eps = hc_eps
|
|
self.hc_sinkhorn_iters = hc_sinkhorn_iters
|
|
self.swiglu_limit = swiglu_limit
|
|
self.index_topk = index_topk
|
|
self.num_nextn_predict_layers = num_nextn_predict_layers
|
|
super().__init__(**kwargs)
|