aye carumba2

deepseek_v4.py

@@ -8,10 +8,66 @@ class DeepseekV4Config(PretrainedConfig):
 
     def __init__(
         self,
-        max_position_embeddings: int = 163840,
+        vocab_size: int = 129024,
+        hidden_size: int = 7168,
+        intermediate_size: int = 18432,
+        moe_intermediate_size: int = 1408,
+        num_hidden_layers: int = 61,
+        num_attention_heads: int = 128,
+        num_key_value_heads: int = 128,
+        n_routed_experts: int = 256,
+        n_shared_experts: int = 1,
+        num_experts_per_tok: int = 8,
+        head_dim: int = 256,
+        q_lora_rank: int = 1536,
+        o_lora_rank: int = 1536,
+        qk_rope_head_dim: int = 64,
+        o_groups: int = 128,
+        rms_norm_eps: float = 1e-6,
+        rope_theta: float = 10000.0,
         rope_scaling: dict | None = None,
+        max_position_embeddings: int = 163840,
+        hidden_act: str = "silu",
+        norm_topk_prob: bool = True,
+        compress_ratios: list | None = None,
+        compress_rope_theta: float = 10000.0,
+        num_hash_layers: int = 0,
+        hc_mult: float = 1.0,
+        hc_eps: float = 1e-6,
+        hc_sinkhorn_iters: int = 8,
+        swiglu_limit: int = 0,
+        index_topk: int = 0,
+        num_nextn_predict_layers: int = 0,
         **kwargs,
     ):
-        self.max_position_embeddings = max_position_embeddings
+        self.vocab_size = vocab_size
+        self.hidden_size = hidden_size
+        self.intermediate_size = intermediate_size
+        self.moe_intermediate_size = moe_intermediate_size
+        self.num_hidden_layers = num_hidden_layers
+        self.num_attention_heads = num_attention_heads
+        self.num_key_value_heads = num_key_value_heads
+        self.n_routed_experts = n_routed_experts
+        self.n_shared_experts = n_shared_experts
+        self.num_experts_per_tok = num_experts_per_tok
+        self.head_dim = head_dim
+        self.q_lora_rank = q_lora_rank
+        self.o_lora_rank = o_lora_rank
+        self.qk_rope_head_dim = qk_rope_head_dim
+        self.o_groups = o_groups
+        self.rms_norm_eps = rms_norm_eps
+        self.rope_theta = rope_theta
         self.rope_scaling = rope_scaling
+        self.max_position_embeddings = max_position_embeddings
+        self.hidden_act = hidden_act
+        self.norm_topk_prob = norm_topk_prob
+        self.compress_ratios = compress_ratios or [1] * num_hidden_layers
+        self.compress_rope_theta = compress_rope_theta
+        self.num_hash_layers = num_hash_layers
+        self.hc_mult = hc_mult
+        self.hc_eps = hc_eps
+        self.hc_sinkhorn_iters = hc_sinkhorn_iters
+        self.swiglu_limit = swiglu_limit
+        self.index_topk = index_topk
+        self.num_nextn_predict_layers = num_nextn_predict_layers
         super().__init__(**kwargs)