[Bugfix] Avoid duplicate k-proj weight emission in helper (#34142)
Signed-off-by: Artus KG <artuskg@gmail.com>
This commit is contained in:
committed by
GitHub
parent
285bab4752
commit
eadb4e868b
@@ -958,8 +958,8 @@ def _create_fake_bias_for_k_proj(
|
||||
So that the bias for k_proj in qkv_proj can be initialized with zeros.
|
||||
"""
|
||||
for name, weight in weights:
|
||||
yield name, weight
|
||||
if name.endswith(fake_bias_key_name):
|
||||
bias = torch.zeros(weight.size(0))
|
||||
bias_name = name.replace("weight", "bias")
|
||||
yield from [(name, weight), (bias_name, bias)]
|
||||
yield name, weight
|
||||
yield bias_name, bias
|
||||
|
||||
Reference in New Issue
Block a user