fam/nanochat-omni
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: initialize smear and backout lambdas in init_weights

Browse Source
This commit is contained in:
Marcin Bogdanski
2026-04-03 20:39:55 +00:00
parent a445144d39
commit 94b73ad29a
1 changed files with 2 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
nanochat/gpt.py
+2
View File
@@ -237,6 +237,8 @@ class GPT(nn.Module):
# Decaying x0 init: earlier layers get more input embedding blending # Decaying x0 init: earlier layers get more input embedding blending
for i in range(n_layer): for i in range(n_layer):
self.x0_lambdas.data[i] = 0.20 - (0.15 * i / max(n_layer - 1, 1)) self.x0_lambdas.data[i] = 0.20 - (0.15 * i / max(n_layer - 1, 1))
self.smear_lambda.fill_(0.0)
self.backout_lambda.fill_(0.2)
# Value embeddings (init like c_v: uniform with same std) # Value embeddings (init like c_v: uniform with same std)
for ve in self.value_embeds.values(): for ve in self.value_embeds.values():