memory: snapshot — R49 1-epoch pilot complete and awaiting adjudication; MeroMero A4B quantized, v2 blocked; althing 3.6.2 rolled

Ships the two pending code changes alongside the memory that describes them:
train_voice_lora.py gains --eval-steps/--save-steps (the 3-epoch pilot overfit
with per-epoch eval and save_strategy="no", so the minimum was neither visible
nor recoverable), and the althing post-office compose is pinned to 3.6.2.

Index rewritten: in-flight compressed from 176 lines to the live state, three
detail files added for R49 D1-D3, the MeroMero four-failure chain, and the
althing rollout. Seven closed pre-08-27 entries archived; the guards held back
the rest because they carry open deferred pointers.
This commit is contained in:
vh
2026-09-10 10:25:52 -07:00
parent 44c853cd20
commit b8dbe71a1c
7 changed files with 435 additions and 205 deletions
+15 -1
View File
@@ -80,6 +80,14 @@ def main() -> int:
ap.add_argument("--batch", type=int, default=1)
ap.add_argument("--accum", type=int, default=8)
ap.add_argument("--seed", type=int, default=4919)
# Step-wise eval + save. The 3-epoch pilot showed held-out loss rising every
# epoch (3.198 -> 3.318 -> 3.385) while train loss fell, so the optimum is
# EARLIER than one epoch-boundary eval can see, and `save_strategy="no"` left
# nothing to fall back to. Evaluate and checkpoint on a step grid so the
# minimum is located from data instead of guessed, and so the best adapter
# actually exists on disk when it is found.
ap.add_argument("--eval-steps", type=int, default=0, help="0 = per-epoch")
ap.add_argument("--save-steps", type=int, default=0, help="0 = no intermediate saves")
a = ap.parse_args()
torch.manual_seed(a.seed); random.seed(a.seed)
@@ -132,6 +140,7 @@ def main() -> int:
prov = {"run": "r49-h02-pilot", "base": a.base, "corpus": a.corpus, "corpus_sha256_16": corpus_sha,
"seq_len": a.seq_len, "lora_rank": a.rank, "lora_alpha": 2 * a.rank, "targets": TARGETS,
"lr": a.lr, "epochs": a.epochs, "batch": a.batch, "grad_accum": a.accum, "seed": a.seed,
"eval_steps": a.eval_steps or "per-epoch", "save_steps": a.save_steps or "none",
"train_blocks": len(train_blocks), "train_tokens": tr_tok, "val_blocks": len(val_blocks),
"trainable_params": trainable, "total_params": total,
"trainable_pct": round(100 * trainable / total, 3),
@@ -151,7 +160,12 @@ def main() -> int:
# longer exists. Read the signature, do not assume the 4.x one.
lr_scheduler_type="cosine", warmup_steps=max(1, int(0.03 * steps_per_epoch * int(a.epochs))),
bf16=True, logging_steps=10,
save_strategy="no", eval_strategy="epoch", report_to=[], seed=a.seed,
save_strategy=("steps" if a.save_steps else "no"),
save_steps=(a.save_steps or 500),
save_total_limit=12,
eval_strategy=("steps" if a.eval_steps else "epoch"),
eval_steps=(a.eval_steps or None),
report_to=[], seed=a.seed,
gradient_checkpointing=True, dataloader_num_workers=2,
)
trainer = Trainer(model=model, args=args, train_dataset=Packed(train_blocks),