memory: snapshot — R49 1-epoch pilot complete and awaiting adjudication; MeroMero A4B quantized, v2 blocked; althing 3.6.2 rolled
Ships the two pending code changes alongside the memory that describes them: train_voice_lora.py gains --eval-steps/--save-steps (the 3-epoch pilot overfit with per-epoch eval and save_strategy="no", so the minimum was neither visible nor recoverable), and the althing post-office compose is pinned to 3.6.2. Index rewritten: in-flight compressed from 176 lines to the live state, three detail files added for R49 D1-D3, the MeroMero four-failure chain, and the althing rollout. Seven closed pre-08-27 entries archived; the guards held back the rest because they carry open deferred pointers.
This commit is contained in:
@@ -80,6 +80,14 @@ def main() -> int:
|
||||
ap.add_argument("--batch", type=int, default=1)
|
||||
ap.add_argument("--accum", type=int, default=8)
|
||||
ap.add_argument("--seed", type=int, default=4919)
|
||||
# Step-wise eval + save. The 3-epoch pilot showed held-out loss rising every
|
||||
# epoch (3.198 -> 3.318 -> 3.385) while train loss fell, so the optimum is
|
||||
# EARLIER than one epoch-boundary eval can see, and `save_strategy="no"` left
|
||||
# nothing to fall back to. Evaluate and checkpoint on a step grid so the
|
||||
# minimum is located from data instead of guessed, and so the best adapter
|
||||
# actually exists on disk when it is found.
|
||||
ap.add_argument("--eval-steps", type=int, default=0, help="0 = per-epoch")
|
||||
ap.add_argument("--save-steps", type=int, default=0, help="0 = no intermediate saves")
|
||||
a = ap.parse_args()
|
||||
|
||||
torch.manual_seed(a.seed); random.seed(a.seed)
|
||||
@@ -132,6 +140,7 @@ def main() -> int:
|
||||
prov = {"run": "r49-h02-pilot", "base": a.base, "corpus": a.corpus, "corpus_sha256_16": corpus_sha,
|
||||
"seq_len": a.seq_len, "lora_rank": a.rank, "lora_alpha": 2 * a.rank, "targets": TARGETS,
|
||||
"lr": a.lr, "epochs": a.epochs, "batch": a.batch, "grad_accum": a.accum, "seed": a.seed,
|
||||
"eval_steps": a.eval_steps or "per-epoch", "save_steps": a.save_steps or "none",
|
||||
"train_blocks": len(train_blocks), "train_tokens": tr_tok, "val_blocks": len(val_blocks),
|
||||
"trainable_params": trainable, "total_params": total,
|
||||
"trainable_pct": round(100 * trainable / total, 3),
|
||||
@@ -151,7 +160,12 @@ def main() -> int:
|
||||
# longer exists. Read the signature, do not assume the 4.x one.
|
||||
lr_scheduler_type="cosine", warmup_steps=max(1, int(0.03 * steps_per_epoch * int(a.epochs))),
|
||||
bf16=True, logging_steps=10,
|
||||
save_strategy="no", eval_strategy="epoch", report_to=[], seed=a.seed,
|
||||
save_strategy=("steps" if a.save_steps else "no"),
|
||||
save_steps=(a.save_steps or 500),
|
||||
save_total_limit=12,
|
||||
eval_strategy=("steps" if a.eval_steps else "epoch"),
|
||||
eval_steps=(a.eval_steps or None),
|
||||
report_to=[], seed=a.seed,
|
||||
gradient_checkpointing=True, dataloader_num_workers=2,
|
||||
)
|
||||
trainer = Trainer(model=model, args=args, train_dataset=Packed(train_blocks),
|
||||
|
||||
Reference in New Issue
Block a user