diff --git a/scripts/r49-corpus/recover_eval_series.py b/scripts/r49-corpus/recover_eval_series.py new file mode 100644 index 0000000..e7b120e --- /dev/null +++ b/scripts/r49-corpus/recover_eval_series.py @@ -0,0 +1,38 @@ +"""Recover eval records a run's log emitted but its loss-series artefact dropped. + +Written because the seed-1 pilot's LossLog callback filtered on `"loss" in logs`, +which excludes every eval record (Trainer emits `eval_loss` with no `loss` key). +The data was never lost -- it was printed and not collected -- so it is parsed +back out of the log rather than re-run. Idempotent: existing eval points are not +duplicated. +""" +import ast, json, re, sys +from pathlib import Path + +log, series = Path(sys.argv[1]), Path(sys.argv[2]) +s = json.loads(series.read_text()) +have = {p["step"] for p in s if "eval_loss" in p} +found = [] +for m in re.finditer(r"\{[^{}]*'eval_loss'[^{}]*\}", log.read_text(errors="replace")): + try: + d = ast.literal_eval(m.group(0)) + except Exception: + continue + d = {k: (float(v) if isinstance(v, str) and re.fullmatch(r"-?[\d.eE+]+", v) else v) + for k, v in d.items()} + found.append(d) +# Trainer logs eval without a step field; epoch is present, so derive step order. +added = 0 +for i, d in enumerate(found): + if d.get("epoch") is None: + continue + step = d.get("step") + if step is None: + after = [p["step"] for p in s if "loss" in p and p.get("epoch", 0) <= d["epoch"]] + step = max(after) if after else 0 + if step in have: + continue + s.append({"step": step, **d}); have.add(step); added += 1 +s.sort(key=lambda p: (p["step"], "eval_loss" in p)) +series.write_text(json.dumps(s, indent=1)) +print(f" recovered {added} eval records from {log.name} ({len(found)} found in log)") diff --git a/scripts/r49-corpus/train_voice_lora.py b/scripts/r49-corpus/train_voice_lora.py index 886d6d5..9e9e8b0 100644 --- a/scripts/r49-corpus/train_voice_lora.py +++ b/scripts/r49-corpus/train_voice_lora.py @@ -57,7 +57,12 @@ def pack(tok, records, seq_len): class LossLog(TrainerCallback): def __init__(self, path): self.path, self.series = path, [] def on_log(self, args, state, control, logs=None, **kw): - if logs and "loss" in logs: + # ⚠ `"loss" in logs` DROPS every eval record, because Trainer emits eval + # under `eval_loss` with no `loss` key. The series then shows zero eval + # points, which reads as "no eval was run" rather than "the collector + # dropped it" -- an artefact silently omitting data that exists in the log + # a metre away. Accept any record carrying a loss of either kind. + if logs and ("loss" in logs or "eval_loss" in logs): self.series.append({"step": state.global_step, **{k: v for k, v in logs.items() if isinstance(v, (int, float))}}) Path(self.path).write_text(json.dumps(self.series, indent=1))