From 20bbb95113cc6725439584286444db21063ef8ba Mon Sep 17 00:00:00 2001
From: Vuong Hoang <vh@phasefinal.com>
Date: Thu, 10 Sep 2026 07:42:17 -0700
Subject: [PATCH] fix(r49): loss-series collector silently dropped every eval
 record

The LossLog callback filtered on `"loss" in logs`. Trainer emits eval under
`eval_loss` with no `loss` key, so every eval record was discarded and
loss-series.json showed zero eval points while the log a metre away carried
`eval_loss: 3.198`. An artefact that omits data which demonstrably exists reads as
"no eval was run" rather than "the collector dropped it", which is the failure mode
that costs someone a re-run.

Collector now accepts either key. Seed 2 gets it from launch; seed 1 is already
running with the old code, so recover_eval_series.py parses the eval records back
out of its log -- the data was printed, not lost -- and the chain runs that
recovery before cutting the generation arms, so the artefact is complete before
anything reads it.

The two seeds therefore differ in logging code but not in training math: the
callback only affects what is recorded, never what is computed, so the weight
trajectories remain comparable. Noting it because a difference between the two
arms whose spread sets the decision threshold is worth stating even when it is
provably inert.
---
 scripts/r49-corpus/recover_eval_series.py | 38 +++++++++++++++++++++++
 scripts/r49-corpus/train_voice_lora.py    |  7 ++++-
 2 files changed, 44 insertions(+), 1 deletion(-)
 create mode 100644 scripts/r49-corpus/recover_eval_series.py

diff --git a/scripts/r49-corpus/recover_eval_series.py b/scripts/r49-corpus/recover_eval_series.py
new file mode 100644
index 0000000..e7b120e
--- /dev/null
+++ b/scripts/r49-corpus/recover_eval_series.py
@@ -0,0 +1,38 @@
+"""Recover eval records a run's log emitted but its loss-series artefact dropped.
+
+Written because the seed-1 pilot's LossLog callback filtered on `"loss" in logs`,
+which excludes every eval record (Trainer emits `eval_loss` with no `loss` key).
+The data was never lost -- it was printed and not collected -- so it is parsed
+back out of the log rather than re-run. Idempotent: existing eval points are not
+duplicated.
+"""
+import ast, json, re, sys
+from pathlib import Path
+
+log, series = Path(sys.argv[1]), Path(sys.argv[2])
+s = json.loads(series.read_text())
+have = {p["step"] for p in s if "eval_loss" in p}
+found = []
+for m in re.finditer(r"\{[^{}]*'eval_loss'[^{}]*\}", log.read_text(errors="replace")):
+    try:
+        d = ast.literal_eval(m.group(0))
+    except Exception:
+        continue
+    d = {k: (float(v) if isinstance(v, str) and re.fullmatch(r"-?[\d.eE+]+", v) else v)
+         for k, v in d.items()}
+    found.append(d)
+# Trainer logs eval without a step field; epoch is present, so derive step order.
+added = 0
+for i, d in enumerate(found):
+    if d.get("epoch") is None:
+        continue
+    step = d.get("step")
+    if step is None:
+        after = [p["step"] for p in s if "loss" in p and p.get("epoch", 0) <= d["epoch"]]
+        step = max(after) if after else 0
+    if step in have:
+        continue
+    s.append({"step": step, **d}); have.add(step); added += 1
+s.sort(key=lambda p: (p["step"], "eval_loss" in p))
+series.write_text(json.dumps(s, indent=1))
+print(f"  recovered {added} eval records from {log.name} ({len(found)} found in log)")
diff --git a/scripts/r49-corpus/train_voice_lora.py b/scripts/r49-corpus/train_voice_lora.py
index 886d6d5..9e9e8b0 100644
--- a/scripts/r49-corpus/train_voice_lora.py
+++ b/scripts/r49-corpus/train_voice_lora.py
@@ -57,7 +57,12 @@ def pack(tok, records, seq_len):
 class LossLog(TrainerCallback):
     def __init__(self, path): self.path, self.series = path, []
     def on_log(self, args, state, control, logs=None, **kw):
-        if logs and "loss" in logs:
+        # ⚠ `"loss" in logs` DROPS every eval record, because Trainer emits eval
+        # under `eval_loss` with no `loss` key. The series then shows zero eval
+        # points, which reads as "no eval was run" rather than "the collector
+        # dropped it" -- an artefact silently omitting data that exists in the log
+        # a metre away. Accept any record carrying a loss of either kind.
+        if logs and ("loss" in logs or "eval_loss" in logs):
             self.series.append({"step": state.global_step, **{k: v for k, v in logs.items()
                                                               if isinstance(v, (int, float))}})
             Path(self.path).write_text(json.dumps(self.series, indent=1))