diff --git a/scripts/yarros-corpus/score_beats.py b/scripts/yarros-corpus/score_beats.py index b89a19a..7525995 100644 --- a/scripts/yarros-corpus/score_beats.py +++ b/scripts/yarros-corpus/score_beats.py @@ -11,8 +11,47 @@ without giving back the direction-following it bought? The raw-text instruct arm profile is on-beat strong, in-band weak, ran-on frequent. In-band is therefore the axis the pilot exists to move, and it is the axis the rule keys on. -DECISION RULE — pairs replace raw-text for Skaldsong iff ALL THREE hold, same harness, -same n, same box, every arm re-measured in one session: +⚠⚠ AMENDED 2026-09-15 BY THE OPERATOR, AFTER THE v1 RULE HAD BEEN RUN AND REPORTED. +The amendment is recorded here rather than applied silently, because a frozen rule edited +in place between a read and a conclusion is indistinguishable from a rule chosen to produce +a verdict. What changed, and why: + + v1 (commit 713e83d, pre-registered before any arm was read) gated on IN-BAND, ON-BEAT and + RAN-ON. It returned DO-NOT-SCALE at n=120: in-band +0.08 against a 0.233 floor (the + +0.45 seen at n=20 was noise), on-beat -0.27, ran-on -0.35. + + The defect in v1: it gated on metrics the UNADAPTED carrier already maxes. Measured, + n=120 -- base-unadapted in-band 0.96. Instruction-following is a property Qwen3-4B-Instruct + ships with, so those three axes can only detect DAMAGE that training does. They cannot + detect the thing an adapter exists to buy, which is VOICE, and v1 contained no voice term + at all. It was a well-formed rule measuring the wrong question. + + v2 therefore gates on the two axes that distinguish the arms, and keeps one v1 term as a + guard rather than a target: + A. VOICE -- delta_cb vs held-out Yarros must beat the base-unadapted control by MORE + than the measured noise floor. (pairs +0.230 vs floor 0.153; rawtext + +0.141, which does NOT clear -- so this term discriminates.) + B. NOT COPIED -- verbatim 8-gram overlap with the training corpus must not exceed the + base-unadapted negative control by a meaningful margin. delta_cb is blind + to regurgitation and a memorising arm scores near the same-author target, + so A without B is a trap. (pairs 0.10 vs control 0.07; rawtext 0.14.) + C. NO DAMAGE -- ran-on must not get worse than raw-text by more than the floor. + Retained because overshoot is the one behavioural axis where the adapters + actually differ from the base carrier. + + NOT carried into v2: in-band (unresolvable -- base maxes it) and on-beat. ⚠ on-beat's + -0.27 was OUTSIDE the floor and is a REAL signal by the keyword proxy; it is dropped from + the gate, not explained away. The open question is whether the proxy punishes + dramatisation -- a generation rendering "she mocks him" as actual mockery scores zero for + "mocks" -- and three read samples is an anecdote, not an answer. It stays an open + follow-up against the full run. + +Authorised by the operator 2026-09-15 ("amend the rule and run the full corpus"). A and B +are evaluated by voice_distance.py and memorization_check.py, which this script does not +recompute; it reports C and prints the v1 table for continuity. + +v1 DECISION RULE (superseded, retained verbatim) — pairs replace raw-text for Skaldsong iff +ALL THREE hold, same harness, same n, same box, every arm re-measured in one session: 1. in-band rate is HIGHER than the raw-text arm by MORE than the pooled within-arm seed spread. A gain inside the spread is noise, not a fix. diff --git a/scripts/yarros-corpus/train_pairs_lora.py b/scripts/yarros-corpus/train_pairs_lora.py index c6f1f67..7df7576 100644 --- a/scripts/yarros-corpus/train_pairs_lora.py +++ b/scripts/yarros-corpus/train_pairs_lora.py @@ -135,6 +135,12 @@ def main() -> int: ap.add_argument("--seed", type=int, default=4919) ap.add_argument("--eval-steps", type=int, default=25) ap.add_argument("--save-steps", type=int, default=25) + # ⚠ Must exceed the number of saves the run will make. The pilot took 9 evals and the + # loss minimum was at the 6th; a full run at 1672 steps saving every 50 makes 33, and + # the hardcoded limit of 12 would have PRUNED an early minimum before it could be read. + # The 4B rung already proved the best checkpoint is not the last one -- a retention + # policy that silently deletes it turns that lesson into a trap rather than a guard. + ap.add_argument("--save-total-limit", type=int, default=12) a = ap.parse_args() torch.manual_seed(a.seed); random.seed(a.seed) @@ -220,7 +226,7 @@ def main() -> int: lr_scheduler_type="cosine", warmup_steps=max(1, int(0.03 * steps_per_epoch * int(a.epochs))), bf16=True, logging_steps=10, - save_strategy="steps", save_steps=a.save_steps, save_total_limit=12, + save_strategy="steps", save_steps=a.save_steps, save_total_limit=a.save_total_limit, eval_strategy="steps", eval_steps=a.eval_steps, report_to=[], seed=a.seed, gradient_checkpointing=True, dataloader_num_workers=2,