131 lines
5.9 KiB
Python
131 lines
5.9 KiB
Python
"""Score beat arms against a decision rule fixed BEFORE any arm's output was read.
|
|
|
|
⚠ THE RULE BELOW IS PRE-REGISTERED. It was written and committed while the generation
|
|
chain was still running, for the reason brokkr's frozen adjudication exists: a threshold
|
|
chosen after seeing the numbers is a threshold chosen to produce a verdict. If it needs to
|
|
change, change it in a commit that says so and re-run every arm -- do not edit it in place
|
|
between a read and a conclusion.
|
|
|
|
THE QUESTION: does pair-SFT fix the length/close discipline that raw-text training cost,
|
|
without giving back the direction-following it bought? The raw-text instruct arm's known
|
|
profile is on-beat strong, in-band weak, ran-on frequent. In-band is therefore the axis the
|
|
pilot exists to move, and it is the axis the rule keys on.
|
|
|
|
DECISION RULE — pairs replace raw-text for Skaldsong iff ALL THREE hold, same harness,
|
|
same n, same box, every arm re-measured in one session:
|
|
|
|
1. in-band rate is HIGHER than the raw-text arm by MORE than the pooled within-arm
|
|
seed spread. A gain inside the spread is noise, not a fix.
|
|
2. on-beat coverage is not WORSE than the raw-text arm by more than that same spread --
|
|
it must not have bought length by losing direction.
|
|
3. ran-on rate is not HIGHER than the raw-text arm by more than that spread.
|
|
|
|
(1) fails -> the pilot did not do its job. Do not scale.
|
|
(1) holds, 2 or 3 no -> mixed result. Surface to the operator; do not auto-scale.
|
|
all three -> scale to the full corpus.
|
|
|
|
on-beat is scored as COVERAGE (fraction of the beat's content keywords that surface in the
|
|
paragraph), thresholded at >= 0.5 for the binary. Both the coverage mean and the binary are
|
|
reported: a rule that reads only the binary hides a large move inside a threshold, and a
|
|
rule that reads only the mean hides a bimodal arm.
|
|
|
|
NOISE FLOOR: the spread is the max-minus-min of a metric across the four SEEDS within an
|
|
arm, pooled (max) over arms. It is measured per run and printed with the verdict, because a
|
|
between-arm gap smaller than it is not a finding. This is the A-vs-A floor; it is not the
|
|
same thing as the distance to real Yarros and must not be reported as if it were.
|
|
"""
|
|
from __future__ import annotations
|
|
import argparse, json, math, statistics as st
|
|
from collections import defaultdict
|
|
from pathlib import Path
|
|
|
|
ON_BEAT_COVERAGE = 0.5
|
|
|
|
|
|
def load(path: Path) -> list[dict]:
|
|
return [json.loads(l) for l in path.read_text(encoding="utf-8").splitlines() if l.strip()]
|
|
|
|
|
|
def per_seed(rows: list[dict]) -> dict[int, dict]:
|
|
by = defaultdict(list)
|
|
for r in rows:
|
|
by[r["seed"]].append(r)
|
|
out = {}
|
|
for seed, rs in by.items():
|
|
cov = [(r["keyword_hits"] / len(r["beat_keywords"])) if r["beat_keywords"] else 0.0
|
|
for r in rs]
|
|
out[seed] = {
|
|
"n": len(rs),
|
|
"in_band": sum(1 for r in rs if r["in_band"]) / len(rs),
|
|
"ran_on": sum(1 for r in rs if r["ran_on"]) / len(rs),
|
|
"coverage": st.mean(cov),
|
|
"on_beat": sum(1 for c in cov if c >= ON_BEAT_COVERAGE) / len(cov),
|
|
"words_median": st.median(r["words"] for r in rs),
|
|
}
|
|
return out
|
|
|
|
|
|
def summarise(name: str, rows: list[dict]) -> dict:
|
|
seeds = per_seed(rows)
|
|
agg = {"arm": name, "n": len(rows), "seeds": len(seeds)}
|
|
for k in ("in_band", "ran_on", "coverage", "on_beat", "words_median"):
|
|
vals = [s[k] for s in seeds.values()]
|
|
agg[k] = st.mean(vals)
|
|
agg[k + "_spread"] = max(vals) - min(vals)
|
|
return agg
|
|
|
|
|
|
def main() -> int:
|
|
ap = argparse.ArgumentParser()
|
|
ap.add_argument("--arm", action="append", required=True, metavar="NAME=PATH")
|
|
ap.add_argument("--baseline", required=True, help="arm name the rule compares against")
|
|
ap.add_argument("--candidate", required=True)
|
|
ap.add_argument("--out", default=None)
|
|
a = ap.parse_args()
|
|
|
|
arms = {}
|
|
for spec in a.arm:
|
|
name, path = spec.split("=", 1)
|
|
arms[name] = summarise(name, load(Path(path)))
|
|
|
|
floor = max(max(v[k + "_spread"] for k in ("in_band", "ran_on", "on_beat", "coverage"))
|
|
for v in arms.values())
|
|
|
|
hdr = f"{'arm':<28} {'n':>3} {'in-band':>8} {'on-beat':>8} {'cover':>7} {'ran-on':>7} {'words':>6}"
|
|
print(hdr); print("-" * len(hdr))
|
|
for v in arms.values():
|
|
print(f"{v['arm']:<28} {v['n']:>3} {v['in_band']:>8.2f} {v['on_beat']:>8.2f} "
|
|
f"{v['coverage']:>7.2f} {v['ran_on']:>7.2f} {v['words_median']:>6.0f}")
|
|
print(f"\nnoise floor (max within-arm spread across seeds): {floor:.3f}")
|
|
print("⚠ any between-arm gap at or under that is NOT a finding\n")
|
|
|
|
b, c = arms[a.baseline], arms[a.candidate]
|
|
d_band = c["in_band"] - b["in_band"]
|
|
d_beat = c["on_beat"] - b["on_beat"]
|
|
d_ran = c["ran_on"] - b["ran_on"]
|
|
c1 = d_band > floor
|
|
c2 = d_beat >= -floor
|
|
c3 = d_ran <= floor
|
|
print(f"1. in-band {c['in_band']:.2f} vs {b['in_band']:.2f} delta {d_band:+.2f} "
|
|
f"{'PASS' if c1 else 'FAIL'} (needs > +{floor:.3f})")
|
|
print(f"2. on-beat {c['on_beat']:.2f} vs {b['on_beat']:.2f} delta {d_beat:+.2f} "
|
|
f"{'PASS' if c2 else 'FAIL'} (needs >= -{floor:.3f})")
|
|
print(f"3. ran-on {c['ran_on']:.2f} vs {b['ran_on']:.2f} delta {d_ran:+.2f} "
|
|
f"{'PASS' if c3 else 'FAIL'} (needs <= +{floor:.3f})")
|
|
|
|
verdict = ("SCALE" if (c1 and c2 and c3) else
|
|
"DO-NOT-SCALE" if not c1 else "MIXED-SURFACE-TO-OPERATOR")
|
|
print(f"\nVERDICT: {verdict}")
|
|
if a.out:
|
|
Path(a.out).write_text(json.dumps(
|
|
{"arms": arms, "noise_floor": floor, "on_beat_coverage_threshold": ON_BEAT_COVERAGE,
|
|
"baseline": a.baseline, "candidate": a.candidate,
|
|
"deltas": {"in_band": d_band, "on_beat": d_beat, "ran_on": d_ran},
|
|
"criteria": {"in_band_gt_floor": c1, "on_beat_not_worse": c2, "ran_on_not_worse": c3},
|
|
"verdict": verdict}, indent=2), encoding="utf-8")
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
raise SystemExit(main())
|