vh
5558d9c7d3
Adds the Yarros-side evaluation the training exists to justify: does the adapter move arbitrary prose toward Yarros, and can the instruct arm still expand a beat to a paragraph on direction. Yarros-flavoured voice prompts (modern/neutral/ romantasy tiers so any Yarros voice in the modern tier is adapter-attributable, not prompt-supplied) and a Yarros-register beat SYS on the chat generator. voice_distance.py is the honest slice of adjudication that needs no seat: Burrows's Delta over character bigrams against held-out Yarros. Its first cut mis-framed the noise floor — it used the same-author distance (held-out vs itself) as the between-arm significance threshold, which is the target, not the threshold. Fixed to the measured floor: the within-arm seed spread, which is this metric's sampling variance at this sample size, computed from the two seeds already generated rather than assumed. Result on the built corpus, ordering base-125-tuned < instruct-tuned < base-unadapted, both adapters clearing the 0.046 measured floor (base +0.157, instruct +0.076), and the ordering corroborating the independent held-out loss ordering (Base below Instruct). One seed-pair per arm, so it corroborates rather than settles; the full frozen adjudication still needs a romantasy control panel, a second seed, and the gen seat for the beat-incumbent leg.
115 lines
6.5 KiB
Python
115 lines
6.5 KiB
Python
"""Build the BabyYarros evaluation booth: voice A/B + beat->paragraph + delta_cb.
|
||
|
||
Reads the eval jsonls and the voice_distance summary, emits a self-contained
|
||
index.html for the Booth. Three panels:
|
||
A VOICE — each opening line, the three arms' continuations side by side, so the
|
||
operator can SEE whether the adapter pulls arbitrary prose toward Yarros.
|
||
B BEAT -> PARAGRAPH — the Skaldsong question: does the Instruct arm still take
|
||
direction (on-beat / in-band / ran-on) after training on raw Yarros text.
|
||
C delta_cb — the seat-free relative measure, with its A-vs-A noise floor.
|
||
"""
|
||
from __future__ import annotations
|
||
import html, json, re, sys
|
||
from pathlib import Path
|
||
|
||
D = Path(sys.argv[1]) # yarros-eval dir
|
||
DIST = Path(sys.argv[2]) if len(sys.argv) > 2 else None # distance stdout captured to a file
|
||
OUT = Path(sys.argv[3]) if len(sys.argv) > 3 else (D / "index.html")
|
||
|
||
def rows(f):
|
||
p = D / f
|
||
return [json.loads(l) for l in p.read_text(encoding="utf-8").splitlines()] if p.exists() else []
|
||
|
||
ARMS = [("base-unadapted", "Base · no adapter (control)"),
|
||
("base-125-tuned", "Base · Yarros LoRA (ckpt-125)"),
|
||
("instruct-tuned", "Instruct · Yarros LoRA")]
|
||
|
||
voice = {}
|
||
for key, _ in ARMS:
|
||
for r in rows(f"voice.{key}.jsonl"):
|
||
voice.setdefault((r["id"], r["seed"]), {})[key] = r
|
||
prompts = {}
|
||
for key, _ in ARMS:
|
||
for r in rows(f"voice.{key}.jsonl"):
|
||
prompts[r["id"]] = (r["tier"], r["prompt"])
|
||
|
||
def wc(t): return len(t.split())
|
||
|
||
def beat_flags(r):
|
||
kh, kws = r.get("keyword_hits", 0), r.get("beat_keywords", [])
|
||
ratio = kh / max(len(kws), 1)
|
||
on_beat = kh >= 1 and ratio >= 0.34 # at least a third of the beat's content words rendered
|
||
return on_beat, f"{kh}/{len(kws)}", r.get("in_band"), r.get("ran_on"), r.get("words", 0)
|
||
|
||
beats = rows("beats.instruct.jsonl")
|
||
|
||
esc = lambda s: html.escape(s or "")
|
||
parts = ["""<title>BabyYarros — voice & beats</title>
|
||
<style>
|
||
:root{--bg:#faf8f5;--fg:#1c1a17;--mut:#6b645c;--line:#e4ded4;--card:#fff;--acc:#8a5a2b;--good:#2e7d43;--bad:#b3402f}
|
||
:root:not([data-theme=light]) @media (prefers-color-scheme:dark){}
|
||
@media (prefers-color-scheme:dark){:root:not([data-theme=light]){--bg:#17150f;--fg:#ece7df;--mut:#a49a8c;--line:#332e26;--card:#201d16;--acc:#d69a5c;--good:#6ecb86;--bad:#e8836f}}
|
||
:root[data-theme=dark]{--bg:#17150f;--fg:#ece7df;--mut:#a49a8c;--line:#332e26;--card:#201d16;--acc:#d69a5c;--good:#6ecb86;--bad:#e8836f}
|
||
body{background:var(--bg);color:var(--fg);font:15px/1.55 -apple-system,BlinkMacSystemFont,'Segoe UI',Roboto,sans-serif;margin:0;padding:2rem}
|
||
h1{font-size:1.5rem;margin:0 0 .2rem} .sub{color:var(--mut);margin:0 0 1.5rem}
|
||
h2{font-size:1.15rem;margin:2rem 0 .6rem;border-bottom:2px solid var(--acc);padding-bottom:.3rem}
|
||
.prompt{color:var(--acc);font-weight:600;margin:1.2rem 0 .4rem}.tier{color:var(--mut);font-size:.8rem;font-weight:400}
|
||
.grid{display:grid;grid-template-columns:repeat(3,1fr);gap:.8rem}
|
||
@media(max-width:900px){.grid{grid-template-columns:1fr}}
|
||
.card{background:var(--card);border:1px solid var(--line);border-radius:8px;padding:.7rem .8rem}
|
||
.arm{font-size:.75rem;color:var(--mut);text-transform:uppercase;letter-spacing:.04em;margin-bottom:.35rem}
|
||
.txt{white-space:pre-wrap;font-size:.92rem}.wc{color:var(--mut);font-size:.75rem;margin-top:.4rem}
|
||
table{border-collapse:collapse;width:100%;margin:.5rem 0}td,th{border:1px solid var(--line);padding:.35rem .5rem;text-align:left;font-size:.9rem}
|
||
.beat{color:var(--acc);font-weight:600}.ok{color:var(--good);font-weight:600}.no{color:var(--bad);font-weight:600}
|
||
pre.dist{background:var(--card);border:1px solid var(--line);border-radius:8px;padding:1rem;overflow-x:auto;font-size:.85rem}
|
||
</style>
|
||
<h1>BabyYarros — does the voice transfer, can it do beats?</h1>
|
||
<p class="sub">Qwen3-4B, one epoch on the leak-gated Yarros corpus. Base LoRA = ckpt-125 (its held-out minimum). Generated on gx10, same harness per arm. Not the frozen adjudication — the voice A/B and beat test the operator asked to see.</p>
|
||
"""]
|
||
|
||
# Panel A — voice
|
||
parts.append('<h2>A · Voice — arbitrary opening line, three arms continue it</h2>')
|
||
order = ["a1","a2","a3","b1","b2","b3","c1","c2","c3"]
|
||
seedpick = 1234
|
||
for pid in order:
|
||
if pid not in prompts: continue
|
||
tier, ptext = prompts[pid]
|
||
parts.append(f'<div class="prompt">{esc(ptext)} <span class="tier">· {tier}</span></div><div class="grid">')
|
||
cell = voice.get((pid, seedpick), {})
|
||
for key, label in ARMS:
|
||
r = cell.get(key)
|
||
body = esc(r["continuation"].strip())[:1400] if r else "<em>—</em>"
|
||
w = wc(r["continuation"]) if r else 0
|
||
parts.append(f'<div class="card"><div class="arm">{esc(label)}</div><div class="txt">{body}</div><div class="wc">{w} words</div></div>')
|
||
parts.append('</div>')
|
||
|
||
# Panel B — beats
|
||
parts.append('<h2>B · Beat → paragraph (Instruct, chat template) — the Skaldsong question</h2>')
|
||
parts.append('<p class="sub">Can the Instruct arm still take direction after training on raw Yarros continuation text? Each beat expanded to one paragraph in Yarros\' voice.</p>')
|
||
if beats:
|
||
parts.append('<table><tr><th>beat</th><th>seed</th><th>on-beat (kw)</th><th>in-band 90–140</th><th>ran-on</th><th>words</th></tr>')
|
||
for r in beats:
|
||
ob, kw, ib, ro, w = beat_flags(r)
|
||
yn = lambda v,good: (f'<span class="ok">{"yes" if v else "no"}</span>' if v==good else f'<span class="no">{"yes" if v else "no"}</span>') if v is not None else '—'
|
||
parts.append(f'<tr><td class="beat">{esc(r.get("beat",r.get("id","")))[:64]}</td>'
|
||
f'<td>{r.get("seed","")}</td>'
|
||
f'<td>{("<span class=ok>yes</span>" if ob else "<span class=no>no</span>")} {kw}</td>'
|
||
f'<td>{yn(ib,True)}</td><td>{yn(ro,False)}</td><td>{w}</td></tr>')
|
||
parts.append('</table>')
|
||
# show the actual paragraphs
|
||
for r in beats[:6]:
|
||
para = esc(r.get("paragraph", r.get("text","")).strip())[:1400]
|
||
parts.append(f'<div class="prompt">{esc(r.get("beat",""))}</div><div class="card"><div class="txt">{para}</div></div>')
|
||
else:
|
||
parts.append('<p class="sub"><em>beats file not present</em></p>')
|
||
|
||
# Panel C — delta_cb
|
||
parts.append('<h2>C · delta_cb — did the adapter move the voice toward held-out Yarros?</h2>')
|
||
if DIST and DIST.exists():
|
||
parts.append(f'<pre class="dist">{esc(DIST.read_text())}</pre>')
|
||
else:
|
||
parts.append('<p class="sub"><em>distance summary not present</em></p>')
|
||
|
||
OUT.write_text("\n".join(parts), encoding="utf-8")
|
||
print(f"wrote {OUT} ({OUT.stat().st_size} bytes)")
|