diff --git a/scripts/r49-corpus/build_booth_four_arms.py b/scripts/r49-corpus/build_booth_four_arms.py new file mode 100644 index 0000000..88891c3 --- /dev/null +++ b/scripts/r49-corpus/build_booth_four_arms.py @@ -0,0 +1,185 @@ +"""Render the four-arm comparison as a self-contained booth page.""" +import json, html, re, sys, statistics as st +from pathlib import Path + +SP = Path("/tmp/claude-1000/-home-lkraven-development-eshpfi-management/d4d5ad1b-76a2-4698-9684-ea6045592d22/scratchpad") +d = json.loads((SP / "booth-cells.json").read_text(encoding="utf-8")) +OUT = Path(sys.argv[1]) + +COLS = [("voices-base-1200", "control", "unadapted carrier"), + ("lv-bronte", "Brontë", "lv-bronte · ckpt475"), + ("lv-yarros", "Yarros", "lv-yarros"), + ("lv-hemingway", "Hemingway", "lv-hemingway · ckpt850")] +SEEDS = [1234, 5678] + + +def split(t): + if "" in t and "" in t: + return t.split("", 1)[1].split("", 1)[0], t.split("", 1)[1].strip() + if "" in t: + return t.split("", 1)[1], "" + return "", t.strip() + + +def paras(text): + out = [] + for p in re.split(r"\n\s*\n", text.strip()): + p = " ".join(p.split()) + if p: + out.append("

" + html.escape(p) + "

") + return "\n".join(out) or '

— the budget ran out before any prose was written —

' + + +# headline numbers, computed not asserted +short_base = [len(split(d["cells"][f"voices-base|{b}|{s}"]["text"])[1].split()) + for b, _ in d["beats"] for s in SEEDS] +stats = {} +for model, label, _ in COLS: + w = [len(split(d["cells"][f"{model}|{b}|{s}"]["text"])[1].split()) + for b, _ in d["beats"] for s in SEEDS] + th = [len(split(d["cells"][f"{model}|{b}|{s}"]["text"])[0].split()) + for b, _ in d["beats"] for s in SEEDS] + stats[model] = (int(st.median(w)), int(st.median(th)), sum(1 for x in w if x == 0)) + +blocks = [] +for bid, beat in d["beats"]: + cells = [] + for model, label, sub in COLS: + panes = [] + for s in SEEDS: + th, prose = split(d["cells"][f"{model}|{bid}|{s}"]["text"]) + tw = len(th.split()) + badge = (f'planned {tw}w first' if tw > 0 + else 'straight to prose') + panes.append( + f'
' + f'
seed {s} · {len(prose.split())}w {badge}
' + f'
{paras(prose)}
') + cells.append(f'
' + f'

{html.escape(label)}' + f'{html.escape(sub)}

' + + "".join(panes) + "
") + blocks.append(f'

{bid}' + f'{html.escape(beat)}

' + "".join(cells) + "
") + +rows = "".join( + f"{html.escape(l)}{stats[m][0]}{stats[m][1]}" + f"{stats[m][2]}/12" for m, l, _ in COLS) + +HTML = f""" + + +Four voices, one beat + +
+

Four voices, one beat

+

The same six beats and the same author-neutral prompt through the + unadapted carrier and the three lv-* LoRA adapters, all served from one + process on vllm-voices (fv-ml1 GPU 0 :8027). Only the adapter changes.

+
system — {html.escape(d["system"])}

user — BEAT: <the beat>
+
    +
  • The prompt names no author, deliberately. Each adapter trained under a prompt + naming its own — driving all four with any one of those would hand that arm a hint the + others do not get, and the page would be measuring the prompt.
    • +
  • One disclosed asymmetry. Brontë and Hemingway trained on “a SHORT PASSAGE … may + run to several paragraphs”; Yarros trained on “ONE paragraph”. The neutral prompt uses + neither qualifier, so it sits slightly off-distribution for all three rather than for one.
    • +
  • The control gets a 4× larger token budget, and that is the fair thing to do. + At the gate's 320-token budget the carrier spends 181–257 words thinking and + 5 of 12 cells never reach the prose at all. The three adapters emit an empty + think block in 12 of 12 — they learned to skip it. Publishing the starved control would + conflate voice with budget discipline, so the control here runs at 1200 tokens and + finishes every time.
    • +
  • Two seeds per cell, because one sample of a sampled process is an anecdote. + Sampler matches the gate harness: temperature 0.9, top_p 0.95.
    • +
  • These 36 adapter generations were checked for verbatim reuse before this page went + up, each arm against its own training corpus (scoring one author against another + returns a clean zero that only means “different book”). Brontë 0, Yarros + 0, Hemingway 2 of 12, longest run 8 words — and the run is + “I don't know, I don't know”. Nothing on this page reproduces anything worth + reproducing.
    • +
  • This is a reading, not a measurement. The numbers that decide anything are in the + gate: persistent-memory.d/2026-09-17-lv-hemingway-gate.md.
    • +
+ + {rows}
armmedian prosemedian planningnever wrote prose
+
+
+ + + column order is fixed: control · Brontë · Yarros · Hemingway +
+
{''.join(blocks)}
+ +""" +OUT.write_text(HTML, encoding="utf-8") +print(f"wrote {OUT} ({len(HTML):,} bytes)") +for m, l, _ in COLS: + print(f" {l:<11} median prose {stats[m][0]:>4}w · median planning {stats[m][1]:>4}w · empty {stats[m][2]}/12") diff --git a/scripts/r49-corpus/gen_booth_four_arms.py b/scripts/r49-corpus/gen_booth_four_arms.py new file mode 100644 index 0000000..bb585a7 --- /dev/null +++ b/scripts/r49-corpus/gen_booth_four_arms.py @@ -0,0 +1,67 @@ +"""Four arms, one prompt, one set of beats — harness-matched on the live seat. + +THE PROMPT IS DELIBERATELY AUTHOR-NEUTRAL. Each adapter was trained under a system +prompt naming its own author ("in the manner of Charlotte Brontë — mid-nineteenth-century +first-person retrospective ..."). Driving all four with any ONE of those would hand that +author's arm a hint the others do not get, and the comparison would measure the prompt. +So the author clause is removed and the shared task skeleton kept. What is left varies +only by which LoRA is loaded. + +⚠ ONE DISCLOSED ASYMMETRY: Brontë and Hemingway trained on "a SHORT PASSAGE ... may run to +several paragraphs"; Yarros trained on "ONE paragraph". The neutral prompt uses neither +qualifier, so it sits slightly off-distribution for all three rather than for one. + +Sampler matches the gate harness exactly: temperature 0.9, top_p 0.95, "BEAT: " prefix. +Two seeds per cell, because one sample of a sampled process is an anecdote — the second is +rendered behind a toggle so the page stays readable but the variance is one click away. +""" +import json, urllib.request, sys, time + +SEAT = "http://10.251.50.54:8027/v1/chat/completions" +ARMS = [("voices-base", "control"), ("lv-bronte", "Brontë"), + ("lv-yarros", "Yarros"), ("lv-hemingway", "Hemingway")] +SEEDS = [1234, 5678] + +SYS = ("You expand a single story beat into a SHORT PASSAGE of prose. Render the beat " + "itself; do not move past it, do not begin a new scene, do not comment, do not " + "write a chapter heading. Output the prose only, 90–140 words.") + +BEATS = [ + ("b1", "She tells him she is leaving in the morning, and he does not ask her to stay."), + ("b2", "He refuses the money a second time, and the other man sets it down on the table anyway."), + ("b3", "She finds him bleeding on the stairs and asks how long he has been sitting there."), + ("b4", "He waits past the hour they agreed, orders another drink, and watches the door."), + ("b5", "She reads the letter twice, then puts it in the fire without saying what it said."), + ("b6", "He tells her the truth about the accident, and she says nothing for a long time."), +] + + +def gen(model, beat, seed): + body = {"model": model, "seed": seed, "temperature": 0.9, "top_p": 0.95, + "max_tokens": 320, + "messages": [{"role": "system", "content": SYS}, + {"role": "user", "content": "BEAT: " + beat}]} + req = urllib.request.Request(SEAT, data=json.dumps(body).encode(), + headers={"Content-Type": "application/json"}) + for attempt in range(3): + try: + r = json.load(urllib.request.urlopen(req, timeout=180)) + return r["choices"][0]["message"]["content"], r.get("model") + except Exception as e: + if attempt == 2: + raise + print(f" retry {attempt+1} after {e}", file=sys.stderr) + time.sleep(5) + + +out = {"system": SYS, "beats": BEATS, "arms": ARMS, "seeds": SEEDS, "cells": {}} +t0 = time.time() +for model, label in ARMS: + for bid, beat in BEATS: + for seed in SEEDS: + txt, served = gen(model, beat, seed) + out["cells"][f"{model}|{bid}|{seed}"] = {"text": txt, "served": served} + print(f" {model:<14} {bid} seed={seed} {len(txt.split()):>4}w " + f"served={served}", flush=True) +print(f"done in {time.time()-t0:.0f}s") +json.dump(out, open(sys.argv[1], "w"), ensure_ascii=False, indent=1)