booth: four arms, one beat, one author-neutral prompt
Six beats through voices-base, lv-bronte, lv-yarros and lv-hemingway, all served from the same process on fv-ml1 :8027 so only the adapter varies. Operator-requested side-by-side. http://10.100.10.50:8090/b/lv-voices-four-arms/ (24h TTL; also on the link board) THE PROMPT NAMES NO AUTHOR, deliberately. Each adapter trained under a prompt naming its own, so driving all four with any one of those hands that arm a hint the others do not get and the page would be measuring the prompt rather than the voice. The shared task skeleton is kept and the author clause removed. One asymmetry is disclosed on the page: Brontë and Hemingway trained on "a SHORT PASSAGE ... may run to several paragraphs" while Yarros trained on "ONE paragraph", so the neutral prompt sits slightly off-distribution for all three rather than for one. THE CONTROL GETS A 4x LARGER TOKEN BUDGET, and publishing it any other way would have been dishonest. Measured at the gate's 320-token budget: voices-base median 26 prose words, 181-257 words of <think> planning first, and 5 of 12 cells never reach the prose at all the adapters 0 of 12 failures each, empty think block in 12 of 12, median 97-105 words The adapters learned to skip the reasoning phase; the carrier has not. Showing the starved control would conflate voice with budget discipline, so the control runs at 1200 tokens and finishes every time, median 121 words. Both numbers are on the page. Two seeds per cell behind a toggle, because one sample of a sampled process is an anecdote, and a blind-mode toggle that hides which column is which. Sampler matches the gate harness (temperature 0.9, top_p 0.95, "BEAT: " prefix). Checked before publishing rather than after: all 36 adapter generations scored for verbatim 8-gram reuse, each arm against ITS OWN corpus. Brontë 0, Yarros 0, Hemingway 2 of 12 with a longest run of 8 words, that run being "i don t know i don t know". Layout verified by rendering it, not by reading the CSS: four equal 374px columns at 1600px wide, no horizontal overflow, 24 cards, 48 panes. ⚠ nh3-dev's shared /opt/ms-playwright tops out at chromium-1234, so playwright must be pinned to 1.61.0; a bare `npm i playwright` pulls 1.63 and asks for a browser build that is not there.
This commit is contained in:
@@ -0,0 +1,185 @@
|
||||
"""Render the four-arm comparison as a self-contained booth page."""
|
||||
import json, html, re, sys, statistics as st
|
||||
from pathlib import Path
|
||||
|
||||
SP = Path("/tmp/claude-1000/-home-lkraven-development-eshpfi-management/d4d5ad1b-76a2-4698-9684-ea6045592d22/scratchpad")
|
||||
d = json.loads((SP / "booth-cells.json").read_text(encoding="utf-8"))
|
||||
OUT = Path(sys.argv[1])
|
||||
|
||||
COLS = [("voices-base-1200", "control", "unadapted carrier"),
|
||||
("lv-bronte", "Brontë", "lv-bronte · ckpt475"),
|
||||
("lv-yarros", "Yarros", "lv-yarros"),
|
||||
("lv-hemingway", "Hemingway", "lv-hemingway · ckpt850")]
|
||||
SEEDS = [1234, 5678]
|
||||
|
||||
|
||||
def split(t):
|
||||
if "<think>" in t and "</think>" in t:
|
||||
return t.split("<think>", 1)[1].split("</think>", 1)[0], t.split("</think>", 1)[1].strip()
|
||||
if "<think>" in t:
|
||||
return t.split("<think>", 1)[1], ""
|
||||
return "", t.strip()
|
||||
|
||||
|
||||
def paras(text):
|
||||
out = []
|
||||
for p in re.split(r"\n\s*\n", text.strip()):
|
||||
p = " ".join(p.split())
|
||||
if p:
|
||||
out.append("<p>" + html.escape(p) + "</p>")
|
||||
return "\n".join(out) or '<p class="none">— the budget ran out before any prose was written —</p>'
|
||||
|
||||
|
||||
# headline numbers, computed not asserted
|
||||
short_base = [len(split(d["cells"][f"voices-base|{b}|{s}"]["text"])[1].split())
|
||||
for b, _ in d["beats"] for s in SEEDS]
|
||||
stats = {}
|
||||
for model, label, _ in COLS:
|
||||
w = [len(split(d["cells"][f"{model}|{b}|{s}"]["text"])[1].split())
|
||||
for b, _ in d["beats"] for s in SEEDS]
|
||||
th = [len(split(d["cells"][f"{model}|{b}|{s}"]["text"])[0].split())
|
||||
for b, _ in d["beats"] for s in SEEDS]
|
||||
stats[model] = (int(st.median(w)), int(st.median(th)), sum(1 for x in w if x == 0))
|
||||
|
||||
blocks = []
|
||||
for bid, beat in d["beats"]:
|
||||
cells = []
|
||||
for model, label, sub in COLS:
|
||||
panes = []
|
||||
for s in SEEDS:
|
||||
th, prose = split(d["cells"][f"{model}|{bid}|{s}"]["text"])
|
||||
tw = len(th.split())
|
||||
badge = (f'<span class="badge think">planned {tw}w first</span>' if tw > 0
|
||||
else '<span class="badge nothink">straight to prose</span>')
|
||||
panes.append(
|
||||
f'<div class="pane" data-seed="{s}">'
|
||||
f'<div class="meta">seed {s} · {len(prose.split())}w {badge}</div>'
|
||||
f'<div class="prose">{paras(prose)}</div></div>')
|
||||
cells.append(f'<div class="col" data-arm="{label}">'
|
||||
f'<h3><span class="nm">{html.escape(label)}</span>'
|
||||
f'<span class="sub">{html.escape(sub)}</span></h3>'
|
||||
+ "".join(panes) + "</div>")
|
||||
blocks.append(f'<section class="beat"><h2><span class="bid">{bid}</span>'
|
||||
f'{html.escape(beat)}</h2><div class="grid">' + "".join(cells) + "</div></section>")
|
||||
|
||||
rows = "".join(
|
||||
f"<tr><td>{html.escape(l)}</td><td>{stats[m][0]}</td><td>{stats[m][1]}</td>"
|
||||
f"<td>{stats[m][2]}/12</td></tr>" for m, l, _ in COLS)
|
||||
|
||||
HTML = f"""<!doctype html>
|
||||
<html lang="en"><head><meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width,initial-scale=1">
|
||||
<title>Four voices, one beat</title>
|
||||
<style>
|
||||
:root {{ --ink:#1e1c19; --dim:#6b645c; --line:#ddd6cb; --bg:#faf7f2; --card:#fff;
|
||||
--acc:#8a5a2b; --ctl:#7a7268; --br:#5c4b8a; --ya:#a33b5e; --he:#2f6b52; }}
|
||||
* {{ box-sizing:border-box; }}
|
||||
body {{ margin:0; background:var(--bg); color:var(--ink);
|
||||
font:16px/1.65 -apple-system,BlinkMacSystemFont,"Segoe UI",Roboto,sans-serif; }}
|
||||
header {{ padding:34px 28px 26px; border-bottom:1px solid var(--line); background:var(--card); }}
|
||||
h1 {{ margin:0 0 6px; font-size:27px; letter-spacing:-.4px; }}
|
||||
.lede {{ color:var(--dim); max-width:78ch; margin:0 0 20px; }}
|
||||
.prompt {{ background:var(--bg); border:1px solid var(--line); border-left:3px solid var(--acc);
|
||||
padding:12px 15px; border-radius:5px; max-width:90ch;
|
||||
font:13.5px/1.6 ui-monospace,SFMono-Regular,Menlo,monospace; }}
|
||||
.prompt b {{ color:var(--acc); }}
|
||||
.notes {{ max-width:82ch; margin:20px 0 0; padding:0; list-style:none; }}
|
||||
.notes li {{ position:relative; padding-left:20px; margin:9px 0; font-size:14.5px; color:var(--dim); }}
|
||||
.notes li::before {{ content:"▸"; position:absolute; left:2px; color:var(--acc); }}
|
||||
.notes b {{ color:var(--ink); }}
|
||||
table {{ border-collapse:collapse; margin:18px 0 0; font-size:14px; }}
|
||||
th,td {{ border:1px solid var(--line); padding:6px 12px; text-align:left; }}
|
||||
th {{ background:var(--bg); font-weight:600; }}
|
||||
td:nth-child(n+2) {{ text-align:right; font-variant-numeric:tabular-nums; }}
|
||||
.bar {{ position:sticky; top:0; z-index:5; display:flex; gap:18px; align-items:center;
|
||||
flex-wrap:wrap; padding:11px 28px; background:rgba(250,247,242,.96);
|
||||
backdrop-filter:blur(8px); border-bottom:1px solid var(--line); font-size:14px; }}
|
||||
.bar label {{ display:flex; gap:6px; align-items:center; cursor:pointer; color:var(--dim); }}
|
||||
main {{ padding:8px 28px 60px; }}
|
||||
.beat {{ margin:36px 0 0; }}
|
||||
.beat h2 {{ font-size:17px; font-weight:600; margin:0 0 14px; padding-bottom:9px;
|
||||
border-bottom:1px solid var(--line); max-width:100%; }}
|
||||
.bid {{ display:inline-block; min-width:28px; color:var(--acc);
|
||||
font:12px ui-monospace,monospace; }}
|
||||
.grid {{ display:grid; grid-template-columns:repeat(4,1fr); gap:16px; }}
|
||||
.col {{ background:var(--card); border:1px solid var(--line); border-radius:7px;
|
||||
padding:14px 16px 6px; min-width:0; }}
|
||||
.col h3 {{ margin:0 0 10px; font-size:13px; text-transform:uppercase;
|
||||
letter-spacing:.7px; display:flex; flex-direction:column; gap:2px; }}
|
||||
.col[data-arm="control"] h3 .nm {{ color:var(--ctl); }}
|
||||
.col[data-arm="Brontë"] h3 .nm {{ color:var(--br); }}
|
||||
.col[data-arm="Yarros"] h3 .nm {{ color:var(--ya); }}
|
||||
.col[data-arm="Hemingway"] h3 .nm {{ color:var(--he); }}
|
||||
.sub {{ font:11px ui-monospace,monospace; text-transform:none;
|
||||
letter-spacing:0; color:var(--dim); }}
|
||||
body.blind .sub, body.blind .nm {{ visibility:hidden; }}
|
||||
body.blind .col h3::after {{ content:"arm " attr(data-n); position:absolute;
|
||||
color:var(--dim); font:12px ui-monospace,monospace; }}
|
||||
body.blind .col h3 {{ position:relative; }}
|
||||
.meta {{ font:11px ui-monospace,monospace; color:var(--dim);
|
||||
padding-bottom:7px; border-bottom:1px dotted var(--line); margin-bottom:9px; }}
|
||||
.badge {{ margin-left:7px; padding:1px 6px; border-radius:9px; font-size:10px; }}
|
||||
.badge.think {{ background:#f6e7d8; color:#8a5a2b; }}
|
||||
.badge.nothink {{ background:#e3efe6; color:#2f6b52; }}
|
||||
.prose p {{ margin:0 0 .75em; font-family:Georgia,"Iowan Old Style",serif;
|
||||
font-size:15px; line-height:1.72; }}
|
||||
.prose .none {{ color:#a33b5e; font-style:italic; font-family:inherit; font-size:13px; }}
|
||||
.pane[data-seed="5678"] {{ display:none; border-top:1px dashed var(--line);
|
||||
margin-top:12px; padding-top:11px; }}
|
||||
body.both .pane[data-seed="5678"] {{ display:block; }}
|
||||
@media (max-width:1100px) {{ .grid {{ grid-template-columns:repeat(2,1fr); }} }}
|
||||
@media (max-width:640px) {{ .grid {{ grid-template-columns:1fr; }}
|
||||
header,main,.bar {{ padding-left:16px; padding-right:16px; }} }}
|
||||
</style></head><body>
|
||||
<header>
|
||||
<h1>Four voices, one beat</h1>
|
||||
<p class="lede">The same six beats and the <b>same author-neutral prompt</b> through the
|
||||
unadapted carrier and the three <code>lv-*</code> LoRA adapters, all served from one
|
||||
process on <code>vllm-voices</code> (fv-ml1 GPU 0 :8027). Only the adapter changes.</p>
|
||||
<div class="prompt"><b>system</b> — {html.escape(d["system"])}<br><br><b>user</b> — BEAT: <the beat></div>
|
||||
<ul class="notes">
|
||||
<li><b>The prompt names no author, deliberately.</b> Each adapter trained under a prompt
|
||||
naming its own — driving all four with any one of those would hand that arm a hint the
|
||||
others do not get, and the page would be measuring the prompt.</li>
|
||||
<li><b>One disclosed asymmetry.</b> Brontë and Hemingway trained on “a SHORT PASSAGE … may
|
||||
run to several paragraphs”; Yarros trained on “ONE paragraph”. The neutral prompt uses
|
||||
neither qualifier, so it sits slightly off-distribution for all three rather than for one.</li>
|
||||
<li><b>The control gets a 4× larger token budget, and that is the fair thing to do.</b>
|
||||
At the gate's 320-token budget the carrier spends 181–257 words <i>thinking</i> and
|
||||
<b>5 of 12 cells never reach the prose at all</b>. The three adapters emit an empty
|
||||
think block in 12 of 12 — they learned to skip it. Publishing the starved control would
|
||||
conflate voice with budget discipline, so the control here runs at 1200 tokens and
|
||||
finishes every time.</li>
|
||||
<li><b>Two seeds per cell</b>, because one sample of a sampled process is an anecdote.
|
||||
Sampler matches the gate harness: temperature 0.9, top_p 0.95.</li>
|
||||
<li><b>These 36 adapter generations were checked for verbatim reuse before this page went
|
||||
up</b>, each arm against its own training corpus (scoring one author against another
|
||||
returns a clean zero that only means “different book”). Brontë <b>0</b>, Yarros
|
||||
<b>0</b>, Hemingway <b>2 of 12, longest run 8 words</b> — and the run is
|
||||
<i>“I don't know, I don't know”</i>. Nothing on this page reproduces anything worth
|
||||
reproducing.</li>
|
||||
<li>This is a <b>reading</b>, not a measurement. The numbers that decide anything are in the
|
||||
gate: <code>persistent-memory.d/2026-09-17-lv-hemingway-gate.md</code>.</li>
|
||||
</ul>
|
||||
<table><thead><tr><th>arm</th><th>median prose</th><th>median planning</th><th>never wrote prose</th></tr></thead>
|
||||
<tbody>{rows}</tbody></table>
|
||||
</header>
|
||||
<div class="bar">
|
||||
<label><input type="checkbox" id="both"> show the second seed</label>
|
||||
<label><input type="checkbox" id="blind"> blind mode (hide which arm is which)</label>
|
||||
<span style="color:var(--dim);font:12px ui-monospace,monospace">column order is fixed: control · Brontë · Yarros · Hemingway</span>
|
||||
</div>
|
||||
<main>{''.join(blocks)}</main>
|
||||
<script>
|
||||
for (const [id, cls] of [["both","both"],["blind","blind"]]) {{
|
||||
document.getElementById(id).addEventListener("change", e =>
|
||||
document.body.classList.toggle(cls, e.target.checked));
|
||||
}}
|
||||
document.querySelectorAll(".grid").forEach(g =>
|
||||
[...g.children].forEach((c,i) => c.querySelector("h3").dataset.n = i+1));
|
||||
</script>
|
||||
</body></html>"""
|
||||
OUT.write_text(HTML, encoding="utf-8")
|
||||
print(f"wrote {OUT} ({len(HTML):,} bytes)")
|
||||
for m, l, _ in COLS:
|
||||
print(f" {l:<11} median prose {stats[m][0]:>4}w · median planning {stats[m][1]:>4}w · empty {stats[m][2]}/12")
|
||||
@@ -0,0 +1,67 @@
|
||||
"""Four arms, one prompt, one set of beats — harness-matched on the live seat.
|
||||
|
||||
THE PROMPT IS DELIBERATELY AUTHOR-NEUTRAL. Each adapter was trained under a system
|
||||
prompt naming its own author ("in the manner of Charlotte Brontë — mid-nineteenth-century
|
||||
first-person retrospective ..."). Driving all four with any ONE of those would hand that
|
||||
author's arm a hint the others do not get, and the comparison would measure the prompt.
|
||||
So the author clause is removed and the shared task skeleton kept. What is left varies
|
||||
only by which LoRA is loaded.
|
||||
|
||||
⚠ ONE DISCLOSED ASYMMETRY: Brontë and Hemingway trained on "a SHORT PASSAGE ... may run to
|
||||
several paragraphs"; Yarros trained on "ONE paragraph". The neutral prompt uses neither
|
||||
qualifier, so it sits slightly off-distribution for all three rather than for one.
|
||||
|
||||
Sampler matches the gate harness exactly: temperature 0.9, top_p 0.95, "BEAT: " prefix.
|
||||
Two seeds per cell, because one sample of a sampled process is an anecdote — the second is
|
||||
rendered behind a toggle so the page stays readable but the variance is one click away.
|
||||
"""
|
||||
import json, urllib.request, sys, time
|
||||
|
||||
SEAT = "http://10.251.50.54:8027/v1/chat/completions"
|
||||
ARMS = [("voices-base", "control"), ("lv-bronte", "Brontë"),
|
||||
("lv-yarros", "Yarros"), ("lv-hemingway", "Hemingway")]
|
||||
SEEDS = [1234, 5678]
|
||||
|
||||
SYS = ("You expand a single story beat into a SHORT PASSAGE of prose. Render the beat "
|
||||
"itself; do not move past it, do not begin a new scene, do not comment, do not "
|
||||
"write a chapter heading. Output the prose only, 90–140 words.")
|
||||
|
||||
BEATS = [
|
||||
("b1", "She tells him she is leaving in the morning, and he does not ask her to stay."),
|
||||
("b2", "He refuses the money a second time, and the other man sets it down on the table anyway."),
|
||||
("b3", "She finds him bleeding on the stairs and asks how long he has been sitting there."),
|
||||
("b4", "He waits past the hour they agreed, orders another drink, and watches the door."),
|
||||
("b5", "She reads the letter twice, then puts it in the fire without saying what it said."),
|
||||
("b6", "He tells her the truth about the accident, and she says nothing for a long time."),
|
||||
]
|
||||
|
||||
|
||||
def gen(model, beat, seed):
|
||||
body = {"model": model, "seed": seed, "temperature": 0.9, "top_p": 0.95,
|
||||
"max_tokens": 320,
|
||||
"messages": [{"role": "system", "content": SYS},
|
||||
{"role": "user", "content": "BEAT: " + beat}]}
|
||||
req = urllib.request.Request(SEAT, data=json.dumps(body).encode(),
|
||||
headers={"Content-Type": "application/json"})
|
||||
for attempt in range(3):
|
||||
try:
|
||||
r = json.load(urllib.request.urlopen(req, timeout=180))
|
||||
return r["choices"][0]["message"]["content"], r.get("model")
|
||||
except Exception as e:
|
||||
if attempt == 2:
|
||||
raise
|
||||
print(f" retry {attempt+1} after {e}", file=sys.stderr)
|
||||
time.sleep(5)
|
||||
|
||||
|
||||
out = {"system": SYS, "beats": BEATS, "arms": ARMS, "seeds": SEEDS, "cells": {}}
|
||||
t0 = time.time()
|
||||
for model, label in ARMS:
|
||||
for bid, beat in BEATS:
|
||||
for seed in SEEDS:
|
||||
txt, served = gen(model, beat, seed)
|
||||
out["cells"][f"{model}|{bid}|{seed}"] = {"text": txt, "served": served}
|
||||
print(f" {model:<14} {bid} seed={seed} {len(txt.split()):>4}w "
|
||||
f"served={served}", flush=True)
|
||||
print(f"done in {time.time()-t0:.0f}s")
|
||||
json.dump(out, open(sys.argv[1], "w"), ensure_ascii=False, indent=1)
|
||||
Reference in New Issue
Block a user