Qwen3-4B, one epoch on the leak-gated Yarros corpus. Base LoRA = ckpt-125 (its held-out minimum). Generated on gx10, same harness per arm. Not the frozen adjudication — the voice A/B and beat test the operator asked to see.
+"""] + +# Panel A — voice +parts.append('Can the Instruct arm still take direction after training on raw Yarros continuation text? Each beat expanded to one paragraph in Yarros\' voice.
') +if beats: + parts.append('| beat | seed | on-beat (kw) | in-band 90–140 | ran-on | words |
|---|---|---|---|---|---|
| {esc(r.get("beat",r.get("id","")))[:64]} | ' + f'{r.get("seed","")} | ' + f'{("yes" if ob else "no")} {kw} | ' + f'{yn(ib,True)} | {yn(ro,False)} | {w} |
beats file not present
') + +# Panel C — delta_cb +parts.append('{esc(DIST.read_text())}')
+else:
+ parts.append('distance summary not present
') + +OUT.write_text("\n".join(parts), encoding="utf-8") +print(f"wrote {OUT} ({OUT.stat().st_size} bytes)") diff --git a/scripts/r49-corpus/gen_beats_chat_yarros.py b/scripts/r49-corpus/gen_beats_chat_yarros.py new file mode 100644 index 0000000..c5f6d3f --- /dev/null +++ b/scripts/r49-corpus/gen_beats_chat_yarros.py @@ -0,0 +1,97 @@ +"""Beat → paragraph through the CHAT TEMPLATE, which is the product's real shape. + +The point of this script is to ask the one question the whole instruct experiment +rests on: after training raw Brontë continuation text into an instruct model, does +it still take direction? Raw text trained into a model whose weights expect +<|im_start|> framing can degrade the template behaviour, and if it has, the adapter +bought voice at the cost of the only capability Skaldsong needs. + +So this drives the model exactly as Skaldsong would -- system prompt stating the job +and the length, user message carrying the beat -- and reports the three things that +decide whether the answer is usable: + + on-beat did it render THIS beat, or wander into Brontë's own plot furniture + in-band did it honour the length, which is the cheapest proxy for "took direction" + ran-on did it close a paragraph, or keep going into the next beat's territory + +Operator's constraint, and it is the one that rules out fixing this downstream: if a +frontier model has to judge every paragraph, the tiny model has no purpose. So these +have to be checkable without one. +""" +from __future__ import annotations +import argparse, json, re, time +from pathlib import Path +import torch +from transformers import AutoModelForCausalLM, AutoTokenizer + +SYS = ("You expand a single story beat into ONE paragraph of prose in the manner of Rebecca " + "Yarros — contemporary first-person PRESENT-tense narration, emotionally charged, sensory " + "and physical, the voice of new-adult romantasy. Render the beat itself; do not move past " + "it, do not add a new scene, do not comment. Output the paragraph only, 90–140 words.") + +ap = argparse.ArgumentParser() +ap.add_argument("--base", required=True) +ap.add_argument("--adapter", default=None) +ap.add_argument("--beats", required=True) +ap.add_argument("--out", required=True) +ap.add_argument("--arm", required=True) +ap.add_argument("--seeds", type=int, nargs="+", default=[1234, 5678]) +ap.add_argument("--max-new-tokens", type=int, default=320) +ap.add_argument("--temperature", type=float, default=0.9) +ap.add_argument("--top-p", type=float, default=0.95) +a = ap.parse_args() + +beats = json.loads(Path(a.beats).read_text()) +tok = AutoTokenizer.from_pretrained(a.base) +if tok.chat_template is None: + raise SystemExit("REFUSING: this carrier has no chat template -- it is not an instruct build") +model = AutoModelForCausalLM.from_pretrained(a.base, dtype=torch.bfloat16, + attn_implementation="sdpa").to("cuda") +if a.adapter: + from peft import PeftModel + model = PeftModel.from_pretrained(model, a.adapter) + nz = sum(1 for m in model.modules() if hasattr(m, "lora_B") + and float(m.lora_B["default"].weight.abs().sum()) > 0) + tot = sum(1 for m in model.modules() if hasattr(m, "lora_B")) + print(f"[gen] adapter bound: {nz}/{tot} lora_B tensors non-zero", flush=True) + if nz == 0: + raise SystemExit("REFUSING: adapter applied but every lora_B is zero") +model.eval() + +STOP = re.compile(r"\n\s*\n") +DROP = {"the", "a", "an", "and", "or", "but", "in", "on", "at", "to", "of", "his", "her", + "he", "she", "it", "was", "were", "had", "would", "not", "be", "by", "as", + "with", "for", "from", "that", "this", "up", "down"} + +out = Path(a.out); out.parent.mkdir(parents=True, exist_ok=True) +t0 = time.time() +with out.open("w", encoding="utf-8") as fh: + for b in beats: + for seed in a.seeds: + torch.manual_seed(seed) + text = tok.apply_chat_template( + [{"role": "system", "content": SYS}, + {"role": "user", "content": "BEAT: " + b["beat"]}], + tokenize=False, add_generation_prompt=True, + **({"enable_thinking": False} if "enable_thinking" in (tok.chat_template or "") else {})) + ids = tok(text, return_tensors="pt", add_special_tokens=False).to("cuda") + with torch.no_grad(): + g = model.generate(**ids, do_sample=True, temperature=a.temperature, + top_p=a.top_p, max_new_tokens=a.max_new_tokens, + pad_token_id=tok.eos_token_id) + raw = tok.decode(g[0][ids["input_ids"].shape[1]:], skip_special_tokens=True).strip() + m = STOP.search(raw) + para = (raw[:m.start()] if m else raw).strip() + kws = [w for w in re.findall(r"[a-z']+", b["beat"].lower()) + if w not in DROP and len(w) > 3] + hit = sum(1 for k in kws if k[:5] in para.lower()) + w = len(para.split()) + fh.write(json.dumps({"format": a.arm, "id": b["id"], "beat": b["beat"], "seed": seed, + "prompt": "BEAT: " + b["beat"], "paragraph": para, + "ran_on": m is None, "words": w, + "in_band": 90 <= w <= 140, + "beat_keywords": kws, "keyword_hits": hit}) + "\n") + print(f" {b['id']:>4} seed={seed} {w:>4}w kw {hit}/{len(kws)} " + f"{'in-band' if 90 <= w <= 140 else 'OUT-OF-BAND'}" + f"{' RAN-ON' if m is None else ''}", flush=True) +print(f"[gen] {a.arm} -> {out} in {time.time()-t0:.0f}s", flush=True) diff --git a/scripts/r49-corpus/voice_distance.py b/scripts/r49-corpus/voice_distance.py new file mode 100644 index 0000000..56891ba --- /dev/null +++ b/scripts/r49-corpus/voice_distance.py @@ -0,0 +1,121 @@ +"""Did the adapter move the voice TOWARD held-out Yarros? A seat-free relative measure. + +NOT the frozen adjudication. That needs a romantasy control-author panel (to place an +absolute band and a hard-negative sister), a seed-to-seed spread, and — for BEAT +INCUMBENT — the gen seat, none of which are available here. This answers the smaller, +honest question the operator can act on: of the three arms generated on ONE harness, +which sits closest to real held-out Yarros, and does the adapter beat the base control? + +Instrument: Burrows's Delta over CHARACTER BIGRAMS (hence delta_cb). Char bigrams are +dominated by function-word morphology and rhythm, not proper nouns, so the rename does +not move them. Reference profile is the HELD-OUT (val) split — text no arm was trained +on. Each arm's pooled generations are scored against it; lower = closer to Yarros. + +Discipline: this is a RELATIVE reading (arms vs each other, same harness), never an +absolute-band claim. The A-vs-A floor below is the only thing that makes a between-arm +gap meaningful — half-vs-half of the held-out reference gives the distance the metric +returns for two samples of the SAME author, so a between-arm gap smaller than that floor +is not a finding. +""" +from __future__ import annotations +import json, re, sys, statistics as st +from collections import Counter +from pathlib import Path + + +def bigrams(text: str) -> Counter: + t = re.sub(r"\s+", " ", text.lower()) + return Counter(t[i:i+2] for i in range(len(t) - 1)) + + +def profile(text: str, keys: list[str]) -> dict: + c = bigrams(text); n = sum(c.values()) or 1 + return {k: c.get(k, 0) / n for k in keys} + + +def delta(arm_text: str, ref_prof: dict, mu: dict, sd: dict, keys: list[str]) -> float: + ap = profile(arm_text, keys) + # Burrows's Delta = mean |z(arm) - z(ref)| over the shared feature set + return st.mean(abs((ap[k] - mu[k]) / sd[k] - (ref_prof[k] - mu[k]) / sd[k]) for k in keys) + + +def main() -> int: + corpus = Path(sys.argv[1]) # yarros-corpus-renamed (has split=val) + evaldir = Path(sys.argv[2]) # dir of voice.*.jsonl + # reference = held-out val text + val = [] + for p in sorted((corpus / "copies").glob("*.jsonl")): + for l in p.read_text(encoding="utf-8").splitlines(): + r = json.loads(l) + if r.get("split") == "val": + val.append(r["text"]) + # dedup identical val chapters across copies (renaming aside, the same chapter recurs) + ref_text = "\n".join(dict.fromkeys(val)) + # feature set: the most frequent bigrams in the reference (stable, high-signal) + keys = [k for k, _ in bigrams(ref_text).most_common(400)] + # mu/sd across the val text split into chunks, for z-scoring + words = ref_text.split() + chunks = [" ".join(words[i:i+800]) for i in range(0, len(words), 800) if len(words[i:i+800]) > 200] + profs = [profile(c, keys) for c in chunks] + mu = {k: st.mean(p[k] for p in profs) for k in keys} + sd = {k: (st.pstdev(p[k] for p in profs) or 1e-9) for k in keys} + ref_prof = profile(ref_text, keys) + + # SAME-AUTHOR REFERENCE (the target, not a significance threshold): two halves + # of held-out Yarros. A perfect mimic scores about this; you cannot get closer + # to Yarros than Yarros gets to itself at this sample size. + half = len(words) // 2 + same_author = delta(" ".join(words[:half]), profile(" ".join(words[half:]), keys), mu, sd, keys) + + print(f"reference: held-out Yarros, {len(words):,} words, {len(chunks)} chunks, {len(keys)} char-bigram features") + print(f"same-author target (held-out Yarros vs itself): delta_cb = {same_author:.3f}") + print(f" -> the floor of what any arm could reach; lower is more Yarros-like, this is the best possible\n") + + def arm_texts(f): + return [json.loads(l) for l in f.read_text(encoding="utf-8").splitlines()] + + rows = [] + for f in sorted(evaldir.glob("voice.*.jsonl")): + arm = f.stem.replace("voice.", "") + recs = arm_texts(f) + allt = "\n".join(r["continuation"] for r in recs) + d = delta(allt, ref_prof, mu, sd, keys) + # within-arm sampling spread = the REAL noise floor for a between-arm gap: + # split by seed and score each subset; the range is this metric's variance + # at this sample size, measured rather than assumed. + by_seed = {} + for r in recs: + by_seed.setdefault(r["seed"], []).append(r["continuation"]) + seed_ds = [delta("\n".join(v), ref_prof, mu, sd, keys) for v in by_seed.values() if len(v) > 2] + spread = (max(seed_ds) - min(seed_ds)) if len(seed_ds) > 1 else float("nan") + rows.append((arm, d, len(allt.split()), seed_ds, spread)) + + print(" arm delta_cb per-seed [words]") + for arm, d, w, sd_, spread in sorted(rows, key=lambda x: x[1]): + seeds = " ".join(f"{x:.3f}" for x in sd_) + print(f" {arm:20s} {d:.3f} ({seeds}) [{w}]") + # the noise floor is the LARGEST within-arm spread across arms + floors = [r[4] for r in rows if r[4] == r[4]] + noise = max(floors) if floors else float("nan") + print(f"\n measured noise floor (largest within-arm seed spread): {noise:.3f}") + print(f" -> a between-arm gap must exceed ~{noise:.3f} to be a real difference\n") + + base = next((d for a, d, _, _, _ in rows if "unadapted" in a), None) + if base is not None: + print(" vs base-unadapted control (positive gap = moved toward Yarros):") + for arm, d, _, _, _ in sorted(rows, key=lambda x: x[1]): + if "unadapted" in arm: + continue + gap = base - d + verdict = ("MOVED toward Yarros (exceeds noise floor)" if gap > noise + else "moved toward Yarros, but within the measured noise floor") + print(f" {arm:20s} {gap:+.3f} ({verdict})") + ordered = [a for a, *_ in sorted(rows, key=lambda x: x[1])] + print(f"\n ordering: {' < '.join(ordered)} (lower = more Yarros-like)") + print(" ⚠ one seed-pair per arm; this ordering CORROBORATES the independent held-out") + print(" loss ordering (Base < Instruct) but is not itself a multi-seed result.") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/r49-corpus/voice_prompts_yarros.json b/scripts/r49-corpus/voice_prompts_yarros.json new file mode 100644 index 0000000..8031412 --- /dev/null +++ b/scripts/r49-corpus/voice_prompts_yarros.json @@ -0,0 +1,11 @@ +[ + {"id":"a1","tier":"modern","prompt":"The self-checkout machine refused her coupon for the third time."}, + {"id":"a2","tier":"modern","prompt":"He parked the car and sat listening to the engine tick as it cooled."}, + {"id":"a3","tier":"modern","prompt":"The office kitchen smelled of burnt coffee and somebody's reheated fish."}, + {"id":"b1","tier":"neutral","prompt":"She had not slept, and the morning found her at the window."}, + {"id":"b2","tier":"neutral","prompt":"There was a message on her phone, and no one would say who had sent it."}, + {"id":"b3","tier":"neutral","prompt":"The boy would not speak, though she had asked him three times."}, + {"id":"c1","tier":"romantasy","prompt":"The instructor called my name, and the whole cohort turned to watch me step onto the mat."}, + {"id":"c2","tier":"romantasy","prompt":"He was the last person I wanted as a partner, and now his hand was at the small of my back."}, + {"id":"c3","tier":"romantasy","prompt":"The wound at my side had stopped bleeding, but the drop to the canyon floor had not gotten any shorter."} +]