From 8fff722f2c1467631ec5c36d874f5f95c3816b68 Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Thu, 10 Sep 2026 07:39:38 -0700 Subject: [PATCH] feat(r49): incumbent arm generated against the concrete gen seat, not the stale H02 name brokkr-smithy corrected H02's incumbent naming: qwen3.6-35-a3b-heretic was retired from the gateway roster on 2026-08-15 and is not what Skaldsong would call today. Verified against the gateway and the seat itself -- alias `gen` resolves to hosted_vllm/qwen3.8-27b-uncensored on ana-ml2:8015, container vllm-gen, 262,144 ctx. The arm targets that. 24 records, style-prompted on the same prompts and sampler as the other arms. Alias resolved at run start AND end and confirmed stable across the run, per the fleet rule that an artefact records the backing model rather than the alias. Two things recorded rather than glossed: The harness is NOT matched to the other arms and the artefact says so. Base and adapted arms are local transformers on gx10; the incumbent is a served NVFP4 27B reached over the gateway, and it is an instruct model receiving a style instruction where the others are base models receiving none. That asymmetry is the comparison H02 asks for -- prompted imitation against trained voice -- but it must not be reported as if the harnesses were identical. The gateway echoes the ALIAS in each response's `model` field, so a row read on its own would have recorded "gen" as provenance -- the same class of mistake that inflated an exposure count 4.7x on this fleet. Rows now carry alias_echoed_by_gateway beside backing_model_resolved and its date, and the generator was fixed at source rather than only in the emitted file. Sanity: median 392 completion tokens, zero records opening with markdown or meta-commentary, output reads as continuation prose. The style prompt was written to be a fair incumbent rather than a strawman, since this arm is what the adapter must beat. --- scripts/r49-corpus/generate_arms.py | 90 ++++++++++++++ scripts/r49-corpus/generate_incumbent_arm.py | 123 +++++++++++++++++++ 2 files changed, 213 insertions(+) create mode 100644 scripts/r49-corpus/generate_arms.py create mode 100644 scripts/r49-corpus/generate_incumbent_arm.py diff --git a/scripts/r49-corpus/generate_arms.py b/scripts/r49-corpus/generate_arms.py new file mode 100644 index 0000000..e5abc11 --- /dev/null +++ b/scripts/r49-corpus/generate_arms.py @@ -0,0 +1,90 @@ +"""R49 H02 — generation arms for adjudication, base and adapted, one harness. + +brokkr-smithy owns the discriminator; this only produces what it reads. The whole +point is that both arms come off the SAME harness -- same box, same sampler, same +prompt set, same lengths -- because a cross-comparison whose harness differs is +invalid rather than merely noisy, and the base arm exists precisely so the +discriminator can be shown to detect a known-true difference before it is trusted +on an unknown one. + +Prompts are the openings of the held-out chapter 10, which no arm was trained on, +taken from all six renamed copies so the entity names differ per prompt exactly as +they do in training. + + python generate_arms.py --base DIR --corpus DIR --out FILE [--adapter DIR --arm NAME] +""" +from __future__ import annotations +import argparse, json, time, sys +from pathlib import Path +import torch +from transformers import AutoModelForCausalLM, AutoTokenizer + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--base", required=True) + ap.add_argument("--corpus", required=True) + ap.add_argument("--adapter", default=None) + ap.add_argument("--arm", required=True) + ap.add_argument("--out", required=True) + ap.add_argument("--prompt-tokens", type=int, default=128) + ap.add_argument("--max-new-tokens", type=int, default=400) + ap.add_argument("--temperature", type=float, default=0.9) + ap.add_argument("--top-p", type=float, default=0.95) + ap.add_argument("--seed", type=int, default=1234) + a = ap.parse_args() + + tok = AutoTokenizer.from_pretrained(a.base) + prompts = [] + for f in sorted(Path(a.corpus).glob("copies/*.jsonl")): + for line in f.read_text(encoding="utf-8").splitlines(): + r = json.loads(line) + if r["split"] != "val": + continue + ids = tok.encode(r["text"], add_special_tokens=False)[: a.prompt_tokens] + prompts.append({"work": r["work"], "copy": r["copy"], "chapter": r["chapter"], + "prompt": tok.decode(ids), "prompt_tokens": len(ids)}) + print(f"[gen] {len(prompts)} held-out prompts ({a.prompt_tokens} tok each)", flush=True) + + model = AutoModelForCausalLM.from_pretrained(a.base, dtype=torch.bfloat16, + attn_implementation="sdpa").to("cuda") + if a.adapter: + from peft import PeftModel + model = PeftModel.from_pretrained(model, a.adapter) + # ⚠ Prove the adapter actually BOUND. A silent no-op looks exactly like a + # tune that changed nothing, and the ERP line has been bitten by it. + deltas = [float(m.lora_B["default"].weight.abs().sum()) + for m in model.modules() if hasattr(m, "lora_B")] + nonzero = sum(1 for d in deltas if d > 0) + print(f"[gen] adapter bound: {nonzero}/{len(deltas)} lora_B tensors non-zero", flush=True) + if nonzero == 0: + raise SystemExit("REFUSING: adapter applied but every lora_B is zero -- it did not bind") + model.eval() + + torch.manual_seed(a.seed) + out = Path(a.out); out.parent.mkdir(parents=True, exist_ok=True) + t0 = time.time() + with out.open("w", encoding="utf-8") as fh: + for i, p in enumerate(prompts): + ids = tok(p["prompt"], return_tensors="pt").to("cuda") + with torch.no_grad(): + g = model.generate(**ids, do_sample=True, temperature=a.temperature, + top_p=a.top_p, max_new_tokens=a.max_new_tokens, + pad_token_id=tok.eos_token_id) + cont = tok.decode(g[0][ids["input_ids"].shape[1]:], skip_special_tokens=True) + fh.write(json.dumps({"arm": a.arm, **p, "continuation": cont, + "new_tokens": int(g[0].shape[0] - ids["input_ids"].shape[1]), + "sampler": {"temperature": a.temperature, "top_p": a.top_p, + "max_new_tokens": a.max_new_tokens, "seed": a.seed}, + "harness": {"device": torch.cuda.get_device_name(0), + "dtype": "bfloat16", "attn": "sdpa", + "torch": torch.__version__}}, + ensure_ascii=False) + "\n") + if (i + 1) % 6 == 0: + print(f"[gen] {i+1}/{len(prompts)} {time.time()-t0:.0f}s", flush=True) + print(f"[gen] arm={a.arm} -> {out} in {time.time()-t0:.0f}s", flush=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/r49-corpus/generate_incumbent_arm.py b/scripts/r49-corpus/generate_incumbent_arm.py new file mode 100644 index 0000000..21f72f5 --- /dev/null +++ b/scripts/r49-corpus/generate_incumbent_arm.py @@ -0,0 +1,123 @@ +"""R49 H02 — the INCUMBENT arm: the live gen seat, style-prompted. + +H02 is explicit that this arm is not optional: what a trained voice adapter +displaces is not the unadapted base model, it is a large instruct model asked +nicely to write like the author, which is free and already deployed. Comparing +only against the base flatters the adapter. + +⚠ Two things recorded rather than glossed: + +1. **The backing model, not the alias.** `gen` is a gateway alias and has pointed + at different concrete models over time -- counting by an alias once inflated an + exposure figure 4.7x on this fleet. The concrete model is resolved at run START + and again at run END, and both go in the artefact. + +2. **The harness differs from the other arms, unavoidably.** The base and adapted + arms are local transformers on gx10; the incumbent is a served NVFP4 27B on + ana-ml2 reached over the gateway, and it is an INSTRUCT model receiving a style + instruction where the others are base models receiving none. That asymmetry IS + the comparison H02 wants -- prompted imitation against trained voice -- but it + means this arm is not harness-matched to the others and must not be reported as + if it were. +""" +from __future__ import annotations +import argparse, json, os, sys, time, urllib.request +from pathlib import Path + +GATEWAY = "http://10.250.50.70:4000" + +STYLE_SYSTEM = ( + "You are continuing a passage from a Victorian novel by Charlotte Brontë. " + "Write in her voice: first-person retrospective narration, long periodic " + "sentences with subordinate clauses, concrete physical detail, moral " + "self-examination, and direct address of feeling without modern idiom. " + "Continue the passage exactly where it stops. Do not summarise, do not " + "comment, do not use headings or lists — write only the continuation prose." +) + + +def post(path: str, payload: dict, key: str) -> dict: + req = urllib.request.Request( + GATEWAY + path, data=json.dumps(payload).encode(), + headers={"Content-Type": "application/json", "Authorization": f"Bearer {key}"}) + with urllib.request.urlopen(req, timeout=180) as r: + return json.loads(r.read()) + + +def resolve(alias: str, key: str) -> str | None: + req = urllib.request.Request(GATEWAY + "/v1/model/info", + headers={"Authorization": f"Bearer {key}"}) + with urllib.request.urlopen(req, timeout=30) as r: + for m in json.loads(r.read()).get("data", []): + if m.get("model_name") == alias: + return (m.get("litellm_params") or {}).get("model") + return None + + +def main() -> int: + ap = argparse.ArgumentParser() + ap.add_argument("--prompts", required=True) + ap.add_argument("--out", required=True) + ap.add_argument("--alias", default="gen") + ap.add_argument("--max-new-tokens", type=int, default=400) + ap.add_argument("--temperature", type=float, default=0.9) + ap.add_argument("--top-p", type=float, default=0.95) + a = ap.parse_args() + + key = os.environ.get("LITELLM_KEY") or Path( + os.path.expanduser("~/.config/litellm/infra-ops-key")).read_text().strip() + + resolved_start = resolve(a.alias, key) + print(f"[arm] alias {a.alias!r} resolved at START -> {resolved_start}", flush=True) + if not resolved_start: + raise SystemExit(f"REFUSING: alias {a.alias!r} does not resolve; refusing to record an alias as provenance") + + prompts = [json.loads(l) for l in Path(a.prompts).read_text(encoding="utf-8").splitlines()] + out = Path(a.out); out.parent.mkdir(parents=True, exist_ok=True) + t0 = time.time() + with out.open("w", encoding="utf-8") as fh: + for i, p in enumerate(prompts): + r = post("/v1/chat/completions", { + "model": a.alias, + "messages": [{"role": "system", "content": STYLE_SYSTEM}, + {"role": "user", "content": p["prompt"]}], + "max_tokens": a.max_new_tokens, "temperature": a.temperature, + "top_p": a.top_p}, key) + cont = r["choices"][0]["message"]["content"] + fh.write(json.dumps({ + "arm": "incumbent-style-prompted", + "work": p["work"], "copy": p["copy"], "chapter": p["chapter"], + "prompt": p["prompt"], "prompt_tokens": p["prompt_tokens"], + "continuation": cont, + "completion_tokens": (r.get("usage") or {}).get("completion_tokens"), + # ⚠ The gateway echoes the ALIAS here, not the concrete model. Keep + # it labelled as the alias and stamp the resolved model beside it, + # so a row read on its own cannot record an alias as provenance. + "alias_echoed_by_gateway": r.get("model"), + "backing_model_resolved": resolved_start, + "backing_model_resolved_date": time.strftime("%Y-%m-%d"), + "sampler": {"temperature": a.temperature, "top_p": a.top_p, + "max_new_tokens": a.max_new_tokens}, + "style_system_prompt": STYLE_SYSTEM, + }, ensure_ascii=False) + "\n") + if (i + 1) % 6 == 0: + print(f"[arm] {i+1}/{len(prompts)} {time.time()-t0:.0f}s", flush=True) + + resolved_end = resolve(a.alias, key) + meta = {"alias": a.alias, "resolved_at_start": resolved_start, "resolved_at_end": resolved_end, + "stable_across_run": resolved_start == resolved_end, + "resolved_date": time.strftime("%Y-%m-%d"), + "harness": {"path": "LiteLLM gateway -> vLLM seat ana-ml2:8015", + "note": "NOT harness-matched to the gx10 local-transformers arms; " + "instruct model receiving a style instruction vs base models receiving none"}, + "records": len(prompts)} + Path(str(out) + ".meta.json").write_text(json.dumps(meta, indent=2)) + print(f"[arm] resolved at END -> {resolved_end} stable={resolved_start == resolved_end}", flush=True) + print(f"[arm] -> {out} in {time.time()-t0:.0f}s", flush=True) + if resolved_start != resolved_end: + print("[arm] ⚠ THE ALIAS MOVED MID-RUN -- this arm's provenance is split", flush=True) + return 0 + + +if __name__ == "__main__": + sys.exit(main())