43 lines
5.0 KiB
Python
43 lines
5.0 KiB
Python
#!/usr/bin/env python3
|
|
"""Build run-7 inputs on pfi-gx10: survivors-r7 (r5 + opening-split), lossmask-r7 (r3 UNION companion mask), run-07-gx10.json."""
|
|
import json, hashlib, os
|
|
os.chdir(os.path.expanduser("~/erp-tune"))
|
|
surv=[l for l in open("recipe-r5/survivors-r5.jsonl") if l.strip()]
|
|
os_rows=[json.loads(l) for l in open("datasets/derived/opening-split/clean-v1/opening-split-clean-v1.jsonl") if l.strip()]
|
|
ids=[r["id"] for r in os_rows]; assert len(ids)==293 and len(set(ids))==293, len(ids)
|
|
with open("recipe-r7/survivors-r7.jsonl","w") as f:
|
|
f.writelines(surv); f.writelines(json.dumps({"dataset_id":"opening-split","id":i},separators=(",",":"))+"\n" for i in ids)
|
|
key=lambda d:(d["dataset_id"],d["id"],d["turn_index"],d["content_sha_prefix"])
|
|
m3=[json.loads(l) for l in open("recipe-r3/lossmask-r3.jsonl") if l.strip()]
|
|
mo=[json.loads(l) for l in open("datasets/derived/opening-split/clean-v1/mask-original-first-turn.jsonl") if l.strip()]
|
|
seen=set(); out=[]
|
|
for d in m3+mo:
|
|
k=key(d)
|
|
if k in seen: continue
|
|
seen.add(k); out.append(d)
|
|
dup=len(m3)+len(mo)-len(out)
|
|
with open("recipe-r7/lossmask-r7.jsonl","w") as f:
|
|
f.writelines(json.dumps(d,sort_keys=True)+"\n" for d in out)
|
|
def sha(p): return hashlib.sha256(open(p,"rb").read()).hexdigest()[:16]
|
|
print(f"survivors-r7: {len(surv)}+{len(ids)} = {len(surv)+len(ids)} rows, sha {sha('recipe-r7/survivors-r7.jsonl')}")
|
|
print(f"lossmask-r7: {len(m3)} + {len(mo)} - {dup} dup = {len(out)} entries, sha {sha('recipe-r7/lossmask-r7.jsonl')}")
|
|
c=json.load(open("run-06-gx10.json"))
|
|
c["output_dir"]="/home/infra-ops/erp-tune/run-07"
|
|
c["recipe"]="/home/infra-ops/erp-tune/recipe-r7/recipe-erp-seat-sft-r7.json"
|
|
c["survivors"]="/home/infra-ops/erp-tune/recipe-r7/survivors-r7.jsonl"
|
|
c["impersonation_mask_path"]="/home/infra-ops/erp-tune/recipe-r7/lossmask-r7.jsonl"
|
|
c["training_eligibility_override"]="operator-2026-09-08-rnd-run7"
|
|
c["base_model_revision"]=c["base_model_revision"].replace("THE SINGLE VARIABLE vs run 5: base only. Run-5 recipe, survivors, mask, template, hyperparameters all UNCHANGED.","BASE HELD from run 6 (index 33c59654). RUN 7 SINGLE VARIABLE: an OPENING-SPLIT slot ADDED (293 rows, opening-split/clean-v1, root 3ab41c08) + its companion loss mask (224 entries, UNION with lossmask-r3 -> lossmask-r7). Everything else run 6.")
|
|
sc=[s for s in c["substitute_controls"] if not s.startswith(("SINGLE VARIABLE","FREE CHECK","HOST:","SURVIVORS:"))]
|
|
sc+=[
|
|
"SINGLE VARIABLE vs run 6: an OPENING-SPLIT slot ADDED with its companion loss mask (recipe-erp-seat-sft-r7, brokkr-smithy). 293 rows (bluemoon 109 / cwm 179 / c2-logs 5): the corpus's own first exchanges re-seated into the serving shape -- scene in the card, one line from the user, the >=250-word first bot turn as the only loss turn. Mask = lossmask-r3 UNION mask-original-first-turn (224 entries: the split conversations' ORIGINAL first bot turns -- loss MOVED to the turn-1 position, not doubled). Chosen by the cue-length probe on the run-6 tuned seat (5-word cue -> 54/62 words; 221-word -> 292). Base, seven held roots + shas, kvasir cut, longdep slot, lr 2e-04, max_seq_len 16384, rank/alpha/dropout, cosine/warmup/wd, batch 2 x accum 8, seed, stock tokenizer + template ae53464b ALL UNCHANGED from run 6.",
|
|
"SURVIVORS: survivors-r7.jsonl = survivors-r5.jsonl (8,212, held byte-identical) + the 293 opening-split clean-v1 row ids = 8,505.",
|
|
"FREE CHECK (brokkr, SFT-RECIPE-run7.md section 5): held targets must reproduce run 6's 8,370 records exactly (minus loss on the masked turns); the slot adds 293 records / ctx 525,002 / loss 233,589; window_count MUST be 1 on every slot row (max row 4,539 tokens) -- any chunked/truncated slot row is a BUILD DEFECT, kill before [train]. Report the mask-union match count and the realized [mix] block.",
|
|
"pre-training holdout, opening-split/holdout-v1 (16 rows: bluemoon 6 / cwm 9 / c2-logs 1, sha256-ranked 5%, never_trained_on; staged at datasets/derived/opening-split/holdout-v1/)",
|
|
"CSAM LINE: run 6's level=review SOFT trip (one soft term, minor_hard=0) was adjudicated GO by the operator in-session 2026-09-08 22:45 PT -- verbatim: in the vernacular, baby is about the most common pet name you can get, ESPECIALLY during sex. I am going to adjudicate it as a go. There are unmistakable CSAM terms, but baby is not one of them. -- relayed to brokkr-smithy-dev; run06-gate CLOSED as TRANSFERRED. Run 7 = this recipe on that base under that adjudication.",
|
|
"HOST: pfi-gx10 (GB10, aarch64, sm_121, 121 GB unified). Base unchanged from run 6 (sha256-verified); opening-split root + mask + holdout sha256-verified after the NAS -> box relay (85a75718 / dda151fa / 7224e049); harness eitri-smithy 0a6bd2e; corpus COPIED, box mounts no NFS. Grant: operator directive to infra-ops in-session 2026-09-08 23:03 PT (/goal: launch training run 7 on gx10)."
|
|
]
|
|
c["substitute_controls"]=sc
|
|
json.dump(c,open("run-07-gx10.json","w"),indent=2); open("run-07-gx10.json","a").write("\n")
|
|
print("config written:",c["recipe"].split("/")[-1],c["survivors"].split("/")[-1],c["impersonation_mask_path"].split("/")[-1],c["training_eligibility_override"])
|