vh
935b35ac2e
Operator: "start the 1.7b training." The 0.6B adapter learned the Gutenberg transcription's ~70-character line breaks along with the prose -- its output wrapped at a mid-length-line ratio of 0.85 against the base model's 0.00. That is typography rather than style, and every further rung would have inherited it, so the corpus is reflowed before rung 2 rather than after the sweep. The reflow joins 57,430 of 85,380 paragraph blocks and keeps 27,950. Verse is the hazard a blind join would destroy, so the decision is per block by median line length: blocks whose lines cluster near the wrap width are flowed prose, blocks of consistently short lines keep their breaks. Every kept multi-line block in the sample was genuinely verse with its lineation intact. No line ended in a lone hyphen, so the space-join could not split a word across lines. The acceptance check is content identity -- " ".join(text.split()) byte-identical before and after -- and it passed on all 852 records, proving only whitespace changed. Concrete cost of the old defect: 5.7% of the training budget was newline tokens. The same words pack to 5,210,112 tokens unwrapped against 5,525,504 wrapped. The 1.7B run is live at 159 steps and roughly 18.7 s/it. Everything but the carrier and the corpus is held from the 0.6B run: seed 4919, rank 32, lr 1e-4, seq 4096, batch 1 by accum 8, one epoch, eval and save every 25 steps so the minimum is located rather than assumed. That corpus change is a second variable and it is named as one. A 0.6B-vs-1.7B comparison is descriptive, not attributable, until the chained 0.6B rerun on the same unwrapped corpus lands behind it -- gated on the 1.7B actually producing an adapter, because a chain that fires on failure turns one lost run into two. "Did sense come back at 1.7B" is a within-arm reading and survives the confound; any between-rung delta does not. The original wrapped corpus is untouched, so the 0.6B run's pinned corpus sha 3959036cf851bf62 stays reproducible.
100 lines
4.3 KiB
Python
100 lines
4.3 KiB
Python
"""Reflow the hard-wrapped renamed corpus into flowing paragraphs.
|
|
|
|
The 0.6B adapter learned the Gutenberg transcription's ~70-character line breaks
|
|
along with Charlotte Brontë's voice: its output wraps at a median mid-length-line
|
|
ratio of 0.85 against the base model's 0.00. That is typography, not style, and
|
|
every further rung would inherit it.
|
|
|
|
⚠ VERSE IS THE HAZARD. These novels contain poems, and a blind join would smear
|
|
them into prose. So the join is decided per paragraph block by MEDIAN LINE LENGTH:
|
|
a block whose lines sit near the wrap width is flowed prose and gets joined; a
|
|
block of consistently short lines is verse (or a heading, or an address) and keeps
|
|
its breaks. Both counts are reported, because a rule that silently reclassified
|
|
half the poetry would look exactly like a rule that worked.
|
|
|
|
The acceptance check is content identity: `" ".join(text.split())` must be
|
|
byte-identical before and after, for every record. That proves ONLY whitespace
|
|
changed -- no word gained, lost, or altered. A reflow that quietly dropped a line
|
|
would otherwise be invisible.
|
|
|
|
Writes to a NEW directory. The original stays exactly as the 0.6B run's provenance
|
|
pins it (corpus_sha256_16 3959036cf851bf62), so that run remains reproducible.
|
|
"""
|
|
import json
|
|
import statistics
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
SRC = Path(sys.argv[1])
|
|
DST = Path(sys.argv[2])
|
|
PROSE_MEDIAN = 55 # a wrapped-prose block's lines cluster near the wrap width
|
|
|
|
stats = {"records": 0, "blocks": 0, "joined": 0, "kept": 0, "hyphen_ends": 0}
|
|
|
|
|
|
def reflow(text: str) -> str:
|
|
out_blocks = []
|
|
for block in text.split("\n\n"):
|
|
lines = block.split("\n")
|
|
body = [l for l in lines if l.strip()]
|
|
if not body:
|
|
out_blocks.append(block)
|
|
continue
|
|
stats["blocks"] += 1
|
|
if len(body) == 1:
|
|
out_blocks.append(block)
|
|
stats["kept"] += 1
|
|
continue
|
|
med = statistics.median(len(l.rstrip()) for l in body[:-1] or body)
|
|
if med >= PROSE_MEDIAN:
|
|
stats["joined"] += 1
|
|
for l in body[:-1]:
|
|
if l.rstrip().endswith("-") and not l.rstrip().endswith("--"):
|
|
stats["hyphen_ends"] += 1
|
|
out_blocks.append(" ".join(l.strip() for l in body))
|
|
else:
|
|
stats["kept"] += 1
|
|
out_blocks.append(block)
|
|
return "\n\n".join(out_blocks)
|
|
|
|
|
|
DST.mkdir(parents=True, exist_ok=True)
|
|
(DST / "copies").mkdir(exist_ok=True)
|
|
for f in sorted((SRC / "copies").glob("*.jsonl")):
|
|
rows_out = []
|
|
for line in f.read_text(encoding="utf-8").splitlines():
|
|
if not line.strip():
|
|
continue
|
|
r = json.loads(line)
|
|
before = r["text"]
|
|
after = reflow(before)
|
|
# ⚠ the acceptance check: content identical, whitespace only
|
|
assert " ".join(before.split()) == " ".join(after.split()), \
|
|
f"CONTENT CHANGED in {f.name} {r.get('work')}/{r.get('chapter')}"
|
|
r["text"] = after
|
|
rows_out.append(json.dumps(r, ensure_ascii=False))
|
|
stats["records"] += 1
|
|
(DST / "copies" / f.name).write_text("\n".join(rows_out) + "\n", encoding="utf-8")
|
|
|
|
for extra in ("rename_stats.json", "manifest.json"):
|
|
if (SRC / extra).exists():
|
|
(DST / extra).write_text((SRC / extra).read_text(encoding="utf-8"), encoding="utf-8")
|
|
|
|
# What the wrap ratio actually became -- the number the defect was measured with.
|
|
def wrap_ratio(root):
|
|
rs = []
|
|
for f in sorted((root / "copies").glob("*.jsonl")):
|
|
for line in f.read_text(encoding="utf-8").splitlines():
|
|
if not line.strip():
|
|
continue
|
|
lines = [l for l in json.loads(line)["text"].split("\n") if l.strip()]
|
|
rs.append(sum(1 for l in lines if 20 < len(l) < 78) / max(1, len(lines)))
|
|
return statistics.median(rs)
|
|
|
|
print(f" records {stats['records']} blocks {stats['blocks']} "
|
|
f"joined {stats['joined']} kept-as-is {stats['kept']} (verse/headings/single-line)")
|
|
print(f" lines ending in a lone hyphen inside joined blocks: {stats['hyphen_ends']} "
|
|
f"(a nonzero count means words were split across lines and a space-join would break them)")
|
|
print(f" mid-length-line ratio before {wrap_ratio(SRC):.2f} -> after {wrap_ratio(DST):.2f}")
|
|
print(f" content identity: PASSED on all {stats['records']} records (whitespace-only change)")
|