"""Reflow the hard-wrapped renamed corpus into flowing paragraphs.

The 0.6B adapter learned the Gutenberg transcription's ~70-character line breaks
along with Charlotte Brontë's voice: its output wraps at a median mid-length-line
ratio of 0.85 against the base model's 0.00. That is typography, not style, and
every further rung would inherit it.

⚠ VERSE IS THE HAZARD. These novels contain poems, and a blind join would smear
them into prose. So the join is decided per paragraph block by MEDIAN LINE LENGTH:
a block whose lines sit near the wrap width is flowed prose and gets joined; a
block of consistently short lines is verse (or a heading, or an address) and keeps
its breaks. Both counts are reported, because a rule that silently reclassified
half the poetry would look exactly like a rule that worked.

The acceptance check is content identity: `" ".join(text.split())` must be
byte-identical before and after, for every record. That proves ONLY whitespace
changed -- no word gained, lost, or altered. A reflow that quietly dropped a line
would otherwise be invisible.

Writes to a NEW directory. The original stays exactly as the 0.6B run's provenance
pins it (corpus_sha256_16 3959036cf851bf62), so that run remains reproducible.
"""
import json
import statistics
import sys
from pathlib import Path

SRC = Path(sys.argv[1])
DST = Path(sys.argv[2])
PROSE_MEDIAN = 55          # a wrapped-prose block's lines cluster near the wrap width

stats = {"records": 0, "blocks": 0, "joined": 0, "kept": 0, "hyphen_ends": 0}


def reflow(text: str) -> str:
    out_blocks = []
    for block in text.split("\n\n"):
        lines = block.split("\n")
        body = [l for l in lines if l.strip()]
        if not body:
            out_blocks.append(block)
            continue
        stats["blocks"] += 1
        if len(body) == 1:
            out_blocks.append(block)
            stats["kept"] += 1
            continue
        med = statistics.median(len(l.rstrip()) for l in body[:-1] or body)
        if med >= PROSE_MEDIAN:
            stats["joined"] += 1
            for l in body[:-1]:
                if l.rstrip().endswith("-") and not l.rstrip().endswith("--"):
                    stats["hyphen_ends"] += 1
            out_blocks.append(" ".join(l.strip() for l in body))
        else:
            stats["kept"] += 1
            out_blocks.append(block)
    return "\n\n".join(out_blocks)


DST.mkdir(parents=True, exist_ok=True)
(DST / "copies").mkdir(exist_ok=True)
for f in sorted((SRC / "copies").glob("*.jsonl")):
    rows_out = []
    for line in f.read_text(encoding="utf-8").splitlines():
        if not line.strip():
            continue
        r = json.loads(line)
        before = r["text"]
        after = reflow(before)
        # ⚠ the acceptance check: content identical, whitespace only
        assert " ".join(before.split()) == " ".join(after.split()), \
            f"CONTENT CHANGED in {f.name} {r.get('work')}/{r.get('chapter')}"
        r["text"] = after
        rows_out.append(json.dumps(r, ensure_ascii=False))
        stats["records"] += 1
    (DST / "copies" / f.name).write_text("\n".join(rows_out) + "\n", encoding="utf-8")

for extra in ("rename_stats.json", "manifest.json"):
    if (SRC / extra).exists():
        (DST / extra).write_text((SRC / extra).read_text(encoding="utf-8"), encoding="utf-8")

# What the wrap ratio actually became -- the number the defect was measured with.
def wrap_ratio(root):
    rs = []
    for f in sorted((root / "copies").glob("*.jsonl")):
        for line in f.read_text(encoding="utf-8").splitlines():
            if not line.strip():
                continue
            lines = [l for l in json.loads(line)["text"].split("\n") if l.strip()]
            rs.append(sum(1 for l in lines if 20 < len(l) < 78) / max(1, len(lines)))
    return statistics.median(rs)

print(f"  records {stats['records']}  blocks {stats['blocks']}  "
      f"joined {stats['joined']}  kept-as-is {stats['kept']} (verse/headings/single-line)")
print(f"  lines ending in a lone hyphen inside joined blocks: {stats['hyphen_ends']} "
      f"(a nonzero count means words were split across lines and a space-join would break them)")
print(f"  mid-length-line ratio  before {wrap_ratio(SRC):.2f}  ->  after {wrap_ratio(DST):.2f}")
print(f"  content identity: PASSED on all {stats['records']} records (whitespace-only change)")