#!/bin/bash
# lv-mccarthy D4 — instruction pairs. Run on pfi-gx10 as infra-ops.
#
# Deviations from the Hemingway D4 recipe, each forced by a measurement:
#   --register mccarthy      new; it names McCarthy's punctuation deliberately, see REGISTERS
#   --source-entities        MANDATORY; Hemingway's pairs predate it
#   --reflow-hard-wraps      DEFECT 4; The Road is hard-wrapped and the other 5 works are not
#   --drop-leading-heading   now also drops Blood Meridian's dash-separated chapter arguments
#   --n 3722 / --n 276       the full passage yield of each split, not a round number
set -e
cd ~/lv-mccarthy
mkdir -p pairs
S=scripts/yarros-corpus/build_sft_pairs.py
COMMON="--corpus corpus-renamed/copies --register mccarthy --model gen
        --key-file $HOME/.config/litellm/all-agents-key
        --source-entities corpus-clean/entities-final.json
        --context-frac 1.0 --drop-leading-heading --reflow-hard-wraps --seed 4919"
python3 $S $COMMON --split train --n 3722 --out pairs/pairs-full.jsonl > pairs/train.log 2>&1
python3 $S $COMMON --split val   --n 276  --out pairs/pairs-val.jsonl  > pairs/val.log  2>&1
echo DONE > pairs/.complete