#!/bin/bash
# lv-hemingway v2 gate. Design is FROZEN in scripts/hemingway-corpus/GATE-PREREG.md
# and was written before this script ever ran. Do not edit the arms, the fixture
# size or the seeds to chase a result -- re-run, do not re-tune.
#
# THREE ARMS. ckpt1750 is the eval-loss minimum (2.2783, epoch 1.973). ckpt850
# (2.2823, epoch 0.959) is +0.0040 against a 0.0044 median neighbour jitter, i.e.
# TIED -- the loss curve cannot pick between them, and on Bronte the earlier
# epoch-1 checkpoint won the tiebreak on the axes that do resolve. `base` is the
# negative control for memorisation (it never saw the corpus) and the voice
# baseline. adapter/ (epoch 3.0, +0.0762 = 17.4x jitter) is NOT gated: that one
# the loss curve settles on its own.
set -o pipefail
cd ~/lv-hemingway || exit 1
PY=/home/infra-ops/ml/.venv/bin/python
RUN=~/r49-runs/hemingway-4b-pairs-3ep
OUT=~/r49-runs/hemingway-eval
BEATS=beats-hemingway-60.json
PROV=pairs/pairs-full.jsonl.provenance.json
SEEDS="1234 5678 9012 3456"
mkdir -p "$OUT"
log(){ echo "[eval $(date +%H:%M:%S)] $*"; }

# --system-from is MANDATORY. The harness's built-in SYS is Yarros's; driving a
# Hemingway arm with it would confound the adapter change with a prompt change.
# This binds the eval prompt to the one the run actually trained under (verified:
# run provenance system_prompt == pairs provenance system_prompt).
[ -f "$PROV" ] || { log "MISSING $PROV"; exit 1; }
[ -f "$BEATS" ] || { log "MISSING $BEATS"; exit 1; }

for arm in base:NONE ckpt1750:$RUN/checkpoints/checkpoint-1750 ckpt850:$RUN/checkpoints/checkpoint-850; do
  name=${arm%%:*}; path=${arm#*:}
  if [ "$name" != "base" ] && [ ! -d "$path" ]; then log "MISSING $path"; exit 1; fi
  if [ -s "$OUT/beats5.$name.jsonl" ]; then
    log "arm $name already has $(wc -l < "$OUT/beats5.$name.jsonl") generations -- skipping"
    continue
  fi
  log "arm $name"
  if [ "$name" = "base" ]; then
    "$PY" scripts/r49-corpus/gen_beats_chat_yarros.py \
      --base ~/carriers/Qwen3-4B-Instruct --beats "$BEATS" \
      --out "$OUT/beats5.$name.jsonl" --arm "$name" --seeds $SEEDS \
      --system-from "$PROV" || exit 1
  else
    "$PY" scripts/r49-corpus/gen_beats_chat_yarros.py \
      --base ~/carriers/Qwen3-4B-Instruct --adapter "$path" --beats "$BEATS" \
      --out "$OUT/beats5.$name.jsonl" --arm "$name" --seeds $SEEDS \
      --system-from "$PROV" || exit 1
  fi
  log "  $(wc -l < "$OUT/beats5.$name.jsonl") generations"
done

log "AXIS B -- MEMORISATION (corpus = the renamed copies the adapter trained on)"
# --corpus and --eval-dir are passed explicitly: the script's Yarros defaults would
# compare a Hemingway arm against the YARROS corpus and report a clean zero that
# means "different book", not "did not memorise".
"$PY" scripts/yarros-corpus/memorization_check.py \
  --eval-dir "$OUT" --corpus corpus-renamed/copies --glob 'beats5.*.jsonl' --strip 'beats5.' -n 8 \
  2>&1 | tee "$OUT/memorization.txt"

log "AXIS C -- DAMAGE (ran-on / out-of-band), ckpt1750 vs base"
"$PY" scripts/yarros-corpus/score_beats.py \
  --arm base="$OUT/beats5.base.jsonl" \
  --arm ckpt1750="$OUT/beats5.ckpt1750.jsonl" \
  --arm ckpt850="$OUT/beats5.ckpt850.jsonl" \
  --baseline base --candidate ckpt1750 --metric-source raw \
  --out "$OUT/score.ckpt1750.json" 2>&1 | tee "$OUT/score.ckpt1750.txt"
log "AXIS C -- DAMAGE, ckpt850 vs base"
"$PY" scripts/yarros-corpus/score_beats.py \
  --arm base="$OUT/beats5.base.jsonl" \
  --arm ckpt1750="$OUT/beats5.ckpt1750.jsonl" \
  --arm ckpt850="$OUT/beats5.ckpt850.jsonl" \
  --baseline base --candidate ckpt850 --metric-source raw \
  --out "$OUT/score.ckpt850.json" 2>&1 | tee "$OUT/score.ckpt850.txt"

log "AXIS A -- VOICE (delta_cb vs held-out Hemingway)"
"$PY" voice-prep.py || exit 1
"$PY" scripts/r49-corpus/voice_distance.py corpus-renamed "$OUT" --author Hemingway \
  2>&1 | tee "$OUT/voice_distance.txt"

echo "rc=0" > ~/lv-hemingway/.eval-complete
log "done"