#!/usr/bin/env bash
# R49 H02 — rung 3, the last of the planned sweep: Qwen3-4B-Base, 1 epoch.
#
# WHAT THIS RUNG IS FOR. 0.6B transferred the voice and not the sense. 1.7B brought
# back LOCAL coherence -- sequential actions that follow one another -- while
# scene-level continuity still broke mid-passage, and half the seeds on a wholesome
# prompt drifted into Brontë's own preoccupations with death. 4B is the test of
# whether continuity closes with carrier size, which is the question that decides
# whether this regime needs a bigger carrier or a different training shape.
#
# Held from both earlier rungs so carrier size stays the ONLY variable: the unwrapped
# corpus (sha 77f37057b2782e49), seed 4919, rank 32, lr 1e-4, seq 4096, batch 1 x
# accum 8, 1 epoch, eval+save every 25 steps.
#
# ⚠ Expect ~105 min. The carrier bench measured 717 tok/s at 4B against 1415 at 1.7B,
# and the 1.7B rung took 52 minutes.
#
# ⚠ All three rungs plateau within one epoch -- the 0.6B series ends 3.173, 3.172,
# 3.172 and the 1.7B ends 3.019, 3.018, 3.019. One epoch is the right length here, so
# a flat tail is the expected shape and NOT evidence of a stuck run.
set -euo pipefail
OUT=/home/infra-ops/r49-runs/h02-4b-1ep
LOG=$OUT/train.log
apps=$(nvidia-smi --query-compute-apps=pid --format=csv,noheader | tr -d '[:space:]')
[ -n "$apps" ] && { echo "REFUSING: GPU not clear" >&2; nvidia-smi --query-compute-apps=pid,process_name,used_memory --format=csv >&2; exit 1; }
[ -f "$OUT/run.pid" ] && kill -0 "$(cat "$OUT/run.pid")" 2>/dev/null && { echo "REFUSING: live pid" >&2; exit 1; }
[ -e "$LOG" ] && { echo "REFUSING: $LOG exists" >&2; exit 1; }
mkdir -p "$OUT"
echo "# launched $(date -Is) Qwen3-4B-Base, 1 epoch, seed 4919, unwrapped corpus, eval+save every 25" > "$LOG"
setsid nohup /home/infra-ops/ml/.venv/bin/python /home/infra-ops/r49-prep/train_voice_lora.py \
  --corpus /home/infra-ops/r49-corpus-renamed-unwrapped \
  --base /home/infra-ops/carriers/Qwen3-4B-Base \
  --seed 4919 --epochs 1 --eval-steps 25 --save-steps 25 \
  --out "$OUT" >> "$LOG" 2>&1 < /dev/null &
echo $! > "$OUT/run.pid"
echo "launched pid $(cat "$OUT/run.pid") -> $LOG"