#!/usr/bin/env bash
# Pull + verify jenerallee78/gemma-4-26B-A4B-it-ara-abliterated @ 0631379a onto pfi-gx10.
# Root shards + small files only; no GGUFs, no mlx-4bit. Verifies bytes against the
# brokkr-smithy pins (base-pin-jenerallee78.json) and installs the STOCK tokenizer set.
# Expects ~/erp-tune/base-pin-jenerallee78-shards.txt (file  sha256  bytes per line).
set -uo pipefail
export PATH="$HOME/.local/bin:$PATH"
REPO=jenerallee78/gemma-4-26B-A4B-it-ara-abliterated
REV=0631379a3d859e0059bc8d9b21ab5b654dfc272c
DEST=$HOME/models/gemma4-26b-a4b-it-ara-abliterated-jenerallee78-0631379a
STOCK=$HOME/models/gemma4-26b-a4b-it-bf16
PINS=$HOME/erp-tune/base-pin-jenerallee78-shards.txt
echo "== start $(date -u +%FT%TZ) on $(hostname)"
mkdir -p "$DEST"
echo "== download"
uv run --quiet --with 'huggingface_hub[hf_transfer]' hf download "$REPO" --revision "$REV" \
  --local-dir "$DEST" \
  --include 'model-*-of-00032.safetensors' --include 'config.json' --include 'generation_config.json' \
  --include 'model.safetensors.index.json' --include 'chat_template.jinja' --include 'ara_config.json' --include 'README.md' \
  --include 'tokenizer.json' --include 'tokenizer_config.json'
rc=$?
echo "== download rc=$rc $(date -u +%FT%TZ)"
[ $rc -eq 0 ] || { echo "DOWNLOAD FAILED rc=$rc"; exit 2; }
cd "$DEST"
echo "== shard sha256 vs pins"
fail=0
while read -r f oid bytes; do
  [ -n "$f" ] || continue
  sz=$(stat -c %s "$f" 2>/dev/null || echo MISSING)
  got=$(sha256sum "$f" 2>/dev/null | cut -d' ' -f1)
  if [ "$sz" = "$bytes" ] && [ "$got" = "$oid" ]; then echo "OK   $f"; else echo "FAIL $f size=$sz want=$bytes sha=$got want=$oid"; fail=$((fail+1)); fi
done < "$PINS"
echo "== shard result: fail=$fail"
echo "== index + config checks"
python3 - "$STOCK" <<'PY'
import json,sys,hashlib
stock=sys.argv[1]
d=json.load(open('model.safetensors.index.json'));s=json.load(open(f'{stock}/model.safetensors.index.json'))
names=set(d['weight_map']);snames=set(s['weight_map'])
print('index weight_map:',len(names),'stock:',len(snames),'set_equal:',names==snames)
print('index total_size:',d['metadata'].get('total_size'),'stock:',s['metadata'].get('total_size'),'equal:',d['metadata'].get('total_size')==s['metadata'].get('total_size'))
c=json.load(open('config.json'))
print('config architectures:',c.get('architectures'),'dtype:',c.get('dtype') or c.get('torch_dtype'))
print('INDEX_SHA256', hashlib.sha256(open('model.safetensors.index.json','rb').read()).hexdigest())
PY
echo "== repo tokenizer set as shipped (kept aside as *.repo)"
sha256sum tokenizer.json tokenizer_config.json chat_template.jinja
python3 -c "import json;print('repo tokenizer.json truncation:',json.load(open('tokenizer.json')).get('truncation'))"
for f in tokenizer.json tokenizer_config.json chat_template.jinja; do mv -n "$f" "$f.repo"; cp "$STOCK/$f" "$f"; done
echo "== STOCK tokenizer set installed (sha256):"
sha256sum tokenizer.json tokenizer_config.json chat_template.jinja
python3 -c "import json;print('installed tokenizer.json truncation:',json.load(open('tokenizer.json')).get('truncation'))"
echo "== listing"; ls -la "$DEST"; du -sh "$DEST"
echo "== done $(date -u +%FT%TZ) shard_fail=$fail"