vh
3fec668bf2
- scripts/erp-tune-gx10/pull-verify-jenerallee78.sh + base-pin-jenerallee78-shards.txt: revision-pinned root-shard pull, 32/32 sha256+size vs brokkr-smithy pins, index set-equal to stock, STOCK tokenizer set installed over the repo's (which bakes in a 256-token truncation); repo originals kept as *.repo - scripts/erp-tune-gx10/run-06-gx10.json + launch-run-06.sh: run-05 config with the base swapped, recipe-r6, survivors-r5 verbatim, stock template path - docs/runbooks/gx10-run-06.md: pull/verify record, free-check result (encode reproduces run 5 exactly), hf download --include gotcha, gate naming (erp-seat-base-ara / erp-tune-v6)
55 lines
3.1 KiB
Bash
55 lines
3.1 KiB
Bash
#!/usr/bin/env bash
|
|
# Pull + verify jenerallee78/gemma-4-26B-A4B-it-ara-abliterated @ 0631379a onto pfi-gx10.
|
|
# Root shards + small files only; no GGUFs, no mlx-4bit. Verifies bytes against the
|
|
# brokkr-smithy pins (base-pin-jenerallee78.json) and installs the STOCK tokenizer set.
|
|
# Expects ~/erp-tune/base-pin-jenerallee78-shards.txt (file sha256 bytes per line).
|
|
set -uo pipefail
|
|
export PATH="$HOME/.local/bin:$PATH"
|
|
REPO=jenerallee78/gemma-4-26B-A4B-it-ara-abliterated
|
|
REV=0631379a3d859e0059bc8d9b21ab5b654dfc272c
|
|
DEST=$HOME/models/gemma4-26b-a4b-it-ara-abliterated-jenerallee78-0631379a
|
|
STOCK=$HOME/models/gemma4-26b-a4b-it-bf16
|
|
PINS=$HOME/erp-tune/base-pin-jenerallee78-shards.txt
|
|
echo "== start $(date -u +%FT%TZ) on $(hostname)"
|
|
mkdir -p "$DEST"
|
|
echo "== download"
|
|
uv run --quiet --with 'huggingface_hub[hf_transfer]' hf download "$REPO" --revision "$REV" \
|
|
--local-dir "$DEST" \
|
|
--include 'model-*-of-00032.safetensors' --include 'config.json' --include 'generation_config.json' \
|
|
--include 'model.safetensors.index.json' --include 'chat_template.jinja' --include 'ara_config.json' --include 'README.md' \
|
|
--include 'tokenizer.json' --include 'tokenizer_config.json'
|
|
rc=$?
|
|
echo "== download rc=$rc $(date -u +%FT%TZ)"
|
|
[ $rc -eq 0 ] || { echo "DOWNLOAD FAILED rc=$rc"; exit 2; }
|
|
cd "$DEST"
|
|
echo "== shard sha256 vs pins"
|
|
fail=0
|
|
while read -r f oid bytes; do
|
|
[ -n "$f" ] || continue
|
|
sz=$(stat -c %s "$f" 2>/dev/null || echo MISSING)
|
|
got=$(sha256sum "$f" 2>/dev/null | cut -d' ' -f1)
|
|
if [ "$sz" = "$bytes" ] && [ "$got" = "$oid" ]; then echo "OK $f"; else echo "FAIL $f size=$sz want=$bytes sha=$got want=$oid"; fail=$((fail+1)); fi
|
|
done < "$PINS"
|
|
echo "== shard result: fail=$fail"
|
|
echo "== index + config checks"
|
|
python3 - "$STOCK" <<'PY'
|
|
import json,sys,hashlib
|
|
stock=sys.argv[1]
|
|
d=json.load(open('model.safetensors.index.json'));s=json.load(open(f'{stock}/model.safetensors.index.json'))
|
|
names=set(d['weight_map']);snames=set(s['weight_map'])
|
|
print('index weight_map:',len(names),'stock:',len(snames),'set_equal:',names==snames)
|
|
print('index total_size:',d['metadata'].get('total_size'),'stock:',s['metadata'].get('total_size'),'equal:',d['metadata'].get('total_size')==s['metadata'].get('total_size'))
|
|
c=json.load(open('config.json'))
|
|
print('config architectures:',c.get('architectures'),'dtype:',c.get('dtype') or c.get('torch_dtype'))
|
|
print('INDEX_SHA256', hashlib.sha256(open('model.safetensors.index.json','rb').read()).hexdigest())
|
|
PY
|
|
echo "== repo tokenizer set as shipped (kept aside as *.repo)"
|
|
sha256sum tokenizer.json tokenizer_config.json chat_template.jinja
|
|
python3 -c "import json;print('repo tokenizer.json truncation:',json.load(open('tokenizer.json')).get('truncation'))"
|
|
for f in tokenizer.json tokenizer_config.json chat_template.jinja; do mv -n "$f" "$f.repo"; cp "$STOCK/$f" "$f"; done
|
|
echo "== STOCK tokenizer set installed (sha256):"
|
|
sha256sum tokenizer.json tokenizer_config.json chat_template.jinja
|
|
python3 -c "import json;print('installed tokenizer.json truncation:',json.load(open('tokenizer.json')).get('truncation'))"
|
|
echo "== listing"; ls -la "$DEST"; du -sh "$DEST"
|
|
echo "== done $(date -u +%FT%TZ) shard_fail=$fail"
|