feat(restic fv-ml1): Diariz in the host backup — run-before pg_dump (soft-fail), stage dir, model cache + raw pgdata excluded; first snapshot 776b700b

This commit is contained in:
vh
2026-10-19 11:23:37 -07:00
parent 366d6fdccd
commit f267d8997a
3 changed files with 52 additions and 0 deletions
+20
View File
@@ -0,0 +1,20 @@
#!/bin/sh
# run-before for fv-ml1's restic backup: a consistent pg_dump of the Diariz pilot's postgres (2026-10-19, Prime: "back it
# up now"). The raw diariz_pgdata volume is excluded, because live PG files copied file by file are not a consistent copy.
#
# SOFT-FAIL BY DESIGN: this is the host's ONLY backup (bare metal), and a pilot app must never block it. On any failure
# the previous dump is REMOVED, so a stale dump is never snapshotted as current. Then a warning, and exit 0 so the rest of
# the host backup proceeds.
set -u
d=/var/lib/restic/stage/diariz; out=$d/diariz.pgdump
install -d -m 0700 "$d"
if docker inspect -f '{{.State.Running}}' diariz-postgres-1 2>/dev/null | grep -q true &&
docker exec diariz-postgres-1 pg_dump -U diariz -d diariz -Fc > "$out.tmp" 2>"$d/dump.err" &&
[ -s "$out.tmp" ]; then
mv -f "$out.tmp" "$out"; date -u +%FT%TZ > "$d/dumped-at"; rm -f "$d/dump.err"
echo "diariz-dump: ok, $(stat -c %s "$out") bytes"
else
rm -f "$out" "$out.tmp" "$d/dumped-at"
echo "diariz-dump: WARNING - no dump this run (postgres down or pg_dump failed: $(head -c 300 "$d/dump.err" 2>/dev/null)); host backup continues" >&2
fi
exit 0
+8
View File
@@ -15,6 +15,9 @@
# the only thing under /tank that upstream cannot hand back — each is # the only thing under /tank that upstream cannot hand back — each is
# ~300 MB and cost 7-14 h of GPU time, and the ERP line's adapters exist # ~300 MB and cost 7-14 h of GPU time, and the ERP line's adapters exist
# nowhere else but pfi-gx10's single NVMe and here. ~2.4 GB total. # nowhere else but pfi-gx10's single NVMe and here. ~2.4 GB total.
# - ONE DB dump (2026-10-19): the Diariz pilot's postgres, by run-before /etc/restic/diariz-dump.sh;
# its raw pgdata volume is excluded. Its recordings (diariz_s3data, SeaweedFS) and DataProtection keys
# (diariz_apikeys) ride the volume backup. Before Diariz:
# - No DB dumps needed. None of the stacks on this host (llama-swap, # - No DB dumps needed. None of the stacks on this host (llama-swap,
# vllm-qwen3, comfyui, kokoro, parakeet, vibevoice, beszel-agent, # vllm-qwen3, comfyui, kokoro, parakeet, vibevoice, beszel-agent,
# dozzle-agent, dockge) store relational data. # dozzle-agent, dockge) store relational data.
@@ -39,6 +42,8 @@ default:
backup: backup:
verbose: 1 verbose: 1
run-before:
- /etc/restic/diariz-dump.sh # Diariz pilot pg_dump -> /var/lib/restic/stage/diariz (soft-fail; see the script)
run-after: run-after:
- date +%s > /var/lib/restic/last-success - date +%s > /var/lib/restic/last-success
source: source:
@@ -46,6 +51,7 @@ default:
- /etc # host config (systemd units, chrony, apparmor, ssh, etc.) - /etc # host config (systemd units, chrony, apparmor, ssh, etc.)
- /root # root shell history, ssh keys, any ad-hoc scripts - /root # root shell history, ssh keys, any ad-hoc scripts
- /var/lib/docker/volumes # named volumes (small; models live on /tank, not here) - /var/lib/docker/volumes # named volumes (small; models live on /tank, not here)
- /var/lib/restic/stage # dumps written by run-before (diariz.pgdump)
- /tank/erp-tune/run-*/adapter # non-regenerable trained LoRA adapters (see header) - /tank/erp-tune/run-*/adapter # non-regenerable trained LoRA adapters (see header)
exclude: exclude:
# Docker internals we never want in a backup # Docker internals we never want in a backup
@@ -55,6 +61,8 @@ default:
# from /tank/ are already outside the source list; parakeet is the # from /tank/ are already outside the source list; parakeet is the
# odd one out using a docker volume). Regenerable from Hugging Face. # odd one out using a docker volume). Regenerable from Hugging Face.
- /var/lib/docker/volumes/parakeet_parakeet_cache - /var/lib/docker/volumes/parakeet_parakeet_cache
- /var/lib/docker/volumes/diariz_workercache # Diariz model weights (~3.4 GB), regenerable from HF
- /var/lib/docker/volumes/diariz_pgdata # live PG files; the consistent copy is stage/diariz/diariz.pgdump
# Ephemeral / regenerable junk # Ephemeral / regenerable junk
- /opt/docker/compose/*/logs - /opt/docker/compose/*/logs
- "**/*.log" - "**/*.log"
+24
View File
@@ -0,0 +1,24 @@
# fv-ml1: add the Diariz pilot to the host's restic backup (Prime 2026-10-19 "back it up now"): a run-before pg_dump
# (soft-fail), /var/lib/restic/stage in the sources, the regenerable model cache and the raw pgdata excluded. Then one
# backup run now and a check that the snapshot holds the dump and the recordings volume.
steps:
- {name: dump hook, upload: {src: configs/restic/ana-ml2/diariz-dump.sh, dest: /etc/restic/diariz-dump.sh, mode: "0755"}, sudo: true}
- name: Back up the live profile
sudo: true
shell: cp -p /etc/restic/profiles.yaml /etc/restic/profiles.yaml.bak-20261019-diariz
creates: /etc/restic/profiles.yaml.bak-20261019-diariz
- {name: profile, upload: {src: configs/restic/ana-ml2/profiles.yaml, dest: /etc/restic/profiles.yaml, mode: "0644"}, sudo: true}
- name: The profile parses
sudo: true
shell: cd /etc/restic && resticprofile show 2>&1 | grep -E "run-before|diariz|stage" | head -6
changed_when: "false"
- name: Back up NOW (the whole host, incremental)
sudo: true
shell: cd /etc/restic && resticprofile backup 2>&1 | grep -E "diariz-dump|processed|snapshot .* saved|Fatal|error" | tail -6
verify:
- name: The latest snapshot holds the dump and the recordings volume
sudo: true
shell: |
cd /etc/restic && . /etc/restic/restic.env 2>/dev/null
restic -r "$(cat /etc/restic/repository)" --password-file /etc/restic/password ls latest /var/lib/restic/stage/diariz /var/lib/docker/volumes/diariz_s3data 2>/dev/null | grep -c -E "diariz.pgdump|diariz_s3data" ; ls -la /var/lib/restic/stage/diariz/
changed_when: "false"