feat(restic fv-ml1): Diariz in the host backup — run-before pg_dump (soft-fail), stage dir, model cache + raw pgdata excluded; first snapshot 776b700b
This commit is contained in:
Executable
+20
@@ -0,0 +1,20 @@
|
|||||||
|
#!/bin/sh
|
||||||
|
# run-before for fv-ml1's restic backup: a consistent pg_dump of the Diariz pilot's postgres (2026-10-19, Prime: "back it
|
||||||
|
# up now"). The raw diariz_pgdata volume is excluded, because live PG files copied file by file are not a consistent copy.
|
||||||
|
#
|
||||||
|
# SOFT-FAIL BY DESIGN: this is the host's ONLY backup (bare metal), and a pilot app must never block it. On any failure
|
||||||
|
# the previous dump is REMOVED, so a stale dump is never snapshotted as current. Then a warning, and exit 0 so the rest of
|
||||||
|
# the host backup proceeds.
|
||||||
|
set -u
|
||||||
|
d=/var/lib/restic/stage/diariz; out=$d/diariz.pgdump
|
||||||
|
install -d -m 0700 "$d"
|
||||||
|
if docker inspect -f '{{.State.Running}}' diariz-postgres-1 2>/dev/null | grep -q true &&
|
||||||
|
docker exec diariz-postgres-1 pg_dump -U diariz -d diariz -Fc > "$out.tmp" 2>"$d/dump.err" &&
|
||||||
|
[ -s "$out.tmp" ]; then
|
||||||
|
mv -f "$out.tmp" "$out"; date -u +%FT%TZ > "$d/dumped-at"; rm -f "$d/dump.err"
|
||||||
|
echo "diariz-dump: ok, $(stat -c %s "$out") bytes"
|
||||||
|
else
|
||||||
|
rm -f "$out" "$out.tmp" "$d/dumped-at"
|
||||||
|
echo "diariz-dump: WARNING - no dump this run (postgres down or pg_dump failed: $(head -c 300 "$d/dump.err" 2>/dev/null)); host backup continues" >&2
|
||||||
|
fi
|
||||||
|
exit 0
|
||||||
@@ -15,6 +15,9 @@
|
|||||||
# the only thing under /tank that upstream cannot hand back — each is
|
# the only thing under /tank that upstream cannot hand back — each is
|
||||||
# ~300 MB and cost 7-14 h of GPU time, and the ERP line's adapters exist
|
# ~300 MB and cost 7-14 h of GPU time, and the ERP line's adapters exist
|
||||||
# nowhere else but pfi-gx10's single NVMe and here. ~2.4 GB total.
|
# nowhere else but pfi-gx10's single NVMe and here. ~2.4 GB total.
|
||||||
|
# - ONE DB dump (2026-10-19): the Diariz pilot's postgres, by run-before /etc/restic/diariz-dump.sh;
|
||||||
|
# its raw pgdata volume is excluded. Its recordings (diariz_s3data, SeaweedFS) and DataProtection keys
|
||||||
|
# (diariz_apikeys) ride the volume backup. Before Diariz:
|
||||||
# - No DB dumps needed. None of the stacks on this host (llama-swap,
|
# - No DB dumps needed. None of the stacks on this host (llama-swap,
|
||||||
# vllm-qwen3, comfyui, kokoro, parakeet, vibevoice, beszel-agent,
|
# vllm-qwen3, comfyui, kokoro, parakeet, vibevoice, beszel-agent,
|
||||||
# dozzle-agent, dockge) store relational data.
|
# dozzle-agent, dockge) store relational data.
|
||||||
@@ -39,6 +42,8 @@ default:
|
|||||||
|
|
||||||
backup:
|
backup:
|
||||||
verbose: 1
|
verbose: 1
|
||||||
|
run-before:
|
||||||
|
- /etc/restic/diariz-dump.sh # Diariz pilot pg_dump -> /var/lib/restic/stage/diariz (soft-fail; see the script)
|
||||||
run-after:
|
run-after:
|
||||||
- date +%s > /var/lib/restic/last-success
|
- date +%s > /var/lib/restic/last-success
|
||||||
source:
|
source:
|
||||||
@@ -46,6 +51,7 @@ default:
|
|||||||
- /etc # host config (systemd units, chrony, apparmor, ssh, etc.)
|
- /etc # host config (systemd units, chrony, apparmor, ssh, etc.)
|
||||||
- /root # root shell history, ssh keys, any ad-hoc scripts
|
- /root # root shell history, ssh keys, any ad-hoc scripts
|
||||||
- /var/lib/docker/volumes # named volumes (small; models live on /tank, not here)
|
- /var/lib/docker/volumes # named volumes (small; models live on /tank, not here)
|
||||||
|
- /var/lib/restic/stage # dumps written by run-before (diariz.pgdump)
|
||||||
- /tank/erp-tune/run-*/adapter # non-regenerable trained LoRA adapters (see header)
|
- /tank/erp-tune/run-*/adapter # non-regenerable trained LoRA adapters (see header)
|
||||||
exclude:
|
exclude:
|
||||||
# Docker internals we never want in a backup
|
# Docker internals we never want in a backup
|
||||||
@@ -55,6 +61,8 @@ default:
|
|||||||
# from /tank/ are already outside the source list; parakeet is the
|
# from /tank/ are already outside the source list; parakeet is the
|
||||||
# odd one out using a docker volume). Regenerable from Hugging Face.
|
# odd one out using a docker volume). Regenerable from Hugging Face.
|
||||||
- /var/lib/docker/volumes/parakeet_parakeet_cache
|
- /var/lib/docker/volumes/parakeet_parakeet_cache
|
||||||
|
- /var/lib/docker/volumes/diariz_workercache # Diariz model weights (~3.4 GB), regenerable from HF
|
||||||
|
- /var/lib/docker/volumes/diariz_pgdata # live PG files; the consistent copy is stage/diariz/diariz.pgdump
|
||||||
# Ephemeral / regenerable junk
|
# Ephemeral / regenerable junk
|
||||||
- /opt/docker/compose/*/logs
|
- /opt/docker/compose/*/logs
|
||||||
- "**/*.log"
|
- "**/*.log"
|
||||||
|
|||||||
@@ -0,0 +1,24 @@
|
|||||||
|
# fv-ml1: add the Diariz pilot to the host's restic backup (Prime 2026-10-19 "back it up now"): a run-before pg_dump
|
||||||
|
# (soft-fail), /var/lib/restic/stage in the sources, the regenerable model cache and the raw pgdata excluded. Then one
|
||||||
|
# backup run now and a check that the snapshot holds the dump and the recordings volume.
|
||||||
|
steps:
|
||||||
|
- {name: dump hook, upload: {src: configs/restic/ana-ml2/diariz-dump.sh, dest: /etc/restic/diariz-dump.sh, mode: "0755"}, sudo: true}
|
||||||
|
- name: Back up the live profile
|
||||||
|
sudo: true
|
||||||
|
shell: cp -p /etc/restic/profiles.yaml /etc/restic/profiles.yaml.bak-20261019-diariz
|
||||||
|
creates: /etc/restic/profiles.yaml.bak-20261019-diariz
|
||||||
|
- {name: profile, upload: {src: configs/restic/ana-ml2/profiles.yaml, dest: /etc/restic/profiles.yaml, mode: "0644"}, sudo: true}
|
||||||
|
- name: The profile parses
|
||||||
|
sudo: true
|
||||||
|
shell: cd /etc/restic && resticprofile show 2>&1 | grep -E "run-before|diariz|stage" | head -6
|
||||||
|
changed_when: "false"
|
||||||
|
- name: Back up NOW (the whole host, incremental)
|
||||||
|
sudo: true
|
||||||
|
shell: cd /etc/restic && resticprofile backup 2>&1 | grep -E "diariz-dump|processed|snapshot .* saved|Fatal|error" | tail -6
|
||||||
|
verify:
|
||||||
|
- name: The latest snapshot holds the dump and the recordings volume
|
||||||
|
sudo: true
|
||||||
|
shell: |
|
||||||
|
cd /etc/restic && . /etc/restic/restic.env 2>/dev/null
|
||||||
|
restic -r "$(cat /etc/restic/repository)" --password-file /etc/restic/password ls latest /var/lib/restic/stage/diariz /var/lib/docker/volumes/diariz_s3data 2>/dev/null | grep -c -E "diariz.pgdump|diariz_s3data" ; ls -la /var/lib/restic/stage/diariz/
|
||||||
|
changed_when: "false"
|
||||||
Reference in New Issue
Block a user