diff --git a/configs/restic/ana-ml2/diariz-dump.sh b/configs/restic/ana-ml2/diariz-dump.sh new file mode 100755 index 0000000..4468fc3 --- /dev/null +++ b/configs/restic/ana-ml2/diariz-dump.sh @@ -0,0 +1,20 @@ +#!/bin/sh +# run-before for fv-ml1's restic backup: a consistent pg_dump of the Diariz pilot's postgres (2026-10-19, Prime: "back it +# up now"). The raw diariz_pgdata volume is excluded, because live PG files copied file by file are not a consistent copy. +# +# SOFT-FAIL BY DESIGN: this is the host's ONLY backup (bare metal), and a pilot app must never block it. On any failure +# the previous dump is REMOVED, so a stale dump is never snapshotted as current. Then a warning, and exit 0 so the rest of +# the host backup proceeds. +set -u +d=/var/lib/restic/stage/diariz; out=$d/diariz.pgdump +install -d -m 0700 "$d" +if docker inspect -f '{{.State.Running}}' diariz-postgres-1 2>/dev/null | grep -q true && + docker exec diariz-postgres-1 pg_dump -U diariz -d diariz -Fc > "$out.tmp" 2>"$d/dump.err" && + [ -s "$out.tmp" ]; then + mv -f "$out.tmp" "$out"; date -u +%FT%TZ > "$d/dumped-at"; rm -f "$d/dump.err" + echo "diariz-dump: ok, $(stat -c %s "$out") bytes" +else + rm -f "$out" "$out.tmp" "$d/dumped-at" + echo "diariz-dump: WARNING - no dump this run (postgres down or pg_dump failed: $(head -c 300 "$d/dump.err" 2>/dev/null)); host backup continues" >&2 +fi +exit 0 diff --git a/configs/restic/ana-ml2/profiles.yaml b/configs/restic/ana-ml2/profiles.yaml index ee973ff..5d83a4a 100644 --- a/configs/restic/ana-ml2/profiles.yaml +++ b/configs/restic/ana-ml2/profiles.yaml @@ -15,6 +15,9 @@ # the only thing under /tank that upstream cannot hand back — each is # ~300 MB and cost 7-14 h of GPU time, and the ERP line's adapters exist # nowhere else but pfi-gx10's single NVMe and here. ~2.4 GB total. +# - ONE DB dump (2026-10-19): the Diariz pilot's postgres, by run-before /etc/restic/diariz-dump.sh; +# its raw pgdata volume is excluded. Its recordings (diariz_s3data, SeaweedFS) and DataProtection keys +# (diariz_apikeys) ride the volume backup. Before Diariz: # - No DB dumps needed. None of the stacks on this host (llama-swap, # vllm-qwen3, comfyui, kokoro, parakeet, vibevoice, beszel-agent, # dozzle-agent, dockge) store relational data. @@ -39,6 +42,8 @@ default: backup: verbose: 1 + run-before: + - /etc/restic/diariz-dump.sh # Diariz pilot pg_dump -> /var/lib/restic/stage/diariz (soft-fail; see the script) run-after: - date +%s > /var/lib/restic/last-success source: @@ -46,6 +51,7 @@ default: - /etc # host config (systemd units, chrony, apparmor, ssh, etc.) - /root # root shell history, ssh keys, any ad-hoc scripts - /var/lib/docker/volumes # named volumes (small; models live on /tank, not here) + - /var/lib/restic/stage # dumps written by run-before (diariz.pgdump) - /tank/erp-tune/run-*/adapter # non-regenerable trained LoRA adapters (see header) exclude: # Docker internals we never want in a backup @@ -55,6 +61,8 @@ default: # from /tank/ are already outside the source list; parakeet is the # odd one out using a docker volume). Regenerable from Hugging Face. - /var/lib/docker/volumes/parakeet_parakeet_cache + - /var/lib/docker/volumes/diariz_workercache # Diariz model weights (~3.4 GB), regenerable from HF + - /var/lib/docker/volumes/diariz_pgdata # live PG files; the consistent copy is stage/diariz/diariz.pgdump # Ephemeral / regenerable junk - /opt/docker/compose/*/logs - "**/*.log" diff --git a/playbooks/fv-ml1-restic-diariz.yaml b/playbooks/fv-ml1-restic-diariz.yaml new file mode 100644 index 0000000..afe4926 --- /dev/null +++ b/playbooks/fv-ml1-restic-diariz.yaml @@ -0,0 +1,24 @@ +# fv-ml1: add the Diariz pilot to the host's restic backup (Prime 2026-10-19 "back it up now"): a run-before pg_dump +# (soft-fail), /var/lib/restic/stage in the sources, the regenerable model cache and the raw pgdata excluded. Then one +# backup run now and a check that the snapshot holds the dump and the recordings volume. +steps: + - {name: dump hook, upload: {src: configs/restic/ana-ml2/diariz-dump.sh, dest: /etc/restic/diariz-dump.sh, mode: "0755"}, sudo: true} + - name: Back up the live profile + sudo: true + shell: cp -p /etc/restic/profiles.yaml /etc/restic/profiles.yaml.bak-20261019-diariz + creates: /etc/restic/profiles.yaml.bak-20261019-diariz + - {name: profile, upload: {src: configs/restic/ana-ml2/profiles.yaml, dest: /etc/restic/profiles.yaml, mode: "0644"}, sudo: true} + - name: The profile parses + sudo: true + shell: cd /etc/restic && resticprofile show 2>&1 | grep -E "run-before|diariz|stage" | head -6 + changed_when: "false" + - name: Back up NOW (the whole host, incremental) + sudo: true + shell: cd /etc/restic && resticprofile backup 2>&1 | grep -E "diariz-dump|processed|snapshot .* saved|Fatal|error" | tail -6 +verify: + - name: The latest snapshot holds the dump and the recordings volume + sudo: true + shell: | + cd /etc/restic && . /etc/restic/restic.env 2>/dev/null + restic -r "$(cat /etc/restic/repository)" --password-file /etc/restic/password ls latest /var/lib/restic/stage/diariz /var/lib/docker/volumes/diariz_s3data 2>/dev/null | grep -c -E "diariz.pgdump|diariz_s3data" ; ls -la /var/lib/restic/stage/diariz/ + changed_when: "false"