diff --git a/configs/restic/esh-vm-db/README.md b/configs/restic/esh-vm-db/README.md index 345928a..5a44f2c 100644 --- a/configs/restic/esh-vm-db/README.md +++ b/configs/restic/esh-vm-db/README.md @@ -31,10 +31,35 @@ profile adds DB-level granularity via pre-backup dumps. 2. Checks mongo ping via `mongosh` — if OK, runs `mongodump` into `$STAGE/mongodump/` -Both dumps are atomic (write to `.tmp`, then rename). If either DB is -unreachable, the script logs a WARN and continues — a failed DB dump -doesn't abort the whole restic run, and restic falls back to whatever -stage content is left over from the prior successful dump. +PostgreSQL uses the local Unix socket and peer authentication as postgres, +not TCP localhost. Dumps are staged before replacement. If either DB dump +fails, the hook returns nonzero and aborts the backup, preserving that DB's +previous dump. The two databases are not a single transactional snapshot. +`RESTIC_STAGE_DIR` supports isolated regression tests. + +## Repair verified 2026-09-12 + +Weekly check failed September 6 on a repository connection timeout after boot. +Nightly backup returned success despite PostgreSQL TCP authentication failures, +reusing a dump last modified April 23. Fixed socket authentication, propagated +both DB failures, and added network-online ordering plus bounded retries to +both services (5-minute delay, maximum 3 starts per hour). + +Deploy with `playbooks/esh-vm-db-restic-repair.yaml`. Service drop-ins survive +regeneration of resticprofile's main units. Previous hook and PG dump retained +under `/var/lib/restic/repair-20260912/` (root-only). + +Fresh snapshot `bc5eeaff` at 07:01 PDT contains today's 3,460,215-byte compressed +PG dump. Retrieved FROM repository, decompressed successfully, and verified its +cluster-dump completion marker. This is not a full database restore test. +Mongo dump also completed. Weekly check rerun at 07:02 passed: 99 snapshots, +configured 10% data sample (19 packs), no errors. Both jobs Result=success, +no failed systemd units, timers active, PostgreSQL/MongoDB remain active. +Inactive/dead between scheduled runs is normal for these finite jobs. + +Tests: `python3 configs/restic/esh-vm-db/test_pre_backup.py` — three passing +regressions for successful peer-auth dump and preservation/failure propagation +for each database. No DB authentication policy or service restarts changed. ## Deploy (one-time) diff --git a/configs/restic/esh-vm-db/backup.contract.md b/configs/restic/esh-vm-db/backup.contract.md new file mode 100644 index 0000000..25224e1 --- /dev/null +++ b/configs/restic/esh-vm-db/backup.contract.md @@ -0,0 +1,6 @@ +Keep the existing hook entrypoint and staged dump names. Allow RESTIC_STAGE_DIR +for isolated tests. Use the local PostgreSQL socket with peer auth, never TCP. +Required PostgreSQL or MongoDB backup failure must exit nonzero and preserve +the prior successful dump. Successful dumps replace the old outputs. +Tests: PostgreSQL failure returns nonzero and preserves previous dump; successful +socket-authenticated PG and Mongo dumps publish fresh outputs. diff --git a/configs/restic/esh-vm-db/pre-backup.sh b/configs/restic/esh-vm-db/pre-backup.sh index 1c46806..a0ebb5e 100644 --- a/configs/restic/esh-vm-db/pre-backup.sh +++ b/configs/restic/esh-vm-db/pre-backup.sh @@ -10,29 +10,32 @@ # raw volume capture isn't in the restic source list so these dumps # are the ONLY way restic sees DB data. # -# Errors in individual blocks log a WARN; whole script doesn't abort. +# Required dump failures return nonzero so restic cannot report stale DB data +# as a successful fresh backup. Previous successful dumps are preserved. set -euo pipefail +ERRORS=0 -STAGE=/var/lib/restic/stage -install -d -o root -g root -m 0700 "$STAGE" +STAGE=${RESTIC_STAGE_DIR:-/var/lib/restic/stage} +install -d -m 0700 "$STAGE" log() { printf '%s pre-backup(esh-vm-db): %s\n' "$(date -Is)" "$*"; } warn() { printf '%s pre-backup(esh-vm-db): WARN: %s\n' "$(date -Is)" "$*" >&2; } +error() { ERRORS=$((ERRORS + 1)); warn "$*"; } # ---- Postgres ---------------------------------------------------------- PG_DUMP="$STAGE/pg_dumpall.sql.gz" -if sudo -u postgres pg_isready -h localhost -p 5432 > /dev/null 2>&1; then +if sudo -u postgres pg_isready -h /var/run/postgresql -p 5432 > /dev/null 2>&1; then log "pg_dumpall starting → $PG_DUMP" - if sudo -u postgres pg_dumpall -h localhost -p 5432 | gzip > "$PG_DUMP.tmp"; then + if (cd /; sudo -u postgres pg_dumpall -w -h /var/run/postgresql -p 5432) | gzip > "$PG_DUMP.tmp"; then mv "$PG_DUMP.tmp" "$PG_DUMP" log "pg_dumpall done ($(du -h "$PG_DUMP" | cut -f1))" else - warn "pg_dumpall failed (exit $?); keeping previous dump if any" + error "pg_dumpall failed (exit $?); keeping previous dump if any" rm -f "$PG_DUMP.tmp" fi else - warn "postgres not ready on :5432 — skipping pg_dumpall" + error "postgres not ready on :5432 — skipping pg_dumpall" fi # ---- MongoDB ----------------------------------------------------------- @@ -45,15 +48,19 @@ if mongosh --quiet --eval 'db.adminCommand({ping: 1}).ok' | grep -q '^1$'; then mv "$MONGO_DIR.tmp" "$MONGO_DIR" log "mongodump done ($(du -sh "$MONGO_DIR" | cut -f1))" else - warn "mongodump failed (exit $?); keeping previous dump if any" + error "mongodump failed (exit $?); keeping previous dump if any" rm -rf "$MONGO_DIR.tmp" fi else - warn "mongo not reachable via mongosh — skipping mongodump" + error "mongo not reachable via mongosh — skipping mongodump" fi # ---- Retention on stage dir -------------------------------------------- # restic dedupes identical dumps at the chunk level, so we can safely keep # overwriting the same files. No explicit rotation needed here. +if [ "$ERRORS" -ne 0 ]; then + log "pre-backup FAILED: $ERRORS required dump(s) failed" + exit 1 +fi log "pre-backup complete" diff --git a/configs/restic/esh-vm-db/retry.conf b/configs/restic/esh-vm-db/retry.conf new file mode 100644 index 0000000..eb34981 --- /dev/null +++ b/configs/restic/esh-vm-db/retry.conf @@ -0,0 +1,9 @@ +[Unit] +Wants=network-online.target +After=network-online.target +StartLimitIntervalSec=1h +StartLimitBurst=3 + +[Service] +Restart=on-failure +RestartSec=5min diff --git a/configs/restic/esh-vm-db/test_pre_backup.py b/configs/restic/esh-vm-db/test_pre_backup.py new file mode 100644 index 0000000..95d0ab0 --- /dev/null +++ b/configs/restic/esh-vm-db/test_pre_backup.py @@ -0,0 +1,39 @@ +import unittest,tempfile,pathlib,subprocess,os,gzip +HOOK=os.environ.get('HOOK',str(pathlib.Path(__file__).with_name('pre-backup.sh'))) +class HookTests(unittest.TestCase): + def setUp(self): + self.tmp=tempfile.TemporaryDirectory(); self.addCleanup(self.tmp.cleanup) + self.root=pathlib.Path(self.tmp.name); self.stage=self.root/'stage'; self.stage.mkdir() + self.old=self.stage/'pg_dumpall.sql.gz'; self.old.write_bytes(b'previous') + self.bin=self.root/'bin'; self.bin.mkdir() + self.command('sudo','''shift 2 +case "$1" in + pg_isready) exit 0;; + pg_dumpall) [ "${FAIL_PG:-0}" = 1 ] && exit 1 + case "$*" in *localhost*) exit 1;; esac + echo fresh-sql;; +esac +''') + self.command('mongosh','echo 1') + self.command('mongodump','mkdir -p "$2"; touch "$2/sample.bson"') + self.env=dict(os.environ,PATH=str(self.bin)+':'+os.environ['PATH'],RESTIC_STAGE_DIR=str(self.stage)) + def command(self,name,body): + p=self.bin/name; p.write_text('#!/bin/sh\n'+body+'\n'); p.chmod(0o755) + def run_hook(self,**env): + return subprocess.run(['bash',HOOK],env=dict(self.env,**env),capture_output=True,text=True) + def test_pg_failure_fails_backup_and_preserves_previous_dump(self): + r=self.run_hook(FAIL_PG='1') + self.assertNotEqual(r.returncode,0,r.stdout+r.stderr) + self.assertEqual(self.old.read_bytes(),b'previous') + def test_successful_socket_dump_replaces_previous(self): + r=self.run_hook() + self.assertEqual(r.returncode,0,r.stdout+r.stderr) + self.assertEqual(gzip.decompress(self.old.read_bytes()),b'fresh-sql\n') + self.assertTrue((self.stage/'mongodump/sample.bson').exists()) + def test_mongo_failure_fails_backup_and_preserves_previous(self): + d=self.stage/'mongodump'; d.mkdir(); (d/'old.bson').write_bytes(b'previous') + self.command('mongodump','exit 1') + r=self.run_hook() + self.assertNotEqual(r.returncode,0,r.stdout+r.stderr) + self.assertEqual((d/'old.bson').read_bytes(),b'previous') +if __name__=='__main__':unittest.main() diff --git a/persistent-memory.d/2026-09-12-esh-vm-db-restic-repair.md b/persistent-memory.d/2026-09-12-esh-vm-db-restic-repair.md new file mode 100644 index 0000000..c4515b1 --- /dev/null +++ b/persistent-memory.d/2026-09-12-esh-vm-db-restic-repair.md @@ -0,0 +1,19 @@ +# esh-vm-db Restic repaired + +User asked diagnose/fix two service issues. Only check was systemd-failed: +Sep6 repository network timeout after boot. Backup exited0 nightly but pg_dumpall +used TCP localhost, required a password, and silently kept April23 dump. +Mongo dumps worked. Root cause was the WARN-only hook masking PG failure. + +Fixed PG to /var/run/postgresql peer auth with -w, both DB failures now fail +backup preserving prior per-DB dump. Three red-green regression tests pass. +Added drop-ins to both jobs: network-online ordering, Restart=on-failure, +RestartSec=5min, StartLimitIntervalSec=1h, StartLimitBurst=3. + +Verified real backup bc5eeaff 07:01PDT; fresh PG gzip 3460215 bytes. Retrieved +from repo, decompression and completion marker pass (not full restore). +Check at07:02 passed 99 snapshots/10% data (19 packs). No failed units remain; +both timers active, PG and Mongo active. Inactive/dead between jobs is normal. +Old hook/dump preserved root-only /var/lib/restic/repair-20260912. +Canonical configs/restic/esh-vm-db and playbooks/esh-vm-db-restic-repair.yaml. +No DB restarts or auth-policy changes; changes saved locally, no commit.