vh
6e8da46a28
Operator decision. paperless-ngx's database lives on esh-vm-db and is backed up at the source by that host's fail-closed pg_dumpall. esh-docker-vm's second copy had failed auth every night since 2026-04-24 behind a WARN. Its `> paperless.pg_dump` redirect left a 0-byte file in every snapshot (confirmed in snapshot 6ec9f74f), which looked like a dump but held nothing. The block was the only consumer of /etc/restic/dbcreds.env, so the creds loader is gone and the template dbcreds.env.example is deleted. The host file was moved (not deleted) to /var/lib/restic/repair-20260923/. Paperless's media volumes are still captured under /var/lib/docker/volumes. Also fixes ownership. elway's sudo upload does scp-as-user then `sudo mv`, so the hook deployed at 08:04 landed infra-ops:infra-ops even though root executes it. Both esh-docker-vm playbooks now chown it back to root and verify root:root 700. Verified: the live hook hash matches canonical (e0d3ddcef1bddf43), and the manual backup saved snapshot decfae71 with 3 staged dumps and no WARN lines.
105 lines
4.4 KiB
Bash
105 lines
4.4 KiB
Bash
#!/bin/bash
|
|
# pre-backup.sh — esh-docker-vm.
|
|
# Runs as root from resticprofile's `run-before`.
|
|
#
|
|
# Produces consistent DB dumps in /var/lib/restic/stage/ for services
|
|
# whose raw volume files risk inconsistency during live restic capture.
|
|
#
|
|
# Unique approach for this host: most containers don't bundle sqlite3,
|
|
# so we run sqlite3 from the HOST against the volume bind-mount paths.
|
|
# Requires sqlite3 installed on esh-docker-vm (apt install sqlite3).
|
|
#
|
|
# Services handled:
|
|
# - home-assistant (local SQLite in volume — sqlite3 .backup from host)
|
|
# - calibre-web-automated (local SQLite — sqlite3 .backup inside container, has sqlite3)
|
|
# - pgadmin (local SQLite in volume — sqlite3 .backup from host)
|
|
#
|
|
# uptime-kuma was here until it moved to ana-docker (2026-09-22). Its block
|
|
# was removed 2026-09-23: with no container left to match, its unguarded
|
|
# `docker ps | grep` lookup exited 1 and set -e aborted this script, and
|
|
# resticprofile then skipped the WHOLE host backup (stale 09-22 → 09-23).
|
|
#
|
|
# paperless-ngx's Postgres (on esh-vm-db, 10.0.50.60) was also dumped from
|
|
# here until 2026-09-23, when the block was removed by operator decision. It
|
|
# is backed up at the source by esh-vm-db's own fail-closed pg_dumpall, and
|
|
# this copy had failed auth every night since 2026-04-24 behind a WARN
|
|
# nobody read, leaving a 0-byte paperless.pg_dump in every snapshot. That
|
|
# block was the only consumer of /etc/restic/dbcreds.env, so the creds
|
|
# loader went with it.
|
|
#
|
|
# Errors in individual blocks log a WARN; whole script doesn't abort.
|
|
# That only holds if every lookup for an optional service sits inside an
|
|
# `if` test or ends in `|| true` — under set -euo pipefail a bare
|
|
# `x=$(… | grep …)` that matches nothing kills the script, and a failed
|
|
# run-before hook means NO snapshot at all, not a partial one.
|
|
|
|
set -euo pipefail
|
|
|
|
STAGE=/var/lib/restic/stage
|
|
install -d -o root -g root -m 0700 "$STAGE"
|
|
|
|
log() { printf '%s pre-backup(esh-docker-vm): %s\n' "$(date -Is)" "$*"; }
|
|
warn() { log "WARN: $*" >&2; }
|
|
|
|
# Purge previous stage so stale dumps don't pile up in the snapshot.
|
|
find "$STAGE" -mindepth 1 -maxdepth 1 -exec rm -rf {} +
|
|
|
|
# Helper: host-side sqlite .backup against a volume-bind path.
|
|
# $1 = source .db path (host absolute, typically under /var/lib/docker/volumes/.../_data/)
|
|
# $2 = stage filename (just the leaf name)
|
|
host_sqlite_backup() {
|
|
local src="$1" dst="$STAGE/$2"
|
|
if ! command -v sqlite3 >/dev/null 2>&1; then
|
|
warn "sqlite3 not on host (apt install sqlite3) — skipping $2"
|
|
return 1
|
|
fi
|
|
if [ ! -f "$src" ]; then
|
|
warn "source db missing: $src — skipping $2"
|
|
return 1
|
|
fi
|
|
if sqlite3 "$src" ".backup '$dst'" 2>/dev/null; then
|
|
log "dumped $2 ($(du -h "$dst" 2>/dev/null | cut -f1))"
|
|
else
|
|
warn "sqlite3 .backup failed for $src"
|
|
rm -f "$dst"
|
|
return 1
|
|
fi
|
|
}
|
|
|
|
# ---------- home-assistant (SQLite in named volume, host-side .backup) --------
|
|
# HA's DB is ~50MB and actively written. SQLite .backup is the proper way
|
|
# to grab a consistent snapshot while HA is running.
|
|
if docker inspect homeassistant >/dev/null 2>&1; then
|
|
HA_DB="/var/lib/docker/volumes/homeassistant_homeassistant_data/_data/home-assistant_v2.db"
|
|
host_sqlite_backup "$HA_DB" "home-assistant.sqlite3" || true
|
|
else
|
|
log "skip home-assistant: container not present"
|
|
fi
|
|
|
|
# ---------- calibre-web-automated (SQLite, sqlite3 inside container) ---------
|
|
if docker inspect calibre-web-automated >/dev/null 2>&1; then
|
|
log "dumping CWA sqlite via in-container .backup"
|
|
if docker exec calibre-web-automated sqlite3 /config/app.db ".backup /tmp/cwa-app.db" 2>/dev/null; then
|
|
docker cp calibre-web-automated:/tmp/cwa-app.db "$STAGE/calibre-web-automated.app.db" \
|
|
&& docker exec calibre-web-automated rm -f /tmp/cwa-app.db \
|
|
|| warn "CWA copy/cleanup failed"
|
|
else
|
|
warn "CWA sqlite .backup failed"
|
|
fi
|
|
else
|
|
log "skip calibre-web-automated: container not present"
|
|
fi
|
|
|
|
# ---------- pgadmin (SQLite in named volume, host-side .backup) --------------
|
|
if docker inspect pgadmin4_container >/dev/null 2>&1; then
|
|
PG_DB="/var/lib/docker/volumes/pgadmin_pgadmin-data/_data/pgadmin4.db"
|
|
host_sqlite_backup "$PG_DB" "pgadmin4.db" || true
|
|
else
|
|
log "skip pgadmin: container not present"
|
|
fi
|
|
|
|
# ---------- summary -----------------------------------------------------------
|
|
size=$(du -sh "$STAGE" 2>/dev/null | awk '{print $1}')
|
|
count=$(find "$STAGE" -type f | wc -l)
|
|
log "stage ready: $count files, $size total"
|