#!/bin/bash # pre-backup.sh — ana-docker. # Runs as root from resticprofile's `run-before`, before `restic backup`. # # Produces DB dumps in /var/lib/restic/stage/ so the nightly restic # snapshot captures consistent point-in-time data for services whose # raw volume files are not safe to back up live. # # Containers handled here: # - synapse-db (internal Postgres 16 — pg_dump) # - seafile-mysql (internal MariaDB 10.6 — mysqldump) # - vaultwarden (external Postgres on PFI-Postgres 10.250.50.80) # - gitea (`gitea dump` captures DB + repos + config + LFS) # - openwebui (local SQLite × 2 — main db + ChromaDB vector store) # # Also staged here (not a container): # - fv-gateway (OPNsense config.xml for the Fountain Valley edge # firewall, pulled over its WAN admin API). ana-docker is # one of the three egress addresses that firewall's # allowlist permits, which is why the pull lives on this # host rather than with the FV hardware — a site that has # lost power cannot back itself up. # # External DB credentials live in /etc/restic/dbcreds.env (root:600). # Template: configs/restic/ana-docker/dbcreds.env.example in the repo. # # Required tooling on the host: # docker — always # pg_dump — for vaultwarden external Postgres; install via # `apt install postgresql-client`. Without it, # the vaultwarden block logs a warning and skips. # # Intentionally NOT handled: # - mattermost (retired 2026-04-21 — stack dir lingers but is not running) # # Required database dump failures abort the backup instead of reporting a # successful snapshot without them. Previous staged dumps remain intact until # all required dumps succeed. Gitea scratch files are isolated and trap-cleaned. set -euo pipefail STAGE=${RESTIC_STAGE_DIR:-/var/lib/restic/stage} install -d -m 0700 "$STAGE" WORK=$(mktemp -d "$STAGE/.pending.XXXXXXXX") trap 'rm -rf -- "$WORK"' EXIT ERRORS=0 log() { printf '%s pre-backup(ana-docker): %s\n' "$(date -Is)" "$*"; } warn() { log "WARN: $*" >&2; } error() { ERRORS=$((ERRORS + 1)); warn "$*"; } # Load external-DB creds. Silently skipped if missing — individual blocks # that need them will log their own WARN. CREDS=${RESTIC_DB_CREDS_FILE:-/etc/restic/dbcreds.env} if [ -r "$CREDS" ]; then set -a; . "$CREDS"; set +a fi # FV gateway API creds, same shape and posture as dbcreds.env (root:600). # Template: configs/restic/ana-docker/fv-gateway.env.example in the repo. FVCREDS=${RESTIC_FV_CREDS_FILE:-/etc/restic/fv-gateway.env} if [ -r "$FVCREDS" ]; then set -a; . "$FVCREDS"; set +a fi # ---------- synapse (internal Postgres) --------------------------------------- if docker inspect synapse-db >/dev/null 2>&1; then log "dumping synapse postgres" docker exec synapse-db \ pg_dump -U synapse -d synapse -Fc --clean --if-exists \ > "$WORK/synapse.pg_dump" \ || error "synapse pg_dump failed" else log "skip synapse: container not present" fi # ---------- seafile (internal MariaDB) ---------------------------------------- if docker inspect seafile-mysql >/dev/null 2>&1; then log "dumping seafile mariadb" docker exec seafile-mysql sh -c \ 'mysqldump -uroot -p"$MYSQL_ROOT_PASSWORD" --all-databases --single-transaction --quick 2>/dev/null' \ | gzip -c > "$WORK/seafile.sql.gz" \ || error "seafile mysqldump failed" else log "skip seafile: container not present" fi # ---------- vaultwarden (external Postgres on PFI-Postgres 10.250.50.80) ------ # The vault moved from SQLite to external Postgres (date unclear). Any # /data/db.sqlite3* files in the container are stale leftovers and should # be deleted separately — this hook captures the live Postgres data only. if docker inspect vaultwarden >/dev/null 2>&1; then if [ -z "${VW_PGPASS:-}" ]; then error "vaultwarden: VW_PGPASS unset in /etc/restic/dbcreds.env" elif ! command -v pg_dump >/dev/null 2>&1; then error "vaultwarden: pg_dump not installed (apt install postgresql-client)" else log "dumping vaultwarden postgres (external: ${VW_PGHOST}:${VW_PGPORT:-5432})" PGPASSWORD="$VW_PGPASS" pg_dump \ -h "$VW_PGHOST" -p "${VW_PGPORT:-5432}" \ -U "$VW_PGUSER" -d "$VW_PGDB" \ -Fc --clean --if-exists \ > "$WORK/vaultwarden.pg_dump" \ || error "vaultwarden pg_dump failed" fi else log "skip vaultwarden: container not present" fi # ---------- gitea (native `gitea dump`) --------------------------------------- # `gitea dump` produces a single archive with the DB dump, repo trees, # config, LFS objects and attachments. The in-container command reads # its own DB creds (from GITEA__database__* env vars) — no external # creds needed. # # `--type tar` produces an UNCOMPRESSED tarball. Compressed formats # (zip/tar.gz) defeat restic's content-defined chunking: each day's # dump looks completely different to restic even when the underlying # data barely changed, so repo grows by ~full dump size every day. # Uncompressed tar lets restic dedup aggressively — after the first # snapshot, daily incrementals only cost the actual new-data delta. # # Tradeoff: on-disk stage file is larger (~2-3x the zip size) but that's # transient (deleted next run). Repo-side storage is much smaller. # # Size trimming flags if the tar becomes excessive: --skip-lfs-data, # --skip-repository, --skip-attachment-data. if docker inspect gitea >/dev/null 2>&1; then log "dumping gitea (gitea dump, uncompressed tar)" if docker exec -u git gitea sh -c ' set -eu scratch=$(mktemp -d /tmp/gitea-backup.XXXXXXXX) trap '\''rm -rf -- "$scratch"'\'' EXIT gitea dump -c /data/gitea/conf/app.ini --tempdir "$scratch" --file - --type tar ' > "$WORK/gitea-dump.tar"; then tar -tf "$WORK/gitea-dump.tar" >/dev/null \ || error "gitea archive validation failed" else error "gitea dump command failed (details above); previous stage preserved" fi else log "skip gitea: container not present" fi # ---------- openwebui (local SQLite × 2) -------------------------------------- # Two SQLite databases: main app (/app/backend/data/webui.db) and the # ChromaDB vector store (.../vector_db/chroma.sqlite3). Uses SQLite's # .backup command for a consistent snapshot if sqlite3 is available in # the container; falls back to restic's volume-level capture otherwise. OWUI_CONTAINER=openwebui-open-webui-1 if docker inspect "$OWUI_CONTAINER" >/dev/null 2>&1; then if docker exec "$OWUI_CONTAINER" sh -c 'command -v sqlite3 >/dev/null 2>&1'; then log "dumping openwebui sqlite (webui.db + chroma.sqlite3) via .backup" for pair in \ "/app/backend/data/webui.db:webui.db" \ "/app/backend/data/vector_db/chroma.sqlite3:chroma.sqlite3"; do src=${pair%:*}; dst=${pair#*:} if docker exec "$OWUI_CONTAINER" sqlite3 "$src" ".backup /tmp/$dst" 2>/dev/null; then docker cp "$OWUI_CONTAINER:/tmp/$dst" "$WORK/openwebui.$dst" \ && docker exec "$OWUI_CONTAINER" rm -f "/tmp/$dst" \ || warn "openwebui copy/cleanup failed for $dst" else warn "openwebui .backup failed for $src (missing file or locked?)" fi done else warn "openwebui: sqlite3 not in container — relying on restic volume-level capture" fi else log "skip openwebui: container not present" fi # ---------- fv-gateway (OPNsense edge firewall config) ------------------------- # Non-fatal by design: a firewall we cannot reach must not abort the nightly # database dumps. But a bad pull must not be PROMOTED either — the summary loop # below only rejects EMPTY files, and this endpoint answers an auth failure or a # captive portal with a perfectly non-empty HTML error page. So validate that the # body is really an OPNsense config and write nothing at all otherwise. FV_HOST=${FV_GATEWAY_HOST:-172.83.89.66} if [ -n "${FV_API_KEY:-}" ] && [ -n "${FV_API_SECRET:-}" ]; then log "pulling fv-gateway config from $FV_HOST" fv_tmp="$WORK/.fv-config.raw" if curl -fsS --max-time 60 -u "$FV_API_KEY:$FV_API_SECRET" \ -o "$fv_tmp" "http://$FV_HOST/api/core/backup/download/this" 2>/dev/null; then if head -c 200 "$fv_tmp" | grep -q ''; then mv -f -- "$fv_tmp" "$WORK/fv-gateway-config.xml" log "fv-gateway config staged ($(wc -c < "$WORK/fv-gateway-config.xml") bytes)" else rm -f -- "$fv_tmp" warn "fv-gateway: response was not an OPNsense config (auth failure or error page?)" fi else rm -f -- "$fv_tmp" warn "fv-gateway: config pull failed (site unreachable?)" fi else log "skip fv-gateway: no API creds in $FVCREDS" fi # ---------- summary ----------------------------------------------------------- if [ "$ERRORS" -ne 0 ]; then log "FAILED: $ERRORS required database dump(s) failed; previous stage preserved" exit 1 fi for dump in "$WORK"/*; do [ -f "$dump" ] || continue if [ ! -s "$dump" ]; then log "FAILED: empty dump ${dump##*/}; previous stage preserved" exit 1 fi done for dump in "$WORK"/*; do [ -f "$dump" ] || continue mv -f -- "$dump" "$STAGE/${dump##*/}" done size=$(du -sh "$STAGE" 2>/dev/null | awk '{print $1}') count=$(find "$STAGE" -type f | wc -l) log "stage ready: $count files, $size total"