esh-pfi-infrastructure/configs/restic/ana-docker/pre-backup.sh

#!/bin/bash
# pre-backup.sh — ana-docker.
# Runs as root from resticprofile's `run-before`, before `restic backup`.
#
# Produces DB dumps in /var/lib/restic/stage/ so the nightly restic
# snapshot captures consistent point-in-time data for services whose
# raw volume files are not safe to back up live.
#
# Containers handled here:
#   - synapse-db        (internal Postgres 16 — pg_dump)
#   - seafile-mysql     (internal MariaDB 10.6 — mysqldump)
#   - vaultwarden       (external Postgres on PFI-Postgres 10.250.50.80)
#   - gitea             (`gitea dump` captures DB + repos + config + LFS)
#   - openwebui         (local SQLite × 2 — main db + ChromaDB vector store)
#
# Also staged here (not a container):
#   - fv-gateway        (OPNsense config.xml for the Fountain Valley edge
#                        firewall, pulled over its WAN admin API). ana-docker is
#                        one of the three egress addresses that firewall's
#                        allowlist permits, which is why the pull lives on this
#                        host rather than with the FV hardware — a site that has
#                        lost power cannot back itself up.
#
# External DB credentials live in /etc/restic/dbcreds.env (root:600).
# Template: configs/restic/ana-docker/dbcreds.env.example in the repo.
#
# Required tooling on the host:
#   docker                  — always
#   pg_dump                 — for vaultwarden external Postgres; install via
#                             `apt install postgresql-client`. Without it,
#                             the vaultwarden block logs a warning and skips.
#
# Intentionally NOT handled:
#   - mattermost (retired 2026-04-21 — stack dir lingers but is not running)
#
# Required database dump failures abort the backup instead of reporting a
# successful snapshot without them. Previous staged dumps remain intact until
# all required dumps succeed. Gitea scratch files are isolated and trap-cleaned.

set -euo pipefail

STAGE=${RESTIC_STAGE_DIR:-/var/lib/restic/stage}
install -d -m 0700 "$STAGE"
WORK=$(mktemp -d "$STAGE/.pending.XXXXXXXX")
trap 'rm -rf -- "$WORK"' EXIT
ERRORS=0

log()  { printf '%s pre-backup(ana-docker): %s\n' "$(date -Is)" "$*"; }
warn() { log "WARN: $*" >&2; }
error() { ERRORS=$((ERRORS + 1)); warn "$*"; }

# Load external-DB creds. Silently skipped if missing — individual blocks
# that need them will log their own WARN.
CREDS=${RESTIC_DB_CREDS_FILE:-/etc/restic/dbcreds.env}
if [ -r "$CREDS" ]; then
  set -a; . "$CREDS"; set +a
fi

# FV gateway API creds, same shape and posture as dbcreds.env (root:600).
# Template: configs/restic/ana-docker/fv-gateway.env.example in the repo.
FVCREDS=${RESTIC_FV_CREDS_FILE:-/etc/restic/fv-gateway.env}
if [ -r "$FVCREDS" ]; then
  set -a; . "$FVCREDS"; set +a
fi

# ---------- synapse (internal Postgres) ---------------------------------------
if docker inspect synapse-db >/dev/null 2>&1; then
  log "dumping synapse postgres"
  docker exec synapse-db \
    pg_dump -U synapse -d synapse -Fc --clean --if-exists \
    > "$WORK/synapse.pg_dump" \
    || error "synapse pg_dump failed"
else
  log "skip synapse: container not present"
fi

# ---------- seafile (internal MariaDB) ----------------------------------------
if docker inspect seafile-mysql >/dev/null 2>&1; then
  log "dumping seafile mariadb"
  docker exec seafile-mysql sh -c \
    'mysqldump -uroot -p"$MYSQL_ROOT_PASSWORD" --all-databases --single-transaction --quick 2>/dev/null' \
    | gzip -c > "$WORK/seafile.sql.gz" \
    || error "seafile mysqldump failed"
else
  log "skip seafile: container not present"
fi

# ---------- vaultwarden (external Postgres on PFI-Postgres 10.250.50.80) ------
# The vault moved from SQLite to external Postgres (date unclear). Any
# /data/db.sqlite3* files in the container are stale leftovers and should
# be deleted separately — this hook captures the live Postgres data only.
if docker inspect vaultwarden >/dev/null 2>&1; then
  if [ -z "${VW_PGPASS:-}" ]; then
    error "vaultwarden: VW_PGPASS unset in /etc/restic/dbcreds.env"
  elif ! command -v pg_dump >/dev/null 2>&1; then
    error "vaultwarden: pg_dump not installed (apt install postgresql-client)"
  else
    log "dumping vaultwarden postgres (external: ${VW_PGHOST}:${VW_PGPORT:-5432})"
    PGPASSWORD="$VW_PGPASS" pg_dump \
      -h "$VW_PGHOST" -p "${VW_PGPORT:-5432}" \
      -U "$VW_PGUSER" -d "$VW_PGDB" \
      -Fc --clean --if-exists \
      > "$WORK/vaultwarden.pg_dump" \
      || error "vaultwarden pg_dump failed"
  fi
else
  log "skip vaultwarden: container not present"
fi

# ---------- gitea (native `gitea dump`) ---------------------------------------
# `gitea dump` produces a single archive with the DB dump, repo trees,
# config, LFS objects and attachments. The in-container command reads
# its own DB creds (from GITEA__database__* env vars) — no external
# creds needed.
#
# `--type tar` produces an UNCOMPRESSED tarball. Compressed formats
# (zip/tar.gz) defeat restic's content-defined chunking: each day's
# dump looks completely different to restic even when the underlying
# data barely changed, so repo grows by ~full dump size every day.
# Uncompressed tar lets restic dedup aggressively — after the first
# snapshot, daily incrementals only cost the actual new-data delta.
#
# Tradeoff: on-disk stage file is larger (~2-3x the zip size) but that's
# transient (deleted next run). Repo-side storage is much smaller.
#
# Size trimming flags if the tar becomes excessive: --skip-lfs-data,
# --skip-repository, --skip-attachment-data.
if docker inspect gitea >/dev/null 2>&1; then
  log "dumping gitea (gitea dump, uncompressed tar)"
  if docker exec -u git gitea sh -c '
       set -eu
       scratch=$(mktemp -d /tmp/gitea-backup.XXXXXXXX)
       trap '\''rm -rf -- "$scratch"'\'' EXIT
       gitea dump -c /data/gitea/conf/app.ini --tempdir "$scratch" --file - --type tar
     ' > "$WORK/gitea-dump.tar"; then
    tar -tf "$WORK/gitea-dump.tar" >/dev/null \
      || error "gitea archive validation failed"
  else
    error "gitea dump command failed (details above); previous stage preserved"
  fi
else
  log "skip gitea: container not present"
fi

# ---------- openwebui (local SQLite × 2) --------------------------------------
# Two SQLite databases: main app (/app/backend/data/webui.db) and the
# ChromaDB vector store (.../vector_db/chroma.sqlite3). Uses SQLite's
# .backup command for a consistent snapshot if sqlite3 is available in
# the container; falls back to restic's volume-level capture otherwise.
OWUI_CONTAINER=openwebui-open-webui-1
if docker inspect "$OWUI_CONTAINER" >/dev/null 2>&1; then
  if docker exec "$OWUI_CONTAINER" sh -c 'command -v sqlite3 >/dev/null 2>&1'; then
    log "dumping openwebui sqlite (webui.db + chroma.sqlite3) via .backup"
    for pair in \
        "/app/backend/data/webui.db:webui.db" \
        "/app/backend/data/vector_db/chroma.sqlite3:chroma.sqlite3"; do
      src=${pair%:*}; dst=${pair#*:}
      if docker exec "$OWUI_CONTAINER" sqlite3 "$src" ".backup /tmp/$dst" 2>/dev/null; then
        docker cp "$OWUI_CONTAINER:/tmp/$dst" "$WORK/openwebui.$dst" \
          && docker exec "$OWUI_CONTAINER" rm -f "/tmp/$dst" \
          || warn "openwebui copy/cleanup failed for $dst"
      else
        warn "openwebui .backup failed for $src (missing file or locked?)"
      fi
    done
  else
    warn "openwebui: sqlite3 not in container — relying on restic volume-level capture"
  fi
else
  log "skip openwebui: container not present"
fi

# ---------- fv-gateway (OPNsense edge firewall config) -------------------------
# Non-fatal by design: a firewall we cannot reach must not abort the nightly
# database dumps. But a bad pull must not be PROMOTED either — the summary loop
# below only rejects EMPTY files, and this endpoint answers an auth failure or a
# captive portal with a perfectly non-empty HTML error page. So validate that the
# body is really an OPNsense config and write nothing at all otherwise.
FV_HOST=${FV_GATEWAY_HOST:-172.83.89.66}
if [ -n "${FV_API_KEY:-}" ] && [ -n "${FV_API_SECRET:-}" ]; then
  log "pulling fv-gateway config from $FV_HOST"
  fv_tmp="$WORK/.fv-config.raw"
  if curl -fsS --max-time 60 -u "$FV_API_KEY:$FV_API_SECRET" \
       -o "$fv_tmp" "http://$FV_HOST/api/core/backup/download/this" 2>/dev/null; then
    if head -c 200 "$fv_tmp" | grep -q '<opnsense>'; then
      mv -f -- "$fv_tmp" "$WORK/fv-gateway-config.xml"
      log "fv-gateway config staged ($(wc -c < "$WORK/fv-gateway-config.xml") bytes)"
    else
      rm -f -- "$fv_tmp"
      warn "fv-gateway: response was not an OPNsense config (auth failure or error page?)"
    fi
  else
    rm -f -- "$fv_tmp"
    warn "fv-gateway: config pull failed (site unreachable?)"
  fi
else
  log "skip fv-gateway: no API creds in $FVCREDS"
fi

# ---------- summary -----------------------------------------------------------
if [ "$ERRORS" -ne 0 ]; then
  log "FAILED: $ERRORS required database dump(s) failed; previous stage preserved"
  exit 1
fi
for dump in "$WORK"/*; do
  [ -f "$dump" ] || continue
  if [ ! -s "$dump" ]; then
    log "FAILED: empty dump ${dump##*/}; previous stage preserved"
    exit 1
  fi
done
for dump in "$WORK"/*; do
  [ -f "$dump" ] || continue
  mv -f -- "$dump" "$STAGE/${dump##*/}"
done
size=$(du -sh "$STAGE" 2>/dev/null | awk '{print $1}')
count=$(find "$STAGE" -type f | wc -l)
log "stage ready: $count files, $size total"