Repair ana-docker database backups and bound CI build caches

configs/restic/ana-docker/pre-backup.sh

@@ -25,25 +25,27 @@
 # Intentionally NOT handled:
 #   - mattermost (retired 2026-04-21 — stack dir lingers but is not running)
 #
-# Idempotent: clears and recreates its staging files each run.
-# Errors in individual blocks are logged as WARN but don't abort the whole
-# script — partial dumps are better than no dumps.
+# Required database dump failures abort the backup instead of reporting a
+# successful snapshot without them. Previous staged dumps remain intact until
+# all required dumps succeed. Gitea scratch files are isolated and trap-cleaned.
 
 set -euo pipefail
 
-STAGE=/var/lib/restic/stage
-install -d -o root -g root -m 0700 "$STAGE"
+STAGE=${RESTIC_STAGE_DIR:-/var/lib/restic/stage}
+install -d -m 0700 "$STAGE"
+WORK=$(mktemp -d "$STAGE/.pending.XXXXXXXX")
+trap 'rm -rf -- "$WORK"' EXIT
+ERRORS=0
 
 log()  { printf '%s pre-backup(ana-docker): %s\n' "$(date -Is)" "$*"; }
 warn() { log "WARN: $*" >&2; }
-
-# Purge previous stage so stale dumps don't pile up into the snapshot.
-find "$STAGE" -mindepth 1 -maxdepth 1 -exec rm -rf {} +
+error() { ERRORS=$((ERRORS + 1)); warn "$*"; }
 
 # Load external-DB creds. Silently skipped if missing — individual blocks
 # that need them will log their own WARN.
-if [ -r /etc/restic/dbcreds.env ]; then
-  set -a; . /etc/restic/dbcreds.env; set +a
+CREDS=${RESTIC_DB_CREDS_FILE:-/etc/restic/dbcreds.env}
+if [ -r "$CREDS" ]; then
+  set -a; . "$CREDS"; set +a
 fi
 
 # ---------- synapse (internal Postgres) ---------------------------------------
@@ -51,8 +53,8 @@ if docker inspect synapse-db >/dev/null 2>&1; then
   log "dumping synapse postgres"
   docker exec synapse-db \
     pg_dump -U synapse -d synapse -Fc --clean --if-exists \
-    > "$STAGE/synapse.pg_dump" \
-    || warn "synapse pg_dump failed"
+    > "$WORK/synapse.pg_dump" \
+    || error "synapse pg_dump failed"
 else
   log "skip synapse: container not present"
 fi
@@ -62,8 +64,8 @@ if docker inspect seafile-mysql >/dev/null 2>&1; then
   log "dumping seafile mariadb"
   docker exec seafile-mysql sh -c \
     'mysqldump -uroot -p"$MYSQL_ROOT_PASSWORD" --all-databases --single-transaction --quick 2>/dev/null' \
-    | gzip -c > "$STAGE/seafile.sql.gz" \
-    || warn "seafile mysqldump failed"
+    | gzip -c > "$WORK/seafile.sql.gz" \
+    || error "seafile mysqldump failed"
 else
   log "skip seafile: container not present"
 fi
@@ -74,17 +76,17 @@ fi
 # be deleted separately — this hook captures the live Postgres data only.
 if docker inspect vaultwarden >/dev/null 2>&1; then
   if [ -z "${VW_PGPASS:-}" ]; then
-    warn "vaultwarden: VW_PGPASS unset in /etc/restic/dbcreds.env — skipping"
+    error "vaultwarden: VW_PGPASS unset in /etc/restic/dbcreds.env"
   elif ! command -v pg_dump >/dev/null 2>&1; then
-    warn "vaultwarden: pg_dump not installed — skipping (apt install postgresql-client)"
+    error "vaultwarden: pg_dump not installed (apt install postgresql-client)"
   else
     log "dumping vaultwarden postgres (external: ${VW_PGHOST}:${VW_PGPORT:-5432})"
     PGPASSWORD="$VW_PGPASS" pg_dump \
       -h "$VW_PGHOST" -p "${VW_PGPORT:-5432}" \
       -U "$VW_PGUSER" -d "$VW_PGDB" \
       -Fc --clean --if-exists \
-      > "$STAGE/vaultwarden.pg_dump" \
-      || warn "vaultwarden pg_dump failed"
+      > "$WORK/vaultwarden.pg_dump" \
+      || error "vaultwarden pg_dump failed"
   fi
 else
   log "skip vaultwarden: container not present"
@@ -110,14 +112,16 @@ fi
 # --skip-repository, --skip-attachment-data.
 if docker inspect gitea >/dev/null 2>&1; then
   log "dumping gitea (gitea dump, uncompressed tar)"
-  if docker exec -u git gitea sh -c \
-       'rm -f /tmp/gitea-dump.tar && gitea dump -c /data/gitea/conf/app.ini -f /tmp/gitea-dump.tar --type tar' \
-       >/dev/null 2>&1; then
-    docker cp gitea:/tmp/gitea-dump.tar "$STAGE/gitea-dump.tar" \
-      && docker exec -u git gitea rm -f /tmp/gitea-dump.tar \
-      || warn "gitea dump copy/cleanup failed"
+  if docker exec -u git gitea sh -c '
+       set -eu
+       scratch=$(mktemp -d /tmp/gitea-backup.XXXXXXXX)
+       trap '\''rm -rf -- "$scratch"'\'' EXIT
+       gitea dump -c /data/gitea/conf/app.ini --tempdir "$scratch" --file - --type tar
+     ' > "$WORK/gitea-dump.tar"; then
+    tar -tf "$WORK/gitea-dump.tar" >/dev/null \
+      || error "gitea archive validation failed"
   else
-    warn "gitea dump command failed"
+    error "gitea dump command failed (details above); previous stage preserved"
   fi
 else
   log "skip gitea: container not present"
@@ -137,7 +141,7 @@ if docker inspect "$OWUI_CONTAINER" >/dev/null 2>&1; then
         "/app/backend/data/vector_db/chroma.sqlite3:chroma.sqlite3"; do
       src=${pair%:*}; dst=${pair#*:}
       if docker exec "$OWUI_CONTAINER" sqlite3 "$src" ".backup /tmp/$dst" 2>/dev/null; then
-        docker cp "$OWUI_CONTAINER:/tmp/$dst" "$STAGE/openwebui.$dst" \
+        docker cp "$OWUI_CONTAINER:/tmp/$dst" "$WORK/openwebui.$dst" \
           && docker exec "$OWUI_CONTAINER" rm -f "/tmp/$dst" \
           || warn "openwebui copy/cleanup failed for $dst"
       else
@@ -152,6 +156,21 @@ else
 fi
 
 # ---------- summary -----------------------------------------------------------
+if [ "$ERRORS" -ne 0 ]; then
+  log "FAILED: $ERRORS required database dump(s) failed; previous stage preserved"
+  exit 1
+fi
+for dump in "$WORK"/*; do
+  [ -f "$dump" ] || continue
+  if [ ! -s "$dump" ]; then
+    log "FAILED: empty dump ${dump##*/}; previous stage preserved"
+    exit 1
+  fi
+done
+for dump in "$WORK"/*; do
+  [ -f "$dump" ] || continue
+  mv -f -- "$dump" "$STAGE/${dump##*/}"
+done
 size=$(du -sh "$STAGE" 2>/dev/null | awk '{print $1}')
 count=$(find "$STAGE" -type f | wc -l)
 log "stage ready: $count files, $size total"