Repair ana-docker database backups and bound CI build caches
This commit is contained in:
@@ -0,0 +1,42 @@
|
||||
# ana-docker database staging
|
||||
|
||||
`pre-backup.sh` runs as root through resticprofile's `run-before`. Deploy and
|
||||
exercise it with `scripts/elway infra-ops@10.250.50.70 --playbook
|
||||
playbooks/ana-docker-backup-repair.yaml` (allow about 12 minutes for Gitea).
|
||||
|
||||
Required dump failures now abort the backup. Dumps are prepared in a private
|
||||
pending directory; previous staged files are replaced only after successful
|
||||
generation and nonempty checks. Gitea additionally gets tar validation and a
|
||||
private in-container scratch directory with exit cleanup. Ordinary failures
|
||||
clean up scratch; SIGKILL or host crashes cannot run shell traps.
|
||||
|
||||
Regression checks: `python3 configs/restic/ana-docker/test_pre_backup.py`.
|
||||
The stage/credential path overrides are for isolated tests; production defaults
|
||||
remain `/var/lib/restic/stage` and `/etc/restic/dbcreds.env`.
|
||||
|
||||
## Repair verified 2026-09-10
|
||||
|
||||
Gitea dumps had failed since June 4 because five root-only historical app.ini
|
||||
copies inside its config directory were unreadable to the git dump user.
|
||||
Those copies retain root-only permissions under
|
||||
`/opt/docker/backups/gitea-config-history/`, included in backups. Do not put
|
||||
unreadable config history back inside Gitea's dump tree.
|
||||
|
||||
Vaultwarden's stale backup credentials were synchronized with its live database
|
||||
connection; the root:600 host file is also saved as Vaultwarden item
|
||||
`ana-docker/restic-dbcreds.env`. No secrets belong in this repository.
|
||||
|
||||
Fresh stage files were saved to Restic snapshot `2ec5a37c`. Both database dumps
|
||||
were restored FROM that repository snapshot into disposable PostgreSQL 16 with
|
||||
network disabled and tmpfs storage: Vaultwarden 7 users, Gitea 9 users and
|
||||
93 repositories. The temporary database container was removed afterward.
|
||||
This verifies database restore, not a complete application disaster-recovery drill.
|
||||
|
||||
Only after successful restores, 101 abandoned Gitea dump files (47.31 GiB logical
|
||||
size) were deleted. Their inventory is root-only at
|
||||
`/opt/docker/backups/gitea-config-history/removed-dump-manifest-20260910.json`.
|
||||
Old failed-dump history itself was not retained; the fresh replacement is in
|
||||
Restic. The verification marker is `/var/lib/restic/verified-repair-20260910`.
|
||||
|
||||
OpenWebUI's existing warning/fallback to volume capture when sqlite3 is absent
|
||||
is unchanged; this repair does not claim a verified OpenWebUI database backup.
|
||||
@@ -25,25 +25,27 @@
|
||||
# Intentionally NOT handled:
|
||||
# - mattermost (retired 2026-04-21 — stack dir lingers but is not running)
|
||||
#
|
||||
# Idempotent: clears and recreates its staging files each run.
|
||||
# Errors in individual blocks are logged as WARN but don't abort the whole
|
||||
# script — partial dumps are better than no dumps.
|
||||
# Required database dump failures abort the backup instead of reporting a
|
||||
# successful snapshot without them. Previous staged dumps remain intact until
|
||||
# all required dumps succeed. Gitea scratch files are isolated and trap-cleaned.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
STAGE=/var/lib/restic/stage
|
||||
install -d -o root -g root -m 0700 "$STAGE"
|
||||
STAGE=${RESTIC_STAGE_DIR:-/var/lib/restic/stage}
|
||||
install -d -m 0700 "$STAGE"
|
||||
WORK=$(mktemp -d "$STAGE/.pending.XXXXXXXX")
|
||||
trap 'rm -rf -- "$WORK"' EXIT
|
||||
ERRORS=0
|
||||
|
||||
log() { printf '%s pre-backup(ana-docker): %s\n' "$(date -Is)" "$*"; }
|
||||
warn() { log "WARN: $*" >&2; }
|
||||
|
||||
# Purge previous stage so stale dumps don't pile up into the snapshot.
|
||||
find "$STAGE" -mindepth 1 -maxdepth 1 -exec rm -rf {} +
|
||||
error() { ERRORS=$((ERRORS + 1)); warn "$*"; }
|
||||
|
||||
# Load external-DB creds. Silently skipped if missing — individual blocks
|
||||
# that need them will log their own WARN.
|
||||
if [ -r /etc/restic/dbcreds.env ]; then
|
||||
set -a; . /etc/restic/dbcreds.env; set +a
|
||||
CREDS=${RESTIC_DB_CREDS_FILE:-/etc/restic/dbcreds.env}
|
||||
if [ -r "$CREDS" ]; then
|
||||
set -a; . "$CREDS"; set +a
|
||||
fi
|
||||
|
||||
# ---------- synapse (internal Postgres) ---------------------------------------
|
||||
@@ -51,8 +53,8 @@ if docker inspect synapse-db >/dev/null 2>&1; then
|
||||
log "dumping synapse postgres"
|
||||
docker exec synapse-db \
|
||||
pg_dump -U synapse -d synapse -Fc --clean --if-exists \
|
||||
> "$STAGE/synapse.pg_dump" \
|
||||
|| warn "synapse pg_dump failed"
|
||||
> "$WORK/synapse.pg_dump" \
|
||||
|| error "synapse pg_dump failed"
|
||||
else
|
||||
log "skip synapse: container not present"
|
||||
fi
|
||||
@@ -62,8 +64,8 @@ if docker inspect seafile-mysql >/dev/null 2>&1; then
|
||||
log "dumping seafile mariadb"
|
||||
docker exec seafile-mysql sh -c \
|
||||
'mysqldump -uroot -p"$MYSQL_ROOT_PASSWORD" --all-databases --single-transaction --quick 2>/dev/null' \
|
||||
| gzip -c > "$STAGE/seafile.sql.gz" \
|
||||
|| warn "seafile mysqldump failed"
|
||||
| gzip -c > "$WORK/seafile.sql.gz" \
|
||||
|| error "seafile mysqldump failed"
|
||||
else
|
||||
log "skip seafile: container not present"
|
||||
fi
|
||||
@@ -74,17 +76,17 @@ fi
|
||||
# be deleted separately — this hook captures the live Postgres data only.
|
||||
if docker inspect vaultwarden >/dev/null 2>&1; then
|
||||
if [ -z "${VW_PGPASS:-}" ]; then
|
||||
warn "vaultwarden: VW_PGPASS unset in /etc/restic/dbcreds.env — skipping"
|
||||
error "vaultwarden: VW_PGPASS unset in /etc/restic/dbcreds.env"
|
||||
elif ! command -v pg_dump >/dev/null 2>&1; then
|
||||
warn "vaultwarden: pg_dump not installed — skipping (apt install postgresql-client)"
|
||||
error "vaultwarden: pg_dump not installed (apt install postgresql-client)"
|
||||
else
|
||||
log "dumping vaultwarden postgres (external: ${VW_PGHOST}:${VW_PGPORT:-5432})"
|
||||
PGPASSWORD="$VW_PGPASS" pg_dump \
|
||||
-h "$VW_PGHOST" -p "${VW_PGPORT:-5432}" \
|
||||
-U "$VW_PGUSER" -d "$VW_PGDB" \
|
||||
-Fc --clean --if-exists \
|
||||
> "$STAGE/vaultwarden.pg_dump" \
|
||||
|| warn "vaultwarden pg_dump failed"
|
||||
> "$WORK/vaultwarden.pg_dump" \
|
||||
|| error "vaultwarden pg_dump failed"
|
||||
fi
|
||||
else
|
||||
log "skip vaultwarden: container not present"
|
||||
@@ -110,14 +112,16 @@ fi
|
||||
# --skip-repository, --skip-attachment-data.
|
||||
if docker inspect gitea >/dev/null 2>&1; then
|
||||
log "dumping gitea (gitea dump, uncompressed tar)"
|
||||
if docker exec -u git gitea sh -c \
|
||||
'rm -f /tmp/gitea-dump.tar && gitea dump -c /data/gitea/conf/app.ini -f /tmp/gitea-dump.tar --type tar' \
|
||||
>/dev/null 2>&1; then
|
||||
docker cp gitea:/tmp/gitea-dump.tar "$STAGE/gitea-dump.tar" \
|
||||
&& docker exec -u git gitea rm -f /tmp/gitea-dump.tar \
|
||||
|| warn "gitea dump copy/cleanup failed"
|
||||
if docker exec -u git gitea sh -c '
|
||||
set -eu
|
||||
scratch=$(mktemp -d /tmp/gitea-backup.XXXXXXXX)
|
||||
trap '\''rm -rf -- "$scratch"'\'' EXIT
|
||||
gitea dump -c /data/gitea/conf/app.ini --tempdir "$scratch" --file - --type tar
|
||||
' > "$WORK/gitea-dump.tar"; then
|
||||
tar -tf "$WORK/gitea-dump.tar" >/dev/null \
|
||||
|| error "gitea archive validation failed"
|
||||
else
|
||||
warn "gitea dump command failed"
|
||||
error "gitea dump command failed (details above); previous stage preserved"
|
||||
fi
|
||||
else
|
||||
log "skip gitea: container not present"
|
||||
@@ -137,7 +141,7 @@ if docker inspect "$OWUI_CONTAINER" >/dev/null 2>&1; then
|
||||
"/app/backend/data/vector_db/chroma.sqlite3:chroma.sqlite3"; do
|
||||
src=${pair%:*}; dst=${pair#*:}
|
||||
if docker exec "$OWUI_CONTAINER" sqlite3 "$src" ".backup /tmp/$dst" 2>/dev/null; then
|
||||
docker cp "$OWUI_CONTAINER:/tmp/$dst" "$STAGE/openwebui.$dst" \
|
||||
docker cp "$OWUI_CONTAINER:/tmp/$dst" "$WORK/openwebui.$dst" \
|
||||
&& docker exec "$OWUI_CONTAINER" rm -f "/tmp/$dst" \
|
||||
|| warn "openwebui copy/cleanup failed for $dst"
|
||||
else
|
||||
@@ -152,6 +156,21 @@ else
|
||||
fi
|
||||
|
||||
# ---------- summary -----------------------------------------------------------
|
||||
if [ "$ERRORS" -ne 0 ]; then
|
||||
log "FAILED: $ERRORS required database dump(s) failed; previous stage preserved"
|
||||
exit 1
|
||||
fi
|
||||
for dump in "$WORK"/*; do
|
||||
[ -f "$dump" ] || continue
|
||||
if [ ! -s "$dump" ]; then
|
||||
log "FAILED: empty dump ${dump##*/}; previous stage preserved"
|
||||
exit 1
|
||||
fi
|
||||
done
|
||||
for dump in "$WORK"/*; do
|
||||
[ -f "$dump" ] || continue
|
||||
mv -f -- "$dump" "$STAGE/${dump##*/}"
|
||||
done
|
||||
size=$(du -sh "$STAGE" 2>/dev/null | awk '{print $1}')
|
||||
count=$(find "$STAGE" -type f | wc -l)
|
||||
log "stage ready: $count files, $size total"
|
||||
|
||||
@@ -0,0 +1,52 @@
|
||||
import os
|
||||
from pathlib import Path
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
|
||||
SCRIPT=Path(__file__).with_name('pre-backup.sh')
|
||||
|
||||
class BackupHookTests(unittest.TestCase):
|
||||
def exercise(self, succeeds):
|
||||
with tempfile.TemporaryDirectory(prefix='backup-hook-test-') as d:
|
||||
root=Path(d); stage=root/'stage'; stage.mkdir(); binpath=root/'bin';binpath.mkdir()
|
||||
previous=stage/'vaultwarden.pg_dump';previous.write_bytes(b'previous-good-backup')
|
||||
docker=binpath/'docker';docker.write_text('#!/bin/sh\n[ "$1" = inspect ] && [ "$2" = vaultwarden ]\n');docker.chmod(0o755)
|
||||
pg=binpath/'pg_dump';pg.write_text('#!/bin/sh\nprintf new-dump\nexit '+('0' if succeeds else '1')+'\n');pg.chmod(0o755)
|
||||
env=dict(os.environ,PATH=str(binpath)+':'+os.environ['PATH'],RESTIC_STAGE_DIR=str(stage),RESTIC_DB_CREDS_FILE=str(root/'absent'),VW_PGPASS='fake',VW_PGHOST='fake',VW_PGUSER='fake',VW_PGDB='fake')
|
||||
r=subprocess.run(['bash',str(SCRIPT)],env=env,capture_output=True,text=True)
|
||||
if succeeds:
|
||||
self.assertEqual(r.returncode,0,r.stdout+r.stderr)
|
||||
self.assertEqual(previous.read_bytes(),b'new-dump')
|
||||
else:
|
||||
self.assertNotEqual(r.returncode,0,r.stdout+r.stderr)
|
||||
self.assertEqual(previous.read_bytes(),b'previous-good-backup')
|
||||
self.assertIn('required database dump(s) failed',r.stdout)
|
||||
self.assertEqual(list(stage.glob('.pending.*')),[])
|
||||
|
||||
def test_failed_required_dump_preserves_previous_backup_and_cleans_scratch(self):
|
||||
self.exercise(False)
|
||||
|
||||
def test_success_publishes_new_dump_and_cleans_scratch(self):
|
||||
self.exercise(True)
|
||||
|
||||
def test_gitea_failure_removes_sql_scratch_and_exposes_error(self):
|
||||
with tempfile.TemporaryDirectory(prefix='backup-gitea-test-') as d:
|
||||
root=Path(d);stage=root/'stage';stage.mkdir();binpath=root/'bin';binpath.mkdir()
|
||||
previous=stage/'gitea-dump.tar';previous.write_bytes(b'previous-good-archive')
|
||||
docker=binpath/'docker'
|
||||
docker.write_text('#!/bin/sh\nif [ "$1" = inspect ]; then [ "$2" = gitea ]; exit $?; fi\nshift 4\nexec "$@"\n')
|
||||
docker.chmod(0o755)
|
||||
gitea=binpath/'gitea'
|
||||
gitea.write_text('#!/bin/sh\nwhile [ "$#" -gt 0 ]; do if [ "$1" = --tempdir ]; then shift; scratch=$1; fi; shift; done\nprintf %s "$scratch" > "$TEST_SCRATCH_PATH"\nprintf partial-sql > "$scratch/gitea-db.sql123"\necho simulated-export-failure >&2\nexit 9\n')
|
||||
gitea.chmod(0o755)
|
||||
path_record=root/'scratch-path'
|
||||
env=dict(os.environ,PATH=str(binpath)+':'+os.environ['PATH'],RESTIC_STAGE_DIR=str(stage),RESTIC_DB_CREDS_FILE=str(root/'absent'),TEST_SCRATCH_PATH=str(path_record))
|
||||
r=subprocess.run(['bash',str(SCRIPT)],env=env,capture_output=True,text=True)
|
||||
self.assertNotEqual(r.returncode,0)
|
||||
self.assertIn('simulated-export-failure',r.stderr)
|
||||
self.assertFalse(Path(path_record.read_text()).exists())
|
||||
self.assertEqual(previous.read_bytes(),b'previous-good-archive')
|
||||
self.assertEqual(list(stage.glob('.pending.*')),[])
|
||||
|
||||
if __name__=='__main__':unittest.main()
|
||||
Reference in New Issue
Block a user