diff --git a/README.md b/README.md index 16ec228..50db1eb 100644 --- a/README.md +++ b/README.md @@ -67,6 +67,7 @@ Per-host snapshots of the running system live under `servers//system-detai **Anaheim non-GPU (ana-docker):** - `traefik`, `crowdsec`, `gitea`, `vaultwarden`, `synapse`, `seafile`, `searxng`, `openwebui`, `sillytavern`, `mailrise`, `rustdesk`, `dockge`, `it-tools` +- (`mattermost` retired 2026-04-21 — compose dir may still linger, containers gone) - Fleet services: `beszel` (metrics hub, port 8090), `dozzle-hub` (log viewer, 8088), `backrest` (restic UI, 9898) - Backup target: `rest-server-ana` on port 8000 diff --git a/configs/restic/ana-docker/dbcreds.env.example b/configs/restic/ana-docker/dbcreds.env.example new file mode 100644 index 0000000..1ba24b1 --- /dev/null +++ b/configs/restic/ana-docker/dbcreds.env.example @@ -0,0 +1,25 @@ +# /etc/restic/dbcreds.env on ana-docker — consumed by pre-backup.sh. +# +# Deploy to the host as: +# sudo install -o root -g root -m 0600 dbcreds.env /etc/restic/dbcreds.env +# +# Never commit the real file — it carries production DB passwords. This +# template is the only thing tracked in the repo. + +# --- Vaultwarden → external Postgres on PFI-Postgres (10.250.50.80) --------- +VW_PGHOST=10.250.50.80 +VW_PGPORT=5432 +VW_PGUSER=vaultwarden +VW_PGDB=vaultwarden +VW_PGPASS=replace-with-postgres-password + +# --- Gitea → external Postgres on PFI-Postgres ------------------------------- +# Not needed: pre-backup.sh uses `gitea dump` which reads DB creds from the +# container's own GITEA__database__* env vars. Listed here for awareness +# in case we switch to direct pg_dump later. +# +# GITEA_PGHOST=10.250.50.80 +# GITEA_PGPORT=5432 +# GITEA_PGUSER=gitea +# GITEA_PGDB=gitea +# GITEA_PGPASS=replace-with-postgres-password diff --git a/configs/restic/ana-docker/pre-backup.sh b/configs/restic/ana-docker/pre-backup.sh index 5c5d9ba..4a7542b 100755 --- a/configs/restic/ana-docker/pre-backup.sh +++ b/configs/restic/ana-docker/pre-backup.sh @@ -7,68 +7,139 @@ # raw volume files are not safe to back up live. # # Containers handled here: -# - synapse-db (Postgres 16) -# - seafile-mysql (MariaDB 10.6) -# - vaultwarden (SQLite w/ WAL; online .backup via sqlite3 if available) +# - synapse-db (internal Postgres 16 — pg_dump) +# - seafile-mysql (internal MariaDB 10.6 — mysqldump) +# - vaultwarden (external Postgres on PFI-Postgres 10.250.50.80) +# - gitea (`gitea dump` captures DB + repos + config + LFS) +# - openwebui (local SQLite × 2 — main db + ChromaDB vector store) # -# Gitea's DB is external (hosted elsewhere in the LAN) so we only back -# up its data volume; whoever owns the gitea Postgres backs it up -# separately. +# External DB credentials live in /etc/restic/dbcreds.env (root:600). +# Template: configs/restic/ana-docker/dbcreds.env.example in the repo. +# +# Required tooling on the host: +# docker — always +# pg_dump — for vaultwarden external Postgres; install via +# `apt install postgresql-client`. Without it, +# the vaultwarden block logs a warning and skips. +# +# Intentionally NOT handled: +# - mattermost (retired 2026-04-21 — stack dir lingers but is not running) # # Idempotent: clears and recreates its staging files each run. +# Errors in individual blocks are logged as WARN but don't abort the whole +# script — partial dumps are better than no dumps. set -euo pipefail STAGE=/var/lib/restic/stage install -d -o root -g root -m 0700 "$STAGE" -log() { printf '%s pre-backup(ana-docker): %s\n' "$(date -Is)" "$*"; } +log() { printf '%s pre-backup(ana-docker): %s\n' "$(date -Is)" "$*"; } +warn() { log "WARN: $*" >&2; } -# Purge previous stage so stale dumps don't pile up and end up in the snapshot. +# Purge previous stage so stale dumps don't pile up into the snapshot. find "$STAGE" -mindepth 1 -maxdepth 1 -exec rm -rf {} + -# ---------- synapse-db (Postgres) --------------------------------------------- +# Load external-DB creds. Silently skipped if missing — individual blocks +# that need them will log their own WARN. +if [ -r /etc/restic/dbcreds.env ]; then + set -a; . /etc/restic/dbcreds.env; set +a +fi + +# ---------- synapse (internal Postgres) --------------------------------------- if docker inspect synapse-db >/dev/null 2>&1; then log "dumping synapse postgres" - # -Fc custom format, internally compressed + restore-to-subset friendly docker exec synapse-db \ pg_dump -U synapse -d synapse -Fc --clean --if-exists \ - > "$STAGE/synapse.pg_dump" + > "$STAGE/synapse.pg_dump" \ + || warn "synapse pg_dump failed" else log "skip synapse: container not present" fi -# ---------- seafile-mysql (MariaDB) ------------------------------------------- +# ---------- seafile (internal MariaDB) ---------------------------------------- if docker inspect seafile-mysql >/dev/null 2>&1; then log "dumping seafile mariadb" - # The root password lives in the container's own env (MYSQL_ROOT_PASSWORD); - # expand it inside the container so it never lands in the host's process list. docker exec seafile-mysql sh -c \ 'mysqldump -uroot -p"$MYSQL_ROOT_PASSWORD" --all-databases --single-transaction --quick 2>/dev/null' \ - | gzip -c > "$STAGE/seafile.sql.gz" + | gzip -c > "$STAGE/seafile.sql.gz" \ + || warn "seafile mysqldump failed" else log "skip seafile: container not present" fi -# ---------- vaultwarden (SQLite + WAL) ---------------------------------------- -# Vaultwarden uses SQLite in WAL mode. A live copy of db.sqlite3 + -wal is -# usually recoverable, but sqlite3's own .backup pragma is the correct way -# to get a consistent snapshot. If the vaultwarden image has sqlite3 -# available, use it; otherwise rely on restic backing up the raw volume. +# ---------- vaultwarden (external Postgres on PFI-Postgres 10.250.50.80) ------ +# The vault moved from SQLite to external Postgres (date unclear). Any +# /data/db.sqlite3* files in the container are stale leftovers and should +# be deleted separately — this hook captures the live Postgres data only. if docker inspect vaultwarden >/dev/null 2>&1; then - if docker exec vaultwarden sh -c 'command -v sqlite3 >/dev/null 2>&1'; then - log "dumping vaultwarden sqlite via .backup" - docker exec vaultwarden sqlite3 /data/db.sqlite3 \ - ".backup /tmp/vaultwarden.sqlite3" - docker cp vaultwarden:/tmp/vaultwarden.sqlite3 "$STAGE/vaultwarden.sqlite3" - docker exec vaultwarden rm -f /tmp/vaultwarden.sqlite3 + if [ -z "${VW_PGPASS:-}" ]; then + warn "vaultwarden: VW_PGPASS unset in /etc/restic/dbcreds.env — skipping" + elif ! command -v pg_dump >/dev/null 2>&1; then + warn "vaultwarden: pg_dump not installed — skipping (apt install postgresql-client)" else - log "skip vaultwarden .backup: sqlite3 not in container (raw volume still included via restic)" + log "dumping vaultwarden postgres (external: ${VW_PGHOST}:${VW_PGPORT:-5432})" + PGPASSWORD="$VW_PGPASS" pg_dump \ + -h "$VW_PGHOST" -p "${VW_PGPORT:-5432}" \ + -U "$VW_PGUSER" -d "$VW_PGDB" \ + -Fc --clean --if-exists \ + > "$STAGE/vaultwarden.pg_dump" \ + || warn "vaultwarden pg_dump failed" fi else log "skip vaultwarden: container not present" fi +# ---------- gitea (native `gitea dump`) --------------------------------------- +# `gitea dump` produces a single ZIP with the DB dump, repo trees, config, +# LFS objects and attachments. The in-container command knows its own DB +# creds (from GITEA__database__* env vars), so no creds needed here. +# +# Size scales with the repo tree. Trim with --skip-lfs-data, +# --skip-repository, --skip-attachment-data if the dump gets too large. +if docker inspect gitea >/dev/null 2>&1; then + log "dumping gitea (gitea dump, zip)" + if docker exec -u git gitea sh -c \ + 'rm -f /tmp/gitea-dump.zip && gitea dump -c /data/gitea/conf/app.ini -f /tmp/gitea-dump.zip' \ + >/dev/null 2>&1; then + docker cp gitea:/tmp/gitea-dump.zip "$STAGE/gitea-dump.zip" \ + && docker exec -u git gitea rm -f /tmp/gitea-dump.zip \ + || warn "gitea dump copy/cleanup failed" + else + warn "gitea dump command failed" + fi +else + log "skip gitea: container not present" +fi + +# ---------- openwebui (local SQLite × 2) -------------------------------------- +# Two SQLite databases: main app (/app/backend/data/webui.db) and the +# ChromaDB vector store (.../vector_db/chroma.sqlite3). Uses SQLite's +# .backup command for a consistent snapshot if sqlite3 is available in +# the container; falls back to restic's volume-level capture otherwise. +OWUI_CONTAINER=openwebui-open-webui-1 +if docker inspect "$OWUI_CONTAINER" >/dev/null 2>&1; then + if docker exec "$OWUI_CONTAINER" sh -c 'command -v sqlite3 >/dev/null 2>&1'; then + log "dumping openwebui sqlite (webui.db + chroma.sqlite3) via .backup" + for pair in \ + "/app/backend/data/webui.db:webui.db" \ + "/app/backend/data/vector_db/chroma.sqlite3:chroma.sqlite3"; do + src=${pair%:*}; dst=${pair#*:} + if docker exec "$OWUI_CONTAINER" sqlite3 "$src" ".backup /tmp/$dst" 2>/dev/null; then + docker cp "$OWUI_CONTAINER:/tmp/$dst" "$STAGE/openwebui.$dst" \ + && docker exec "$OWUI_CONTAINER" rm -f "/tmp/$dst" \ + || warn "openwebui copy/cleanup failed for $dst" + else + warn "openwebui .backup failed for $src (missing file or locked?)" + fi + done + else + warn "openwebui: sqlite3 not in container — relying on restic volume-level capture" + fi +else + log "skip openwebui: container not present" +fi + # ---------- summary ----------------------------------------------------------- size=$(du -sh "$STAGE" 2>/dev/null | awk '{print $1}') count=$(find "$STAGE" -type f | wc -l)