From 25e41d2ab59b454bd06cae6f38357dfe1f1d7e78 Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Wed, 23 Sep 2026 08:05:47 -0700 Subject: [PATCH] fix(restic/esh-docker-vm): drop the uptime-kuma hook block that aborted every backup Uptime Kuma moved from esh-docker-vm to ana-docker on 2026-09-22. The pre-backup hook's fallback lookup, `docker ps | grep -E "uptime.kuma"`, then matched nothing and exited 1. Under set -euo pipefail that aborted the hook, and resticprofile treats a failed run-before as fatal, so no snapshot was taken from 2026-09-22 01:00 until this fix (backup-freshness: 54h stale). The block is removed rather than guarded because there is nothing on this host left for it to back up. The header now records the invariant the "blocks only WARN" promise depends on: every optional-service lookup must sit inside an `if` test or end in `|| true`. The remaining blocks were checked and all do. Deployed with playbooks/esh-docker-vm-restic-drop-kuma-block.yaml (the pre-fix hook is kept in /var/lib/restic/repair-20260923/). The live hook hash matches the canonical copy (43e6bea8b8569602). The manual backup saved snapshot 6ec9f74f, and backup-freshness now reports all backups fresh. --- configs/restic/esh-docker-vm/README.md | 8 ++-- configs/restic/esh-docker-vm/pre-backup.sh | 22 +++++------ .../esh-docker-vm-restic-drop-kuma-block.yaml | 38 +++++++++++++++++++ 3 files changed, 51 insertions(+), 17 deletions(-) create mode 100644 playbooks/esh-docker-vm-restic-drop-kuma-block.yaml diff --git a/configs/restic/esh-docker-vm/README.md b/configs/restic/esh-docker-vm/README.md index cae409f..066f3ab 100644 --- a/configs/restic/esh-docker-vm/README.md +++ b/configs/restic/esh-docker-vm/README.md @@ -30,15 +30,16 @@ Cross-site writes to `rest-server-ana` at `10.250.50.70:8000/esh-docker-vm/`. Unique to this host: most containers don't bundle sqlite3, so `pre-backup.sh` runs sqlite3 and pg_dump **from the host** against the -volume bind-mount paths. Simpler than building custom images for HA, -pgadmin, and uptime-kuma. +volume bind-mount paths. Simpler than building custom images for HA +and pgadmin. (uptime-kuma moved to ana-docker on 2026-09-22 and its block +was removed; see the header of `pre-backup.sh` for why a stale block took +the whole backup down.) | Service | DB | Approach | |---|---|---| | paperless-ngx | external Postgres `10.0.50.60` / `paperless-ng` | host pg_dump | | home-assistant | `/var/lib/docker/.../homeassistant_v2.db` | host sqlite3 | | pgadmin | `/var/lib/docker/.../pgadmin4.db` | host sqlite3 | -| uptime-kuma | `/var/lib/docker/.../kuma.db` | host sqlite3 | | calibre-web-automated | `/config/app.db` inside container | in-container sqlite3 (it has the binary) | ## Host prerequisites @@ -129,7 +130,6 @@ Expected files: - `home-assistant.sqlite3` — ~50 MB (matches live DB size) - `calibre-web-automated.app.db` — ~250 KB - `pgadmin4.db` — ~200 KB -- `uptime-kuma.kuma.db` — varies (history retention) Any `WARN:` lines in the hook output indicate a block that was skipped — read them, debug one at a time. diff --git a/configs/restic/esh-docker-vm/pre-backup.sh b/configs/restic/esh-docker-vm/pre-backup.sh index ced29ad..0a825cd 100644 --- a/configs/restic/esh-docker-vm/pre-backup.sh +++ b/configs/restic/esh-docker-vm/pre-backup.sh @@ -15,12 +15,20 @@ # - home-assistant (local SQLite in volume — sqlite3 .backup from host) # - calibre-web-automated (local SQLite — sqlite3 .backup inside container, has sqlite3) # - pgadmin (local SQLite in volume — sqlite3 .backup from host) -# - uptime-kuma (local SQLite in volume — sqlite3 .backup from host) +# +# uptime-kuma was here until it moved to ana-docker (2026-09-22). Its block +# was removed 2026-09-23: with no container left to match, its unguarded +# `docker ps | grep` lookup exited 1 and set -e aborted this script, and +# resticprofile then skipped the WHOLE host backup (stale 09-22 → 09-23). # # External DB credentials live in /etc/restic/dbcreds.env (root:600). # Template: configs/restic/esh-docker-vm/dbcreds.env.example. # # Errors in individual blocks log a WARN; whole script doesn't abort. +# That only holds if every lookup for an optional service sits inside an +# `if` test or ends in `|| true` — under set -euo pipefail a bare +# `x=$(… | grep …)` that matches nothing kills the script, and a failed +# run-before hook means NO snapshot at all, not a partial one. set -euo pipefail @@ -111,18 +119,6 @@ else log "skip pgadmin: container not present" fi -# ---------- uptime-kuma (SQLite in named volume, host-side .backup) ---------- -# Container name may vary after force-recreates (e.g. _uptime-kuma). -# Detect by label rather than hardcoded name. -UK_CONTAINER=$(docker ps --filter "label=com.docker.compose.project=uptimekuma" --format "{{.Names}}" | head -1) -[ -z "$UK_CONTAINER" ] && UK_CONTAINER=$(docker ps --format "{{.Names}}" | grep -E "uptime.kuma" | head -1) -if [ -n "$UK_CONTAINER" ]; then - UK_DB="/var/lib/docker/volumes/uptimekuma_uptime-kuma/_data/kuma.db" - host_sqlite_backup "$UK_DB" "uptime-kuma.kuma.db" || true -else - log "skip uptime-kuma: no container matching" -fi - # ---------- summary ----------------------------------------------------------- size=$(du -sh "$STAGE" 2>/dev/null | awk '{print $1}') count=$(find "$STAGE" -type f | wc -l) diff --git a/playbooks/esh-docker-vm-restic-drop-kuma-block.yaml b/playbooks/esh-docker-vm-restic-drop-kuma-block.yaml new file mode 100644 index 0000000..fc612d0 --- /dev/null +++ b/playbooks/esh-docker-vm-restic-drop-kuma-block.yaml @@ -0,0 +1,38 @@ +# esh-docker-vm: install the pre-backup hook without the uptime-kuma block. +# +# Kuma moved to ana-docker on 2026-09-22. The old block's unguarded +# `docker ps | grep uptime.kuma` lookup then matched nothing, exited 1, and +# set -euo pipefail aborted the hook. resticprofile treats a failed run-before +# as fatal, so every nightly backup since 2026-09-22 01:00 was skipped. +# +# Rerunnable: the preserve step is `creates:`-guarded, and the upload only +# changes the file when the content differs. + +steps: + - name: Preserve the pre-fix hook + sudo: true + shell: | + set -eu + install -d -m 0700 /var/lib/restic/repair-20260923 + cp -p /etc/restic/pre-backup.sh /var/lib/restic/repair-20260923/pre-backup.sh + creates: /var/lib/restic/repair-20260923/pre-backup.sh + + - name: Install the hook without the uptime-kuma block + sudo: true + upload: + src: configs/restic/esh-docker-vm/pre-backup.sh + dest: /etc/restic/pre-backup.sh + mode: '0700' + +verify: + - name: Hook parses under bash + sudo: true + shell: bash -n /etc/restic/pre-backup.sh + + - name: No uptime-kuma lookup left in the live hook + sudo: true + shell: "! grep -q 'UK_CONTAINER' /etc/restic/pre-backup.sh" + + - name: Hook runs to completion (stage summary line reached) + sudo: true + shell: /etc/restic/pre-backup.sh 2>&1 | grep -q 'stage ready:'