diff --git a/docs/runbooks/backups.md b/docs/runbooks/backups.md index 060e613..4164f8c 100644 --- a/docs/runbooks/backups.md +++ b/docs/runbooks/backups.md @@ -209,6 +209,32 @@ reachability): hops through `infra-ops@localhost`, which has it. The credentials under `/etc/restic` are deliberately root-only. +### Repos are DISCOVERED, not enumerated + +The host list is no longer hand-written. `check-backup-freshness.sh` asks each +NAS which repositories exist — a directory counts as one when it has a +`snapshots/` child, which distinguishes a real repo from a container dir +(`/volume1/Backup/restic/repo/` holds `ana|esh|nh3` namespaces and no snapshots +of its own). + +**Why (svos-dev's framing, 2026-09-22):** adding nh3-dev fixed the *instance*; +the *class* was that a hand-maintained list of things to watch, sitting beside a +NAS that already knows which repos exist, means **the next repo added is +unwatched by default and nothing says so.** The list of what to check can +silently disagree with the set of what exists. + +The hand-written list survives **demoted**, as `EXPECTED_RESTIC_ANA` / +`EXPECTED_RESTIC_NH3`, used only to report a repo that has *vanished*. That +keeps two facts distinct that would otherwise both read as silence: + +| fact | now | +|---|---| +| a repo exists that nobody watches | **impossible** — it is discovered | +| a repo we expected is gone | reported as `EXPECTED REPO NOT FOUND` | + +Currently 8 repos: 5 on ana-nas (`repo/ana/*`), 3 on nh3-nas. The `esh` and +`nh3` namespaces under `repo/` on ana-nas are empty. + ⚠ **`restic ls` always prints a header line**, matched or not — a path absent from the repo returns 1 line, a real one returned 6. Counting with `grep -c .` read the header as a hit and made the ABSENT case fall through to *"blobs diff --git a/scripts/check-backup-freshness.sh b/scripts/check-backup-freshness.sh index 5367f54..b8973c5 100755 --- a/scripts/check-backup-freshness.sh +++ b/scripts/check-backup-freshness.sh @@ -128,21 +128,65 @@ report() { # $1=label $2=epoch("" = none) echo "=== Backup freshness (threshold ${MAX_AGE_H}h) — $(date '+%Y-%m-%d %H:%M %Z') ===" -# --- Layer: restic file+DB, ANA side (rest-server-ana) --- -for c in ana-docker ana-ml2 esh-docker-vm esh-vm-db vm-esh-nas; do - report "restic/ana/$c" "$(newest_epoch ana-nas "/mnt/backup/restic/repo/ana/$c/snapshots/*")" +# --- Layer: restic file+DB — DISCOVERED, not enumerated --------------------- +# +# ⚠ THIS LIST USED TO BE HAND-WRITTEN, AND ON 2026-09-22 IT WAS WRONG. It read +# `irv-ml1 nh3-docker` and omitted nh3-dev -- the repo holding every Claude Code +# transcript on that box, althing routes, hermes history and Miranda's +# conversation. The repo had always existed and always been written; it was +# simply never enumerated, so the one repository nobody could reconstruct was +# the one nothing watched. +# +# Adding nh3-dev fixed the instance. Deriving fixes the CLASS (svos-dev's +# framing): a hand-maintained list of things to watch, sitting beside a NAS that +# already knows which repos exist, means the next repo added is unwatched BY +# DEFAULT and nothing says so. The list of what to check can silently disagree +# with the set of what exists -- the same shape as every other instrument fault +# found that day, only slower-acting. +# +# So: ask the NAS what is there. A directory counts as a repository when it has +# a `snapshots/` child, which distinguishes a real repo from a container dir +# (/volume1/Backup/restic/repo/ holds ana|esh|nh3 namespaces and no snapshots +# of its own). +# +# The hand-written list survives DEMOTED, as an EXPECTED set -- used only to +# report a repo that has VANISHED. That keeps two different facts distinct that +# would otherwise both read as silence: +# "a repo exists that nobody watches" -> impossible now, it is discovered +# "a repo we expected is gone" -> reported below +discover_repos() { # $1=host $2=parent dir — prints "namenewest_epoch" + $SSH "$1" "for d in $2/*/; do [ -d \"\$d/snapshots\" ] || continue + printf '%s\t%s\n' \"\$(basename \"\$d\")\" \"\$(stat -c %Y \"\$d\"snapshots/* 2>/dev/null | sort -n | tail -1)\" + done" 2>/dev/null +} + +EXPECTED_ANA="${EXPECTED_RESTIC_ANA:-ana-docker ana-ml2 esh-docker-vm esh-vm-db vm-esh-nas}" +EXPECTED_NH3="${EXPECTED_RESTIC_NH3:-irv-ml1 nh3-docker nh3-dev}" + +found_ana=""; found_nh3="" +while IFS=$'\t' read -r name ep; do + [ -n "$name" ] || continue + found_ana="$found_ana $name" + report "restic/ana/$name" "$ep" +done < <(discover_repos ana-nas /mnt/backup/restic/repo/ana) + +while IFS=$'\t' read -r name ep; do + [ -n "$name" ] || continue + found_nh3="$found_nh3 $name" + report "restic/nh3/$name" "$ep" +done < <(discover_repos nh3-nas /volume1/Backup/restic) + +# A repo we expected and did NOT discover is a different fault from a stale one: +# the repository is gone, not behind. Say so in those words. +for want in $EXPECTED_ANA; do + in_csv "$want" "$(echo $found_ana | tr ' ' ',')" || \ + stale+=("restic/ana/$want: EXPECTED REPO NOT FOUND on ana-nas (deleted, renamed, or never created)") done -# --- Layer: restic file+DB, NH3 side (rest-server-nh3) --- -# ⚠ nh3-dev WAS MISSING FROM THIS LIST until 2026-09-22, and it is the host -# whose restic repo holds the most irreplaceable data on the fleet: every -# Claude Code session transcript, althing routes, hermes run history, and -# Miranda's conversation. Its repo has always existed at -# /volume1/Backup/restic/nh3-dev/ and has always been written nightly -- it was -# simply never enumerated here, so a total failure of the one repo nobody could -# reconstruct would have gone unreported indefinitely. -for c in irv-ml1 nh3-docker nh3-dev; do - report "restic/nh3/$c" "$(newest_epoch nh3-nas "/volume1/Backup/restic/$c/snapshots/*")" +for want in $EXPECTED_NH3; do + in_csv "$want" "$(echo $found_nh3 | tr ' ' ',')" || \ + stale+=("restic/nh3/$want: EXPECTED REPO NOT FOUND on nh3-nas (deleted, renamed, or never created)") done + # --- Layer: PBS VM images (newest per guest, all namespaces) --- pbs=$($SSH pbs-ana 'for ns in /mnt/pbs-datastore/ns/*/; do n=$(basename "$ns") for d in vm ct; do for g in "$ns$d"/*/; do [ -d "$g" ] || continue