From 4979869731649332a63176bb9107dea1e8be0ce7 Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Tue, 22 Sep 2026 13:36:57 -0700 Subject: [PATCH] feat(backups): discover restic repos instead of enumerating them Adding nh3-dev to the host list fixed the instance. This fixes the class, on svos-dev's framing: a hand-maintained list of things to watch, sitting beside a NAS that already knows which repos exist, means the next repo added is unwatched BY DEFAULT and nothing says so. The list of what to check can silently disagree with the set of what exists -- the same shape as every other instrument fault found this day, only slower-acting. The check now asks each NAS. A directory is a repository when it has a snapshots/ child, which cleanly separates real repos from container dirs (/volume1/Backup/restic/repo/ holds ana|esh|nh3 namespaces and no snapshots of its own -- verified rather than assumed before building discovery on the layout). The hand-written list survives DEMOTED to an EXPECTED set, used only to report a repo that has VANISHED. Two facts that would otherwise both read as silence stay distinct: "a repo exists that nobody watches" -> impossible now, it is discovered "a repo we expected is gone" -> EXPECTED REPO NOT FOUND Preventive, not corrective: all 8 repos currently discovered are already in the expected sets, so this found no live gap. It removes the possibility of the next one. Controls run, since a check only ever seen passing is untested: a bogus expected repo reports EXPECTED REPO NOT FOUND and turns the verdict STALE; unchanged expectations still report all-fresh; all 8 repos report their age. Observed while testing, not a fault: restic/ana/esh-docker-vm is 36h old against 12h for every other repo. Inside the 48h threshold so correctly green, but it is a day behind the fleet and worth a look. --- docs/runbooks/backups.md | 26 ++++++++++++ scripts/check-backup-freshness.sh | 70 +++++++++++++++++++++++++------ 2 files changed, 83 insertions(+), 13 deletions(-) diff --git a/docs/runbooks/backups.md b/docs/runbooks/backups.md index 060e613..4164f8c 100644 --- a/docs/runbooks/backups.md +++ b/docs/runbooks/backups.md @@ -209,6 +209,32 @@ reachability): hops through `infra-ops@localhost`, which has it. The credentials under `/etc/restic` are deliberately root-only. +### Repos are DISCOVERED, not enumerated + +The host list is no longer hand-written. `check-backup-freshness.sh` asks each +NAS which repositories exist — a directory counts as one when it has a +`snapshots/` child, which distinguishes a real repo from a container dir +(`/volume1/Backup/restic/repo/` holds `ana|esh|nh3` namespaces and no snapshots +of its own). + +**Why (svos-dev's framing, 2026-09-22):** adding nh3-dev fixed the *instance*; +the *class* was that a hand-maintained list of things to watch, sitting beside a +NAS that already knows which repos exist, means **the next repo added is +unwatched by default and nothing says so.** The list of what to check can +silently disagree with the set of what exists. + +The hand-written list survives **demoted**, as `EXPECTED_RESTIC_ANA` / +`EXPECTED_RESTIC_NH3`, used only to report a repo that has *vanished*. That +keeps two facts distinct that would otherwise both read as silence: + +| fact | now | +|---|---| +| a repo exists that nobody watches | **impossible** — it is discovered | +| a repo we expected is gone | reported as `EXPECTED REPO NOT FOUND` | + +Currently 8 repos: 5 on ana-nas (`repo/ana/*`), 3 on nh3-nas. The `esh` and +`nh3` namespaces under `repo/` on ana-nas are empty. + ⚠ **`restic ls` always prints a header line**, matched or not — a path absent from the repo returns 1 line, a real one returned 6. Counting with `grep -c .` read the header as a hit and made the ABSENT case fall through to *"blobs diff --git a/scripts/check-backup-freshness.sh b/scripts/check-backup-freshness.sh index 5367f54..b8973c5 100755 --- a/scripts/check-backup-freshness.sh +++ b/scripts/check-backup-freshness.sh @@ -128,21 +128,65 @@ report() { # $1=label $2=epoch("" = none) echo "=== Backup freshness (threshold ${MAX_AGE_H}h) — $(date '+%Y-%m-%d %H:%M %Z') ===" -# --- Layer: restic file+DB, ANA side (rest-server-ana) --- -for c in ana-docker ana-ml2 esh-docker-vm esh-vm-db vm-esh-nas; do - report "restic/ana/$c" "$(newest_epoch ana-nas "/mnt/backup/restic/repo/ana/$c/snapshots/*")" +# --- Layer: restic file+DB — DISCOVERED, not enumerated --------------------- +# +# ⚠ THIS LIST USED TO BE HAND-WRITTEN, AND ON 2026-09-22 IT WAS WRONG. It read +# `irv-ml1 nh3-docker` and omitted nh3-dev -- the repo holding every Claude Code +# transcript on that box, althing routes, hermes history and Miranda's +# conversation. The repo had always existed and always been written; it was +# simply never enumerated, so the one repository nobody could reconstruct was +# the one nothing watched. +# +# Adding nh3-dev fixed the instance. Deriving fixes the CLASS (svos-dev's +# framing): a hand-maintained list of things to watch, sitting beside a NAS that +# already knows which repos exist, means the next repo added is unwatched BY +# DEFAULT and nothing says so. The list of what to check can silently disagree +# with the set of what exists -- the same shape as every other instrument fault +# found that day, only slower-acting. +# +# So: ask the NAS what is there. A directory counts as a repository when it has +# a `snapshots/` child, which distinguishes a real repo from a container dir +# (/volume1/Backup/restic/repo/ holds ana|esh|nh3 namespaces and no snapshots +# of its own). +# +# The hand-written list survives DEMOTED, as an EXPECTED set -- used only to +# report a repo that has VANISHED. That keeps two different facts distinct that +# would otherwise both read as silence: +# "a repo exists that nobody watches" -> impossible now, it is discovered +# "a repo we expected is gone" -> reported below +discover_repos() { # $1=host $2=parent dir — prints "namenewest_epoch" + $SSH "$1" "for d in $2/*/; do [ -d \"\$d/snapshots\" ] || continue + printf '%s\t%s\n' \"\$(basename \"\$d\")\" \"\$(stat -c %Y \"\$d\"snapshots/* 2>/dev/null | sort -n | tail -1)\" + done" 2>/dev/null +} + +EXPECTED_ANA="${EXPECTED_RESTIC_ANA:-ana-docker ana-ml2 esh-docker-vm esh-vm-db vm-esh-nas}" +EXPECTED_NH3="${EXPECTED_RESTIC_NH3:-irv-ml1 nh3-docker nh3-dev}" + +found_ana=""; found_nh3="" +while IFS=$'\t' read -r name ep; do + [ -n "$name" ] || continue + found_ana="$found_ana $name" + report "restic/ana/$name" "$ep" +done < <(discover_repos ana-nas /mnt/backup/restic/repo/ana) + +while IFS=$'\t' read -r name ep; do + [ -n "$name" ] || continue + found_nh3="$found_nh3 $name" + report "restic/nh3/$name" "$ep" +done < <(discover_repos nh3-nas /volume1/Backup/restic) + +# A repo we expected and did NOT discover is a different fault from a stale one: +# the repository is gone, not behind. Say so in those words. +for want in $EXPECTED_ANA; do + in_csv "$want" "$(echo $found_ana | tr ' ' ',')" || \ + stale+=("restic/ana/$want: EXPECTED REPO NOT FOUND on ana-nas (deleted, renamed, or never created)") done -# --- Layer: restic file+DB, NH3 side (rest-server-nh3) --- -# ⚠ nh3-dev WAS MISSING FROM THIS LIST until 2026-09-22, and it is the host -# whose restic repo holds the most irreplaceable data on the fleet: every -# Claude Code session transcript, althing routes, hermes run history, and -# Miranda's conversation. Its repo has always existed at -# /volume1/Backup/restic/nh3-dev/ and has always been written nightly -- it was -# simply never enumerated here, so a total failure of the one repo nobody could -# reconstruct would have gone unreported indefinitely. -for c in irv-ml1 nh3-docker nh3-dev; do - report "restic/nh3/$c" "$(newest_epoch nh3-nas "/volume1/Backup/restic/$c/snapshots/*")" +for want in $EXPECTED_NH3; do + in_csv "$want" "$(echo $found_nh3 | tr ' ' ',')" || \ + stale+=("restic/nh3/$want: EXPECTED REPO NOT FOUND on nh3-nas (deleted, renamed, or never created)") done + # --- Layer: PBS VM images (newest per guest, all namespaces) --- pbs=$($SSH pbs-ana 'for ns in /mnt/pbs-datastore/ns/*/; do n=$(basename "$ns") for d in vm ct; do for g in "$ns$d"/*/; do [ -d "$g" ] || continue