diff --git a/STATUS.md b/STATUS.md index dcfe399..a203d6a 100644 --- a/STATUS.md +++ b/STATUS.md @@ -183,10 +183,15 @@ significant work lands — don't let it drift quietly. Unlocks cross-site rsync (item 6), rest-server-nh3 healthcheck deploy, and `.htpasswd` edits on the NH3 side. -9. **`scripts/restic-prune.sh`** — temporarily flip `--append-only` off, - run forget + prune across all hosts, flip back on. Needed quarterly - for disk hygiene. Not urgent; blocks only the "I need to reclaim - disk space now" scenario. +9. ~~**`scripts/restic-prune.sh`**~~ — **done 2026-04-24.** Quarterly + disk-hygiene tool. Drops `--append-only` on the rest-server (via a + temporary `docker-compose.override.yaml` — never edits the canonical + compose), runs `resticprofile forget --prune --verbose` on each + client, restores `--append-only` (with `trap` so it runs even on + partial failure). ANA side fully automated (5 clients); NH3 side + prints a manual ceremony because DSM Container Manager + sudo on + syncuser aren't cleanly scriptable from this workstation. Run with + `scripts/restic-prune.sh ana|nh3|all`, optionally `--dry-run`. 10. ~~**Retire `offen/docker-volume-backup` sidecars**~~ — **done 2026-04-23**. Removed from paperless-ngx and pgadmin composes on diff --git a/scripts/restic-prune.sh b/scripts/restic-prune.sh new file mode 100755 index 0000000..6aa403c --- /dev/null +++ b/scripts/restic-prune.sh @@ -0,0 +1,198 @@ +#!/usr/bin/env bash +# restic-prune.sh — quarterly disk-hygiene tool for the rest-server fleet. +# +# Each client repo has its own retention policy under +# `forget:` in configs/restic//profiles.yaml. Those policies are +# evaluated by the daily timer's `backup` run but they don't reclaim +# anything on disk because the rest-servers run with `--append-only` +# (compromised client can't wipe its own history). +# +# This script does the periodic ceremony to actually reclaim space: +# +# 1. Disable --append-only on the rest-server (via temporary +# docker-compose.override.yaml — never edits the canonical compose) +# 2. SSH to each client whose repo lives on that rest-server and run +# `resticprofile forget --prune --verbose` +# 3. Restore --append-only (with a `trap` so it runs even if step 2 +# bails partway through). +# +# Run it manually, on demand. Frequency: quarterly is plenty unless +# disk pressure becomes a concern. +# +# Usage: +# scripts/restic-prune.sh # show this help +# scripts/restic-prune.sh ana # prune the ANA repo (5 clients) +# scripts/restic-prune.sh nh3 # NH3 — prints manual ceremony +# # (DSM sudo can't be scripted +# # cleanly from this workstation) +# scripts/restic-prune.sh all # ana + then nh3 instructions +# +# Add --dry-run to print every command without executing or mutating. + +set -euo pipefail + +DRY_RUN=0 +TARGET="" + +# ── client → rest-server topology ──────────────────────────────────── +ANA_CLIENTS=(ana-docker ana-ml2 esh-docker-vm vm-esh-nas esh-vm-db) +NH3_CLIENTS=(nh3-docker nh3-dev irv-ml1) + +ANA_REST_HOST="ana-docker" +ANA_REST_COMPOSE_DIR="/opt/docker/compose/rest-server-ana" +ANA_REST_SERVICE="rest-server" + +NH3_REST_HOST="nh3-nas" +NH3_REST_COMPOSE_DIR="/volume1/docker/rest-server" + +# ── argument parsing ───────────────────────────────────────────────── +for arg in "$@"; do + case "$arg" in + --dry-run|-n) DRY_RUN=1 ;; + ana|nh3|all) TARGET="$arg" ;; + -h|--help) TARGET="" ;; + *) echo "unknown arg: $arg" >&2; TARGET=""; break ;; + esac +done + +if [[ -z "$TARGET" ]]; then + sed -n '3,30p' "$0" | sed 's/^# \?//' + exit 0 +fi + +# ── helpers ────────────────────────────────────────────────────────── +say() { printf '\033[1;36m▸ %s\033[0m\n' "$*"; } +ok() { printf '\033[1;32m✓ %s\033[0m\n' "$*"; } +warn() { printf '\033[1;33m⚠ %s\033[0m\n' "$*"; } +err() { printf '\033[1;31m✗ %s\033[0m\n' "$*" >&2; } + +run() { + if (( DRY_RUN )); then + printf '\033[2m (dry) %s\033[0m\n' "$*" + else + "$@" + fi +} + +# Run a remote command via ssh; with -t when sudo is in the command line +# so the user can be prompted if needed. Heredoc'd scripts go via stdin +# with plain ssh (no -t — pipe and tty are mutually exclusive). +ssh_run() { + local host="$1"; shift + if (( DRY_RUN )); then + printf '\033[2m (dry) ssh %s -- %s\033[0m\n' "$host" "$*" + else + ssh "$host" "$@" + fi +} + +ssh_sudo() { + local host="$1"; shift + if (( DRY_RUN )); then + printf '\033[2m (dry) ssh -t %s -- %s\033[0m\n' "$host" "$*" + else + ssh -t "$host" "$@" + fi +} + +# ── ANA flow ───────────────────────────────────────────────────────── +ana_disable_append_only() { + say "ANA: writing docker-compose.override.yaml to drop --append-only" + local override + override=$(cat < ${ANA_REST_COMPOSE_DIR}/docker-compose.override.yaml" <<<"$override" + ssh_run "$ANA_REST_HOST" "cd ${ANA_REST_COMPOSE_DIR} && docker compose up -d" + ok "ANA: rest-server now writable for prune" +} + +ana_restore_append_only() { + say "ANA: removing override and restoring --append-only" + ssh_run "$ANA_REST_HOST" "rm -f ${ANA_REST_COMPOSE_DIR}/docker-compose.override.yaml" + ssh_run "$ANA_REST_HOST" "cd ${ANA_REST_COMPOSE_DIR} && docker compose up -d" + ok "ANA: rest-server back to append-only" +} + +prune_client() { + local host="$1" + say "client: ${host} — forget --prune" + if ssh_sudo "$host" "sudo resticprofile -c /etc/restic/profiles.yaml forget --prune --verbose 2>&1 | tail -30"; then + ok "client: ${host} done" + return 0 + else + err "client: ${host} failed (continuing — append-only will be restored)" + return 1 + fi +} + +ana_run() { + local fail=0 + # Always restore append-only, even if one of the client prunes blows up. + trap 'ana_restore_append_only' EXIT + ana_disable_append_only + for c in "${ANA_CLIENTS[@]}"; do + prune_client "$c" || fail=$(( fail + 1 )) + done + trap - EXIT + ana_restore_append_only + if (( fail > 0 )); then + err "ANA prune complete: ${fail} client(s) failed (see above)" + return 1 + else + ok "ANA prune complete: ${#ANA_CLIENTS[@]}/${#ANA_CLIENTS[@]} clients done" + fi +} + +# ── NH3 flow (instructions, not automated) ─────────────────────────── +nh3_print_ceremony() { + cat <<'EOF' + +──────────────────────────────────────────────────────────────────── +NH3 manual prune ceremony +──────────────────────────────────────────────────────────────────── +The Synology DSM Container Manager doesn't expose docker on the +expected paths and sudo on syncuser isn't NOPASSWD, so the toggle +isn't safely scriptable from this workstation. Walk through these +steps in DSM + an interactive ssh: + +1. Drop --append-only via DSM: + - DSM web UI → Container Manager → Container → rest-server + - Edit → Environment → change OPTIONS to: + --private-repos --prometheus + - Apply (DSM recreates the container). + +2. Run forget --prune on each NH3 client (in any order): +EOF + for c in "${NH3_CLIENTS[@]}"; do + printf ' ssh -t %s "sudo resticprofile -c /etc/restic/profiles.yaml forget --prune --verbose"\n' "$c" + done + cat <<'EOF' + +3. Restore --append-only via DSM: + - DSM → Container Manager → Container → rest-server + - Edit → Environment → set OPTIONS back to: + --private-repos --append-only --prometheus + - Apply. + +When the DSM workflow gets too tedious to repeat, replace this +block with a real ssh-driven flow (probe `which docker` via +`ssh -t nh3-nas sudo …` first to find the actual binary path). +──────────────────────────────────────────────────────────────────── + +EOF +} + +# ── main dispatch ──────────────────────────────────────────────────── +case "$TARGET" in + ana) ana_run ;; + nh3) nh3_print_ceremony ;; + all) ana_run; nh3_print_ceremony ;; +esac