feat(sudo-io-archive): weekly fleet archive of sudo session recordings (pack, verify, then clear); fv-ml1 HF_HOME on tank and root-pool cleanup

2026-10-04 22:48:05 -07:00
parent 626e0c7f94
commit 008bd685aa
8 changed files with 248 additions and 0 deletions
@@ -0,0 +1,60 @@
+#!/usr/bin/env bash
+# Collector for the fleet's sudo I/O logs. Runs on nh3-dev (weekly user timer, or by hand).
+#   services/sudo-io-archive/sudo-io-archive                 # every host in ./hosts
+#   services/sudo-io-archive/sudo-io-archive fv-ml1 pbs-ana  # just these (names from ./hosts)
+#   SUDO_IO_DAYS=30 (default)   SUDO_IO_ARCHIVE_DIR=~/archive/sudo-io (default)
+#
+# Per host: the host packs sessions older than SUDO_IO_DAYS into one outbox archive (sudo-io-archive-host.sh pack),
+# this side pulls it, checks its sha256 and gzip integrity, keeps it under <dir>/<host>/, and only then tells the
+# host to delete exactly what it archived (… commit <sha>). A host that fails at any step keeps its logs and its
+# outbox; the next run re-ships the same outbox. Backed up because nh3-dev (VM 102 on nh3-pve) is in the nightly
+# vzdump to pbs-ana, which pbs-nh3 mirrors.
+#
+# Nothing here deletes anything locally (auto-memory feedback_no_rm_on_loop_variable): a failed pull leaves a
+# `.part` file behind, which the summary reports.
+set -uo pipefail
+umask 077   # the archives hold whatever the sessions printed; owner-only
+here=$(cd "$(dirname "$0")" && pwd)
+days=${SUDO_IO_DAYS:-30}
+dest=${SUDO_IO_ARCHIVE_DIR:-$HOME/archive/sudo-io}
+hostscript="$here/sudo-io-archive-host.sh"
+ssh_opts=(-o BatchMode=yes -o ConnectTimeout=10 -o ServerAliveInterval=30)
+opslog="$here/../../scripts/ops-log"
+want=("$@")
+fail=0
+summary=()
+
+# hosts come in on fd 3: ssh reads stdin, and on fd 0 the first pull swallowed the rest of the list (2026-10-04)
+while read -r -u 3 name target; do
+  [[ -z ${name:-} || $name == \#* ]] && continue
+  if [ ${#want[@]} -gt 0 ] && [[ ! " ${want[*]} " =~ " $name " ]]; then continue; fi
+  r=$(ssh "${ssh_opts[@]}" "$target" "sudo -n bash -s -- pack $days" < "$hostscript" 2>&1 | grep '^RESULT' | tail -1)
+  read -r _ kind sha sessions files _ <<<"${r:-RESULT ERROR}"
+  case "$kind" in
+    NOSUDOIO|NOTHING) summary+=("$name: ${kind,,}"); continue ;;
+    PACKED) ;;
+    *) summary+=("$name: PACK FAILED (${r:-no answer})"); fail=1; continue ;;
+  esac
+  install -d -m 700 "$dest/$name"
+  f="$dest/$name/$name-$(date -u +%Y%m%dT%H%M%SZ)-older-than-${days}d.tar.gz"
+  if ! ssh -n "${ssh_opts[@]}" "$target" 'cat /var/log/sudo-io-archive/outbox.tar.gz' > "$f.part"; then
+    summary+=("$name: PULL FAILED, left $f.part"); fail=1; continue
+  fi
+  got=$(sha256sum "$f.part" | cut -d' ' -f1)
+  if [ "$got" != "$sha" ] || ! gzip -t "$f.part" 2>/dev/null; then
+    summary+=("$name: VERIFY FAILED (sha $got vs $sha), left $f.part"); fail=1; continue
+  fi
+  mv -n "$f.part" "$f"
+  echo "$sha  $(basename "$f")" > "$f.sha256"
+  c=$(ssh "${ssh_opts[@]}" "$target" "sudo -n bash -s -- commit $sha" < "$hostscript" 2>&1 | grep '^RESULT' | tail -1)
+  if [[ $c != "RESULT COMMITTED"* ]]; then
+    summary+=("$name: archived to $f but COMMIT FAILED (${c:-no answer}); host keeps its logs"); fail=1; continue
+  fi
+  size=$(du -h "$f" | cut -f1)
+  summary+=("$name: $sessions sessions / $files files -> $f ($size); host ${c#RESULT COMMITTED }")
+  [ -x "$opslog" ] && "$opslog" record --host "$name" --action archive --target sudo-io \
+    --detail "sudo I/O logs older than ${days}d: $sessions sessions archived to nh3-dev $f ($size, sha256 ${sha:0:12}) and cleared on the host" >/dev/null 2>&1
+done 3< "$here/hosts"
+
+printf '%s\n' "${summary[@]}"
+exit $fail