feat(sudo-io-archive): weekly fleet archive of sudo session recordings (pack, verify, then clear); fv-ml1 HF_HOME on tank and root-pool cleanup

services/sudo-io-archive/sudo-io-archive-host.sh

@@ -0,0 +1,91 @@
+#!/usr/bin/env bash
+# Host side of the sudo I/O log archiver. Runs as ROOT on a fleet host, fed over ssh by the collector:
+#   ssh infra-ops@<host> 'sudo -n bash -s -- pack 30'        < sudo-io-archive-host.sh
+#   ssh infra-ops@<host> 'sudo -n bash -s -- commit <sha256>' < sudo-io-archive-host.sh
+#
+# Why this exists: every infra-ops sudoers file carries `Defaults:infra-ops log_output`, so sudo records each
+# session under /var/log/sudo-io/XX/YY/ZZ with no retention at all. fv-ml1 reached 16 G / 14,076 sessions by
+# 2026-10-04. Prime: "those audits should be archived and backed up, then cleared. That should be a regular thing."
+#
+# Safety rules this script keeps (eshpfi auto-memory feedback_no_rm_on_loop_variable):
+#   - every destructive target is a LITERAL path; nothing is deleted through a shell variable;
+#   - the session logs themselves are removed only by `tar --remove-files` reading the SAME list the shipped archive
+#     was built from, and only after the collector has confirmed the archive's sha256 on its side;
+#   - a session is eligible only when NO file in it is newer than the cutoff, so a still-running session is never
+#     touched, and the list is refused if any entry is absolute or contains `..`.
+# Reading stdin is off limits here: bash -s is reading this very script from it.
+set -euo pipefail
+phase=${1:?usage: pack <days> | commit <sha256> | status}
+arg=${2:-}
+
+if [ ! -d /var/log/sudo-io ]; then echo "RESULT NOSUDOIO"; exit 0; fi
+install -d -m 700 /var/log/sudo-io-archive
+
+case "$phase" in
+pack)
+  [[ $arg =~ ^[0-9]+$ ]] || { echo "pack needs a day count"; exit 2; }
+  if [ -f /var/log/sudo-io-archive/outbox.tar.gz ]; then
+    # a previous run packed but never committed: ship that same archive again rather than repacking
+    chgrp infra-ops /var/log/sudo-io-archive /var/log/sudo-io-archive/outbox.tar.gz
+    chmod 750 /var/log/sudo-io-archive
+    chmod 640 /var/log/sudo-io-archive/outbox.tar.gz
+    echo "RESULT PACKED $(cut -d' ' -f1 /var/log/sudo-io-archive/outbox.sha256) $(cat /var/log/sudo-io-archive/count) reused"
+    exit 0
+  fi
+  touch -d "-${arg} days" /var/log/sudo-io-archive/cutoff
+  : > /var/log/sudo-io-archive/list
+  # session dirs are XX/YY/ZZ; include one only if nothing inside it changed after the cutoff
+  while IFS= read -r -d '' rel; do
+    if [ -z "$(find "/var/log/sudo-io/$rel" -newer /var/log/sudo-io-archive/cutoff -print -quit)" ]; then
+      printf '%s\0' "$rel" >> /var/log/sudo-io-archive/list
+    fi
+  done < <(find /var/log/sudo-io -mindepth 3 -maxdepth 3 -type d -printf '%P\0')
+  sessions=$(tr -cd '\0' < /var/log/sudo-io-archive/list | wc -c)
+  if [ "$sessions" -eq 0 ]; then
+    rm -f /var/log/sudo-io-archive/list
+    rm -f /var/log/sudo-io-archive/cutoff
+    echo "RESULT NOTHING"
+    exit 0
+  fi
+  # sh -c so the session dirs land BEFORE -type (plain `xargs find -type f` appends them after the predicate)
+  files=$(cd /var/log/sudo-io && xargs -0 -a /var/log/sudo-io-archive/list sh -c 'find "$@" -type f' sh | wc -l)
+  # gzip, not zstd: it is on every Debian host, and sudo already gzips the stream files inside each session
+  tar -C /var/log/sudo-io --null -T /var/log/sudo-io-archive/list -cf - | gzip -1 > /var/log/sudo-io-archive/outbox.tar.gz
+  gzip -t /var/log/sudo-io-archive/outbox.tar.gz
+  members=$(tar -tzf /var/log/sudo-io-archive/outbox.tar.gz | grep -vc '/$' || true)
+  if [ "$members" -ne "$files" ]; then
+    echo "archive holds $members files, the sessions hold $files: refusing"; exit 3
+  fi
+  echo "$sessions $files" > /var/log/sudo-io-archive/count
+  sha256sum /var/log/sudo-io-archive/outbox.tar.gz > /var/log/sudo-io-archive/outbox.sha256
+  # the collector reads the outbox as plain infra-ops, NOT through sudo: sudo's log_output would record the whole
+  # archive as a new session, and sudo >= 1.9.14 (Debian 13) truncated the piped stream at exit (2026-10-04:
+  # fv-ml1 and corviduo-dev pulls came back 52 K and 7 K short).
+  chgrp infra-ops /var/log/sudo-io-archive /var/log/sudo-io-archive/outbox.tar.gz
+  chmod 750 /var/log/sudo-io-archive
+  chmod 640 /var/log/sudo-io-archive/outbox.tar.gz
+  echo "RESULT PACKED $(cut -d' ' -f1 /var/log/sudo-io-archive/outbox.sha256) $sessions $files"
+  ;;
+commit)
+  [[ $arg =~ ^[0-9a-f]{64}$ ]] || { echo "commit needs the collector's sha256"; exit 2; }
+  [ -f /var/log/sudo-io-archive/outbox.tar.gz ] || { echo "no outbox to commit"; exit 3; }
+  [ "$(cut -d' ' -f1 /var/log/sudo-io-archive/outbox.sha256)" = "$arg" ] || { echo "sha mismatch with the collector: refusing"; exit 3; }
+  [ "$(sha256sum /var/log/sudo-io-archive/outbox.tar.gz | cut -d' ' -f1)" = "$arg" ] || { echo "outbox changed since pack: refusing"; exit 3; }
+  if grep -z -q -E '(^/|(^|/)\.\.(/|$))' /var/log/sudo-io-archive/list; then echo "unsafe path in list: refusing"; exit 3; fi
+  # removal = exactly the archived list, via tar itself (no rm on a variable). Output discarded; the archive that
+  # matters is the one the collector already holds.
+  # --ignore-failed-read: a retried commit (files already gone) must still finish its cleanup, not wedge forever
+  tar -C /var/log/sudo-io --null -T /var/log/sudo-io-archive/list --remove-files --ignore-failed-read -cf - | cat > /dev/null
+  find /var/log/sudo-io -mindepth 1 -maxdepth 2 -type d -empty -delete
+  rm -f /var/log/sudo-io-archive/outbox.tar.gz
+  rm -f /var/log/sudo-io-archive/outbox.sha256
+  rm -f /var/log/sudo-io-archive/list
+  rm -f /var/log/sudo-io-archive/cutoff
+  rm -f /var/log/sudo-io-archive/count
+  echo "RESULT COMMITTED $(du -sh /var/log/sudo-io | cut -f1) left"
+  ;;
+status)
+  echo "RESULT STATUS $(du -sh /var/log/sudo-io | cut -f1) $(find /var/log/sudo-io -mindepth 3 -maxdepth 3 -type d | wc -l) sessions outbox=$([ -f /var/log/sudo-io-archive/outbox.tar.gz ] && echo yes || echo no)"
+  ;;
+*) echo "unknown phase $phase"; exit 2 ;;
+esac