#!/usr/bin/env bash
# Collector for the fleet's sudo I/O logs. Runs on nh3-dev (weekly user timer, or by hand).
#   services/sudo-io-archive/sudo-io-archive                 # every host in ./hosts
#   services/sudo-io-archive/sudo-io-archive fv-ml1 pbs-ana  # just these (names from ./hosts)
#   SUDO_IO_DAYS=30 (default)   SUDO_IO_ARCHIVE_DIR=~/archive/sudo-io (default)
#
# Per host: the host packs sessions older than SUDO_IO_DAYS into one outbox archive (sudo-io-archive-host.sh pack),
# this side pulls it, checks its sha256 and gzip integrity, keeps it under <dir>/<host>/, and only then tells the
# host to delete exactly what it archived (… commit <sha>). A host that fails at any step keeps its logs and its
# outbox; the next run re-ships the same outbox. Backed up because nh3-dev (VM 102 on nh3-pve) is in the nightly
# vzdump to pbs-ana, which pbs-nh3 mirrors.
#
# Nothing here deletes anything locally (auto-memory feedback_no_rm_on_loop_variable): a failed pull leaves a
# `.part` file behind, which the summary reports.
set -uo pipefail
umask 077   # the archives hold whatever the sessions printed; owner-only
here=$(cd "$(dirname "$0")" && pwd)
days=${SUDO_IO_DAYS:-30}
dest=${SUDO_IO_ARCHIVE_DIR:-$HOME/archive/sudo-io}
hostscript="$here/sudo-io-archive-host.sh"
ssh_opts=(-o BatchMode=yes -o ConnectTimeout=10 -o ServerAliveInterval=30)
opslog="$here/../../scripts/ops-log"
want=("$@")
fail=0
summary=()

# hosts come in on fd 3: ssh reads stdin, and on fd 0 the first pull swallowed the rest of the list (2026-10-04)
while read -r -u 3 name target; do
  [[ -z ${name:-} || $name == \#* ]] && continue
  if [ ${#want[@]} -gt 0 ] && [[ ! " ${want[*]} " =~ " $name " ]]; then continue; fi
  r=$(ssh "${ssh_opts[@]}" "$target" "sudo -n bash -s -- pack $days" < "$hostscript" 2>&1 | grep '^RESULT' | tail -1)
  read -r _ kind sha sessions files _ <<<"${r:-RESULT ERROR}"
  case "$kind" in
    NOSUDOIO|NOTHING) summary+=("$name: ${kind,,}"); continue ;;
    PACKED) ;;
    *) summary+=("$name: PACK FAILED (${r:-no answer})"); fail=1; continue ;;
  esac
  install -d -m 700 "$dest/$name"
  f="$dest/$name/$name-$(date -u +%Y%m%dT%H%M%SZ)-older-than-${days}d.tar.gz"
  if ! ssh -n "${ssh_opts[@]}" "$target" 'cat /var/log/sudo-io-archive/outbox.tar.gz' > "$f.part"; then
    summary+=("$name: PULL FAILED, left $f.part"); fail=1; continue
  fi
  got=$(sha256sum "$f.part" | cut -d' ' -f1)
  if [ "$got" != "$sha" ] || ! gzip -t "$f.part" 2>/dev/null; then
    summary+=("$name: VERIFY FAILED (sha $got vs $sha), left $f.part"); fail=1; continue
  fi
  mv -n "$f.part" "$f"
  echo "$sha  $(basename "$f")" > "$f.sha256"
  c=$(ssh "${ssh_opts[@]}" "$target" "sudo -n bash -s -- commit $sha" < "$hostscript" 2>&1 | grep '^RESULT' | tail -1)
  if [[ $c != "RESULT COMMITTED"* ]]; then
    summary+=("$name: archived to $f but COMMIT FAILED (${c:-no answer}); host keeps its logs"); fail=1; continue
  fi
  size=$(du -h "$f" | cut -f1)
  summary+=("$name: $sessions sessions / $files files -> $f ($size); host ${c#RESULT COMMITTED }")
  [ -x "$opslog" ] && "$opslog" record --host "$name" --action archive --target sudo-io \
    --detail "sudo I/O logs older than ${days}d: $sessions sessions archived to nh3-dev $f ($size, sha256 ${sha:0:12}) and cleared on the host" >/dev/null 2>&1
done 3< "$here/hosts"

printf '%s\n' "${summary[@]}"
exit $fail
