61 lines
3.4 KiB
Bash
Executable File
61 lines
3.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# Collector for the fleet's sudo I/O logs. Runs on nh3-dev (weekly user timer, or by hand).
|
|
# services/sudo-io-archive/sudo-io-archive # every host in ./hosts
|
|
# services/sudo-io-archive/sudo-io-archive fv-ml1 pbs-ana # just these (names from ./hosts)
|
|
# SUDO_IO_DAYS=30 (default) SUDO_IO_ARCHIVE_DIR=~/archive/sudo-io (default)
|
|
#
|
|
# Per host: the host packs sessions older than SUDO_IO_DAYS into one outbox archive (sudo-io-archive-host.sh pack),
|
|
# this side pulls it, checks its sha256 and gzip integrity, keeps it under <dir>/<host>/, and only then tells the
|
|
# host to delete exactly what it archived (… commit <sha>). A host that fails at any step keeps its logs and its
|
|
# outbox; the next run re-ships the same outbox. Backed up because nh3-dev (VM 102 on nh3-pve) is in the nightly
|
|
# vzdump to pbs-ana, which pbs-nh3 mirrors.
|
|
#
|
|
# Nothing here deletes anything locally (auto-memory feedback_no_rm_on_loop_variable): a failed pull leaves a
|
|
# `.part` file behind, which the summary reports.
|
|
set -uo pipefail
|
|
umask 077 # the archives hold whatever the sessions printed; owner-only
|
|
here=$(cd "$(dirname "$0")" && pwd)
|
|
days=${SUDO_IO_DAYS:-30}
|
|
dest=${SUDO_IO_ARCHIVE_DIR:-$HOME/archive/sudo-io}
|
|
hostscript="$here/sudo-io-archive-host.sh"
|
|
ssh_opts=(-o BatchMode=yes -o ConnectTimeout=10 -o ServerAliveInterval=30)
|
|
opslog="$here/../../scripts/ops-log"
|
|
want=("$@")
|
|
fail=0
|
|
summary=()
|
|
|
|
# hosts come in on fd 3: ssh reads stdin, and on fd 0 the first pull swallowed the rest of the list (2026-10-04)
|
|
while read -r -u 3 name target; do
|
|
[[ -z ${name:-} || $name == \#* ]] && continue
|
|
if [ ${#want[@]} -gt 0 ] && [[ ! " ${want[*]} " =~ " $name " ]]; then continue; fi
|
|
r=$(ssh "${ssh_opts[@]}" "$target" "sudo -n bash -s -- pack $days" < "$hostscript" 2>&1 | grep '^RESULT' | tail -1)
|
|
read -r _ kind sha sessions files oversize _ <<<"${r:-RESULT ERROR}"
|
|
case "$kind" in
|
|
NOSUDOIO|NOTHING) summary+=("$name: ${kind,,}"); continue ;;
|
|
PACKED) ;;
|
|
*) summary+=("$name: PACK FAILED (${r:-no answer})"); fail=1; continue ;;
|
|
esac
|
|
install -d -m 700 "$dest/$name"
|
|
f="$dest/$name/$name-$(date -u +%Y%m%dT%H%M%SZ)-older-than-${days}d.tar.gz"
|
|
if ! ssh -n "${ssh_opts[@]}" "$target" 'cat /var/log/sudo-io-archive/outbox.tar.gz' > "$f.part"; then
|
|
summary+=("$name: PULL FAILED, left $f.part"); fail=1; continue
|
|
fi
|
|
got=$(sha256sum "$f.part" | cut -d' ' -f1)
|
|
if [ "$got" != "$sha" ] || ! gzip -t "$f.part" 2>/dev/null; then
|
|
summary+=("$name: VERIFY FAILED (sha $got vs $sha), left $f.part"); fail=1; continue
|
|
fi
|
|
mv -n "$f.part" "$f"
|
|
echo "$sha $(basename "$f")" > "$f.sha256"
|
|
c=$(ssh "${ssh_opts[@]}" "$target" "sudo -n bash -s -- commit $sha" < "$hostscript" 2>&1 | grep '^RESULT' | tail -1)
|
|
if [[ $c != "RESULT COMMITTED"* ]]; then
|
|
summary+=("$name: archived to $f but COMMIT FAILED (${c:-no answer}); host keeps its logs"); fail=1; continue
|
|
fi
|
|
size=$(du -h "$f" | cut -f1)
|
|
summary+=("$name: $sessions sessions / $files files (${oversize:-0} oversize streams not kept) -> $f ($size); host ${c#RESULT COMMITTED }")
|
|
[ -x "$opslog" ] && "$opslog" record --host "$name" --action archive --target sudo-io \
|
|
--detail "sudo I/O logs older than ${days}d: $sessions sessions archived to nh3-dev $f ($size, sha256 ${sha:0:12}; ${oversize:-0} oversize streams dropped) and cleared on the host" >/dev/null 2>&1
|
|
done 3< "$here/hosts"
|
|
|
|
printf '%s\n' "${summary[@]}"
|
|
exit $fail
|