109 lines
7.3 KiB
Bash
109 lines
7.3 KiB
Bash
#!/usr/bin/env bash
|
|
# Host side of the sudo I/O log archiver. Runs as ROOT on a fleet host, fed over ssh by the collector:
|
|
# ssh infra-ops@<host> 'sudo -n bash -s -- pack 30' < sudo-io-archive-host.sh
|
|
# ssh infra-ops@<host> 'sudo -n bash -s -- commit <sha256>' < sudo-io-archive-host.sh
|
|
#
|
|
# Why this exists: every infra-ops sudoers file carries `Defaults:infra-ops log_output`, so sudo records each
|
|
# session under /var/log/sudo-io/XX/YY/ZZ with no retention at all. fv-ml1 reached 16 G / 14,076 sessions by
|
|
# 2026-10-04. Prime: "those audits should be archived and backed up, then cleared. That should be a regular thing."
|
|
#
|
|
# Safety rules this script keeps (eshpfi auto-memory feedback_no_rm_on_loop_variable):
|
|
# - every destructive target is a LITERAL path; nothing is deleted through a shell variable;
|
|
# - the session logs themselves are removed only by `tar --remove-files` reading the SAME list the shipped archive
|
|
# was built from, and only after the collector has confirmed the archive's sha256 on its side;
|
|
# - a session is eligible only when NO file in it is newer than the cutoff, so a still-running session is never
|
|
# touched, and the list is refused if any entry is absolute or contains `..`.
|
|
# Reading stdin is off limits here: bash -s is reading this very script from it.
|
|
set -euo pipefail
|
|
phase=${1:?usage: pack <days> | commit <sha256> | status}
|
|
arg=${2:-}
|
|
|
|
if [ ! -d /var/log/sudo-io ]; then echo "RESULT NOSUDOIO"; exit 0; fi
|
|
install -d -m 700 /var/log/sudo-io-archive
|
|
|
|
case "$phase" in
|
|
pack)
|
|
[[ $arg =~ ^[0-9]+$ ]] || { echo "pack needs a day count"; exit 2; }
|
|
if [ -f /var/log/sudo-io-archive/outbox.tar.gz ] && [ -f /var/log/sudo-io-archive/outbox.sha256 ] && [ -f /var/log/sudo-io-archive/count ]; then
|
|
# a previous run packed but never committed: ship that same archive again rather than repacking
|
|
chgrp infra-ops /var/log/sudo-io-archive /var/log/sudo-io-archive/outbox.tar.gz
|
|
chmod 750 /var/log/sudo-io-archive
|
|
chmod 640 /var/log/sudo-io-archive/outbox.tar.gz
|
|
echo "RESULT PACKED $(cut -d' ' -f1 /var/log/sudo-io-archive/outbox.sha256) $(cat /var/log/sudo-io-archive/count) reused"
|
|
exit 0
|
|
fi
|
|
# leftovers of a pack that failed its own checks (no sha/count written, so never shipped): start over
|
|
rm -f /var/log/sudo-io-archive/outbox.tar.gz
|
|
rm -f /var/log/sudo-io-archive/outbox.tar.gz.new
|
|
touch -d "-${arg} days" /var/log/sudo-io-archive/cutoff
|
|
: > /var/log/sudo-io-archive/list
|
|
# session dirs are XX/YY/ZZ; include one only if nothing inside it changed after the cutoff
|
|
while IFS= read -r -d '' rel; do
|
|
if [ -z "$(find "/var/log/sudo-io/$rel" -newer /var/log/sudo-io-archive/cutoff -print -quit)" ]; then
|
|
printf '%s\0' "$rel" >> /var/log/sudo-io-archive/list
|
|
fi
|
|
done < <(find /var/log/sudo-io -mindepth 3 -maxdepth 3 -type d -printf '%P\0')
|
|
sessions=$(tr -cd '\0' < /var/log/sudo-io-archive/list | wc -c)
|
|
if [ "$sessions" -eq 0 ]; then
|
|
rm -f /var/log/sudo-io-archive/list
|
|
rm -f /var/log/sudo-io-archive/cutoff
|
|
echo "RESULT NOTHING"
|
|
exit 0
|
|
fi
|
|
# sh -c so the session dirs land BEFORE -type (plain `xargs find -type f` appends them after the predicate)
|
|
files=$(cd /var/log/sudo-io && xargs -0 -a /var/log/sudo-io-archive/list sh -c 'find "$@" -type f' sh | wc -l)
|
|
# Oversize output streams are not archived (Prime 2026-10-04: "we don't need to durably back up weights and rsyncs
|
|
# forever"). Any stdout/stdin/tty stream over 64 MiB is DATA moving through sudo, not an audit record; the session's
|
|
# log, log.json and timing are still archived, and commit deletes the stream with everything else. Since the same
|
|
# day the infra-ops sudoers also stops recording rsync/tar/cat/dd/docker save at all, so this is the backstop.
|
|
(cd /var/log/sudo-io && xargs -0 -a /var/log/sudo-io-archive/list sh -c \
|
|
'find "$@" -type f \( -name stdout -o -name stdin -o -name ttyout -o -name ttyin -o -name stderr \) -size +64M -print0' sh) \
|
|
> /var/log/sudo-io-archive/oversize
|
|
# NUL-separated on purpose: tar's --null applies to -X as well as -T (a newline list silently excluded nothing,
|
|
# caught by the member-count check in the 2026-10-04 positive control)
|
|
oversize=$(tr -cd '\0' < /var/log/sudo-io-archive/oversize | wc -c)
|
|
# gzip, not zstd: it is on every Debian host, and sudo already gzips the stream files inside each session
|
|
# -X before -T: GNU tar options are positional, and an exclude placed after the file list "has no effect"
|
|
tar -C /var/log/sudo-io --null -X /var/log/sudo-io-archive/oversize -T /var/log/sudo-io-archive/list -cf - \
|
|
| gzip -1 > /var/log/sudo-io-archive/outbox.tar.gz.new
|
|
gzip -t /var/log/sudo-io-archive/outbox.tar.gz.new
|
|
members=$(tar -tzf /var/log/sudo-io-archive/outbox.tar.gz.new | grep -vc '/$' || true)
|
|
if [ "$members" -ne $((files - oversize)) ]; then
|
|
echo "archive holds $members files, expected $files minus $oversize oversize streams: refusing"; exit 3
|
|
fi
|
|
mv /var/log/sudo-io-archive/outbox.tar.gz.new /var/log/sudo-io-archive/outbox.tar.gz
|
|
echo "$sessions $files $oversize" > /var/log/sudo-io-archive/count
|
|
sha256sum /var/log/sudo-io-archive/outbox.tar.gz > /var/log/sudo-io-archive/outbox.sha256
|
|
# the collector reads the outbox as plain infra-ops, NOT through sudo: sudo's log_output would record the whole
|
|
# archive as a new session, and sudo >= 1.9.14 (Debian 13) truncated the piped stream at exit (2026-10-04:
|
|
# fv-ml1 and corviduo-dev pulls came back 52 K and 7 K short).
|
|
chgrp infra-ops /var/log/sudo-io-archive /var/log/sudo-io-archive/outbox.tar.gz
|
|
chmod 750 /var/log/sudo-io-archive
|
|
chmod 640 /var/log/sudo-io-archive/outbox.tar.gz
|
|
echo "RESULT PACKED $(cut -d' ' -f1 /var/log/sudo-io-archive/outbox.sha256) $sessions $files $oversize"
|
|
;;
|
|
commit)
|
|
[[ $arg =~ ^[0-9a-f]{64}$ ]] || { echo "commit needs the collector's sha256"; exit 2; }
|
|
[ -f /var/log/sudo-io-archive/outbox.tar.gz ] || { echo "no outbox to commit"; exit 3; }
|
|
[ "$(cut -d' ' -f1 /var/log/sudo-io-archive/outbox.sha256)" = "$arg" ] || { echo "sha mismatch with the collector: refusing"; exit 3; }
|
|
[ "$(sha256sum /var/log/sudo-io-archive/outbox.tar.gz | cut -d' ' -f1)" = "$arg" ] || { echo "outbox changed since pack: refusing"; exit 3; }
|
|
if grep -z -q -E '(^/|(^|/)\.\.(/|$))' /var/log/sudo-io-archive/list; then echo "unsafe path in list: refusing"; exit 3; fi
|
|
# removal = exactly the archived list, via tar itself (no rm on a variable). Output discarded; the archive that
|
|
# matters is the one the collector already holds.
|
|
# --ignore-failed-read: a retried commit (files already gone) must still finish its cleanup, not wedge forever
|
|
tar -C /var/log/sudo-io --null -T /var/log/sudo-io-archive/list --remove-files --ignore-failed-read -cf - | cat > /dev/null
|
|
find /var/log/sudo-io -mindepth 1 -maxdepth 2 -type d -empty -delete
|
|
rm -f /var/log/sudo-io-archive/outbox.tar.gz
|
|
rm -f /var/log/sudo-io-archive/outbox.sha256
|
|
rm -f /var/log/sudo-io-archive/list
|
|
rm -f /var/log/sudo-io-archive/cutoff
|
|
rm -f /var/log/sudo-io-archive/count
|
|
rm -f /var/log/sudo-io-archive/oversize
|
|
echo "RESULT COMMITTED $(du -sh /var/log/sudo-io | cut -f1) left"
|
|
;;
|
|
status)
|
|
echo "RESULT STATUS $(du -sh /var/log/sudo-io | cut -f1) $(find /var/log/sudo-io -mindepth 3 -maxdepth 3 -type d | wc -l) sessions outbox=$([ -f /var/log/sudo-io-archive/outbox.tar.gz ] && echo yes || echo no)"
|
|
;;
|
|
*) echo "unknown phase $phase"; exit 2 ;;
|
|
esac
|