feat(sudo-io): stop recording bulk data (sudoers BULK_DATA !log_output fleet-wide), drop >64 MiB streams from archives, prune existing bulk recordings
This commit is contained in:
@@ -34,10 +34,26 @@ ssh infra-ops@<ip> 'sudo -n bash -s -- status' < services/sudo-io-archive/sudo-i
|
||||
|
||||
Replay an archived session: untar it anywhere, then `sudoreplay -d <dir> <XX/YY/ZZ>` (as root).
|
||||
|
||||
## Where the volume comes from (2026-10-04)
|
||||
## Bulk data is not kept (Prime 2026-10-04: "we don't need to durably back up weights and rsyncs forever")
|
||||
|
||||
Almost all of it is DATA piped through sudo, not commands: one fv-ml1 session is a 14.45 G recording of
|
||||
`sudo tar -C /tank/aimodels -cf - erp-tune-v6-nvfp4a16` (a model copy), and esh-docker-vm records a ~380 M
|
||||
`sudo rsync --server --sender … /var/lib/docker/volumes/homeassistant…` stream every night. Exempting bulk-transfer
|
||||
commands from output recording (`Defaults!/usr/bin/rsync,/usr/bin/tar !log_output` in the infra-ops sudoers) would keep
|
||||
the command line in the audit but stop recording the data; that is a policy change for Prime, not made here.
|
||||
Almost all of the volume was DATA piped through sudo, not commands: a 14.45 G recording of
|
||||
`sudo tar -C /tank/aimodels -cf - erp-tune-v6-nvfp4a16` on fv-ml1, a ~380 M `sudo rsync --server --sender` of the
|
||||
Home Assistant volume every night on esh-docker-vm, Worldtree volume rsyncs on corviduo-dev, a `docker save`. Three
|
||||
layers now keep it out:
|
||||
|
||||
1. **Not recorded.** Every infra-ops sudoers file carries a `BULK_DATA` exemption (rsync, tar, cat, dd, zstd, zcat, gzip,
|
||||
pv, `docker save|export|cp`): `Defaults!BULK_DATA !log_output`. sudo still logs who ran the command; it no longer
|
||||
records the bytes. `playbooks/sudoers-infra-ops-bulk-no-log-output.yaml` rolled it out to all 19 recording hosts on
|
||||
2026-10-04 (controls on each: `sudo id` adds a session, `sudo tar --version` does not), and
|
||||
`playbooks/bootstrap-infra-ops-user.yaml` writes it for new hosts.
|
||||
2. **Not archived.** `pack` leaves out any stdout/stdin/tty stream over 64 MiB (the session's log, log.json, timing
|
||||
and stderr are still archived) and `commit` deletes it with the rest. Proven with a labelled 70 MB test session
|
||||
(`TE/ST/OV` in the 2026-10-04 pfi-pteradactyl archive).
|
||||
3. **Pruned once (2026-10-04).** The 36 existing bulk streams on hosts (≈31 G: fv-ml1 15 G, esh-docker-vm 15 G,
|
||||
corviduo-dev, nh3-dev, nh3-docker) were deleted with literal-path `rm`, keeping each session's metadata, and the three
|
||||
collected archives that held bulk streams were rebuilt without them (`*-bulk-pruned.tar.gz`; the store went from
|
||||
4.8 G to 83 M).
|
||||
|
||||
Lessons kept in the scripts: pull the outbox WITHOUT sudo (sudo ≥ 1.9.14 truncated piped output, and log_output
|
||||
re-recorded the archive); read the host list on fd 3 (ssh swallowed it from stdin); GNU tar reads `-X` NUL-separated
|
||||
under `--null` and only honours it BEFORE `-T`. Each of those was caught by a check, not by luck.
|
||||
|
||||
@@ -29,7 +29,7 @@ while read -r -u 3 name target; do
|
||||
[[ -z ${name:-} || $name == \#* ]] && continue
|
||||
if [ ${#want[@]} -gt 0 ] && [[ ! " ${want[*]} " =~ " $name " ]]; then continue; fi
|
||||
r=$(ssh "${ssh_opts[@]}" "$target" "sudo -n bash -s -- pack $days" < "$hostscript" 2>&1 | grep '^RESULT' | tail -1)
|
||||
read -r _ kind sha sessions files _ <<<"${r:-RESULT ERROR}"
|
||||
read -r _ kind sha sessions files oversize _ <<<"${r:-RESULT ERROR}"
|
||||
case "$kind" in
|
||||
NOSUDOIO|NOTHING) summary+=("$name: ${kind,,}"); continue ;;
|
||||
PACKED) ;;
|
||||
@@ -51,9 +51,9 @@ while read -r -u 3 name target; do
|
||||
summary+=("$name: archived to $f but COMMIT FAILED (${c:-no answer}); host keeps its logs"); fail=1; continue
|
||||
fi
|
||||
size=$(du -h "$f" | cut -f1)
|
||||
summary+=("$name: $sessions sessions / $files files -> $f ($size); host ${c#RESULT COMMITTED }")
|
||||
summary+=("$name: $sessions sessions / $files files (${oversize:-0} oversize streams not kept) -> $f ($size); host ${c#RESULT COMMITTED }")
|
||||
[ -x "$opslog" ] && "$opslog" record --host "$name" --action archive --target sudo-io \
|
||||
--detail "sudo I/O logs older than ${days}d: $sessions sessions archived to nh3-dev $f ($size, sha256 ${sha:0:12}) and cleared on the host" >/dev/null 2>&1
|
||||
--detail "sudo I/O logs older than ${days}d: $sessions sessions archived to nh3-dev $f ($size, sha256 ${sha:0:12}; ${oversize:-0} oversize streams dropped) and cleared on the host" >/dev/null 2>&1
|
||||
done 3< "$here/hosts"
|
||||
|
||||
printf '%s\n' "${summary[@]}"
|
||||
|
||||
@@ -24,7 +24,7 @@ install -d -m 700 /var/log/sudo-io-archive
|
||||
case "$phase" in
|
||||
pack)
|
||||
[[ $arg =~ ^[0-9]+$ ]] || { echo "pack needs a day count"; exit 2; }
|
||||
if [ -f /var/log/sudo-io-archive/outbox.tar.gz ]; then
|
||||
if [ -f /var/log/sudo-io-archive/outbox.tar.gz ] && [ -f /var/log/sudo-io-archive/outbox.sha256 ] && [ -f /var/log/sudo-io-archive/count ]; then
|
||||
# a previous run packed but never committed: ship that same archive again rather than repacking
|
||||
chgrp infra-ops /var/log/sudo-io-archive /var/log/sudo-io-archive/outbox.tar.gz
|
||||
chmod 750 /var/log/sudo-io-archive
|
||||
@@ -32,6 +32,9 @@ pack)
|
||||
echo "RESULT PACKED $(cut -d' ' -f1 /var/log/sudo-io-archive/outbox.sha256) $(cat /var/log/sudo-io-archive/count) reused"
|
||||
exit 0
|
||||
fi
|
||||
# leftovers of a pack that failed its own checks (no sha/count written, so never shipped): start over
|
||||
rm -f /var/log/sudo-io-archive/outbox.tar.gz
|
||||
rm -f /var/log/sudo-io-archive/outbox.tar.gz.new
|
||||
touch -d "-${arg} days" /var/log/sudo-io-archive/cutoff
|
||||
: > /var/log/sudo-io-archive/list
|
||||
# session dirs are XX/YY/ZZ; include one only if nothing inside it changed after the cutoff
|
||||
@@ -49,14 +52,27 @@ pack)
|
||||
fi
|
||||
# sh -c so the session dirs land BEFORE -type (plain `xargs find -type f` appends them after the predicate)
|
||||
files=$(cd /var/log/sudo-io && xargs -0 -a /var/log/sudo-io-archive/list sh -c 'find "$@" -type f' sh | wc -l)
|
||||
# Oversize output streams are not archived (Prime 2026-10-04: "we don't need to durably back up weights and rsyncs
|
||||
# forever"). Any stdout/stdin/tty stream over 64 MiB is DATA moving through sudo, not an audit record; the session's
|
||||
# log, log.json and timing are still archived, and commit deletes the stream with everything else. Since the same
|
||||
# day the infra-ops sudoers also stops recording rsync/tar/cat/dd/docker save at all, so this is the backstop.
|
||||
(cd /var/log/sudo-io && xargs -0 -a /var/log/sudo-io-archive/list sh -c \
|
||||
'find "$@" -type f \( -name stdout -o -name stdin -o -name ttyout -o -name ttyin -o -name stderr \) -size +64M -print0' sh) \
|
||||
> /var/log/sudo-io-archive/oversize
|
||||
# NUL-separated on purpose: tar's --null applies to -X as well as -T (a newline list silently excluded nothing,
|
||||
# caught by the member-count check in the 2026-10-04 positive control)
|
||||
oversize=$(tr -cd '\0' < /var/log/sudo-io-archive/oversize | wc -c)
|
||||
# gzip, not zstd: it is on every Debian host, and sudo already gzips the stream files inside each session
|
||||
tar -C /var/log/sudo-io --null -T /var/log/sudo-io-archive/list -cf - | gzip -1 > /var/log/sudo-io-archive/outbox.tar.gz
|
||||
gzip -t /var/log/sudo-io-archive/outbox.tar.gz
|
||||
members=$(tar -tzf /var/log/sudo-io-archive/outbox.tar.gz | grep -vc '/$' || true)
|
||||
if [ "$members" -ne "$files" ]; then
|
||||
echo "archive holds $members files, the sessions hold $files: refusing"; exit 3
|
||||
# -X before -T: GNU tar options are positional, and an exclude placed after the file list "has no effect"
|
||||
tar -C /var/log/sudo-io --null -X /var/log/sudo-io-archive/oversize -T /var/log/sudo-io-archive/list -cf - \
|
||||
| gzip -1 > /var/log/sudo-io-archive/outbox.tar.gz.new
|
||||
gzip -t /var/log/sudo-io-archive/outbox.tar.gz.new
|
||||
members=$(tar -tzf /var/log/sudo-io-archive/outbox.tar.gz.new | grep -vc '/$' || true)
|
||||
if [ "$members" -ne $((files - oversize)) ]; then
|
||||
echo "archive holds $members files, expected $files minus $oversize oversize streams: refusing"; exit 3
|
||||
fi
|
||||
echo "$sessions $files" > /var/log/sudo-io-archive/count
|
||||
mv /var/log/sudo-io-archive/outbox.tar.gz.new /var/log/sudo-io-archive/outbox.tar.gz
|
||||
echo "$sessions $files $oversize" > /var/log/sudo-io-archive/count
|
||||
sha256sum /var/log/sudo-io-archive/outbox.tar.gz > /var/log/sudo-io-archive/outbox.sha256
|
||||
# the collector reads the outbox as plain infra-ops, NOT through sudo: sudo's log_output would record the whole
|
||||
# archive as a new session, and sudo >= 1.9.14 (Debian 13) truncated the piped stream at exit (2026-10-04:
|
||||
@@ -64,7 +80,7 @@ pack)
|
||||
chgrp infra-ops /var/log/sudo-io-archive /var/log/sudo-io-archive/outbox.tar.gz
|
||||
chmod 750 /var/log/sudo-io-archive
|
||||
chmod 640 /var/log/sudo-io-archive/outbox.tar.gz
|
||||
echo "RESULT PACKED $(cut -d' ' -f1 /var/log/sudo-io-archive/outbox.sha256) $sessions $files"
|
||||
echo "RESULT PACKED $(cut -d' ' -f1 /var/log/sudo-io-archive/outbox.sha256) $sessions $files $oversize"
|
||||
;;
|
||||
commit)
|
||||
[[ $arg =~ ^[0-9a-f]{64}$ ]] || { echo "commit needs the collector's sha256"; exit 2; }
|
||||
@@ -82,6 +98,7 @@ commit)
|
||||
rm -f /var/log/sudo-io-archive/list
|
||||
rm -f /var/log/sudo-io-archive/cutoff
|
||||
rm -f /var/log/sudo-io-archive/count
|
||||
rm -f /var/log/sudo-io-archive/oversize
|
||||
echo "RESULT COMMITTED $(du -sh /var/log/sudo-io | cut -f1) left"
|
||||
;;
|
||||
status)
|
||||
|
||||
Reference in New Issue
Block a user