esh-pfi-infrastructure/playbooks/nh3-dev-memory-forensics.yaml

# nh3-dev: make memory exhaustion diagnosable after the fact.
#
# Context: three memory-exhaustion events in 14 days (2026-08-14, 2026-08-26,
# 2026-08-28) with the interval halving. forseti reported that none could be
# attributed because "kernel messages are not being persisted to journald".
#
# ⚠ THAT PREMISE WAS WRONG, and the way it was wrong matters more than the fix.
# journald on this box IS persistent and HAS every OOM report: 15,068 kernel
# entries in the 82-day previous boot, 351 OOM records overall, full task tables
# with per-process RSS. `journalctl -b -1 -k` returned "one entry" because it was
# run by a user in neither `adm` nor `systemd-journal` — journalctl silently shows
# you only your OWN messages and prints the reason as a hint. The same artifact
# also produced the "journal stops mid-line at 05:36:08 with no shutdown
# sequence" claim: the true boot -1 boundary is 05:47:04, and OOM kills are
# recorded at 05:38, 05:40 and 05:42.
#
# So step 1 is a GROUP MEMBERSHIP fix, not a logging fix. The evidence was
# always there and unreadable.
#
# What the evidence says, now that it can be read: the hog is Claude Code.
# `/home/lkraven/.local/share/claude/versions/2.1.220` is the versioned CC binary,
# so OOM victims named `2.1.220` / `2.1.177` / `2.1.168` are CC sessions, as are
# the ones named `claude`. Largest single anon-rss recorded: 18.4 GB (2.1.177,
# Aug 14), with 15.8 GB seen twice. Everything else killed — althing-forseti,
# caddy, ttyd, zellij, the althing daemons — is 30-55 MB collateral.
#
# steps 2-4 add the timeseries that the journal cannot give: the journal records
# the moment of the kill, not the ramp toward it, and the Aug 28 event was a hard
# lockup where the box never got far enough to log a coherent sweep.
#   - sysstat  -> system-wide memory/CPU timeseries (the ramp)
#   - atop     -> PER-PROCESS timeseries (which session, and how fast)
# atop is the one that answers "which of the dozen sessions", which sar cannot.
#
# Run:  scripts/elway infra-ops@nh3-dev --playbook playbooks/nh3-dev-memory-forensics.yaml
# Rerunnable: a second run shows every step `skipped` or `ok`.

vars:
  # The interactive/agent user whose sessions read the journal.
  journal_user: lkraven
  # Debian's journald ACL grants read to `adm` explicitly (getfacl shows
  # group:adm:r-x); `systemd-journal` owns the files. `adm` is the documented
  # Debian path and the one the ACL names, so use it.
  journal_group: adm
  # 5 min, not Debian's default 10 — a CC session can add several GB inside one
  # 10-minute bucket, which is exactly the resolution the ramp needs.
  sar_interval: "*:00/05"
  # 60s per-process sample. ~7 generations keeps this under ~1 GB against 80 GB free.
  atop_interval: "60"
  atop_generations: "7"

steps:
  - name: Grant the agent user journal read access (THE actual fix for "no evidence")
    shell: sudo usermod -aG {{ journal_group }} {{ journal_user }}
    when: "! id -nG {{ journal_user }} | grep -qw {{ journal_group }}"

  - name: Install sysstat and atop
    shell: sudo DEBIAN_FRONTEND=noninteractive apt-get install -y sysstat atop
    when: "! dpkg -s sysstat >/dev/null 2>&1 || ! dpkg -s atop >/dev/null 2>&1"

  - name: Enable sysstat collection in /etc/default/sysstat
    # The package ships ENABLED="false" and the timer is a no-op until this flips.
    shell: sudo sed -i 's/^ENABLED=.*/ENABLED="true"/' /etc/default/sysstat
    when: "! grep -qxF 'ENABLED=\"true\"' /etc/default/sysstat 2>/dev/null"

  - name: Tighten the sysstat collection interval to 5 minutes
    shell: |
      sudo mkdir -p /etc/systemd/system/sysstat-collect.timer.d
      printf '[Timer]\n# Default is */10. A CC session can add several GB inside one 10-minute\n# bucket; 5 min is the resolution the memory ramp actually needs.\nOnCalendar=\nOnCalendar=%s\n' '{{ sar_interval }}' | sudo tee /etc/systemd/system/sysstat-collect.timer.d/override.conf >/dev/null
    when: "! grep -qxF 'OnCalendar={{ sar_interval }}' /etc/systemd/system/sysstat-collect.timer.d/override.conf 2>/dev/null"

  - name: Configure atop for 60s per-process sampling with 7-day retention
    shell: |
      sudo sed -i 's/^LOGINTERVAL=.*/LOGINTERVAL={{ atop_interval }}/' /etc/default/atop
      sudo sed -i 's/^LOGGENERATIONS=.*/LOGGENERATIONS={{ atop_generations }}/' /etc/default/atop
    when: "! grep -qxF 'LOGINTERVAL={{ atop_interval }}' /etc/default/atop 2>/dev/null || ! grep -qxF 'LOGGENERATIONS={{ atop_generations }}' /etc/default/atop 2>/dev/null"

  - name: Reload systemd and enable the collectors
    shell: |
      sudo systemctl daemon-reload
      sudo systemctl enable --now sysstat.service sysstat-collect.timer sysstat-summary.timer
      sudo systemctl enable --now atopacct.service atop.service atop-rotate.timer
      sudo systemctl restart atop.service

  - name: Seed one sysstat sample so sar has data immediately
    shell: sudo /usr/lib/sysstat/sa1 1 1

verify:
  - name: Agent user is now in the journal-reading group
    # `sg` evaluates the membership WITHOUT waiting for a re-login, so this
    # asserts the effective grant rather than the /etc/group substring.
    shell: sudo -u {{ journal_user }} sg {{ journal_group }} -c 'journalctl -b -1 -k --no-pager 2>/dev/null | wc -l' | awk '{ if ($1 > 100) exit 0; else exit 1 }'
    changed_when: "false"

  - name: sysstat collection timer is active
    shell: systemctl is-active --quiet sysstat-collect.timer
    changed_when: "false"

  - name: sysstat is collecting at the 5-minute cadence
    # ⚠ Assert the EFFECTIVE value, not the string we wrote. systemd normalises
    # `*:00/05` to `*-*-* *:00/5:00`, so grepping for our own input fails while
    # the setting is live — which is exactly how this verify failed on the first
    # run and briefly looked like the override had not applied.
    shell: systemctl show sysstat-collect.timer -p TimersCalendar | grep -qF '*:00/5:00'
    changed_when: "false"

  - name: sar can actually read a memory timeseries (not just that the timer exists)
    shell: sar -r 2>/dev/null | tail -2 | grep -qE '[0-9]'
    changed_when: "false"

  - name: atop daemon is running
    shell: systemctl is-active --quiet atop.service
    changed_when: "false"

  - name: atop is writing a readable per-process log
    shell: sudo test -s /var/log/atop/atop_$(date +%Y%m%d)
    changed_when: "false"