esh-pfi-infrastructure/playbooks/esh-pve-hardware-watchdog.yaml

# esh-pve — move from the software watchdog to the PCH hardware watchdog.
#
# WHY: esh-pve hard-froze at 03:34 on 2026-08-19 (no panic, no OOM, no MCE —
# the journal simply stops mid-line) and stayed frozen for ~4.5 hours until it
# was power-cycled by hand. Everything on it went with it, including the only
# DNS resolver the esh-userland VLAN is handed, so the whole house lost name
# resolution.
#
# Nothing on the box could have recovered it:
#   - `softdog` was the loaded watchdog. A SOFTWARE watchdog cannot rescue a
#     hard kernel freeze, because the frozen kernel is the thing that would
#     have to fire its timer.
#   - Proxmox's `watchdog-mux` held /dev/watchdog but never armed it: it only
#     pets the device while an HA client is connected, and this cluster has no
#     HA resources configured (`ha-manager status` reports quorum only).
#
# The board's PCH TCO timer is present and NOT blocked by firmware — verified
# before writing this:
#     iTCO_wdt: Found a Intel PCH TCO device (Version=6, TCOBASE=0x0400)
#     iTCO_wdt: initialized. heartbeat=30 sec (nowayout=0)
# (no "unable to reset NO_REBOOT flag" line, which is the BIOS-blocked case).
#
# APPROACH: systemd owns the hardware watchdog directly. Setting
# WATCHDOG_MODULE=iTCO_wdt in /etc/default/pve-ha-manager would point
# watchdog-mux at the right device but would still never arm it without HA, so
# it does not solve this. systemd's RuntimeWatchdogSec pets unconditionally,
# which is what "reboot me if I wedge" actually requires.
#
# ⚠️ CONSEQUENCE — READ BEFORE ENABLING PROXMOX HA ON THIS CLUSTER.
# This masks `watchdog-mux`. If HA is ever configured on esh-pve, watchdog-mux
# must own /dev/watchdog again and this must be reverted, or HA fencing will
# not work. That is not a near-term concern: esh-pve-cluster is TWO nodes with
# no qdevice, so a single node loss already costs quorum and the survivor would
# fence itself. HA here would make availability worse, not better.
#
# Revert: unmask + enable watchdog-mux, delete the three dropped files,
# `systemctl daemon-reexec`, reboot.
#
# Run:  scripts/elway esh-pve --playbook playbooks/esh-pve-hardware-watchdog.yaml

vars:
  # 60s: long enough that a busy-but-healthy host is never reset, short enough
  # that a freeze costs a minute rather than half a working day. systemd pets
  # at half this interval. PID 1 does not block on filesystem I/O, so the known
  # NFS-wedge history on this host does not put it at risk of a false trip.
  runtime_watchdog_sec: 60

steps:
  - name: Load iTCO_wdt at every boot
    shell: |
      printf '# PCH hardware watchdog — see playbooks/esh-pve-hardware-watchdog.yaml\niTCO_wdt\n' \
        > /etc/modules-load.d/itco-watchdog.conf
    when: "! grep -qx 'iTCO_wdt' /etc/modules-load.d/itco-watchdog.conf 2>/dev/null"

  - name: Stop softdog being auto-loaded so iTCO_wdt claims watchdog0
    shell: |
      printf '# softdog cannot rescue a hard freeze; iTCO_wdt can.\n# See playbooks/esh-pve-hardware-watchdog.yaml\nblacklist softdog\n' \
        > /etc/modprobe.d/blacklist-softdog.conf
    when: "! grep -qx 'blacklist softdog' /etc/modprobe.d/blacklist-softdog.conf 2>/dev/null"

  - name: Mask watchdog-mux (idle without HA, and it holds the device)
    shell: systemctl disable --now watchdog-mux.service && systemctl mask watchdog-mux.service
    when: "[ \"$(systemctl is-enabled watchdog-mux.service 2>/dev/null)\" != masked ]"

  - name: Hand the watchdog to systemd
    shell: |
      mkdir -p /etc/systemd/system.conf.d
      cat > /etc/systemd/system.conf.d/watchdog.conf <<'EOF'
      # Hardware watchdog (iTCO_wdt). See playbooks/esh-pve-hardware-watchdog.yaml
      # for why systemd owns this rather than Proxmox's watchdog-mux.
      [Manager]
      RuntimeWatchdogSec={{ runtime_watchdog_sec }}
      RebootWatchdogSec=10min
      EOF
    when: "! grep -q 'RuntimeWatchdogSec={{ runtime_watchdog_sec }}' /etc/systemd/system.conf.d/watchdog.conf 2>/dev/null"

  # Renumber live so the change takes effect without waiting for a reboot:
  # softdog currently holds watchdog0, so systemd would otherwise arm the
  # software watchdog — the exact device that failed us.
  - name: Make iTCO_wdt the active watchdog0 now
    shell: |
      rmmod softdog 2>/dev/null || true
      rmmod iTCO_wdt 2>/dev/null || true
      modprobe iTCO_wdt
    # Gated, not unconditional: once systemd holds /dev/watchdog0 the rmmod
    # would fail anyway, and re-running this on an already-correct host would
    # otherwise churn the device for no reason. Only renumber when watchdog0
    # is NOT already iTCO_wdt.
    when: "! grep -qx 'iTCO_wdt' /sys/class/watchdog/watchdog0/identity 2>/dev/null"

  - name: Re-exec systemd so RuntimeWatchdogSec takes effect
    # daemon-reload does NOT apply [Manager] settings; a re-exec is required.
    # Skipped once systemd already reports it owns the hardware watchdog.
    shell: systemctl daemon-reexec
    when: "! journalctl -b --no-pager | grep -q 'Using hardware watchdog .iTCO_wdt.'"

verify:
  - name: iTCO_wdt is the kernel's watchdog0
    shell: grep -qx 'iTCO_wdt' /sys/class/watchdog/watchdog0/identity
    changed_when: "false"

  - name: The watchdog is ARMED, not merely present
    shell: grep -qx 'active' /sys/class/watchdog/watchdog0/state
    changed_when: "false"

  - name: systemd reports it owns a hardware watchdog
    shell: journalctl -b --no-pager | grep -q 'Using hardware watchdog .iTCO_wdt.'
    changed_when: "false"

  - name: softdog is not loaded
    shell: "! lsmod | grep -qE '^softdog'"
    changed_when: "false"

  - name: watchdog-mux is masked
    shell: "[ \"$(systemctl is-enabled watchdog-mux.service 2>/dev/null)\" = masked ]"
    changed_when: "false"

  - name: Config survives a reboot
    shell: grep -qx 'iTCO_wdt' /etc/modules-load.d/itco-watchdog.conf && grep -q RuntimeWatchdogSec /etc/systemd/system.conf.d/watchdog.conf
    changed_when: "false"