#!/bin/bash
# esh-vm-docker-watchdog — bound the NFS D-state wedge on esh-vm-docker (VMID 100).
#
# WHY THIS EXISTS
#   esh-vm-docker mounts /mnt/books from the NAS at 10.0.50.50 with `hard` NFS
#   semantics (required: calibre's library holds a SQLite metadata.db, and soft
#   semantics risk corrupting it). When the NAS stalls, I/O blocks in
#   uninterruptible sleep — the classic D-state — and NOTHING clears it from
#   inside the guest. Confirmed 2026-07-15: `docker stop/rm -f`, `ctr task
#   delete`, `systemctl restart docker`, and even `docker exec` all fail. Only a
#   host reset recovers. This watchdog does not PREVENT the wedge; it bounds the
#   outage from "until someone notices" to a few minutes.
#
# THE PROBE, AND WHY IT IS SHAPED THIS WAY
#   The wedge signature is specifically "guest OS alive, services dead" — sshd
#   keeps answering because / is local disk, so an SSH or ping check reports
#   HEALTHY through a total service outage. That is why the trigger is an HTTP
#   probe of traefik (the ingress every service sits behind) and NOT a TCP/ping
#   check. The guest-agent ping is recorded only to CLASSIFY the failure in the
#   log, never to veto a reset.
#
# SAFETY RAILS
#   - Resets only when `qm status` reports the VM `running`. A deliberately
#     stopped VM is left alone.
#   - Requires FAIL_THRESHOLD consecutive failures, so a traefik redeploy or a
#     brief blip cannot trigger a power cycle.
#   - COOLDOWN_SEC between resets, so a genuinely broken VM cannot be
#     reset-looped forever.
#   - Touch the disable flag to suspend it during planned maintenance:
#         touch /etc/esh-vm-docker-watchdog.disabled
#
# Install: see README.md in this directory. Runs on esh-pve (10.0.250.35), NOT
# on the guest — a watchdog living inside the thing it watches is no watchdog.

set -uo pipefail

VMID="${VMID:-100}"
TARGET="${TARGET:-10.0.50.45}"
PROBE_URL="${PROBE_URL:-http://${TARGET}:80/}"
PROBE_TIMEOUT="${PROBE_TIMEOUT:-10}"
FAIL_THRESHOLD="${FAIL_THRESHOLD:-5}"
COOLDOWN_SEC="${COOLDOWN_SEC:-1800}"
STATE_DIR="${STATE_DIR:-/var/lib/esh-vm-docker-watchdog}"
LOG="${LOG:-/var/log/esh-vm-docker-watchdog.log}"
DISABLE_FLAG="${DISABLE_FLAG:-/etc/esh-vm-docker-watchdog.disabled}"

mkdir -p "$STATE_DIR"
FAILFILE="$STATE_DIR/consecutive_failures"
LASTRESET="$STATE_DIR/last_reset_epoch"
[ -f "$FAILFILE" ]  || echo 0 > "$FAILFILE"
[ -f "$LASTRESET" ] || echo 0 > "$LASTRESET"

log() { printf '%s %s\n' "$(date -Is)" "$*" >> "$LOG"; }

if [ -f "$DISABLE_FLAG" ]; then
    echo 0 > "$FAILFILE"
    log "SKIP disabled by $DISABLE_FLAG"
    exit 0
fi

# Only act on a VM that is SUPPOSED to be up.
status=$(qm status "$VMID" 2>/dev/null | awk '{print $2}')
if [ "$status" != "running" ]; then
    echo 0 > "$FAILFILE"
    log "SKIP vm $VMID status=${status:-unknown} (not running)"
    exit 0
fi

# Any HTTP response at all means the ingress is serving. --max-time bounds it so
# the probe itself can never hang the watchdog.
if curl -sS -o /dev/null --max-time "$PROBE_TIMEOUT" "$PROBE_URL" 2>/dev/null; then
    prev=$(cat "$FAILFILE")
    echo 0 > "$FAILFILE"
    [ "$prev" -gt 0 ] && log "OK probe recovered after $prev consecutive failure(s)"
    exit 0
fi

fails=$(( $(cat "$FAILFILE") + 1 ))
echo "$fails" > "$FAILFILE"

# Classification only — an agent that still answers while HTTP is dead is the
# textbook D-state wedge, and is the case we most want in the log.
if qm agent "$VMID" ping >/dev/null 2>&1; then
    kind="guest-agent ALIVE (services wedged — D-state signature)"
else
    kind="guest-agent DEAD (guest hung or down)"
fi
log "FAIL $fails/$FAIL_THRESHOLD probe=$PROBE_URL $kind"

[ "$fails" -ge "$FAIL_THRESHOLD" ] || exit 0

now=$(date +%s)
since=$(( now - $(cat "$LASTRESET") ))
if [ "$since" -lt "$COOLDOWN_SEC" ]; then
    log "HOLD threshold reached but only ${since}s since last reset (cooldown ${COOLDOWN_SEC}s) — NOT resetting"
    exit 0
fi

log "RESET issuing 'qm reset $VMID' after $fails consecutive failures — $kind"
if qm reset "$VMID" >/dev/null 2>&1; then
    echo "$now" > "$LASTRESET"
    echo 0 > "$FAILFILE"
    log "RESET ok"
else
    log "RESET FAILED — qm reset returned non-zero; manual intervention needed"
    exit 1
fi