feat(cira-tunnel-watchdog): auto-drop stale AMT CIRA tunnels on MeshCentral's MPS

2026-10-03 12:34:35 -07:00
parent fe1d472aae
commit 00a6fbd849
9 changed files with 161 additions and 0 deletions
@@ -0,0 +1,56 @@
+#!/bin/bash
+# cira-tunnel-watchdog: drop half-open Intel AMT phone-home (CIRA) tunnels on MeshCentral's MPS.
+#
+# Why (servers/pfi-tacticalrmm/README.md): when a host whose AMT shares the network chip power-cycles, AMT opens
+# a new CIRA tunnel without closing the old one. MeshCentral 1.2.0 keeps both. Its web relay (KVM "HW Connect",
+# SOL, IDE-R, MeshCommander) can pick the dead one and hang at "Setup..." forever. Its own 90 s idle timeout
+# does not fire, because its writes to the dead socket keep resetting it. Seen 3 of 3 power events, at two sites
+# (2026-10-02/03).
+#
+# What: every run, any ESTABLISHED socket on the MPS port that has received nothing for MAX_SILENT_MS is
+# destroyed with `ss -K`. MeshCentral then drops it, and the live tunnel serves. Healthy tunnels hear from their
+# AMT every ~6-30 s (measured), so the 180 s default leaves a wide margin. A tunnel dropped by mistake comes
+# back on its own: the AMT policy reconnects every 10 s.
+#
+#   cira-tunnel-watchdog                 act (root: ss -K needs CAP_NET_ADMIN)
+#   cira-tunnel-watchdog --dry-run       report what would be dropped
+#   cira-tunnel-watchdog --dry-run --from-file F   decide from a saved `ss -tnio` capture (tests)
+set -euo pipefail
+PORT="${CIRA_PORT:-4433}"
+MAX_SILENT_MS="${CIRA_MAX_SILENT_MS:-180000}"
+DRY=0
+SRC=""
+while [ $# -gt 0 ]; do
+  case "$1" in
+    --dry-run) DRY=1 ;;
+    --from-file) SRC="${2:?--from-file needs a path}"; shift ;;
+    *) echo "usage: $0 [--dry-run] [--from-file <ss capture>]" >&2; exit 2 ;;
+  esac
+  shift
+done
+if [ -n "$SRC" ]; then
+  [ "$DRY" = 1 ] || { echo "--from-file is a test mode and needs --dry-run" >&2; exit 2; }
+  out="$(cat "$SRC")"
+else
+  out="$(ss -tnio state established "( sport = :$PORT )")"
+fi
+# `ss -tnio state established` prints a socket line (Recv-Q Send-Q Local Peer) followed by an indented info line.
+stale="$(printf '%s\n' "$out" | awk -v max="$MAX_SILENT_MS" '
+  /^[0-9]/ { peer = $4; next }
+  /lastrcv:/ {
+    lr = -1
+    for (i = 1; i <= NF; i++) if ($i ~ /^lastrcv:/) { split($i, a, ":"); lr = a[2] + 0 }
+    if (peer != "" && lr > max) print peer, lr
+    peer = ""
+  }')"
+[ -z "$stale" ] && exit 0
+while read -r peer lr; do
+  [[ "$peer" =~ ^\[[0-9a-fA-F:.]+\]:[0-9]+$ ]] || { echo "skip unparsable peer [$peer]" >&2; continue; }
+  if [ "$DRY" = 1 ]; then
+    echo "would drop $peer (silent ${lr} ms > ${MAX_SILENT_MS})"
+  else
+    ss -K -tn "( sport = :$PORT ) and dst $peer" >/dev/null
+    logger -t cira-tunnel-watchdog "dropped stale CIRA tunnel $peer (silent ${lr} ms > ${MAX_SILENT_MS})"
+    echo "dropped $peer (silent ${lr} ms)"
+  fi
+done <<<"$stale"