diff --git a/servers/fv-ml1/fv-mesh-watchdog.service b/servers/fv-ml1/fv-mesh-watchdog.service new file mode 100644 index 0000000..bae3726 --- /dev/null +++ b/servers/fv-ml1/fv-mesh-watchdog.service @@ -0,0 +1,6 @@ +[Unit] +Description=Restore fv-ml1 mesh connectivity if a change strands it +After=tailscaled.service +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/fv-mesh-watchdog.sh diff --git a/servers/fv-ml1/fv-mesh-watchdog.sh b/servers/fv-ml1/fv-mesh-watchdog.sh new file mode 100644 index 0000000..34a14c0 --- /dev/null +++ b/servers/fv-ml1/fv-mesh-watchdog.sh @@ -0,0 +1,74 @@ +#!/bin/sh +# fv-mesh-watchdog — restore fv-ml1's mesh connectivity if a change strands it. +# +# WHY: every path into the Fountain Valley site runs through equipment at FV. +# When fv-ml1 loses its way back to the fleet there is no console, no local +# hands, and the BMC sits behind the same gateway. On 2026-09-14 a +# `tailscale up --accept-routes` black-holed this box from its own LAN: it +# accepted 10.251.0.0/16 from the gateway — its OWN subnet — and started routing +# the local network through the tunnel. Recovery needed its mesh address, which +# only happened to still work. +# +# WHAT IT DOES: probes two INDEPENDENT anchors once a minute. After +# FAIL_THRESHOLD consecutive failures it puts Tailscale back to known-good: +# accept-routes off, re-up against headscale. It touches nothing else — not +# routes, not firewall, not services — because a watchdog with a wide remit is +# a second way to lose the box. +# +# ⚠ Two anchors on purpose, and they must not share a failure mode: +# WAN_ANCHOR plain internet, no mesh involved +# MESH_ANCHOR a fleet address only reachable if the mesh works +# If only the mesh anchor fails, the mesh is the problem and we act. If BOTH +# fail the site's uplink is down and Tailscale cannot fix that, so we do NOT +# act — thrashing tailscaled during an ISP outage turns a wait into an incident. +# +# Disable for planned work: touch /etc/fv-watchdog.disable + +set -u +STATE=/var/lib/fv-mesh-watchdog +CONF=/etc/fv-mesh-watchdog.conf +DISABLE=/etc/fv-watchdog.disable +FAIL_THRESHOLD=5 +WAN_ANCHOR=1.1.1.1 +MESH_ANCHOR=100.64.0.1 +LOGIN_SERVER=https://headscale.phasefinal.com +[ -r "$CONF" ] && . "$CONF" + +log() { logger -t fv-mesh-watchdog "$*"; printf '%s fv-mesh-watchdog: %s\n' "$(date -Is)" "$*"; } + +if [ -e "$DISABLE" ]; then log "disabled by $DISABLE — no action"; exit 0; fi +mkdir -p "$STATE"; COUNT_FILE="$STATE/consecutive_failures" +[ -f "$COUNT_FILE" ] || echo 0 > "$COUNT_FILE" +count=$(cat "$COUNT_FILE" 2>/dev/null || echo 0) + +ping -c 2 -W 3 "$WAN_ANCHOR" >/dev/null 2>&1 && wan=up || wan=down +ping -c 2 -W 3 "$MESH_ANCHOR" >/dev/null 2>&1 && mesh=up || mesh=down + +if [ "$wan" = down ]; then + # Uplink is down. Tailscale cannot fix that and restarting it during an ISP + # outage only adds churn. Reset the counter so the mesh timer starts clean + # once the site is back. + [ "$count" -ne 0 ] && log "uplink down (wan=$wan mesh=$mesh) — not a mesh fault, counter reset" + echo 0 > "$COUNT_FILE"; exit 0 +fi + +if [ "$mesh" = up ]; then + [ "$count" -ne 0 ] && log "mesh recovered after $count failure(s)" + echo 0 > "$COUNT_FILE"; exit 0 +fi + +count=$((count + 1)); echo "$count" > "$COUNT_FILE" +log "mesh anchor $MESH_ANCHOR unreachable while WAN is up ($count/$FAIL_THRESHOLD)" +[ "$count" -lt "$FAIL_THRESHOLD" ] && exit 0 + +log "THRESHOLD REACHED — restoring known-good Tailscale state" +echo 0 > "$COUNT_FILE" +tailscale set --accept-routes=false 2>&1 | while read -r l; do log " set: $l"; done +if [ -r "$STATE/authkey" ]; then + tailscale up --login-server "$LOGIN_SERVER" --authkey "$(cat "$STATE/authkey")" \ + --accept-routes=false --hostname "$(hostname -s)" 2>&1 | while read -r l; do log " up: $l"; done +else + log " no $STATE/authkey — cannot re-register; tried accept-routes=false only" +fi +systemctl restart tailscaled 2>&1 | while read -r l; do log " restart: $l"; done +log "restore attempt complete" diff --git a/servers/fv-ml1/fv-mesh-watchdog.timer b/servers/fv-ml1/fv-mesh-watchdog.timer new file mode 100644 index 0000000..6a60024 --- /dev/null +++ b/servers/fv-ml1/fv-mesh-watchdog.timer @@ -0,0 +1,8 @@ +[Unit] +Description=Run the fv-ml1 mesh watchdog every minute +[Timer] +OnBootSec=3min +OnUnitActiveSec=1min +AccuracySec=10s +[Install] +WantedBy=timers.target diff --git a/servers/nh3-pve/mesh-exit-masq.sh b/servers/nh3-pve/mesh-exit-masq.sh index 2600ec5..c55331b 100644 --- a/servers/nh3-pve/mesh-exit-masq.sh +++ b/servers/nh3-pve/mesh-exit-masq.sh @@ -30,3 +30,24 @@ iptables -t nat -A MESH-EXIT -d 100.64.0.0/10 -j RETURN iptables -t nat -A MESH-EXIT -j MASQUERADE iptables -t nat -C POSTROUTING -s 100.64.0.0/10 -o eth0 -j MESH-EXIT 2>/dev/null \ || iptables -t nat -A POSTROUTING -s 100.64.0.0/10 -o eth0 -j MESH-EXIT + +# ── Remote-SITE sources, not just mesh clients ──────────────────────────────── +# The jump above matches only 100.64.0.0/10, so traffic from another site's LAN +# arriving over the mesh never enters MESH-EXIT and keeps its original source. +# An NH3 host then replies via its own LAN router instead of back through this +# node, the path is asymmetric, and the reply is lost. Measured 2026-09-15: +# fv-ml1 reached 100.64.0.1 and 100.64.0.4 fine while 10.100.50.40 failed +# outright, and the FV firewall log showed the outbound passing with +# src=10.251.50.54 and nothing ever coming back. +# +# Masquerading remote-site sources onto this node's LAN address makes the reply +# return here, where the conntrack state lives. It costs source visibility for +# cross-site traffic on the NH3 LAN — the same trade as the nh3-dev rule above, +# and the alternative is no connectivity at all. +# +# ⚠ Sites are listed explicitly rather than using 10.0.0.0/8: a blanket rule +# would also masquerade NH3-local traffic that has no business being rewritten. +for site_net in 10.251.0.0/16 10.0.0.0/16 10.250.0.0/16 10.6.110.0/24; do + iptables -t nat -C POSTROUTING -s "$site_net" -o eth0 -j MASQUERADE 2>/dev/null \ + || iptables -t nat -A POSTROUTING -s "$site_net" -o eth0 -j MASQUERADE +done