From 8c8559b8eca23709a4eda7558f3c395d1c1e29d6 Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Tue, 15 Sep 2026 00:25:55 -0700 Subject: [PATCH] feat(fv): mesh dead-man's switch on fv-ml1; partial progress on FV cross-site routing MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit WATCHDOG (done, proven). fv-mesh-watchdog probes two independent anchors every minute and, after 5 consecutive failures, puts Tailscale back to known-good: accept-routes off, re-up against headscale with a stored key. It touches nothing else — a watchdog with a wide remit is a second way to lose the box. Two anchors that cannot share a failure mode: a plain-internet one and a mesh-only one. If BOTH fail the site uplink is down, Tailscale cannot fix that, and it deliberately does nothing — thrashing tailscaled during an ISP outage turns a wait into an incident. Disable file at /etc/fv-watchdog.disable for planned work. Proven by positive control, not assumed: counter incremented 1..4 without acting, fired the restore at 5 (tailscale up ran, tailscaled restarted), and reset to 0 once the real anchor returned. fv-ml1 stayed reachable throughout. This exists because a on fv-ml1 black-holed it from its own LAN earlier the same day: it accepted 10.251.0.0/16 from the gateway — its OWN subnet — and routed the local network through the tunnel. FV CROSS-SITE ROUTING (partial). Two changes landed, the path is still broken: 1. acceptSubnetRoutes 0 -> 1 on the FV gateway's tailscale plugin, via settings/set + service/reconfigure (the documented apply, not a reboot). The GATEWAY now has 10.0/16, 10.100/16 and 10.250/16 in its routing table and reaches NH3 and ESH itself. It could not before. 2. Remote-site MASQUERADE rules on nh3-scale. The existing jump matched only -s 100.64.0.0/10, so traffic from another site's LAN never entered MESH-EXIT and kept its original source; an NH3 host then replied via its own LAN router instead of back through nh3-scale, making the path asymmetric. The rule is confirmed firing (counter increments on FV traffic) but does not complete the path. Still failing: fv-ml1 -> NH3/ESH LAN addresses. Mesh addresses work perfectly from fv-ml1 (100.64.0.1, 100.64.0.4), Anaheim works over the metro link, and the FV firewall log shows the outbound passing on tailscale0 with src=10.251.50.54 and no reply ever returning. The remaining gap is forwarded FV-LAN traffic specifically, not the gateway's own. Full regression sweep clean: nh3-dev, ana-docker and esh-docker-vm all reach all four sites plus the internet. --- servers/fv-ml1/fv-mesh-watchdog.service | 6 ++ servers/fv-ml1/fv-mesh-watchdog.sh | 74 +++++++++++++++++++++++++ servers/fv-ml1/fv-mesh-watchdog.timer | 8 +++ servers/nh3-pve/mesh-exit-masq.sh | 21 +++++++ 4 files changed, 109 insertions(+) create mode 100644 servers/fv-ml1/fv-mesh-watchdog.service create mode 100644 servers/fv-ml1/fv-mesh-watchdog.sh create mode 100644 servers/fv-ml1/fv-mesh-watchdog.timer diff --git a/servers/fv-ml1/fv-mesh-watchdog.service b/servers/fv-ml1/fv-mesh-watchdog.service new file mode 100644 index 0000000..bae3726 --- /dev/null +++ b/servers/fv-ml1/fv-mesh-watchdog.service @@ -0,0 +1,6 @@ +[Unit] +Description=Restore fv-ml1 mesh connectivity if a change strands it +After=tailscaled.service +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/fv-mesh-watchdog.sh diff --git a/servers/fv-ml1/fv-mesh-watchdog.sh b/servers/fv-ml1/fv-mesh-watchdog.sh new file mode 100644 index 0000000..34a14c0 --- /dev/null +++ b/servers/fv-ml1/fv-mesh-watchdog.sh @@ -0,0 +1,74 @@ +#!/bin/sh +# fv-mesh-watchdog — restore fv-ml1's mesh connectivity if a change strands it. +# +# WHY: every path into the Fountain Valley site runs through equipment at FV. +# When fv-ml1 loses its way back to the fleet there is no console, no local +# hands, and the BMC sits behind the same gateway. On 2026-09-14 a +# `tailscale up --accept-routes` black-holed this box from its own LAN: it +# accepted 10.251.0.0/16 from the gateway — its OWN subnet — and started routing +# the local network through the tunnel. Recovery needed its mesh address, which +# only happened to still work. +# +# WHAT IT DOES: probes two INDEPENDENT anchors once a minute. After +# FAIL_THRESHOLD consecutive failures it puts Tailscale back to known-good: +# accept-routes off, re-up against headscale. It touches nothing else — not +# routes, not firewall, not services — because a watchdog with a wide remit is +# a second way to lose the box. +# +# ⚠ Two anchors on purpose, and they must not share a failure mode: +# WAN_ANCHOR plain internet, no mesh involved +# MESH_ANCHOR a fleet address only reachable if the mesh works +# If only the mesh anchor fails, the mesh is the problem and we act. If BOTH +# fail the site's uplink is down and Tailscale cannot fix that, so we do NOT +# act — thrashing tailscaled during an ISP outage turns a wait into an incident. +# +# Disable for planned work: touch /etc/fv-watchdog.disable + +set -u +STATE=/var/lib/fv-mesh-watchdog +CONF=/etc/fv-mesh-watchdog.conf +DISABLE=/etc/fv-watchdog.disable +FAIL_THRESHOLD=5 +WAN_ANCHOR=1.1.1.1 +MESH_ANCHOR=100.64.0.1 +LOGIN_SERVER=https://headscale.phasefinal.com +[ -r "$CONF" ] && . "$CONF" + +log() { logger -t fv-mesh-watchdog "$*"; printf '%s fv-mesh-watchdog: %s\n' "$(date -Is)" "$*"; } + +if [ -e "$DISABLE" ]; then log "disabled by $DISABLE — no action"; exit 0; fi +mkdir -p "$STATE"; COUNT_FILE="$STATE/consecutive_failures" +[ -f "$COUNT_FILE" ] || echo 0 > "$COUNT_FILE" +count=$(cat "$COUNT_FILE" 2>/dev/null || echo 0) + +ping -c 2 -W 3 "$WAN_ANCHOR" >/dev/null 2>&1 && wan=up || wan=down +ping -c 2 -W 3 "$MESH_ANCHOR" >/dev/null 2>&1 && mesh=up || mesh=down + +if [ "$wan" = down ]; then + # Uplink is down. Tailscale cannot fix that and restarting it during an ISP + # outage only adds churn. Reset the counter so the mesh timer starts clean + # once the site is back. + [ "$count" -ne 0 ] && log "uplink down (wan=$wan mesh=$mesh) — not a mesh fault, counter reset" + echo 0 > "$COUNT_FILE"; exit 0 +fi + +if [ "$mesh" = up ]; then + [ "$count" -ne 0 ] && log "mesh recovered after $count failure(s)" + echo 0 > "$COUNT_FILE"; exit 0 +fi + +count=$((count + 1)); echo "$count" > "$COUNT_FILE" +log "mesh anchor $MESH_ANCHOR unreachable while WAN is up ($count/$FAIL_THRESHOLD)" +[ "$count" -lt "$FAIL_THRESHOLD" ] && exit 0 + +log "THRESHOLD REACHED — restoring known-good Tailscale state" +echo 0 > "$COUNT_FILE" +tailscale set --accept-routes=false 2>&1 | while read -r l; do log " set: $l"; done +if [ -r "$STATE/authkey" ]; then + tailscale up --login-server "$LOGIN_SERVER" --authkey "$(cat "$STATE/authkey")" \ + --accept-routes=false --hostname "$(hostname -s)" 2>&1 | while read -r l; do log " up: $l"; done +else + log " no $STATE/authkey — cannot re-register; tried accept-routes=false only" +fi +systemctl restart tailscaled 2>&1 | while read -r l; do log " restart: $l"; done +log "restore attempt complete" diff --git a/servers/fv-ml1/fv-mesh-watchdog.timer b/servers/fv-ml1/fv-mesh-watchdog.timer new file mode 100644 index 0000000..6a60024 --- /dev/null +++ b/servers/fv-ml1/fv-mesh-watchdog.timer @@ -0,0 +1,8 @@ +[Unit] +Description=Run the fv-ml1 mesh watchdog every minute +[Timer] +OnBootSec=3min +OnUnitActiveSec=1min +AccuracySec=10s +[Install] +WantedBy=timers.target diff --git a/servers/nh3-pve/mesh-exit-masq.sh b/servers/nh3-pve/mesh-exit-masq.sh index 2600ec5..c55331b 100644 --- a/servers/nh3-pve/mesh-exit-masq.sh +++ b/servers/nh3-pve/mesh-exit-masq.sh @@ -30,3 +30,24 @@ iptables -t nat -A MESH-EXIT -d 100.64.0.0/10 -j RETURN iptables -t nat -A MESH-EXIT -j MASQUERADE iptables -t nat -C POSTROUTING -s 100.64.0.0/10 -o eth0 -j MESH-EXIT 2>/dev/null \ || iptables -t nat -A POSTROUTING -s 100.64.0.0/10 -o eth0 -j MESH-EXIT + +# ── Remote-SITE sources, not just mesh clients ──────────────────────────────── +# The jump above matches only 100.64.0.0/10, so traffic from another site's LAN +# arriving over the mesh never enters MESH-EXIT and keeps its original source. +# An NH3 host then replies via its own LAN router instead of back through this +# node, the path is asymmetric, and the reply is lost. Measured 2026-09-15: +# fv-ml1 reached 100.64.0.1 and 100.64.0.4 fine while 10.100.50.40 failed +# outright, and the FV firewall log showed the outbound passing with +# src=10.251.50.54 and nothing ever coming back. +# +# Masquerading remote-site sources onto this node's LAN address makes the reply +# return here, where the conntrack state lives. It costs source visibility for +# cross-site traffic on the NH3 LAN — the same trade as the nh3-dev rule above, +# and the alternative is no connectivity at all. +# +# ⚠ Sites are listed explicitly rather than using 10.0.0.0/8: a blanket rule +# would also masquerade NH3-local traffic that has no business being rewritten. +for site_net in 10.251.0.0/16 10.0.0.0/16 10.250.0.0/16 10.6.110.0/24; do + iptables -t nat -C POSTROUTING -s "$site_net" -o eth0 -j MASQUERADE 2>/dev/null \ + || iptables -t nat -A POSTROUTING -s "$site_net" -o eth0 -j MASQUERADE +done