#!/bin/sh
# fv-mesh-watchdog — restore fv-ml1's mesh connectivity if a change strands it.
#
# WHY: every path into the Fountain Valley site runs through equipment at FV.
# When fv-ml1 loses its way back to the fleet there is no console, no local
# hands, and the BMC sits behind the same gateway. On 2026-09-14 a
# `tailscale up --accept-routes` black-holed this box from its own LAN: it
# accepted 10.251.0.0/16 from the gateway — its OWN subnet — and started routing
# the local network through the tunnel. Recovery needed its mesh address, which
# only happened to still work.
#
# WHAT IT DOES: probes two INDEPENDENT anchors once a minute. After
# FAIL_THRESHOLD consecutive failures it puts Tailscale back to known-good:
# accept-routes off, re-up against headscale. It touches nothing else — not
# routes, not firewall, not services — because a watchdog with a wide remit is
# a second way to lose the box.
#
# ⚠ Two anchors on purpose, and they must not share a failure mode:
#   WAN_ANCHOR   plain internet, no mesh involved
#   MESH_ANCHOR  a fleet address only reachable if the mesh works
# If only the mesh anchor fails, the mesh is the problem and we act. If BOTH
# fail the site's uplink is down and Tailscale cannot fix that, so we do NOT
# act — thrashing tailscaled during an ISP outage turns a wait into an incident.
#
# Disable for planned work:  touch /etc/fv-watchdog.disable

set -u
STATE=/var/lib/fv-mesh-watchdog
CONF=/etc/fv-mesh-watchdog.conf
DISABLE=/etc/fv-watchdog.disable
FAIL_THRESHOLD=5
WAN_ANCHOR=1.1.1.1
MESH_ANCHOR=100.64.0.1
LOGIN_SERVER=https://headscale.phasefinal.com
[ -r "$CONF" ] && . "$CONF"

log() { logger -t fv-mesh-watchdog "$*"; printf '%s fv-mesh-watchdog: %s\n' "$(date -Is)" "$*"; }

if [ -e "$DISABLE" ]; then log "disabled by $DISABLE — no action"; exit 0; fi
mkdir -p "$STATE"; COUNT_FILE="$STATE/consecutive_failures"
[ -f "$COUNT_FILE" ] || echo 0 > "$COUNT_FILE"
count=$(cat "$COUNT_FILE" 2>/dev/null || echo 0)

ping -c 2 -W 3 "$WAN_ANCHOR"  >/dev/null 2>&1 && wan=up   || wan=down
ping -c 2 -W 3 "$MESH_ANCHOR" >/dev/null 2>&1 && mesh=up  || mesh=down

if [ "$wan" = down ]; then
  # Uplink is down. Tailscale cannot fix that and restarting it during an ISP
  # outage only adds churn. Reset the counter so the mesh timer starts clean
  # once the site is back.
  [ "$count" -ne 0 ] && log "uplink down (wan=$wan mesh=$mesh) — not a mesh fault, counter reset"
  echo 0 > "$COUNT_FILE"; exit 0
fi

if [ "$mesh" = up ]; then
  [ "$count" -ne 0 ] && log "mesh recovered after $count failure(s)"
  echo 0 > "$COUNT_FILE"; exit 0
fi

count=$((count + 1)); echo "$count" > "$COUNT_FILE"
log "mesh anchor $MESH_ANCHOR unreachable while WAN is up ($count/$FAIL_THRESHOLD)"
[ "$count" -lt "$FAIL_THRESHOLD" ] && exit 0

log "THRESHOLD REACHED — restoring known-good Tailscale state"
echo 0 > "$COUNT_FILE"
tailscale set --accept-routes=false 2>&1 | while read -r l; do log "  set: $l"; done
if [ -r "$STATE/authkey" ]; then
  tailscale up --login-server "$LOGIN_SERVER" --authkey "$(cat "$STATE/authkey")" \
      --accept-routes=false --hostname "$(hostname -s)" 2>&1 | while read -r l; do log "  up: $l"; done
else
  log "  no $STATE/authkey — cannot re-register; tried accept-routes=false only"
fi
systemctl restart tailscaled 2>&1 | while read -r l; do log "  restart: $l"; done
log "restore attempt complete"