#!/bin/sh
# fv-mesh-watchdog — BREAK-GLASS: join the mesh when fv-ml1 loses the fleet.
#
# ⭐ INVERTED DESIGN (operator, 2026-09-15). fv-ml1 is normally OFF the mesh: its
# cross-site traffic is carried by outbound SNAT on the FV OPNsense gateway
# (docs/runbooks/fv-to-ana-nat.md), so mesh membership is redundant for routing
# and its only value is as a SECOND way in when the first one breaks. Keeping it
# permanently enrolled bought a standing second door; joining ON DEMAND buys the
# same recovery path without one.
#
# This is therefore not a "restore Tailscale" watchdog. It is a door that opens
# when the normal path is gone.
#
#   normal    tailscaled stopped+disabled; fleet reached via the gateway's SNAT
#   fault     FLEET_ANCHORS unreachable while the WAN is up
#   action    start tailscaled and `tailscale up` -> the box is reachable at its
#             mesh address even though its LAN path is broken
#
# ⚠ TWO ANCHOR CLASSES, and they must not share a failure mode:
#   WAN_ANCHOR    plain internet. If THIS is down the site uplink is gone,
#                 Tailscale cannot fix it, and we do nothing -- thrashing
#                 during an ISP outage turns a wait into an incident.
#   FLEET_ANCHORS hosts reachable ONLY through the gateway's SNAT path. Their
#                 loss is exactly the fault this exists for.
#
# ⚠⚠ THE REJOIN MUST USE --accept-routes=false. On 2026-09-14 a
# `tailscale up --accept-routes` on this box accepted 10.251.0.0/16 from the
# gateway -- its OWN subnet -- and routed the local network through the tunnel,
# black-holing it. That happened with a human watching. Here it would run
# unattended, during an incident, on a box already in trouble.
#
# ⚠ NO AUTO-LEAVE. Once the door is open it stays open until a human closes it
# (`systemctl disable --now tailscaled`). A watchdog that re-closes on recovery
# flaps, and a flapping recovery path is worse than none -- it would be down
# exactly when someone finally looked.
#
# Disable for planned work:  touch /etc/fv-watchdog.disable

set -u
STATE=/var/lib/fv-mesh-watchdog
CONF=/etc/fv-mesh-watchdog.conf
DISABLE=/etc/fv-watchdog.disable
FAIL_THRESHOLD=5
WAN_ANCHOR=1.1.1.1
FLEET_ANCHORS="10.100.10.50 10.100.50.40"    # nh3-dev, nh3-docker
LOGIN_SERVER=https://headscale.phasefinal.com
[ -r "$CONF" ] && . "$CONF"

log() { logger -t fv-mesh-watchdog "$*"; printf '%s fv-mesh-watchdog: %s\n' "$(date -Is)" "$*"; }

if [ -e "$DISABLE" ]; then log "disabled by $DISABLE — no action"; exit 0; fi
mkdir -p "$STATE"; COUNT_FILE="$STATE/consecutive_failures"
[ -f "$COUNT_FILE" ] || echo 0 > "$COUNT_FILE"
count=$(cat "$COUNT_FILE" 2>/dev/null || echo 0)

# Already on the mesh? Then the door is open and there is nothing to do. This is
# also what makes the script idempotent after it has fired.
if [ "$(systemctl is-active tailscaled 2>/dev/null)" = active ] \
   && tailscale status >/dev/null 2>&1; then
  [ "$count" -ne 0 ] && echo 0 > "$COUNT_FILE"
  exit 0
fi

ping -c 2 -W 3 "$WAN_ANCHOR" >/dev/null 2>&1 && wan=up || wan=down
if [ "$wan" = down ]; then
  [ "$count" -ne 0 ] && log "uplink down — not a fleet-path fault, counter reset"
  echo 0 > "$COUNT_FILE"; exit 0
fi

# Any fleet anchor answering means the normal path works.
fleet=down
for a in $FLEET_ANCHORS; do
  if ping -c 2 -W 3 "$a" >/dev/null 2>&1; then fleet=up; break; fi
done

if [ "$fleet" = up ]; then
  [ "$count" -ne 0 ] && log "fleet path recovered after $count failure(s)"
  echo 0 > "$COUNT_FILE"; exit 0
fi

count=$((count + 1)); echo "$count" > "$COUNT_FILE"
log "fleet unreachable via the gateway path while WAN is up ($count/$FAIL_THRESHOLD) — anchors: $FLEET_ANCHORS"
[ "$count" -lt "$FAIL_THRESHOLD" ] && exit 0

log "THRESHOLD REACHED — opening the break-glass mesh path"
echo 0 > "$COUNT_FILE"
systemctl start tailscaled 2>&1 | while read -r l; do log "  start: $l"; done
if [ -r "$STATE/authkey" ]; then
  tailscale up --login-server "$LOGIN_SERVER" --authkey "$(cat "$STATE/authkey")" \
      --accept-routes=false --hostname "$(hostname -s)" 2>&1 | while read -r l; do log "  up: $l"; done
  log "mesh path OPEN — reach this host at its 100.64.x address; close with 'systemctl disable --now tailscaled'"
else
  log "  NO $STATE/authkey — cannot join. The break-glass path is UNARMED."
fi