diff --git a/scripts/discover-fortigate.sh b/scripts/discover-fortigate.sh index 908103a..05b4f74 100755 --- a/scripts/discover-fortigate.sh +++ b/scripts/discover-fortigate.sh @@ -100,7 +100,7 @@ awk -v src="fortigate:${HOST}" ' # Lease row $1 ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/ && $2 ~ /^[0-9a-fA-F]{2}:/ { ip = $1 - mac = $2 + mac = tolower($2) # normalize to lowercase — matches UniFi output host = (NF >= 3 && $3 !~ /^[0-9]+$/) ? $3 : "-" printf "%s\t%s\t%s\t%s\t%s\n", ip, mac, host, (iface==""?"-":iface), src } diff --git a/scripts/discover-gaps.sh b/scripts/discover-gaps.sh index 47a328e..ef9fc68 100755 --- a/scripts/discover-gaps.sh +++ b/scripts/discover-gaps.sh @@ -3,14 +3,25 @@ # servers/*/ssh-target (the authoritative "managed hosts" list). # # Input: one or more TSV files, each a line of: -# IP MAC HOSTNAME (EXTRA1) (EXTRA2) … +# IP MAC HOSTNAME (EXTRA1) (EXTRA2) … SOURCE # Produced by discover-fortigate.sh, discover-unifi.sh, or equivalent. # # Usage: -# scripts/discover-gaps.sh leases-ana.tsv leases-nh3.tsv +# scripts/discover-gaps.sh [--ignore-unifi] leases.tsv [more.tsv ...] # -# Output: rows where IP does NOT match any IP in servers/*/ssh-target, -# sorted for readability. Same TSV format as input. +# Flags: +# --ignore-unifi Drop rows whose final column starts with "unifi:". +# Useful when you want to focus on servers / IoT / +# endpoints rather than UniFi infrastructure +# (APs, switches, UDMs). +# +# An IP is considered "known" if it matches any of: +# - an IPv4 in servers//ssh-target +# - a resolvable hostname in ssh-target +# - any IPv4 token found in servers//README.md +# - any IPv4 token in servers//proxmox-details.txt +# (so Proxmox-tracked VM IPs surfaced by proxmox_inspect.sh are +# automatically absorbed) # # Exit code: # 0 if zero gaps found @@ -22,14 +33,33 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)" SERVERS_DIR="$REPO_ROOT/servers" -if [ "$#" -lt 1 ]; then - echo "usage: $(basename "$0") [more.tsv ...]" >&2 +IGNORE_UNIFI=0 +FILES=() +for arg in "$@"; do + case "$arg" in + --ignore-unifi) IGNORE_UNIFI=1 ;; + -h|--help) + sed -n '2,30p' "$0" + exit 0 + ;; + -*) + echo "error: unknown flag $arg" >&2 + exit 2 + ;; + *) + FILES+=("$arg") + ;; + esac +done + +if [ "${#FILES[@]}" -lt 1 ]; then + echo "usage: $(basename "$0") [--ignore-unifi] [more.tsv ...]" >&2 exit 2 fi -# Build the set of managed IPs from servers/*/ssh-target. -# ssh-target may contain "user@IP" or a bare IP/hostname. We extract any -# IPv4 and (if the value is a hostname) try to resolve it. +# ---- Build the set of managed IPs --------------------------------------- + +# 1. servers/*/ssh-target managed_ips=$( for stf in "$SERVERS_DIR"/*/ssh-target; do [ -r "$stf" ] || continue @@ -38,7 +68,6 @@ managed_ips=$( if [[ "$val" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then echo "$val" else - # getent hosts returns "IP name [alias…]"; take first field if resolved ip=$(getent hosts "$val" 2>/dev/null | awk 'NR==1{print $1}') [ -n "$ip" ] && echo "$ip" fi @@ -49,23 +78,44 @@ if [ -z "$managed_ips" ]; then echo "warning: no managed IPs found under $SERVERS_DIR/*/ssh-target" >&2 fi -# Also tolerate IPs that appear in servers/*/README.md (for hosts that -# have snapshot info but no ssh-target — e.g. Proxmox hosts documented -# there). Extract any IPv4-looking token from READMEs. +# 2. Any IPv4 in README.md files under servers/ readme_ips=$( - grep -rhEo '([0-9]+\.){3}[0-9]+' "$SERVERS_DIR"/*/README.md 2>/dev/null | sort -u || true + grep -rhEo '([0-9]+\.){3}[0-9]+' "$SERVERS_DIR"/*/README.md 2>/dev/null \ + | sort -u || true ) -all_known=$(printf '%s\n%s\n' "$managed_ips" "$readme_ips" | sort -u) +# 3. Any IPv4 in proxmox-details.txt (VM IPs + cluster-resources table) +pve_ips=$( + grep -rhEo '([0-9]+\.){3}[0-9]+' "$SERVERS_DIR"/*/proxmox-details.txt 2>/dev/null \ + | sort -u || true +) -# Read the discovery TSVs, filter out lines whose IP is in all_known. +# 4. Any IPv4 in system-details.txt (ip addr show output, docker inspect, etc.) +sys_ips=$( + grep -rhEo '([0-9]+\.){3}[0-9]+' "$SERVERS_DIR"/*/system-details.txt 2>/dev/null \ + | sort -u || true +) + +# Drop meaningless addresses from the "known" set (they'd falsely mask +# real unmanaged hosts that happen to also be 127.0.0.1 in some probe). +all_known=$( + printf '%s\n%s\n%s\n%s\n' "$managed_ips" "$readme_ips" "$pve_ips" "$sys_ips" \ + | grep -Ev '^(0\.0\.0\.0|127\.|255\.255\.255\.255|169\.254\.)' \ + | sort -u +) + +# ---- Filter discovery TSVs --------------------------------------------- unmanaged=$( - cat "$@" | awk -v known="$all_known" ' + cat "${FILES[@]}" | awk -F'\t' -v known="$all_known" -v drop_unifi="$IGNORE_UNIFI" ' BEGIN { - n=split(known, arr, "\n") + n = split(known, arr, "\n") for (i=1; i<=n; i++) if (arr[i] != "") k[arr[i]] = 1 } - NF >= 1 && $1 ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/ && !($1 in k) + # Need a valid IP in column 1 and IP must not be "known". + NF >= 1 && $1 ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/ && !($1 in k) { + if (drop_unifi && $NF ~ /^unifi:/) next + print + } ' | sort -t$'\t' -k1,1 -V -u ) diff --git a/scripts/discover-unifi.sh b/scripts/discover-unifi.sh index 3ae616b..9a7f1dc 100755 --- a/scripts/discover-unifi.sh +++ b/scripts/discover-unifi.sh @@ -110,12 +110,31 @@ ui_get() { # Endpoint-specific TSV formatters # ---------------------------------------------------------------------- +# Shared jq prelude: normalize MAC to xx:xx:xx:xx:xx:xx lowercase so +# output aligns with the FortiGate format. +JQ_PRELUDE=' + def norm_mac: + . as $m | + if $m == null or $m == "-" then "-" + else + ($m | ascii_downcase | gsub("[^0-9a-f]"; "")) as $h | + if ($h | length) == 12 then + ($h[0:2] + ":" + $h[2:4] + ":" + $h[4:6] + ":" + $h[6:8] + ":" + $h[8:10] + ":" + $h[10:12]) + else $m + end + end; +' + +# LAN filter regex. Default matches the fleet's 10.x.x.x space; override +# with UNIFI_LAN_FILTER if you run a different private range. +LAN_FILTER="${UNIFI_LAN_FILTER:-^10\\.}" + emit_hosts() { # LAN IP is the first RFC1918 entry in reportedState.ipAddrs that is # NOT also present as a WAN ipv4 (reportedState.wans[]) — UDMs with # RFC1918-addressed WAN2 interfaces would otherwise get mis-picked as # their LAN IP. Falls back through the usual chain if nothing matches. - ui_get /ea/hosts | jq -r ' + ui_get /ea/hosts | jq -r "$JQ_PRELUDE"' . as $h | (($h.reportedState.wans // []) | map(.ipv4 // empty)) as $wans | [ @@ -129,7 +148,7 @@ emit_hosts() { )] | .[0]) // .reportedState.ip // .ipAddress // "-" ), - (.reportedState.mac // "-"), + ((.reportedState.mac // "-") | norm_mac), (.reportedState.hostname // .reportedState.name // "-"), (.reportedState.hardware.shortname // .reportedState.hardware.name // .type // "-"), (.reportedState.version // "-"), @@ -152,13 +171,30 @@ emit_sites() { } emit_devices() { - # /ea/devices returns per-host wrappers; flatten into one row per AP/switch. - ui_get /ea/devices | jq -r ' + # /ea/devices returns per-host wrappers; flatten into one row per + # AP/switch. Drop entries that clutter gap analysis without adding value: + # - IPs outside the fleet LAN range (UDM's public WAN IP listed as + # a "device", etc.) + # - UDM self-records (isConsole=true or their WAN IP matches a + # wans[].ipv4 — the UDM itself shows up in its own devices list) + # - UCI records (UniFi Cable Internet = ISP uplink tracking, model="UCI") + ui_get /ea/devices | jq -r \ + --arg lan "$LAN_FILTER" \ + "$JQ_PRELUDE"' . as $h | + (($h.wans // []) | map(.ipv4 // empty)) as $wans | (.devices // [])[] | + select( + (.ip // "") | test($lan) + ) | + select( + (.isConsole // false) != true and + (.model // "") != "UCI" and + (.ip as $ip | ($wans | index($ip)) == null) + ) | [ (.ip // "-"), - (.mac // .id // "-"), + ((.mac // .id // "-") | norm_mac), (.name // "-"), (.model // .shortname // "-"), ($h.hostName // $h.hostId // "-"),