scripts/discover-*: four QoL upgrades from first live gap-analysis run
First real run surfaced 31 gap rows, ~20 of which were noise. These
changes reduce the output to actionable signal.
1. discover-unifi: /ea/devices now filters out
- IPs outside the fleet LAN range (UDM's WAN IP appearing as a
"device", ISP uplink records with public IPs)
- UDM self-records (isConsole=true, or IP matches wans[].ipv4)
- UCI records (UniFi Cable Internet = ISP modem tracking)
LAN filter regex defaults to ^10\. (matches 10.0.0.0/8); override
via UNIFI_LAN_FILTER env var if you run other private ranges.
2. discover-gaps: new --ignore-unifi flag drops rows where the final
SOURCE column starts with "unifi:". Useful for "show me servery
things to manage, not the fleet's network hardware."
3. discover-gaps: known-IP set now pulls IPs from
servers/*/proxmox-details.txt AND servers/*/system-details.txt in
addition to README.md and ssh-target. Consequence: VMs tracked by
proxmox_inspect.sh are automatically counted as known without
needing a separate servers/<vmname>/ dir. Also strips meaningless
addresses (127.*, 0.0.0.0, 169.254.*) so they can't false-positive
a "known" match.
4. MAC normalization: both discover-fortigate and discover-unifi now
emit xx:xx:xx:xx:xx:xx lowercase. Previously FortiGate used colon
format, UniFi used no-separator uppercase — same MAC looked
different per source. Fortigate does tolower() in awk; UniFi uses
a shared jq `norm_mac` function.
This commit is contained in:
@@ -100,7 +100,7 @@ awk -v src="fortigate:${HOST}" '
|
||||
# Lease row
|
||||
$1 ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/ && $2 ~ /^[0-9a-fA-F]{2}:/ {
|
||||
ip = $1
|
||||
mac = $2
|
||||
mac = tolower($2) # normalize to lowercase — matches UniFi output
|
||||
host = (NF >= 3 && $3 !~ /^[0-9]+$/) ? $3 : "-"
|
||||
printf "%s\t%s\t%s\t%s\t%s\n", ip, mac, host, (iface==""?"-":iface), src
|
||||
}
|
||||
|
||||
+69
-19
@@ -3,14 +3,25 @@
|
||||
# servers/*/ssh-target (the authoritative "managed hosts" list).
|
||||
#
|
||||
# Input: one or more TSV files, each a line of:
|
||||
# IP MAC HOSTNAME (EXTRA1) (EXTRA2) …
|
||||
# IP MAC HOSTNAME (EXTRA1) (EXTRA2) … SOURCE
|
||||
# Produced by discover-fortigate.sh, discover-unifi.sh, or equivalent.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/discover-gaps.sh leases-ana.tsv leases-nh3.tsv
|
||||
# scripts/discover-gaps.sh [--ignore-unifi] leases.tsv [more.tsv ...]
|
||||
#
|
||||
# Output: rows where IP does NOT match any IP in servers/*/ssh-target,
|
||||
# sorted for readability. Same TSV format as input.
|
||||
# Flags:
|
||||
# --ignore-unifi Drop rows whose final column starts with "unifi:".
|
||||
# Useful when you want to focus on servers / IoT /
|
||||
# endpoints rather than UniFi infrastructure
|
||||
# (APs, switches, UDMs).
|
||||
#
|
||||
# An IP is considered "known" if it matches any of:
|
||||
# - an IPv4 in servers/<host>/ssh-target
|
||||
# - a resolvable hostname in ssh-target
|
||||
# - any IPv4 token found in servers/<host>/README.md
|
||||
# - any IPv4 token in servers/<host>/proxmox-details.txt
|
||||
# (so Proxmox-tracked VM IPs surfaced by proxmox_inspect.sh are
|
||||
# automatically absorbed)
|
||||
#
|
||||
# Exit code:
|
||||
# 0 if zero gaps found
|
||||
@@ -22,14 +33,33 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
SERVERS_DIR="$REPO_ROOT/servers"
|
||||
|
||||
if [ "$#" -lt 1 ]; then
|
||||
echo "usage: $(basename "$0") <leases.tsv> [more.tsv ...]" >&2
|
||||
IGNORE_UNIFI=0
|
||||
FILES=()
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--ignore-unifi) IGNORE_UNIFI=1 ;;
|
||||
-h|--help)
|
||||
sed -n '2,30p' "$0"
|
||||
exit 0
|
||||
;;
|
||||
-*)
|
||||
echo "error: unknown flag $arg" >&2
|
||||
exit 2
|
||||
;;
|
||||
*)
|
||||
FILES+=("$arg")
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ "${#FILES[@]}" -lt 1 ]; then
|
||||
echo "usage: $(basename "$0") [--ignore-unifi] <leases.tsv> [more.tsv ...]" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# Build the set of managed IPs from servers/*/ssh-target.
|
||||
# ssh-target may contain "user@IP" or a bare IP/hostname. We extract any
|
||||
# IPv4 and (if the value is a hostname) try to resolve it.
|
||||
# ---- Build the set of managed IPs ---------------------------------------
|
||||
|
||||
# 1. servers/*/ssh-target
|
||||
managed_ips=$(
|
||||
for stf in "$SERVERS_DIR"/*/ssh-target; do
|
||||
[ -r "$stf" ] || continue
|
||||
@@ -38,7 +68,6 @@ managed_ips=$(
|
||||
if [[ "$val" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
|
||||
echo "$val"
|
||||
else
|
||||
# getent hosts returns "IP name [alias…]"; take first field if resolved
|
||||
ip=$(getent hosts "$val" 2>/dev/null | awk 'NR==1{print $1}')
|
||||
[ -n "$ip" ] && echo "$ip"
|
||||
fi
|
||||
@@ -49,23 +78,44 @@ if [ -z "$managed_ips" ]; then
|
||||
echo "warning: no managed IPs found under $SERVERS_DIR/*/ssh-target" >&2
|
||||
fi
|
||||
|
||||
# Also tolerate IPs that appear in servers/*/README.md (for hosts that
|
||||
# have snapshot info but no ssh-target — e.g. Proxmox hosts documented
|
||||
# there). Extract any IPv4-looking token from READMEs.
|
||||
# 2. Any IPv4 in README.md files under servers/
|
||||
readme_ips=$(
|
||||
grep -rhEo '([0-9]+\.){3}[0-9]+' "$SERVERS_DIR"/*/README.md 2>/dev/null | sort -u || true
|
||||
grep -rhEo '([0-9]+\.){3}[0-9]+' "$SERVERS_DIR"/*/README.md 2>/dev/null \
|
||||
| sort -u || true
|
||||
)
|
||||
|
||||
all_known=$(printf '%s\n%s\n' "$managed_ips" "$readme_ips" | sort -u)
|
||||
# 3. Any IPv4 in proxmox-details.txt (VM IPs + cluster-resources table)
|
||||
pve_ips=$(
|
||||
grep -rhEo '([0-9]+\.){3}[0-9]+' "$SERVERS_DIR"/*/proxmox-details.txt 2>/dev/null \
|
||||
| sort -u || true
|
||||
)
|
||||
|
||||
# Read the discovery TSVs, filter out lines whose IP is in all_known.
|
||||
# 4. Any IPv4 in system-details.txt (ip addr show output, docker inspect, etc.)
|
||||
sys_ips=$(
|
||||
grep -rhEo '([0-9]+\.){3}[0-9]+' "$SERVERS_DIR"/*/system-details.txt 2>/dev/null \
|
||||
| sort -u || true
|
||||
)
|
||||
|
||||
# Drop meaningless addresses from the "known" set (they'd falsely mask
|
||||
# real unmanaged hosts that happen to also be 127.0.0.1 in some probe).
|
||||
all_known=$(
|
||||
printf '%s\n%s\n%s\n%s\n' "$managed_ips" "$readme_ips" "$pve_ips" "$sys_ips" \
|
||||
| grep -Ev '^(0\.0\.0\.0|127\.|255\.255\.255\.255|169\.254\.)' \
|
||||
| sort -u
|
||||
)
|
||||
|
||||
# ---- Filter discovery TSVs ---------------------------------------------
|
||||
unmanaged=$(
|
||||
cat "$@" | awk -v known="$all_known" '
|
||||
cat "${FILES[@]}" | awk -F'\t' -v known="$all_known" -v drop_unifi="$IGNORE_UNIFI" '
|
||||
BEGIN {
|
||||
n=split(known, arr, "\n")
|
||||
n = split(known, arr, "\n")
|
||||
for (i=1; i<=n; i++) if (arr[i] != "") k[arr[i]] = 1
|
||||
}
|
||||
NF >= 1 && $1 ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/ && !($1 in k)
|
||||
# Need a valid IP in column 1 and IP must not be "known".
|
||||
NF >= 1 && $1 ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/ && !($1 in k) {
|
||||
if (drop_unifi && $NF ~ /^unifi:/) next
|
||||
print
|
||||
}
|
||||
' | sort -t$'\t' -k1,1 -V -u
|
||||
)
|
||||
|
||||
|
||||
@@ -110,12 +110,31 @@ ui_get() {
|
||||
# Endpoint-specific TSV formatters
|
||||
# ----------------------------------------------------------------------
|
||||
|
||||
# Shared jq prelude: normalize MAC to xx:xx:xx:xx:xx:xx lowercase so
|
||||
# output aligns with the FortiGate format.
|
||||
JQ_PRELUDE='
|
||||
def norm_mac:
|
||||
. as $m |
|
||||
if $m == null or $m == "-" then "-"
|
||||
else
|
||||
($m | ascii_downcase | gsub("[^0-9a-f]"; "")) as $h |
|
||||
if ($h | length) == 12 then
|
||||
($h[0:2] + ":" + $h[2:4] + ":" + $h[4:6] + ":" + $h[6:8] + ":" + $h[8:10] + ":" + $h[10:12])
|
||||
else $m
|
||||
end
|
||||
end;
|
||||
'
|
||||
|
||||
# LAN filter regex. Default matches the fleet's 10.x.x.x space; override
|
||||
# with UNIFI_LAN_FILTER if you run a different private range.
|
||||
LAN_FILTER="${UNIFI_LAN_FILTER:-^10\\.}"
|
||||
|
||||
emit_hosts() {
|
||||
# LAN IP is the first RFC1918 entry in reportedState.ipAddrs that is
|
||||
# NOT also present as a WAN ipv4 (reportedState.wans[]) — UDMs with
|
||||
# RFC1918-addressed WAN2 interfaces would otherwise get mis-picked as
|
||||
# their LAN IP. Falls back through the usual chain if nothing matches.
|
||||
ui_get /ea/hosts | jq -r '
|
||||
ui_get /ea/hosts | jq -r "$JQ_PRELUDE"'
|
||||
. as $h |
|
||||
(($h.reportedState.wans // []) | map(.ipv4 // empty)) as $wans |
|
||||
[
|
||||
@@ -129,7 +148,7 @@ emit_hosts() {
|
||||
)] | .[0])
|
||||
// .reportedState.ip // .ipAddress // "-"
|
||||
),
|
||||
(.reportedState.mac // "-"),
|
||||
((.reportedState.mac // "-") | norm_mac),
|
||||
(.reportedState.hostname // .reportedState.name // "-"),
|
||||
(.reportedState.hardware.shortname // .reportedState.hardware.name // .type // "-"),
|
||||
(.reportedState.version // "-"),
|
||||
@@ -152,13 +171,30 @@ emit_sites() {
|
||||
}
|
||||
|
||||
emit_devices() {
|
||||
# /ea/devices returns per-host wrappers; flatten into one row per AP/switch.
|
||||
ui_get /ea/devices | jq -r '
|
||||
# /ea/devices returns per-host wrappers; flatten into one row per
|
||||
# AP/switch. Drop entries that clutter gap analysis without adding value:
|
||||
# - IPs outside the fleet LAN range (UDM's public WAN IP listed as
|
||||
# a "device", etc.)
|
||||
# - UDM self-records (isConsole=true or their WAN IP matches a
|
||||
# wans[].ipv4 — the UDM itself shows up in its own devices list)
|
||||
# - UCI records (UniFi Cable Internet = ISP uplink tracking, model="UCI")
|
||||
ui_get /ea/devices | jq -r \
|
||||
--arg lan "$LAN_FILTER" \
|
||||
"$JQ_PRELUDE"'
|
||||
. as $h |
|
||||
(($h.wans // []) | map(.ipv4 // empty)) as $wans |
|
||||
(.devices // [])[] |
|
||||
select(
|
||||
(.ip // "") | test($lan)
|
||||
) |
|
||||
select(
|
||||
(.isConsole // false) != true and
|
||||
(.model // "") != "UCI" and
|
||||
(.ip as $ip | ($wans | index($ip)) == null)
|
||||
) |
|
||||
[
|
||||
(.ip // "-"),
|
||||
(.mac // .id // "-"),
|
||||
((.mac // .id // "-") | norm_mac),
|
||||
(.name // "-"),
|
||||
(.model // .shortname // "-"),
|
||||
($h.hostName // $h.hostId // "-"),
|
||||
|
||||
Reference in New Issue
Block a user