scripts/discover-*: four QoL upgrades from first live gap-analysis run
First real run surfaced 31 gap rows, ~20 of which were noise. These
changes reduce the output to actionable signal.
1. discover-unifi: /ea/devices now filters out
- IPs outside the fleet LAN range (UDM's WAN IP appearing as a
"device", ISP uplink records with public IPs)
- UDM self-records (isConsole=true, or IP matches wans[].ipv4)
- UCI records (UniFi Cable Internet = ISP modem tracking)
LAN filter regex defaults to ^10\. (matches 10.0.0.0/8); override
via UNIFI_LAN_FILTER env var if you run other private ranges.
2. discover-gaps: new --ignore-unifi flag drops rows where the final
SOURCE column starts with "unifi:". Useful for "show me servery
things to manage, not the fleet's network hardware."
3. discover-gaps: known-IP set now pulls IPs from
servers/*/proxmox-details.txt AND servers/*/system-details.txt in
addition to README.md and ssh-target. Consequence: VMs tracked by
proxmox_inspect.sh are automatically counted as known without
needing a separate servers/<vmname>/ dir. Also strips meaningless
addresses (127.*, 0.0.0.0, 169.254.*) so they can't false-positive
a "known" match.
4. MAC normalization: both discover-fortigate and discover-unifi now
emit xx:xx:xx:xx:xx:xx lowercase. Previously FortiGate used colon
format, UniFi used no-separator uppercase — same MAC looked
different per source. Fortigate does tolower() in awk; UniFi uses
a shared jq `norm_mac` function.
This commit is contained in:
+69
-19
@@ -3,14 +3,25 @@
|
||||
# servers/*/ssh-target (the authoritative "managed hosts" list).
|
||||
#
|
||||
# Input: one or more TSV files, each a line of:
|
||||
# IP MAC HOSTNAME (EXTRA1) (EXTRA2) …
|
||||
# IP MAC HOSTNAME (EXTRA1) (EXTRA2) … SOURCE
|
||||
# Produced by discover-fortigate.sh, discover-unifi.sh, or equivalent.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/discover-gaps.sh leases-ana.tsv leases-nh3.tsv
|
||||
# scripts/discover-gaps.sh [--ignore-unifi] leases.tsv [more.tsv ...]
|
||||
#
|
||||
# Output: rows where IP does NOT match any IP in servers/*/ssh-target,
|
||||
# sorted for readability. Same TSV format as input.
|
||||
# Flags:
|
||||
# --ignore-unifi Drop rows whose final column starts with "unifi:".
|
||||
# Useful when you want to focus on servers / IoT /
|
||||
# endpoints rather than UniFi infrastructure
|
||||
# (APs, switches, UDMs).
|
||||
#
|
||||
# An IP is considered "known" if it matches any of:
|
||||
# - an IPv4 in servers/<host>/ssh-target
|
||||
# - a resolvable hostname in ssh-target
|
||||
# - any IPv4 token found in servers/<host>/README.md
|
||||
# - any IPv4 token in servers/<host>/proxmox-details.txt
|
||||
# (so Proxmox-tracked VM IPs surfaced by proxmox_inspect.sh are
|
||||
# automatically absorbed)
|
||||
#
|
||||
# Exit code:
|
||||
# 0 if zero gaps found
|
||||
@@ -22,14 +33,33 @@ SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
SERVERS_DIR="$REPO_ROOT/servers"
|
||||
|
||||
if [ "$#" -lt 1 ]; then
|
||||
echo "usage: $(basename "$0") <leases.tsv> [more.tsv ...]" >&2
|
||||
IGNORE_UNIFI=0
|
||||
FILES=()
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--ignore-unifi) IGNORE_UNIFI=1 ;;
|
||||
-h|--help)
|
||||
sed -n '2,30p' "$0"
|
||||
exit 0
|
||||
;;
|
||||
-*)
|
||||
echo "error: unknown flag $arg" >&2
|
||||
exit 2
|
||||
;;
|
||||
*)
|
||||
FILES+=("$arg")
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ "${#FILES[@]}" -lt 1 ]; then
|
||||
echo "usage: $(basename "$0") [--ignore-unifi] <leases.tsv> [more.tsv ...]" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# Build the set of managed IPs from servers/*/ssh-target.
|
||||
# ssh-target may contain "user@IP" or a bare IP/hostname. We extract any
|
||||
# IPv4 and (if the value is a hostname) try to resolve it.
|
||||
# ---- Build the set of managed IPs ---------------------------------------
|
||||
|
||||
# 1. servers/*/ssh-target
|
||||
managed_ips=$(
|
||||
for stf in "$SERVERS_DIR"/*/ssh-target; do
|
||||
[ -r "$stf" ] || continue
|
||||
@@ -38,7 +68,6 @@ managed_ips=$(
|
||||
if [[ "$val" =~ ^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$ ]]; then
|
||||
echo "$val"
|
||||
else
|
||||
# getent hosts returns "IP name [alias…]"; take first field if resolved
|
||||
ip=$(getent hosts "$val" 2>/dev/null | awk 'NR==1{print $1}')
|
||||
[ -n "$ip" ] && echo "$ip"
|
||||
fi
|
||||
@@ -49,23 +78,44 @@ if [ -z "$managed_ips" ]; then
|
||||
echo "warning: no managed IPs found under $SERVERS_DIR/*/ssh-target" >&2
|
||||
fi
|
||||
|
||||
# Also tolerate IPs that appear in servers/*/README.md (for hosts that
|
||||
# have snapshot info but no ssh-target — e.g. Proxmox hosts documented
|
||||
# there). Extract any IPv4-looking token from READMEs.
|
||||
# 2. Any IPv4 in README.md files under servers/
|
||||
readme_ips=$(
|
||||
grep -rhEo '([0-9]+\.){3}[0-9]+' "$SERVERS_DIR"/*/README.md 2>/dev/null | sort -u || true
|
||||
grep -rhEo '([0-9]+\.){3}[0-9]+' "$SERVERS_DIR"/*/README.md 2>/dev/null \
|
||||
| sort -u || true
|
||||
)
|
||||
|
||||
all_known=$(printf '%s\n%s\n' "$managed_ips" "$readme_ips" | sort -u)
|
||||
# 3. Any IPv4 in proxmox-details.txt (VM IPs + cluster-resources table)
|
||||
pve_ips=$(
|
||||
grep -rhEo '([0-9]+\.){3}[0-9]+' "$SERVERS_DIR"/*/proxmox-details.txt 2>/dev/null \
|
||||
| sort -u || true
|
||||
)
|
||||
|
||||
# Read the discovery TSVs, filter out lines whose IP is in all_known.
|
||||
# 4. Any IPv4 in system-details.txt (ip addr show output, docker inspect, etc.)
|
||||
sys_ips=$(
|
||||
grep -rhEo '([0-9]+\.){3}[0-9]+' "$SERVERS_DIR"/*/system-details.txt 2>/dev/null \
|
||||
| sort -u || true
|
||||
)
|
||||
|
||||
# Drop meaningless addresses from the "known" set (they'd falsely mask
|
||||
# real unmanaged hosts that happen to also be 127.0.0.1 in some probe).
|
||||
all_known=$(
|
||||
printf '%s\n%s\n%s\n%s\n' "$managed_ips" "$readme_ips" "$pve_ips" "$sys_ips" \
|
||||
| grep -Ev '^(0\.0\.0\.0|127\.|255\.255\.255\.255|169\.254\.)' \
|
||||
| sort -u
|
||||
)
|
||||
|
||||
# ---- Filter discovery TSVs ---------------------------------------------
|
||||
unmanaged=$(
|
||||
cat "$@" | awk -v known="$all_known" '
|
||||
cat "${FILES[@]}" | awk -F'\t' -v known="$all_known" -v drop_unifi="$IGNORE_UNIFI" '
|
||||
BEGIN {
|
||||
n=split(known, arr, "\n")
|
||||
n = split(known, arr, "\n")
|
||||
for (i=1; i<=n; i++) if (arr[i] != "") k[arr[i]] = 1
|
||||
}
|
||||
NF >= 1 && $1 ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/ && !($1 in k)
|
||||
# Need a valid IP in column 1 and IP must not be "known".
|
||||
NF >= 1 && $1 ~ /^[0-9]+\.[0-9]+\.[0-9]+\.[0-9]+$/ && !($1 in k) {
|
||||
if (drop_unifi && $NF ~ /^unifi:/) next
|
||||
print
|
||||
}
|
||||
' | sort -t$'\t' -k1,1 -V -u
|
||||
)
|
||||
|
||||
|
||||
Reference in New Issue
Block a user