esh-pfi-infrastructure/scripts/proxmox_inspect.sh

#!/usr/bin/env bash
# proxmox_inspect.sh — read-only discovery probe for Proxmox VE hosts.
#
# Companion to server_inspect.sh. Where that script focuses on OS / docker
# detail, this one enumerates VMs, LXC containers, storage pools, and
# backup job coverage — the Proxmox-specific surface that the generic
# inspect misses.
#
# Run remotely via stdin the same way server_inspect.sh does:
#
#   ssh pfi-pve 'bash -s' < scripts/proxmox_inspect.sh > servers/pfi-pve/proxmox-details.txt
#
# Output format: sectioned plain text, easy to `grep` / diff.
#
# Expects to run as root on a Proxmox node (that's the default SSH user
# on the fleet's PVE hosts). Safe to run repeatedly — no writes, no
# mutations. Missing tools just emit a note and continue.

set -u  # fail on unset vars; keep going on command errors

have()  { command -v "$1" >/dev/null 2>&1; }
hdr()   { printf '\n===== %s =====\n\n' "$1"; }
sub()   { printf '\n----- %s -----\n' "$1"; }
note()  { printf '(%s)\n' "$*"; }

# ------------------------------------------------------------
# 0. sanity check — is this actually Proxmox?
# ------------------------------------------------------------
if ! have pveversion; then
  echo "error: pveversion not found. This doesn't look like a Proxmox host." >&2
  echo "       Use scripts/server_inspect.sh for plain Linux hosts instead." >&2
  exit 2
fi

# ------------------------------------------------------------
# 1. Host summary (mirrors server_inspect.sh top section)
# ------------------------------------------------------------
hdr "HOST"
printf 'Hostname:   %s\n' "$(hostname -f 2>/dev/null || hostname)"
printf 'Date:       %s\n' "$(date -Iseconds)"
printf 'Uptime:     %s\n' "$(uptime -p 2>/dev/null || uptime)"
printf 'PVE:        %s\n' "$(pveversion 2>/dev/null | head -1)"
printf 'Kernel:     %s\n' "$(uname -r)"
printf 'Arch:       %s\n' "$(uname -m)"

hdr "HARDWARE"
printf 'CPU model:  %s\n' "$(awk -F': ' '/^model name/ {print $2; exit}' /proc/cpuinfo)"
printf 'CPU cores:  %s\n' "$(nproc)"
printf 'MemTotal:   %s\n' "$(awk '/^MemTotal/ {printf "%.1f GB", $2/1024/1024}' /proc/meminfo)"
printf 'MemAvail:   %s\n' "$(awk '/^MemAvailable/ {printf "%.1f GB", $2/1024/1024}' /proc/meminfo)"

# ------------------------------------------------------------
# 2. Cluster status (single-node hosts print a polite note)
# ------------------------------------------------------------
hdr "CLUSTER"
if have pvecm; then
  out=$(pvecm status 2>&1)
  if grep -qi "cluster information" <<<"$out"; then
    printf '%s\n' "$out"
  else
    note "single-node host (or pvecm returned no cluster info)"
  fi
else
  note "pvecm not available"
fi

# ------------------------------------------------------------
# 3. Node resources (one line per VM / LXC / node / storage)
# ------------------------------------------------------------
hdr "CLUSTER RESOURCES (pvesh)"
if have pvesh; then
  # Plain text form is human-readable; JSON form is machine-parseable.
  # We emit plain here and a JSON dump in the BACKUP COVERAGE section.
  pvesh get /cluster/resources --output-format text 2>/dev/null || \
    note "pvesh /cluster/resources failed"
else
  note "pvesh not available"
fi

# ------------------------------------------------------------
# 4. VMs (KVM) — per-VM summary + config highlights
# ------------------------------------------------------------
hdr "VMs (qm list)"
if have qm; then
  qm list 2>/dev/null || note "qm list failed"

  hdr "VM CONFIGS (qm config per id)"
  vmids=$(qm list 2>/dev/null | awk 'NR>1 {print $1}')
  if [ -n "$vmids" ]; then
    for id in $vmids; do
      sub "VMID $id"
      qm config "$id" 2>/dev/null | awk '
        /^(name|cores|memory|balloon|net[0-9]+|scsi[0-9]+|virtio[0-9]+|ide[0-9]+|sata[0-9]+|ostype|onboot|agent|tags|boot|bootdisk|description):/
      '
    done
  else
    note "no VMs found"
  fi
else
  note "qm not available (not a Proxmox VE node?)"
fi

# ------------------------------------------------------------
# 5. LXC containers
# ------------------------------------------------------------
hdr "LXC CONTAINERS (pct list)"
if have pct; then
  pct list 2>/dev/null || note "pct list failed"

  hdr "LXC CONFIGS (pct config per id)"
  ctids=$(pct list 2>/dev/null | awk 'NR>1 {print $1}')
  if [ -n "$ctids" ]; then
    for id in $ctids; do
      sub "CTID $id"
      pct config "$id" 2>/dev/null | awk '
        /^(hostname|cores|memory|swap|net[0-9]+|rootfs|mp[0-9]+|onboot|ostype|unprivileged|features|tags|description):/
      '
    done
  else
    note "no LXC containers found"
  fi
else
  note "pct not available"
fi

# ------------------------------------------------------------
# 6. Storage — pools, usage, ZFS if present
# ------------------------------------------------------------
hdr "STORAGE (pvesm status)"
if have pvesm; then
  pvesm status 2>/dev/null || note "pvesm status failed"
else
  note "pvesm not available"
fi

hdr "DATASTORE CONFIG (/etc/pve/storage.cfg)"
if [ -r /etc/pve/storage.cfg ]; then
  cat /etc/pve/storage.cfg
else
  note "/etc/pve/storage.cfg not readable"
fi

hdr "ZFS POOLS (zpool list -v)"
if have zpool; then
  zpool list -v 2>/dev/null || note "zpool list returned no pools"
else
  note "no zpool on this host (not ZFS)"
fi

hdr "FILESYSTEMS (df -h, local fs only)"
df -hT -x tmpfs -x devtmpfs -x overlay -x squashfs 2>/dev/null | head -40

hdr "MOUNTED NFS / CIFS"
awk '$3 ~ /^(nfs|nfs4|cifs|smb)/ {print}' /proc/mounts || note "no network mounts"

# ------------------------------------------------------------
# 7. Backup jobs + coverage analysis
# ------------------------------------------------------------
hdr "BACKUP JOBS (/etc/pve/jobs.cfg + /etc/vzdump.conf)"
if [ -r /etc/pve/jobs.cfg ]; then
  cat /etc/pve/jobs.cfg
else
  note "/etc/pve/jobs.cfg not readable (no scheduled backups?)"
fi

sub "vzdump defaults"
if [ -r /etc/vzdump.conf ]; then
  awk '!/^#/ && NF' /etc/vzdump.conf || note "vzdump.conf has no non-comment lines"
else
  note "/etc/vzdump.conf not present"
fi

hdr "BACKUP COVERAGE ANALYSIS"
# Parse /etc/pve/jobs.cfg directly — it's always present on PVE nodes and
# doesn't need jq. Format is stanzas separated by blank lines; each stanza
# starts with `vzdump: <id>` and is followed by indented `key value` pairs.
covered_all=0
covered_ids=""
pool_jobs=""
excluded_ids=""

parse_jobs_cfg() {
  # $1 = jobs.cfg path; $2 = "table" for human-readable, "data" for TSV.
  awk -v mode="${2:-data}" '
    BEGIN { RS=""; FS="\n" }
    /^vzdump:/ {
      jid="-"; sched="-"; all="0"; enabled="1"
      vmids=""; excl=""; pool=""; storage="-"
      for (i=1; i<=NF; i++) {
        line=$i
        if (line ~ /^vzdump:/) { split(line, a, " "); jid=a[2]; continue }
        sub(/^[[:space:]]+/, "", line)
        if (line == "") continue
        key=line; val=""
        sp=index(line, " ")
        if (sp > 0) { key=substr(line,1,sp-1); val=substr(line,sp+1); sub(/^[[:space:]]+/, "", val) }
        if (key=="schedule") sched=val
        else if (key=="all") all=val
        else if (key=="enabled") enabled=val
        else if (key=="vmid") vmids=val
        else if (key=="exclude") excl=val
        else if (key=="pool") pool=val
        else if (key=="storage") storage=val
      }
      if (mode == "table") {
        printf "%-24s %-14s %-4s %-8s %-40s %-20s %s\n", jid, sched, all, enabled, (vmids==""?"-":vmids), (excl==""?"-":excl), storage
      } else {
        printf "%s\t%s\t%s\t%s\t%s\n", all, vmids, excl, pool, enabled
      }
    }
  ' "$1"
}

if [ -r /etc/pve/jobs.cfg ]; then
  printf '%-24s %-14s %-4s %-8s %-40s %-20s %s\n' \
    "JOB-ID" "SCHEDULE" "ALL" "ENABLED" "VMID-LIST" "EXCLUDE" "STORAGE"
  parse_jobs_cfg /etc/pve/jobs.cfg table

  while IFS=$'\t' read -r all vmids excl pool enabled; do
    [ -z "${all:-}" ] && continue
    [ "$enabled" = "0" ] && continue
    [ "$all" = "1" ] && covered_all=1
    [ -n "$vmids" ] && covered_ids="$covered_ids $(echo "$vmids" | tr ',' ' ')"
    [ -n "$excl" ] && excluded_ids="$excluded_ids $(echo "$excl" | tr ',' ' ')"
    [ -n "$pool" ] && pool_jobs="$pool_jobs $pool"
  done < <(parse_jobs_cfg /etc/pve/jobs.cfg data)
else
  note "/etc/pve/jobs.cfg not readable — no coverage data"
fi

# Per-guest coverage verdict
sub "per-guest coverage"
all_guest_ids=""
if have qm;  then all_guest_ids="$all_guest_ids $(qm list 2>/dev/null | awk 'NR>1 {print $1}')"; fi
if have pct; then all_guest_ids="$all_guest_ids $(pct list 2>/dev/null | awk 'NR>1 {print $1}')"; fi

if [ -n "$all_guest_ids" ]; then
  printf '%-6s  %-10s  %s\n' "ID" "VERDICT" "REASON"
  for id in $all_guest_ids; do
    verdict="NO"; reason="no job targets this id"
    if [ "$covered_all" = "1" ]; then
      verdict="YES"; reason="covered by 'all' job"
    fi
    for c in $covered_ids; do
      if [ "$c" = "$id" ]; then verdict="YES"; reason="explicit vmid match"; break; fi
    done
    for x in $excluded_ids; do
      if [ "$x" = "$id" ]; then verdict="NO";  reason="explicitly excluded"; break; fi
    done
    if [ -n "$pool_jobs" ] && [ "$verdict" = "NO" ]; then
      reason="$reason (pool job exists: $pool_jobs — manual audit)"
    fi
    printf '%-6s  %-10s  %s\n' "$id" "$verdict" "$reason"
  done
else
  note "no guests on this node"
fi

# ------------------------------------------------------------
# 8. Most-recent dump files per guest (local dump storages)
# ------------------------------------------------------------
hdr "RECENT BACKUPS ON LOCAL DUMP STORAGES"
if [ -r /etc/pve/storage.cfg ]; then
  # Find all storages flagged as dump-capable (content includes 'backup').
  dump_paths=$(awk '
    /^[a-z0-9_-]+: / { stname=$2; path=""; content="" }
    /^[[:space:]]+path / { path=$2 }
    /^[[:space:]]+content / { content=$0 }
    /^$/ { if (path != "" && content ~ /backup/) print path; path=""; content="" }
    END { if (path != "" && content ~ /backup/) print path }
  ' /etc/pve/storage.cfg)
  if [ -z "$dump_paths" ]; then
    note "no local dump storages configured (backups may target remote/PBS)"
  else
    for p in $dump_paths; do
      sub "$p/dump/"
      if [ -d "$p/dump" ]; then
        # Latest actual backup (not .notes sidecar) per VMID.
        # Filename pattern: vzdump-{qemu,lxc}-<VMID>-<YYYY_MM_DD>-<HH_MM_SS>.{vma,tar}.{zst,gz,lzo}
        find "$p/dump" -maxdepth 1 -type f \( \
          -name '*.vma.zst' -o -name '*.vma.gz' -o -name '*.vma.lzo' \
          -o -name '*.tar.zst' -o -name '*.tar.gz' -o -name '*.tar.lzo' \
        \) -printf '%T@ %p\n' 2>/dev/null | sort -nr | awk '
          {
            path=$2
            # extract vmid from filename (3rd hyphen-delimited field after vzdump-)
            fname=path; sub(/.*\//, "", fname)
            n=split(fname, a, "-")
            id=a[3]
            if (!(id in seen)) {
              seen[id]=1
              # human date: extract date+time from filename
              cmd="date -d @" $1 " +%Y-%m-%d\\ %H:%M"
              cmd | getline when
              close(cmd)
              printf "  VMID %-6s  %s  %s\n", id, when, fname
            }
          }
        ' | sort -k2
      else
        note "$p/dump does not exist"
      fi
    done
  fi
fi

# ------------------------------------------------------------
# 9. Proxmox Backup Server targets
# ------------------------------------------------------------
hdr "PBS TARGETS"
if [ -r /etc/pve/storage.cfg ]; then
  awk '/^pbs:/ {show=1; print; next} /^[a-z]/ {show=0} show {print}' /etc/pve/storage.cfg
fi
if [ -z "$(awk '/^pbs:/ {print}' /etc/pve/storage.cfg 2>/dev/null)" ]; then
  note "no PBS storage configured on this node"
fi

# ------------------------------------------------------------
# 10. Replication (cluster-only)
# ------------------------------------------------------------
hdr "REPLICATION JOBS (pvesr status)"
if have pvesr; then
  pvesr status 2>/dev/null | head -40 || note "pvesr status failed (single-node?)"
else
  note "pvesr not available"
fi

# ------------------------------------------------------------
# 11. Listening ports (quick attack surface glance)
# ------------------------------------------------------------
hdr "LISTENING PORTS"
if have ss; then
  ss -ltn 2>/dev/null | awk 'NR>1 {print $4}' | sort -u
fi

# ------------------------------------------------------------
# 12. Subscription + updates
# ------------------------------------------------------------
hdr "SUBSCRIPTION / UPDATES"
if have pvesubscription; then
  pvesubscription get 2>/dev/null | grep -E '^(status|productname|key):' || note "pvesubscription returned nothing"
fi
sub "apt upgradable (top 20)"
apt list --upgradable 2>/dev/null | grep -v '^Listing' | head -20 || note "apt not available"

hdr "DONE"
echo "Review this output for backup gaps (VERDICT=NO in coverage section)"
echo "and anomalies in storage / PBS / replication sections."