esh-pfi-infrastructure/scripts/refresh-proxmox-info.sh

#!/usr/bin/env bash
# refresh-proxmox-info.sh — pull a fresh proxmox-details.txt from PVE nodes.
#
# Companion to refresh-server-info.sh. Same discovery pattern (hosts under
# servers/*/, ssh-target fallback), but pipes scripts/proxmox_inspect.sh
# over SSH and writes to servers/<host>/proxmox-details.txt.
#
# Host selection:
#   - `all` expands to every servers/*/ dir name matching `*-pve` or whose
#     existing proxmox-details.txt suggests it's a PVE node. Use the name
#     explicitly to target a PVE host that doesn't match the pattern.
#   - Explicit names are never filtered — they're trusted.
#
# For each host:
#   1. Run scripts/proxmox_inspect.sh on the remote via `ssh <target> 'bash -s'`.
#   2. Write output atomically to servers/<host>/proxmox-details.txt.
#      A failed SSH/run never clobbers the previous good snapshot.
#
# Exit status is non-zero if any host failed.
#
# Usage:
#   scripts/refresh-proxmox-info.sh                         show this help
#   scripts/refresh-proxmox-info.sh all                     refresh every PVE host
#   scripts/refresh-proxmox-info.sh pfi-pve                 refresh one host
#   scripts/refresh-proxmox-info.sh pfi-pve nh3-pve         refresh several
#   scripts/refresh-proxmox-info.sh --dry-run all           preview, no ssh
#   scripts/refresh-proxmox-info.sh --validate-only all     checks only
#
# Validation surfaces problems in the captured snapshot — missing `=====
# DONE =====` trailer, absent `pveversion` output (wrong script pointed at
# a non-PVE host), empty guest list when /etc/pve/jobs.cfg mentions vmids
# (backup job targeting retired VMs), etc.

set -euo pipefail

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
INSPECT="$SCRIPT_DIR/proxmox_inspect.sh"
SERVERS_DIR="$REPO_ROOT/servers"

if [ ! -f "$INSPECT" ]; then
  echo "error: $INSPECT not found" >&2
  exit 2
fi

DRY_RUN=0
VALIDATE_ONLY=0
REQUESTED=()
for arg in "$@"; do
  case "$arg" in
    --dry-run) DRY_RUN=1 ;;
    --validate-only|--validate) VALIDATE_ONLY=1 ;;
    -h|--help)
      sed -n '2,32p' "$0"
      exit 0
      ;;
    -*) echo "error: unknown flag $arg" >&2; exit 2 ;;
    *)  REQUESTED+=("$arg") ;;
  esac
done

discover_proxmox_hosts() {
  # Default discovery: match names containing `-pve` either at the end
  # (e.g. pfi-pve, nh3-pve, esh-pve) or as a segment (e.g. esh-pve-nas).
  # Hosts with other naming that happen to be PVE should be called by
  # name; they're outside the fleet-wide `all` sweep.
  find "$SERVERS_DIR" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' \
    | awk '/-pve(-|$)/' | sort
}

# No positional args → show help. Fleet-wide runs must be explicit.
if [ "${#REQUESTED[@]}" -eq 0 ]; then
  sed -n '2,32p' "$0"
  exit 0
fi

if [ "${#REQUESTED[@]}" -eq 1 ] && [ "${REQUESTED[0]}" = "all" ]; then
  mapfile -t HOSTS < <(discover_proxmox_hosts)
else
  for r in "${REQUESTED[@]}"; do
    if [ "$r" = "all" ]; then
      echo "error: 'all' must be the only argument when used" >&2
      exit 2
    fi
  done
  HOSTS=("${REQUESTED[@]}")
fi

if [ "${#HOSTS[@]}" -eq 0 ]; then
  echo "error: no PVE hosts found (expected servers/*-pve/ dirs under $SERVERS_DIR)" >&2
  exit 2
fi

resolve_target() {
  # Identical logic to refresh-server-info.sh: ssh-target file wins when
  # present, then ssh_config + /etc/hosts, finally the dir name as-is.
  # Proxmox hosts in this fleet typically use `root@<name>` via key auth.
  local host="$1"
  local fallback="$SERVERS_DIR/$host/ssh-target"
  if [ -f "$fallback" ]; then
    local target
    target=$(awk 'NF{print $1; exit}' "$fallback")
    if [ -n "$target" ]; then echo "$target"; return; fi
  fi
  local effective
  effective=$(ssh -G "$host" 2>/dev/null | awk '/^hostname /{print $2; exit}')
  if [ -n "$effective" ] && getent hosts "$effective" >/dev/null 2>&1; then
    echo "$host"
    return
  fi
  echo "$host"
}

validate_host() {
  local host="$1"
  local dir="$SERVERS_DIR/$host"

  if [ ! -d "$dir" ]; then
    printf '%s\n' "dir missing: $dir"
    return
  fi
  if [ ! -r "$dir" ] || [ ! -x "$dir" ]; then
    printf '%s\n' "dir not readable/searchable (check permissions)"
    return
  fi

  local stf="$dir/ssh-target"
  if [ -e "$stf" ]; then
    if [ ! -f "$stf" ]; then
      printf '%s\n' "ssh-target is not a regular file"
    elif [ ! -r "$stf" ]; then
      printf '%s\n' "ssh-target not readable"
    elif [ ! -s "$stf" ]; then
      printf '%s\n' "ssh-target is empty"
    else
      local t
      t=$(awk 'NF{print $1; exit}' "$stf" 2>/dev/null || true)
      if [ -z "$t" ]; then
        printf '%s\n' "ssh-target has no non-blank content"
      elif [[ "$t" =~ [[:space:]] ]]; then
        printf '%s\n' "ssh-target first token contains whitespace ('$t')"
      fi
    fi
  fi

  local effective=""
  effective=$(ssh -G "$host" 2>/dev/null | awk '/^hostname /{print $2; exit}')
  local resolves=0
  if [ -n "$effective" ] && getent hosts "$effective" >/dev/null 2>&1; then
    resolves=1
  fi
  if [ "$resolves" -eq 0 ] && [ ! -s "$stf" ]; then
    printf '%s\n' "name '$host' does not resolve and no ssh-target fallback present"
  fi

  [ -f "$dir/README.md" ] || printf '%s\n' "README.md missing"
  if [ ! -f "$dir/proxmox-details.txt" ]; then
    printf '%s\n' "proxmox-details.txt missing (never refreshed)"
  else
    validate_snapshot "$dir/proxmox-details.txt"
  fi
}

validate_snapshot() {
  local file="$1"
  if ! grep -q '^===== DONE =====' "$file"; then
    printf '%s\n' "snapshot appears truncated (missing trailing '===== DONE =====' marker)"
  fi
  if ! grep -q '^PVE:' "$file"; then
    printf '%s\n' "PVE version line missing — snapshot may have been captured against a non-PVE host"
  fi
  if grep -q 'pveversion not found' "$file"; then
    printf '%s\n' "remote reports pveversion not found — target is not a Proxmox node"
  fi
  # Backup coverage sanity: flag any VERDICT=NO lines so gaps surface.
  if grep -qE '^\s*[0-9]+\s+NO\s' "$file"; then
    local n
    n=$(grep -cE '^\s*[0-9]+\s+NO\s' "$file")
    printf '%s\n' "backup coverage: $n guest(s) marked NO — review 'per-guest coverage' section"
  fi
}

print_warnings() {
  local indent="$1"; shift
  local w
  for w in "$@"; do
    printf '%s! %s\n' "$indent" "$w"
  done
}

pad=0
for h in "${HOSTS[@]}"; do (( ${#h} > pad )) && pad=${#h}; done

if [ "$VALIDATE_ONLY" -eq 1 ]; then
  printf 'Validating %d PVE host(s):\n' "${#HOSTS[@]}"
  total_warn=0
  for host in "${HOSTS[@]}"; do
    mapfile -t warnings < <(validate_host "$host")
    if [ "${#warnings[@]}" -eq 0 ]; then
      printf '  %-*s  ok\n' "$pad" "$host"
    else
      printf '  %-*s  %d warning(s)\n' "$pad" "$host" "${#warnings[@]}"
      print_warnings "      " "${warnings[@]}"
      total_warn=$((total_warn + ${#warnings[@]}))
    fi
  done
  if [ "$total_warn" -gt 0 ]; then exit 1; fi
  exit 0
fi

printf 'Refreshing %d PVE host(s):\n' "${#HOSTS[@]}"
failed=()
for host in "${HOSTS[@]}"; do
  out="$SERVERS_DIR/$host/proxmox-details.txt"
  tmp="$out.new"

  mapfile -t warnings < <(validate_host "$host")

  target=$(resolve_target "$host")
  if [ "$target" = "$host" ]; then
    label="$host"
  else
    label="$host → $target"
  fi

  printf '  %-*s  ' "$pad" "$label"

  if [ "$DRY_RUN" -eq 1 ]; then
    printf 'would run: ssh %s bash -s < %s > %s\n' "$target" "$INSPECT" "$out"
    [ "${#warnings[@]}" -gt 0 ] && print_warnings "      " "${warnings[@]}"
    continue
  fi

  mkdir -p "$SERVERS_DIR/$host"

  if ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "$target" 'bash -s' < "$INSPECT" > "$tmp" 2> "$tmp.err"; then
    # ⚠ ssh exiting 0 is NOT proof the capture is usable. The inspect script can
    # emit nothing (a shell that dies before its first write, output swallowed by
    # a remote wrapper) and this would then promote an EMPTY file over a good
    # snapshot and report "ok (0 bytes)" — the readers below all test the
    # snapshot with `-s`, so the writer must not produce something they consider
    # invalid. A guard whose test disagrees with its writer's contract has
    # quietly stopped guarding. Refuse the promotion, keep the old snapshot,
    # and count it as a failure so the exit code carries it.
    new_bytes=$(wc -c < "$tmp" 2>/dev/null || echo 0)
    if [ "$new_bytes" -eq 0 ]; then
      rm -f "$tmp" "$tmp.err"
      printf 'FAIL (empty capture — previous snapshot kept)\n'
      failed+=("$host")
      [ "${#warnings[@]}" -gt 0 ] && print_warnings "      " "${warnings[@]}"
      continue
    fi
    # A capture that collapses to a fraction of the previous one is suspicious
    # but not provably wrong (a host really can shed services), so this WARNS and
    # still promotes — the operator sees it rather than the script guessing.
    shrink=""
    if [ -s "$out" ]; then
      old_bytes=$(wc -c < "$out")
      if [ "$old_bytes" -gt 0 ] && [ $((new_bytes * 4)) -lt "$old_bytes" ]; then
        shrink=" ⚠ shrank from ${old_bytes}B — check before trusting"
      fi
    fi
    mv "$tmp" "$out"
    rm -f "$tmp.err"
    printf 'ok (%s bytes)%s\n' "$new_bytes" "$shrink"
  else
    rc=$?
    rm -f "$tmp"
    err=$(head -n 1 "$tmp.err" 2>/dev/null || true)
    rm -f "$tmp.err"
    printf 'FAIL (rc=%d) %s\n' "$rc" "$err"
    failed+=("$host")
  fi

  [ "${#warnings[@]}" -gt 0 ] && print_warnings "      " "${warnings[@]}"
done

if [ "${#failed[@]}" -gt 0 ]; then
  printf '\n%d host(s) failed: %s\n' "${#failed[@]}" "${failed[*]}" >&2
  exit 1
fi