esh-pfi-infrastructure/scripts/server_inspect.sh

#!/usr/bin/env bash
# server_inspect.sh — collect server details for writing Docker Compose files.
#
# Conventions this script assumes and reports on:
#   - Compose files:   /opt/docker/compose/<stack>/{docker-compose.yml,compose.yaml}
#   - Config mounts:   /opt/docker/conf/<stack>/...
#   - Named volumes preferred over bind mounts for persistent state.
#
# Safe: read-only. No modifications are made.
# Usage:
#   bash server_inspect.sh                 # print to stdout
#   bash server_inspect.sh /tmp/report.txt # also save to file

set -u

OUT="${1:-}"
if [ -n "$OUT" ]; then exec > >(tee "$OUT") 2>&1; fi

hr()   { printf '\n===== %s =====\n\n' "$*"; }
sub()  { printf '\n----- %s -----\n' "$*"; }
have() { command -v "$1" >/dev/null 2>&1; }

# --- HOST --------------------------------------------------------------------
hr "HOST"
echo "Hostname:   $(hostname -f 2>/dev/null || hostname)"
echo "Date:       $(date -Iseconds)"
echo "Uptime:     $(uptime -p 2>/dev/null || uptime)"
if [ -f /etc/os-release ]; then
  . /etc/os-release
  echo "OS:         ${PRETTY_NAME:-unknown}"
fi
echo "Kernel:     $(uname -r)"
echo "Arch:       $(uname -m)"

# --- HARDWARE ----------------------------------------------------------------
hr "HARDWARE"
if [ -f /proc/cpuinfo ]; then
  echo "CPU cores:  $(grep -c ^processor /proc/cpuinfo)"
  echo "CPU model:  $(awk -F: '/model name/ {print $2; exit}' /proc/cpuinfo | sed 's/^[[:space:]]*//')"
fi
if [ -f /proc/meminfo ]; then
  awk '/^MemTotal:|^MemAvailable:/ {printf "%-11s %.1f GB\n", $1, $2/1024/1024}' /proc/meminfo
fi

# --- GPUS --------------------------------------------------------------------
hr "GPUS"
if have nvidia-smi; then
  nvidia-smi --query-gpu=index,name,memory.total,memory.free,driver_version --format=csv
else
  echo "nvidia-smi not present (no NVIDIA GPUs or driver not installed)"
fi

# --- FILESYSTEM --------------------------------------------------------------
hr "FILESYSTEMS (df)"
df -h -x tmpfs -x devtmpfs -x overlay 2>/dev/null

hr "PERSISTENT MOUNTS (/etc/fstab, non-comment)"
if [ -r /etc/fstab ]; then
  grep -vE '^\s*(#|$)' /etc/fstab
fi

hr "TARGETED DATA PATHS"
for path in /tank /models /opt /opt/docker /opt/docker/compose /opt/docker/conf \
            /var/lib/docker /data /srv; do
  if [ -d "$path" ]; then
    size=$(du -sh "$path" 2>/dev/null | awk '{print $1}')
    echo "$path  (total: ${size:-?})"
    ls -la --time-style=long-iso "$path" 2>/dev/null | sed 's/^/  /' | head -30
    echo
  fi
done

# --- DOCKER ------------------------------------------------------------------
hr "DOCKER"
if ! have docker; then
  echo "docker not installed"
else
  docker version --format 'Server: {{.Server.Version}}   Client: {{.Client.Version}}' 2>/dev/null \
    || echo "docker daemon not reachable by current user"

  sub "docker info"
  docker info --format 'Containers:      {{.Containers}} (running {{.ContainersRunning}}, paused {{.ContainersPaused}}, stopped {{.ContainersStopped}})
Images:          {{.Images}}
Runtimes:        {{.Runtimes}}
Default runtime: {{.DefaultRuntime}}
Storage driver:  {{.Driver}}
Root dir:        {{.DockerRootDir}}
Server version:  {{.ServerVersion}}' 2>/dev/null

  sub "running containers"
  docker ps --format 'table {{.Names}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}' 2>/dev/null

  sub "all containers"
  docker ps -a --format 'table {{.Names}}\t{{.Image}}\t{{.Status}}' 2>/dev/null

  sub "networks"
  docker network ls --format 'table {{.Name}}\t{{.Driver}}\t{{.Scope}}' 2>/dev/null

  sub "networks (external, non-default — worth knowing for compose external: true)"
  docker network ls --filter driver=bridge --format '{{.Name}}' 2>/dev/null \
    | grep -vE '^(bridge|host|none)$' || true

  sub "named volumes"
  docker volume ls --format 'table {{.Name}}\t{{.Driver}}' 2>/dev/null

  sub "compose projects currently running"
  docker ps --format '{{.Label "com.docker.compose.project"}}' 2>/dev/null \
    | sort -u | grep -v '^$' || echo "(none)"
fi

# --- COMPOSE FILES -----------------------------------------------------------
hr "COMPOSE FILES (/opt/docker/compose/)"
if [ -d /opt/docker/compose ]; then
  find /opt/docker/compose -maxdepth 3 -type f \
    \( -name 'docker-compose.y*ml' -o -name 'compose.y*ml' \) 2>/dev/null \
    | sort | while read -r f; do
      printf '\n>>> %s\n' "$f"
      cat "$f"
    done
else
  echo "/opt/docker/compose not present"
fi

# --- CONFIG LAYOUT -----------------------------------------------------------
hr "CONFIG LAYOUT (/opt/docker/conf/ — top 200 entries)"
if [ -d /opt/docker/conf ]; then
  find /opt/docker/conf -maxdepth 4 2>/dev/null | sort | head -200
else
  echo "/opt/docker/conf not present"
fi

# --- PORTS -------------------------------------------------------------------
hr "LISTENING PORTS"
if have ss; then
  ss -tlnH 2>/dev/null | awk '{print $4}' | sort -u
elif have netstat; then
  netstat -tln 2>/dev/null | awk 'NR>2 {print $4}' | sort -u
else
  echo "ss and netstat both unavailable"
fi

# --- HF / MODEL CACHES -------------------------------------------------------
hr "MODEL / HUGGINGFACE CACHES"
for path in /tank/aimodels/huggingface /tank/aimodels/llm ~/.cache/huggingface \
            /data/huggingface /opt/huggingface; do
  if [ -d "$path" ]; then
    size=$(du -sh "$path" 2>/dev/null | awk '{print $1}')
    echo "$path  (${size:-?})"
    if [ -d "$path/hub" ]; then
      echo "  hub entries:"
      ls "$path/hub" 2>/dev/null | sed 's/^/    /' | head -30
    fi
    echo
  fi
done

# --- SYSTEMD SERVICES (docker-adjacent) --------------------------------------
hr "DOCKER-ADJACENT SYSTEMD SERVICES"
if have systemctl; then
  systemctl list-units --type=service --state=running --no-pager --no-legend 2>/dev/null \
    | awk '{print $1, $4}' \
    | grep -Ei 'docker|container|traefik|nvidia' || echo "(none matching)"
fi

hr "DONE"
echo "Paste the above back into the chat, or pass a path as argv[1] to save."