esh-pfi-infrastructure/scripts/sync-stacks.sh

#!/usr/bin/env bash
# sync-stacks.sh — pull /opt/docker/{compose,conf}/<stack>/ from every
# server into the gitignored `stacks-mirror/<host>/<stack>/` snapshot tree.
#
# Role: drift detection. The mirror reflects what's actually running on
# each host RIGHT NOW. The canonical source of truth lives at
# `stacks/<stack>/` (git-tracked), and `deploy-stack.sh` pushes from
# there. Use the mirror to compare what we have to what's deployed:
#   diff -ru stacks/<stack>/ stacks-mirror/<host>/<stack>/
# See CLAUDE.md "Stack tree convention" for the full split.
#
# Layout (flat per stack):
#   stacks-mirror/<host>/<stack>/          <- mirrors /opt/docker/compose/<stack>/
#   stacks-mirror/<host>/<stack>/conf/     <- mirrors /opt/docker/conf/<stack>/
#
# Opt-out (per-stack, per-kind):
#   stacks-mirror/<host>/<stack>/.no-sync        → skip stack entirely
#   stacks-mirror/<host>/<stack>/conf/.no-sync   → skip conf only
# The marker file is preserved; only the rsync is suppressed. Create the
# marker manually for any stack you don't want mirrored.
#
# Secrets and runtime state are always excluded regardless of opt-out:
#   .env, .env.*, acme.json, client_secrets.json,
#   *.pem, *.key, *.crt, *.pfx,
#   *.sqlite, *.sqlite3, *.db, *.log, *.log.*, *.pid,
#   hub/, logs/
#
# Usage:
#   scripts/sync-stacks.sh                  # pull from every discovered host
#   scripts/sync-stacks.sh ana-docker nh3-docker
#   scripts/sync-stacks.sh --dry-run        # show what would change, no writes

set -euo pipefail

if ! command -v rsync >/dev/null 2>&1; then
  echo "error: rsync is not installed on this workstation" >&2
  echo "       install it (e.g. 'sudo apt install rsync') and ensure every remote host has it too" >&2
  exit 2
fi

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
SERVERS_DIR="$REPO_ROOT/servers"
MIRROR_DIR="$REPO_ROOT/stacks-mirror"

EXCLUDES=(
  # Include .env.example / *.env.example templates before the broader
  # .env* exclude — rsync processes these in order, first match wins.
  --include='.env.example'
  --include='*.env.example'
  --exclude=.env
  --exclude='.env.*'
  --exclude=acme.json
  --exclude=client_secrets.json
  --exclude='*.pem'
  --exclude='*.key'
  --exclude='*.crt'
  --exclude='*.pfx'
  --exclude='*.sqlite'
  --exclude='*.sqlite3'
  --exclude='*.db'
  --exclude='*.log'
  --exclude='*.log.*'
  --exclude='*.pid'
  --exclude='hub/'
  --exclude='logs/'
)

DRY_RUN=0
REQUESTED=()
for arg in "$@"; do
  case "$arg" in
    --dry-run) DRY_RUN=1 ;;
    -h|--help) sed -n '2,24p' "$0"; exit 0 ;;
    -*) echo "error: unknown flag $arg" >&2; exit 2 ;;
    *)  REQUESTED+=("$arg") ;;
  esac
done

resolve_target() {
  # ssh-target file wins when present (may carry user@ or non-default port);
  # /etc/hosts + ssh_config is the fallback.
  local host="$1"
  local fb="$SERVERS_DIR/$host/ssh-target"
  if [ -f "$fb" ]; then
    local t
    t=$(awk 'NF{print $1; exit}' "$fb")
    if [ -n "$t" ]; then echo "$t"; return; fi
  fi
  local effective
  effective=$(ssh -G "$host" 2>/dev/null | awk '/^hostname /{print $2; exit}')
  if [ -n "$effective" ] && getent hosts "$effective" >/dev/null 2>&1; then
    echo "$host"; return
  fi
  echo "$host"
}

list_remote_subdirs() {
  # $1 = ssh target, $2 = remote parent path
  ssh -o BatchMode=yes -o ConnectTimeout=10 "$1" \
    "find '$2' -maxdepth 1 -mindepth 1 -type d -printf '%f\n' 2>/dev/null | sort" \
    2>/dev/null || true
}

sync_one() {
  # $1 = host, $2 = ssh target, $3 = stack,
  # $4 = 'compose'|'conf' (kind),
  # $5 = local dest dir
  local host="$1" target="$2" stack="$3" kind="$4" dest="$5"
  local remote_src="/opt/docker/$kind/$stack/"
  local skip="$dest/.no-sync"
  local label
  if [ "$kind" = conf ]; then label='conf   '; else label='compose'; fi

  mkdir -p "$dest"

  if [ -f "$skip" ]; then
    printf '      %s  skip (.no-sync)\n' "$label"
    return 0
  fi

  local extra=()
  # Don't recurse into conf/ from the compose side — it's its own mirror target.
  [ "$kind" = compose ] && extra+=(--exclude='conf/')
  [ "$DRY_RUN" -eq 1 ]   && extra+=(--dry-run)

  local err rc=0
  err=$(
    rsync -az --delete --info=stats0,flist0 \
      "${EXCLUDES[@]}" "${extra[@]}" \
      "$target:$remote_src" "$dest/" 2>&1
  ) || rc=$?

  if [ $rc -ne 0 ]; then
    printf '      %s  FAIL (rc=%d) %s\n' "$label" "$rc" "$(echo "$err" | head -n 1)"
    return 1
  fi
  if [ "$DRY_RUN" -eq 1 ]; then
    printf '      %s  dry-run ok\n' "$label"
  else
    printf '      %s  ok\n' "$label"
  fi
  return 0
}

if [ "${#REQUESTED[@]}" -eq 0 ]; then
  mapfile -t HOSTS < <(find "$SERVERS_DIR" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' | sort)
else
  HOSTS=("${REQUESTED[@]}")
fi

[ "${#HOSTS[@]}" -eq 0 ] && { echo "error: no hosts found" >&2; exit 2; }
[ "$DRY_RUN" -eq 1 ] && echo "(dry-run)"

mkdir -p "$MIRROR_DIR"
total_fail=0

for host in "${HOSTS[@]}"; do
  target=$(resolve_target "$host")
  printf '%s (%s):\n' "$host" "$target"

  mapfile -t compose_stacks < <(list_remote_subdirs "$target" /opt/docker/compose)
  mapfile -t conf_stacks    < <(list_remote_subdirs "$target" /opt/docker/conf)

  if [ "${#compose_stacks[@]}" -eq 0 ] && [ "${#conf_stacks[@]}" -eq 0 ]; then
    printf '  (no stacks discovered — check ssh + remote /opt/docker layout)\n'
    continue
  fi

  # Union of stack names.
  mapfile -t all_stacks < <(printf '%s\n' "${compose_stacks[@]}" "${conf_stacks[@]}" | sort -u | grep .)

  # Warn about local stacks that no longer exist on the remote.
  if [ -d "$MIRROR_DIR/$host" ]; then
    for local_stack in "$MIRROR_DIR/$host"/*/; do
      [ -d "$local_stack" ] || continue
      name=$(basename "$local_stack")
      if ! printf '%s\n' "${all_stacks[@]}" | grep -qxF "$name"; then
        printf '  ! %s  exists locally but not on remote (stale — remove manually if intentional)\n' "$name"
      fi
    done
  fi

  for stack in "${all_stacks[@]}"; do
    printf '  %s\n' "$stack"
    stack_root="$MIRROR_DIR/$host/$stack"
    if [ -f "$stack_root/.no-sync" ]; then
      printf '      skip (.no-sync at stack root)\n'
      continue
    fi
    if printf '%s\n' "${compose_stacks[@]}" | grep -qxF "$stack"; then
      sync_one "$host" "$target" "$stack" compose "$stack_root" || total_fail=$((total_fail+1))
    fi
    if printf '%s\n' "${conf_stacks[@]}" | grep -qxF "$stack"; then
      sync_one "$host" "$target" "$stack" conf "$stack_root/conf" || total_fail=$((total_fail+1))
    fi
  done
done

if [ "$total_fail" -gt 0 ]; then
  printf '\n%d sync operation(s) failed\n' "$total_fail" >&2
  exit 1
fi