esh-pfi-infrastructure/scripts/deploy-stack.sh

#!/usr/bin/env bash
# deploy-stack.sh — push a canonical stack from `stacks/` to a server,
# with per-file diff and confirmation prompt.
#
# Source of truth is `stacks/<stack>/` (the canonical, git-tracked tree).
# `stacks-mirror/` is a separate, gitignored snapshot of what's currently
# on each host (pulled by sync-stacks.sh) — it's used for drift detection,
# NOT as the deploy source. See CLAUDE.md "Stack tree convention".
#
# Layout assumed:
#   stacks/<stack>/<file>         → <host>:/opt/docker/compose/<stack>/<file>
#   stacks/<stack>/conf/<file>    → <host>:/opt/docker/conf/<stack>/<file>
#
# Secrets / runtime state are never pushed (same exclude list as
# sync-stacks.sh): .env*, acme.json, *.key/crt/pem/pfx, *.sqlite*, *.db,
# *.log*, *.pid, hub/, logs/, client_secrets.json.
#
# The script:
#   1. Runs rsync --dry-run to find which files would change.
#   2. Prints a unified diff for each changed/added file (deletions noted).
#   3. Prompts [y/N]; applies the rsync only on 'y'.
#
# Usage:
#   scripts/deploy-stack.sh <host> <stack>
#   scripts/deploy-stack.sh <host> <stack> --yes     # skip prompt (use sparingly)
#   scripts/deploy-stack.sh <host> <stack> --compose # push only compose side
#   scripts/deploy-stack.sh <host> <stack> --conf    # push only conf side
# Optional environment:
#   DEPLOY_DEST_STACK=<name>  retain a legacy remote stack directory/project
#   DEPLOY_SUDO=1            use passwordless sudo for remote files and rsync
#   DEPLOY_NO_CLAIM=1        skip the ops-log claim (see below — use sparingly)
#   DEPLOY_CLAIM_TTL=<dur>   how long the claim stays live (default 30m)
#
# OPS LOG + CLAIM (added 2026-09-19)
#   `infra-ops` and `infra-hermes` are two agents sharing ONE OS identity, so
#   host-side changes are otherwise fingerprint-less. Before touching the
#   host this script claims <host>/<stack> via scripts/ops-log and holds the
#   claim across the diff, the y/N prompt and the apply — that whole window is
#   where the 2026-09-18 searxng collision happened, not just the rsync. It
#   then records what it pushed.
#   A REFUSAL (another agent holds the claim) is fatal. ops-log being absent
#   or broken is NOT: the deploy path must not gain a new single point of
#   failure just because it grew an audit trail.

set -euo pipefail

if ! command -v rsync >/dev/null 2>&1; then
  echo "error: rsync is not installed on this workstation" >&2
  echo "       install it (e.g. 'sudo apt install rsync') and ensure the target host has it too" >&2
  exit 2
fi

SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
SERVERS_DIR="$REPO_ROOT/servers"
STACKS_DIR="$REPO_ROOT/stacks"

EXCLUDES=(
  # Include .env.example / *.env.example templates before the broader
  # .env* exclude — rsync processes these in order, first match wins.
  --include='.env.example'
  --include='*.env.example'
  --exclude=.env
  --exclude='.env.*'
  --exclude=acme.json
  --exclude=client_secrets.json
  --exclude='*.pem'
  --exclude='*.key'
  --exclude='*.crt'
  --exclude='*.pfx'
  --exclude='*.sqlite'
  --exclude='*.sqlite3'
  --exclude='*.db'
  --exclude='*.log'
  --exclude='*.log.*'
  --exclude='*.pid'
  --exclude='hub/'
  --exclude='logs/'
)

HOST=
STACK=
ASSUME_YES=0
DO_COMPOSE=1
DO_CONF=1
DEST_STACK=${DEPLOY_DEST_STACK:-}
for a in "$@"; do
  case "$a" in
    --yes|-y)   ASSUME_YES=1 ;;
    --compose)  DO_CONF=0 ;;
    --conf)     DO_COMPOSE=0 ;;
    -h|--help)  sed -n '2,43p' "$0"; exit 0 ;;
    -*) echo "error: unknown flag $a" >&2; exit 2 ;;
    *)
      if   [ -z "$HOST"  ]; then HOST="$a"
      elif [ -z "$STACK" ]; then STACK="$a"
      else echo "error: unexpected positional '$a'" >&2; exit 2
      fi
      ;;
  esac
done

[ -n "$HOST"  ] || { echo "usage: $(basename "$0") <host> <stack>" >&2; exit 2; }
[ -n "$STACK" ] || { echo "usage: $(basename "$0") <host> <stack>" >&2; exit 2; }
DEST_STACK=${DEST_STACK:-$STACK}
# Dots ARE legal — several stacks carry a version in the name (qwen3.5-122b,
# qwopus3.5-122b, mistral-medium-3.5). The previous pattern excluded them, so
# those stacks could not be deployed by this script AT ALL; it surfaced on
# 2026-09-13 as "invalid DEPLOY_DEST_STACK" during the fv-ml1 label repoint.
# The check exists to stop path traversal and shell metacharacters, which means
# it has to reject `..` and `/` — not every dot.
case "$DEST_STACK" in
  *..*|*/*) echo "invalid stack name (path traversal): $DEST_STACK" >&2; exit 2 ;;
esac
[[ "$DEST_STACK" =~ ^[a-zA-Z0-9][a-zA-Z0-9_.-]*$ ]] || { echo "invalid stack name: $DEST_STACK" >&2; exit 2; }

# --------- Claim the stack before any remote work. --------------------
OPS_LOG="$SCRIPT_DIR/ops-log"
CLAIMED=0
release_claim() {
  if [ "$CLAIMED" -eq 1 ]; then
    "$OPS_LOG" release "$HOST" "$STACK" -q >/dev/null 2>&1 || true
    CLAIMED=0
  fi
  return 0
}
trap release_claim EXIT

if [ -x "$OPS_LOG" ] && [ "${DEPLOY_NO_CLAIM:-0}" != 1 ]; then
  claim_rc=0
  "$OPS_LOG" claim "$HOST" "$STACK" --ttl "${DEPLOY_CLAIM_TTL:-30m}" \
    --why "deploy-stack.sh $HOST $STACK" -q || claim_rc=$?
  case "$claim_rc" in
    0) CLAIMED=1 ;;
    3) echo "error: refused — see the claim above. Wait for the holder, coordinate" >&2
       echo "       on althing, or override with DEPLOY_NO_CLAIM=1 if it is dead." >&2
       exit 3 ;;
    *) echo "warning: ops-log claim failed (exit $claim_rc) — deploying UNCLAIMED." >&2 ;;
  esac
fi

resolve_target() {
  # ssh-target file wins when present (may carry user@ or non-default port);
  # /etc/hosts + ssh_config is the fallback.
  local host="$1"
  local fb="$SERVERS_DIR/$host/ssh-target"
  if [ -f "$fb" ]; then
    local t
    t=$(awk 'NF{print $1; exit}' "$fb")
    if [ -n "$t" ]; then echo "$t"; return; fi
  fi
  local effective
  effective=$(ssh -G "$host" 2>/dev/null | awk '/^hostname /{print $2; exit}')
  if [ -n "$effective" ] && getent hosts "$effective" >/dev/null 2>&1; then
    echo "$host"; return
  fi
  echo "$host"
}

TARGET=$(resolve_target "$HOST")
REMOTE_PREFIX=
RSYNC_REMOTE=()
if [ "${DEPLOY_SUDO:-0}" = 1 ]; then
  REMOTE_PREFIX='sudo -n '
  RSYNC_REMOTE=(--rsync-path='sudo -n rsync')
fi
STACK_DIR="$STACKS_DIR/$STACK"

[ -d "$STACK_DIR" ] || { echo "error: $STACK_DIR not found — author the canonical stack first (see stacks/<other>/ for examples)" >&2; exit 2; }

# Collect the two src/dest pairs we need to consider.
PAIRS=()      # each entry: "<kind>|<src>|<dest>"
if [ "$DO_COMPOSE" -eq 1 ]; then
  PAIRS+=("compose|$STACK_DIR/|$TARGET:/opt/docker/compose/$DEST_STACK/")
fi
if [ "$DO_CONF" -eq 1 ] && [ -d "$STACK_DIR/conf" ]; then
  PAIRS+=("conf|$STACK_DIR/conf/|$TARGET:/opt/docker/conf/$DEST_STACK/")
fi

[ "${#PAIRS[@]}" -gt 0 ] || { echo "nothing to deploy"; exit 0; }

# --------- Dry-run summary: which files would change, per kind. --------
declare -A CHANGED_FILES_BY_KIND=()   # kind → newline-separated list
declare -A DELETED_FILES_BY_KIND=()
declare -A RAW_RSYNC_OUT_BY_KIND=()   # kind → raw rsync itemize output
any_change=0

for entry in "${PAIRS[@]}"; do
  IFS='|' read -r kind src dest <<<"$entry"
  extra=()
  [ "$kind" = compose ] && extra+=(--exclude='conf/')

  # Pre-create the remote dir. Without this, rsync against a nonexistent
  # destination can fail in ways the dry-run doesn't surface cleanly.
  remote_path="/opt/docker/$kind/$DEST_STACK/"
  if ! ssh -n -o BatchMode=yes -o ConnectTimeout=10 "$TARGET" \
       "${REMOTE_PREFIX}mkdir -p '$remote_path'" 2>/dev/null; then
    echo "error: could not create $remote_path on $TARGET (check perms / ssh)" >&2
    exit 2
  fi

  tmp_out=$(mktemp) tmp_err=$(mktemp)
  rc=0
  # THE RULE: the deploy syncs CONTENT; the conventions own METADATA.
  #
  # --no-o --no-g --no-perms --omit-dir-times. The deploy tree is
  # root:docker 2775 (setgid) since the
  # 2026-09-14 fleet normalisation, and plain -a makes rsync try to chgrp the
  # destination as the deploy identity, which is not root. It fails with
  # "chgrp ... Operation not permitted" and exits 23 AFTER transferring the
  # content — a loud error on a deploy that actually succeeded. The setgid bit
  # already assigns the right group, so rsync should not be fighting it.
  # This was fixed three times in one session before the rule above was
  # written down, because `-a` is `-rlptgoD` and a non-root identity cannot
  # apply ANY of owner, group, permissions or times to a root-owned directory.
  # Each patch fixed one letter and the next deploy failed on the next one:
  #   chgrp failed          -> --no-o --no-g
  #   failed to set times   -> --omit-dir-times
  #   failed to set perms   -> --no-perms
  # Every one of them exited 23 AFTER transferring the content — a loud error
  # on a deploy that had succeeded — so each partial fix looked complete until
  # the next run. Enumerate the flag set, don't chase the symptom.
  #
  # ⚠ --no-perms means a NEW file lands with the remote umask rather than the
  # source's mode, so a stack shipping an executable script needs its +x set by
  # the convention (playbooks/normalize-docker-tree.yaml preserves exec bits),
  # not by the deploy. Existing files keep their modes.
  rsync -az --no-o --no-g --no-perms --omit-dir-times --delete --dry-run \
      "${RSYNC_REMOTE[@]}" \
      --out-format='%i %n' \
      "${EXCLUDES[@]}" "${extra[@]}" \
      "$src" "$dest" >"$tmp_out" 2>"$tmp_err" || rc=$?
  if [ "$rc" -ne 0 ]; then
    echo "error: rsync dry-run failed (exit $rc) for $src → $dest" >&2
    sed 's/^/  /' "$tmp_err" >&2
    rm -f "$tmp_out" "$tmp_err"
    exit 2
  fi
  mapfile -t lines < "$tmp_out"
  RAW_RSYNC_OUT_BY_KIND[$kind]=$(cat "$tmp_out")
  rm -f "$tmp_out" "$tmp_err"

  changed=""
  deleted=""
  for ln in "${lines[@]}"; do
    # Itemized codes (rsync uses '<' for push, '>' for pull):
    #   <f+++++++++ newfile           (new file, push)
    #   >f+++++++++ newfile           (new file, pull)
    #   <f..t...... file.yaml         (content update, push)
    #   *deleting   oldfile           (deletion, either direction)
    #   .d..t...... ./                (metadata on a dir — skip)
    #   cd+++++++++ somedir/          (new dir — skip, we diff files only)
    [ -z "$ln" ] && continue
    code=$(awk '{print $1}' <<<"$ln")
    name=$(awk '{ $1=""; sub(/^ /,""); print }' <<<"$ln")
    [ -z "$name" ] && continue
    [ "${name: -1}" = "/" ] && continue   # directory entry

    case "$code" in
      '*deleting') deleted+="$name"$'\n' ;;
      '<f'*|'>f'*) changed+="$name"$'\n' ;;
      *)           : ;;     # dir entries, metadata-only, unknown
    esac
  done
  CHANGED_FILES_BY_KIND[$kind]="$changed"
  DELETED_FILES_BY_KIND[$kind]="$deleted"
  if [ -n "$changed$deleted" ]; then any_change=1; fi
done

if [ "$any_change" -eq 0 ]; then
  echo "up to date: $HOST/$STACK is already in sync with server."
  # Diagnostic: if the remote dir actually looks empty, we may have been
  # fooled by an rsync quirk — dump what rsync saw so the user can tell.
  for entry in "${PAIRS[@]}"; do
    IFS='|' read -r kind _ _ <<<"$entry"
    raw=${RAW_RSYNC_OUT_BY_KIND[$kind]:-}
    remote_path="/opt/docker/$kind/$DEST_STACK/"
    remote_count=$(ssh -n -o BatchMode=yes "$TARGET" \
      "${REMOTE_PREFIX}find '$remote_path' -mindepth 1 -maxdepth 1 2>/dev/null | wc -l" \
      2>/dev/null || echo "?")
    printf '  %s: remote has %s entries, rsync itemize output:\n' "$kind" "$remote_count"
    if [ -z "$raw" ]; then
      printf '    (empty — rsync reported no work)\n'
    else
      sed 's/^/    /' <<<"$raw"
    fi
  done
  exit 0
fi

# --------- Print diffs. -----------------------------------------------
divider() { printf '\n%s\n' "------------------------------------------------------------"; }

for entry in "${PAIRS[@]}"; do
  IFS='|' read -r kind src dest <<<"$entry"
  remote_base="/opt/docker/$kind/$DEST_STACK"
  changed=${CHANGED_FILES_BY_KIND[$kind]:-}
  deleted=${DELETED_FILES_BY_KIND[$kind]:-}
  [ -z "$changed$deleted" ] && continue

  printf '\n=== %s → %s ===\n' "$src" "$dest"

  # ssh below runs with -n: without it, ssh slurps the loop's heredoc
  # stdin ($changed / $deleted) and eats the remaining iterations,
  # silently truncating the diff output to the first file only.
  while IFS= read -r rel; do
    [ -z "$rel" ] && continue
    local_file="$src$rel"
    remote_file="$remote_base/$rel"
    divider
    if ssh -n -o BatchMode=yes "$TARGET" "${REMOTE_PREFIX}test -f '$remote_file'" 2>/dev/null; then
      printf 'MODIFY  %s\n' "$rel"
      diff -u --label "a/$rel (remote)" --label "b/$rel (local)" \
        <(ssh -n -o BatchMode=yes "$TARGET" "${REMOTE_PREFIX}cat '$remote_file'" 2>/dev/null) \
        "$local_file" || true
    else
      printf 'ADD     %s\n' "$rel"
      diff -u --label /dev/null --label "b/$rel (local)" \
        /dev/null "$local_file" || true
    fi
  done <<<"$changed"

  while IFS= read -r rel; do
    [ -z "$rel" ] && continue
    remote_file="$remote_base/$rel"
    divider
    printf 'DELETE  %s\n' "$rel"
    diff -u --label "a/$rel (remote)" --label /dev/null \
      <(ssh -n -o BatchMode=yes "$TARGET" "${REMOTE_PREFIX}cat '$remote_file'" 2>/dev/null) \
      /dev/null || true
  done <<<"$deleted"
done

divider

if [ "$ASSUME_YES" -ne 1 ]; then
  read -r -p "Apply these changes to $TARGET? [y/N] " ans
  case "$ans" in
    y|Y|yes|YES) ;;
    *) echo "aborted."; exit 1 ;;
  esac
fi

# --------- Apply. -----------------------------------------------------
for entry in "${PAIRS[@]}"; do
  IFS='|' read -r kind src dest <<<"$entry"
  extra=()
  [ "$kind" = compose ] && extra+=(--exclude='conf/')
  printf 'pushing %s → %s\n' "$src" "$dest"
  rsync -az --no-o --no-g --no-perms --omit-dir-times --delete \
    "${RSYNC_REMOTE[@]}" \
    "${EXCLUDES[@]}" "${extra[@]}" \
    "$src" "$dest"
done

# --------- Record what we just did. -----------------------------------
# Counted from the dry-run itemize, which is what the operator actually
# reviewed and approved — not re-derived after the fact.
if [ -x "$OPS_LOG" ]; then
  summary=""
  for entry in "${PAIRS[@]}"; do
    IFS='|' read -r kind _ _ <<<"$entry"
    n_ch=$(grep -c . <<<"${CHANGED_FILES_BY_KIND[$kind]:-}" || true)
    n_del=$(grep -c . <<<"${DELETED_FILES_BY_KIND[$kind]:-}" || true)
    summary+="${summary:+, }$kind ${n_ch:-0} changed/${n_del:-0} deleted"
  done
  [ "$DEST_STACK" != "$STACK" ] && summary+=" (remote dir $DEST_STACK)"
  "$OPS_LOG" record --host "$HOST" --action deploy-stack --target "$STACK" \
    --outcome changed --detail "$summary" -q || true
fi

echo "done."