vh
3e7d3a3e3d
deploy-stack.sh claims and releases around its own work. When the agent already held a longer claim for a multi-step operation, the deploy refreshed it, then released it on exit -- silently dropping the protection partway through the very operation it was guarding. Caught live: a 45-minute claim on nh3-docker/althing-post-office, taken to cover a build-push-deploy-verify rollout, was gone by the time the rollout finished. Nothing refused anything, and nothing said so. `ops-log claim` now exits 10 when the claim was already the caller's, and leaves the holder file untouched. deploy-stack.sh treats 10 as "not mine to release". Untouched matters as much as the exit code: a refresh would overwrite the reason and TTL the original claimant chose, so a deliberate 45-minute "3.7.0 rollout in progress" would degrade into "deploy-stack.sh <host> <stack>" and whoever got refused would read the wrong story. Verified in three states: no pre-existing claim -> deploy claims and releases; caller's own wider claim -> deploy refreshes nothing and leaves it standing with its original reason; another agent's claim -> still refused with exit 3.
379 lines
15 KiB
Bash
Executable File
379 lines
15 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
# deploy-stack.sh — push a canonical stack from `stacks/` to a server,
|
|
# with per-file diff and confirmation prompt.
|
|
#
|
|
# Source of truth is `stacks/<stack>/` (the canonical, git-tracked tree).
|
|
# `stacks-mirror/` is a separate, gitignored snapshot of what's currently
|
|
# on each host (pulled by sync-stacks.sh) — it's used for drift detection,
|
|
# NOT as the deploy source. See CLAUDE.md "Stack tree convention".
|
|
#
|
|
# Layout assumed:
|
|
# stacks/<stack>/<file> → <host>:/opt/docker/compose/<stack>/<file>
|
|
# stacks/<stack>/conf/<file> → <host>:/opt/docker/conf/<stack>/<file>
|
|
#
|
|
# Secrets / runtime state are never pushed (same exclude list as
|
|
# sync-stacks.sh): .env*, acme.json, *.key/crt/pem/pfx, *.sqlite*, *.db,
|
|
# *.log*, *.pid, hub/, logs/, client_secrets.json.
|
|
#
|
|
# The script:
|
|
# 1. Runs rsync --dry-run to find which files would change.
|
|
# 2. Prints a unified diff for each changed/added file (deletions noted).
|
|
# 3. Prompts [y/N]; applies the rsync only on 'y'.
|
|
#
|
|
# Usage:
|
|
# scripts/deploy-stack.sh <host> <stack>
|
|
# scripts/deploy-stack.sh <host> <stack> --yes # skip prompt (use sparingly)
|
|
# scripts/deploy-stack.sh <host> <stack> --compose # push only compose side
|
|
# scripts/deploy-stack.sh <host> <stack> --conf # push only conf side
|
|
# Optional environment:
|
|
# DEPLOY_DEST_STACK=<name> retain a legacy remote stack directory/project
|
|
# DEPLOY_SUDO=1 use passwordless sudo for remote files and rsync
|
|
# DEPLOY_NO_CLAIM=1 skip the ops-log claim (see below — use sparingly)
|
|
# DEPLOY_CLAIM_TTL=<dur> how long the claim stays live (default 30m)
|
|
#
|
|
# OPS LOG + CLAIM (added 2026-09-19)
|
|
# `infra-ops` and `infra-hermes` are two agents sharing ONE OS identity, so
|
|
# host-side changes are otherwise fingerprint-less. Before touching the
|
|
# host this script claims <host>/<stack> via scripts/ops-log and holds the
|
|
# claim across the diff, the y/N prompt and the apply — that whole window is
|
|
# where the 2026-09-18 searxng collision happened, not just the rsync. It
|
|
# then records what it pushed.
|
|
# A REFUSAL (another agent holds the claim) is fatal. ops-log being absent
|
|
# or broken is NOT: the deploy path must not gain a new single point of
|
|
# failure just because it grew an audit trail.
|
|
|
|
set -euo pipefail
|
|
|
|
if ! command -v rsync >/dev/null 2>&1; then
|
|
echo "error: rsync is not installed on this workstation" >&2
|
|
echo " install it (e.g. 'sudo apt install rsync') and ensure the target host has it too" >&2
|
|
exit 2
|
|
fi
|
|
|
|
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
|
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
|
SERVERS_DIR="$REPO_ROOT/servers"
|
|
STACKS_DIR="$REPO_ROOT/stacks"
|
|
|
|
EXCLUDES=(
|
|
# Include .env.example / *.env.example templates before the broader
|
|
# .env* exclude — rsync processes these in order, first match wins.
|
|
--include='.env.example'
|
|
--include='*.env.example'
|
|
--exclude=.env
|
|
--exclude='.env.*'
|
|
--exclude=acme.json
|
|
--exclude=client_secrets.json
|
|
--exclude='*.pem'
|
|
--exclude='*.key'
|
|
--exclude='*.crt'
|
|
--exclude='*.pfx'
|
|
--exclude='*.sqlite'
|
|
--exclude='*.sqlite3'
|
|
--exclude='*.db'
|
|
--exclude='*.log'
|
|
--exclude='*.log.*'
|
|
--exclude='*.pid'
|
|
--exclude='hub/'
|
|
--exclude='logs/'
|
|
)
|
|
|
|
HOST=
|
|
STACK=
|
|
ASSUME_YES=0
|
|
DO_COMPOSE=1
|
|
DO_CONF=1
|
|
DEST_STACK=${DEPLOY_DEST_STACK:-}
|
|
for a in "$@"; do
|
|
case "$a" in
|
|
--yes|-y) ASSUME_YES=1 ;;
|
|
--compose) DO_CONF=0 ;;
|
|
--conf) DO_COMPOSE=0 ;;
|
|
-h|--help) sed -n '2,43p' "$0"; exit 0 ;;
|
|
-*) echo "error: unknown flag $a" >&2; exit 2 ;;
|
|
*)
|
|
if [ -z "$HOST" ]; then HOST="$a"
|
|
elif [ -z "$STACK" ]; then STACK="$a"
|
|
else echo "error: unexpected positional '$a'" >&2; exit 2
|
|
fi
|
|
;;
|
|
esac
|
|
done
|
|
|
|
[ -n "$HOST" ] || { echo "usage: $(basename "$0") <host> <stack>" >&2; exit 2; }
|
|
[ -n "$STACK" ] || { echo "usage: $(basename "$0") <host> <stack>" >&2; exit 2; }
|
|
DEST_STACK=${DEST_STACK:-$STACK}
|
|
# Dots ARE legal — several stacks carry a version in the name (qwen3.5-122b,
|
|
# qwopus3.5-122b, mistral-medium-3.5). The previous pattern excluded them, so
|
|
# those stacks could not be deployed by this script AT ALL; it surfaced on
|
|
# 2026-09-13 as "invalid DEPLOY_DEST_STACK" during the fv-ml1 label repoint.
|
|
# The check exists to stop path traversal and shell metacharacters, which means
|
|
# it has to reject `..` and `/` — not every dot.
|
|
case "$DEST_STACK" in
|
|
*..*|*/*) echo "invalid stack name (path traversal): $DEST_STACK" >&2; exit 2 ;;
|
|
esac
|
|
[[ "$DEST_STACK" =~ ^[a-zA-Z0-9][a-zA-Z0-9_.-]*$ ]] || { echo "invalid stack name: $DEST_STACK" >&2; exit 2; }
|
|
|
|
# --------- Claim the stack before any remote work. --------------------
|
|
OPS_LOG="$SCRIPT_DIR/ops-log"
|
|
CLAIMED=0
|
|
release_claim() {
|
|
if [ "$CLAIMED" -eq 1 ]; then
|
|
"$OPS_LOG" release "$HOST" "$STACK" -q >/dev/null 2>&1 || true
|
|
CLAIMED=0
|
|
fi
|
|
return 0
|
|
}
|
|
trap release_claim EXIT
|
|
|
|
if [ -x "$OPS_LOG" ] && [ "${DEPLOY_NO_CLAIM:-0}" != 1 ]; then
|
|
claim_rc=0
|
|
"$OPS_LOG" claim "$HOST" "$STACK" --ttl "${DEPLOY_CLAIM_TTL:-30m}" \
|
|
--why "deploy-stack.sh $HOST $STACK" -q || claim_rc=$?
|
|
case "$claim_rc" in
|
|
0) CLAIMED=1 ;;
|
|
10) # We only refreshed a claim this agent already held for a wider
|
|
# operation. Do NOT release it on the way out — dropping someone's
|
|
# multi-step claim mid-operation is exactly the exposure the claim
|
|
# exists to prevent.
|
|
CLAIMED=0
|
|
echo "note: $HOST/$STACK was already claimed by you — leaving that claim in place." ;;
|
|
3) echo "error: refused — see the claim above. Wait for the holder, coordinate" >&2
|
|
echo " on althing, or override with DEPLOY_NO_CLAIM=1 if it is dead." >&2
|
|
exit 3 ;;
|
|
*) echo "warning: ops-log claim failed (exit $claim_rc) — deploying UNCLAIMED." >&2 ;;
|
|
esac
|
|
fi
|
|
|
|
resolve_target() {
|
|
# ssh-target file wins when present (may carry user@ or non-default port);
|
|
# /etc/hosts + ssh_config is the fallback.
|
|
local host="$1"
|
|
local fb="$SERVERS_DIR/$host/ssh-target"
|
|
if [ -f "$fb" ]; then
|
|
local t
|
|
t=$(awk 'NF{print $1; exit}' "$fb")
|
|
if [ -n "$t" ]; then echo "$t"; return; fi
|
|
fi
|
|
local effective
|
|
effective=$(ssh -G "$host" 2>/dev/null | awk '/^hostname /{print $2; exit}')
|
|
if [ -n "$effective" ] && getent hosts "$effective" >/dev/null 2>&1; then
|
|
echo "$host"; return
|
|
fi
|
|
echo "$host"
|
|
}
|
|
|
|
TARGET=$(resolve_target "$HOST")
|
|
REMOTE_PREFIX=
|
|
RSYNC_REMOTE=()
|
|
if [ "${DEPLOY_SUDO:-0}" = 1 ]; then
|
|
REMOTE_PREFIX='sudo -n '
|
|
RSYNC_REMOTE=(--rsync-path='sudo -n rsync')
|
|
fi
|
|
STACK_DIR="$STACKS_DIR/$STACK"
|
|
|
|
[ -d "$STACK_DIR" ] || { echo "error: $STACK_DIR not found — author the canonical stack first (see stacks/<other>/ for examples)" >&2; exit 2; }
|
|
|
|
# Collect the two src/dest pairs we need to consider.
|
|
PAIRS=() # each entry: "<kind>|<src>|<dest>"
|
|
if [ "$DO_COMPOSE" -eq 1 ]; then
|
|
PAIRS+=("compose|$STACK_DIR/|$TARGET:/opt/docker/compose/$DEST_STACK/")
|
|
fi
|
|
if [ "$DO_CONF" -eq 1 ] && [ -d "$STACK_DIR/conf" ]; then
|
|
PAIRS+=("conf|$STACK_DIR/conf/|$TARGET:/opt/docker/conf/$DEST_STACK/")
|
|
fi
|
|
|
|
[ "${#PAIRS[@]}" -gt 0 ] || { echo "nothing to deploy"; exit 0; }
|
|
|
|
# --------- Dry-run summary: which files would change, per kind. --------
|
|
declare -A CHANGED_FILES_BY_KIND=() # kind → newline-separated list
|
|
declare -A DELETED_FILES_BY_KIND=()
|
|
declare -A RAW_RSYNC_OUT_BY_KIND=() # kind → raw rsync itemize output
|
|
any_change=0
|
|
|
|
for entry in "${PAIRS[@]}"; do
|
|
IFS='|' read -r kind src dest <<<"$entry"
|
|
extra=()
|
|
[ "$kind" = compose ] && extra+=(--exclude='conf/')
|
|
|
|
# Pre-create the remote dir. Without this, rsync against a nonexistent
|
|
# destination can fail in ways the dry-run doesn't surface cleanly.
|
|
remote_path="/opt/docker/$kind/$DEST_STACK/"
|
|
if ! ssh -n -o BatchMode=yes -o ConnectTimeout=10 "$TARGET" \
|
|
"${REMOTE_PREFIX}mkdir -p '$remote_path'" 2>/dev/null; then
|
|
echo "error: could not create $remote_path on $TARGET (check perms / ssh)" >&2
|
|
exit 2
|
|
fi
|
|
|
|
tmp_out=$(mktemp) tmp_err=$(mktemp)
|
|
rc=0
|
|
# THE RULE: the deploy syncs CONTENT; the conventions own METADATA.
|
|
#
|
|
# --no-o --no-g --no-perms --omit-dir-times. The deploy tree is
|
|
# root:docker 2775 (setgid) since the
|
|
# 2026-09-14 fleet normalisation, and plain -a makes rsync try to chgrp the
|
|
# destination as the deploy identity, which is not root. It fails with
|
|
# "chgrp ... Operation not permitted" and exits 23 AFTER transferring the
|
|
# content — a loud error on a deploy that actually succeeded. The setgid bit
|
|
# already assigns the right group, so rsync should not be fighting it.
|
|
# This was fixed three times in one session before the rule above was
|
|
# written down, because `-a` is `-rlptgoD` and a non-root identity cannot
|
|
# apply ANY of owner, group, permissions or times to a root-owned directory.
|
|
# Each patch fixed one letter and the next deploy failed on the next one:
|
|
# chgrp failed -> --no-o --no-g
|
|
# failed to set times -> --omit-dir-times
|
|
# failed to set perms -> --no-perms
|
|
# Every one of them exited 23 AFTER transferring the content — a loud error
|
|
# on a deploy that had succeeded — so each partial fix looked complete until
|
|
# the next run. Enumerate the flag set, don't chase the symptom.
|
|
#
|
|
# ⚠ --no-perms means a NEW file lands with the remote umask rather than the
|
|
# source's mode, so a stack shipping an executable script needs its +x set by
|
|
# the convention (playbooks/normalize-docker-tree.yaml preserves exec bits),
|
|
# not by the deploy. Existing files keep their modes.
|
|
rsync -az --no-o --no-g --no-perms --omit-dir-times --delete --dry-run \
|
|
"${RSYNC_REMOTE[@]}" \
|
|
--out-format='%i %n' \
|
|
"${EXCLUDES[@]}" "${extra[@]}" \
|
|
"$src" "$dest" >"$tmp_out" 2>"$tmp_err" || rc=$?
|
|
if [ "$rc" -ne 0 ]; then
|
|
echo "error: rsync dry-run failed (exit $rc) for $src → $dest" >&2
|
|
sed 's/^/ /' "$tmp_err" >&2
|
|
rm -f "$tmp_out" "$tmp_err"
|
|
exit 2
|
|
fi
|
|
mapfile -t lines < "$tmp_out"
|
|
RAW_RSYNC_OUT_BY_KIND[$kind]=$(cat "$tmp_out")
|
|
rm -f "$tmp_out" "$tmp_err"
|
|
|
|
changed=""
|
|
deleted=""
|
|
for ln in "${lines[@]}"; do
|
|
# Itemized codes (rsync uses '<' for push, '>' for pull):
|
|
# <f+++++++++ newfile (new file, push)
|
|
# >f+++++++++ newfile (new file, pull)
|
|
# <f..t...... file.yaml (content update, push)
|
|
# *deleting oldfile (deletion, either direction)
|
|
# .d..t...... ./ (metadata on a dir — skip)
|
|
# cd+++++++++ somedir/ (new dir — skip, we diff files only)
|
|
[ -z "$ln" ] && continue
|
|
code=$(awk '{print $1}' <<<"$ln")
|
|
name=$(awk '{ $1=""; sub(/^ /,""); print }' <<<"$ln")
|
|
[ -z "$name" ] && continue
|
|
[ "${name: -1}" = "/" ] && continue # directory entry
|
|
|
|
case "$code" in
|
|
'*deleting') deleted+="$name"$'\n' ;;
|
|
'<f'*|'>f'*) changed+="$name"$'\n' ;;
|
|
*) : ;; # dir entries, metadata-only, unknown
|
|
esac
|
|
done
|
|
CHANGED_FILES_BY_KIND[$kind]="$changed"
|
|
DELETED_FILES_BY_KIND[$kind]="$deleted"
|
|
if [ -n "$changed$deleted" ]; then any_change=1; fi
|
|
done
|
|
|
|
if [ "$any_change" -eq 0 ]; then
|
|
echo "up to date: $HOST/$STACK is already in sync with server."
|
|
# Diagnostic: if the remote dir actually looks empty, we may have been
|
|
# fooled by an rsync quirk — dump what rsync saw so the user can tell.
|
|
for entry in "${PAIRS[@]}"; do
|
|
IFS='|' read -r kind _ _ <<<"$entry"
|
|
raw=${RAW_RSYNC_OUT_BY_KIND[$kind]:-}
|
|
remote_path="/opt/docker/$kind/$DEST_STACK/"
|
|
remote_count=$(ssh -n -o BatchMode=yes "$TARGET" \
|
|
"${REMOTE_PREFIX}find '$remote_path' -mindepth 1 -maxdepth 1 2>/dev/null | wc -l" \
|
|
2>/dev/null || echo "?")
|
|
printf ' %s: remote has %s entries, rsync itemize output:\n' "$kind" "$remote_count"
|
|
if [ -z "$raw" ]; then
|
|
printf ' (empty — rsync reported no work)\n'
|
|
else
|
|
sed 's/^/ /' <<<"$raw"
|
|
fi
|
|
done
|
|
exit 0
|
|
fi
|
|
|
|
# --------- Print diffs. -----------------------------------------------
|
|
divider() { printf '\n%s\n' "------------------------------------------------------------"; }
|
|
|
|
for entry in "${PAIRS[@]}"; do
|
|
IFS='|' read -r kind src dest <<<"$entry"
|
|
remote_base="/opt/docker/$kind/$DEST_STACK"
|
|
changed=${CHANGED_FILES_BY_KIND[$kind]:-}
|
|
deleted=${DELETED_FILES_BY_KIND[$kind]:-}
|
|
[ -z "$changed$deleted" ] && continue
|
|
|
|
printf '\n=== %s → %s ===\n' "$src" "$dest"
|
|
|
|
# ssh below runs with -n: without it, ssh slurps the loop's heredoc
|
|
# stdin ($changed / $deleted) and eats the remaining iterations,
|
|
# silently truncating the diff output to the first file only.
|
|
while IFS= read -r rel; do
|
|
[ -z "$rel" ] && continue
|
|
local_file="$src$rel"
|
|
remote_file="$remote_base/$rel"
|
|
divider
|
|
if ssh -n -o BatchMode=yes "$TARGET" "${REMOTE_PREFIX}test -f '$remote_file'" 2>/dev/null; then
|
|
printf 'MODIFY %s\n' "$rel"
|
|
diff -u --label "a/$rel (remote)" --label "b/$rel (local)" \
|
|
<(ssh -n -o BatchMode=yes "$TARGET" "${REMOTE_PREFIX}cat '$remote_file'" 2>/dev/null) \
|
|
"$local_file" || true
|
|
else
|
|
printf 'ADD %s\n' "$rel"
|
|
diff -u --label /dev/null --label "b/$rel (local)" \
|
|
/dev/null "$local_file" || true
|
|
fi
|
|
done <<<"$changed"
|
|
|
|
while IFS= read -r rel; do
|
|
[ -z "$rel" ] && continue
|
|
remote_file="$remote_base/$rel"
|
|
divider
|
|
printf 'DELETE %s\n' "$rel"
|
|
diff -u --label "a/$rel (remote)" --label /dev/null \
|
|
<(ssh -n -o BatchMode=yes "$TARGET" "${REMOTE_PREFIX}cat '$remote_file'" 2>/dev/null) \
|
|
/dev/null || true
|
|
done <<<"$deleted"
|
|
done
|
|
|
|
divider
|
|
|
|
if [ "$ASSUME_YES" -ne 1 ]; then
|
|
read -r -p "Apply these changes to $TARGET? [y/N] " ans
|
|
case "$ans" in
|
|
y|Y|yes|YES) ;;
|
|
*) echo "aborted."; exit 1 ;;
|
|
esac
|
|
fi
|
|
|
|
# --------- Apply. -----------------------------------------------------
|
|
for entry in "${PAIRS[@]}"; do
|
|
IFS='|' read -r kind src dest <<<"$entry"
|
|
extra=()
|
|
[ "$kind" = compose ] && extra+=(--exclude='conf/')
|
|
printf 'pushing %s → %s\n' "$src" "$dest"
|
|
rsync -az --no-o --no-g --no-perms --omit-dir-times --delete \
|
|
"${RSYNC_REMOTE[@]}" \
|
|
"${EXCLUDES[@]}" "${extra[@]}" \
|
|
"$src" "$dest"
|
|
done
|
|
|
|
# --------- Record what we just did. -----------------------------------
|
|
# Counted from the dry-run itemize, which is what the operator actually
|
|
# reviewed and approved — not re-derived after the fact.
|
|
if [ -x "$OPS_LOG" ]; then
|
|
summary=""
|
|
for entry in "${PAIRS[@]}"; do
|
|
IFS='|' read -r kind _ _ <<<"$entry"
|
|
n_ch=$(grep -c . <<<"${CHANGED_FILES_BY_KIND[$kind]:-}" || true)
|
|
n_del=$(grep -c . <<<"${DELETED_FILES_BY_KIND[$kind]:-}" || true)
|
|
summary+="${summary:+, }$kind ${n_ch:-0} changed/${n_del:-0} deleted"
|
|
done
|
|
[ "$DEST_STACK" != "$STACK" ] && summary+=" (remote dir $DEST_STACK)"
|
|
"$OPS_LOG" record --host "$HOST" --action deploy-stack --target "$STACK" \
|
|
--outcome changed --detail "$summary" -q || true
|
|
fi
|
|
|
|
echo "done."
|