Initial commit: PFI fleet inventory, stacks, tooling, and backup pipeline
Captures the full workspace state built up to this point:
- CLAUDE.md + README.md describing conventions and the four-host fleet
(ana-ml2, ana-docker, nh3-docker, esh-docker-vm).
- Per-host notes under servers/<host>/ with ssh-target fallback files
and latest system-details snapshots (two in-compose credential leaks
scrubbed; the upstream compose files still need to move those to .env).
- scripts/: server_inspect.sh (read-only remote diagnostic),
refresh-server-info.sh (dir-driven discovery + snapshot capture with
validation warnings), add-host.sh, sync-stacks.sh (pull
compose/conf trees), deploy-stack.sh (push with per-file diff + prompt).
- stacks/: canonical compose for backrest, beszel, dozzle, llama-swap,
rest-server-ana, rest-server-nh3, vllm-qwen3, plus the retired
infinity reference. All use the .env-driven + traefik-net + homepage
label pattern.
- configs/restic/ana-docker/: first resticprofile config + pre-backup
hook (Synapse pg_dump, Seafile mysqldump, Vaultwarden SQLite); templates
for the other three hosts to come.
- docs/pfi/: general infrastructure reference carried over.
- .gitignore excludes .env, stacks-mirror/, and assorted secret/state
filenames to prevent re-leaks on later commits.
This commit is contained in:
Executable
+69
@@ -0,0 +1,69 @@
|
||||
#!/usr/bin/env bash
|
||||
# add-host.sh — register a new server so refresh-server-info.sh picks it up.
|
||||
#
|
||||
# Creates servers/<name>/ and writes servers/<name>/ssh-target with the
|
||||
# given IP (or user@ip). The next `scripts/refresh-server-info.sh <name>`
|
||||
# run will discover the host and pull its first system-details.txt.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/add-host.sh <name> <ip-or-user@ip>
|
||||
# scripts/add-host.sh <name> <ip-or-user@ip> --force # overwrite existing
|
||||
#
|
||||
# Example:
|
||||
# scripts/add-host.sh la-docker 10.60.50.12
|
||||
# scripts/add-host.sh edge-01 admin@203.0.113.9
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
SERVERS_DIR="$REPO_ROOT/servers"
|
||||
|
||||
FORCE=0
|
||||
POSITIONAL=()
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
-f|--force) FORCE=1 ;;
|
||||
-h|--help) sed -n '2,14p' "$0"; exit 0 ;;
|
||||
-*) echo "error: unknown flag $arg" >&2; exit 2 ;;
|
||||
*) POSITIONAL+=("$arg") ;;
|
||||
esac
|
||||
done
|
||||
|
||||
if [ "${#POSITIONAL[@]}" -ne 2 ]; then
|
||||
echo "usage: $(basename "$0") <name> <ip-or-user@ip> [--force]" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
name="${POSITIONAL[0]}"
|
||||
target="${POSITIONAL[1]}"
|
||||
|
||||
if [[ ! "$name" =~ ^[a-zA-Z0-9][a-zA-Z0-9._-]*$ ]]; then
|
||||
echo "error: invalid host name '$name' (expected [a-zA-Z0-9._-])" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
# Trivial sanity check on the target — not exhaustive, just catches typos.
|
||||
if [[ -z "$target" ]] || [[ "$target" =~ [[:space:]] ]]; then
|
||||
echo "error: invalid ssh target '$target'" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
host_dir="$SERVERS_DIR/$name"
|
||||
ssh_target_file="$host_dir/ssh-target"
|
||||
|
||||
if [ -f "$ssh_target_file" ] && [ "$FORCE" -ne 1 ]; then
|
||||
existing=$(awk 'NF{print $1; exit}' "$ssh_target_file")
|
||||
if [ "$existing" = "$target" ]; then
|
||||
echo "host '$name' already registered with target '$target' — nothing to do"
|
||||
exit 0
|
||||
fi
|
||||
echo "error: $ssh_target_file already exists (currently '$existing'); pass --force to overwrite" >&2
|
||||
exit 1
|
||||
fi
|
||||
|
||||
mkdir -p "$host_dir"
|
||||
printf '%s\n' "$target" > "$ssh_target_file"
|
||||
|
||||
printf 'registered: %s → %s\n' "$name" "$target"
|
||||
printf 'next: scripts/refresh-server-info.sh %s\n' "$name"
|
||||
Executable
+257
@@ -0,0 +1,257 @@
|
||||
#!/usr/bin/env bash
|
||||
# deploy-stack.sh — push a local stacks-mirror dir to a server, with
|
||||
# per-file diff and confirmation prompt.
|
||||
#
|
||||
# Layout assumed:
|
||||
# stacks-mirror/<host>/<stack>/<file> → <host>:/opt/docker/compose/<stack>/<file>
|
||||
# stacks-mirror/<host>/<stack>/conf/<file> → <host>:/opt/docker/conf/<stack>/<file>
|
||||
#
|
||||
# Secrets / runtime state are never pushed (same exclude list as
|
||||
# sync-stacks.sh): .env*, acme.json, *.key/crt/pem/pfx, *.sqlite*, *.db,
|
||||
# *.log*, *.pid, hub/, logs/, client_secrets.json.
|
||||
#
|
||||
# The script:
|
||||
# 1. Runs rsync --dry-run to find which files would change.
|
||||
# 2. Prints a unified diff for each changed/added file (deletions noted).
|
||||
# 3. Prompts [y/N]; applies the rsync only on 'y'.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/deploy-stack.sh <host> <stack>
|
||||
# scripts/deploy-stack.sh <host> <stack> --yes # skip prompt (use sparingly)
|
||||
# scripts/deploy-stack.sh <host> <stack> --compose # push only compose side
|
||||
# scripts/deploy-stack.sh <host> <stack> --conf # push only conf side
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
if ! command -v rsync >/dev/null 2>&1; then
|
||||
echo "error: rsync is not installed on this workstation" >&2
|
||||
echo " install it (e.g. 'sudo apt install rsync') and ensure the target host has it too" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
SERVERS_DIR="$REPO_ROOT/servers"
|
||||
MIRROR_DIR="$REPO_ROOT/stacks-mirror"
|
||||
|
||||
EXCLUDES=(
|
||||
# Include .env.example / *.env.example templates before the broader
|
||||
# .env* exclude — rsync processes these in order, first match wins.
|
||||
--include='.env.example'
|
||||
--include='*.env.example'
|
||||
--exclude=.env
|
||||
--exclude='.env.*'
|
||||
--exclude=acme.json
|
||||
--exclude=client_secrets.json
|
||||
--exclude='*.pem'
|
||||
--exclude='*.key'
|
||||
--exclude='*.crt'
|
||||
--exclude='*.pfx'
|
||||
--exclude='*.sqlite'
|
||||
--exclude='*.sqlite3'
|
||||
--exclude='*.db'
|
||||
--exclude='*.log'
|
||||
--exclude='*.log.*'
|
||||
--exclude='*.pid'
|
||||
--exclude='hub/'
|
||||
--exclude='logs/'
|
||||
)
|
||||
|
||||
HOST=
|
||||
STACK=
|
||||
ASSUME_YES=0
|
||||
DO_COMPOSE=1
|
||||
DO_CONF=1
|
||||
for a in "$@"; do
|
||||
case "$a" in
|
||||
--yes|-y) ASSUME_YES=1 ;;
|
||||
--compose) DO_CONF=0 ;;
|
||||
--conf) DO_COMPOSE=0 ;;
|
||||
-h|--help) sed -n '2,22p' "$0"; exit 0 ;;
|
||||
-*) echo "error: unknown flag $a" >&2; exit 2 ;;
|
||||
*)
|
||||
if [ -z "$HOST" ]; then HOST="$a"
|
||||
elif [ -z "$STACK" ]; then STACK="$a"
|
||||
else echo "error: unexpected positional '$a'" >&2; exit 2
|
||||
fi
|
||||
;;
|
||||
esac
|
||||
done
|
||||
|
||||
[ -n "$HOST" ] || { echo "usage: $(basename "$0") <host> <stack>" >&2; exit 2; }
|
||||
[ -n "$STACK" ] || { echo "usage: $(basename "$0") <host> <stack>" >&2; exit 2; }
|
||||
|
||||
resolve_target() {
|
||||
local host="$1"
|
||||
local effective
|
||||
effective=$(ssh -G "$host" 2>/dev/null | awk '/^hostname /{print $2; exit}')
|
||||
if [ -n "$effective" ] && getent hosts "$effective" >/dev/null 2>&1; then
|
||||
echo "$host"; return
|
||||
fi
|
||||
local fb="$SERVERS_DIR/$host/ssh-target"
|
||||
if [ -f "$fb" ]; then awk 'NF{print $1; exit}' "$fb"; return; fi
|
||||
echo "$host"
|
||||
}
|
||||
|
||||
TARGET=$(resolve_target "$HOST")
|
||||
STACK_DIR="$MIRROR_DIR/$HOST/$STACK"
|
||||
|
||||
[ -d "$STACK_DIR" ] || { echo "error: $STACK_DIR not found (pull with sync-stacks.sh first)" >&2; exit 2; }
|
||||
|
||||
# Collect the two src/dest pairs we need to consider.
|
||||
PAIRS=() # each entry: "<kind>|<src>|<dest>"
|
||||
if [ "$DO_COMPOSE" -eq 1 ]; then
|
||||
PAIRS+=("compose|$STACK_DIR/|$TARGET:/opt/docker/compose/$STACK/")
|
||||
fi
|
||||
if [ "$DO_CONF" -eq 1 ] && [ -d "$STACK_DIR/conf" ]; then
|
||||
PAIRS+=("conf|$STACK_DIR/conf/|$TARGET:/opt/docker/conf/$STACK/")
|
||||
fi
|
||||
|
||||
[ "${#PAIRS[@]}" -gt 0 ] || { echo "nothing to deploy"; exit 0; }
|
||||
|
||||
# --------- Dry-run summary: which files would change, per kind. --------
|
||||
declare -A CHANGED_FILES_BY_KIND=() # kind → newline-separated list
|
||||
declare -A DELETED_FILES_BY_KIND=()
|
||||
declare -A RAW_RSYNC_OUT_BY_KIND=() # kind → raw rsync itemize output
|
||||
any_change=0
|
||||
|
||||
for entry in "${PAIRS[@]}"; do
|
||||
IFS='|' read -r kind src dest <<<"$entry"
|
||||
extra=()
|
||||
[ "$kind" = compose ] && extra+=(--exclude='conf/')
|
||||
|
||||
# Pre-create the remote dir. Without this, rsync against a nonexistent
|
||||
# destination can fail in ways the dry-run doesn't surface cleanly.
|
||||
remote_path="/opt/docker/$kind/$STACK/"
|
||||
if ! ssh -o BatchMode=yes -o ConnectTimeout=10 "$TARGET" \
|
||||
"mkdir -p '$remote_path'" 2>/dev/null; then
|
||||
echo "error: could not create $remote_path on $TARGET (check perms / ssh)" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
tmp_out=$(mktemp) tmp_err=$(mktemp)
|
||||
rc=0
|
||||
rsync -az --delete --dry-run \
|
||||
--out-format='%i %n' \
|
||||
"${EXCLUDES[@]}" "${extra[@]}" \
|
||||
"$src" "$dest" >"$tmp_out" 2>"$tmp_err" || rc=$?
|
||||
if [ "$rc" -ne 0 ]; then
|
||||
echo "error: rsync dry-run failed (exit $rc) for $src → $dest" >&2
|
||||
sed 's/^/ /' "$tmp_err" >&2
|
||||
rm -f "$tmp_out" "$tmp_err"
|
||||
exit 2
|
||||
fi
|
||||
mapfile -t lines < "$tmp_out"
|
||||
RAW_RSYNC_OUT_BY_KIND[$kind]=$(cat "$tmp_out")
|
||||
rm -f "$tmp_out" "$tmp_err"
|
||||
|
||||
changed=""
|
||||
deleted=""
|
||||
for ln in "${lines[@]}"; do
|
||||
# Itemized codes (rsync uses '<' for push, '>' for pull):
|
||||
# <f+++++++++ newfile (new file, push)
|
||||
# >f+++++++++ newfile (new file, pull)
|
||||
# <f..t...... file.yaml (content update, push)
|
||||
# *deleting oldfile (deletion, either direction)
|
||||
# .d..t...... ./ (metadata on a dir — skip)
|
||||
# cd+++++++++ somedir/ (new dir — skip, we diff files only)
|
||||
[ -z "$ln" ] && continue
|
||||
code=$(awk '{print $1}' <<<"$ln")
|
||||
name=$(awk '{ $1=""; sub(/^ /,""); print }' <<<"$ln")
|
||||
[ -z "$name" ] && continue
|
||||
[ "${name: -1}" = "/" ] && continue # directory entry
|
||||
|
||||
case "$code" in
|
||||
'*deleting') deleted+="$name"$'\n' ;;
|
||||
'<f'*|'>f'*) changed+="$name"$'\n' ;;
|
||||
*) : ;; # dir entries, metadata-only, unknown
|
||||
esac
|
||||
done
|
||||
CHANGED_FILES_BY_KIND[$kind]="$changed"
|
||||
DELETED_FILES_BY_KIND[$kind]="$deleted"
|
||||
if [ -n "$changed$deleted" ]; then any_change=1; fi
|
||||
done
|
||||
|
||||
if [ "$any_change" -eq 0 ]; then
|
||||
echo "up to date: $HOST/$STACK is already in sync with server."
|
||||
# Diagnostic: if the remote dir actually looks empty, we may have been
|
||||
# fooled by an rsync quirk — dump what rsync saw so the user can tell.
|
||||
for entry in "${PAIRS[@]}"; do
|
||||
IFS='|' read -r kind _ _ <<<"$entry"
|
||||
raw=${RAW_RSYNC_OUT_BY_KIND[$kind]:-}
|
||||
remote_path="/opt/docker/$kind/$STACK/"
|
||||
remote_count=$(ssh -o BatchMode=yes "$TARGET" \
|
||||
"find '$remote_path' -mindepth 1 -maxdepth 1 2>/dev/null | wc -l" \
|
||||
2>/dev/null || echo "?")
|
||||
printf ' %s: remote has %s entries, rsync itemize output:\n' "$kind" "$remote_count"
|
||||
if [ -z "$raw" ]; then
|
||||
printf ' (empty — rsync reported no work)\n'
|
||||
else
|
||||
sed 's/^/ /' <<<"$raw"
|
||||
fi
|
||||
done
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# --------- Print diffs. -----------------------------------------------
|
||||
divider() { printf '\n%s\n' "------------------------------------------------------------"; }
|
||||
|
||||
for entry in "${PAIRS[@]}"; do
|
||||
IFS='|' read -r kind src dest <<<"$entry"
|
||||
remote_base="/opt/docker/$kind/$STACK"
|
||||
changed=${CHANGED_FILES_BY_KIND[$kind]:-}
|
||||
deleted=${DELETED_FILES_BY_KIND[$kind]:-}
|
||||
[ -z "$changed$deleted" ] && continue
|
||||
|
||||
printf '\n=== %s → %s ===\n' "$src" "$dest"
|
||||
|
||||
while IFS= read -r rel; do
|
||||
[ -z "$rel" ] && continue
|
||||
local_file="$src$rel"
|
||||
remote_file="$remote_base/$rel"
|
||||
divider
|
||||
if ssh -o BatchMode=yes "$TARGET" "[ -f '$remote_file' ]" 2>/dev/null; then
|
||||
printf 'MODIFY %s\n' "$rel"
|
||||
diff -u --label "a/$rel (remote)" --label "b/$rel (local)" \
|
||||
<(ssh -o BatchMode=yes "$TARGET" "cat '$remote_file'" 2>/dev/null) \
|
||||
"$local_file" || true
|
||||
else
|
||||
printf 'ADD %s\n' "$rel"
|
||||
diff -u --label /dev/null --label "b/$rel (local)" \
|
||||
/dev/null "$local_file" || true
|
||||
fi
|
||||
done <<<"$changed"
|
||||
|
||||
while IFS= read -r rel; do
|
||||
[ -z "$rel" ] && continue
|
||||
remote_file="$remote_base/$rel"
|
||||
divider
|
||||
printf 'DELETE %s\n' "$rel"
|
||||
diff -u --label "a/$rel (remote)" --label /dev/null \
|
||||
<(ssh -o BatchMode=yes "$TARGET" "cat '$remote_file'" 2>/dev/null) \
|
||||
/dev/null || true
|
||||
done <<<"$deleted"
|
||||
done
|
||||
|
||||
divider
|
||||
|
||||
if [ "$ASSUME_YES" -ne 1 ]; then
|
||||
read -r -p "Apply these changes to $TARGET? [y/N] " ans
|
||||
case "$ans" in
|
||||
y|Y|yes|YES) ;;
|
||||
*) echo "aborted."; exit 1 ;;
|
||||
esac
|
||||
fi
|
||||
|
||||
# --------- Apply. -----------------------------------------------------
|
||||
for entry in "${PAIRS[@]}"; do
|
||||
IFS='|' read -r kind src dest <<<"$entry"
|
||||
extra=()
|
||||
[ "$kind" = compose ] && extra+=(--exclude='conf/')
|
||||
printf 'pushing %s → %s\n' "$src" "$dest"
|
||||
rsync -az --delete \
|
||||
"${EXCLUDES[@]}" "${extra[@]}" \
|
||||
"$src" "$dest"
|
||||
done
|
||||
|
||||
echo "done."
|
||||
Executable
+260
@@ -0,0 +1,260 @@
|
||||
#!/usr/bin/env bash
|
||||
# refresh-server-info.sh — pull a fresh system-details.txt from servers.
|
||||
#
|
||||
# Hosts are discovered by listing `servers/*/` directory names. Each name
|
||||
# is used as the SSH target, so put matching entries in ~/.ssh/config to
|
||||
# customize user / port / identity.
|
||||
#
|
||||
# Fallback: if the dir name doesn't resolve (and ssh_config doesn't rewrite
|
||||
# it), the script looks for `servers/<host>/ssh-target` (one line, containing
|
||||
# an IP or `user@ip`) and uses that instead. This keeps the tool working
|
||||
# from a fresh clone without requiring DNS or ssh_config setup.
|
||||
#
|
||||
# For each host:
|
||||
# 1. Run scripts/server_inspect.sh on the remote via `ssh <target> 'bash -s'`.
|
||||
# 2. Write output atomically to `servers/<host>/system-details.txt`.
|
||||
# A failed SSH/run never clobbers the previous good snapshot.
|
||||
#
|
||||
# Exit status is non-zero if any host failed.
|
||||
#
|
||||
# Usage:
|
||||
# scripts/refresh-server-info.sh show this help
|
||||
# scripts/refresh-server-info.sh all refresh every host
|
||||
# scripts/refresh-server-info.sh ana-docker refresh one host
|
||||
# scripts/refresh-server-info.sh ana-docker ana-ml2 refresh several
|
||||
# scripts/refresh-server-info.sh --dry-run all preview, no ssh
|
||||
# scripts/refresh-server-info.sh --validate-only all checks only
|
||||
# scripts/refresh-server-info.sh --validate-only <h> checks one host
|
||||
#
|
||||
# Discovery is validated before any ssh is attempted. Each host prints
|
||||
# an indented "! <reason>" line per warning (unreadable files, empty or
|
||||
# missing ssh-target, unresolvable name with no fallback, missing README
|
||||
# or system-details, etc). Warnings never block the refresh — they're
|
||||
# informational — but are surfaced so drift is visible.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
INSPECT="$SCRIPT_DIR/server_inspect.sh"
|
||||
SERVERS_DIR="$REPO_ROOT/servers"
|
||||
|
||||
if [ ! -f "$INSPECT" ]; then
|
||||
echo "error: $INSPECT not found" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
DRY_RUN=0
|
||||
VALIDATE_ONLY=0
|
||||
REQUESTED=()
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--dry-run) DRY_RUN=1 ;;
|
||||
--validate-only|--validate) VALIDATE_ONLY=1 ;;
|
||||
-h|--help)
|
||||
sed -n '2,32p' "$0"
|
||||
exit 0
|
||||
;;
|
||||
-*) echo "error: unknown flag $arg" >&2; exit 2 ;;
|
||||
*) REQUESTED+=("$arg") ;;
|
||||
esac
|
||||
done
|
||||
|
||||
discover_hosts() {
|
||||
find "$SERVERS_DIR" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' | sort
|
||||
}
|
||||
|
||||
# No positional args → show help. Fleet-wide operations must be explicit.
|
||||
if [ "${#REQUESTED[@]}" -eq 0 ]; then
|
||||
sed -n '2,32p' "$0"
|
||||
exit 0
|
||||
fi
|
||||
|
||||
# The literal keyword "all" expands to every discovered host. Using it as
|
||||
# a sentinel (rather than making "no args" mean "all") makes fleet-wide
|
||||
# runs deliberate — useful when you might otherwise hit 20 hosts by accident.
|
||||
if [ "${#REQUESTED[@]}" -eq 1 ] && [ "${REQUESTED[0]}" = "all" ]; then
|
||||
mapfile -t HOSTS < <(discover_hosts)
|
||||
else
|
||||
# Check for 'all' mixed with other names — almost certainly a mistake.
|
||||
for r in "${REQUESTED[@]}"; do
|
||||
if [ "$r" = "all" ]; then
|
||||
echo "error: 'all' must be the only argument when used" >&2
|
||||
exit 2
|
||||
fi
|
||||
done
|
||||
HOSTS=("${REQUESTED[@]}")
|
||||
fi
|
||||
|
||||
if [ "${#HOSTS[@]}" -eq 0 ]; then
|
||||
echo "error: no hosts found under $SERVERS_DIR" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
resolve_target() {
|
||||
# Prints the SSH target to use for a given dir name.
|
||||
# Preference order:
|
||||
# 1. The dir name — if ssh's effective hostname (after ssh_config) resolves.
|
||||
# 2. Contents of servers/<host>/ssh-target (first whitespace token).
|
||||
# 3. The dir name as-is — let ssh fail with its own error.
|
||||
local host="$1"
|
||||
local effective
|
||||
effective=$(ssh -G "$host" 2>/dev/null | awk '/^hostname /{print $2; exit}')
|
||||
if [ -n "$effective" ] && getent hosts "$effective" >/dev/null 2>&1; then
|
||||
echo "$host"
|
||||
return
|
||||
fi
|
||||
local fallback="$SERVERS_DIR/$host/ssh-target"
|
||||
if [ -f "$fallback" ]; then
|
||||
awk 'NF{print $1; exit}' "$fallback"
|
||||
return
|
||||
fi
|
||||
echo "$host"
|
||||
}
|
||||
|
||||
validate_host() {
|
||||
# Prints one warning per line. Empty output = clean.
|
||||
local host="$1"
|
||||
local dir="$SERVERS_DIR/$host"
|
||||
|
||||
if [ ! -d "$dir" ]; then
|
||||
printf '%s\n' "dir missing: $dir"
|
||||
return
|
||||
fi
|
||||
if [ ! -r "$dir" ] || [ ! -x "$dir" ]; then
|
||||
printf '%s\n' "dir not readable/searchable (check permissions)"
|
||||
return
|
||||
fi
|
||||
|
||||
local stf="$dir/ssh-target"
|
||||
if [ -e "$stf" ]; then
|
||||
if [ ! -f "$stf" ]; then
|
||||
printf '%s\n' "ssh-target is not a regular file"
|
||||
elif [ ! -r "$stf" ]; then
|
||||
printf '%s\n' "ssh-target not readable"
|
||||
elif [ ! -s "$stf" ]; then
|
||||
printf '%s\n' "ssh-target is empty"
|
||||
else
|
||||
local t
|
||||
t=$(awk 'NF{print $1; exit}' "$stf" 2>/dev/null || true)
|
||||
if [ -z "$t" ]; then
|
||||
printf '%s\n' "ssh-target has no non-blank content"
|
||||
elif [[ "$t" =~ [[:space:]] ]]; then
|
||||
printf '%s\n' "ssh-target first token contains whitespace ('$t')"
|
||||
fi
|
||||
fi
|
||||
fi
|
||||
|
||||
local effective=""
|
||||
effective=$(ssh -G "$host" 2>/dev/null | awk '/^hostname /{print $2; exit}')
|
||||
local resolves=0
|
||||
if [ -n "$effective" ] && getent hosts "$effective" >/dev/null 2>&1; then
|
||||
resolves=1
|
||||
fi
|
||||
if [ "$resolves" -eq 0 ] && [ ! -s "$stf" ]; then
|
||||
printf '%s\n' "name '$host' does not resolve and no ssh-target fallback present"
|
||||
fi
|
||||
|
||||
[ -f "$dir/README.md" ] || printf '%s\n' "README.md missing"
|
||||
if [ ! -f "$dir/system-details.txt" ]; then
|
||||
printf '%s\n' "system-details.txt missing (never refreshed)"
|
||||
else
|
||||
validate_snapshot "$dir/system-details.txt"
|
||||
fi
|
||||
}
|
||||
|
||||
validate_snapshot() {
|
||||
# Read the captured system-details.txt and yield a warning for each
|
||||
# known problem marker that indicates the remote inspect ran but didn't
|
||||
# have the rights / environment it needed.
|
||||
local file="$1"
|
||||
if grep -q 'docker daemon not reachable by current user' "$file"; then
|
||||
printf '%s\n' "remote user cannot reach docker daemon (add to 'docker' group or fix socket perms)"
|
||||
fi
|
||||
if grep -qE '^Server:[[:space:]]*$' "$file" \
|
||||
&& ! grep -q 'docker daemon not reachable by current user' "$file"; then
|
||||
printf '%s\n' "docker 'Server:' line is blank (daemon down or inaccessible)"
|
||||
fi
|
||||
if grep -q '^nvidia-smi present, but failed' "$file"; then
|
||||
printf '%s\n' "nvidia-smi failed on remote (driver broken or no permission)"
|
||||
fi
|
||||
if ! grep -q '^===== DONE =====' "$file"; then
|
||||
printf '%s\n' "snapshot appears truncated (missing trailing '===== DONE =====' marker)"
|
||||
fi
|
||||
}
|
||||
|
||||
print_warnings() {
|
||||
# $1 = indent, rest = warning lines
|
||||
local indent="$1"; shift
|
||||
local w
|
||||
for w in "$@"; do
|
||||
printf '%s! %s\n' "$indent" "$w"
|
||||
done
|
||||
}
|
||||
|
||||
pad=0
|
||||
for h in "${HOSTS[@]}"; do (( ${#h} > pad )) && pad=${#h}; done
|
||||
|
||||
if [ "$VALIDATE_ONLY" -eq 1 ]; then
|
||||
printf 'Validating %d host(s):\n' "${#HOSTS[@]}"
|
||||
total_warn=0
|
||||
for host in "${HOSTS[@]}"; do
|
||||
mapfile -t warnings < <(validate_host "$host")
|
||||
if [ "${#warnings[@]}" -eq 0 ]; then
|
||||
printf ' %-*s ok\n' "$pad" "$host"
|
||||
else
|
||||
printf ' %-*s %d warning(s)\n' "$pad" "$host" "${#warnings[@]}"
|
||||
print_warnings " " "${warnings[@]}"
|
||||
total_warn=$((total_warn + ${#warnings[@]}))
|
||||
fi
|
||||
done
|
||||
if [ "$total_warn" -gt 0 ]; then exit 1; fi
|
||||
exit 0
|
||||
fi
|
||||
|
||||
printf 'Refreshing %d host(s):\n' "${#HOSTS[@]}"
|
||||
failed=()
|
||||
for host in "${HOSTS[@]}"; do
|
||||
out="$SERVERS_DIR/$host/system-details.txt"
|
||||
tmp="$out.new"
|
||||
|
||||
mapfile -t warnings < <(validate_host "$host")
|
||||
|
||||
target=$(resolve_target "$host")
|
||||
if [ "$target" = "$host" ]; then
|
||||
label="$host"
|
||||
else
|
||||
label="$host → $target"
|
||||
fi
|
||||
|
||||
printf ' %-*s ' "$pad" "$label"
|
||||
|
||||
if [ "$DRY_RUN" -eq 1 ]; then
|
||||
printf 'would run: ssh %s bash -s < %s > %s\n' "$target" "$INSPECT" "$out"
|
||||
[ "${#warnings[@]}" -gt 0 ] && print_warnings " " "${warnings[@]}"
|
||||
continue
|
||||
fi
|
||||
|
||||
mkdir -p "$SERVERS_DIR/$host"
|
||||
|
||||
if ssh -o BatchMode=yes -o ConnectTimeout=10 "$target" 'bash -s' < "$INSPECT" > "$tmp" 2> "$tmp.err"; then
|
||||
mv "$tmp" "$out"
|
||||
rm -f "$tmp.err"
|
||||
bytes=$(wc -c < "$out")
|
||||
printf 'ok (%s bytes)\n' "$bytes"
|
||||
else
|
||||
rc=$?
|
||||
rm -f "$tmp"
|
||||
err=$(head -n 1 "$tmp.err" 2>/dev/null || true)
|
||||
rm -f "$tmp.err"
|
||||
printf 'FAIL (rc=%d) %s\n' "$rc" "$err"
|
||||
failed+=("$host")
|
||||
fi
|
||||
|
||||
[ "${#warnings[@]}" -gt 0 ] && print_warnings " " "${warnings[@]}"
|
||||
done
|
||||
|
||||
if [ "${#failed[@]}" -gt 0 ]; then
|
||||
printf '\n%d host(s) failed: %s\n' "${#failed[@]}" "${failed[*]}" >&2
|
||||
exit 1
|
||||
fi
|
||||
Executable
+166
@@ -0,0 +1,166 @@
|
||||
#!/usr/bin/env bash
|
||||
# server_inspect.sh — collect server details for writing Docker Compose files.
|
||||
#
|
||||
# Conventions this script assumes and reports on:
|
||||
# - Compose files: /opt/docker/compose/<stack>/{docker-compose.yml,compose.yaml}
|
||||
# - Config mounts: /opt/docker/conf/<stack>/...
|
||||
# - Named volumes preferred over bind mounts for persistent state.
|
||||
#
|
||||
# Safe: read-only. No modifications are made.
|
||||
# Usage:
|
||||
# bash server_inspect.sh # print to stdout
|
||||
# bash server_inspect.sh /tmp/report.txt # also save to file
|
||||
|
||||
set -u
|
||||
|
||||
OUT="${1:-}"
|
||||
if [ -n "$OUT" ]; then exec > >(tee "$OUT") 2>&1; fi
|
||||
|
||||
hr() { printf '\n===== %s =====\n\n' "$*"; }
|
||||
sub() { printf '\n----- %s -----\n' "$*"; }
|
||||
have() { command -v "$1" >/dev/null 2>&1; }
|
||||
|
||||
# --- HOST --------------------------------------------------------------------
|
||||
hr "HOST"
|
||||
echo "Hostname: $(hostname -f 2>/dev/null || hostname)"
|
||||
echo "Date: $(date -Iseconds)"
|
||||
echo "Uptime: $(uptime -p 2>/dev/null || uptime)"
|
||||
if [ -f /etc/os-release ]; then
|
||||
. /etc/os-release
|
||||
echo "OS: ${PRETTY_NAME:-unknown}"
|
||||
fi
|
||||
echo "Kernel: $(uname -r)"
|
||||
echo "Arch: $(uname -m)"
|
||||
|
||||
# --- HARDWARE ----------------------------------------------------------------
|
||||
hr "HARDWARE"
|
||||
if [ -f /proc/cpuinfo ]; then
|
||||
echo "CPU cores: $(grep -c ^processor /proc/cpuinfo)"
|
||||
echo "CPU model: $(awk -F: '/model name/ {print $2; exit}' /proc/cpuinfo | sed 's/^[[:space:]]*//')"
|
||||
fi
|
||||
if [ -f /proc/meminfo ]; then
|
||||
awk '/^MemTotal:|^MemAvailable:/ {printf "%-11s %.1f GB\n", $1, $2/1024/1024}' /proc/meminfo
|
||||
fi
|
||||
|
||||
# --- GPUS --------------------------------------------------------------------
|
||||
hr "GPUS"
|
||||
if have nvidia-smi; then
|
||||
nvidia-smi --query-gpu=index,name,memory.total,memory.free,driver_version --format=csv
|
||||
else
|
||||
echo "nvidia-smi not present (no NVIDIA GPUs or driver not installed)"
|
||||
fi
|
||||
|
||||
# --- FILESYSTEM --------------------------------------------------------------
|
||||
hr "FILESYSTEMS (df)"
|
||||
df -h -x tmpfs -x devtmpfs -x overlay 2>/dev/null
|
||||
|
||||
hr "PERSISTENT MOUNTS (/etc/fstab, non-comment)"
|
||||
if [ -r /etc/fstab ]; then
|
||||
grep -vE '^\s*(#|$)' /etc/fstab
|
||||
fi
|
||||
|
||||
hr "TARGETED DATA PATHS"
|
||||
for path in /tank /models /opt /opt/docker /opt/docker/compose /opt/docker/conf \
|
||||
/var/lib/docker /data /srv; do
|
||||
if [ -d "$path" ]; then
|
||||
size=$(du -sh "$path" 2>/dev/null | awk '{print $1}')
|
||||
echo "$path (total: ${size:-?})"
|
||||
ls -la --time-style=long-iso "$path" 2>/dev/null | sed 's/^/ /' | head -30
|
||||
echo
|
||||
fi
|
||||
done
|
||||
|
||||
# --- DOCKER ------------------------------------------------------------------
|
||||
hr "DOCKER"
|
||||
if ! have docker; then
|
||||
echo "docker not installed"
|
||||
else
|
||||
docker version --format 'Server: {{.Server.Version}} Client: {{.Client.Version}}' 2>/dev/null \
|
||||
|| echo "docker daemon not reachable by current user"
|
||||
|
||||
sub "docker info"
|
||||
docker info --format 'Containers: {{.Containers}} (running {{.ContainersRunning}}, paused {{.ContainersPaused}}, stopped {{.ContainersStopped}})
|
||||
Images: {{.Images}}
|
||||
Runtimes: {{.Runtimes}}
|
||||
Default runtime: {{.DefaultRuntime}}
|
||||
Storage driver: {{.Driver}}
|
||||
Root dir: {{.DockerRootDir}}
|
||||
Server version: {{.ServerVersion}}' 2>/dev/null
|
||||
|
||||
sub "running containers"
|
||||
docker ps --format 'table {{.Names}}\t{{.Image}}\t{{.Status}}\t{{.Ports}}' 2>/dev/null
|
||||
|
||||
sub "all containers"
|
||||
docker ps -a --format 'table {{.Names}}\t{{.Image}}\t{{.Status}}' 2>/dev/null
|
||||
|
||||
sub "networks"
|
||||
docker network ls --format 'table {{.Name}}\t{{.Driver}}\t{{.Scope}}' 2>/dev/null
|
||||
|
||||
sub "networks (external, non-default — worth knowing for compose external: true)"
|
||||
docker network ls --filter driver=bridge --format '{{.Name}}' 2>/dev/null \
|
||||
| grep -vE '^(bridge|host|none)$' || true
|
||||
|
||||
sub "named volumes"
|
||||
docker volume ls --format 'table {{.Name}}\t{{.Driver}}' 2>/dev/null
|
||||
|
||||
sub "compose projects currently running"
|
||||
docker ps --format '{{.Label "com.docker.compose.project"}}' 2>/dev/null \
|
||||
| sort -u | grep -v '^$' || echo "(none)"
|
||||
fi
|
||||
|
||||
# --- COMPOSE FILES -----------------------------------------------------------
|
||||
hr "COMPOSE FILES (/opt/docker/compose/)"
|
||||
if [ -d /opt/docker/compose ]; then
|
||||
find /opt/docker/compose -maxdepth 3 -type f \
|
||||
\( -name 'docker-compose.y*ml' -o -name 'compose.y*ml' \) 2>/dev/null \
|
||||
| sort | while read -r f; do
|
||||
printf '\n>>> %s\n' "$f"
|
||||
cat "$f"
|
||||
done
|
||||
else
|
||||
echo "/opt/docker/compose not present"
|
||||
fi
|
||||
|
||||
# --- CONFIG LAYOUT -----------------------------------------------------------
|
||||
hr "CONFIG LAYOUT (/opt/docker/conf/ — top 200 entries)"
|
||||
if [ -d /opt/docker/conf ]; then
|
||||
find /opt/docker/conf -maxdepth 4 2>/dev/null | sort | head -200
|
||||
else
|
||||
echo "/opt/docker/conf not present"
|
||||
fi
|
||||
|
||||
# --- PORTS -------------------------------------------------------------------
|
||||
hr "LISTENING PORTS"
|
||||
if have ss; then
|
||||
ss -tlnH 2>/dev/null | awk '{print $4}' | sort -u
|
||||
elif have netstat; then
|
||||
netstat -tln 2>/dev/null | awk 'NR>2 {print $4}' | sort -u
|
||||
else
|
||||
echo "ss and netstat both unavailable"
|
||||
fi
|
||||
|
||||
# --- HF / MODEL CACHES -------------------------------------------------------
|
||||
hr "MODEL / HUGGINGFACE CACHES"
|
||||
for path in /tank/aimodels/huggingface /tank/aimodels/llm ~/.cache/huggingface \
|
||||
/data/huggingface /opt/huggingface; do
|
||||
if [ -d "$path" ]; then
|
||||
size=$(du -sh "$path" 2>/dev/null | awk '{print $1}')
|
||||
echo "$path (${size:-?})"
|
||||
if [ -d "$path/hub" ]; then
|
||||
echo " hub entries:"
|
||||
ls "$path/hub" 2>/dev/null | sed 's/^/ /' | head -30
|
||||
fi
|
||||
echo
|
||||
fi
|
||||
done
|
||||
|
||||
# --- SYSTEMD SERVICES (docker-adjacent) --------------------------------------
|
||||
hr "DOCKER-ADJACENT SYSTEMD SERVICES"
|
||||
if have systemctl; then
|
||||
systemctl list-units --type=service --state=running --no-pager --no-legend 2>/dev/null \
|
||||
| awk '{print $1, $4}' \
|
||||
| grep -Ei 'docker|container|traefik|nvidia' || echo "(none matching)"
|
||||
fi
|
||||
|
||||
hr "DONE"
|
||||
echo "Paste the above back into the chat, or pass a path as argv[1] to save."
|
||||
Executable
+190
@@ -0,0 +1,190 @@
|
||||
#!/usr/bin/env bash
|
||||
# sync-stacks.sh — pull /opt/docker/{compose,conf}/<stack>/ from every
|
||||
# server into version-controlled `stacks-mirror/<host>/<stack>/`.
|
||||
#
|
||||
# Layout (flat per stack):
|
||||
# stacks-mirror/<host>/<stack>/ <- mirrors /opt/docker/compose/<stack>/
|
||||
# stacks-mirror/<host>/<stack>/conf/ <- mirrors /opt/docker/conf/<stack>/
|
||||
#
|
||||
# Opt-out (per-stack, per-kind):
|
||||
# stacks-mirror/<host>/<stack>/.no-sync → skip stack entirely
|
||||
# stacks-mirror/<host>/<stack>/conf/.no-sync → skip conf only
|
||||
# The marker file is preserved; only the rsync is suppressed. Create the
|
||||
# marker manually for any stack you don't want mirrored.
|
||||
#
|
||||
# Secrets and runtime state are always excluded regardless of opt-out:
|
||||
# .env, .env.*, acme.json, client_secrets.json,
|
||||
# *.pem, *.key, *.crt, *.pfx,
|
||||
# *.sqlite, *.sqlite3, *.db, *.log, *.log.*, *.pid,
|
||||
# hub/, logs/
|
||||
#
|
||||
# Usage:
|
||||
# scripts/sync-stacks.sh # pull from every discovered host
|
||||
# scripts/sync-stacks.sh ana-docker nh3-docker
|
||||
# scripts/sync-stacks.sh --dry-run # show what would change, no writes
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
if ! command -v rsync >/dev/null 2>&1; then
|
||||
echo "error: rsync is not installed on this workstation" >&2
|
||||
echo " install it (e.g. 'sudo apt install rsync') and ensure every remote host has it too" >&2
|
||||
exit 2
|
||||
fi
|
||||
|
||||
SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
|
||||
REPO_ROOT="$(cd "$SCRIPT_DIR/.." && pwd)"
|
||||
SERVERS_DIR="$REPO_ROOT/servers"
|
||||
MIRROR_DIR="$REPO_ROOT/stacks-mirror"
|
||||
|
||||
EXCLUDES=(
|
||||
# Include .env.example / *.env.example templates before the broader
|
||||
# .env* exclude — rsync processes these in order, first match wins.
|
||||
--include='.env.example'
|
||||
--include='*.env.example'
|
||||
--exclude=.env
|
||||
--exclude='.env.*'
|
||||
--exclude=acme.json
|
||||
--exclude=client_secrets.json
|
||||
--exclude='*.pem'
|
||||
--exclude='*.key'
|
||||
--exclude='*.crt'
|
||||
--exclude='*.pfx'
|
||||
--exclude='*.sqlite'
|
||||
--exclude='*.sqlite3'
|
||||
--exclude='*.db'
|
||||
--exclude='*.log'
|
||||
--exclude='*.log.*'
|
||||
--exclude='*.pid'
|
||||
--exclude='hub/'
|
||||
--exclude='logs/'
|
||||
)
|
||||
|
||||
DRY_RUN=0
|
||||
REQUESTED=()
|
||||
for arg in "$@"; do
|
||||
case "$arg" in
|
||||
--dry-run) DRY_RUN=1 ;;
|
||||
-h|--help) sed -n '2,24p' "$0"; exit 0 ;;
|
||||
-*) echo "error: unknown flag $arg" >&2; exit 2 ;;
|
||||
*) REQUESTED+=("$arg") ;;
|
||||
esac
|
||||
done
|
||||
|
||||
resolve_target() {
|
||||
local host="$1"
|
||||
local effective
|
||||
effective=$(ssh -G "$host" 2>/dev/null | awk '/^hostname /{print $2; exit}')
|
||||
if [ -n "$effective" ] && getent hosts "$effective" >/dev/null 2>&1; then
|
||||
echo "$host"; return
|
||||
fi
|
||||
local fb="$SERVERS_DIR/$host/ssh-target"
|
||||
if [ -f "$fb" ]; then awk 'NF{print $1; exit}' "$fb"; return; fi
|
||||
echo "$host"
|
||||
}
|
||||
|
||||
list_remote_subdirs() {
|
||||
# $1 = ssh target, $2 = remote parent path
|
||||
ssh -o BatchMode=yes -o ConnectTimeout=10 "$1" \
|
||||
"find '$2' -maxdepth 1 -mindepth 1 -type d -printf '%f\n' 2>/dev/null | sort" \
|
||||
2>/dev/null || true
|
||||
}
|
||||
|
||||
sync_one() {
|
||||
# $1 = host, $2 = ssh target, $3 = stack,
|
||||
# $4 = 'compose'|'conf' (kind),
|
||||
# $5 = local dest dir
|
||||
local host="$1" target="$2" stack="$3" kind="$4" dest="$5"
|
||||
local remote_src="/opt/docker/$kind/$stack/"
|
||||
local skip="$dest/.no-sync"
|
||||
local label
|
||||
if [ "$kind" = conf ]; then label='conf '; else label='compose'; fi
|
||||
|
||||
mkdir -p "$dest"
|
||||
|
||||
if [ -f "$skip" ]; then
|
||||
printf ' %s skip (.no-sync)\n' "$label"
|
||||
return 0
|
||||
fi
|
||||
|
||||
local extra=()
|
||||
# Don't recurse into conf/ from the compose side — it's its own mirror target.
|
||||
[ "$kind" = compose ] && extra+=(--exclude='conf/')
|
||||
[ "$DRY_RUN" -eq 1 ] && extra+=(--dry-run)
|
||||
|
||||
local err rc=0
|
||||
err=$(
|
||||
rsync -az --delete --info=stats0,flist0 \
|
||||
"${EXCLUDES[@]}" "${extra[@]}" \
|
||||
"$target:$remote_src" "$dest/" 2>&1
|
||||
) || rc=$?
|
||||
|
||||
if [ $rc -ne 0 ]; then
|
||||
printf ' %s FAIL (rc=%d) %s\n' "$label" "$rc" "$(echo "$err" | head -n 1)"
|
||||
return 1
|
||||
fi
|
||||
if [ "$DRY_RUN" -eq 1 ]; then
|
||||
printf ' %s dry-run ok\n' "$label"
|
||||
else
|
||||
printf ' %s ok\n' "$label"
|
||||
fi
|
||||
return 0
|
||||
}
|
||||
|
||||
if [ "${#REQUESTED[@]}" -eq 0 ]; then
|
||||
mapfile -t HOSTS < <(find "$SERVERS_DIR" -mindepth 1 -maxdepth 1 -type d -printf '%f\n' | sort)
|
||||
else
|
||||
HOSTS=("${REQUESTED[@]}")
|
||||
fi
|
||||
|
||||
[ "${#HOSTS[@]}" -eq 0 ] && { echo "error: no hosts found" >&2; exit 2; }
|
||||
[ "$DRY_RUN" -eq 1 ] && echo "(dry-run)"
|
||||
|
||||
mkdir -p "$MIRROR_DIR"
|
||||
total_fail=0
|
||||
|
||||
for host in "${HOSTS[@]}"; do
|
||||
target=$(resolve_target "$host")
|
||||
printf '%s (%s):\n' "$host" "$target"
|
||||
|
||||
mapfile -t compose_stacks < <(list_remote_subdirs "$target" /opt/docker/compose)
|
||||
mapfile -t conf_stacks < <(list_remote_subdirs "$target" /opt/docker/conf)
|
||||
|
||||
if [ "${#compose_stacks[@]}" -eq 0 ] && [ "${#conf_stacks[@]}" -eq 0 ]; then
|
||||
printf ' (no stacks discovered — check ssh + remote /opt/docker layout)\n'
|
||||
continue
|
||||
fi
|
||||
|
||||
# Union of stack names.
|
||||
mapfile -t all_stacks < <(printf '%s\n' "${compose_stacks[@]}" "${conf_stacks[@]}" | sort -u | grep .)
|
||||
|
||||
# Warn about local stacks that no longer exist on the remote.
|
||||
if [ -d "$MIRROR_DIR/$host" ]; then
|
||||
for local_stack in "$MIRROR_DIR/$host"/*/; do
|
||||
[ -d "$local_stack" ] || continue
|
||||
name=$(basename "$local_stack")
|
||||
if ! printf '%s\n' "${all_stacks[@]}" | grep -qxF "$name"; then
|
||||
printf ' ! %s exists locally but not on remote (stale — remove manually if intentional)\n' "$name"
|
||||
fi
|
||||
done
|
||||
fi
|
||||
|
||||
for stack in "${all_stacks[@]}"; do
|
||||
printf ' %s\n' "$stack"
|
||||
stack_root="$MIRROR_DIR/$host/$stack"
|
||||
if [ -f "$stack_root/.no-sync" ]; then
|
||||
printf ' skip (.no-sync at stack root)\n'
|
||||
continue
|
||||
fi
|
||||
if printf '%s\n' "${compose_stacks[@]}" | grep -qxF "$stack"; then
|
||||
sync_one "$host" "$target" "$stack" compose "$stack_root" || total_fail=$((total_fail+1))
|
||||
fi
|
||||
if printf '%s\n' "${conf_stacks[@]}" | grep -qxF "$stack"; then
|
||||
sync_one "$host" "$target" "$stack" conf "$stack_root/conf" || total_fail=$((total_fail+1))
|
||||
fi
|
||||
done
|
||||
done
|
||||
|
||||
if [ "$total_fail" -gt 0 ]; then
|
||||
printf '\n%d sync operation(s) failed\n' "$total_fail" >&2
|
||||
exit 1
|
||||
fi
|
||||
Reference in New Issue
Block a user