From 9175fbd539063effa2a22730f4dc7d8ab6b7b86a Mon Sep 17 00:00:00 2001
From: Vuong Hoang <vh@phasefinal.com>
Date: Mon, 5 Oct 2026 19:25:14 -0700
Subject: [PATCH] feat(qemu-ga): failure-propagating fsfreeze dispatcher;
 enabled on nh3-svos (hooks were never run)

---
 configs/qemu-ga/fsfreeze-dispatch | 66 +++++++++++++++++++++++++++++++
 servers/nh3-svos/README.md        |  2 +-
 2 files changed, 67 insertions(+), 1 deletion(-)
 create mode 100644 configs/qemu-ga/fsfreeze-dispatch

diff --git a/configs/qemu-ga/fsfreeze-dispatch b/configs/qemu-ga/fsfreeze-dispatch
new file mode 100644
index 0000000..60d1aea
--- /dev/null
+++ b/configs/qemu-ga/fsfreeze-dispatch
@@ -0,0 +1,66 @@
+#!/bin/bash
+# fsfreeze-dispatch: qemu-ga fsfreeze hook dispatcher that PROPAGATES failure (eshpfi configs/qemu-ga/, 2026-10-05).
+#
+# Debian's stock /etc/qemu/fsfreeze-hook logs a child hook's nonzero status but always exits 0, so a hook that
+# refuses to quiesce still lets the snapshot proceed (found by svos-dev on nh3-svos). This one does not:
+#   freeze: run hooks in name order; on the first failure, thaw the FAILING hook and then every hook that had
+#           completed, in reverse order (best effort), log any unwind failure, and exit 1.
+#   thaw:   run every hook in reverse name order, exit 1 if any failed (never stop early).
+# Wired in by /etc/qemu/qemu-ga.conf `fsfreeze-hook=/usr/local/sbin/fsfreeze-dispatch`. The packaged
+# /etc/qemu/fsfreeze-hook (a dpkg conffile) is left untouched. Same hook dir and log file as the stock hook.
+set -u
+HOOKDIR=/etc/qemu/fsfreeze-hook.d
+LOGFILE=/var/log/qga-fsfreeze-hook.log
+action=${1:-}
+
+log() { printf '%s: %s\n' "$(date)" "$1" >>"$LOGFILE" 2>/dev/null || logger -t qemu-ga-freeze-hook "$1"; }
+
+ignored() {
+    case "$1" in
+        *~ | *.bak | *.orig | *.rpmnew | *.rpmorig | *.rpmsave | *.sample | *.dpkg-old | *.dpkg-new | *.dpkg-tmp | \
+        *.dpkg-dist | *.dpkg-bak | *.dpkg-backup | *.dpkg-remove) return 0 ;;
+    esac
+    return 1
+}
+
+hooks=()
+if [ -d "$HOOKDIR" ]; then
+    while IFS= read -r -d '' f; do
+        ignored "$f" && continue
+        [ -f "$f" ] && [ -x "$f" ] && hooks+=("$f")
+    done < <(find "$HOOKDIR" -mindepth 1 -maxdepth 1 -print0 | LC_ALL=C sort -z)
+fi
+
+run() { # $1 hook, $2 action
+    log "Executing $1 $2"
+    "$1" "$2" >>"$LOGFILE" 2>&1
+    local st=$?
+    [ "$st" -ne 0 ] && log "Error: $1 $2 finished with status=$st"
+    return "$st"
+}
+
+case "$action" in
+    freeze)
+        completed=()
+        for h in "${hooks[@]}"; do
+            if run "$h" freeze; then
+                completed+=("$h")
+            else
+                log "freeze FAILED at $h: unwinding (thaw $h, then ${#completed[@]} completed hook(s) in reverse)"
+                unwind_failed=0
+                run "$h" thaw || unwind_failed=1
+                for ((i = ${#completed[@]} - 1; i >= 0; i--)); do run "${completed[$i]}" thaw || unwind_failed=1; done
+                [ "$unwind_failed" -ne 0 ] && log "UNWIND FAILURE: at least one thaw during the freeze unwind failed"
+                exit 1
+            fi
+        done
+        exit 0 ;;
+    thaw)
+        rc=0
+        for ((i = ${#hooks[@]} - 1; i >= 0; i--)); do run "${hooks[$i]}" thaw || rc=1; done
+        [ "$rc" -ne 0 ] && log "thaw: at least one hook failed"
+        exit "$rc" ;;
+    *)
+        log "fsfreeze-dispatch: unknown action '$action'"
+        exit 1 ;;
+esac
diff --git a/servers/nh3-svos/README.md b/servers/nh3-svos/README.md
index 78eec9d..3230231 100644
--- a/servers/nh3-svos/README.md
+++ b/servers/nh3-svos/README.md
@@ -20,7 +20,7 @@ configuration, enrollment and acceptance.
 | Releases | `/opt/svos` root:svc-svos 2775, with `releases/` for svos-dev's layout. The git checkout is not a runtime dependency |
 | Docker | none, by design |
 | Backups | nh3-pve-2 vzdump job, nightly **21:30**, `all 1` → pbs-ana namespace `nh3-pve-2` (mirrored to pbs-nh3 at 06:30), snapshot mode with guest-agent fs-freeze |
-| App-consistent hook | `/etc/qemu/fsfreeze-hook.d/` exists. The guest agent runs those scripts on vzdump's freeze/thaw, which is where svos-dev's SQLite checkpoint hook belongs. Until it lands, backups are crash-consistent (fine for WAL + synchronous FULL, but not the stated requirement) |
+| App-consistent hook | svos-dev's `/etc/qemu/fsfreeze-hook.d/50-svos` stops the SVOS services on freeze and restarts the ones that were running on thaw. ⚠ **Debian's qemu-ga does NOT run fsfreeze hooks unless configured** (no `-F` in the unit, no `qemu-ga.conf` by default). Here `/etc/qemu/qemu-ga.conf` points it at **`/usr/local/sbin/fsfreeze-dispatch`** (`configs/qemu-ga/fsfreeze-dispatch`), which unlike the stock `/etc/qemu/fsfreeze-hook` propagates failure: if a freeze hook fails, it thaws that hook and the completed ones and exits 1, so qemu-ga refuses the freeze. Tested 2026-10-05 via `qm guest cmd 111 fsfreeze-*`, both success and failure paths. The live-service backup drill is still pending |
 | Monitoring | Beszel native agent 0.18.7 (`playbooks/beszel-native.yaml`, `configs/beszel-agent/hosts/nh3-svos.env`), hub system `t6y646ifa3x7xxv`, alerts CPU>95%/15m, Disk>85%/5m, Memory>90%/10m, offline/2m. Up 2026-10-05 |
 
 ## Build (reproducible)