From 6d66bc2f3005e4f91f342342b4375ec4dace9aae Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Sat, 13 Jun 2026 17:17:49 -0700 Subject: [PATCH] feat(arbo): track webhook deploy scripts (arbo-deploy.sh + arbo-webhook.py) Operator's call: keep the arbo stack in eshpfi and version its deploy machinery alongside the compose (was host-only on irv-ml1 = recoverability foot-gun). - arbo-webhook.py: :9009 HMAC listener (secret externalized to host file, not git) - arbo-deploy.sh: internal-route fetch + catalog-only targeted restart Document both in the README Q5 section + the internal-gitea-route gotcha. --- stacks/arbo/README.md | 30 +++++++++++++++++++++++------- stacks/arbo/arbo-deploy.sh | 30 ++++++++++++++++++++++++++++++ stacks/arbo/arbo-webhook.py | 23 +++++++++++++++++++++++ 3 files changed, 76 insertions(+), 7 deletions(-) create mode 100755 stacks/arbo/arbo-deploy.sh create mode 100755 stacks/arbo/arbo-webhook.py diff --git a/stacks/arbo/README.md b/stacks/arbo/README.md index 3fd4ec9..4945275 100644 --- a/stacks/arbo/README.md +++ b/stacks/arbo/README.md @@ -59,14 +59,30 @@ SQLite-on-NFS locking is a foot-gun). The gallery DB is the durable asset → it joins the restic file backup. (Adds irv-ml1's `arbo_db` path to a Backrest repo — small; flag if irv-ml1 isn't yet a restic source.) -## Q5 — Catalog-pull automation +## Q5 — Catalog-pull automation (IMPLEMENTED) -- **Day 1:** manual `git pull` in `/worktank/arbo/repo` + `deploy-stack.sh - irv-ml1 arbo` restart (or `docker compose restart arbo`). -- **Follow-on (recommended):** mirror the **yt-voice-clipper webhook** already - live on irv-ml1 — gitea webhook → HMAC listener → `git pull` + `compose - restart`, so a comfy-dev catalog push reaches prod in one action (D2's "cheap - to reach prod"). Layered after the stack is up. +A gitea push-webhook → HMAC listener → `git pull --reset` reaches prod in one +action. The two host-side scripts are tracked here (they live on irv-ml1 at +`~/arbo-webhook.py` + `~/arbo-deploy.sh`; copy them back if rebuilding the host): + +- **[`arbo-webhook.py`](arbo-webhook.py)** — HTTP listener on `:9009`. Validates + the gitea `X-Gitea-Signature` HMAC-SHA256 against `~/.config/arbo/webhook-secret` + (secret stays on the host, NOT in git), and on a verified push to `refs/heads/main` + fires `arbo-deploy.sh` in a daemon thread. +- **[`arbo-deploy.sh`](arbo-deploy.sh)** — `git fetch` + `reset --hard origin/main` + in `/worktank/arbo/repo`, then a **targeted** restart: + - **`catalog/`** changed → `compose restart engine` (catalog is loaded once at + startup into `app.state.cat`). + - **`graphs/` / `frontend/`** → no restart (read per-request). + - **`src/` or `Dockerfile`** → warns "NEW IMAGE required" + skips (baked code + needs a rebuild, not a restart; comfy-dev pins a tag, infra-ops redeploys). + - **`pyproject.toml` / `uv.lock`** → ignored (they bump on every commit via + SemVer etiquette, so they're not image signals). + +**Gotcha:** the `gitea-arbo` ssh remote on irv-ml1 must point at the **internal** +gitea route `10.250.50.70:222` (the ana-docker container's git-SSH), NOT the public +`gitea.phasefinal.com:22` — the public path fail2bans the host's egress IP and +silently wedges the webhook fetch. See `docs/orientation.md` → Git / gitea. ## Items needing comfy-dev's image (jointly owned) diff --git a/stacks/arbo/arbo-deploy.sh b/stacks/arbo/arbo-deploy.sh new file mode 100755 index 0000000..11ac902 --- /dev/null +++ b/stacks/arbo/arbo-deploy.sh @@ -0,0 +1,30 @@ +#!/usr/bin/env bash +# Auto-deploy arbo CATALOG on a verified main push (run by the webhook listener). +# Restart the engine ONLY when catalog/ changes (loaded once at startup -> app.state.cat). +# graphs/ + frontend/ are read PER-REQUEST (no restart). A NEW IMAGE (not a restart) is +# needed only for src/ or Dockerfile changes. NOTE: pyproject.toml + uv.lock churn on +# EVERY commit (per-commit SemVer version bump) so they are NOT image signals -> ignored. +# Confirmed against engine code w/ comfy-dev 2026-06-13. +set -euo pipefail +REPO=/worktank/arbo/repo +STACK=/opt/docker/compose/arbo +LOG="$HOME/.config/arbo/deploy.log" +{ + echo "=== arbo catalog deploy $(date -u +%FT%TZ) ===" + cd "$REPO" + git fetch -q origin main + before=$(git rev-parse HEAD); git reset --hard origin/main; after=$(git rev-parse HEAD) + echo "main $before -> $after" + [ "$before" = "$after" ] && { echo "no change"; exit 0; } + changed=$(git diff --name-only "$before" "$after") + if echo "$changed" | grep -qE '^(src/|Dockerfile)'; then + echo "WARNING: src/ or Dockerfile change — a NEW IMAGE is required (D2). NOT restarting; comfy-dev pins a tag + infra-ops redeploys." + exit 0 + fi + if echo "$changed" | grep -qE '^catalog/'; then + cd "$STACK" && docker compose restart engine + echo "=== catalog/ changed -> restarted; catalog reloaded $(date -u +%FT%TZ) ===" + else + echo "=== no catalog/ change (graphs/frontend/version-bump only) -> no restart $(date -u +%FT%TZ) ===" + fi +} >> "$LOG" 2>&1 diff --git a/stacks/arbo/arbo-webhook.py b/stacks/arbo/arbo-webhook.py new file mode 100755 index 0000000..f8ba661 --- /dev/null +++ b/stacks/arbo/arbo-webhook.py @@ -0,0 +1,23 @@ +#!/usr/bin/env python3 +"""Gitea push-webhook listener: on a verified push to main, trigger ~/arbo-deploy.sh. +HMAC-SHA256 validated against ~/.config/arbo/webhook-secret (X-Gitea-Signature).""" +import hashlib, hmac, json, os, subprocess, threading +from http.server import BaseHTTPRequestHandler, HTTPServer +SECRET = open(os.path.expanduser("~/.config/arbo/webhook-secret"), "rb").read().strip() +DEPLOY = os.path.expanduser("~/arbo-deploy.sh") +class H(BaseHTTPRequestHandler): + def do_POST(self): + body = self.rfile.read(int(self.headers.get("Content-Length", 0))) + mac = hmac.new(SECRET, body, hashlib.sha256).hexdigest() + if not hmac.compare_digest(mac, self.headers.get("X-Gitea-Signature", "")): + self.send_response(401); self.end_headers(); self.wfile.write(b"bad signature\n"); return + try: ref = json.loads(body).get("ref", "") + except Exception: self.send_response(400); self.end_headers(); return + if ref != "refs/heads/main": + self.send_response(200); self.end_headers(); self.wfile.write(b"ignored " + ref.encode() + b"\n"); return + self.send_response(202); self.end_headers(); self.wfile.write(b"deploying\n") + threading.Thread(target=lambda: subprocess.run(["bash", DEPLOY]), daemon=True).start() + def do_GET(self): + self.send_response(200); self.end_headers(); self.wfile.write(b"arbo-webhook ok\n") + def log_message(self, *a): pass +HTTPServer(("0.0.0.0", 9009), H).serve_forever()