From fb556586e3b27a7a09f52206b79e8043db5a9512 Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Mon, 13 Jul 2026 14:16:26 -0700 Subject: [PATCH] feat(soong-lab-ci): green-gated push-to-deploy CI/CD for the soong-lab studio MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Vuong-directed. gitea webhook (push→main) → HMAC listener on corviduo-dev:9010 → clone (read-only deploy key) → uv sync + pytest → redeploy soong-lab-studio.service ONLY on green (running studio untouched on red). Validated end-to-end 2026-07-13. Canonical copies of the deploy script + listener + unit; runbook in docs/runbooks. --- docs/runbooks/soong-lab-autodeploy.md | 58 +++++++++++++++++++++ services/soong-lab-ci/soong-lab-deploy.sh | 31 +++++++++++ services/soong-lab-ci/soong-webhook.py | 27 ++++++++++ services/soong-lab-ci/soong-webhook.service | 13 +++++ 4 files changed, 129 insertions(+) create mode 100644 docs/runbooks/soong-lab-autodeploy.md create mode 100755 services/soong-lab-ci/soong-lab-deploy.sh create mode 100644 services/soong-lab-ci/soong-webhook.py create mode 100644 services/soong-lab-ci/soong-webhook.service diff --git a/docs/runbooks/soong-lab-autodeploy.md b/docs/runbooks/soong-lab-autodeploy.md new file mode 100644 index 0000000..9924de0 --- /dev/null +++ b/docs/runbooks/soong-lab-autodeploy.md @@ -0,0 +1,58 @@ +# soong-lab push-to-deploy (gitea webhook → corviduo-dev, test-gated) + +Green-gated CI/CD for the soong-lab studio: **push to `main` → run the test +suite → redeploy the studio ONLY if tests pass** (running studio is never +touched on a red run). Built 2026-07-13 (Vuong-directed). Adapts the +[ytvc-autodeploy](./ytvc-autodeploy.md) webhook pattern. + +## Flow + +``` +push→main → gitea webhook (POST, HMAC) → soong-webhook listener :9010 on corviduo-dev + → ~/soong-lab-deploy.sh: + git clone (read-only deploy key, internal SSH :222) + uv sync ; uv run pytest ── RED → abort, studio UNTOUCHED, status=red + rsync backend/ → studio dir ; uv sync --no-dev ; systemctl restart + → status=green, studio healthy +``` + +## Components (all on corviduo-dev, user `infra-ops`) + +- `~/soong-lab-deploy.sh` — clone → test → deploy-on-green. Logs to + `~/soong-lab-deploy.log`; writes `~/.config/soong/last-deploy.json` + (`{result: green|red, stage, sha, at}`). +- `~/soong-webhook.py` — HTTP listener on `:9010`. HMAC-SHA256 (`X-Gitea-Signature`) + vs `~/.config/soong/webhook-secret` (mode 600); fires the deploy only on + `ref == refs/heads/main`. `GET /` returns `ok | last: `. +- `soong-webhook.service` (system unit, enabled) — runs the listener. +- Read-only deploy key `~/.ssh/soong-deploy_ed25519` → gitea repo key id 5 on + `vh/soong-lab` (read_only). Clone via `ssh://git@10.250.50.70:222/vh/soong-lab.git`. +- Studio unit `soong-lab-studio.service` (WD `/home/infra-ops/soong-lab/backend`); + restart needs infra-ops NOPASSWD sudo (present). +- Gitea webhook: repo `vh/soong-lab` hook id 3 → `http://10.250.50.152:9010/`, + JSON, Push events, the shared secret. + +## Verify / operate + +```bash +ssh corviduo-dev 'systemctl is-active soong-webhook.service; curl -s localhost:9010/' +ssh corviduo-dev 'tail -30 ~/soong-lab-deploy.log' # deploy history +# manual deploy (same as the webhook does): +ssh corviduo-dev 'bash ~/soong-lab-deploy.sh' +``` + +## Notes / gotchas + +- **Green-gated by construction**: `pytest || fail` runs BEFORE any studio touch, + so a red suite aborts with the studio still on the old version. Validated + 2026-07-13 (a mid-deploy rsync failure left the studio untouched/active). +- **rsync is required** on corviduo-dev (`apt install rsync` — installed 2026-07-13; + it wasn't present initially). +- **bifrost dep** resolves from the internal Gitea PyPI via `~/.netrc` (already + present on corviduo-dev); no extra auth in the deploy script. +- **SSRF**: gitea reached corviduo-dev `10.250.50.152` fine (test-delivery 204) — + no `ALLOWED_HOST_LIST` relax needed (unlike the ytvc/WG case). +- **No althing on corviduo-dev** → red-run notify is log/status-file based + (`last-deploy.json` + `GET :9010`). A gitea commit-status or althing relay + could be added if push-notify on red is wanted. +- Test suite: `uv run pytest` in `backend/` (242 tests as of v0.3.6). diff --git a/services/soong-lab-ci/soong-lab-deploy.sh b/services/soong-lab-ci/soong-lab-deploy.sh new file mode 100755 index 0000000..c713b98 --- /dev/null +++ b/services/soong-lab-ci/soong-lab-deploy.sh @@ -0,0 +1,31 @@ +#!/usr/bin/env bash +# soong-lab CI/CD deploy: clone -> uv sync + pytest -> deploy studio ONLY on green. +# Running studio is NEVER touched on a red test run (safe-by-construction). +set -uo pipefail +LOG=/home/infra-ops/soong-lab-deploy.log +exec >>"$LOG" 2>&1 +echo; echo "========== deploy run $(date -u) ==========" +SRC=/tmp/soong-lab-ci +STUDIO=/home/infra-ops/soong-lab/backend +KEY=/home/infra-ops/.ssh/soong-deploy_ed25519 +UV=/home/infra-ops/.local/bin/uv +STATUS=/home/infra-ops/.config/soong/last-deploy.json +export GIT_SSH_COMMAND="ssh -i $KEY -o IdentitiesOnly=yes -p 222" +SHA="?" +fail(){ echo "DEPLOY ABORTED [$2]: $1"; printf "{\"result\":\"red\",\"stage\":\"%s\",\"detail\":\"%s\",\"sha\":\"%s\",\"at\":\"%s\"}\n" "$2" "$1" "$SHA" "$(date -u +%FT%TZ)" > "$STATUS"; exit 1; } +rm -rf "$SRC" +git clone --depth 1 ssh://git@10.250.50.70:222/vh/soong-lab.git "$SRC" || fail "clone failed" clone +SHA=$(git -C "$SRC" rev-parse --short HEAD); echo "cloned $SHA" +cd "$SRC/backend" +"$UV" sync || fail "uv sync (test env) failed" sync +echo "=== test suite ===" +"$UV" run pytest -q || fail "TESTS RED — studio left untouched" tests +echo "=== tests GREEN — deploying to studio ===" +rsync -a --delete --exclude .venv/ --exclude "*.env" --exclude "*.db" --exclude "*.sqlite*" --exclude "data/" --exclude "state/" --exclude "logs/" "$SRC/backend/" "$STUDIO/" || fail "rsync to studio failed" rsync +cd "$STUDIO" +"$UV" sync --no-dev || fail "studio uv sync failed" studio_sync +sudo systemctl restart soong-lab-studio.service || fail "systemctl restart failed" restart +sleep 3 +systemctl is-active --quiet soong-lab-studio.service || fail "studio not active post-restart" health +echo "DEPLOYED $SHA — studio healthy" +printf "{\"result\":\"green\",\"sha\":\"%s\",\"at\":\"%s\"}\n" "$SHA" "$(date -u +%FT%TZ)" > "$STATUS" diff --git a/services/soong-lab-ci/soong-webhook.py b/services/soong-lab-ci/soong-webhook.py new file mode 100644 index 0000000..12ec1de --- /dev/null +++ b/services/soong-lab-ci/soong-webhook.py @@ -0,0 +1,27 @@ +#!/usr/bin/env python3 +"""Gitea push-webhook listener for soong-lab: on a verified push to main, run +~/soong-lab-deploy.sh. HMAC-SHA256 (X-Gitea-Signature) vs ~/.config/soong/webhook-secret.""" +import hashlib, hmac, json, os, subprocess, threading +from http.server import BaseHTTPRequestHandler, HTTPServer +SECRET = open(os.path.expanduser("~/.config/soong/webhook-secret"), "rb").read().strip() +DEPLOY = os.path.expanduser("~/soong-lab-deploy.sh") +STATUS = os.path.expanduser("~/.config/soong/last-deploy.json") +class H(BaseHTTPRequestHandler): + def do_POST(self): + body = self.rfile.read(int(self.headers.get("Content-Length", 0))) + mac = hmac.new(SECRET, body, hashlib.sha256).hexdigest() + if not hmac.compare_digest(mac, self.headers.get("X-Gitea-Signature", "")): + self.send_response(401); self.end_headers(); self.wfile.write(b"bad signature\n"); return + try: ref = json.loads(body).get("ref", "") + except Exception: self.send_response(400); self.end_headers(); return + if ref != "refs/heads/main": + self.send_response(200); self.end_headers(); self.wfile.write(b"ignored " + ref.encode() + b"\n"); return + self.send_response(202); self.end_headers(); self.wfile.write(b"deploying\n") + threading.Thread(target=lambda: subprocess.run(["bash", DEPLOY]), daemon=True).start() + def do_GET(self): + self.send_response(200); self.end_headers() + try: st = open(STATUS).read().strip() + except Exception: st = "no deploy yet" + self.wfile.write(b"soong-webhook ok | last: " + st.encode()) + def log_message(self, *a): pass +HTTPServer(("0.0.0.0", 9010), H).serve_forever() diff --git a/services/soong-lab-ci/soong-webhook.service b/services/soong-lab-ci/soong-webhook.service new file mode 100644 index 0000000..054be3d --- /dev/null +++ b/services/soong-lab-ci/soong-webhook.service @@ -0,0 +1,13 @@ +[Unit] +Description=soong-lab CI webhook listener (push->main -> test-gated redeploy) +After=network-online.target + +[Service] +Type=simple +User=infra-ops +ExecStart=/usr/bin/python3 /home/infra-ops/soong-webhook.py +Restart=on-failure +RestartSec=5 + +[Install] +WantedBy=multi-user.target