From cc0e3af87d192806c15ed6bc0f3200872795d389 Mon Sep 17 00:00:00 2001
From: Vuong Hoang <vh@phasefinal.com>
Date: Mon, 13 Jul 2026 14:28:39 -0700
Subject: [PATCH] feat(soong-lab-ci): red-run althing relay (nh3-dev poll ->
 ping soong-dev)

Per operator call (no gitea write token on the Worldtree-team VM): a 2-min systemd
--user timer on nh3-dev polls corviduo's last-deploy.json and pings soong-dev via
althing on a NEW red deploy (green stays silent). Delivers soong-dev's red-run
visibility without a credential on corviduo. Tested (red detect+format DRY, green quiet).
---
 docs/runbooks/soong-lab-autodeploy.md        | 10 ++++++---
 services/soong-lab-ci/soong-ci-relay.service |  6 +++++
 services/soong-lab-ci/soong-ci-relay.sh      | 23 ++++++++++++++++++++
 services/soong-lab-ci/soong-ci-relay.timer   |  7 ++++++
 4 files changed, 43 insertions(+), 3 deletions(-)
 create mode 100644 services/soong-lab-ci/soong-ci-relay.service
 create mode 100755 services/soong-lab-ci/soong-ci-relay.sh
 create mode 100644 services/soong-lab-ci/soong-ci-relay.timer

diff --git a/docs/runbooks/soong-lab-autodeploy.md b/docs/runbooks/soong-lab-autodeploy.md
index 9924de0..0087ff3 100644
--- a/docs/runbooks/soong-lab-autodeploy.md
+++ b/docs/runbooks/soong-lab-autodeploy.md
@@ -52,7 +52,11 @@ ssh corviduo-dev 'bash ~/soong-lab-deploy.sh'
   present on corviduo-dev); no extra auth in the deploy script.
 - **SSRF**: gitea reached corviduo-dev `10.250.50.152` fine (test-delivery 204) —
   no `ALLOWED_HOST_LIST` relax needed (unlike the ytvc/WG case).
-- **No althing on corviduo-dev** → red-run notify is log/status-file based
-  (`last-deploy.json` + `GET :9010`). A gitea commit-status or althing relay
-  could be added if push-notify on red is wanted.
+- **Red-run push-notify** via an **althing relay on nh3-dev** (`soong-ci-relay.timer`,
+  2-min poll of corviduo's `last-deploy.json` → pings **soong-dev** via althing on a
+  NEW red run; green runs stay silent = fire-and-forget). corviduo itself has no
+  althing, so the relay lives on nh3-dev (which does), needing no gitea write token
+  on the Worldtree-team VM. Files: `services/soong-lab-ci/soong-ci-relay.{sh,service,timer}`;
+  state `~/.local/state/soong-ci-relay/last-at.txt`. (A gitea commit-status was the
+  alternative but needs a write token gitea won't mint without basic-auth.)
 - Test suite: `uv run pytest` in `backend/` (242 tests as of v0.3.6).
diff --git a/services/soong-lab-ci/soong-ci-relay.service b/services/soong-lab-ci/soong-ci-relay.service
new file mode 100644
index 0000000..011cc3c
--- /dev/null
+++ b/services/soong-lab-ci/soong-ci-relay.service
@@ -0,0 +1,6 @@
+[Unit]
+Description=soong-lab CI red-run relay (poll corviduo -> ping soong-dev on red)
+[Service]
+Type=oneshot
+Environment=PATH=%h/.local/bin:/usr/local/bin:/usr/bin:/bin
+ExecStart=%h/.local/bin/soong-ci-relay.sh
diff --git a/services/soong-lab-ci/soong-ci-relay.sh b/services/soong-lab-ci/soong-ci-relay.sh
new file mode 100755
index 0000000..40ef9b1
--- /dev/null
+++ b/services/soong-lab-ci/soong-ci-relay.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+# soong-lab CI red-run relay (runs on nh3-dev, which has althing): poll corviduo's
+# last-deploy.json; on a NEW red run, ping soong-dev via althing. No cred on corviduo.
+set -uo pipefail
+export ALTHING_HANDLE=infra-ops
+STATE="$HOME/.local/state/soong-ci-relay/last-at.txt"
+mkdir -p "$(dirname "$STATE")"
+CUR=$(ssh -o BatchMode=yes -o ConnectTimeout=10 corviduo-dev 'cat ~/.config/soong/last-deploy.json 2>/dev/null' 2>/dev/null) || exit 0
+[ -z "$CUR" ] && exit 0
+get(){ printf '%s' "$CUR" | python3 -c "import sys,json;print(json.load(sys.stdin).get('$1',''))" 2>/dev/null; }
+AT=$(get at); RESULT=$(get result)
+[ -z "$AT" ] && exit 0
+LAST=$(cat "$STATE" 2>/dev/null || true)
+[ "$AT" = "$LAST" ] && exit 0
+printf '%s' "$AT" > "$STATE"
+[ "$RESULT" != "red" ] && exit 0
+SHA=$(get sha); STAGE=$(get stage); DETAIL=$(get detail)
+MSG="soong-lab CI deploy FAILED (RED) — sha=$SHA, stage=$STAGE, detail: $DETAIL.
+The studio was NOT touched (still on the previous good version — safe-abort).
+A red runner-run despite green-local usually means a runner-vs-local env divergence.
+Log: corviduo-dev:~/soong-lab-deploy.log  |  status: curl corviduo-dev:9010/"
+if [ "${DRY_RUN:-0}" = "1" ]; then echo "[DRY] would post to soong-dev:"; echo "$MSG"; exit 0; fi
+printf '%s' "$MSG" | althing-cli post --to soong-dev --subject "soong-lab CI: RED deploy ($SHA) — studio untouched"
diff --git a/services/soong-lab-ci/soong-ci-relay.timer b/services/soong-lab-ci/soong-ci-relay.timer
new file mode 100644
index 0000000..1e649bc
--- /dev/null
+++ b/services/soong-lab-ci/soong-ci-relay.timer
@@ -0,0 +1,7 @@
+[Unit]
+Description=poll soong-lab CI status every 2 min
+[Timer]
+OnBootSec=90
+OnUnitActiveSec=120
+[Install]
+WantedBy=timers.target