gitea-runner: stack + playbook for self-hosted Actions

Central runner on ana-docker (gitea is local; existing fleet tooling
already SSHes from there). Playbook is parameterized so future
site-local runners (nh3-docker, esh-docker-vm) drop in via --var
overrides instead of copy-paste.

Includes a workflow template for vh/task-board that calls the existing
deploy-task-board.yaml playbook — keeps the playbook as the single
source of truth for "how task-board is deployed", manual or automated.

Labels embed `:docker://node:20-bookworm-slim` schema; without it,
act_runner v0.6+ silently falls back to host-mode and runs job steps
inside the Alpine runner container (no apt/python/node), breaking any
real workflow. node:20-bookworm-slim is small + has git + node so
actions/checkout works out of the box.

playbooks/deploy-gitea-runner.yaml

@@ -0,0 +1,139 @@
+# Deploy a Gitea Actions self-hosted runner to a Docker host.
+#
+# We currently run ONE central runner on ana-docker (gitea is local
+# there and SSH-from-there to the rest of the fleet already works).
+# This playbook is parameterized so the same file can stand up
+# site-local runners later without copy-paste — see
+# stacks/gitea-runner/README.md "Topology" for when to do that.
+#
+# Usage:
+#   # Central runner (defaults match ana-docker)
+#   scripts/elway ana-docker --playbook playbooks/deploy-gitea-runner.yaml
+#
+#   # Site-local runner — override identity at the CLI; the .env on
+#   # the host carries the registration token + image pin.
+#   scripts/elway nh3-docker --playbook playbooks/deploy-gitea-runner.yaml \
+#     --var runner_name=nh3-docker-runner \
+#     --var runner_labels=pfi-fleet,nh3-docker
+#
+#   scripts/elway esh-docker-vm --playbook playbooks/deploy-gitea-runner.yaml \
+#     --var runner_name=esh-runner \
+#     --var runner_labels=pfi-fleet,esh
+#
+# Prereqs on the target host:
+#   - Docker + compose plugin
+#   - traefik-net network exists (external)
+#   - /opt/docker/{compose,conf} convention (any PFI host already has this)
+#
+# Prereqs in gitea (do once before the first run):
+#   - Generate a registration token at the appropriate scope
+#     (admin / org / repo). Paste into .env on the target host
+#     under GITEA_RUNNER_REGISTRATION_TOKEN before this playbook runs
+#     `docker compose up -d`. See stacks/gitea-runner/README.md.
+
+vars:
+  compose_dir: /opt/docker/compose/gitea-runner
+  data_dir: /opt/docker/conf/gitea-runner/data
+  # Optional overrides — leave blank to take whatever's already in the
+  # host-side .env. Setting these here patches the .env in place,
+  # which is how site-local runners get unique names/labels without
+  # editing files on the host by hand.
+  runner_name: ""
+  runner_labels: ""
+
+steps:
+  # ── host-side directory prep ────────────────────────────────────────
+  - name: Ensure compose dir exists
+    shell: mkdir -p {{ compose_dir }}
+    creates: "{{ compose_dir }}"
+
+  - name: Ensure data dir exists
+    shell: mkdir -p {{ data_dir }}
+    creates: "{{ data_dir }}"
+
+  # ── compose + config files ──────────────────────────────────────────
+  - name: Upload compose.yaml
+    upload:
+      src: stacks/gitea-runner/compose.yaml
+      dest: "{{ compose_dir }}/compose.yaml"
+      mode: "0644"
+
+  - name: Upload runner config.yaml
+    upload:
+      src: stacks/gitea-runner/conf/config.yaml
+      dest: "{{ data_dir }}/config.yaml"
+      mode: "0644"
+
+  - name: Seed .env from template (only if absent)
+    upload:
+      src: stacks/gitea-runner/.env.example
+      dest: "{{ compose_dir }}/.env"
+      mode: "0644"
+    when: "[ ! -f {{ compose_dir }}/.env ]"
+
+  # ── optional CLI-driven identity overrides ──────────────────────────
+  # If runner_name was passed as --var, patch the line in the host's
+  # .env so subsequent compose-up uses it. Idempotent: changed_when
+  # checks whether the value differs from what's already there.
+  - name: Patch GITEA_RUNNER_NAME in .env (when --var runner_name=)
+    shell: sed -i 's|^GITEA_RUNNER_NAME=.*|GITEA_RUNNER_NAME={{ runner_name }}|' {{ compose_dir }}/.env
+    when: '[ -n "{{ runner_name }}" ]'
+    changed_when: '[ -n "{{ runner_name }}" ] && ! grep -q "^GITEA_RUNNER_NAME={{ runner_name }}$" {{ compose_dir }}/.env'
+
+  - name: Patch GITEA_RUNNER_LABELS in .env (when --var runner_labels=)
+    shell: sed -i 's|^GITEA_RUNNER_LABELS=.*|GITEA_RUNNER_LABELS={{ runner_labels }}|' {{ compose_dir }}/.env
+    when: '[ -n "{{ runner_labels }}" ]'
+    changed_when: '[ -n "{{ runner_labels }}" ] && ! grep -q "^GITEA_RUNNER_LABELS={{ runner_labels }}$" {{ compose_dir }}/.env'
+
+  # ── safety: surface missing registration token before bring-up ──────
+  - name: Verify registration token is present (or runner is already registered)
+    # Either the .env carries a non-empty token (first run) or
+    # data_dir/.runner exists (already registered). If neither, the
+    # container will start but spin in a registration loop — fail
+    # fast with a useful message instead.
+    shell: |
+      if [ -f {{ data_dir }}/.runner ]; then
+        echo "already registered (.runner present)"
+        exit 0
+      fi
+      tok=$(grep -E '^GITEA_RUNNER_REGISTRATION_TOKEN=' {{ compose_dir }}/.env | cut -d= -f2-)
+      if [ -z "$tok" ]; then
+        echo "ERROR: no .runner cached and GITEA_RUNNER_REGISTRATION_TOKEN is empty in {{ compose_dir }}/.env"
+        echo "Generate one at https://gitea.phasefinal.com/-/admin/actions/runners and paste it into the .env, then re-run."
+        exit 1
+      fi
+      echo "registration token present"
+    changed_when: "false"
+
+  # ── bring up + wait for runner to come online ───────────────────────
+  - name: docker compose up -d
+    shell: cd {{ compose_dir }} && docker compose up -d
+
+  - name: Wait for runner to register / start polling
+    # The runner logs "Runner registered" on first start, then enters
+    # a polling loop ("Listening for tasks" / "polling"). Either is a
+    # success signal.
+    shell: |
+      for i in $(seq 1 30); do
+        if docker logs gitea-runner 2>&1 | grep -qE "Runner registered|Listening for tasks|poll|Starting runner"; then
+          exit 0
+        fi
+        sleep 2
+      done
+      echo "--- last 80 lines of gitea-runner logs ---"
+      docker logs gitea-runner --tail 80
+      exit 1
+    changed_when: "false"
+
+verify:
+  - name: Container is running
+    shell: docker inspect gitea-runner --format '{{.State.Status}}' | grep -q running
+    changed_when: "false"
+
+  - name: Runner credentials are cached on disk
+    shell: test -f {{ data_dir }}/.runner
+    changed_when: "false"
+
+  - name: Runner can reach gitea (no TLS / auth errors in last 50 log lines)
+    shell: '! docker logs gitea-runner --tail 50 2>&1 | grep -qiE "tls|x509|unauthorized|forbidden|connection refused"'
+    changed_when: "false"

stacks/gitea-runner/.env.example

@@ -0,0 +1,42 @@
+# Gitea instance the runner polls.
+GITEA_INSTANCE_URL=https://gitea.phasefinal.com
+
+# Runner identity. NAME shows in the runner list in gitea admin;
+# LABELS is what `runs-on:` in workflow YAML matches against.
+#
+# `pfi-fleet` is the cross-host label used by all PFI runners — keep
+# it on every runner. The host-specific label (ana-docker, nh3-docker,
+# esh-docker-vm) lets a workflow pin to a particular site if it has to
+# (e.g. needs LAN access to a host only reachable from there).
+#
+# Each label MUST carry a `:docker://<image>` schema. Without it,
+# act_runner v0.6+ falls back to "host" mode — jobs would execute
+# inside the runner container itself (Alpine, no apt/python/node)
+# instead of in a spawned job container, breaking any workflow that
+# expects a normal Linux userspace.
+#
+# `node:20-bookworm-slim` is a sane default: small (~150 MB), has git
+# + node (so `actions/checkout@v4` and other JS-based actions work),
+# debian-based so `apt-get install` works. Workflows can still override
+# with their own job-level `container: image:` if they need something
+# different.
+GITEA_RUNNER_NAME=ana-docker-runner
+GITEA_RUNNER_LABELS=pfi-fleet:docker://node:20-bookworm-slim,ana-docker:docker://node:20-bookworm-slim
+
+# One-time registration token. Generate at:
+#   https://gitea.phasefinal.com/-/admin/actions/runners (org/global)
+# or:
+#   https://gitea.phasefinal.com/<owner>/<repo>/settings/actions/runners (repo)
+#
+# Used only on first start; afterwards the runner caches a permanent
+# token at ${DATA_DIR}/.runner. Safe to clear after the runner shows
+# up in the gitea admin runner list.
+GITEA_RUNNER_REGISTRATION_TOKEN=
+
+# Persistence (.runner credentials, build cache, working dirs).
+DATA_DIR=/opt/docker/conf/gitea-runner/data
+
+# Pin to a tagged release in production; :latest is fine for
+# bootstrapping. Check https://gitea.com/gitea/act_runner/releases
+# for the current stable.
+RUNNER_IMAGE=gitea/act_runner:latest

stacks/gitea-runner/README.md

@@ -0,0 +1,134 @@
+# gitea-runner
+
+Self-hosted [Gitea Actions](https://docs.gitea.com/usage/actions/overview)
+runner. Polls `gitea.phasefinal.com` for jobs from any repo that has a
+`.gitea/workflows/` directory and runs them in ephemeral docker
+containers on this host.
+
+**Server:** ana-docker (single central runner — see "Topology" below
+for when to add more)
+**Image:** `gitea/act_runner:latest`
+**Outbound only** — no host port published; the runner connects out to
+gitea, gitea never connects in.
+
+## Topology
+
+We run **one central runner on ana-docker**. Reasoning:
+
+- gitea is on ana-docker, so runner→API is local
+- existing fleet tooling (`elway`, `sync-stacks`, `deploy-stack`,
+  `refresh-server-info`) already SSHes from one origin to all hosts;
+  the runner inherits that pattern
+- single point to manage SSH keys, secrets, and runner upgrades
+
+The deploy playbook (`playbooks/deploy-gitea-runner.yaml`) is
+parameterized by host / runner-name / labels, so spinning up
+`nh3-docker-runner` or an ESH runner later is a one-line elway
+invocation — not a copy-pasted playbook.
+
+**When to add a site-local runner:**
+
+- Cross-site SSH from ana-docker to that site has become unreliable
+- A workflow needs LAN access to something only reachable from inside
+  that site's network segment
+- You want failure isolation (NH3 can deploy itself when Anaheim is down)
+
+Until one of those bites, one runner is enough.
+
+## Prereqs
+
+Before running the deploy playbook:
+
+1. **Verify Gitea Actions is enabled.** In gitea 1.21+ Actions ships
+   on by default, but check `/-/admin/actions` resolves. If not, add
+   `GITEA__actions__ENABLED=true` to the gitea stack env and bounce.
+
+2. **Generate a registration token.** Pick the scope:
+
+   | Scope | URL | Use when |
+   |---|---|---|
+   | Global (admin) | `https://gitea.phasefinal.com/-/admin/actions/runners` | runner serves any repo on the instance (recommended for the central PFI runner) |
+   | Org/user | `https://gitea.phasefinal.com/<owner>/-/actions/runners` | runner serves all repos under one owner |
+   | Repo | `https://gitea.phasefinal.com/<owner>/<repo>/settings/actions/runners` | runner serves one repo |
+
+   Register-as-admin is right for our use case: one runner, fleet-wide.
+
+3. **Create an SSH deploy key for the runner** that lets it execute
+   the elway playbooks against fleet hosts. The key lives only on
+   ana-docker (passed in as a workflow secret per repo, or mounted
+   into the runner via volume — see "Wiring deploys" below).
+
+4. **(Optional) Create a Gitea PAT** with `read:repository` scope on
+   `vh/esh-pfi-infrastructure`. Workflows need to check out the
+   management repo to get at the playbooks; the auto-injected
+   `GITHUB_TOKEN` only works for the triggering repo.
+
+## Deploy
+
+```bash
+# Edit .env first if not using defaults — at minimum paste the registration token
+$EDITOR stacks/gitea-runner/.env.example   # template
+
+# First-time deploy
+scripts/elway ana-docker --playbook playbooks/deploy-gitea-runner.yaml
+
+# Site-local runner later (NH3 or ESH)
+scripts/elway nh3-docker --playbook playbooks/deploy-gitea-runner.yaml \
+  --var runner_name=nh3-docker-runner --var runner_labels=pfi-fleet,nh3-docker
+```
+
+The playbook seeds `.env` from `.env.example` only if absent; for the
+first run, copy `.env.example` to `/opt/docker/compose/gitea-runner/.env`
+on the host and paste the registration token in before running, OR
+let the playbook seed it and edit on the host before the
+`docker compose up -d` step (it's idempotent — second run will pick up
+the edited token).
+
+After successful registration, the token is consumed (it's one-time
+use). You can clear `GITEA_RUNNER_REGISTRATION_TOKEN` from `.env`;
+the runner reads its permanent credentials from `${DATA_DIR}/.runner`
+on subsequent starts.
+
+## Wiring deploys
+
+A workflow that runs on the central runner needs three things:
+
+1. **`runs-on:`** matching a runner label — `pfi-fleet` (cross-fleet)
+   or `ana-docker` (pin to that host).
+2. **An SSH key** to reach the deploy target. Stored as a repo or
+   org-level Actions secret named e.g. `DEPLOY_SSH_KEY`. The
+   corresponding public key must be in `~lkraven/.ssh/authorized_keys`
+   on every host the workflow targets.
+3. **A token to clone `vh/esh-pfi-infrastructure`** if the workflow
+   wants to invoke an elway playbook from this repo. Stored as
+   `MGMT_REPO_TOKEN` (Gitea PAT, `read:repository` scope).
+
+See `stacks/task-board/gitea-workflow-deploy.yaml.example` for a
+complete deploy workflow that consumes all three.
+
+## Path layout (on ana-docker)
+
+| Host path | Container path | Purpose | Restic? |
+|---|---|---|---|
+| `/opt/docker/compose/gitea-runner/` | — | compose.yaml + .env | included (via `/opt/docker`) |
+| `/opt/docker/conf/gitea-runner/data/` | `/data` | `.runner` creds, cache, job workspaces | excluded (regenerable; nothing irreplaceable) |
+
+## Operations
+
+```bash
+# Tail runner logs
+ssh ana-docker docker logs -f gitea-runner
+
+# List currently registered runners (admin)
+# https://gitea.phasefinal.com/-/admin/actions/runners
+
+# Re-register (lost the .runner file? regenerate token, then:)
+ssh ana-docker docker compose -f /opt/docker/compose/gitea-runner/compose.yaml down
+ssh ana-docker rm /opt/docker/conf/gitea-runner/data/.runner
+# paste new GITEA_RUNNER_REGISTRATION_TOKEN into .env
+scripts/elway ana-docker --playbook playbooks/deploy-gitea-runner.yaml
+
+# Pin to a specific act_runner version
+# Edit RUNNER_IMAGE in /opt/docker/compose/gitea-runner/.env, then:
+scripts/elway ana-docker --playbook playbooks/deploy-gitea-runner.yaml
+```

stacks/gitea-runner/compose.yaml

@@ -0,0 +1,38 @@
+# Gitea Actions self-hosted runner.
+#
+# Polls https://gitea.phasefinal.com for queued jobs and runs them in
+# ephemeral job-containers spawned via the host docker socket. The
+# runner registers itself on first start using
+# GITEA_RUNNER_REGISTRATION_TOKEN; subsequent starts reuse credentials
+# cached at ${DATA_DIR}/.runner.
+#
+# Tunables live in .env. config.yaml controls runner internals (job
+# timeouts, capacity, container engine settings) — edit conf/config.yaml,
+# don't inline that here.
+
+services:
+  runner:
+    image: ${RUNNER_IMAGE}
+    container_name: gitea-runner
+    restart: unless-stopped
+    environment:
+      - GITEA_INSTANCE_URL=${GITEA_INSTANCE_URL}
+      - GITEA_RUNNER_REGISTRATION_TOKEN=${GITEA_RUNNER_REGISTRATION_TOKEN}
+      - GITEA_RUNNER_NAME=${GITEA_RUNNER_NAME}
+      - GITEA_RUNNER_LABELS=${GITEA_RUNNER_LABELS}
+      - CONFIG_FILE=/data/config.yaml
+    volumes:
+      - ${DATA_DIR}:/data
+      - /var/run/docker.sock:/var/run/docker.sock
+    networks:
+      - tnet
+    labels:
+      - homepage.group=Toolchain
+      - homepage.name=gitea-runner
+      - homepage.icon=mdi-cog-play
+      - homepage.description=Gitea Actions self-hosted runner (${GITEA_RUNNER_NAME})
+
+networks:
+  tnet:
+    name: traefik-net
+    external: true

stacks/gitea-runner/conf/config.yaml

@@ -0,0 +1,55 @@
+# act_runner config. Mounted into the runner container at
+# /data/config.yaml (CONFIG_FILE env var points here).
+#
+# Reference: https://docs.gitea.com/usage/actions/act-runner
+
+log:
+  level: info
+
+runner:
+  # Persisted registration credentials. Created on first successful
+  # `register`; reused on subsequent starts.
+  file: /data/.runner
+
+  # Max parallel jobs this runner will accept.
+  capacity: 2
+
+  # Hard timeout per job — covers a wedged docker build, a hung ssh,
+  # etc. 30m is generous for our deploy workflows (mostly seconds).
+  timeout: 30m
+
+  # Time given to a job to clean up after a SIGTERM before SIGKILL.
+  shutdown_timeout: 1m
+
+  # TLS verification when talking to gitea. KEEP true in prod.
+  insecure: false
+
+  # Polling cadence + per-poll HTTP timeout.
+  fetch_timeout: 5s
+  fetch_interval: 2s
+
+cache:
+  # Provides actions-cache-compatible storage for `actions/cache`.
+  enabled: true
+  dir: /data/cache
+
+container:
+  # Job containers join this docker network. Lets workflow steps
+  # talk to other compose services (gitea itself, registries, etc.)
+  # by container name.
+  network: traefik-net
+
+  privileged: false
+
+  # Job containers' working dir is mounted under here on the host
+  # (via the runner's docker.sock spawning). Kept on the runner's
+  # /data volume so workspaces persist briefly between steps.
+  workdir_parent: /data/workspace
+
+  # Volumes the runner allows job containers to bind-mount. Keep tight.
+  valid_volumes: []
+
+  force_pull: false
+
+host:
+  workdir_parent: /data/host-workspace

stacks/task-board/README.md

@@ -11,8 +11,24 @@ the deploy playbook. Not pulled from a registry.
 
 ## Deploy
 
-Via elway — see `playbooks/deploy-task-board.yaml` in the eshpfi-management
+Two paths — automated (preferred) and manual (escape hatch / first-time).
-root. The playbook owns the full flow: clone/update the source repo,
+
+### Automated (Gitea Actions, push-to-main)
+
+Once the central gitea-runner is up (see `stacks/gitea-runner/README.md`),
+every push to `main` on `vh/task-board` triggers a deploy. The workflow
+just calls the same elway playbook below; the playbook stays the
+single source of truth for "how task-board is deployed."
+
+Workflow template lives next to this README at
+[`gitea-workflow-deploy.yaml.example`](gitea-workflow-deploy.yaml.example);
+copy it into the task-board repo at `.gitea/workflows/deploy.yaml`.
+The example header lists the two repo secrets required
+(`DEPLOY_SSH_KEY`, `MGMT_REPO_TOKEN`).
+
+### Manual (elway from a workstation)
+
+The playbook owns the full flow: clone/update the source repo,
 `docker build`, install compose + seed .env, bring up, verify health.
 
 ```bash

stacks/task-board/gitea-workflow-deploy.yaml.example

@@ -0,0 +1,91 @@
+# Gitea Actions workflow for task-board.
+#
+# THIS FILE LIVES IN THE TASK-BOARD REPO, NOT HERE.
+# Copy to vh/task-board:.gitea/workflows/deploy.yaml and commit.
+#
+# What it does on every push to main (and on manual workflow_dispatch):
+#   1. Checks out task-board itself (the triggering repo).
+#   2. Checks out vh/esh-pfi-infrastructure to pick up the elway
+#      playbook and helper scripts.
+#   3. Configures SSH so elway can reach ana-docker.
+#   4. Runs `scripts/elway ana-docker --playbook playbooks/deploy-task-board.yaml`
+#      pinning to the commit SHA that triggered the workflow.
+#
+# Required Actions secrets (configure under
+# https://gitea.phasefinal.com/vh/task-board/settings/actions/secrets,
+# or org-level for reuse across repos):
+#
+#   DEPLOY_SSH_KEY    Private SSH key whose pubkey is in
+#                     ~lkraven/.ssh/authorized_keys on ana-docker.
+#                     Used by the runner to invoke the elway playbook.
+#                     Generate fresh; don't reuse a personal key.
+#
+#   MGMT_REPO_TOKEN   Gitea PAT (read:repository scope) on
+#                     vh/esh-pfi-infrastructure, used to clone the
+#                     management repo. Generate at
+#                     https://gitea.phasefinal.com/-/user/settings/applications.
+
+name: Deploy task-board
+
+on:
+  push:
+    branches: [main]
+  workflow_dispatch:
+
+jobs:
+  deploy:
+    # `pfi-fleet` matches the central runner on ana-docker. Pin to
+    # `ana-docker` instead if you want to refuse running on a future
+    # site-local runner.
+    runs-on: pfi-fleet
+
+    # Debian image picked because the elway playbook uses python3 +
+    # pyyaml (Debian package: python3-yaml). Keeps the install step short.
+    container:
+      image: debian:bookworm-slim
+
+    steps:
+      - name: Install playbook prerequisites
+        run: |
+          apt-get update -qq
+          apt-get install -y --no-install-recommends \
+            python3 python3-yaml openssh-client ca-certificates curl git
+          rm -rf /var/lib/apt/lists/*
+
+      - name: Checkout task-board (triggering repo)
+        uses: actions/checkout@v4
+
+      - name: Checkout management repo (eshpfi-management)
+        uses: actions/checkout@v4
+        with:
+          repository: vh/esh-pfi-infrastructure
+          token: ${{ secrets.MGMT_REPO_TOKEN }}
+          path: _mgmt
+
+      - name: Configure SSH to ana-docker
+        run: |
+          mkdir -p ~/.ssh
+          # The DEPLOY_SSH_KEY secret is the full private key contents,
+          # newline-terminated. ssh refuses keys that aren't 0600.
+          printf '%s\n' "${{ secrets.DEPLOY_SSH_KEY }}" > ~/.ssh/id_ed25519
+          chmod 600 ~/.ssh/id_ed25519
+
+          # ssh_config alias so elway resolves "ana-docker" the same
+          # way it would on a workstation. accept-new is fine for a
+          # fresh job container — host key gets cached for the lifetime
+          # of this job only.
+          cat > ~/.ssh/config <<'EOF'
+          Host ana-docker
+            HostName 10.250.50.70
+            User lkraven
+            IdentityFile ~/.ssh/id_ed25519
+            StrictHostKeyChecking accept-new
+          EOF
+          chmod 600 ~/.ssh/config
+
+      - name: Deploy task-board (elway playbook, pinned to this commit)
+        working-directory: _mgmt
+        run: |
+          scripts/elway ana-docker \
+            --playbook playbooks/deploy-task-board.yaml \
+            --var ref=${{ github.sha }}