diff --git a/playbooks/deploy-gitea-runner.yaml b/playbooks/deploy-gitea-runner.yaml new file mode 100644 index 0000000..41c095d --- /dev/null +++ b/playbooks/deploy-gitea-runner.yaml @@ -0,0 +1,139 @@ +# Deploy a Gitea Actions self-hosted runner to a Docker host. +# +# We currently run ONE central runner on ana-docker (gitea is local +# there and SSH-from-there to the rest of the fleet already works). +# This playbook is parameterized so the same file can stand up +# site-local runners later without copy-paste — see +# stacks/gitea-runner/README.md "Topology" for when to do that. +# +# Usage: +# # Central runner (defaults match ana-docker) +# scripts/elway ana-docker --playbook playbooks/deploy-gitea-runner.yaml +# +# # Site-local runner — override identity at the CLI; the .env on +# # the host carries the registration token + image pin. +# scripts/elway nh3-docker --playbook playbooks/deploy-gitea-runner.yaml \ +# --var runner_name=nh3-docker-runner \ +# --var runner_labels=pfi-fleet,nh3-docker +# +# scripts/elway esh-docker-vm --playbook playbooks/deploy-gitea-runner.yaml \ +# --var runner_name=esh-runner \ +# --var runner_labels=pfi-fleet,esh +# +# Prereqs on the target host: +# - Docker + compose plugin +# - traefik-net network exists (external) +# - /opt/docker/{compose,conf} convention (any PFI host already has this) +# +# Prereqs in gitea (do once before the first run): +# - Generate a registration token at the appropriate scope +# (admin / org / repo). Paste into .env on the target host +# under GITEA_RUNNER_REGISTRATION_TOKEN before this playbook runs +# `docker compose up -d`. See stacks/gitea-runner/README.md. + +vars: + compose_dir: /opt/docker/compose/gitea-runner + data_dir: /opt/docker/conf/gitea-runner/data + # Optional overrides — leave blank to take whatever's already in the + # host-side .env. Setting these here patches the .env in place, + # which is how site-local runners get unique names/labels without + # editing files on the host by hand. + runner_name: "" + runner_labels: "" + +steps: + # ── host-side directory prep ──────────────────────────────────────── + - name: Ensure compose dir exists + shell: mkdir -p {{ compose_dir }} + creates: "{{ compose_dir }}" + + - name: Ensure data dir exists + shell: mkdir -p {{ data_dir }} + creates: "{{ data_dir }}" + + # ── compose + config files ────────────────────────────────────────── + - name: Upload compose.yaml + upload: + src: stacks/gitea-runner/compose.yaml + dest: "{{ compose_dir }}/compose.yaml" + mode: "0644" + + - name: Upload runner config.yaml + upload: + src: stacks/gitea-runner/conf/config.yaml + dest: "{{ data_dir }}/config.yaml" + mode: "0644" + + - name: Seed .env from template (only if absent) + upload: + src: stacks/gitea-runner/.env.example + dest: "{{ compose_dir }}/.env" + mode: "0644" + when: "[ ! -f {{ compose_dir }}/.env ]" + + # ── optional CLI-driven identity overrides ────────────────────────── + # If runner_name was passed as --var, patch the line in the host's + # .env so subsequent compose-up uses it. Idempotent: changed_when + # checks whether the value differs from what's already there. + - name: Patch GITEA_RUNNER_NAME in .env (when --var runner_name=) + shell: sed -i 's|^GITEA_RUNNER_NAME=.*|GITEA_RUNNER_NAME={{ runner_name }}|' {{ compose_dir }}/.env + when: '[ -n "{{ runner_name }}" ]' + changed_when: '[ -n "{{ runner_name }}" ] && ! grep -q "^GITEA_RUNNER_NAME={{ runner_name }}$" {{ compose_dir }}/.env' + + - name: Patch GITEA_RUNNER_LABELS in .env (when --var runner_labels=) + shell: sed -i 's|^GITEA_RUNNER_LABELS=.*|GITEA_RUNNER_LABELS={{ runner_labels }}|' {{ compose_dir }}/.env + when: '[ -n "{{ runner_labels }}" ]' + changed_when: '[ -n "{{ runner_labels }}" ] && ! grep -q "^GITEA_RUNNER_LABELS={{ runner_labels }}$" {{ compose_dir }}/.env' + + # ── safety: surface missing registration token before bring-up ────── + - name: Verify registration token is present (or runner is already registered) + # Either the .env carries a non-empty token (first run) or + # data_dir/.runner exists (already registered). If neither, the + # container will start but spin in a registration loop — fail + # fast with a useful message instead. + shell: | + if [ -f {{ data_dir }}/.runner ]; then + echo "already registered (.runner present)" + exit 0 + fi + tok=$(grep -E '^GITEA_RUNNER_REGISTRATION_TOKEN=' {{ compose_dir }}/.env | cut -d= -f2-) + if [ -z "$tok" ]; then + echo "ERROR: no .runner cached and GITEA_RUNNER_REGISTRATION_TOKEN is empty in {{ compose_dir }}/.env" + echo "Generate one at https://gitea.phasefinal.com/-/admin/actions/runners and paste it into the .env, then re-run." + exit 1 + fi + echo "registration token present" + changed_when: "false" + + # ── bring up + wait for runner to come online ─────────────────────── + - name: docker compose up -d + shell: cd {{ compose_dir }} && docker compose up -d + + - name: Wait for runner to register / start polling + # The runner logs "Runner registered" on first start, then enters + # a polling loop ("Listening for tasks" / "polling"). Either is a + # success signal. + shell: | + for i in $(seq 1 30); do + if docker logs gitea-runner 2>&1 | grep -qE "Runner registered|Listening for tasks|poll|Starting runner"; then + exit 0 + fi + sleep 2 + done + echo "--- last 80 lines of gitea-runner logs ---" + docker logs gitea-runner --tail 80 + exit 1 + changed_when: "false" + +verify: + - name: Container is running + shell: docker inspect gitea-runner --format '{{.State.Status}}' | grep -q running + changed_when: "false" + + - name: Runner credentials are cached on disk + shell: test -f {{ data_dir }}/.runner + changed_when: "false" + + - name: Runner can reach gitea (no TLS / auth errors in last 50 log lines) + shell: '! docker logs gitea-runner --tail 50 2>&1 | grep -qiE "tls|x509|unauthorized|forbidden|connection refused"' + changed_when: "false" diff --git a/stacks/gitea-runner/.env.example b/stacks/gitea-runner/.env.example new file mode 100644 index 0000000..4da293a --- /dev/null +++ b/stacks/gitea-runner/.env.example @@ -0,0 +1,42 @@ +# Gitea instance the runner polls. +GITEA_INSTANCE_URL=https://gitea.phasefinal.com + +# Runner identity. NAME shows in the runner list in gitea admin; +# LABELS is what `runs-on:` in workflow YAML matches against. +# +# `pfi-fleet` is the cross-host label used by all PFI runners — keep +# it on every runner. The host-specific label (ana-docker, nh3-docker, +# esh-docker-vm) lets a workflow pin to a particular site if it has to +# (e.g. needs LAN access to a host only reachable from there). +# +# Each label MUST carry a `:docker://` schema. Without it, +# act_runner v0.6+ falls back to "host" mode — jobs would execute +# inside the runner container itself (Alpine, no apt/python/node) +# instead of in a spawned job container, breaking any workflow that +# expects a normal Linux userspace. +# +# `node:20-bookworm-slim` is a sane default: small (~150 MB), has git +# + node (so `actions/checkout@v4` and other JS-based actions work), +# debian-based so `apt-get install` works. Workflows can still override +# with their own job-level `container: image:` if they need something +# different. +GITEA_RUNNER_NAME=ana-docker-runner +GITEA_RUNNER_LABELS=pfi-fleet:docker://node:20-bookworm-slim,ana-docker:docker://node:20-bookworm-slim + +# One-time registration token. Generate at: +# https://gitea.phasefinal.com/-/admin/actions/runners (org/global) +# or: +# https://gitea.phasefinal.com///settings/actions/runners (repo) +# +# Used only on first start; afterwards the runner caches a permanent +# token at ${DATA_DIR}/.runner. Safe to clear after the runner shows +# up in the gitea admin runner list. +GITEA_RUNNER_REGISTRATION_TOKEN= + +# Persistence (.runner credentials, build cache, working dirs). +DATA_DIR=/opt/docker/conf/gitea-runner/data + +# Pin to a tagged release in production; :latest is fine for +# bootstrapping. Check https://gitea.com/gitea/act_runner/releases +# for the current stable. +RUNNER_IMAGE=gitea/act_runner:latest diff --git a/stacks/gitea-runner/README.md b/stacks/gitea-runner/README.md new file mode 100644 index 0000000..acbeffa --- /dev/null +++ b/stacks/gitea-runner/README.md @@ -0,0 +1,134 @@ +# gitea-runner + +Self-hosted [Gitea Actions](https://docs.gitea.com/usage/actions/overview) +runner. Polls `gitea.phasefinal.com` for jobs from any repo that has a +`.gitea/workflows/` directory and runs them in ephemeral docker +containers on this host. + +**Server:** ana-docker (single central runner — see "Topology" below +for when to add more) +**Image:** `gitea/act_runner:latest` +**Outbound only** — no host port published; the runner connects out to +gitea, gitea never connects in. + +## Topology + +We run **one central runner on ana-docker**. Reasoning: + +- gitea is on ana-docker, so runner→API is local +- existing fleet tooling (`elway`, `sync-stacks`, `deploy-stack`, + `refresh-server-info`) already SSHes from one origin to all hosts; + the runner inherits that pattern +- single point to manage SSH keys, secrets, and runner upgrades + +The deploy playbook (`playbooks/deploy-gitea-runner.yaml`) is +parameterized by host / runner-name / labels, so spinning up +`nh3-docker-runner` or an ESH runner later is a one-line elway +invocation — not a copy-pasted playbook. + +**When to add a site-local runner:** + +- Cross-site SSH from ana-docker to that site has become unreliable +- A workflow needs LAN access to something only reachable from inside + that site's network segment +- You want failure isolation (NH3 can deploy itself when Anaheim is down) + +Until one of those bites, one runner is enough. + +## Prereqs + +Before running the deploy playbook: + +1. **Verify Gitea Actions is enabled.** In gitea 1.21+ Actions ships + on by default, but check `/-/admin/actions` resolves. If not, add + `GITEA__actions__ENABLED=true` to the gitea stack env and bounce. + +2. **Generate a registration token.** Pick the scope: + + | Scope | URL | Use when | + |---|---|---| + | Global (admin) | `https://gitea.phasefinal.com/-/admin/actions/runners` | runner serves any repo on the instance (recommended for the central PFI runner) | + | Org/user | `https://gitea.phasefinal.com//-/actions/runners` | runner serves all repos under one owner | + | Repo | `https://gitea.phasefinal.com///settings/actions/runners` | runner serves one repo | + + Register-as-admin is right for our use case: one runner, fleet-wide. + +3. **Create an SSH deploy key for the runner** that lets it execute + the elway playbooks against fleet hosts. The key lives only on + ana-docker (passed in as a workflow secret per repo, or mounted + into the runner via volume — see "Wiring deploys" below). + +4. **(Optional) Create a Gitea PAT** with `read:repository` scope on + `vh/esh-pfi-infrastructure`. Workflows need to check out the + management repo to get at the playbooks; the auto-injected + `GITHUB_TOKEN` only works for the triggering repo. + +## Deploy + +```bash +# Edit .env first if not using defaults — at minimum paste the registration token +$EDITOR stacks/gitea-runner/.env.example # template + +# First-time deploy +scripts/elway ana-docker --playbook playbooks/deploy-gitea-runner.yaml + +# Site-local runner later (NH3 or ESH) +scripts/elway nh3-docker --playbook playbooks/deploy-gitea-runner.yaml \ + --var runner_name=nh3-docker-runner --var runner_labels=pfi-fleet,nh3-docker +``` + +The playbook seeds `.env` from `.env.example` only if absent; for the +first run, copy `.env.example` to `/opt/docker/compose/gitea-runner/.env` +on the host and paste the registration token in before running, OR +let the playbook seed it and edit on the host before the +`docker compose up -d` step (it's idempotent — second run will pick up +the edited token). + +After successful registration, the token is consumed (it's one-time +use). You can clear `GITEA_RUNNER_REGISTRATION_TOKEN` from `.env`; +the runner reads its permanent credentials from `${DATA_DIR}/.runner` +on subsequent starts. + +## Wiring deploys + +A workflow that runs on the central runner needs three things: + +1. **`runs-on:`** matching a runner label — `pfi-fleet` (cross-fleet) + or `ana-docker` (pin to that host). +2. **An SSH key** to reach the deploy target. Stored as a repo or + org-level Actions secret named e.g. `DEPLOY_SSH_KEY`. The + corresponding public key must be in `~lkraven/.ssh/authorized_keys` + on every host the workflow targets. +3. **A token to clone `vh/esh-pfi-infrastructure`** if the workflow + wants to invoke an elway playbook from this repo. Stored as + `MGMT_REPO_TOKEN` (Gitea PAT, `read:repository` scope). + +See `stacks/task-board/gitea-workflow-deploy.yaml.example` for a +complete deploy workflow that consumes all three. + +## Path layout (on ana-docker) + +| Host path | Container path | Purpose | Restic? | +|---|---|---|---| +| `/opt/docker/compose/gitea-runner/` | — | compose.yaml + .env | included (via `/opt/docker`) | +| `/opt/docker/conf/gitea-runner/data/` | `/data` | `.runner` creds, cache, job workspaces | excluded (regenerable; nothing irreplaceable) | + +## Operations + +```bash +# Tail runner logs +ssh ana-docker docker logs -f gitea-runner + +# List currently registered runners (admin) +# https://gitea.phasefinal.com/-/admin/actions/runners + +# Re-register (lost the .runner file? regenerate token, then:) +ssh ana-docker docker compose -f /opt/docker/compose/gitea-runner/compose.yaml down +ssh ana-docker rm /opt/docker/conf/gitea-runner/data/.runner +# paste new GITEA_RUNNER_REGISTRATION_TOKEN into .env +scripts/elway ana-docker --playbook playbooks/deploy-gitea-runner.yaml + +# Pin to a specific act_runner version +# Edit RUNNER_IMAGE in /opt/docker/compose/gitea-runner/.env, then: +scripts/elway ana-docker --playbook playbooks/deploy-gitea-runner.yaml +``` diff --git a/stacks/gitea-runner/compose.yaml b/stacks/gitea-runner/compose.yaml new file mode 100644 index 0000000..965985e --- /dev/null +++ b/stacks/gitea-runner/compose.yaml @@ -0,0 +1,38 @@ +# Gitea Actions self-hosted runner. +# +# Polls https://gitea.phasefinal.com for queued jobs and runs them in +# ephemeral job-containers spawned via the host docker socket. The +# runner registers itself on first start using +# GITEA_RUNNER_REGISTRATION_TOKEN; subsequent starts reuse credentials +# cached at ${DATA_DIR}/.runner. +# +# Tunables live in .env. config.yaml controls runner internals (job +# timeouts, capacity, container engine settings) — edit conf/config.yaml, +# don't inline that here. + +services: + runner: + image: ${RUNNER_IMAGE} + container_name: gitea-runner + restart: unless-stopped + environment: + - GITEA_INSTANCE_URL=${GITEA_INSTANCE_URL} + - GITEA_RUNNER_REGISTRATION_TOKEN=${GITEA_RUNNER_REGISTRATION_TOKEN} + - GITEA_RUNNER_NAME=${GITEA_RUNNER_NAME} + - GITEA_RUNNER_LABELS=${GITEA_RUNNER_LABELS} + - CONFIG_FILE=/data/config.yaml + volumes: + - ${DATA_DIR}:/data + - /var/run/docker.sock:/var/run/docker.sock + networks: + - tnet + labels: + - homepage.group=Toolchain + - homepage.name=gitea-runner + - homepage.icon=mdi-cog-play + - homepage.description=Gitea Actions self-hosted runner (${GITEA_RUNNER_NAME}) + +networks: + tnet: + name: traefik-net + external: true diff --git a/stacks/gitea-runner/conf/config.yaml b/stacks/gitea-runner/conf/config.yaml new file mode 100644 index 0000000..fded738 --- /dev/null +++ b/stacks/gitea-runner/conf/config.yaml @@ -0,0 +1,55 @@ +# act_runner config. Mounted into the runner container at +# /data/config.yaml (CONFIG_FILE env var points here). +# +# Reference: https://docs.gitea.com/usage/actions/act-runner + +log: + level: info + +runner: + # Persisted registration credentials. Created on first successful + # `register`; reused on subsequent starts. + file: /data/.runner + + # Max parallel jobs this runner will accept. + capacity: 2 + + # Hard timeout per job — covers a wedged docker build, a hung ssh, + # etc. 30m is generous for our deploy workflows (mostly seconds). + timeout: 30m + + # Time given to a job to clean up after a SIGTERM before SIGKILL. + shutdown_timeout: 1m + + # TLS verification when talking to gitea. KEEP true in prod. + insecure: false + + # Polling cadence + per-poll HTTP timeout. + fetch_timeout: 5s + fetch_interval: 2s + +cache: + # Provides actions-cache-compatible storage for `actions/cache`. + enabled: true + dir: /data/cache + +container: + # Job containers join this docker network. Lets workflow steps + # talk to other compose services (gitea itself, registries, etc.) + # by container name. + network: traefik-net + + privileged: false + + # Job containers' working dir is mounted under here on the host + # (via the runner's docker.sock spawning). Kept on the runner's + # /data volume so workspaces persist briefly between steps. + workdir_parent: /data/workspace + + # Volumes the runner allows job containers to bind-mount. Keep tight. + valid_volumes: [] + + force_pull: false + +host: + workdir_parent: /data/host-workspace diff --git a/stacks/task-board/README.md b/stacks/task-board/README.md index a2944cc..9941951 100644 --- a/stacks/task-board/README.md +++ b/stacks/task-board/README.md @@ -11,8 +11,24 @@ the deploy playbook. Not pulled from a registry. ## Deploy -Via elway — see `playbooks/deploy-task-board.yaml` in the eshpfi-management -root. The playbook owns the full flow: clone/update the source repo, +Two paths — automated (preferred) and manual (escape hatch / first-time). + +### Automated (Gitea Actions, push-to-main) + +Once the central gitea-runner is up (see `stacks/gitea-runner/README.md`), +every push to `main` on `vh/task-board` triggers a deploy. The workflow +just calls the same elway playbook below; the playbook stays the +single source of truth for "how task-board is deployed." + +Workflow template lives next to this README at +[`gitea-workflow-deploy.yaml.example`](gitea-workflow-deploy.yaml.example); +copy it into the task-board repo at `.gitea/workflows/deploy.yaml`. +The example header lists the two repo secrets required +(`DEPLOY_SSH_KEY`, `MGMT_REPO_TOKEN`). + +### Manual (elway from a workstation) + +The playbook owns the full flow: clone/update the source repo, `docker build`, install compose + seed .env, bring up, verify health. ```bash diff --git a/stacks/task-board/gitea-workflow-deploy.yaml.example b/stacks/task-board/gitea-workflow-deploy.yaml.example new file mode 100644 index 0000000..c61494d --- /dev/null +++ b/stacks/task-board/gitea-workflow-deploy.yaml.example @@ -0,0 +1,91 @@ +# Gitea Actions workflow for task-board. +# +# THIS FILE LIVES IN THE TASK-BOARD REPO, NOT HERE. +# Copy to vh/task-board:.gitea/workflows/deploy.yaml and commit. +# +# What it does on every push to main (and on manual workflow_dispatch): +# 1. Checks out task-board itself (the triggering repo). +# 2. Checks out vh/esh-pfi-infrastructure to pick up the elway +# playbook and helper scripts. +# 3. Configures SSH so elway can reach ana-docker. +# 4. Runs `scripts/elway ana-docker --playbook playbooks/deploy-task-board.yaml` +# pinning to the commit SHA that triggered the workflow. +# +# Required Actions secrets (configure under +# https://gitea.phasefinal.com/vh/task-board/settings/actions/secrets, +# or org-level for reuse across repos): +# +# DEPLOY_SSH_KEY Private SSH key whose pubkey is in +# ~lkraven/.ssh/authorized_keys on ana-docker. +# Used by the runner to invoke the elway playbook. +# Generate fresh; don't reuse a personal key. +# +# MGMT_REPO_TOKEN Gitea PAT (read:repository scope) on +# vh/esh-pfi-infrastructure, used to clone the +# management repo. Generate at +# https://gitea.phasefinal.com/-/user/settings/applications. + +name: Deploy task-board + +on: + push: + branches: [main] + workflow_dispatch: + +jobs: + deploy: + # `pfi-fleet` matches the central runner on ana-docker. Pin to + # `ana-docker` instead if you want to refuse running on a future + # site-local runner. + runs-on: pfi-fleet + + # Debian image picked because the elway playbook uses python3 + + # pyyaml (Debian package: python3-yaml). Keeps the install step short. + container: + image: debian:bookworm-slim + + steps: + - name: Install playbook prerequisites + run: | + apt-get update -qq + apt-get install -y --no-install-recommends \ + python3 python3-yaml openssh-client ca-certificates curl git + rm -rf /var/lib/apt/lists/* + + - name: Checkout task-board (triggering repo) + uses: actions/checkout@v4 + + - name: Checkout management repo (eshpfi-management) + uses: actions/checkout@v4 + with: + repository: vh/esh-pfi-infrastructure + token: ${{ secrets.MGMT_REPO_TOKEN }} + path: _mgmt + + - name: Configure SSH to ana-docker + run: | + mkdir -p ~/.ssh + # The DEPLOY_SSH_KEY secret is the full private key contents, + # newline-terminated. ssh refuses keys that aren't 0600. + printf '%s\n' "${{ secrets.DEPLOY_SSH_KEY }}" > ~/.ssh/id_ed25519 + chmod 600 ~/.ssh/id_ed25519 + + # ssh_config alias so elway resolves "ana-docker" the same + # way it would on a workstation. accept-new is fine for a + # fresh job container — host key gets cached for the lifetime + # of this job only. + cat > ~/.ssh/config <<'EOF' + Host ana-docker + HostName 10.250.50.70 + User lkraven + IdentityFile ~/.ssh/id_ed25519 + StrictHostKeyChecking accept-new + EOF + chmod 600 ~/.ssh/config + + - name: Deploy task-board (elway playbook, pinned to this commit) + working-directory: _mgmt + run: | + scripts/elway ana-docker \ + --playbook playbooks/deploy-task-board.yaml \ + --var ref=${{ github.sha }}