From cd3c4b8be99fcca621fba4159e1d6a9e70fd4557 Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Mon, 12 Oct 2026 17:27:46 -0700 Subject: [PATCH] feat(gitea-runner): opt-in fast runner on nh3-docker (label nh3-docker), no docker socket in jobs Prime's CI fix A. ana-docker's runner sits on HDD mirrors with no SLOG; synced 8 KiB writes measured ~34 ms there vs ~2.7 ms on nh3-docker (same probe, one run, n=3 each). Repos opt in with runs-on: nh3-docker; the runner carries no pfi-fleet label. Its config is tighter because nh3-docker also runs NH3 DNS and the post office: docker_host '-' (verified absent in the job, present on the ana-docker baseline), capacity 1, jobs capped at 4 GiB / 6 CPUs, a network per job. The deploy playbook takes runner_config; each runner's homepage name is its runner name. --- playbooks/deploy-gitea-runner.yaml | 12 ++++-- stacks/gitea-runner/README.md | 40 ++++++++++++++++- stacks/gitea-runner/compose.yaml | 2 +- .../gitea-runner/conf/config.nh3-docker.yaml | 43 +++++++++++++++++++ 4 files changed, 90 insertions(+), 7 deletions(-) create mode 100644 stacks/gitea-runner/conf/config.nh3-docker.yaml diff --git a/playbooks/deploy-gitea-runner.yaml b/playbooks/deploy-gitea-runner.yaml index 41c095d..7293086 100644 --- a/playbooks/deploy-gitea-runner.yaml +++ b/playbooks/deploy-gitea-runner.yaml @@ -10,11 +10,12 @@ # # Central runner (defaults match ana-docker) # scripts/elway ana-docker --playbook playbooks/deploy-gitea-runner.yaml # -# # Site-local runner — override identity at the CLI; the .env on -# # the host carries the registration token + image pin. +# # The OPT-IN fast runner on nh3-docker (Prime's CI fix A, 2026-10-12): its own label only, and its own config. +# # The .env on the host carries the registration token + image pin. # scripts/elway nh3-docker --playbook playbooks/deploy-gitea-runner.yaml \ # --var runner_name=nh3-docker-runner \ -# --var runner_labels=pfi-fleet,nh3-docker +# --var runner_labels=nh3-docker:docker://node:20-bookworm-slim \ +# --var runner_config=stacks/gitea-runner/conf/config.nh3-docker.yaml # # scripts/elway esh-docker-vm --playbook playbooks/deploy-gitea-runner.yaml \ # --var runner_name=esh-runner \ @@ -40,6 +41,9 @@ vars: # editing files on the host by hand. runner_name: "" runner_labels: "" + # Which act_runner config to upload. The nh3-docker fast runner uses its own, tighter one + # (conf/config.nh3-docker.yaml: no docker socket in jobs, capacity 1, memory-capped jobs). + runner_config: stacks/gitea-runner/conf/config.yaml steps: # ── host-side directory prep ──────────────────────────────────────── @@ -60,7 +64,7 @@ steps: - name: Upload runner config.yaml upload: - src: stacks/gitea-runner/conf/config.yaml + src: "{{ runner_config }}" dest: "{{ data_dir }}/config.yaml" mode: "0644" diff --git a/stacks/gitea-runner/README.md b/stacks/gitea-runner/README.md index 75eb57f..2181123 100644 --- a/stacks/gitea-runner/README.md +++ b/stacks/gitea-runner/README.md @@ -5,8 +5,7 @@ runner. Polls `gitea.phasefinal.com` for jobs from any repo that has a `.gitea/workflows/` directory and runs them in ephemeral docker containers on this host. -**Server:** ana-docker (single central runner — see "Topology" below -for when to add more) +**Server:** ana-docker (the central runner), plus the opt-in fast runner on nh3-docker (see "Topology") **Image:** `gitea/act_runner:latest` **Outbound only** — no host port published; the runner connects out to gitea, gitea never connects in. @@ -35,6 +34,43 @@ invocation — not a copy-pasted playbook. Until one of those bites, one runner is enough. +### The opt-in fast runner on nh3-docker (Prime's CI fix A, 2026-10-12) + +`nh3-docker-runner` (gitea runner id 4, act_runner v0.6.1) carries **one label, `nh3-docker`**, and NOT `pfi-fleet`. +Nothing lands on it unless a workflow says `runs-on: nh3-docker`. It exists because ana-docker's disk is slow to +fsync: pfi-pve's `ospool` is HDD mirrors with no SLOG. Jobs that fsync a lot (a Postgres service, pytest temp dirs) +crawled into the 30 m timeout there. + +Measured 2026-10-12 1725 in a throwaway repo (`vh/ci-probe-nh3`, deleted after). One run had a job on each runner +with the same steps (8 KiB `dd oflag=dsync` × 200, n = 3 each): + +| | nh3-docker-runner | ana-docker-runner | +|---|---|---| +| 200 synced 8 KiB writes | 0.51 / 0.54 / 0.57 s (~2.7 ms each) | 6.36 / 6.82 / 7.21 s (~34 ms each) | +| docker socket in the job | **absent** | present (root over the host) | +| job memory / CPU limit | 4 GiB / 6 CPUs | none | +| `services:` container | Postgres reachable after 1 s | (not tested here) | + +Its config is `conf/config.nh3-docker.yaml`, tighter than ana-docker's because nh3-docker also runs NH3's DNS +(adguardhome) and the althing post office on 8 vCPU / 7 GiB: +- `docker_host: "-"` keeps the docker socket out of jobs (the probe above proves it). `services:` still work. +- `capacity: 1`, and each job container is capped at `--memory=4g --memory-swap=4g --cpus=6`. +- `network: ""` gives every job its own network instead of traefik-net. +- The timeout stays at 30 m. + +A job that needs docker inside it (buildx and the like) cannot run here; keep those on `pfi-fleet`. ⚠ nh3-docker's +vCPU has no AVX2 either, so CPU-bound torch tests are no faster here. This runner fixes the disk, not the CPU. + +Deployed with: +```bash +scripts/elway infra-ops@10.100.50.40 --playbook playbooks/deploy-gitea-runner.yaml \ + --var runner_name=nh3-docker-runner --var runner_labels=nh3-docker:docker://node:20-bookworm-slim \ + --var runner_config=stacks/gitea-runner/conf/config.nh3-docker.yaml +``` +Use `infra-ops@`, not the `nh3-docker` alias: the alias logs in as lkraven, and the first attempt left a +lkraven-owned data dir that infra-ops could not write. The registration token came from the admin API +(`POST /api/v1/admin/actions/runners/registration-token`) and was cleared from `.env` once the runner registered. + ## What a job on this runner can actually do Measured 2026-09-02 on `pfi-fleet` (throwaway repo, three jobs, since deleted). diff --git a/stacks/gitea-runner/compose.yaml b/stacks/gitea-runner/compose.yaml index 167a6a1..e783600 100644 --- a/stacks/gitea-runner/compose.yaml +++ b/stacks/gitea-runner/compose.yaml @@ -28,7 +28,7 @@ services: - tnet labels: - homepage.group=Agents (no UI) - - homepage.name=gitea-runner + - homepage.name=${GITEA_RUNNER_NAME} - homepage.icon=mdi-cog-play - homepage.description=Gitea Actions self-hosted runner (${GITEA_RUNNER_NAME}) diff --git a/stacks/gitea-runner/conf/config.nh3-docker.yaml b/stacks/gitea-runner/conf/config.nh3-docker.yaml new file mode 100644 index 0000000..fb1d122 --- /dev/null +++ b/stacks/gitea-runner/conf/config.nh3-docker.yaml @@ -0,0 +1,43 @@ +# act_runner config for the OPT-IN fast runner on nh3-docker (Prime's CI fix A, 2026-10-12). +# Mounted at /data/config.yaml. Deployed by playbooks/deploy-gitea-runner.yaml with +# --var runner_config=stacks/gitea-runner/conf/config.nh3-docker.yaml +# +# Why it exists: the central runner on ana-docker sits on pfi-pve's `ospool` (HDD mirrors, no SLOG); 8 KiB fsync +# writes measured ~145 ms there vs ~2.7 ms on nh3-docker. Jobs that fsync a lot (a Postgres service, pytest temp +# dirs) ran past the 30 m job timeout. Repos opt in with `runs-on: nh3-docker`. The runner carries ONLY that label, +# never `pfi-fleet`, so nothing lands here by accident. +# +# Why it is tighter than ana-docker's config: nh3-docker also runs NH3's DNS (adguardhome) and the althing post +# office, on 8 vCPU / 7 GiB. So: +# - docker_host "-": the job containers do NOT get the host docker socket (ana-docker's runner hands every job root +# over its host). `services:` still work, because the runner creates them through its own socket. +# - capacity 1, and each job container is capped at 4 GiB / 6 CPUs, leaving room for DNS and the bus. +# - network "": every job gets its own throwaway network, not traefik-net. + +log: + level: info + +runner: + file: /data/.runner + capacity: 1 + timeout: 30m + shutdown_timeout: 1m + insecure: false + fetch_timeout: 5s + fetch_interval: 2s + +cache: + enabled: true + dir: /data/cache + +container: + network: "" + privileged: false + options: "--memory=4g --memory-swap=4g --cpus=6" + workdir_parent: /data/workspace + valid_volumes: [] + docker_host: "-" + force_pull: false + +host: + workdir_parent: /data/host-workspace