From d3f59aa2c541a8b571300ad536bbf15ed21fac2b Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Sun, 4 Oct 2026 23:33:24 -0700 Subject: [PATCH] feat(albok): move albok-service to a dedicated VM, nh3-albok (VM 110 on nh3-pve-2) albok-service leaves nh3-docker before unit 4 puts the personal vault and licensed books into restricted wings. On nh3-docker, portainer, portainer_agent, dockge (socket rw) and dockerd on plaintext tcp/2375 were each root over /srv/albok. The new host runs only albok-service: no socket containers and no TCP Docker API, so its Homepage tile is a static services.yaml entry. - nh3-pve-2: vmbr0 made VLAN-aware (applied by reboot, no guests running); pbs-ana storage in the new namespace nh3-pve-2, plus a 21:30 vzdump job (pbs-nh3's recursive sync mirrors it) - VM 110 nh3-albok, 10.100.50.44 on VLAN 50, Debian 13 cloud-init, 4 vCPU x86-64-v3 / 12 GiB / 100 GB; built to fleet conventions (vh 1000, infra-ops 850, docker 851) - compose: mem_limit 4g -> 8g, homepage labels removed; host port 8392 kept - cutover by tar stream, verified by digest (contents plus owner, group, mode, mtime and type identical); albok.nh3.internal now points at nh3-albok - old /srv/albok stays on nh3-docker (container removed, compose renamed) until acceptance and the operator's approval to delete it - sudo-io archive hosts and ops-log STACK_HOSTS include nh3-albok --- CLAUDE.md | 2 + dns/internal.yaml | 3 +- ...-10-04-albok-service-moved-to-nh3-albok.md | 35 ++++ persistent-memory.md | 4 +- playbooks/albok-service-host.yaml | 4 +- playbooks/nh3-albok-host.yaml | 95 +++++++++ playbooks/nh3-albok-vm.yaml | 98 +++++++++ playbooks/nh3-pve-2-vmbr0-vlan-aware.yaml | 63 ++++++ scripts/ops-log | 2 +- servers/nh3-albok/README.md | 57 ++++++ servers/nh3-albok/ssh-target | 1 + servers/nh3-albok/system-details.txt | 189 ++++++++++++++++++ servers/nh3-pve-2/README.md | 30 ++- services/sudo-io-archive/hosts | 1 + stacks/albok-service/README.md | 24 ++- stacks/albok-service/compose.yaml | 19 +- stacks/homepage/conf/services.yaml | 10 + 17 files changed, 607 insertions(+), 30 deletions(-) create mode 100644 persistent-memory.d/2026-10-04-albok-service-moved-to-nh3-albok.md create mode 100644 playbooks/nh3-albok-host.yaml create mode 100644 playbooks/nh3-albok-vm.yaml create mode 100644 playbooks/nh3-pve-2-vmbr0-vlan-aware.yaml create mode 100644 servers/nh3-albok/README.md create mode 100644 servers/nh3-albok/ssh-target create mode 100644 servers/nh3-albok/system-details.txt diff --git a/CLAUDE.md b/CLAUDE.md index 9d56edd..3dcdb67 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -360,6 +360,8 @@ Observed and standardized across servers: | nh3-dev | 10.100.10.50 | NH3 | Dev box — fleet sidecars (egress SOCKS5 proxy, ttyd seat, mead-hall, volva) + live Claude Code sessions; not a Docker-stack host | `servers/nh3-dev/README.md` | | nh3-extdev | 10.100.50.42 | NH3 | Manager / external-dev box (VM on nh3-pve, Debian 13); infra-ops is user-level and has no Docker, but **does have NOPASSWD sudo** — measured 2026-09-05, `sudo -n whoami` → `root`; this row previously said "sudo-less", which was wrong; successor to retired nh3-ansible | `servers/nh3-extdev/README.md` | | nh3-pve | 10.100.250.60 | NH3 | Proxmox VE hypervisor | `servers/nh3-pve/README.md` | +| nh3-pve-2 | 10.100.250.62 | NH3 | Proxmox VE hypervisor (Minisforum MS-03, standalone, not clustered); purpose TBD, first guest nh3-albok | `servers/nh3-pve-2/README.md` | +| nh3-albok | 10.100.50.44 | NH3 | VM 110 on nh3-pve-2 — runs **only** albok-service (fleet knowledgebase); no Docker API on tcp, no socket containers, by design | `servers/nh3-albok/README.md` | | nh3-nas | 10.100.50.50 | NH3 | Synology RS2418+ — NFS exports, rest-server-nh3, PBS-NH3 datastore backend | `servers/nh3-nas/README.md` | | pbs-nh3 | 10.100.50.90 | NH3 | Proxmox Backup Server — DR mirror (VM on nh3-pve, NFS datastore on nh3-nas); syncs from pbs-ana | `servers/pbs-nh3/README.md` | | esh-docker-vm | 10.0.50.45 | ESH home lab (`esteban.net`, `10.0.50.0/24`) | Home-lab Docker host (VM on esh-pve) | `servers/esh-docker-vm/README.md` | diff --git a/dns/internal.yaml b/dns/internal.yaml index 530d3f1..5fe8c44 100644 --- a/dns/internal.yaml +++ b/dns/internal.yaml @@ -85,6 +85,7 @@ hosts: - {name: nh3-pve-2, site: nh3, v4: 10.100.250.62, note: Proxmox host nh3-pve-2 (MS-03) on its X710 SFP+ (38:05:25:3b:a0:a8) — USW Pro 24 port 25 (native nh3-mgmt, tagged auto), DHCP reservation} - {name: nh3-pve-2-amt, site: nh3, v4: 10.100.250.63, note: Intel AMT 21 (vPro) on nh3-pve-2's (MS-03) I226-LM — UDM port 5 (native nh3-mgmt), DHCP reservation for 38:05:25:3b:a0:a6; LAN mgmt goes dark once CIRA is set up} - {name: pbs-nh3, site: nh3, v4: 10.100.50.90, note: Proxmox Backup Server — DR mirror} + - {name: nh3-albok, site: nh3, v4: 10.100.50.44, note: VM 110 on nh3-pve-2 — runs ONLY albok-service (no Docker API on tcp, no socket containers)} - {name: nh3-gw, site: nh3, v4: 10.100.0.1, note: UniFi UDM Pro SE — gateway + controller} # Irvine is not its own DNS site: it is a single box (irv-ml1) reached over # the headscale mesh, so it is named under nh3 rather than getting its own @@ -122,7 +123,7 @@ hosts: # runs becomes a one-line edit here instead of a hunt through configs. aliases: - - {name: albok, site: nh3, target: nh3-docker, note: albok-service (fleet knowledgebase) :8392 — container :8390; 8390 on the host is the post office} + - {name: albok, site: nh3, target: nh3-albok, note: albok-service (fleet knowledgebase) :8392 — container :8390; on nh3-albok (VM 110, nh3-pve-2) since 2026-10-04, nh3-docker before} - {name: searxng, site: nh3, target: nh3-docker, note: moved off ana-docker 2026-09-03 — colo egress (38.120.12.42) is CAPTCHA-gated by search engines; NH3 egresses residentially} - {name: gateway, site: ana, target: ana-docker, note: LiteLLM gateway :4000} - {name: booth, site: nh3, target: nh3-dev, note: The Booth :8090} diff --git a/persistent-memory.d/2026-10-04-albok-service-moved-to-nh3-albok.md b/persistent-memory.d/2026-10-04-albok-service-moved-to-nh3-albok.md new file mode 100644 index 0000000..75035d7 --- /dev/null +++ b/persistent-memory.d/2026-10-04-albok-service-moved-to-nh3-albok.md @@ -0,0 +1,35 @@ +# albok-service moved to its own VM: nh3-albok (VM 110 on nh3-pve-2) + +`[2026-10-04]` **albok-service moved off nh3-docker onto nh3-albok, a VM that runs nothing else** (albok-dev relaying +Prime ~2235; built and cut over by infra-ops 2311–2332). → `servers/nh3-albok/README.md`, `stacks/albok-service/README.md` + +**Why.** Unit 4 will ingest the operator's personal vault (~7,600 notes) and licensed books into restricted wings. On +nh3-docker, four things were root over `/srv/albok`: portainer, portainer_agent and dockge (socket rw), and dockerd on +plaintext **tcp/2375** for Homepage. albok-dev's list named the three containers; the 2375 exposure was found during +this work and is the bigger one, since any LAN host can reach it. Memory too: 2.4 of a 4 GiB cap on a 7.7 GiB host shared with the post office +and AdGuard. + +**What was built.** +- nh3-pve-2 `vmbr0` made VLAN-aware (`playbooks/nh3-pve-2-vmbr0-vlan-aware.yaml`), applied by a **reboot at 2311 with no + guests**, never a live ifreload (the 10-03 lesson). AMT port came back up at 1 Gb. +- VM 110 `nh3-albok`, 10.100.50.44 on VLAN 50, Debian 13 genericcloud + cloud-init, 4 vCPU x86-64-v3 / 12 GiB / 100 GB + vmstore (`playbooks/nh3-albok-vm.yaml`). `vh` 1000 via cloud-init → infra-ops 850 bootstrapped as vh → the bootstrap + key taken back off vh (`playbooks/nh3-albok-host.yaml`). Docker 29.8 with `docker` pinned at 851 before install. +- **Deliberately single-tenant:** no socket containers, no `-H tcp://`, so Homepage cannot discover it. The Albok tile is a static + `services.yaml` entry; it hot-reloaded without a container recreate. `mem_limit` 4g → 8g. +- PBS: nh3-pve-2 got storage `pbs-ana` (namespace **`nh3-pve-2`**, created on pbs-ana) and a 21:30 `all 1` vzdump job. + pbs-nh3's sync is root-namespace recursive, so the DR mirror needed nothing. The `root@pam!fleet-vzdump` secret is + now vaulted as `pbs-ana/fleet-vzdump-token`. albok-dev asked for "a PBS job on pbs-nh3"; the pbs-ana job keeps the fleet + pattern and keeps an off-site copy, and they were told why. + +**Cutover (2330–2332, ~2.5 min down).** Container removed on nh3-docker and its `compose.yaml` renamed +`compose.yaml.moved-to-nh3-albok-2026-10-04` (a stray `up` there would be a second writer on a stale copy, i.e. split +brain). One `tar --format=posix --numeric-owner` stream under `sudo /usr/bin/tar` (BULK_DATA, so nothing is recorded). +Verified by digests only, so no restricted path was printed: 166,305 entries each side, contents identical, and owner/group/ +mode/mtime/type identical. ⚠ **A full-manifest digest that includes `%s` will MISMATCH on directories**: ext4 directory +st_size reflects blocks once allocated, so it is not data. Split the manifest by field before deciding a copy is bad. +New host healthy at 2331:25: 4 wings ready, drift 0, needs_reindex 0, so no reindex was needed. Then DNS moved. + +**Still open (as of 2332):** albok-dev's acceptance (albok-check + one Nemi walk); the first manual PBS backup of VM +110 after acceptance; **deleting the old `/srv/albok` on nh3-docker needs Prime's word directly** (it cannot be undone, +and the request came as a relay). Until then it sits under the same 2375 exposure as before, with nothing reading it. diff --git a/persistent-memory.md b/persistent-memory.md index 74774ab..17a4745 100644 --- a/persistent-memory.md +++ b/persistent-memory.md @@ -1,6 +1,6 @@ # Persistent memory — eshpfi-management -_Last updated: 2026-10-04 ~2300 PT (every hypervisor on PVE 9.2.21 + both PBS on 4.2.7; pfi-ana-webhost destroyed, phasefinal.com on Cloudflare Pages; sudo-io archiver + bulk-data exemption; fv-ml1 root pool cleaned; fv-ml1 Docker move + guest patch campaign parked.)_ +_Last updated: 2026-10-04 ~2335 PT (albok-service moved to its own VM nh3-albok on nh3-pve-2; every hypervisor on PVE 9.2.21 + both PBS on 4.2.7; pfi-ana-webhost destroyed, phasefinal.com on Cloudflare Pages; sudo-io archiver + bulk-data exemption; fv-ml1 root pool cleaned; fv-ml1 Docker move + guest patch campaign parked.)_ > **Always check for `/tmp/infra-ops-handoff.md`** — if it exists and its > `Written:` stamp is under **8 hours** old, read it (it carries the in-flight @@ -119,6 +119,7 @@ _As of 2026-10-04 ~2300 PT. The newest subsection is first; older subsections ca ### 2026-10-04: live now (as of 2026-10-04 ~2300 PT) +- **albok-service now on nh3-albok (VM 110, nh3-pve-2, 10.100.50.44) since 2332; DNS moved.** Open: albok-dev acceptance, first manual PBS backup of VM 110, and Prime's word before deleting the old `/srv/albok` on nh3-docker (container gone, compose renamed). → `persistent-memory.d/2026-10-04-albok-service-moved-to-nh3-albok.md` - **Nothing is mid-action.** Every hypervisor is on PVE 9.2.21 / 7.0.14-20 (pfi-pve 0656, esh-pve 0742, nh3-pve + esh-nas-pve 10-03, nh3-pve-2 fresh) and both PBS on 4.2.7 (1144). → `persistent-memory.d/2026-10-04-fleet-on-pve-9-and-pbs-4.md` - **Scheduled with infra-hermes (do not redo):** 10-05 0815 PT check that sfsrv-ana's (PVE 8) 04:00 backup reached pbs-ana, then `qm delsnapshot` 100/105 `pre-pbs4` (if stale/errored he sends infra-ops URGENT and keeps them); on/after 10-05 0900 drop `pve/root-pre-pve9` on pfi-pve + esh-pve (job `pve9-drop-pre-snapshot`); 10-11 0900 delete the two one-off PBS VM vzdumps. esh-nas-pve `nvme/ROOT@pre-pve9`: destroy ~10-10 (ours). Never `zpool upgrade` any pool yet. - **Weekly sudo-io archiver** first scheduled run Sun 2026-10-11 05:40 (nh3-dev user timer); bulk data no longer recorded fleet-wide. → `services/sudo-io-archive/README.md` @@ -298,6 +299,7 @@ _As of 2026-10-04 ~2300 PT. The newest subsection is first; older subsections ca ## Recent decisions +- `[2026-10-04]` **albok-service moved to nh3-albok, a VM that runs only albok (albok-dev relaying Prime)** — nh3-docker's socket containers and plaintext tcp/2375 were root over the restricted wings; nh3-pve-2 vmbr0 VLAN-aware by reboot; PBS ns `nh3-pve-2`; copy verified by digest (dir st_size differs on ext4, not data). → `persistent-memory.d/2026-10-04-albok-service-moved-to-nh3-albok.md` - `[2026-10-04]` **The whole fleet is on PVE 9.2.21 and PBS 4.2.7** — per-host traps (pfi-pve's grub-pc on UEFI, esh-pve's dead bridge port, nh3-pve's missing 7.0 headers, PBS via guest agent) and what a pfi-pve reboot costs. → `persistent-memory.d/2026-10-04-fleet-on-pve-9-and-pbs-4.md` - `[2026-10-04]` **pfi-ana-webhost destroyed; phasefinal.com on Cloudflare Pages; token-creator + `infra-ops-pages` tokens** — the dragonfire sites were never ours to serve; 38.120.12.45 now unmapped. → `persistent-memory.d/2026-10-04-webhost-destroyed-phasefinal-on-pages.md` - `[2026-10-04]` **sudo session recordings: archived weekly, bulk data never kept (Prime)** — `BULK_DATA !log_output` on all 19 recording hosts, archiver drops streams >64 MiB, ~31 G pruned. → `services/sudo-io-archive/README.md` diff --git a/playbooks/albok-service-host.yaml b/playbooks/albok-service-host.yaml index 33f1f59..3d63687 100644 --- a/playbooks/albok-service-host.yaml +++ b/playbooks/albok-service-host.yaml @@ -1,5 +1,5 @@ -# Host preparation for albok-service on nh3-docker (2026-10-02). Idempotent. -# scripts/elway infra-ops@10.100.50.40 --playbook playbooks/albok-service-host.yaml +# Host preparation for albok-service (2026-10-02; nh3-docker, then nh3-albok from 2026-10-04). Idempotent. +# scripts/elway infra-ops@10.100.50.44 --playbook playbooks/albok-service-host.yaml # Creates the service identity and read groups with FIXED numeric ids (the contract requires them), # and the two local roots owned by uid 1500. The config file itself (it carries the embedder key) # is uploaded separately to /srv/albok/etc/albok.yaml: see stacks/albok-service/README.md. diff --git a/playbooks/nh3-albok-host.yaml b/playbooks/nh3-albok-host.yaml new file mode 100644 index 0000000..fa97ac2 --- /dev/null +++ b/playbooks/nh3-albok-host.yaml @@ -0,0 +1,95 @@ +# Base build for nh3-albok (VM 110 on nh3-pve-2, 10.100.50.44): the dedicated albok-service host. +# scripts/elway infra-ops@10.100.50.44 --playbook playbooks/nh3-albok-host.yaml +# Order: playbooks/nh3-albok-vm.yaml (on nh3-pve-2) → bootstrap-infra-ops-user.yaml run AS vh → this +# → albok-service-host.yaml (ids + /srv/albok) → stacks/albok-service/README.md deploy steps. +# +# Built to docs/pfi/fleet-conventions.md: `vh` 1000 (cloud-init), `infra-ops` 850, `docker` gid 851 +# created BEFORE docker-ce so the package adopts it, /opt/docker root:docker 2775. +# +# ⚠ This host exists to keep root-over-docker away from the albok store. So, unlike the general docker +# hosts, it gets NO portainer/dockge/agent containers, NO socket mounts, and dockerd does NOT listen on +# tcp/2375 (Homepage cannot discover it; the Homepage entry for albok is static in services.yaml). +# Do not add any of those here. A host-level read of containers is `ssh` + `docker ps`. +steps: + - name: take the bootstrap infra-ops key back off vh (vh keeps the operator's keys) + sudo: true + shell: | + set -e + f=/home/vh/.ssh/authorized_keys + grep -v 'infra-ops@pfi-fleet' "$f" > "$f.new" + test -s "$f.new" + chown vh:vh "$f.new"; chmod 600 "$f.new"; mv "$f.new" "$f" + when: "sudo -n grep -q 'infra-ops@pfi-fleet' /home/vh/.ssh/authorized_keys" + + - name: timezone America/Los_Angeles (the fleet's; trixie ships US/Pacific only in tzdata-legacy) + sudo: true + shell: timedatectl set-timezone America/Los_Angeles + when: "[ \"$(timedatectl show -p Timezone --value)\" != America/Los_Angeles ]" + + - name: base packages (guest agent for vzdump freeze/thaw, rsync for the store move) + sudo: true + shell: DEBIAN_FRONTEND=noninteractive apt-get install -y -q qemu-guest-agent rsync jq curl ca-certificates gnupg + when: "! dpkg -s qemu-guest-agent rsync jq >/dev/null 2>&1" + + - name: qemu-guest-agent running + sudo: true + shell: systemctl start qemu-guest-agent + when: "! systemctl is-active --quiet qemu-guest-agent" + + - name: docker group pinned at gid 851 BEFORE docker-ce installs + sudo: true + shell: groupadd -g 851 docker + when: "! getent group docker >/dev/null" + + - name: Docker's apt repo (trixie) + sudo: true + shell: | + set -e + install -d -m 0755 /etc/apt/keyrings + curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor --yes -o /etc/apt/keyrings/docker.gpg + chmod a+r /etc/apt/keyrings/docker.gpg + echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian $(. /etc/os-release; echo $VERSION_CODENAME) stable" \ + > /etc/apt/sources.list.d/docker.list + apt-get update -qq + creates: /etc/apt/sources.list.d/docker.list + + - name: docker-ce + compose plugin + sudo: true + shell: DEBIAN_FRONTEND=noninteractive apt-get install -y -q docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin + when: "! dpkg -s docker-ce docker-compose-plugin >/dev/null 2>&1" + + - name: vh and infra-ops in docker + sudo: true + shell: usermod -aG docker vh && usermod -aG docker infra-ops + when: "! id -nG infra-ops | grep -qw docker || ! id -nG vh | grep -qw docker" + + - name: deploy tree /opt/docker{,/compose,/conf} root:docker 2775 + sudo: true + shell: | + install -d -o root -g docker -m 2775 /opt/docker /opt/docker/compose /opt/docker/conf + changed_when: "false" + + - name: traefik-net exists (the fleet compose convention names it; no traefik runs here) + sudo: true + shell: docker network create traefik-net + when: "! sudo -n docker network inspect traefik-net >/dev/null 2>&1" + +verify: + - name: identities and groups match the fleet pin + shell: | + test "$(id -u vh)" = 1000 && test "$(id -u infra-ops)" = 850 && test "$(getent group docker | cut -d: -f3)" = 851 + id -nG infra-ops | grep -qw docker + changed_when: "false" + - name: vh no longer carries the infra-ops key + sudo: true + shell: "! grep -q 'infra-ops@pfi-fleet' /home/vh/.ssh/authorized_keys && grep -c . /home/vh/.ssh/authorized_keys" + changed_when: "false" + - name: dockerd up, listening on its unix socket only (nothing on tcp/2375 or 2376) + sudo: true + shell: | + docker version --format '{{.Server.Version}}' + ! ss -ltn | grep -E ':(2375|2376)\b' + changed_when: "false" + - name: guest agent active + shell: systemctl is-active qemu-guest-agent + changed_when: "false" diff --git a/playbooks/nh3-albok-vm.yaml b/playbooks/nh3-albok-vm.yaml new file mode 100644 index 0000000..484cb10 --- /dev/null +++ b/playbooks/nh3-albok-vm.yaml @@ -0,0 +1,98 @@ +# Create VM 110 `nh3-albok` on nh3-pve-2: the dedicated albok-service host (2026-10-04). +# scripts/elway infra-ops@10.100.250.62 --playbook playbooks/nh3-albok-vm.yaml +# Needs vmbr0 VLAN-aware first (playbooks/nh3-pve-2-vmbr0-vlan-aware.yaml + a reboot). +# +# Why a VM of its own (albok-dev, relaying Prime 2026-10-04 ~2235): on nh3-docker the store shared a +# host with three containers holding the docker socket read-write (portainer, portainer_agent, dockge) +# plus dockerd on plaintext tcp/2375 for Homepage. Each of those is root over /srv/albok, and unit 4 +# puts the operator's personal vault and licensed books into restricted wings. Also memory: albok sat +# at 2.4 of a 4 GiB cap on a 7.7 GiB host it shared with the post office and AdGuard. +# +# Shape: Debian 13 genericcloud + cloud-init. 4 vCPU (x86-64-v3: AVX2 for the index code, and still +# restorable on nh3-pve's i9-13900H), 12 GiB, 100 GB on `vmstore`, VLAN 50 (nh3-servers) at +# 10.100.50.44 — below the UDM's DHCP pool (.150–.249), like every other static on that VLAN. +# VMID 110 continues the NH3 numbering (nh3-pve holds 100–109) so "VM 110" is unambiguous on the site; +# PBS keeps nodes apart by namespace anyway. +# +# cloud-init creates `vh` (uid 1000, the fleet's human account) with NOPASSWD sudo. infra-ops (uid 850) +# is then built by playbooks/bootstrap-infra-ops-user.yaml run AS vh, after which the infra-ops key is +# taken back off vh (playbooks/nh3-albok-host.yaml). +vars: + vmid: "110" + name: nh3-albok + ip: 10.100.50.44/24 + gw: 10.100.50.1 + # nh3-docker AdGuard first, ana-docker AdGuard second: the same pair the UDM hands out on VLAN 50 + dns: 10.100.50.40 10.250.50.70 + img_url: https://cloud.debian.org/images/cloud/trixie/latest + img: debian-13-genericcloud-amd64.qcow2 + img_dir: /var/lib/vz/import + +steps: + - name: refuse unless this is nh3-pve-2, vmbr0 is VLAN-aware, and the VMID and address are free + sudo: true + shell: | + set -e + test "$(hostname)" = nh3-pve-2 + test "$(cat /sys/class/net/vmbr0/bridge/vlan_filtering)" = 1 + if ping -c2 -W1 $(echo {{ ip }} | cut -d/ -f1) >/dev/null 2>&1; then echo "{{ ip }} ANSWERS — taken"; exit 1; fi + changed_when: "false" + when: "! qm status {{ vmid }} >/dev/null 2>&1" + + - name: fetch the Debian 13 genericcloud image and check it against SHA512SUMS + sudo: true + shell: | + set -e + install -d -m 0755 {{ img_dir }} + cd {{ img_dir }} + curl -fsSL -o SHA512SUMS.trixie {{ img_url }}/SHA512SUMS + curl -fsSL -o {{ img }}.part {{ img_url }}/{{ img }} + want=$(awk '$2=="{{ img }}"{print $1}' SHA512SUMS.trixie) + test -n "$want" + echo "$want {{ img }}.part" | sha512sum -c - + mv {{ img }}.part {{ img }} + when: "! qm status {{ vmid }} >/dev/null 2>&1" + + - name: create VM {{ vmid }} ({{ name }}) + sudo: true + shell: | + set -e + qm create {{ vmid }} --name {{ name }} --ostype l26 --cpu x86-64-v3 --cores 4 --memory 12288 \ + --scsihw virtio-scsi-single --agent enabled=1 --onboot 1 --tags albok \ + --net0 virtio,bridge=vmbr0,tag=50,firewall=1 \ + --serial0 socket --vga serial0 \ + --description "albok-service host (fleet knowledgebase). Runs ONLY albok-service. Built 2026-10-04, eshpfi playbooks/nh3-albok-vm.yaml + nh3-albok-host.yaml; servers/nh3-albok/README.md" + qm set {{ vmid }} --scsi0 vmstore:0,import-from={{ img_dir }}/{{ img }},discard=on,ssd=1,iothread=1 + qm disk resize {{ vmid }} scsi0 100G + qm set {{ vmid }} --ide2 vmstore:cloudinit --boot order=scsi0 + creates: /etc/pve/qemu-server/{{ vmid }}.conf + + - name: cloud-init — user vh, keys, static address, resolvers + sudo: true + shell: | + set -e + k=$(mktemp) + cat > "$k" <<'EOF' + ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN+1HBwfXrkfTYWdcnWCjLJ6VLAGC87gxH5h5vKaaA3c infra-ops@pfi-fleet + ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKGm/lJslGEL2xDBY/7LdUc0Q4GyuxwFtGspYTe6WsVE lkrav@VHHomeG13 + ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDP/AwgaLghPoQi9RDt+m/9H9jLHDoP93WgzczlhZlSvZoQ5qDpWu5oAQlRwImiWhVL1ClK4FwEz9JULsta7Uq3FaiQvs2ltevrW/EpNbhAfxZa4Z1ELrPAOO8SSlhZ+Zer33SPkfJD9t3Hi9TXheCaANiEtE6hH1nCVLpHWQdbzy+L7EyG8z+/RzNXwwIL8f0BZ8n7FvFujgwc1OQTXlg4fiW3FdB3uzJMVRT59IIsPEq/ODGZZ6XGtHy24Ewcsi8l78I+oKaPdMl8ZXNBQrNb0MPIJnHkQwTrzlDKZ0Oh1UKm7I9i9hDMB/M2+aQ+6MxzY5TLZyDvRYd8R3j9Mnrp3ouCnNcklGAkX7i8hNJ2y8xCbwOeP8sMd+VtQ6hBbNa6KAEaAgGuWHHS0KGGO4HjtvDxJH8e7BnGzveIeCUGLVKMrg08w9nqF6Kzf3Y72ou0yCmnu7kKqocS1Pcr+e5E1MdT4brc2BfYkzqExLp8gxQaJ1Zas2OXPNXpAxL58xc= lkraven@nh3-dev + EOF + sed -i 's/^ *//' "$k" + qm set {{ vmid }} --ciuser vh --sshkeys "$k" --ipconfig0 ip={{ ip }},gw={{ gw }} \ + --nameserver "{{ dns }}" --searchdomain nh3.internal --ciupgrade 1 + rm -f "$k" + when: "! qm config {{ vmid }} | grep -q '^ciuser: vh'" + + - name: start VM {{ vmid }} + sudo: true + shell: qm start {{ vmid }} + when: "! qm status {{ vmid }} | grep -q running" + +verify: + - name: VM is running with the intended shape + sudo: true + shell: | + qm status {{ vmid }} + qm config {{ vmid }} | grep -E '^(name|cores|cpu|memory|net0|scsi0|ipconfig0|onboot|ciuser):' + qm status {{ vmid }} | grep -q running + changed_when: "false" diff --git a/playbooks/nh3-pve-2-vmbr0-vlan-aware.yaml b/playbooks/nh3-pve-2-vmbr0-vlan-aware.yaml new file mode 100644 index 0000000..3ba49c8 --- /dev/null +++ b/playbooks/nh3-pve-2-vmbr0-vlan-aware.yaml @@ -0,0 +1,63 @@ +# nh3-pve-2: make vmbr0 VLAN-aware so guests can sit on tagged VLANs (2026-10-04). +# scripts/elway infra-ops@10.100.250.62 --playbook playbooks/nh3-pve-2-vmbr0-vlan-aware.yaml +# then REBOOT the node (the playbook does not). First consumer: VM 110 nh3-albok on VLAN 50 +# (nh3-servers), the albok-service host. +# +# Same shape as nh3-pve's vmbr0 (`bridge-vlan-aware yes`, `bridge-vids 2-4094`), where every guest +# carries `tag=50`. The host address stays untagged on the bridge (PVID 1 = the switch port's native +# nh3-mgmt), so 10.100.250.62 does not move. +# +# ⚠ Applied by REBOOT, never by a live `ifreload -a`. On 2026-10-03 a live ifreload here read the new +# vmbr0 as "not a bridge" and downed it (servers/nh3-pve-2/README.md). The syntax check below +# (`ifreload -a -s`) parses the file without touching any interface. +# ⚠ Only the vmbr0 stanza changes. `auto nic0` (the AMT port) must stay as it is: if Linux leaves that +# port down, igc powers off the PHY and AMT goes dark. +steps: + - name: refuse unless this is nh3-pve-2 with vmbr0 on nic3 and no guests + sudo: true + shell: | + set -e + test "$(hostname)" = nh3-pve-2 + grep -q '^iface vmbr0 inet static' /etc/network/interfaces + grep -q 'bridge-ports nic3$' /etc/network/interfaces + grep -q '^auto nic0$' /etc/network/interfaces + test -z "$(qm list | tail -n +2)" + changed_when: "false" + when: "! grep -q 'bridge-vlan-aware yes' /etc/network/interfaces" + + - name: back up /etc/network/interfaces + sudo: true + shell: cp -a /etc/network/interfaces /etc/network/interfaces.pre-vlan-aware-2026-10-04 + creates: /etc/network/interfaces.pre-vlan-aware-2026-10-04 + + - name: add bridge-vlan-aware + bridge-vids to the vmbr0 stanza (after bridge-fd) + sudo: true + shell: | + set -e + tmp=$(mktemp) + awk '{print} /^iface vmbr0 /{inv=1} inv && /^\tbridge-fd /{print "\tbridge-vlan-aware yes"; print "\tbridge-vids 2-4094"; inv=0}' \ + /etc/network/interfaces > "$tmp" + test "$(grep -c 'bridge-vlan-aware yes' "$tmp")" = 1 + cat "$tmp" > /etc/network/interfaces + rm -f "$tmp" + when: "! grep -q 'bridge-vlan-aware yes' /etc/network/interfaces" + + # `ifup -s` exits 1 on the PVE-default `bridge-fd 0` ("out of range 2-255") even on the file the node + # booted from, so its rc carries no signal here. The check is instead: the new file draws exactly the + # warnings the original did, nothing more. + - name: syntax-check the file without applying it (no warnings beyond the original's) + sudo: true + shell: | + set -e + old=$(ifup -a -s -i /etc/network/interfaces.pre-vlan-aware-2026-10-04 2>&1 || true) + new=$(ifup -a -s -i /etc/network/interfaces 2>&1 || true) + echo "$new" + test "$old" = "$new" + changed_when: "false" + +verify: + - name: vmbr0 stanza carries both lines, nic0 untouched + shell: | + sed -n '/^iface vmbr0/,/^$/p;/^auto nic0/,/^iface nic0/p' /etc/network/interfaces + grep -q 'bridge-vlan-aware yes' /etc/network/interfaces && grep -q 'bridge-vids 2-4094' /etc/network/interfaces && grep -q '^auto nic0$' /etc/network/interfaces + changed_when: "false" diff --git a/scripts/ops-log b/scripts/ops-log index afd579c..28e19d9 100755 --- a/scripts/ops-log +++ b/scripts/ops-log @@ -470,7 +470,7 @@ def cmd_audit(args) -> int: # corviduo-dev is excluded on purpose: Worldtree deploys there are CI/CD-driven # and rewrite the tree constantly, so it would report unlogged changes forever # and train us to ignore the output. -STACK_HOSTS = ["ana-docker", "nh3-docker", "esh-docker-vm", "vm-esh-nas", +STACK_HOSTS = ["ana-docker", "nh3-docker", "nh3-albok", "esh-docker-vm", "vm-esh-nas", "fv-ml1", "irv-ml1"] diff --git a/servers/nh3-albok/README.md b/servers/nh3-albok/README.md new file mode 100644 index 0000000..990ae46 --- /dev/null +++ b/servers/nh3-albok/README.md @@ -0,0 +1,57 @@ +# nh3-albok: the albok-service host (VM 110 on nh3-pve-2, NH3) + +A VM that runs **only** `albok-service`, the fleet knowledgebase (`stacks/albok-service/`). Built 2026-10-04 at +albok-dev's request, relaying Prime (~2235): albok moved off nh3-docker before unit 4 puts the operator's personal +vault and licensed books into restricted wings. + +**Why it has its own VM.** On nh3-docker, four things were root over `/srv/albok`, restricted wings included: +portainer, portainer_agent and dockge (docker socket read-write), and dockerd itself on plaintext **tcp/2375** +for Homepage discovery, reachable from the whole LAN. Memory was the second reason: albok sat at 2.4 of a 4 GiB +cap on a 7.7 GiB host shared with the post office and AdGuard, so a reindex under pressure could take DNS or the +bus with it. Unit 6 (encrypted wings keyed from the vault) also wants a host whose only tenant is albok. + +| | | +|---|---| +| Address | `10.100.50.44`, VLAN 50 (nh3-servers), static via cloud-init; `nh3-albok.nh3.internal`. Service alias `albok.nh3.internal` points here | +| Guest | VM **110** on **nh3-pve-2** (first guest there). `onboot 1`, tag `albok` | +| Shape | 4 vCPU `x86-64-v3` (AVX2; still restorable on nh3-pve's i9-13900H), 12 GiB, 100 GB `vmstore` (LVM-thin, discard on), virtio-scsi-single + iothread | +| OS | Debian 13 genericcloud + cloud-init; Docker CE 29.8 from Docker's repo | +| Identities | `vh` 1000 (operator keys), `infra-ops` 850 (NOPASSWD sudo, `log_output`, BULK_DATA exempt), `docker` gid 851; albok ids 1500/1510/1511/1512 (see the stack README) | +| Service | `albok-service` on host **8392** → container 8390, `mem_limit 8g`. `/srv/albok/{store,private,etc}` on the root ext4 | +| Backups | nh3-pve-2 vzdump job, nightly **2130**, snapshot mode → **pbs-ana**, namespace **`nh3-pve-2`**; pbs-nh3's 0630 sync mirrors it | + +## ⚠ Keep it single-tenant + +The point of this host is that nothing else can reach the store as root. So: + +- **No** portainer, dockge, beszel-agent-with-socket or any container that mounts `/var/run/docker.sock`. +- **No** `-H tcp://` on dockerd. Homepage therefore cannot discover it; the Albok tile is a **static** entry in + `stacks/homepage/conf/services.yaml`. Do not add this host to `stacks/homepage/conf/docker.yaml`. +- **Nothing else mounts `/srv/albok/store` read-write.** Viewers follow the rules in the stack README. +- Who is root here anyway: anyone with root on **nh3-pve-2** (reads the disk), the **PBS** datastores (backups are + not client-side encrypted), and the `vh` / `infra-ops` sudoers. That circle is the residual exposure. + +## Build (reproducible) + +```bash +scripts/elway infra-ops@10.100.250.62 --playbook playbooks/nh3-pve-2-vmbr0-vlan-aware.yaml # once, then reboot nh3-pve-2 +scripts/elway infra-ops@10.100.250.62 --playbook playbooks/nh3-albok-vm.yaml # VM 110 + cloud-init +scripts/elway vh@10.100.50.44 --playbook playbooks/bootstrap-infra-ops-user.yaml # infra-ops 850, as vh +scripts/elway infra-ops@10.100.50.44 --playbook playbooks/nh3-albok-host.yaml # docker, tree, takes the bootstrap key off vh +scripts/elway infra-ops@10.100.50.44 --playbook playbooks/albok-service-host.yaml # albok ids + /srv/albok +# then the stack README's deploy steps (config, deploy-stack.sh nh3-albok albok-service, registry login, up -d) +``` + +Registry: root is logged in to `gitea.phasefinal.com` as `claude-bot` with the vaulted +`nh3-dev/.config/claude-bot/gitea-token`, the same credential nh3-docker's root uses (matched by hash, 2026-10-04). + +## Move from nh3-docker (2026-10-04) + +The store, private root and config moved as one `tar` stream (`--format=posix --numeric-owner --xattrs --acls`, +both ends under `sudo /usr/bin/tar`, which the BULK_DATA rule keeps out of the sudo-io recordings). It was checked +by a manifest digest (path, uid, gid, mode, size, mtime, type) and a content-hash digest compared on both ends. +Only digests were printed, so the restricted wings' paths never reached a terminal or a recording. + +On nh3-docker the container was removed and `compose.yaml` renamed to +`compose.yaml.moved-to-nh3-albok-2026-10-04`, so a stray `compose up` there cannot start a second writer on a stale +copy. **`/srv/albok` is still on nh3-docker** until Prime approves deleting it after acceptance. diff --git a/servers/nh3-albok/ssh-target b/servers/nh3-albok/ssh-target new file mode 100644 index 0000000..5682890 --- /dev/null +++ b/servers/nh3-albok/ssh-target @@ -0,0 +1 @@ +infra-ops@10.100.50.44 diff --git a/servers/nh3-albok/system-details.txt b/servers/nh3-albok/system-details.txt new file mode 100644 index 0000000..639ccc1 --- /dev/null +++ b/servers/nh3-albok/system-details.txt @@ -0,0 +1,189 @@ + +===== HOST ===== + +Hostname: nh3-albok.nh3.internal +Date: 2026-10-04T23:32:32-07:00 +Uptime: up 19 minutes +OS: Debian GNU/Linux 13 (trixie) +Kernel: 6.12.111+deb13-cloud-amd64 +Arch: x86_64 + +===== HARDWARE ===== + +CPU cores: 4 +CPU model: QEMU Virtual CPU version 2.5+ +MemTotal: 11.7 GB +MemAvailable: 10.6 GB + +===== GPUS ===== + +nvidia-smi not present (no NVIDIA GPUs or driver not installed) + +===== FILESYSTEMS (df) ===== + +Filesystem Size Used Avail Use% Mounted on +/dev/sda1 99G 4.8G 90G 6% / +/dev/sda15 124M 8.9M 115M 8% /boot/efi + +===== PERSISTENT MOUNTS (/etc/fstab, non-comment) ===== + +PARTUUID=fd202e72-0241-4c3b-953d-ffe506ae6cfc / ext4 rw,discard,errors=remount-ro,x-systemd.growfs 0 1 +PARTUUID=b558256e-e822-4787-9bd2-c4c47dfb54d9 /boot/efi vfat defaults,umask=077 0 2 + +===== TARGETED DATA PATHS ===== + +/opt (total: 48K) + total 16 + drwxr-xr-x 4 root root 4096 2026-10-04 23:15 . + drwxr-xr-x 18 root root 4096 2026-10-04 23:13 .. + drwx--x--x 4 root root 4096 2026-10-04 23:15 containerd + drwxrwsr-x 4 root docker 4096 2026-10-04 23:15 docker + +/opt/docker (total: 40K) + total 16 + drwxrwsr-x 4 root docker 4096 2026-10-04 23:15 . + drwxr-xr-x 4 root root 4096 2026-10-04 23:15 .. + drwxrwsr-x 3 root docker 4096 2026-10-04 23:16 compose + drwxrwsr-x 3 root docker 4096 2026-10-04 23:16 conf + +/opt/docker/compose (total: 20K) + total 12 + drwxrwsr-x 3 root docker 4096 2026-10-04 23:16 . + drwxrwsr-x 4 root docker 4096 2026-10-04 23:15 .. + drwxrwsr-x 2 infra-ops docker 4096 2026-10-04 23:16 albok-service + +/opt/docker/conf (total: 16K) + total 12 + drwxrwsr-x 3 root docker 4096 2026-10-04 23:16 . + drwxrwsr-x 4 root docker 4096 2026-10-04 23:15 .. + drwxrwsr-x 2 infra-ops docker 4096 2026-10-04 23:16 albok-service + +/var/lib/docker (total: 4.0K) + +/srv (total: 20K) + total 12 + drwxr-xr-x 3 root root 4096 2026-10-04 23:15 . + drwxr-xr-x 18 root root 4096 2026-10-04 23:13 .. + drwxr-xr-x 5 root root 4096 2026-10-04 23:15 albok + + +===== DOCKER ===== + +Server: 29.8.2 Client: 29.8.2 + +----- docker info ----- +Containers: 1 (running 1, paused 0, stopped 0) +Images: 1 +Runtimes: map[io.containerd.runc.v2:{{runc [] map[]} map[org.opencontainers.runtime-spec.features:{"ociVersionMin":"1.0.0","ociVersionMax":"1.3.0","hooks":["prestart","createRuntime","createContainer","startContainer","poststart","poststop"],"mountOptions":["async","atime","bind","defaults","dev","diratime","dirsync","exec","iversion","lazytime","loud","mand","noatime","nodev","nodiratime","noexec","noiversion","nolazytime","nomand","norelatime","nostrictatime","nosuid","nosymfollow","private","ratime","rbind","rdev","rdiratime","relatime","remount","rexec","rnoatime","rnodev","rnodiratime","rnoexec","rnorelatime","rnostrictatime","rnosuid","rnosymfollow","ro","rprivate","rrelatime","rro","rrw","rshared","rslave","rstrictatime","rsuid","rsymfollow","runbindable","rw","shared","silent","slave","strictatime","suid","symfollow","sync","tmpcopyup","unbindable"],"linux":{"namespaces":["cgroup","ipc","mount","network","pid","time","user","uts"],"capabilities":["CAP_CHOWN","CAP_DAC_OVERRIDE","CAP_DAC_READ_SEARCH","CAP_FOWNER","CAP_FSETID","CAP_KILL","CAP_SETGID","CAP_SETUID","CAP_SETPCAP","CAP_LINUX_IMMUTABLE","CAP_NET_BIND_SERVICE","CAP_NET_BROADCAST","CAP_NET_ADMIN","CAP_NET_RAW","CAP_IPC_LOCK","CAP_IPC_OWNER","CAP_SYS_MODULE","CAP_SYS_RAWIO","CAP_SYS_CHROOT","CAP_SYS_PTRACE","CAP_SYS_PACCT","CAP_SYS_ADMIN","CAP_SYS_BOOT","CAP_SYS_NICE","CAP_SYS_RESOURCE","CAP_SYS_TIME","CAP_SYS_TTY_CONFIG","CAP_MKNOD","CAP_LEASE","CAP_AUDIT_WRITE","CAP_AUDIT_CONTROL","CAP_SETFCAP","CAP_MAC_OVERRIDE","CAP_MAC_ADMIN","CAP_SYSLOG","CAP_WAKE_ALARM","CAP_BLOCK_SUSPEND","CAP_AUDIT_READ","CAP_PERFMON","CAP_BPF","CAP_CHECKPOINT_RESTORE"],"cgroup":{"v1":true,"v2":true,"systemd":true,"systemdUser":true,"rdma":true},"seccomp":{"enabled":true,"actions":["SCMP_ACT_ALLOW","SCMP_ACT_ERRNO","SCMP_ACT_KILL","SCMP_ACT_KILL_PROCESS","SCMP_ACT_KILL_THREAD","SCMP_ACT_LOG","SCMP_ACT_NOTIFY","SCMP_ACT_TRACE","SCMP_ACT_TRAP"],"operators":["SCMP_CMP_EQ","SCMP_CMP_GE","SCMP_CMP_GT","SCMP_CMP_LE","SCMP_CMP_LT","SCMP_CMP_MASKED_EQ","SCMP_CMP_NE"],"archs":["SCMP_ARCH_AARCH64","SCMP_ARCH_ARM","SCMP_ARCH_LOONGARCH64","SCMP_ARCH_MIPS","SCMP_ARCH_MIPS64","SCMP_ARCH_MIPS64N32","SCMP_ARCH_MIPSEL","SCMP_ARCH_MIPSEL64","SCMP_ARCH_MIPSEL64N32","SCMP_ARCH_PPC","SCMP_ARCH_PPC64","SCMP_ARCH_PPC64LE","SCMP_ARCH_RISCV64","SCMP_ARCH_S390","SCMP_ARCH_S390X","SCMP_ARCH_X32","SCMP_ARCH_X86","SCMP_ARCH_X86_64"],"knownFlags":["SECCOMP_FILTER_FLAG_TSYNC","SECCOMP_FILTER_FLAG_SPEC_ALLOW","SECCOMP_FILTER_FLAG_LOG","SECCOMP_FILTER_FLAG_WAIT_KILLABLE_RECV"],"supportedFlags":["SECCOMP_FILTER_FLAG_TSYNC","SECCOMP_FILTER_FLAG_SPEC_ALLOW","SECCOMP_FILTER_FLAG_LOG","SECCOMP_FILTER_FLAG_WAIT_KILLABLE_RECV"]},"apparmor":{"enabled":true},"selinux":{"enabled":true},"intelRdt":{"enabled":true,"schemata":true,"monitoring":true},"memoryPolicy":{"modes":["MPOL_BIND","MPOL_DEFAULT","MPOL_INTERLEAVE","MPOL_LOCAL","MPOL_PREFERRED","MPOL_PREFERRED_MANY","MPOL_WEIGHTED_INTERLEAVE"],"flags":["MPOL_F_NUMA_BALANCING","MPOL_F_RELATIVE_NODES","MPOL_F_STATIC_NODES"]},"mountExtensions":{"idmap":{"enabled":true}},"netDevices":{"enabled":true}},"annotations":{"io.github.seccomp.libseccomp.version":"2.6.0","org.opencontainers.runc.checkpoint.enabled":"true","org.opencontainers.runc.commit":"v1.5.1-0-g8f2685a4","org.opencontainers.runc.version":"1.5.1"},"potentiallyUnsafeConfigAnnotations":["bundle","org.systemd.property.","org.criu.config"]}]} runc:{{runc [] map[]} map[org.opencontainers.runtime-spec.features:{"ociVersionMin":"1.0.0","ociVersionMax":"1.3.0","hooks":["prestart","createRuntime","createContainer","startContainer","poststart","poststop"],"mountOptions":["async","atime","bind","defaults","dev","diratime","dirsync","exec","iversion","lazytime","loud","mand","noatime","nodev","nodiratime","noexec","noiversion","nolazytime","nomand","norelatime","nostrictatime","nosuid","nosymfollow","private","ratime","rbind","rdev","rdiratime","relatime","remount","rexec","rnoatime","rnodev","rnodiratime","rnoexec","rnorelatime","rnostrictatime","rnosuid","rnosymfollow","ro","rprivate","rrelatime","rro","rrw","rshared","rslave","rstrictatime","rsuid","rsymfollow","runbindable","rw","shared","silent","slave","strictatime","suid","symfollow","sync","tmpcopyup","unbindable"],"linux":{"namespaces":["cgroup","ipc","mount","network","pid","time","user","uts"],"capabilities":["CAP_CHOWN","CAP_DAC_OVERRIDE","CAP_DAC_READ_SEARCH","CAP_FOWNER","CAP_FSETID","CAP_KILL","CAP_SETGID","CAP_SETUID","CAP_SETPCAP","CAP_LINUX_IMMUTABLE","CAP_NET_BIND_SERVICE","CAP_NET_BROADCAST","CAP_NET_ADMIN","CAP_NET_RAW","CAP_IPC_LOCK","CAP_IPC_OWNER","CAP_SYS_MODULE","CAP_SYS_RAWIO","CAP_SYS_CHROOT","CAP_SYS_PTRACE","CAP_SYS_PACCT","CAP_SYS_ADMIN","CAP_SYS_BOOT","CAP_SYS_NICE","CAP_SYS_RESOURCE","CAP_SYS_TIME","CAP_SYS_TTY_CONFIG","CAP_MKNOD","CAP_LEASE","CAP_AUDIT_WRITE","CAP_AUDIT_CONTROL","CAP_SETFCAP","CAP_MAC_OVERRIDE","CAP_MAC_ADMIN","CAP_SYSLOG","CAP_WAKE_ALARM","CAP_BLOCK_SUSPEND","CAP_AUDIT_READ","CAP_PERFMON","CAP_BPF","CAP_CHECKPOINT_RESTORE"],"cgroup":{"v1":true,"v2":true,"systemd":true,"systemdUser":true,"rdma":true},"seccomp":{"enabled":true,"actions":["SCMP_ACT_ALLOW","SCMP_ACT_ERRNO","SCMP_ACT_KILL","SCMP_ACT_KILL_PROCESS","SCMP_ACT_KILL_THREAD","SCMP_ACT_LOG","SCMP_ACT_NOTIFY","SCMP_ACT_TRACE","SCMP_ACT_TRAP"],"operators":["SCMP_CMP_EQ","SCMP_CMP_GE","SCMP_CMP_GT","SCMP_CMP_LE","SCMP_CMP_LT","SCMP_CMP_MASKED_EQ","SCMP_CMP_NE"],"archs":["SCMP_ARCH_AARCH64","SCMP_ARCH_ARM","SCMP_ARCH_LOONGARCH64","SCMP_ARCH_MIPS","SCMP_ARCH_MIPS64","SCMP_ARCH_MIPS64N32","SCMP_ARCH_MIPSEL","SCMP_ARCH_MIPSEL64","SCMP_ARCH_MIPSEL64N32","SCMP_ARCH_PPC","SCMP_ARCH_PPC64","SCMP_ARCH_PPC64LE","SCMP_ARCH_RISCV64","SCMP_ARCH_S390","SCMP_ARCH_S390X","SCMP_ARCH_X32","SCMP_ARCH_X86","SCMP_ARCH_X86_64"],"knownFlags":["SECCOMP_FILTER_FLAG_TSYNC","SECCOMP_FILTER_FLAG_SPEC_ALLOW","SECCOMP_FILTER_FLAG_LOG","SECCOMP_FILTER_FLAG_WAIT_KILLABLE_RECV"],"supportedFlags":["SECCOMP_FILTER_FLAG_TSYNC","SECCOMP_FILTER_FLAG_SPEC_ALLOW","SECCOMP_FILTER_FLAG_LOG","SECCOMP_FILTER_FLAG_WAIT_KILLABLE_RECV"]},"apparmor":{"enabled":true},"selinux":{"enabled":true},"intelRdt":{"enabled":true,"schemata":true,"monitoring":true},"memoryPolicy":{"modes":["MPOL_BIND","MPOL_DEFAULT","MPOL_INTERLEAVE","MPOL_LOCAL","MPOL_PREFERRED","MPOL_PREFERRED_MANY","MPOL_WEIGHTED_INTERLEAVE"],"flags":["MPOL_F_NUMA_BALANCING","MPOL_F_RELATIVE_NODES","MPOL_F_STATIC_NODES"]},"mountExtensions":{"idmap":{"enabled":true}},"netDevices":{"enabled":true}},"annotations":{"io.github.seccomp.libseccomp.version":"2.6.0","org.opencontainers.runc.checkpoint.enabled":"true","org.opencontainers.runc.commit":"v1.5.1-0-g8f2685a4","org.opencontainers.runc.version":"1.5.1"},"potentiallyUnsafeConfigAnnotations":["bundle","org.systemd.property.","org.criu.config"]}]}] +Default runtime: runc +Storage driver: overlayfs +Root dir: /var/lib/docker +Server version: 29.8.2 + +----- running containers ----- +NAMES IMAGE STATUS PORTS +albok-service gitea.phasefinal.com/pfi/albok-service:0.1.8 Up About a minute (healthy) 0.0.0.0:8392->8390/tcp, [::]:8392->8390/tcp + +----- all containers ----- +NAMES IMAGE STATUS +albok-service gitea.phasefinal.com/pfi/albok-service:0.1.8 Up About a minute (healthy) + +----- networks ----- +NAME DRIVER SCOPE +bridge bridge local +host host local +none null local +traefik-net bridge local + +----- networks (external, non-default — worth knowing for compose external: true) ----- +traefik-net + +----- named volumes ----- +VOLUME NAME DRIVER + +----- compose projects currently running ----- +albok-service + +===== COMPOSE FILES (/opt/docker/compose/) ===== + + +>>> /opt/docker/compose/albok-service/compose.yaml +# albok-service — the fleet knowledgebase service (vh/albok, packages/albok-service). The only +# writer of the buildings-and-wings store. Requested by albok-dev 2026-10-02 (operator-approved). +# Host: nh3-albok (VM 110 on nh3-pve-2, 10.100.50.44) since 2026-10-04, a VM that runs ONLY this. +# It moved off nh3-docker, where portainer/dockge/the agent and dockerd on tcp/2375 were all root +# over the store (servers/nh3-albok/README.md). +# +# Deploy: scripts/elway infra-ops@10.100.50.44 --playbook playbooks/albok-service-host.yaml (host prep, config) +# scripts/deploy-stack.sh nh3-albok albok-service (this file + conf/) +# Image: built from a clean `git archive` of vh/albok, pushed to the pfi org (see README.md). +# +# ⚠ Host port 8392 (kept from nh3-docker, where 8390 was the post office) so clients did not change. +# ⚠ Exactly ONE instance per private root: a second one exits 75 on the lease. Never scale this. +# ⚠ Nothing else may mount /srv/albok/store read-write. Viewers mount it :ro with group_add by gid +# (1510 albok-read, 1511 albok-personal) and need GIT_OPTIONAL_LOCKS=0 + a safe.directory entry. +# 1512 albok-feedback (restricted wing personal/agent-feedback) is default-deny: no viewer gets it. +services: + albok-service: + image: gitea.phasefinal.com/pfi/albok-service:0.1.8@sha256:239f900ee39a26414f34afe9b7583253986d838df9988228fab1039d18c414df + container_name: albok-service + restart: unless-stopped + ports: + - "8392:8390" + # Membership the service needs to chgrp wing dirs to the read groups (it runs as uid 1500, not root). + group_add: + - "1510" + - "1511" + - "1512" # albok-feedback: restricted wing personal/agent-feedback + volumes: + - /srv/albok/store:/srv/albok/store + - /srv/albok/private:/srv/albok/private + - /srv/albok/etc/albok.yaml:/etc/albok/albok.yaml:ro + # the read groups must RESOLVE BY NAME inside the container (layout.py uses grp.getgrnam) + - /opt/docker/conf/albok-service/group:/etc/group:ro + mem_limit: 8g # 4g on nh3-docker (7.7 GiB shared host); 12 GiB VM to itself now + healthcheck: + test: ["CMD", "python", "-c", "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8390/health', timeout=5).status == 200 else 1)"] + interval: 30s + timeout: 10s + retries: 3 + start_period: 60s + networks: + - tnet + # No homepage.* labels: Homepage discovers containers over a host's tcp/2375, and this host must not + # expose its Docker API. The Albok tile is a static entry in stacks/homepage/conf/services.yaml. + +networks: + tnet: + name: traefik-net + external: true + +===== CONFIG LAYOUT (/opt/docker/conf/ — top 200 entries) ===== + +/opt/docker/conf +/opt/docker/conf/albok-service +/opt/docker/conf/albok-service/albok.yaml.template +/opt/docker/conf/albok-service/group + +===== LISTENING PORTS ===== + +0.0.0.0:22 +0.0.0.0:5355 +0.0.0.0:8392 +127.0.0.53%lo:53 +127.0.0.54:53 +[::]:22 +[::]:5355 +[::]:8392 + +===== MODEL / HUGGINGFACE CACHES ===== + + +===== DOCKER-ADJACENT SYSTEMD SERVICES ===== + +containerd.service running +docker.service running + +===== DONE ===== + +Paste the above back into the chat, or pass a path as argv[1] to save. diff --git a/servers/nh3-pve-2/README.md b/servers/nh3-pve-2/README.md index 51c295e..b6fde63 100644 --- a/servers/nh3-pve-2/README.md +++ b/servers/nh3-pve-2/README.md @@ -1,12 +1,28 @@ # nh3-pve-2: Minisforum MS-03, Proxmox VE 9 (NH3) The NH3 MS-03, twin of esh-pve-2. **Purpose is TBD on purpose** (Prime, 2026-10-02): a high-powered PVE host, -possibly a dev environment for security-software work. Standalone node, not clustered with nh3-pve. +possibly a dev environment for security-software work. First guest: nh3-albok (2026-10-04, below). Standalone node, not clustered with nh3-pve. - **Host:** `10.100.250.62` on nh3-mgmt (`nh3-pve-2.nh3.internal`, `nh3-pve-2.phasefinal.com` in /etc/hosts). Static in `/etc/network/interfaces`, plus a UDM reservation on `38:05:25:3b:a0:a8`. `ssh infra-ops@10.100.250.62` (fleet key, NOPASSWD sudo with `log_output`). -- PVE 9.2.21, installed 2026-10-03 on site to the right disk (the 256 GB Toshiba). **Follow-ups DONE 2026-10-04:** Prime rebooted it at 0724 onto 7.0.14-20; verified 1635: `.62` on vmbr0, `nic0` up at 1 Gb with `arp_ignore=8` + IPv6 off, MeshCentral shows `nh3-pve-2-amt` connected (CIRA, power on), ceph + pve enterprise sources `Enabled: false`, 0 upgradable, no failed units. No guests yet. +- PVE 9.2.21, installed 2026-10-03 on site to the right disk (the 256 GB Toshiba). **Follow-ups DONE 2026-10-04:** Prime rebooted it at 0724 onto 7.0.14-20; verified 1635: `.62` on vmbr0, `nic0` up at 1 Gb with `arp_ignore=8` + IPv6 off, MeshCentral shows `nh3-pve-2-amt` connected (CIRA, power on), ceph + pve enterprise sources `Enabled: false`, 0 upgradable, no failed units. + +## Guests + +| ID | guest | onboot | notes | +|---|---|---|---| +| 110 | **nh3-albok** (`10.100.50.44`, VLAN 50) | 1 | First guest, 2026-10-04 (albok-dev relaying Prime): runs only albok-service. `servers/nh3-albok/README.md` | + +VMIDs continue the NH3 numbering (nh3-pve holds 100–109) so a VMID names one guest on the site. + +**Backups:** storage `pbs-ana` (10.250.50.90, datastore `backups`, **namespace `nh3-pve-2`**, token +`root@pam!fleet-vzdump`, secret vaulted as `pbs-ana/fleet-vzdump-token`) and vzdump job **21:30 nightly, `all 1`**, snapshot +mode, fleecing on `vmstore`, the same prune as nh3-pve. 30 min after nh3-pve's 21:00 so the two do not contend on pbs-ana. +pbs-nh3's sync (root namespace, recursive) mirrors the new namespace without any change on the DR side. +⚠ `pvesm add pbs` refuses without `--password` on the command line (it test-logs in), which would put the token in the +sudo log. The stanza was appended to `/etc/pve/storage.cfg` and the secret written to `/etc/pve/priv/storage/pbs-ana.pw` +(37 bytes, value + newline) with `sudo dd` instead. | NIC | MAC | Chip | Use | |---|---|---|---| @@ -17,9 +33,13 @@ possibly a dev environment for security-software work. Standalone node, not clus ⚠ **Keep `auto nic0`.** If Linux leaves the AMT port down, igc powers off the PHY and AMT goes dark. That happened 2026-10-03 when the bridge moved to the SFP+. -⚠ `vmbr0` is a plain (not VLAN-aware) bridge. Making it VLAN-aware is a separate change: apply it with a reboot, -not a live `ifreload`. On 2026-10-03 a live ifreload that also re-IP'd read the new vmbr0 as "not a bridge" and -downed it (auto-memory `feedback_no_remote_reip_thrash_when_operator_has_console`). +**`vmbr0` is VLAN-aware since 2026-10-04 2311** (`bridge-vlan-aware yes`, `bridge-vids 2-4094`, same as nh3-pve), applied by +`playbooks/nh3-pve-2-vmbr0-vlan-aware.yaml` + a reboot with no guests running. The host address stays untagged (PVID 1 = +the port's native nh3-mgmt); guests take `tag=`. Pre-change file kept at `/etc/network/interfaces.pre-vlan-aware-2026-10-04`. +⚠ Change this bridge only by reboot, never a live `ifreload`: on 2026-10-03 a live ifreload that also re-IP'd read the new +vmbr0 as "not a bridge" and downed it (auto-memory `feedback_no_remote_reip_thrash_when_operator_has_console`). +⚠ `ifup -a -s` exits 1 on this host even for a good file: it warns on the PVE-default `bridge-fd 0`. Compare its output +against the previous file's rather than trusting the exit code. **Disks:** boot = Toshiba 256 GB (`69CA12VQK65N`). **VM storage = LVM-thin `vmstore`, 913 GiB**, on the Crucial 1 TB (`2610EAD05AB7`). It held the old wrong-drive install, which was wiped 2026-10-03 by `playbooks/nh3-pve-2-disk-prep.yaml`, diff --git a/services/sudo-io-archive/hosts b/services/sudo-io-archive/hosts index c5424e6..9a1c163 100644 --- a/services/sudo-io-archive/hosts +++ b/services/sudo-io-archive/hosts @@ -18,6 +18,7 @@ irv-ml1 infra-ops@irv-ml1.nh3.internal pfi-gx10 infra-ops@10.100.50.60 nh3-dev infra-ops@10.100.10.50 nh3-docker infra-ops@10.100.50.40 +nh3-albok infra-ops@10.100.50.44 nh3-extdev infra-ops@10.100.50.42 nh3-ml1 infra-ops@10.100.50.80 esh-docker-vm infra-ops@10.0.50.45 diff --git a/stacks/albok-service/README.md b/stacks/albok-service/README.md index bec837c..4b517c7 100644 --- a/stacks/albok-service/README.md +++ b/stacks/albok-service/README.md @@ -1,11 +1,13 @@ -# albok-service — fleet knowledgebase service (nh3-docker) +# albok-service — fleet knowledgebase service (nh3-albok) The FastAPI process that owns Albok's buildings-and-wings store (`vh/albok`, `packages/albok-service`, contract `docs/contracts/unit2_service.contract.md`). It is the **only writer** of the store. Deployed 2026-10-02 at albok-dev's request (operator-approved). -- **URL:** `http://albok.nh3.internal:8392` (= `http://10.100.50.40:8392`). Container port 8390. - ⚠ Host port **8392**, because **8390 on nh3-docker is the althing post office**. +- **Host:** **nh3-albok** (VM 110 on nh3-pve-2, `10.100.50.44`) since 2026-10-04, a VM that runs nothing else + (`servers/nh3-albok/README.md` says why and what must never be added there). Before that: nh3-docker, 2026-10-02 to 2026-10-04. +- **URL:** `http://albok.nh3.internal:8392` (= `http://10.100.50.44:8392`). Container port 8390. + Host port **8392** is kept from nh3-docker, where 8390 was the althing post office, so no client changed. - **Image:** `gitea.phasefinal.com/pfi/albok-service:0.1.8` @ `sha256:239f900e…` (pinned in `compose.yaml`), built 2026-10-04 ~1115 from vh/albok `e653c29` (tag albok-service/v0.1.8; heid re-pass fixes over 0.1.7: keyword-index lock mismatch, journal-stranding replay path, bypassable 400, link-kind handling in the collapse). fleet/memory @@ -65,8 +67,10 @@ The previous token (`01M40979…`, without agent-feedback) is still live until a `albok/litellm-key-nemi`, alias `albok-nemi`, scoped to `gen-small`, `summarizer` and `qwen3-embedding`. Verified 200 on all three and 403 on another model. -**Backups:** nh3-docker is VM 100 on nh3-pve, in the nightly 21:00 vzdump (snapshot mode → pbs-ana), -so `/srv/albok` is covered whole-VM. No file-level restic job for it. +**Backups:** nh3-albok is VM 110 on nh3-pve-2, in that node's nightly **21:30** vzdump (snapshot mode → pbs-ana, +namespace `nh3-pve-2`; pbs-nh3's 06:30 sync mirrors it), so `/srv/albok` is covered whole-VM. No file-level restic +job. ⚠ The PBS datastores are not client-side encrypted: anyone with PBS or NAS root can read the restricted wings +out of a backup. That matters once unit 4 ingests the personal vault; unit 6's encrypted wings close it. ## Deploy / redeploy @@ -77,15 +81,15 @@ work and the service commits per batch, but avoidable. Check first, on nh3-dev a ```bash # 1. host prep (idempotent) -scripts/elway infra-ops@10.100.50.40 --playbook playbooks/albok-service-host.yaml +scripts/elway infra-ops@10.100.50.44 --playbook playbooks/albok-service-host.yaml # 2. config: render the template with the vaulted key (never commit the rendered file) umask 077; secret get albok/litellm-key | tr -d '\n' | python3 -c "import sys; k=sys.stdin.read(); \ open('/tmp/albok.yaml','w').write(open('stacks/albok-service/conf/albok.yaml.template').read().replace('__ALBOK_LITELLM_KEY__', k))" -scripts/elway infra-ops@10.100.50.40 --upload /tmp/albok.yaml:/srv/albok/etc/albok.yaml:0640 --sudo --owner root:albok +scripts/elway infra-ops@10.100.50.44 --upload /tmp/albok.yaml:/srv/albok/etc/albok.yaml:0640 --sudo --owner root:albok # then delete /tmp/albok.yaml by its literal path -# 3. compose + conf, then start (root holds the registry login on nh3-docker) -scripts/deploy-stack.sh nh3-docker albok-service -ssh infra-ops@10.100.50.40 'cd /opt/docker/compose/albok-service && sudo docker compose up -d' +# 3. compose + conf, then start (root holds the registry login on nh3-albok: claude-bot, vaulted nh3-dev/.config/claude-bot/gitea-token) +scripts/deploy-stack.sh nh3-albok albok-service +ssh infra-ops@10.100.50.44 'cd /opt/docker/compose/albok-service && sudo docker compose up -d' ``` **Image rebuild** (from a clean archive, never a working tree): diff --git a/stacks/albok-service/compose.yaml b/stacks/albok-service/compose.yaml index a759704..48435bb 100644 --- a/stacks/albok-service/compose.yaml +++ b/stacks/albok-service/compose.yaml @@ -1,11 +1,14 @@ # albok-service — the fleet knowledgebase service (vh/albok, packages/albok-service). The only # writer of the buildings-and-wings store. Requested by albok-dev 2026-10-02 (operator-approved). +# Host: nh3-albok (VM 110 on nh3-pve-2, 10.100.50.44) since 2026-10-04, a VM that runs ONLY this. +# It moved off nh3-docker, where portainer/dockge/the agent and dockerd on tcp/2375 were all root +# over the store (servers/nh3-albok/README.md). # -# Deploy: scripts/elway infra-ops@10.100.50.40 --playbook playbooks/albok-service-host.yaml (host prep, config) -# scripts/deploy-stack.sh nh3-docker albok-service (this file + conf/) +# Deploy: scripts/elway infra-ops@10.100.50.44 --playbook playbooks/albok-service-host.yaml (host prep, config) +# scripts/deploy-stack.sh nh3-albok albok-service (this file + conf/) # Image: built from a clean `git archive` of vh/albok, pushed to the pfi org (see README.md). # -# ⚠ Host port 8392, NOT 8390: 8390 on nh3-docker is the althing post office. +# ⚠ Host port 8392 (kept from nh3-docker, where 8390 was the post office) so clients did not change. # ⚠ Exactly ONE instance per private root: a second one exits 75 on the lease. Never scale this. # ⚠ Nothing else may mount /srv/albok/store read-write. Viewers mount it :ro with group_add by gid # (1510 albok-read, 1511 albok-personal) and need GIT_OPTIONAL_LOCKS=0 + a safe.directory entry. @@ -28,7 +31,7 @@ services: - /srv/albok/etc/albok.yaml:/etc/albok/albok.yaml:ro # the read groups must RESOLVE BY NAME inside the container (layout.py uses grp.getgrnam) - /opt/docker/conf/albok-service/group:/etc/group:ro - mem_limit: 4g + mem_limit: 8g # 4g on nh3-docker (7.7 GiB shared host); 12 GiB VM to itself now healthcheck: test: ["CMD", "python", "-c", "import urllib.request,sys; sys.exit(0 if urllib.request.urlopen('http://127.0.0.1:8390/health', timeout=5).status == 200 else 1)"] interval: 30s @@ -37,12 +40,8 @@ services: start_period: 60s networks: - tnet - labels: - - homepage.group=Agents (no UI) - - homepage.name=Albok - - homepage.icon=mdi-book-search - - homepage.description=Fleet knowledgebase service (albok-service) on nh3-docker - - homepage.href=http://10.100.50.40:8392/health + # No homepage.* labels: Homepage discovers containers over a host's tcp/2375, and this host must not + # expose its Docker API. The Albok tile is a static entry in stacks/homepage/conf/services.yaml. networks: tnet: diff --git a/stacks/homepage/conf/services.yaml b/stacks/homepage/conf/services.yaml index 78b8abb..5a95785 100644 --- a/stacks/homepage/conf/services.yaml +++ b/stacks/homepage/conf/services.yaml @@ -85,6 +85,16 @@ # well or it renders twice. To move a service between AI groups, change the # label on its compose file and recreate the container — labels are read at # creation, so `restart` will not do it. +- Agents (no UI): + # Manual entry — albok-service moved to nh3-albok (2026-10-04), a host that deliberately does NOT + # expose its Docker API on tcp/2375 (it holds restricted wings), so it is not in docker.yaml and its + # container carries no homepage labels. Do not add the host to docker.yaml to "fix" discovery. + - Albok: + href: http://10.100.50.44:8392/health + icon: mdi-book-search + siteMonitor: http://10.100.50.44:8392/health + description: Fleet knowledgebase service (albok-service) on nh3-albok + - AI - Studios: # Manual entry — Voice Design Studio is a user-level systemd service on # irv-ml1, not a Docker-labeled stack, so it cannot auto-discover. It sits