feat(albok): move albok-service to a dedicated VM, nh3-albok (VM 110 on nh3-pve-2)

albok-service leaves nh3-docker before unit 4 puts the personal vault and licensed books into restricted wings. On nh3-docker, portainer, portainer_agent, dockge (socket rw) and dockerd on plaintext tcp/2375 were each root over /srv/albok. The new host runs only albok-service: no socket containers and no TCP Docker API, so its Homepage tile is a static services.yaml entry. - nh3-pve-2: vmbr0 made VLAN-aware (applied by reboot, no guests running); pbs-ana storage in the new namespace nh3-pve-2, plus a 21:30 vzdump job (pbs-nh3's recursive sync mirrors it) - VM 110 nh3-albok, 10.100.50.44 on VLAN 50, Debian 13 cloud-init, 4 vCPU x86-64-v3 / 12 GiB / 100 GB; built to fleet conventions (vh 1000, infra-ops 850, docker 851) - compose: mem_limit 4g -> 8g, homepage labels removed; host port 8392 kept - cutover by tar stream, verified by digest (contents plus owner, group, mode, mtime and type identical); albok.nh3.internal now points at nh3-albok - old /srv/albok stays on nh3-docker (container removed, compose renamed) until acceptance and the operator's approval to delete it - sudo-io archive hosts and ops-log STACK_HOSTS include nh3-albok
2026-10-04 23:33:24 -07:00
parent 867f33d672
commit d3f59aa2c5
17 changed files with 607 additions and 30 deletions
@@ -1,5 +1,5 @@
-# Host preparation for albok-service on nh3-docker (2026-10-02). Idempotent.
-#   scripts/elway infra-ops@10.100.50.40 --playbook playbooks/albok-service-host.yaml
+# Host preparation for albok-service (2026-10-02; nh3-docker, then nh3-albok from 2026-10-04). Idempotent.
+#   scripts/elway infra-ops@10.100.50.44 --playbook playbooks/albok-service-host.yaml
 # Creates the service identity and read groups with FIXED numeric ids (the contract requires them),
 # and the two local roots owned by uid 1500. The config file itself (it carries the embedder key)
 # is uploaded separately to /srv/albok/etc/albok.yaml: see stacks/albok-service/README.md.
@@ -0,0 +1,95 @@
+# Base build for nh3-albok (VM 110 on nh3-pve-2, 10.100.50.44): the dedicated albok-service host.
+#   scripts/elway infra-ops@10.100.50.44 --playbook playbooks/nh3-albok-host.yaml
+# Order: playbooks/nh3-albok-vm.yaml (on nh3-pve-2) → bootstrap-infra-ops-user.yaml run AS vh → this
+# → albok-service-host.yaml (ids + /srv/albok) → stacks/albok-service/README.md deploy steps.
+#
+# Built to docs/pfi/fleet-conventions.md: `vh` 1000 (cloud-init), `infra-ops` 850, `docker` gid 851
+# created BEFORE docker-ce so the package adopts it, /opt/docker root:docker 2775.
+#
+# ⚠ This host exists to keep root-over-docker away from the albok store. So, unlike the general docker
+#   hosts, it gets NO portainer/dockge/agent containers, NO socket mounts, and dockerd does NOT listen on
+#   tcp/2375 (Homepage cannot discover it; the Homepage entry for albok is static in services.yaml).
+#   Do not add any of those here. A host-level read of containers is `ssh` + `docker ps`.
+steps:
+  - name: take the bootstrap infra-ops key back off vh (vh keeps the operator's keys)
+    sudo: true
+    shell: |
+      set -e
+      f=/home/vh/.ssh/authorized_keys
+      grep -v 'infra-ops@pfi-fleet' "$f" > "$f.new"
+      test -s "$f.new"
+      chown vh:vh "$f.new"; chmod 600 "$f.new"; mv "$f.new" "$f"
+    when: "sudo -n grep -q 'infra-ops@pfi-fleet' /home/vh/.ssh/authorized_keys"
+
+  - name: timezone America/Los_Angeles (the fleet's; trixie ships US/Pacific only in tzdata-legacy)
+    sudo: true
+    shell: timedatectl set-timezone America/Los_Angeles
+    when: "[ \"$(timedatectl show -p Timezone --value)\" != America/Los_Angeles ]"
+
+  - name: base packages (guest agent for vzdump freeze/thaw, rsync for the store move)
+    sudo: true
+    shell: DEBIAN_FRONTEND=noninteractive apt-get install -y -q qemu-guest-agent rsync jq curl ca-certificates gnupg
+    when: "! dpkg -s qemu-guest-agent rsync jq >/dev/null 2>&1"
+
+  - name: qemu-guest-agent running
+    sudo: true
+    shell: systemctl start qemu-guest-agent
+    when: "! systemctl is-active --quiet qemu-guest-agent"
+
+  - name: docker group pinned at gid 851 BEFORE docker-ce installs
+    sudo: true
+    shell: groupadd -g 851 docker
+    when: "! getent group docker >/dev/null"
+
+  - name: Docker's apt repo (trixie)
+    sudo: true
+    shell: |
+      set -e
+      install -d -m 0755 /etc/apt/keyrings
+      curl -fsSL https://download.docker.com/linux/debian/gpg | gpg --dearmor --yes -o /etc/apt/keyrings/docker.gpg
+      chmod a+r /etc/apt/keyrings/docker.gpg
+      echo "deb [arch=$(dpkg --print-architecture) signed-by=/etc/apt/keyrings/docker.gpg] https://download.docker.com/linux/debian $(. /etc/os-release; echo $VERSION_CODENAME) stable" \
+        > /etc/apt/sources.list.d/docker.list
+      apt-get update -qq
+    creates: /etc/apt/sources.list.d/docker.list
+
+  - name: docker-ce + compose plugin
+    sudo: true
+    shell: DEBIAN_FRONTEND=noninteractive apt-get install -y -q docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
+    when: "! dpkg -s docker-ce docker-compose-plugin >/dev/null 2>&1"
+
+  - name: vh and infra-ops in docker
+    sudo: true
+    shell: usermod -aG docker vh && usermod -aG docker infra-ops
+    when: "! id -nG infra-ops | grep -qw docker || ! id -nG vh | grep -qw docker"
+
+  - name: deploy tree /opt/docker{,/compose,/conf} root:docker 2775
+    sudo: true
+    shell: |
+      install -d -o root -g docker -m 2775 /opt/docker /opt/docker/compose /opt/docker/conf
+    changed_when: "false"
+
+  - name: traefik-net exists (the fleet compose convention names it; no traefik runs here)
+    sudo: true
+    shell: docker network create traefik-net
+    when: "! sudo -n docker network inspect traefik-net >/dev/null 2>&1"
+
+verify:
+  - name: identities and groups match the fleet pin
+    shell: |
+      test "$(id -u vh)" = 1000 && test "$(id -u infra-ops)" = 850 && test "$(getent group docker | cut -d: -f3)" = 851
+      id -nG infra-ops | grep -qw docker
+    changed_when: "false"
+  - name: vh no longer carries the infra-ops key
+    sudo: true
+    shell: "! grep -q 'infra-ops@pfi-fleet' /home/vh/.ssh/authorized_keys && grep -c . /home/vh/.ssh/authorized_keys"
+    changed_when: "false"
+  - name: dockerd up, listening on its unix socket only (nothing on tcp/2375 or 2376)
+    sudo: true
+    shell: |
+      docker version --format '{{.Server.Version}}'
+      ! ss -ltn | grep -E ':(2375|2376)\b'
+    changed_when: "false"
+  - name: guest agent active
+    shell: systemctl is-active qemu-guest-agent
+    changed_when: "false"
@@ -0,0 +1,98 @@
+# Create VM 110 `nh3-albok` on nh3-pve-2: the dedicated albok-service host (2026-10-04).
+#   scripts/elway infra-ops@10.100.250.62 --playbook playbooks/nh3-albok-vm.yaml
+# Needs vmbr0 VLAN-aware first (playbooks/nh3-pve-2-vmbr0-vlan-aware.yaml + a reboot).
+#
+# Why a VM of its own (albok-dev, relaying Prime 2026-10-04 ~2235): on nh3-docker the store shared a
+# host with three containers holding the docker socket read-write (portainer, portainer_agent, dockge)
+# plus dockerd on plaintext tcp/2375 for Homepage. Each of those is root over /srv/albok, and unit 4
+# puts the operator's personal vault and licensed books into restricted wings. Also memory: albok sat
+# at 2.4 of a 4 GiB cap on a 7.7 GiB host it shared with the post office and AdGuard.
+#
+# Shape: Debian 13 genericcloud + cloud-init. 4 vCPU (x86-64-v3: AVX2 for the index code, and still
+# restorable on nh3-pve's i9-13900H), 12 GiB, 100 GB on `vmstore`, VLAN 50 (nh3-servers) at
+# 10.100.50.44 — below the UDM's DHCP pool (.150–.249), like every other static on that VLAN.
+# VMID 110 continues the NH3 numbering (nh3-pve holds 100–109) so "VM 110" is unambiguous on the site;
+# PBS keeps nodes apart by namespace anyway.
+#
+# cloud-init creates `vh` (uid 1000, the fleet's human account) with NOPASSWD sudo. infra-ops (uid 850)
+# is then built by playbooks/bootstrap-infra-ops-user.yaml run AS vh, after which the infra-ops key is
+# taken back off vh (playbooks/nh3-albok-host.yaml).
+vars:
+  vmid: "110"
+  name: nh3-albok
+  ip: 10.100.50.44/24
+  gw: 10.100.50.1
+  # nh3-docker AdGuard first, ana-docker AdGuard second: the same pair the UDM hands out on VLAN 50
+  dns: 10.100.50.40 10.250.50.70
+  img_url: https://cloud.debian.org/images/cloud/trixie/latest
+  img: debian-13-genericcloud-amd64.qcow2
+  img_dir: /var/lib/vz/import
+
+steps:
+  - name: refuse unless this is nh3-pve-2, vmbr0 is VLAN-aware, and the VMID and address are free
+    sudo: true
+    shell: |
+      set -e
+      test "$(hostname)" = nh3-pve-2
+      test "$(cat /sys/class/net/vmbr0/bridge/vlan_filtering)" = 1
+      if ping -c2 -W1 $(echo {{ ip }} | cut -d/ -f1) >/dev/null 2>&1; then echo "{{ ip }} ANSWERS — taken"; exit 1; fi
+    changed_when: "false"
+    when: "! qm status {{ vmid }} >/dev/null 2>&1"
+
+  - name: fetch the Debian 13 genericcloud image and check it against SHA512SUMS
+    sudo: true
+    shell: |
+      set -e
+      install -d -m 0755 {{ img_dir }}
+      cd {{ img_dir }}
+      curl -fsSL -o SHA512SUMS.trixie {{ img_url }}/SHA512SUMS
+      curl -fsSL -o {{ img }}.part {{ img_url }}/{{ img }}
+      want=$(awk '$2=="{{ img }}"{print $1}' SHA512SUMS.trixie)
+      test -n "$want"
+      echo "$want  {{ img }}.part" | sha512sum -c -
+      mv {{ img }}.part {{ img }}
+    when: "! qm status {{ vmid }} >/dev/null 2>&1"
+
+  - name: create VM {{ vmid }} ({{ name }})
+    sudo: true
+    shell: |
+      set -e
+      qm create {{ vmid }} --name {{ name }} --ostype l26 --cpu x86-64-v3 --cores 4 --memory 12288 \
+        --scsihw virtio-scsi-single --agent enabled=1 --onboot 1 --tags albok \
+        --net0 virtio,bridge=vmbr0,tag=50,firewall=1 \
+        --serial0 socket --vga serial0 \
+        --description "albok-service host (fleet knowledgebase). Runs ONLY albok-service. Built 2026-10-04, eshpfi playbooks/nh3-albok-vm.yaml + nh3-albok-host.yaml; servers/nh3-albok/README.md"
+      qm set {{ vmid }} --scsi0 vmstore:0,import-from={{ img_dir }}/{{ img }},discard=on,ssd=1,iothread=1
+      qm disk resize {{ vmid }} scsi0 100G
+      qm set {{ vmid }} --ide2 vmstore:cloudinit --boot order=scsi0
+    creates: /etc/pve/qemu-server/{{ vmid }}.conf
+
+  - name: cloud-init — user vh, keys, static address, resolvers
+    sudo: true
+    shell: |
+      set -e
+      k=$(mktemp)
+      cat > "$k" <<'EOF'
+      ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN+1HBwfXrkfTYWdcnWCjLJ6VLAGC87gxH5h5vKaaA3c infra-ops@pfi-fleet
+      ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIKGm/lJslGEL2xDBY/7LdUc0Q4GyuxwFtGspYTe6WsVE lkrav@VHHomeG13
+      ssh-rsa AAAAB3NzaC1yc2EAAAADAQABAAABgQDP/AwgaLghPoQi9RDt+m/9H9jLHDoP93WgzczlhZlSvZoQ5qDpWu5oAQlRwImiWhVL1ClK4FwEz9JULsta7Uq3FaiQvs2ltevrW/EpNbhAfxZa4Z1ELrPAOO8SSlhZ+Zer33SPkfJD9t3Hi9TXheCaANiEtE6hH1nCVLpHWQdbzy+L7EyG8z+/RzNXwwIL8f0BZ8n7FvFujgwc1OQTXlg4fiW3FdB3uzJMVRT59IIsPEq/ODGZZ6XGtHy24Ewcsi8l78I+oKaPdMl8ZXNBQrNb0MPIJnHkQwTrzlDKZ0Oh1UKm7I9i9hDMB/M2+aQ+6MxzY5TLZyDvRYd8R3j9Mnrp3ouCnNcklGAkX7i8hNJ2y8xCbwOeP8sMd+VtQ6hBbNa6KAEaAgGuWHHS0KGGO4HjtvDxJH8e7BnGzveIeCUGLVKMrg08w9nqF6Kzf3Y72ou0yCmnu7kKqocS1Pcr+e5E1MdT4brc2BfYkzqExLp8gxQaJ1Zas2OXPNXpAxL58xc= lkraven@nh3-dev
+      EOF
+      sed -i 's/^ *//' "$k"
+      qm set {{ vmid }} --ciuser vh --sshkeys "$k" --ipconfig0 ip={{ ip }},gw={{ gw }} \
+        --nameserver "{{ dns }}" --searchdomain nh3.internal --ciupgrade 1
+      rm -f "$k"
+    when: "! qm config {{ vmid }} | grep -q '^ciuser: vh'"
+
+  - name: start VM {{ vmid }}
+    sudo: true
+    shell: qm start {{ vmid }}
+    when: "! qm status {{ vmid }} | grep -q running"
+
+verify:
+  - name: VM is running with the intended shape
+    sudo: true
+    shell: |
+      qm status {{ vmid }}
+      qm config {{ vmid }} | grep -E '^(name|cores|cpu|memory|net0|scsi0|ipconfig0|onboot|ciuser):'
+      qm status {{ vmid }} | grep -q running
+    changed_when: "false"
@@ -0,0 +1,63 @@
+# nh3-pve-2: make vmbr0 VLAN-aware so guests can sit on tagged VLANs (2026-10-04).
+#   scripts/elway infra-ops@10.100.250.62 --playbook playbooks/nh3-pve-2-vmbr0-vlan-aware.yaml
+# then REBOOT the node (the playbook does not). First consumer: VM 110 nh3-albok on VLAN 50
+# (nh3-servers), the albok-service host.
+#
+# Same shape as nh3-pve's vmbr0 (`bridge-vlan-aware yes`, `bridge-vids 2-4094`), where every guest
+# carries `tag=50`. The host address stays untagged on the bridge (PVID 1 = the switch port's native
+# nh3-mgmt), so 10.100.250.62 does not move.
+#
+# ⚠ Applied by REBOOT, never by a live `ifreload -a`. On 2026-10-03 a live ifreload here read the new
+#   vmbr0 as "not a bridge" and downed it (servers/nh3-pve-2/README.md). The syntax check below
+#   (`ifreload -a -s`) parses the file without touching any interface.
+# ⚠ Only the vmbr0 stanza changes. `auto nic0` (the AMT port) must stay as it is: if Linux leaves that
+#   port down, igc powers off the PHY and AMT goes dark.
+steps:
+  - name: refuse unless this is nh3-pve-2 with vmbr0 on nic3 and no guests
+    sudo: true
+    shell: |
+      set -e
+      test "$(hostname)" = nh3-pve-2
+      grep -q '^iface vmbr0 inet static' /etc/network/interfaces
+      grep -q 'bridge-ports nic3$' /etc/network/interfaces
+      grep -q '^auto nic0$' /etc/network/interfaces
+      test -z "$(qm list | tail -n +2)"
+    changed_when: "false"
+    when: "! grep -q 'bridge-vlan-aware yes' /etc/network/interfaces"
+
+  - name: back up /etc/network/interfaces
+    sudo: true
+    shell: cp -a /etc/network/interfaces /etc/network/interfaces.pre-vlan-aware-2026-10-04
+    creates: /etc/network/interfaces.pre-vlan-aware-2026-10-04
+
+  - name: add bridge-vlan-aware + bridge-vids to the vmbr0 stanza (after bridge-fd)
+    sudo: true
+    shell: |
+      set -e
+      tmp=$(mktemp)
+      awk '{print} /^iface vmbr0 /{inv=1} inv && /^\tbridge-fd /{print "\tbridge-vlan-aware yes"; print "\tbridge-vids 2-4094"; inv=0}' \
+        /etc/network/interfaces > "$tmp"
+      test "$(grep -c 'bridge-vlan-aware yes' "$tmp")" = 1
+      cat "$tmp" > /etc/network/interfaces
+      rm -f "$tmp"
+    when: "! grep -q 'bridge-vlan-aware yes' /etc/network/interfaces"
+
+  # `ifup -s` exits 1 on the PVE-default `bridge-fd 0` ("out of range 2-255") even on the file the node
+  # booted from, so its rc carries no signal here. The check is instead: the new file draws exactly the
+  # warnings the original did, nothing more.
+  - name: syntax-check the file without applying it (no warnings beyond the original's)
+    sudo: true
+    shell: |
+      set -e
+      old=$(ifup -a -s -i /etc/network/interfaces.pre-vlan-aware-2026-10-04 2>&1 || true)
+      new=$(ifup -a -s -i /etc/network/interfaces 2>&1 || true)
+      echo "$new"
+      test "$old" = "$new"
+    changed_when: "false"
+
+verify:
+  - name: vmbr0 stanza carries both lines, nic0 untouched
+    shell: |
+      sed -n '/^iface vmbr0/,/^$/p;/^auto nic0/,/^iface nic0/p' /etc/network/interfaces
+      grep -q 'bridge-vlan-aware yes' /etc/network/interfaces && grep -q 'bridge-vids 2-4094' /etc/network/interfaces && grep -q '^auto nic0$' /etc/network/interfaces
+    changed_when: "false"