feat(esh-ml1): RTX 2000E Ada on esh-pve serves embed + rerank as a LiteLLM failover

esh-pve: NVIDIA 580.178.04 (open modules, DKMS) installed on the host from
NVIDIA's .run and loaded live, no reboot. nvidia-persistenced unit creates the
device nodes before pve-guests; the T400's vfio-pci ids and `blacklist nvidia`
retired. playbooks/esh-pve-nvidia-host.yaml.

esh-ml1: CT 110, unprivileged Debian 12, 10.0.50.80, GPU nodes via devN,
NVIDIA userspace from the same .run (--no-kernel-modules), docker-ce +
nvidia-container-toolkit (no-cgroups). playbooks/esh-ml1-lxc.yaml. Not in the
vzdump job on purpose. DNS esh-ml1.esh.internal.

stacks/embed-rerank: Qwen3-Embedding-0.6B :8001 + bge-reranker-v2-m3 :8013 on
the same vLLM v0.24.0 digest and flags as fv-ml1. Measured parity: embed
cosine FV-vs-ESH median 0.999908 (min 0.999772), inside both self-noise
floors; rerank max |delta| 0.000145 vs floor 0.000181, identical ranking.

litellm: qwen3-embedding and reranker gain an esh-ml1 deployment at order 2
behind fv-ml1 (order 1). Order fallback proven with throwaway groups:
refused primary +0.15 s, host-down primary ~18.7 s per call, dead-only 500.

Also: repaired the DB-only alias reranker-a3-bge-v2-m3, dead since the
fv-ml1 relocation (still named 10.250.50.54); documented the third
unkillable homepage wedge on esh-docker-vm.

playbooks/esh-ml1-lxc.yaml

@@ -0,0 +1,175 @@
+# esh-ml1 — the GPU LXC on esh-pve that serves embedding + reranking on the
+# RTX 2000E Ada (16 GB). Runs against the HYPERVISOR and drives the container
+# through `pct`, so it works before the container has any users or keys.
+#
+# Prerequisite: playbooks/esh-pve-nvidia-host.yaml (host kernel module, device
+# nodes, persistence unit ordered before pve-guests).
+#
+# WHAT IT BUILDS
+#   - CT {{ ctid }} `esh-ml1`, unprivileged, Debian 12 (the template the host's
+#     own bookworm matches), nesting+keyctl for Docker, static 10.0.50.80 on
+#     VLAN 50 (outside the UDM's .150–.250 DHCP pool), onboot AFTER esh-scale
+#     and the two VMs so a GPU fault can never delay DNS or the mesh route.
+#   - /dev/nvidia0, nvidiactl, nvidia-uvm, nvidia-uvm-tools bound in via
+#     `devN:` (PVE handles the cgroup rules and node creation).
+#   - NVIDIA userspace from THE SAME .run the host installed, with
+#     --no-kernel-modules. Host module and container libraries must be the same
+#     version or every CUDA call fails with "driver/library version mismatch".
+#   - Fleet identities per docs/pfi/fleet-conventions.md: docker gid 851
+#     (created BEFORE docker-ce so the package does not pick one), infra-ops
+#     850 NOPASSWD, vh 1000, /opt/docker root:docker 2775.
+#   - docker-ce + nvidia-container-toolkit with no-cgroups=true (an unprivileged
+#     LXC cannot write the device cgroup; the LXC's own devN rules already
+#     confine the GPU).
+#
+# NOT BACKED UP, on purpose: esh-pve's vzdump job lists vmids explicitly and
+# this CT is not in it. Everything here is rebuilt by this playbook plus
+# stacks/embed-rerank; the models re-download from Hugging Face.
+#
+# Run:  scripts/elway root@esh-pve --playbook playbooks/esh-ml1-lxc.yaml
+
+vars:
+  ctid: 110
+  hostname: esh-ml1
+  ip_cidr: 10.0.50.80/24
+  gateway: 10.0.50.1
+  vlan: 50
+  cores: 6
+  memory_mb: 16384
+  swap_mb: 2048
+  rootfs_gb: 80
+  template: debian-12-standard_12.12-1_amd64.tar.zst
+  # Must equal playbooks/esh-pve-nvidia-host.yaml's driver_version.
+  driver_version: 580.178.04
+  stage_dir: /root/nvidia
+  infra_ops_pubkey: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN+1HBwfXrkfTYWdcnWCjLJ6VLAGC87gxH5h5vKaaA3c infra-ops@pfi-fleet"
+
+steps:
+  - name: Fetch the Debian 12 template
+    shell: pveam update >/dev/null && pveam download local {{ template }}
+    creates: /var/lib/vz/template/cache/{{ template }}
+
+  - name: Create CT {{ ctid }} ({{ hostname }}) with the GPU device nodes bound in
+    shell: |
+      pct create {{ ctid }} local:vztmpl/{{ template }} \
+        --hostname {{ hostname }} --unprivileged 1 --features nesting=1,keyctl=1 \
+        --cores {{ cores }} --memory {{ memory_mb }} --swap {{ swap_mb }} \
+        --rootfs local-lvm:{{ rootfs_gb }} \
+        --net0 name=eth0,bridge=vmbr0,firewall=1,gw={{ gateway }},ip={{ ip_cidr }},tag={{ vlan }},type=veth \
+        --onboot 1 --startup order=30,up=10 \
+        --dev0 /dev/nvidia0 --dev1 /dev/nvidiactl \
+        --dev2 /dev/nvidia-uvm --dev3 /dev/nvidia-uvm-tools \
+        --description "esh-ml1 — RTX 2000E Ada embedding + reranking (vLLM). Built by eshpfi playbooks/esh-ml1-lxc.yaml; stack stacks/embed-rerank. NOT in the vzdump job on purpose (rebuildable)."
+    when: "! pct status {{ ctid }} >/dev/null 2>&1"
+
+  - name: Start the container
+    shell: pct start {{ ctid }} && sleep 5
+    when: "! pct status {{ ctid }} | grep -q running"
+
+  - name: Base packages + bookworm point upgrade
+    shell: |
+      pct exec {{ ctid }} -- bash -s <<'EOF'
+      set -euo pipefail
+      export DEBIAN_FRONTEND=noninteractive
+      for i in $(seq 1 30); do getent hosts deb.debian.org >/dev/null && break; sleep 1; done
+      apt-get update -qq
+      apt-get -y -qq full-upgrade
+      apt-get install -y -qq --no-install-recommends ca-certificates curl gnupg sudo kmod jq less rsync locales
+      EOF
+    # rsync: scripts/deploy-stack.sh pushes stacks with it.
+    when: "! pct exec {{ ctid }} -- sh -c 'command -v sudo && command -v jq && command -v rsync && command -v locale-gen' >/dev/null 2>&1"
+
+  - name: Generate the en_US.UTF-8 locale the template's LANG names
+    # The Debian template sets LANG=en_US.UTF-8 but ships no locale, so every
+    # apt and perl run warns about it.
+    shell: |
+      pct exec {{ ctid }} -- bash -s <<'EOF'
+      set -euo pipefail
+      sed -i 's/^# *en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen
+      locale-gen >/dev/null
+      EOF
+    when: "! pct exec {{ ctid }} -- locale -a 2>/dev/null | grep -qi '^en_US.utf8$'"
+
+  - name: Fleet identities (docker 851, infra-ops 850, vh 1000) + /opt/docker tree
+    shell: |
+      pct exec {{ ctid }} -- bash -s <<'EOF'
+      set -euo pipefail
+      getent group docker >/dev/null || groupadd -g 851 docker
+      getent group infra-ops >/dev/null || groupadd -g 850 infra-ops
+      id infra-ops >/dev/null 2>&1 || useradd -u 850 -g 850 -G docker -m -s /bin/bash infra-ops
+      chmod 0700 /home/infra-ops
+      install -d -m 0700 -o infra-ops -g infra-ops /home/infra-ops/.ssh
+      echo '{{ infra_ops_pubkey }}' > /home/infra-ops/.ssh/authorized_keys
+      chown infra-ops:infra-ops /home/infra-ops/.ssh/authorized_keys
+      chmod 0600 /home/infra-ops/.ssh/authorized_keys
+      echo 'infra-ops ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/infra-ops
+      chmod 0440 /etc/sudoers.d/infra-ops
+      id vh >/dev/null 2>&1 || useradd -u 1000 -U -G docker,sudo -m -s /bin/bash vh
+      chmod 0700 /home/vh
+      install -d -m 2775 -o root -g docker /opt/docker /opt/docker/compose /opt/docker/conf
+      EOF
+    when: "! pct exec {{ ctid }} -- sh -c 'test \"$(id -u infra-ops)\" = 850 && test \"$(getent group docker | cut -d: -f3)\" = 851 && test -d /opt/docker/compose'"
+
+  - name: Hugging Face cache for the embed-rerank stack
+    shell: pct exec {{ ctid }} -- install -d -m 2775 -o root -g docker /opt/aimodels /opt/aimodels/huggingface
+    when: "! pct exec {{ ctid }} -- test -d /opt/aimodels/huggingface"
+
+  - name: NVIDIA userspace {{ driver_version }} from the host's .run (no kernel modules)
+    shell: |
+      set -e
+      f=NVIDIA-Linux-x86_64-{{ driver_version }}-no-compat32.run
+      pct push {{ ctid }} {{ stage_dir }}/$f /root/$f --perms 0755
+      pct exec {{ ctid }} -- sh /root/$f --no-kernel-modules --silent --no-x-check --no-opengl-files
+      pct exec {{ ctid }} -- rm -f /root/$f
+    when: "! pct exec {{ ctid }} -- sh -c 'nvidia-smi --query-gpu=driver_version --format=csv,noheader 2>/dev/null' | grep -qx '{{ driver_version }}'"
+
+  - name: docker-ce from Docker's apt repo
+    shell: |
+      pct exec {{ ctid }} -- bash -s <<'EOF'
+      set -euo pipefail
+      export DEBIAN_FRONTEND=noninteractive
+      install -m 0755 -d /etc/apt/keyrings
+      curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc
+      chmod a+r /etc/apt/keyrings/docker.asc
+      echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian bookworm stable" \
+        > /etc/apt/sources.list.d/docker.list
+      apt-get update -qq
+      apt-get install -y -qq docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
+      EOF
+    when: "! pct exec {{ ctid }} -- sh -c 'command -v docker' >/dev/null 2>&1"
+
+  - name: nvidia-container-toolkit, wired into dockerd, no-cgroups for the LXC
+    shell: |
+      pct exec {{ ctid }} -- bash -s <<'EOF'
+      set -euo pipefail
+      export DEBIAN_FRONTEND=noninteractive
+      curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey \
+        | gpg --dearmor --yes -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg
+      curl -fsSL https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list \
+        | sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' \
+        > /etc/apt/sources.list.d/nvidia-container-toolkit.list
+      apt-get update -qq
+      apt-get install -y -qq nvidia-container-toolkit
+      nvidia-ctk runtime configure --runtime=docker
+      nvidia-ctk config --set nvidia-container-cli.no-cgroups=true --in-place
+      systemctl restart docker
+      EOF
+    when: "! pct exec {{ ctid }} -- sh -c 'grep -q nvidia /etc/docker/daemon.json && grep -Eq \"^no-cgroups *= *true\" /etc/nvidia-container-runtime/config.toml' 2>/dev/null"
+
+verify:
+  - name: Container is running with onboot set, started after the core guests
+    shell: "pct status {{ ctid }} | grep -q running && pct config {{ ctid }} | grep -q '^onboot: 1' && pct config {{ ctid }} | grep -q '^startup: order=30'"
+    changed_when: "false"
+
+  - name: nvidia-smi inside the CT reports the host's driver version
+    shell: pct exec {{ ctid }} -- nvidia-smi --query-gpu=name,driver_version --format=csv,noheader | grep -q 'RTX 2000.*, {{ driver_version }}$'
+    changed_when: "false"
+
+  - name: A Docker container gets the GPU (toolkit + no-cgroups work end to end)
+    shell: pct exec {{ ctid }} -- docker run --rm --gpus all debian:12-slim nvidia-smi -L | grep -q 'RTX 2000'
+    changed_when: "false"
+
+  - name: Fleet identities are the pinned ids
+    shell: |
+      pct exec {{ ctid }} -- sh -c 'test "$(id -u infra-ops)" = 850 && test "$(id -u vh)" = 1000 && test "$(getent group docker | cut -d: -f3)" = 851 && id -nG infra-ops | grep -qw docker'
+    changed_when: "false"

playbooks/esh-pve-nvidia-host.yaml

@@ -0,0 +1,137 @@
+# esh-pve — NVIDIA driver on the HYPERVISOR, for the RTX 2000 Ada that an LXC
+# (esh-ml1, embedding + reranking) uses.
+#
+# WHY A HOST DRIVER: Prime's decision 2026-09-24 — LXC + host driver, NOT a
+# VFIO VM. Both esh-pve hangs this year (the 2026-08-19 hard freeze and the
+# 2026-09-10 vzdump that left `kvm -id 102` in D state for 14 days) came from
+# VFIO passthrough on this box. An LXC shares the host kernel's driver, so no
+# passthrough, no pinned VM RAM. Accepted cost: a driver on the hypervisor,
+# rebuilt by DKMS on every PVE kernel update, and the LXC's userspace libraries
+# must be the SAME version as this kernel module (see
+# playbooks/esh-ml1-lxc.yaml, which installs from the same .run file).
+#
+# WHY THE .run FILE, NOT AN APT REPO: the LXC must match the host module
+# exactly. One file, installed with the kernel module here and with
+# --no-kernel-module in the container, makes the match true by construction.
+# An apt repo on both sides would let a routine `apt upgrade` move one side
+# and not the other, which breaks every CUDA call in the container.
+#
+# WHY 580: the fleet's vLLM image (vllm/vllm-openai:v0.24.0) is a CUDA 13.0
+# build and needs a >= 580 driver. fv-ml1 runs 580.65.06; this is the newest
+# point release on the same branch. `-no-compat32` because a headless compute
+# box has no use for 32-bit libraries.
+#
+# NO REBOOT NEEDED: nouveau is blacklisted and was never loaded, and the T400's
+# vfio-pci ids no longer match any device, so nothing holds the card. The
+# module loads live. (esh-pve is ESH's only DNS and mesh route; a reboot takes
+# the whole site dark from outside — avoid one.)
+#
+# Upgrading the driver: bump driver_version + driver_sha256 here AND in
+# playbooks/esh-ml1-lxc.yaml, run this, then that. Until both have run, CUDA
+# in the container fails with "driver/library version mismatch".
+#
+# Run:  scripts/elway root@esh-pve --playbook playbooks/esh-pve-nvidia-host.yaml
+
+vars:
+  driver_version: 580.178.04
+  # From download.nvidia.com/XFree86/Linux-x86_64/580.178.04/*.run.sha256sum
+  driver_sha256: 1833c8c5c53481dfab46df442c3de26ec5d4c35084e03d94ad93746d29899750
+  stage_dir: /root/nvidia
+
+steps:
+  - name: Stage the driver .run and check it against NVIDIA's published sha256
+    shell: |
+      set -e
+      mkdir -p {{ stage_dir }}
+      f={{ stage_dir }}/NVIDIA-Linux-x86_64-{{ driver_version }}-no-compat32.run
+      if ! echo "{{ driver_sha256 }}  $f" | sha256sum -c --status 2>/dev/null; then
+        curl -fsSL -o "$f.part" \
+          https://download.nvidia.com/XFree86/Linux-x86_64/{{ driver_version }}/NVIDIA-Linux-x86_64-{{ driver_version }}-no-compat32.run
+        echo "{{ driver_sha256 }}  $f.part" | sha256sum -c
+        mv "$f.part" "$f"
+        chmod 0755 "$f"
+      fi
+    when: "! echo '{{ driver_sha256 }}  {{ stage_dir }}/NVIDIA-Linux-x86_64-{{ driver_version }}-no-compat32.run' | sha256sum -c --status 2>/dev/null"
+
+  # The T400 was passed to VM 102 through vfio-pci by id. The card and the VM
+  # are both gone; the ids match nothing. Kept (not deleted) so the retirement
+  # is readable later.
+  - name: Retire the T400's vfio-pci id binding
+    shell: mv /etc/modprobe.d/vfio.conf {{ stage_dir }}/vfio.conf.retired-2026-09-24
+    when: "grep -q '10de:1ff2' /etc/modprobe.d/vfio.conf 2>/dev/null"
+
+  # blacklist.conf blacklisted `nvidia` alongside nouveau (VM-passthrough era,
+  # when the host must never touch the card). nouveau stays blacklisted.
+  - name: Stop blacklisting the nvidia module (nouveau stays blacklisted)
+    shell: |
+      cp -n /etc/modprobe.d/blacklist.conf {{ stage_dir }}/blacklist.conf.pre-nvidia
+      sed -i '/^blacklist nvidia$/d' /etc/modprobe.d/blacklist.conf
+    when: "grep -qx 'blacklist nvidia' /etc/modprobe.d/blacklist.conf"
+
+  - name: Install the driver with DKMS (open kernel modules)
+    # --no-opengl-files: headless compute. --kernel-module-type=open: Ada
+    # (Turing+) is supported by the open modules and fv-ml1 runs them too.
+    # DKMS rebuilds the module whenever a new proxmox-headers package lands;
+    # `proxmox-headers-6.8` (the meta-package) is installed so new headers
+    # arrive with each new kernel.
+    shell: |
+      sh {{ stage_dir }}/NVIDIA-Linux-x86_64-{{ driver_version }}-no-compat32.run \
+        --dkms --silent --no-x-check --no-opengl-files --kernel-module-type=open \
+        --kernel-name="$(uname -r)"
+    when: "! dkms status -m nvidia -v {{ driver_version }} -k $(uname -r) 2>/dev/null | grep -q installed"
+
+  - name: Load nvidia + nvidia_uvm at every boot
+    shell: |
+      printf '# RTX 2000 Ada for the esh-ml1 LXC — see playbooks/esh-pve-nvidia-host.yaml\nnvidia\nnvidia_uvm\n' \
+        > /etc/modules-load.d/nvidia.conf
+    when: "! grep -qx 'nvidia_uvm' /etc/modules-load.d/nvidia.conf 2>/dev/null"
+
+  # The LXC binds /dev/nvidia0, /dev/nvidiactl and /dev/nvidia-uvm{,-tools}.
+  # Those nodes do not exist until something asks for them, and `pct start`
+  # refuses a dev passthrough whose host path is missing. This unit creates
+  # them and keeps the GPU initialised (persistence mode), and runs BEFORE
+  # pve-guests so onboot guests find them. If it fails, only the GPU LXC fails
+  # to start; the other guests do not depend on it.
+  - name: Install the GPU-init + persistence unit, ordered before pve-guests
+    shell: |
+      cat > /etc/systemd/system/nvidia-persistenced.service <<'EOF'
+      # See playbooks/esh-pve-nvidia-host.yaml
+      [Unit]
+      Description=NVIDIA persistence daemon + device nodes for the GPU LXC
+      Before=pve-guests.service
+      After=systemd-modules-load.service
+
+      [Service]
+      Type=forking
+      # -c0 -u: load nvidia_uvm and create /dev/nvidia-uvm{,-tools} and /dev/nvidia0.
+      ExecStartPre=/usr/bin/nvidia-modprobe -c0 -u
+      ExecStart=/usr/bin/nvidia-persistenced --user root
+      ExecStopPost=/bin/rm -rf /var/run/nvidia-persistenced
+
+      [Install]
+      WantedBy=multi-user.target
+      EOF
+      systemctl daemon-reload
+      systemctl enable --now nvidia-persistenced.service
+    when: "! systemctl is-active --quiet nvidia-persistenced.service || ! grep -q 'Before=pve-guests.service' /etc/systemd/system/nvidia-persistenced.service"
+
+verify:
+  - name: nvidia-smi sees the RTX 2000 Ada at the pinned driver version
+    shell: nvidia-smi --query-gpu=name,driver_version --format=csv,noheader | grep -q 'RTX 2000.*, {{ driver_version }}$'
+    changed_when: "false"
+
+  - name: Kernel module in use is the DKMS-built {{ driver_version }}
+    shell: grep -q '{{ driver_version }}' /proc/driver/nvidia/version && dkms status -m nvidia -v {{ driver_version }} -k $(uname -r) | grep -q installed
+    changed_when: "false"
+
+  - name: Device nodes the LXC binds all exist
+    shell: test -c /dev/nvidia0 && test -c /dev/nvidiactl && test -c /dev/nvidia-uvm && test -c /dev/nvidia-uvm-tools
+    changed_when: "false"
+
+  - name: Persistence mode is on
+    shell: nvidia-smi --query-gpu=persistence_mode --format=csv,noheader | grep -qx Enabled
+    changed_when: "false"
+
+  - name: Survives a reboot (modules-load + unit enabled before pve-guests)
+    shell: grep -qx nvidia_uvm /etc/modules-load.d/nvidia.conf && systemctl is-enabled --quiet nvidia-persistenced.service && ! grep -qx 'blacklist nvidia' /etc/modprobe.d/blacklist.conf
+    changed_when: "false"