refactor(playbooks): host-generic GPU host + GPU LXC playbooks for nh3-ml1

- esh-pve-nvidia-host -> pve-nvidia-host: headers/dkms/build-essential step,
  nouveau blacklist + guarded unload (refuses if nouveau bound a device)
- esh-ml1-lxc -> gpu-lxc: host vars have no defaults (elway aborts on undefined),
  rootfs storage/startup order parameterized, CT kept out of all-guests vzdump jobs
- embed-rerank: Homepage labels take HOST_NAME/HOST_IP, defaults = esh-ml1
This commit is contained in:
vh
2026-09-25 14:17:31 -07:00
parent cdd7605e89
commit bc278d4ba8
10 changed files with 170 additions and 72 deletions
@@ -1,15 +1,25 @@
# esh-ml1 — the GPU LXC on esh-pve that serves embedding + reranking on the
# RTX 2000E Ada (16 GB). Runs against the HYPERVISOR and drives the container
# through `pct`, so it works before the container has any users or keys.
# A GPU LXC on a Proxmox host whose NVIDIA driver lives on the HYPERVISOR.
# Host-generic since 2026-09-25 (was esh-ml1-lxc). Runs against the HYPERVISOR
# and drives the container through `pct`, so it works before the container has
# any users or keys.
#
# Prerequisite: playbooks/esh-pve-nvidia-host.yaml (host kernel module, device
# nodes, persistence unit ordered before pve-guests).
# CT host IP role
# 110 esh-ml1 esh-pve 10.0.50.80 v50 fleet embed/rerank (TEI) + reward seat
# 109 nh3-ml1 nh3-pve 10.100.50.80 v50 embed/rerank (TEI), second RTX 2000
#
# Prerequisite: playbooks/pve-nvidia-host.yaml on the same host (kernel module,
# device nodes, persistence unit ordered before pve-guests).
#
# HOST-SPECIFIC VARS HAVE NO DEFAULTS, on purpose: elway refuses to run with an
# undefined {{ var }}, so forgetting one aborts before anything is touched
# instead of silently building esh-ml1's twin. Pass them all with --var (the
# exact invocations are at the bottom of this header and in each server README).
#
# WHAT IT BUILDS
# - CT {{ ctid }} `esh-ml1`, unprivileged, Debian 12 (the template the host's
# own bookworm matches), nesting+keyctl for Docker, static 10.0.50.80 on
# VLAN 50 (outside the UDM's .150–.250 DHCP pool), onboot AFTER esh-scale
# and the two VMs so a GPU fault can never delay DNS or the mesh route.
# - CT {{ ctid }}, unprivileged, Debian 12 (PVE < 8.4.20 rejects the Debian 13
# template, and both hosts are bookworm), nesting+keyctl for Docker, static
# IP outside the site's DHCP pool, onboot AFTER the site's core guests
# (startup order 30) so a GPU fault can never delay DNS or the mesh route.
# - /dev/nvidia0, nvidiactl, nvidia-uvm, nvidia-uvm-tools bound in via
# `devN:` (PVE handles the cgroup rules and node creation).
# - NVIDIA userspace from THE SAME .run the host installed, with
@@ -22,25 +32,34 @@
# LXC cannot write the device cgroup; the LXC's own devN rules already
# confine the GPU).
#
# NOT BACKED UP, on purpose: esh-pve's vzdump job lists vmids explicitly and
# this CT is not in it. Everything here is rebuilt by this playbook plus
# stacks/embed-rerank; the models re-download from Hugging Face.
# NOT BACKED UP, on purpose: everything here is rebuilt by this playbook plus
# the stack, and the models re-download from Hugging Face. esh-pve's vzdump job
# lists vmids explicitly, so it never included 110. nh3-pve's job is `all 1`,
# so the playbook adds the CT to that job's `exclude` list.
#
# Run: scripts/elway root@esh-pve --playbook playbooks/esh-ml1-lxc.yaml
# Run (esh-ml1):
# scripts/elway root@esh-pve --playbook playbooks/gpu-lxc.yaml \
# --var ctid=110 --var hostname=esh-ml1 --var ip_cidr=10.0.50.80/24 \
# --var ip_addr=10.0.50.80 --var gateway=10.0.50.1 --var vlan=50 \
# --var rootfs_storage=local-lvm \
# --var "role=the fleet embedding + reranking service (TEI)"
# Run (nh3-ml1):
# scripts/elway root@nh3-pve.nh3.internal --playbook playbooks/gpu-lxc.yaml \
# --var ctid=109 --var hostname=nh3-ml1 --var ip_cidr=10.100.50.80/24 \
# --var ip_addr=10.100.50.80 --var gateway=10.100.50.1 --var vlan=50 \
# --var rootfs_storage=local-zfs \
# --var "role=second embed/rerank backend (TEI)"
vars:
ctid: 110
hostname: esh-ml1
ip_cidr: 10.0.50.80/24
ip_addr: 10.0.50.80
gateway: 10.0.50.1
vlan: 50
# Host-specific, NO defaults (see header): ctid, hostname, ip_cidr, ip_addr,
# gateway, vlan, rootfs_storage, role.
cores: 6
memory_mb: 16384
swap_mb: 2048
rootfs_gb: 80
startup_order: 30
template: debian-12-standard_12.12-1_amd64.tar.zst
# Must equal playbooks/esh-pve-nvidia-host.yaml's driver_version.
# Must equal playbooks/pve-nvidia-host.yaml's driver_version.
driver_version: 580.178.04
stage_dir: /root/nvidia
infra_ops_pubkey: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN+1HBwfXrkfTYWdcnWCjLJ6VLAGC87gxH5h5vKaaA3c infra-ops@pfi-fleet"
@@ -55,14 +74,33 @@ steps:
pct create {{ ctid }} local:vztmpl/{{ template }} \
--hostname {{ hostname }} --unprivileged 1 --features nesting=1,keyctl=1 \
--cores {{ cores }} --memory {{ memory_mb }} --swap {{ swap_mb }} \
--rootfs local-lvm:{{ rootfs_gb }} \
--rootfs {{ rootfs_storage }}:{{ rootfs_gb }} \
--net0 name=eth0,bridge=vmbr0,firewall=1,gw={{ gateway }},ip={{ ip_cidr }},tag={{ vlan }},type=veth \
--onboot 1 --startup order=30,up=10 \
--onboot 1 --startup order={{ startup_order }},up=10 \
--dev0 /dev/nvidia0 --dev1 /dev/nvidiactl \
--dev2 /dev/nvidia-uvm --dev3 /dev/nvidia-uvm-tools \
--description "esh-ml1 — RTX 2000E Ada: the fleet embedding + reranking service (TEI). Built by eshpfi playbooks/esh-ml1-lxc.yaml; stack stacks/embed-rerank. NOT in the vzdump job on purpose (rebuildable)."
--description "{{ hostname }} — RTX 2000E Ada: {{ role }}. Built by eshpfi playbooks/gpu-lxc.yaml; stack stacks/embed-rerank. NOT backed up on purpose (rebuildable)."
when: "! pct status {{ ctid }} >/dev/null 2>&1"
# A job with `all 1` would pick the new CT up tonight. Add it to that job's
# exclude list; jobs that name their vmids (esh-pve's) are left alone.
- name: Keep CT {{ ctid }} out of every all-guests vzdump job
shell: |
set -e
CT={{ ctid }} JOBS="$(pvesh get /cluster/backup --output-format json)" python3 - <<'PY'
import json, os, subprocess
ct = os.environ["CT"]
for j in json.loads(os.environ["JOBS"]):
if str(j.get("all", 0)) != "1":
continue
ex = [x for x in str(j.get("exclude", "")).split(",") if x]
if ct in ex:
continue
subprocess.run(["pvesh", "set", "/cluster/backup/" + j["id"], "--exclude", ",".join(ex + [ct])], check=True)
print("excluded", ct, "from", j["id"])
PY
when: "awk -v ct={{ ctid }} 'function fl(){ if(isv&&all&&!hit) need=1 } /^[^ \\t]/{fl(); isv=($1==\"vzdump:\"); all=0; hit=0; next} isv&&$1==\"all\"&&$2==\"1\"{all=1} isv&&$1==\"exclude\"{n=split($2,a,\",\"); for(i=1;i<=n;i++) if(a[i]==ct) hit=1} END{fl(); exit !need}' /etc/pve/jobs.cfg"
- name: Start the container
shell: pct start {{ ctid }} && sleep 5
when: "! pct status {{ ctid }} | grep -q running"
@@ -168,7 +206,7 @@ steps:
set -euo pipefail
install -d /etc/systemd/system/docker.service.d
cat > /etc/systemd/system/docker.service.d/override.conf <<'EOC'
# Homepage discovery — see eshpfi playbooks/esh-ml1-lxc.yaml
# Homepage discovery — see eshpfi playbooks/gpu-lxc.yaml
[Service]
ExecStart=
ExecStart=/usr/bin/dockerd -H fd:// -H tcp://{{ ip_addr }}:2375 --containerd=/run/containerd/containerd.sock
@@ -180,7 +218,7 @@ steps:
verify:
- name: Container is running with onboot set, started after the core guests
shell: "pct status {{ ctid }} | grep -q running && pct config {{ ctid }} | grep -q '^onboot: 1' && pct config {{ ctid }} | grep -q '^startup: order=30'"
shell: "pct status {{ ctid }} | grep -q running && pct config {{ ctid }} | grep -q '^onboot: 1' && pct config {{ ctid }} | grep -q '^startup: order={{ startup_order }}'"
changed_when: "false"
- name: nvidia-smi inside the CT reports the host's driver version
@@ -195,3 +233,7 @@ verify:
shell: |
pct exec {{ ctid }} -- sh -c 'test "$(id -u infra-ops)" = 850 && test "$(id -u vh)" = 1000 && test "$(getent group docker | cut -d: -f3)" = 851 && id -nG infra-ops | grep -qw docker'
changed_when: "false"
- name: CT {{ ctid }} is in no all-guests vzdump job
shell: "! awk -v ct={{ ctid }} 'function fl(){ if(isv&&all&&!hit) need=1 } /^[^ \\t]/{fl(); isv=($1==\"vzdump:\"); all=0; hit=0; next} isv&&$1==\"all\"&&$2==\"1\"{all=1} isv&&$1==\"exclude\"{n=split($2,a,\",\"); for(i=1;i<=n;i++) if(a[i]==ct) hit=1} END{fl(); exit !need}' /etc/pve/jobs.cfg"
changed_when: "false"
@@ -1,14 +1,18 @@
# esh-pve — NVIDIA driver on the HYPERVISOR, for the RTX 2000 Ada that an LXC
# (esh-ml1, embedding + reranking) uses.
# NVIDIA driver on a Proxmox HYPERVISOR, for a GPU that an LXC uses through the
# host's kernel module. Host-generic since 2026-09-25 (was esh-pve-nvidia-host).
#
# host GPU LXC run
# esh-pve RTX 2000E Ada (01:00.0) esh-ml1 (CT 110) 2026-09-24
# nh3-pve RTX 2000E Ada (01:00.0) nh3-ml1 (CT 109) 2026-09-25
#
# WHY A HOST DRIVER: Prime's decision 2026-09-24 — LXC + host driver, NOT a
# VFIO VM. Both esh-pve hangs this year (the 2026-08-19 hard freeze and the
# 2026-09-10 vzdump that left `kvm -id 102` in D state for 14 days) came from
# VFIO passthrough on this box. An LXC shares the host kernel's driver, so no
# VFIO passthrough on that box. An LXC shares the host kernel's driver, so no
# passthrough, no pinned VM RAM. Accepted cost: a driver on the hypervisor,
# rebuilt by DKMS on every PVE kernel update, and the LXC's userspace libraries
# must be the SAME version as this kernel module (see
# playbooks/esh-ml1-lxc.yaml, which installs from the same .run file).
# must be the SAME version as this kernel module (see playbooks/gpu-lxc.yaml,
# which installs from the same .run file).
#
# WHY THE .run FILE, NOT AN APT REPO: the LXC must match the host module
# exactly. One file, installed with the kernel module here and with
@@ -19,26 +23,45 @@
# WHY 580: the fleet's vLLM image (vllm/vllm-openai:v0.24.0) is a CUDA 13.0
# build and needs a >= 580 driver. fv-ml1 runs 580.65.06; this is the newest
# point release on the same branch. `-no-compat32` because a headless compute
# box has no use for 32-bit libraries.
# box has no use for 32-bit libraries. ONE version fleet-wide for the RTX 2000
# hosts, so a container image that works on one works on the other.
#
# NO REBOOT NEEDED: nouveau is blacklisted and was never loaded, and the T400's
# vfio-pci ids no longer match any device, so nothing holds the card. The
# module loads live. (esh-pve is ESH's only DNS and mesh route; a reboot takes
# the whole site dark from outside — avoid one.)
# NO REBOOT NEEDED, on either host, and each PVE host is its site's DNS/mesh
# path, so avoid one. esh-pve: nouveau was blacklisted and never loaded, and the
# T400's vfio-pci ids matched nothing. nh3-pve: nouveau auto-loaded at boot but
# FAILED to bind the Ada card (`gsp ctor failed: -2`, no GSP firmware), so the
# module holds nothing and unloads cleanly. The unload step refuses if nouveau
# has actually bound a device; then the blacklist plus a reboot is the only way.
#
# Upgrading the driver: bump driver_version + driver_sha256 here AND in
# playbooks/esh-ml1-lxc.yaml, run this, then that. Until both have run, CUDA
# in the container fails with "driver/library version mismatch".
# Upgrading the driver: bump driver_version + driver_sha256 here AND
# driver_version in playbooks/gpu-lxc.yaml, run this on every GPU host, then
# gpu-lxc.yaml for every GPU LXC. Until both have run, CUDA in the container
# fails with "driver/library version mismatch".
#
# Run: scripts/elway root@esh-pve --playbook playbooks/esh-pve-nvidia-host.yaml
# Run: scripts/elway root@esh-pve --playbook playbooks/pve-nvidia-host.yaml
# scripts/elway root@nh3-pve.nh3.internal --playbook playbooks/pve-nvidia-host.yaml
vars:
driver_version: 580.178.04
# From download.nvidia.com/XFree86/Linux-x86_64/580.178.04/*.run.sha256sum
driver_sha256: 1833c8c5c53481dfab46df442c3de26ec5d4c35084e03d94ad93746d29899750
stage_dir: /root/nvidia
# What `nvidia-smi --query-gpu=name` must contain on this host.
gpu_match: RTX 2000
steps:
# DKMS needs the headers for the RUNNING kernel (built now) and the series
# meta-package (so each future kernel's headers arrive with it and DKMS
# rebuilds on upgrade). nh3-pve had neither, nor dkms or a compiler.
- name: Kernel headers (running kernel + series meta), dkms, build tools
shell: |
set -e
export DEBIAN_FRONTEND=noninteractive
k=$(uname -r); series=$(echo "$k" | cut -d. -f1,2)
apt-get update -qq
apt-get install -y -qq "proxmox-headers-$k" "proxmox-headers-$series" dkms build-essential
when: "! dpkg -s proxmox-headers-$(uname -r) proxmox-headers-$(uname -r | cut -d. -f1,2) dkms build-essential >/dev/null 2>&1"
- name: Stage the driver .run and check it against NVIDIA's published sha256
shell: |
set -e
@@ -53,27 +76,45 @@ steps:
fi
when: "! echo '{{ driver_sha256 }} {{ stage_dir }}/NVIDIA-Linux-x86_64-{{ driver_version }}-no-compat32.run' | sha256sum -c --status 2>/dev/null"
# The T400 was passed to VM 102 through vfio-pci by id. The card and the VM
# are both gone; the ids match nothing. Kept (not deleted) so the retirement
# is readable later.
# esh-pve only: the T400 was passed to VM 102 through vfio-pci by id. The card
# and the VM are both gone; the ids match nothing. Kept (not deleted) so the
# retirement is readable later.
- name: Retire the T400's vfio-pci id binding
shell: mv /etc/modprobe.d/vfio.conf {{ stage_dir }}/vfio.conf.retired-2026-09-24
when: "grep -q '10de:1ff2' /etc/modprobe.d/vfio.conf 2>/dev/null"
# blacklist.conf blacklisted `nvidia` alongside nouveau (VM-passthrough era,
# when the host must never touch the card). nouveau stays blacklisted.
# esh-pve only: blacklist.conf blacklisted `nvidia` alongside nouveau
# (VM-passthrough era, when the host must never touch the card). nouveau
# stays blacklisted.
- name: Stop blacklisting the nvidia module (nouveau stays blacklisted)
shell: |
cp -n /etc/modprobe.d/blacklist.conf {{ stage_dir }}/blacklist.conf.pre-nvidia
sed -i '/^blacklist nvidia$/d' /etc/modprobe.d/blacklist.conf
when: "grep -qx 'blacklist nvidia' /etc/modprobe.d/blacklist.conf"
when: "grep -qx 'blacklist nvidia' /etc/modprobe.d/blacklist.conf 2>/dev/null"
# nouveau is not in PVE's initramfs (checked on nh3-pve: 0 files), so the
# rootfs modprobe.d is where udev's autoload reads the blacklist. No
# initramfs rebuild or ESP sync needed.
- name: Blacklist nouveau
shell: |
printf '# nvidia owns the GPU (the LXC uses it) — see playbooks/pve-nvidia-host.yaml\nblacklist nouveau\noptions nouveau modeset=0\n' \
> /etc/modprobe.d/blacklist-nouveau.conf
when: "! grep -rqx 'blacklist nouveau' /etc/modprobe.d/"
- name: Unload nouveau (only when it holds no device)
shell: |
set -e
if ls /sys/bus/pci/drivers/nouveau/ 2>/dev/null | grep -q '^0000:'; then
echo "nouveau is BOUND to a device: it is blacklisted now, but only a reboot releases it" >&2
exit 1
fi
modprobe -r nouveau
when: "grep -q '^nouveau ' /proc/modules"
- name: Install the driver with DKMS (open kernel modules)
# --no-opengl-files: headless compute. --kernel-module-type=open: Ada
# (Turing+) is supported by the open modules and fv-ml1 runs them too.
# DKMS rebuilds the module whenever a new proxmox-headers package lands;
# `proxmox-headers-6.8` (the meta-package) is installed so new headers
# arrive with each new kernel.
# DKMS rebuilds the module whenever a new proxmox-headers package lands.
shell: |
sh {{ stage_dir }}/NVIDIA-Linux-x86_64-{{ driver_version }}-no-compat32.run \
--dkms --silent --no-x-check --no-opengl-files --kernel-module-type=open \
@@ -82,7 +123,7 @@ steps:
- name: Load nvidia + nvidia_uvm at every boot
shell: |
printf '# RTX 2000 Ada for the esh-ml1 LXC — see playbooks/esh-pve-nvidia-host.yaml\nnvidia\nnvidia_uvm\n' \
printf '# GPU for the host GPU LXC — see playbooks/pve-nvidia-host.yaml\nnvidia\nnvidia_uvm\n' \
> /etc/modules-load.d/nvidia.conf
when: "! grep -qx 'nvidia_uvm' /etc/modules-load.d/nvidia.conf 2>/dev/null"
@@ -95,7 +136,7 @@ steps:
- name: Install the GPU-init + persistence unit, ordered before pve-guests
shell: |
cat > /etc/systemd/system/nvidia-persistenced.service <<'EOF'
# See playbooks/esh-pve-nvidia-host.yaml
# See playbooks/pve-nvidia-host.yaml
[Unit]
Description=NVIDIA persistence daemon + device nodes for the GPU LXC
Before=pve-guests.service
@@ -116,8 +157,8 @@ steps:
when: "! systemctl is-active --quiet nvidia-persistenced.service || ! grep -q 'Before=pve-guests.service' /etc/systemd/system/nvidia-persistenced.service"
verify:
- name: nvidia-smi sees the RTX 2000 Ada at the pinned driver version
shell: nvidia-smi --query-gpu=name,driver_version --format=csv,noheader | grep -q 'RTX 2000.*, {{ driver_version }}$'
- name: nvidia-smi sees the {{ gpu_match }} at the pinned driver version
shell: nvidia-smi --query-gpu=name,driver_version --format=csv,noheader | grep -q '{{ gpu_match }}.*, {{ driver_version }}$'
changed_when: "false"
- name: Kernel module in use is the DKMS-built {{ driver_version }}
@@ -132,6 +173,10 @@ verify:
shell: nvidia-smi --query-gpu=persistence_mode --format=csv,noheader | grep -qx Enabled
changed_when: "false"
- name: Survives a reboot (modules-load + unit enabled before pve-guests)
shell: grep -qx nvidia_uvm /etc/modules-load.d/nvidia.conf && systemctl is-enabled --quiet nvidia-persistenced.service && ! grep -qx 'blacklist nvidia' /etc/modprobe.d/blacklist.conf
- name: Survives a reboot (modules-load + unit enabled + nouveau blacklisted)
shell: grep -qx nvidia_uvm /etc/modules-load.d/nvidia.conf && systemctl is-enabled --quiet nvidia-persistenced.service && ! grep -qx 'blacklist nvidia' /etc/modprobe.d/blacklist.conf 2>/dev/null && grep -rqx 'blacklist nouveau' /etc/modprobe.d/
changed_when: "false"
- name: DKMS will rebuild for the next kernel (series headers meta installed)
shell: dpkg -s proxmox-headers-$(uname -r | cut -d. -f1,2) >/dev/null 2>&1
changed_when: "false"