From bc278d4ba8f6b660fc2f8d9fc1acc9a8616a9c9e Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Fri, 25 Sep 2026 14:17:31 -0700 Subject: [PATCH] refactor(playbooks): host-generic GPU host + GPU LXC playbooks for nh3-ml1 - esh-pve-nvidia-host -> pve-nvidia-host: headers/dkms/build-essential step, nouveau blacklist + guarded unload (refuses if nouveau bound a device) - esh-ml1-lxc -> gpu-lxc: host vars have no defaults (elway aborts on undefined), rootfs storage/startup order parameterized, CT kept out of all-guests vzdump jobs - embed-rerank: Homepage labels take HOST_NAME/HOST_IP, defaults = esh-ml1 --- .../2026-09-24-esh-ml1-embed-rerank.md | 4 +- persistent-memory.md | 2 +- playbooks/{esh-ml1-lxc.yaml => gpu-lxc.yaml} | 92 +++++++++++----- ...-nvidia-host.yaml => pve-nvidia-host.yaml} | 103 +++++++++++++----- servers/esh-ml1/README.md | 9 +- servers/esh-pve/README.md | 2 +- stacks/beszel/hosts/esh-ml1.yaml | 2 +- stacks/embed-rerank/.env.example | 8 +- stacks/embed-rerank/compose.yaml | 18 +-- stacks/homepage/conf/docker.yaml | 2 +- 10 files changed, 170 insertions(+), 72 deletions(-) rename playbooks/{esh-ml1-lxc.yaml => gpu-lxc.yaml} (67%) rename playbooks/{esh-pve-nvidia-host.yaml => pve-nvidia-host.yaml} (56%) diff --git a/persistent-memory.d/2026-09-24-esh-ml1-embed-rerank.md b/persistent-memory.d/2026-09-24-esh-ml1-embed-rerank.md index 3d75470..34de46b 100644 --- a/persistent-memory.d/2026-09-24-esh-ml1-embed-rerank.md +++ b/persistent-memory.d/2026-09-24-esh-ml1-embed-rerank.md @@ -5,7 +5,7 @@ Executes Prime's 2026-09-24 decision (LXC + host driver, not a VFIO VM). Done **Host (esh-pve).** NVIDIA **580.178.04** open modules via DKMS from NVIDIA's `-no-compat32.run` (sha256 checked against NVIDIA's published sum), applied -live by `playbooks/esh-pve-nvidia-host.yaml`: nouveau was never loaded and the +live by `playbooks/pve-nvidia-host.yaml`: nouveau was never loaded and the T400's vfio ids matched nothing, so nothing held the card. Our own `nvidia-persistenced.service` creates `/dev/nvidia{0,ctl,-uvm,-uvm-tools}` **before `pve-guests`** (else `pct start` refuses the `devN:` entries). Retired @@ -15,7 +15,7 @@ rebuilt on purpose (boot path untouched). 580 because the fleet's vLLM not an apt repo: one file on both sides makes the host-module/container-lib version match true by construction; apt would let an upgrade move one side. -**CT 110 `esh-ml1`** (`playbooks/esh-ml1-lxc.yaml`): unprivileged Debian 12, +**CT 110 `esh-ml1`** (`playbooks/gpu-lxc.yaml`): unprivileged Debian 12, nesting+keyctl, 6c/16G/80G `local-lvm`, `10.0.50.80` VLAN 50 (static; UDM pool is `.150–.250`), `startup order=30` after esh-scale/VMs. NVIDIA userspace from the same `.run` with `--no-kernel-modules`; docker-ce; nvidia-container-toolkit diff --git a/persistent-memory.md b/persistent-memory.md index fcd1b91..36168c5 100644 --- a/persistent-memory.md +++ b/persistent-memory.md @@ -167,7 +167,7 @@ Pre-flight done before shutdown: 8. **pfi-gx10:** if Prime ran the AC-pull test on this visit, verify it came back by itself. 9. **Next work:** the GPU's purpose is TBD with Prime; likely the second embed/rerank - backend. Reuse `playbooks/esh-pve-nvidia-host.yaml` / `esh-ml1-lxc.yaml` (make + backend. Reuse `playbooks/pve-nvidia-host.yaml` / `gpu-lxc.yaml` (make them host-generic). nh3-pve runs kernel 6.8.12-11 with **no matching proxmox-headers installed**, on PVE 8.4.1 (Debian 12 template only). If the LM port was cabled, MEBx provisioning can be done through the NanoKVM. diff --git a/playbooks/esh-ml1-lxc.yaml b/playbooks/gpu-lxc.yaml similarity index 67% rename from playbooks/esh-ml1-lxc.yaml rename to playbooks/gpu-lxc.yaml index 20042cb..5600dc1 100644 --- a/playbooks/esh-ml1-lxc.yaml +++ b/playbooks/gpu-lxc.yaml @@ -1,15 +1,25 @@ -# esh-ml1 — the GPU LXC on esh-pve that serves embedding + reranking on the -# RTX 2000E Ada (16 GB). Runs against the HYPERVISOR and drives the container -# through `pct`, so it works before the container has any users or keys. +# A GPU LXC on a Proxmox host whose NVIDIA driver lives on the HYPERVISOR. +# Host-generic since 2026-09-25 (was esh-ml1-lxc). Runs against the HYPERVISOR +# and drives the container through `pct`, so it works before the container has +# any users or keys. # -# Prerequisite: playbooks/esh-pve-nvidia-host.yaml (host kernel module, device -# nodes, persistence unit ordered before pve-guests). +# CT host IP role +# 110 esh-ml1 esh-pve 10.0.50.80 v50 fleet embed/rerank (TEI) + reward seat +# 109 nh3-ml1 nh3-pve 10.100.50.80 v50 embed/rerank (TEI), second RTX 2000 +# +# Prerequisite: playbooks/pve-nvidia-host.yaml on the same host (kernel module, +# device nodes, persistence unit ordered before pve-guests). +# +# HOST-SPECIFIC VARS HAVE NO DEFAULTS, on purpose: elway refuses to run with an +# undefined {{ var }}, so forgetting one aborts before anything is touched +# instead of silently building esh-ml1's twin. Pass them all with --var (the +# exact invocations are at the bottom of this header and in each server README). # # WHAT IT BUILDS -# - CT {{ ctid }} `esh-ml1`, unprivileged, Debian 12 (the template the host's -# own bookworm matches), nesting+keyctl for Docker, static 10.0.50.80 on -# VLAN 50 (outside the UDM's .150–.250 DHCP pool), onboot AFTER esh-scale -# and the two VMs so a GPU fault can never delay DNS or the mesh route. +# - CT {{ ctid }}, unprivileged, Debian 12 (PVE < 8.4.20 rejects the Debian 13 +# template, and both hosts are bookworm), nesting+keyctl for Docker, static +# IP outside the site's DHCP pool, onboot AFTER the site's core guests +# (startup order 30) so a GPU fault can never delay DNS or the mesh route. # - /dev/nvidia0, nvidiactl, nvidia-uvm, nvidia-uvm-tools bound in via # `devN:` (PVE handles the cgroup rules and node creation). # - NVIDIA userspace from THE SAME .run the host installed, with @@ -22,25 +32,34 @@ # LXC cannot write the device cgroup; the LXC's own devN rules already # confine the GPU). # -# NOT BACKED UP, on purpose: esh-pve's vzdump job lists vmids explicitly and -# this CT is not in it. Everything here is rebuilt by this playbook plus -# stacks/embed-rerank; the models re-download from Hugging Face. +# NOT BACKED UP, on purpose: everything here is rebuilt by this playbook plus +# the stack, and the models re-download from Hugging Face. esh-pve's vzdump job +# lists vmids explicitly, so it never included 110. nh3-pve's job is `all 1`, +# so the playbook adds the CT to that job's `exclude` list. # -# Run: scripts/elway root@esh-pve --playbook playbooks/esh-ml1-lxc.yaml +# Run (esh-ml1): +# scripts/elway root@esh-pve --playbook playbooks/gpu-lxc.yaml \ +# --var ctid=110 --var hostname=esh-ml1 --var ip_cidr=10.0.50.80/24 \ +# --var ip_addr=10.0.50.80 --var gateway=10.0.50.1 --var vlan=50 \ +# --var rootfs_storage=local-lvm \ +# --var "role=the fleet embedding + reranking service (TEI)" +# Run (nh3-ml1): +# scripts/elway root@nh3-pve.nh3.internal --playbook playbooks/gpu-lxc.yaml \ +# --var ctid=109 --var hostname=nh3-ml1 --var ip_cidr=10.100.50.80/24 \ +# --var ip_addr=10.100.50.80 --var gateway=10.100.50.1 --var vlan=50 \ +# --var rootfs_storage=local-zfs \ +# --var "role=second embed/rerank backend (TEI)" vars: - ctid: 110 - hostname: esh-ml1 - ip_cidr: 10.0.50.80/24 - ip_addr: 10.0.50.80 - gateway: 10.0.50.1 - vlan: 50 + # Host-specific, NO defaults (see header): ctid, hostname, ip_cidr, ip_addr, + # gateway, vlan, rootfs_storage, role. cores: 6 memory_mb: 16384 swap_mb: 2048 rootfs_gb: 80 + startup_order: 30 template: debian-12-standard_12.12-1_amd64.tar.zst - # Must equal playbooks/esh-pve-nvidia-host.yaml's driver_version. + # Must equal playbooks/pve-nvidia-host.yaml's driver_version. driver_version: 580.178.04 stage_dir: /root/nvidia infra_ops_pubkey: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN+1HBwfXrkfTYWdcnWCjLJ6VLAGC87gxH5h5vKaaA3c infra-ops@pfi-fleet" @@ -55,14 +74,33 @@ steps: pct create {{ ctid }} local:vztmpl/{{ template }} \ --hostname {{ hostname }} --unprivileged 1 --features nesting=1,keyctl=1 \ --cores {{ cores }} --memory {{ memory_mb }} --swap {{ swap_mb }} \ - --rootfs local-lvm:{{ rootfs_gb }} \ + --rootfs {{ rootfs_storage }}:{{ rootfs_gb }} \ --net0 name=eth0,bridge=vmbr0,firewall=1,gw={{ gateway }},ip={{ ip_cidr }},tag={{ vlan }},type=veth \ - --onboot 1 --startup order=30,up=10 \ + --onboot 1 --startup order={{ startup_order }},up=10 \ --dev0 /dev/nvidia0 --dev1 /dev/nvidiactl \ --dev2 /dev/nvidia-uvm --dev3 /dev/nvidia-uvm-tools \ - --description "esh-ml1 — RTX 2000E Ada: the fleet embedding + reranking service (TEI). Built by eshpfi playbooks/esh-ml1-lxc.yaml; stack stacks/embed-rerank. NOT in the vzdump job on purpose (rebuildable)." + --description "{{ hostname }} — RTX 2000E Ada: {{ role }}. Built by eshpfi playbooks/gpu-lxc.yaml; stack stacks/embed-rerank. NOT backed up on purpose (rebuildable)." when: "! pct status {{ ctid }} >/dev/null 2>&1" + # A job with `all 1` would pick the new CT up tonight. Add it to that job's + # exclude list; jobs that name their vmids (esh-pve's) are left alone. + - name: Keep CT {{ ctid }} out of every all-guests vzdump job + shell: | + set -e + CT={{ ctid }} JOBS="$(pvesh get /cluster/backup --output-format json)" python3 - <<'PY' + import json, os, subprocess + ct = os.environ["CT"] + for j in json.loads(os.environ["JOBS"]): + if str(j.get("all", 0)) != "1": + continue + ex = [x for x in str(j.get("exclude", "")).split(",") if x] + if ct in ex: + continue + subprocess.run(["pvesh", "set", "/cluster/backup/" + j["id"], "--exclude", ",".join(ex + [ct])], check=True) + print("excluded", ct, "from", j["id"]) + PY + when: "awk -v ct={{ ctid }} 'function fl(){ if(isv&&all&&!hit) need=1 } /^[^ \\t]/{fl(); isv=($1==\"vzdump:\"); all=0; hit=0; next} isv&&$1==\"all\"&&$2==\"1\"{all=1} isv&&$1==\"exclude\"{n=split($2,a,\",\"); for(i=1;i<=n;i++) if(a[i]==ct) hit=1} END{fl(); exit !need}' /etc/pve/jobs.cfg" + - name: Start the container shell: pct start {{ ctid }} && sleep 5 when: "! pct status {{ ctid }} | grep -q running" @@ -168,7 +206,7 @@ steps: set -euo pipefail install -d /etc/systemd/system/docker.service.d cat > /etc/systemd/system/docker.service.d/override.conf <<'EOC' - # Homepage discovery — see eshpfi playbooks/esh-ml1-lxc.yaml + # Homepage discovery — see eshpfi playbooks/gpu-lxc.yaml [Service] ExecStart= ExecStart=/usr/bin/dockerd -H fd:// -H tcp://{{ ip_addr }}:2375 --containerd=/run/containerd/containerd.sock @@ -180,7 +218,7 @@ steps: verify: - name: Container is running with onboot set, started after the core guests - shell: "pct status {{ ctid }} | grep -q running && pct config {{ ctid }} | grep -q '^onboot: 1' && pct config {{ ctid }} | grep -q '^startup: order=30'" + shell: "pct status {{ ctid }} | grep -q running && pct config {{ ctid }} | grep -q '^onboot: 1' && pct config {{ ctid }} | grep -q '^startup: order={{ startup_order }}'" changed_when: "false" - name: nvidia-smi inside the CT reports the host's driver version @@ -195,3 +233,7 @@ verify: shell: | pct exec {{ ctid }} -- sh -c 'test "$(id -u infra-ops)" = 850 && test "$(id -u vh)" = 1000 && test "$(getent group docker | cut -d: -f3)" = 851 && id -nG infra-ops | grep -qw docker' changed_when: "false" + + - name: CT {{ ctid }} is in no all-guests vzdump job + shell: "! awk -v ct={{ ctid }} 'function fl(){ if(isv&&all&&!hit) need=1 } /^[^ \\t]/{fl(); isv=($1==\"vzdump:\"); all=0; hit=0; next} isv&&$1==\"all\"&&$2==\"1\"{all=1} isv&&$1==\"exclude\"{n=split($2,a,\",\"); for(i=1;i<=n;i++) if(a[i]==ct) hit=1} END{fl(); exit !need}' /etc/pve/jobs.cfg" + changed_when: "false" diff --git a/playbooks/esh-pve-nvidia-host.yaml b/playbooks/pve-nvidia-host.yaml similarity index 56% rename from playbooks/esh-pve-nvidia-host.yaml rename to playbooks/pve-nvidia-host.yaml index 8a218c9..2cf61c6 100644 --- a/playbooks/esh-pve-nvidia-host.yaml +++ b/playbooks/pve-nvidia-host.yaml @@ -1,14 +1,18 @@ -# esh-pve — NVIDIA driver on the HYPERVISOR, for the RTX 2000 Ada that an LXC -# (esh-ml1, embedding + reranking) uses. +# NVIDIA driver on a Proxmox HYPERVISOR, for a GPU that an LXC uses through the +# host's kernel module. Host-generic since 2026-09-25 (was esh-pve-nvidia-host). +# +# host GPU LXC run +# esh-pve RTX 2000E Ada (01:00.0) esh-ml1 (CT 110) 2026-09-24 +# nh3-pve RTX 2000E Ada (01:00.0) nh3-ml1 (CT 109) 2026-09-25 # # WHY A HOST DRIVER: Prime's decision 2026-09-24 — LXC + host driver, NOT a # VFIO VM. Both esh-pve hangs this year (the 2026-08-19 hard freeze and the # 2026-09-10 vzdump that left `kvm -id 102` in D state for 14 days) came from -# VFIO passthrough on this box. An LXC shares the host kernel's driver, so no +# VFIO passthrough on that box. An LXC shares the host kernel's driver, so no # passthrough, no pinned VM RAM. Accepted cost: a driver on the hypervisor, # rebuilt by DKMS on every PVE kernel update, and the LXC's userspace libraries -# must be the SAME version as this kernel module (see -# playbooks/esh-ml1-lxc.yaml, which installs from the same .run file). +# must be the SAME version as this kernel module (see playbooks/gpu-lxc.yaml, +# which installs from the same .run file). # # WHY THE .run FILE, NOT AN APT REPO: the LXC must match the host module # exactly. One file, installed with the kernel module here and with @@ -19,26 +23,45 @@ # WHY 580: the fleet's vLLM image (vllm/vllm-openai:v0.24.0) is a CUDA 13.0 # build and needs a >= 580 driver. fv-ml1 runs 580.65.06; this is the newest # point release on the same branch. `-no-compat32` because a headless compute -# box has no use for 32-bit libraries. +# box has no use for 32-bit libraries. ONE version fleet-wide for the RTX 2000 +# hosts, so a container image that works on one works on the other. # -# NO REBOOT NEEDED: nouveau is blacklisted and was never loaded, and the T400's -# vfio-pci ids no longer match any device, so nothing holds the card. The -# module loads live. (esh-pve is ESH's only DNS and mesh route; a reboot takes -# the whole site dark from outside — avoid one.) +# NO REBOOT NEEDED, on either host, and each PVE host is its site's DNS/mesh +# path, so avoid one. esh-pve: nouveau was blacklisted and never loaded, and the +# T400's vfio-pci ids matched nothing. nh3-pve: nouveau auto-loaded at boot but +# FAILED to bind the Ada card (`gsp ctor failed: -2`, no GSP firmware), so the +# module holds nothing and unloads cleanly. The unload step refuses if nouveau +# has actually bound a device; then the blacklist plus a reboot is the only way. # -# Upgrading the driver: bump driver_version + driver_sha256 here AND in -# playbooks/esh-ml1-lxc.yaml, run this, then that. Until both have run, CUDA -# in the container fails with "driver/library version mismatch". +# Upgrading the driver: bump driver_version + driver_sha256 here AND +# driver_version in playbooks/gpu-lxc.yaml, run this on every GPU host, then +# gpu-lxc.yaml for every GPU LXC. Until both have run, CUDA in the container +# fails with "driver/library version mismatch". # -# Run: scripts/elway root@esh-pve --playbook playbooks/esh-pve-nvidia-host.yaml +# Run: scripts/elway root@esh-pve --playbook playbooks/pve-nvidia-host.yaml +# scripts/elway root@nh3-pve.nh3.internal --playbook playbooks/pve-nvidia-host.yaml vars: driver_version: 580.178.04 # From download.nvidia.com/XFree86/Linux-x86_64/580.178.04/*.run.sha256sum driver_sha256: 1833c8c5c53481dfab46df442c3de26ec5d4c35084e03d94ad93746d29899750 stage_dir: /root/nvidia + # What `nvidia-smi --query-gpu=name` must contain on this host. + gpu_match: RTX 2000 steps: + # DKMS needs the headers for the RUNNING kernel (built now) and the series + # meta-package (so each future kernel's headers arrive with it and DKMS + # rebuilds on upgrade). nh3-pve had neither, nor dkms or a compiler. + - name: Kernel headers (running kernel + series meta), dkms, build tools + shell: | + set -e + export DEBIAN_FRONTEND=noninteractive + k=$(uname -r); series=$(echo "$k" | cut -d. -f1,2) + apt-get update -qq + apt-get install -y -qq "proxmox-headers-$k" "proxmox-headers-$series" dkms build-essential + when: "! dpkg -s proxmox-headers-$(uname -r) proxmox-headers-$(uname -r | cut -d. -f1,2) dkms build-essential >/dev/null 2>&1" + - name: Stage the driver .run and check it against NVIDIA's published sha256 shell: | set -e @@ -53,27 +76,45 @@ steps: fi when: "! echo '{{ driver_sha256 }} {{ stage_dir }}/NVIDIA-Linux-x86_64-{{ driver_version }}-no-compat32.run' | sha256sum -c --status 2>/dev/null" - # The T400 was passed to VM 102 through vfio-pci by id. The card and the VM - # are both gone; the ids match nothing. Kept (not deleted) so the retirement - # is readable later. + # esh-pve only: the T400 was passed to VM 102 through vfio-pci by id. The card + # and the VM are both gone; the ids match nothing. Kept (not deleted) so the + # retirement is readable later. - name: Retire the T400's vfio-pci id binding shell: mv /etc/modprobe.d/vfio.conf {{ stage_dir }}/vfio.conf.retired-2026-09-24 when: "grep -q '10de:1ff2' /etc/modprobe.d/vfio.conf 2>/dev/null" - # blacklist.conf blacklisted `nvidia` alongside nouveau (VM-passthrough era, - # when the host must never touch the card). nouveau stays blacklisted. + # esh-pve only: blacklist.conf blacklisted `nvidia` alongside nouveau + # (VM-passthrough era, when the host must never touch the card). nouveau + # stays blacklisted. - name: Stop blacklisting the nvidia module (nouveau stays blacklisted) shell: | cp -n /etc/modprobe.d/blacklist.conf {{ stage_dir }}/blacklist.conf.pre-nvidia sed -i '/^blacklist nvidia$/d' /etc/modprobe.d/blacklist.conf - when: "grep -qx 'blacklist nvidia' /etc/modprobe.d/blacklist.conf" + when: "grep -qx 'blacklist nvidia' /etc/modprobe.d/blacklist.conf 2>/dev/null" + + # nouveau is not in PVE's initramfs (checked on nh3-pve: 0 files), so the + # rootfs modprobe.d is where udev's autoload reads the blacklist. No + # initramfs rebuild or ESP sync needed. + - name: Blacklist nouveau + shell: | + printf '# nvidia owns the GPU (the LXC uses it) — see playbooks/pve-nvidia-host.yaml\nblacklist nouveau\noptions nouveau modeset=0\n' \ + > /etc/modprobe.d/blacklist-nouveau.conf + when: "! grep -rqx 'blacklist nouveau' /etc/modprobe.d/" + + - name: Unload nouveau (only when it holds no device) + shell: | + set -e + if ls /sys/bus/pci/drivers/nouveau/ 2>/dev/null | grep -q '^0000:'; then + echo "nouveau is BOUND to a device: it is blacklisted now, but only a reboot releases it" >&2 + exit 1 + fi + modprobe -r nouveau + when: "grep -q '^nouveau ' /proc/modules" - name: Install the driver with DKMS (open kernel modules) # --no-opengl-files: headless compute. --kernel-module-type=open: Ada # (Turing+) is supported by the open modules and fv-ml1 runs them too. - # DKMS rebuilds the module whenever a new proxmox-headers package lands; - # `proxmox-headers-6.8` (the meta-package) is installed so new headers - # arrive with each new kernel. + # DKMS rebuilds the module whenever a new proxmox-headers package lands. shell: | sh {{ stage_dir }}/NVIDIA-Linux-x86_64-{{ driver_version }}-no-compat32.run \ --dkms --silent --no-x-check --no-opengl-files --kernel-module-type=open \ @@ -82,7 +123,7 @@ steps: - name: Load nvidia + nvidia_uvm at every boot shell: | - printf '# RTX 2000 Ada for the esh-ml1 LXC — see playbooks/esh-pve-nvidia-host.yaml\nnvidia\nnvidia_uvm\n' \ + printf '# GPU for the host GPU LXC — see playbooks/pve-nvidia-host.yaml\nnvidia\nnvidia_uvm\n' \ > /etc/modules-load.d/nvidia.conf when: "! grep -qx 'nvidia_uvm' /etc/modules-load.d/nvidia.conf 2>/dev/null" @@ -95,7 +136,7 @@ steps: - name: Install the GPU-init + persistence unit, ordered before pve-guests shell: | cat > /etc/systemd/system/nvidia-persistenced.service <<'EOF' - # See playbooks/esh-pve-nvidia-host.yaml + # See playbooks/pve-nvidia-host.yaml [Unit] Description=NVIDIA persistence daemon + device nodes for the GPU LXC Before=pve-guests.service @@ -116,8 +157,8 @@ steps: when: "! systemctl is-active --quiet nvidia-persistenced.service || ! grep -q 'Before=pve-guests.service' /etc/systemd/system/nvidia-persistenced.service" verify: - - name: nvidia-smi sees the RTX 2000 Ada at the pinned driver version - shell: nvidia-smi --query-gpu=name,driver_version --format=csv,noheader | grep -q 'RTX 2000.*, {{ driver_version }}$' + - name: nvidia-smi sees the {{ gpu_match }} at the pinned driver version + shell: nvidia-smi --query-gpu=name,driver_version --format=csv,noheader | grep -q '{{ gpu_match }}.*, {{ driver_version }}$' changed_when: "false" - name: Kernel module in use is the DKMS-built {{ driver_version }} @@ -132,6 +173,10 @@ verify: shell: nvidia-smi --query-gpu=persistence_mode --format=csv,noheader | grep -qx Enabled changed_when: "false" - - name: Survives a reboot (modules-load + unit enabled before pve-guests) - shell: grep -qx nvidia_uvm /etc/modules-load.d/nvidia.conf && systemctl is-enabled --quiet nvidia-persistenced.service && ! grep -qx 'blacklist nvidia' /etc/modprobe.d/blacklist.conf + - name: Survives a reboot (modules-load + unit enabled + nouveau blacklisted) + shell: grep -qx nvidia_uvm /etc/modules-load.d/nvidia.conf && systemctl is-enabled --quiet nvidia-persistenced.service && ! grep -qx 'blacklist nvidia' /etc/modprobe.d/blacklist.conf 2>/dev/null && grep -rqx 'blacklist nouveau' /etc/modprobe.d/ + changed_when: "false" + + - name: DKMS will rebuild for the next kernel (series headers meta installed) + shell: dpkg -s proxmox-headers-$(uname -r | cut -d. -f1,2) >/dev/null 2>&1 changed_when: "false" diff --git a/servers/esh-ml1/README.md b/servers/esh-ml1/README.md index fcf4cec..772510b 100644 --- a/servers/esh-ml1/README.md +++ b/servers/esh-ml1/README.md @@ -111,14 +111,17 @@ free. ## How it is built -1. [`playbooks/esh-pve-nvidia-host.yaml`](../../playbooks/esh-pve-nvidia-host.yaml) +1. [`playbooks/pve-nvidia-host.yaml`](../../playbooks/pve-nvidia-host.yaml) — driver **580.178.04** (open modules, DKMS) on the **hypervisor**, the `nvidia-persistenced` unit that creates the device nodes before `pve-guests`, and removal of the old VFIO/blacklist config. -2. [`playbooks/esh-ml1-lxc.yaml`](../../playbooks/esh-ml1-lxc.yaml) — the CT, +2. [`playbooks/gpu-lxc.yaml`](../../playbooks/gpu-lxc.yaml) — the CT, `dev0–3` GPU nodes, the NVIDIA userspace from the **same `.run`** with `--no-kernel-modules`, fleet ids (infra-ops 850, docker 851, vh 1000), docker-ce and nvidia-container-toolkit with `no-cgroups = true`. + Host-generic since 2026-09-25 (it also builds nh3-ml1). Its host vars have + **no defaults**, so pass esh-ml1's set: the exact `--var` line is in the + playbook header under "Run (esh-ml1)". 3. `scripts/deploy-stack.sh esh-ml1 embed-rerank`, then `docker compose up -d`. Both playbooks are idempotent; re-run them to repair. @@ -142,7 +145,7 @@ build and this CT will fail to start at the next boot. |---|---|---| | **Beszel** (host + GPU telemetry) | NVIDIA agent `henrygd/beszel-agent-nvidia:0.18.7`, `stacks/beszel` + `hosts/esh-ml1.yaml`. Hub system `ridkfdpwfq3f730`. GPU util, VRAM, power and temperature are sampled. | alerts → infra-ops: Status down 2 m, Disk >85% 5 m, CPU >95% 15 m, Memory >90% 10 m, Temperature >85 °C 5 m (GPU included) | | **Uptime Kuma** (service) | `Embed — Qwen3 0.6B (TEI, esh-ml1)` → `:8001/health` (#27); `Rerank — bge-v2-m3 (TEI, esh-ml1)` → `:8013/health` (#28). TEI's health runs the backend. | alerts → infra-ops via althing-alert-bridge; `stacks/uptimekuma/monitors.yaml` | -| **Homepage** | three cards under *AI - Eval & Retrieval*. dockerd exposes tcp/2375 on `10.0.50.80` only (fleet norm, `playbooks/esh-ml1-lxc.yaml`). | `stacks/homepage/conf/docker.yaml` → `esh-ml1-docker` | +| **Homepage** | three cards under *AI - Eval & Retrieval*. dockerd exposes tcp/2375 on `10.0.50.80` only (fleet norm, `playbooks/gpu-lxc.yaml`). | `stacks/homepage/conf/docker.yaml` → `esh-ml1-docker` | | **Dozzle** (logs) | agent `v10.4.1` on `10.0.50.80:7007`, compose dir `dozzle-agent` | hub on ana-docker :8088 | The reward seat has **no Kuma check on purpose**: seats are outside Kuma's lane, diff --git a/servers/esh-pve/README.md b/servers/esh-pve/README.md index ed5a1c0..f1a8df0 100644 --- a/servers/esh-pve/README.md +++ b/servers/esh-pve/README.md @@ -59,7 +59,7 @@ and reranking ([`servers/esh-ml1/README.md`](../esh-ml1/README.md)). - **Driver 580.178.04, open kernel modules, DKMS**, from NVIDIA's `.run` (`/root/nvidia/`). Applied by - [`playbooks/esh-pve-nvidia-host.yaml`](../../playbooks/esh-pve-nvidia-host.yaml) + [`playbooks/pve-nvidia-host.yaml`](../../playbooks/pve-nvidia-host.yaml) **live, with no reboot**: nouveau was never loaded and nothing held the card. - **`nvidia-persistenced.service`** (ours, in `/etc/systemd/system`) runs `nvidia-modprobe -c0 -u` and the persistence daemon **before diff --git a/stacks/beszel/hosts/esh-ml1.yaml b/stacks/beszel/hosts/esh-ml1.yaml index 3f9a0c9..3b8579e 100644 --- a/stacks/beszel/hosts/esh-ml1.yaml +++ b/stacks/beszel/hosts/esh-ml1.yaml @@ -1,6 +1,6 @@ # esh-ml1 (CT 110 on esh-pve) — Beszel agent with NVIDIA GPU telemetry for the # RTX 2000E Ada (utilization, VRAM, temperature, power). Docker-in-LXC: the -# NVIDIA container toolkit runs with no-cgroups=true (playbooks/esh-ml1-lxc.yaml). +# NVIDIA container toolkit runs with no-cgroups=true (playbooks/gpu-lxc.yaml). # No extra filesystems: the root filesystem holds everything, models included. services: beszel-agent: diff --git a/stacks/embed-rerank/.env.example b/stacks/embed-rerank/.env.example index e004530..50252c3 100644 --- a/stacks/embed-rerank/.env.example +++ b/stacks/embed-rerank/.env.example @@ -1,4 +1,10 @@ -# embed-rerank tunables (esh-ml1, TEI). Copy to `.env` on the server. +# embed-rerank tunables (TEI). Copy to `.env` on the server. + +# Per-host Homepage labels. Unset = esh-ml1's values. nh3-ml1 sets: +# HOST_NAME=nh3-ml1 +# HOST_IP=10.100.50.80 +#HOST_NAME=esh-ml1 +#HOST_IP=10.0.50.80 # TEI image tag. `89-` = the Ada Lovelace (sm_89) build; a different GPU # generation needs a different prefix (see the TEI README's image table). diff --git a/stacks/embed-rerank/compose.yaml b/stacks/embed-rerank/compose.yaml index 3e4da5f..3184bff 100644 --- a/stacks/embed-rerank/compose.yaml +++ b/stacks/embed-rerank/compose.yaml @@ -1,6 +1,8 @@ # embed-rerank — THE fleet's embedding + reranking service, on esh-ml1 (CT 110 on # esh-pve, RTX 2000E Ada, 16 GB). Served by Hugging Face Text Embeddings -# Inference (TEI). +# Inference (TEI). A second instance runs on nh3-ml1 (CT 109 on nh3-pve, the same +# card) since 2026-09-25; the per-host bits are HOST_NAME / HOST_IP in .env, and +# their defaults are esh-ml1's, so esh-ml1's live .env needs no change. # # Prime, 2026-09-25: "TEI is embed/reranker server for esh-ml1 and the FLEET in # general, in future." It replaced vLLM here the same day, after a side-by-side @@ -30,7 +32,7 @@ # ports, and in practice only the LiteLLM gateway does (verified from the seats' # logs 2026-09-25: every request matched a gateway spend-log row). # -# Host setup: playbooks/esh-pve-nvidia-host.yaml, then playbooks/esh-ml1-lxc.yaml. +# Host setup: playbooks/pve-nvidia-host.yaml, then playbooks/gpu-lxc.yaml. # Tunables live in .env. name: embed-rerank @@ -76,10 +78,10 @@ services: start_period: 60s labels: - homepage.group=AI - Eval & Retrieval - - homepage.name=Embed — Qwen3 0.6B (TEI, esh-ml1) + - homepage.name=Embed — Qwen3 0.6B (TEI, ${HOST_NAME:-esh-ml1}) - homepage.icon=mdi-vector-arrange-below - - homepage.description=Fleet embeddings (qwen3-embedding) via TEI on esh-ml1 - - homepage.href=http://10.0.50.80:${EMBED_PORT}/docs + - homepage.description=Fleet embeddings (qwen3-embedding) via TEI on ${HOST_NAME:-esh-ml1} + - homepage.href=http://${HOST_IP:-10.0.50.80}:${EMBED_PORT}/docs tei-rerank: image: ghcr.io/huggingface/text-embeddings-inference:${TEI_TAG} @@ -117,7 +119,7 @@ services: start_period: 60s labels: - homepage.group=AI - Eval & Retrieval - - homepage.name=Rerank — bge-v2-m3 (TEI, esh-ml1) + - homepage.name=Rerank — bge-v2-m3 (TEI, ${HOST_NAME:-esh-ml1}) - homepage.icon=mdi-sort-variant - - homepage.description=Fleet reranker (reranker) via TEI on esh-ml1 - - homepage.href=http://10.0.50.80:${RERANK_PORT}/docs + - homepage.description=Fleet reranker (reranker) via TEI on ${HOST_NAME:-esh-ml1} + - homepage.href=http://${HOST_IP:-10.0.50.80}:${RERANK_PORT}/docs diff --git a/stacks/homepage/conf/docker.yaml b/stacks/homepage/conf/docker.yaml index 6fdef26..8afa0a5 100644 --- a/stacks/homepage/conf/docker.yaml +++ b/stacks/homepage/conf/docker.yaml @@ -32,7 +32,7 @@ irv-ml1-docker: port: 2375 # esh-ml1 — CT 110 on esh-pve: the fleet embed/rerank (TEI) + reward seats. -# dockerd listens only on its own address (playbooks/esh-ml1-lxc.yaml), 2026-09-25. +# dockerd listens only on its own address (playbooks/gpu-lxc.yaml), 2026-09-25. esh-ml1-docker: host: 10.0.50.80 port: 2375