esh-pfi-infrastructure/playbooks/esh-pve-nvidia-host.yaml

# esh-pve — NVIDIA driver on the HYPERVISOR, for the RTX 2000 Ada that an LXC
# (esh-ml1, embedding + reranking) uses.
#
# WHY A HOST DRIVER: Prime's decision 2026-09-24 — LXC + host driver, NOT a
# VFIO VM. Both esh-pve hangs this year (the 2026-08-19 hard freeze and the
# 2026-09-10 vzdump that left `kvm -id 102` in D state for 14 days) came from
# VFIO passthrough on this box. An LXC shares the host kernel's driver, so no
# passthrough, no pinned VM RAM. Accepted cost: a driver on the hypervisor,
# rebuilt by DKMS on every PVE kernel update, and the LXC's userspace libraries
# must be the SAME version as this kernel module (see
# playbooks/esh-ml1-lxc.yaml, which installs from the same .run file).
#
# WHY THE .run FILE, NOT AN APT REPO: the LXC must match the host module
# exactly. One file, installed with the kernel module here and with
# --no-kernel-module in the container, makes the match true by construction.
# An apt repo on both sides would let a routine `apt upgrade` move one side
# and not the other, which breaks every CUDA call in the container.
#
# WHY 580: the fleet's vLLM image (vllm/vllm-openai:v0.24.0) is a CUDA 13.0
# build and needs a >= 580 driver. fv-ml1 runs 580.65.06; this is the newest
# point release on the same branch. `-no-compat32` because a headless compute
# box has no use for 32-bit libraries.
#
# NO REBOOT NEEDED: nouveau is blacklisted and was never loaded, and the T400's
# vfio-pci ids no longer match any device, so nothing holds the card. The
# module loads live. (esh-pve is ESH's only DNS and mesh route; a reboot takes
# the whole site dark from outside — avoid one.)
#
# Upgrading the driver: bump driver_version + driver_sha256 here AND in
# playbooks/esh-ml1-lxc.yaml, run this, then that. Until both have run, CUDA
# in the container fails with "driver/library version mismatch".
#
# Run:  scripts/elway root@esh-pve --playbook playbooks/esh-pve-nvidia-host.yaml

vars:
  driver_version: 580.178.04
  # From download.nvidia.com/XFree86/Linux-x86_64/580.178.04/*.run.sha256sum
  driver_sha256: 1833c8c5c53481dfab46df442c3de26ec5d4c35084e03d94ad93746d29899750
  stage_dir: /root/nvidia

steps:
  - name: Stage the driver .run and check it against NVIDIA's published sha256
    shell: |
      set -e
      mkdir -p {{ stage_dir }}
      f={{ stage_dir }}/NVIDIA-Linux-x86_64-{{ driver_version }}-no-compat32.run
      if ! echo "{{ driver_sha256 }}  $f" | sha256sum -c --status 2>/dev/null; then
        curl -fsSL -o "$f.part" \
          https://download.nvidia.com/XFree86/Linux-x86_64/{{ driver_version }}/NVIDIA-Linux-x86_64-{{ driver_version }}-no-compat32.run
        echo "{{ driver_sha256 }}  $f.part" | sha256sum -c
        mv "$f.part" "$f"
        chmod 0755 "$f"
      fi
    when: "! echo '{{ driver_sha256 }}  {{ stage_dir }}/NVIDIA-Linux-x86_64-{{ driver_version }}-no-compat32.run' | sha256sum -c --status 2>/dev/null"

  # The T400 was passed to VM 102 through vfio-pci by id. The card and the VM
  # are both gone; the ids match nothing. Kept (not deleted) so the retirement
  # is readable later.
  - name: Retire the T400's vfio-pci id binding
    shell: mv /etc/modprobe.d/vfio.conf {{ stage_dir }}/vfio.conf.retired-2026-09-24
    when: "grep -q '10de:1ff2' /etc/modprobe.d/vfio.conf 2>/dev/null"

  # blacklist.conf blacklisted `nvidia` alongside nouveau (VM-passthrough era,
  # when the host must never touch the card). nouveau stays blacklisted.
  - name: Stop blacklisting the nvidia module (nouveau stays blacklisted)
    shell: |
      cp -n /etc/modprobe.d/blacklist.conf {{ stage_dir }}/blacklist.conf.pre-nvidia
      sed -i '/^blacklist nvidia$/d' /etc/modprobe.d/blacklist.conf
    when: "grep -qx 'blacklist nvidia' /etc/modprobe.d/blacklist.conf"

  - name: Install the driver with DKMS (open kernel modules)
    # --no-opengl-files: headless compute. --kernel-module-type=open: Ada
    # (Turing+) is supported by the open modules and fv-ml1 runs them too.
    # DKMS rebuilds the module whenever a new proxmox-headers package lands;
    # `proxmox-headers-6.8` (the meta-package) is installed so new headers
    # arrive with each new kernel.
    shell: |
      sh {{ stage_dir }}/NVIDIA-Linux-x86_64-{{ driver_version }}-no-compat32.run \
        --dkms --silent --no-x-check --no-opengl-files --kernel-module-type=open \
        --kernel-name="$(uname -r)"
    when: "! dkms status -m nvidia -v {{ driver_version }} -k $(uname -r) 2>/dev/null | grep -q installed"

  - name: Load nvidia + nvidia_uvm at every boot
    shell: |
      printf '# RTX 2000 Ada for the esh-ml1 LXC — see playbooks/esh-pve-nvidia-host.yaml\nnvidia\nnvidia_uvm\n' \
        > /etc/modules-load.d/nvidia.conf
    when: "! grep -qx 'nvidia_uvm' /etc/modules-load.d/nvidia.conf 2>/dev/null"

  # The LXC binds /dev/nvidia0, /dev/nvidiactl and /dev/nvidia-uvm{,-tools}.
  # Those nodes do not exist until something asks for them, and `pct start`
  # refuses a dev passthrough whose host path is missing. This unit creates
  # them and keeps the GPU initialised (persistence mode), and runs BEFORE
  # pve-guests so onboot guests find them. If it fails, only the GPU LXC fails
  # to start; the other guests do not depend on it.
  - name: Install the GPU-init + persistence unit, ordered before pve-guests
    shell: |
      cat > /etc/systemd/system/nvidia-persistenced.service <<'EOF'
      # See playbooks/esh-pve-nvidia-host.yaml
      [Unit]
      Description=NVIDIA persistence daemon + device nodes for the GPU LXC
      Before=pve-guests.service
      After=systemd-modules-load.service

      [Service]
      Type=forking
      # -c0 -u: load nvidia_uvm and create /dev/nvidia-uvm{,-tools} and /dev/nvidia0.
      ExecStartPre=/usr/bin/nvidia-modprobe -c0 -u
      ExecStart=/usr/bin/nvidia-persistenced --user root
      ExecStopPost=/bin/rm -rf /var/run/nvidia-persistenced

      [Install]
      WantedBy=multi-user.target
      EOF
      systemctl daemon-reload
      systemctl enable --now nvidia-persistenced.service
    when: "! systemctl is-active --quiet nvidia-persistenced.service || ! grep -q 'Before=pve-guests.service' /etc/systemd/system/nvidia-persistenced.service"

verify:
  - name: nvidia-smi sees the RTX 2000 Ada at the pinned driver version
    shell: nvidia-smi --query-gpu=name,driver_version --format=csv,noheader | grep -q 'RTX 2000.*, {{ driver_version }}$'
    changed_when: "false"

  - name: Kernel module in use is the DKMS-built {{ driver_version }}
    shell: grep -q '{{ driver_version }}' /proc/driver/nvidia/version && dkms status -m nvidia -v {{ driver_version }} -k $(uname -r) | grep -q installed
    changed_when: "false"

  - name: Device nodes the LXC binds all exist
    shell: test -c /dev/nvidia0 && test -c /dev/nvidiactl && test -c /dev/nvidia-uvm && test -c /dev/nvidia-uvm-tools
    changed_when: "false"

  - name: Persistence mode is on
    shell: nvidia-smi --query-gpu=persistence_mode --format=csv,noheader | grep -qx Enabled
    changed_when: "false"

  - name: Survives a reboot (modules-load + unit enabled before pve-guests)
    shell: grep -qx nvidia_uvm /etc/modules-load.d/nvidia.conf && systemctl is-enabled --quiet nvidia-persistenced.service && ! grep -qx 'blacklist nvidia' /etc/modprobe.d/blacklist.conf
    changed_when: "false"