202 lines
11 KiB
YAML
202 lines
11 KiB
YAML
# NVIDIA driver on a Proxmox HYPERVISOR, for a GPU that an LXC uses through the
|
|
# host's kernel module. Host-generic since 2026-09-25 (was esh-pve-nvidia-host).
|
|
#
|
|
# host GPU LXC run
|
|
# esh-pve RTX 2000E Ada (01:00.0) esh-ml1 (CT 110) 2026-09-24
|
|
# nh3-pve RTX 2000E Ada (01:00.0) nh3-ml1 (CT 109) 2026-09-25
|
|
#
|
|
# WHY A HOST DRIVER: Prime's decision 2026-09-24 — LXC + host driver, NOT a
|
|
# VFIO VM. Both esh-pve hangs this year (the 2026-08-19 hard freeze and the
|
|
# 2026-09-10 vzdump that left `kvm -id 102` in D state for 14 days) came from
|
|
# VFIO passthrough on that box. An LXC shares the host kernel's driver, so no
|
|
# passthrough, no pinned VM RAM. Accepted cost: a driver on the hypervisor,
|
|
# rebuilt by DKMS on every PVE kernel update, and the LXC's userspace libraries
|
|
# must be the SAME version as this kernel module (see playbooks/gpu-lxc.yaml,
|
|
# which installs from the same .run file).
|
|
#
|
|
# WHY THE .run FILE, NOT AN APT REPO: the LXC must match the host module
|
|
# exactly. One file, installed with the kernel module here and with
|
|
# --no-kernel-module in the container, makes the match true by construction.
|
|
# An apt repo on both sides would let a routine `apt upgrade` move one side
|
|
# and not the other, which breaks every CUDA call in the container.
|
|
#
|
|
# WHY 580: the fleet's vLLM image (vllm/vllm-openai:v0.24.0) is a CUDA 13.0
|
|
# build and needs a >= 580 driver. fv-ml1 runs 580.65.06; this is the newest
|
|
# point release on the same branch. `-no-compat32` because a headless compute
|
|
# box has no use for 32-bit libraries. ONE version fleet-wide for the RTX 2000
|
|
# hosts, so a container image that works on one works on the other.
|
|
#
|
|
# NO REBOOT NEEDED, on either host, and each PVE host is its site's DNS/mesh
|
|
# path, so avoid one. esh-pve: nouveau was blacklisted and never loaded, and the
|
|
# T400's vfio-pci ids matched nothing. nh3-pve: nouveau auto-loaded at boot but
|
|
# FAILED to bind the Ada card (`gsp ctor failed: -2`, no GSP firmware), so the
|
|
# module holds nothing and unloads cleanly. The unload step refuses if nouveau
|
|
# has actually bound a device; then the blacklist plus a reboot is the only way.
|
|
#
|
|
# ⚠ SECURE BOOT: a DKMS-built module only loads under Secure Boot if its signing
|
|
# key is enrolled as a MOK, and enrolling needs a person at the console at boot
|
|
# (MokManager). esh-pve has Secure Boot OFF. nh3-pve has it ON (lockdown =
|
|
# integrity), and the 2026-09-25 install failed there with "module failed to load".
|
|
# The installer rolled itself back. The first step below now refuses early
|
|
# instead of getting that far.
|
|
#
|
|
# Upgrading the driver: bump driver_version + driver_sha256 here AND
|
|
# driver_version in playbooks/gpu-lxc.yaml, run this on every GPU host, then
|
|
# gpu-lxc.yaml for every GPU LXC. Until both have run, CUDA in the container
|
|
# fails with "driver/library version mismatch".
|
|
#
|
|
# Run: scripts/elway root@esh-pve --playbook playbooks/pve-nvidia-host.yaml
|
|
# scripts/elway root@nh3-pve.nh3.internal --playbook playbooks/pve-nvidia-host.yaml
|
|
|
|
vars:
|
|
driver_version: 580.178.04
|
|
# From download.nvidia.com/XFree86/Linux-x86_64/580.178.04/*.run.sha256sum
|
|
driver_sha256: 1833c8c5c53481dfab46df442c3de26ec5d4c35084e03d94ad93746d29899750
|
|
stage_dir: /root/nvidia
|
|
# What `nvidia-smi --query-gpu=name` must contain on this host.
|
|
gpu_match: RTX 2000
|
|
|
|
steps:
|
|
- name: Pre-flight — Secure Boot off, or the DKMS signing key already enrolled
|
|
shell: |
|
|
if mokutil --sb-state 2>/dev/null | grep -q 'SecureBoot enabled'; then
|
|
if [ -f /var/lib/dkms/mok.pub ] && mokutil --test-key /var/lib/dkms/mok.pub 2>&1 | grep -q 'already enrolled'; then
|
|
exit 0
|
|
fi
|
|
echo "Secure Boot is ON and no enrolled DKMS key: the nvidia module would be refused." >&2
|
|
echo "Needs the console at boot: disable Secure Boot in the BIOS, or enroll /var/lib/dkms/mok.pub via MokManager." >&2
|
|
exit 1
|
|
fi
|
|
changed_when: "false"
|
|
|
|
# DKMS needs the headers for the RUNNING kernel (built now) and the series
|
|
# meta-package (so each future kernel's headers arrive with it and DKMS
|
|
# rebuilds on upgrade). nh3-pve had neither, nor dkms or a compiler.
|
|
- name: Kernel headers (running kernel + series meta), dkms, build tools
|
|
shell: |
|
|
set -e
|
|
export DEBIAN_FRONTEND=noninteractive
|
|
k=$(uname -r); series=$(echo "$k" | cut -d. -f1,2)
|
|
apt-get update -qq
|
|
apt-get install -y -qq "proxmox-headers-$k" "proxmox-headers-$series" dkms build-essential
|
|
when: "! dpkg -s proxmox-headers-$(uname -r) proxmox-headers-$(uname -r | cut -d. -f1,2) dkms build-essential >/dev/null 2>&1"
|
|
|
|
- name: Stage the driver .run and check it against NVIDIA's published sha256
|
|
shell: |
|
|
set -e
|
|
mkdir -p {{ stage_dir }}
|
|
f={{ stage_dir }}/NVIDIA-Linux-x86_64-{{ driver_version }}-no-compat32.run
|
|
if ! echo "{{ driver_sha256 }} $f" | sha256sum -c --status 2>/dev/null; then
|
|
curl -fsSL -o "$f.part" \
|
|
https://download.nvidia.com/XFree86/Linux-x86_64/{{ driver_version }}/NVIDIA-Linux-x86_64-{{ driver_version }}-no-compat32.run
|
|
echo "{{ driver_sha256 }} $f.part" | sha256sum -c
|
|
mv "$f.part" "$f"
|
|
chmod 0755 "$f"
|
|
fi
|
|
when: "! echo '{{ driver_sha256 }} {{ stage_dir }}/NVIDIA-Linux-x86_64-{{ driver_version }}-no-compat32.run' | sha256sum -c --status 2>/dev/null"
|
|
|
|
# esh-pve only: the T400 was passed to VM 102 through vfio-pci by id. The card
|
|
# and the VM are both gone; the ids match nothing. Kept (not deleted) so the
|
|
# retirement is readable later.
|
|
- name: Retire the T400's vfio-pci id binding
|
|
shell: mv /etc/modprobe.d/vfio.conf {{ stage_dir }}/vfio.conf.retired-2026-09-24
|
|
when: "grep -q '10de:1ff2' /etc/modprobe.d/vfio.conf 2>/dev/null"
|
|
|
|
# esh-pve only: blacklist.conf blacklisted `nvidia` alongside nouveau
|
|
# (VM-passthrough era, when the host must never touch the card). nouveau
|
|
# stays blacklisted.
|
|
- name: Stop blacklisting the nvidia module (nouveau stays blacklisted)
|
|
shell: |
|
|
cp -n /etc/modprobe.d/blacklist.conf {{ stage_dir }}/blacklist.conf.pre-nvidia
|
|
sed -i '/^blacklist nvidia$/d' /etc/modprobe.d/blacklist.conf
|
|
when: "grep -qx 'blacklist nvidia' /etc/modprobe.d/blacklist.conf 2>/dev/null"
|
|
|
|
# nouveau is not in PVE's initramfs (checked on nh3-pve: 0 files), so the
|
|
# rootfs modprobe.d is where udev's autoload reads the blacklist. No
|
|
# initramfs rebuild or ESP sync needed.
|
|
- name: Blacklist nouveau
|
|
shell: |
|
|
printf '# nvidia owns the GPU (the LXC uses it) — see playbooks/pve-nvidia-host.yaml\nblacklist nouveau\noptions nouveau modeset=0\n' \
|
|
> /etc/modprobe.d/blacklist-nouveau.conf
|
|
when: "! grep -rqx 'blacklist nouveau' /etc/modprobe.d/"
|
|
|
|
- name: Unload nouveau (only when it holds no device)
|
|
shell: |
|
|
set -e
|
|
if ls /sys/bus/pci/drivers/nouveau/ 2>/dev/null | grep -q '^0000:'; then
|
|
echo "nouveau is BOUND to a device: it is blacklisted now, but only a reboot releases it" >&2
|
|
exit 1
|
|
fi
|
|
modprobe -r nouveau
|
|
when: "grep -q '^nouveau ' /proc/modules"
|
|
|
|
- name: Install the driver with DKMS (open kernel modules)
|
|
# --no-opengl-files: headless compute. --kernel-module-type=open: Ada
|
|
# (Turing+) is supported by the open modules and fv-ml1 runs them too.
|
|
# DKMS rebuilds the module whenever a new proxmox-headers package lands.
|
|
shell: |
|
|
sh {{ stage_dir }}/NVIDIA-Linux-x86_64-{{ driver_version }}-no-compat32.run \
|
|
--dkms --silent --no-x-check --no-opengl-files --kernel-module-type=open \
|
|
--kernel-name="$(uname -r)"
|
|
when: "! dkms status -m nvidia -v {{ driver_version }} -k $(uname -r) 2>/dev/null | grep -q installed"
|
|
|
|
- name: Load nvidia + nvidia_uvm at every boot
|
|
shell: |
|
|
printf '# GPU for the host GPU LXC — see playbooks/pve-nvidia-host.yaml\nnvidia\nnvidia_uvm\n' \
|
|
> /etc/modules-load.d/nvidia.conf
|
|
when: "! grep -qx 'nvidia_uvm' /etc/modules-load.d/nvidia.conf 2>/dev/null"
|
|
|
|
# The LXC binds /dev/nvidia0, /dev/nvidiactl and /dev/nvidia-uvm{,-tools}.
|
|
# Those nodes do not exist until something asks for them, and `pct start`
|
|
# refuses a dev passthrough whose host path is missing. This unit creates
|
|
# them and keeps the GPU initialised (persistence mode), and runs BEFORE
|
|
# pve-guests so onboot guests find them. If it fails, only the GPU LXC fails
|
|
# to start; the other guests do not depend on it.
|
|
- name: Install the GPU-init + persistence unit, ordered before pve-guests
|
|
shell: |
|
|
cat > /etc/systemd/system/nvidia-persistenced.service <<'EOF'
|
|
# See playbooks/pve-nvidia-host.yaml
|
|
[Unit]
|
|
Description=NVIDIA persistence daemon + device nodes for the GPU LXC
|
|
Before=pve-guests.service
|
|
After=systemd-modules-load.service
|
|
|
|
[Service]
|
|
Type=forking
|
|
# -c0 -u: load nvidia_uvm and create /dev/nvidia-uvm{,-tools} and /dev/nvidia0.
|
|
ExecStartPre=/usr/bin/nvidia-modprobe -c0 -u
|
|
ExecStart=/usr/bin/nvidia-persistenced --user root
|
|
ExecStopPost=/bin/rm -rf /var/run/nvidia-persistenced
|
|
|
|
[Install]
|
|
WantedBy=multi-user.target
|
|
EOF
|
|
systemctl daemon-reload
|
|
systemctl enable --now nvidia-persistenced.service
|
|
when: "! systemctl is-active --quiet nvidia-persistenced.service || ! grep -q 'Before=pve-guests.service' /etc/systemd/system/nvidia-persistenced.service"
|
|
|
|
verify:
|
|
- name: nvidia-smi sees the {{ gpu_match }} at the pinned driver version
|
|
shell: nvidia-smi --query-gpu=name,driver_version --format=csv,noheader | grep -q '{{ gpu_match }}.*, {{ driver_version }}$'
|
|
changed_when: "false"
|
|
|
|
- name: Kernel module in use is the DKMS-built {{ driver_version }}
|
|
shell: grep -q '{{ driver_version }}' /proc/driver/nvidia/version && dkms status -m nvidia -v {{ driver_version }} -k $(uname -r) | grep -q installed
|
|
changed_when: "false"
|
|
|
|
- name: Device nodes the LXC binds all exist
|
|
shell: test -c /dev/nvidia0 && test -c /dev/nvidiactl && test -c /dev/nvidia-uvm && test -c /dev/nvidia-uvm-tools
|
|
changed_when: "false"
|
|
|
|
- name: Persistence mode is on
|
|
shell: nvidia-smi --query-gpu=persistence_mode --format=csv,noheader | grep -qx Enabled
|
|
changed_when: "false"
|
|
|
|
- name: Survives a reboot (modules-load + unit enabled + nouveau blacklisted)
|
|
shell: grep -qx nvidia_uvm /etc/modules-load.d/nvidia.conf && systemctl is-enabled --quiet nvidia-persistenced.service && ! grep -qx 'blacklist nvidia' /etc/modprobe.d/blacklist.conf 2>/dev/null && grep -rqx 'blacklist nouveau' /etc/modprobe.d/
|
|
changed_when: "false"
|
|
|
|
- name: DKMS will rebuild for the next kernel (series headers meta installed)
|
|
shell: dpkg -s proxmox-headers-$(uname -r | cut -d. -f1,2) >/dev/null 2>&1
|
|
changed_when: "false"
|