# A GPU LXC on a Proxmox host whose NVIDIA driver lives on the HYPERVISOR. # Host-generic since 2026-09-25 (was esh-ml1-lxc). Runs against the HYPERVISOR # and drives the container through `pct`, so it works before the container has # any users or keys. # # CT host IP role # 110 esh-ml1 esh-pve 10.0.50.80 v50 fleet embed/rerank (TEI) + reward seat # 109 nh3-ml1 nh3-pve 10.100.50.80 v50 embed/rerank (TEI), second RTX 2000 # # Prerequisite: playbooks/pve-nvidia-host.yaml on the same host (kernel module, # device nodes, persistence unit ordered before pve-guests). # # HOST-SPECIFIC VARS HAVE NO DEFAULTS, on purpose: elway refuses to run with an # undefined {{ var }}, so forgetting one aborts before anything is touched # instead of silently building esh-ml1's twin. Pass them all with --var (the # exact invocations are at the bottom of this header and in each server README). # # WHAT IT BUILDS # - CT {{ ctid }}, unprivileged, Debian 12 (PVE < 8.4.20 rejects the Debian 13 # template, and both hosts are bookworm), nesting+keyctl for Docker, static # IP outside the site's DHCP pool, onboot AFTER the site's core guests # (startup order 30) so a GPU fault can never delay DNS or the mesh route. # - /dev/nvidia0, nvidiactl, nvidia-uvm, nvidia-uvm-tools bound in via # `devN:` (PVE handles the cgroup rules and node creation). # - NVIDIA userspace from THE SAME .run the host installed, with # --no-kernel-modules. Host module and container libraries must be the same # version or every CUDA call fails with "driver/library version mismatch". # - Fleet identities per docs/pfi/fleet-conventions.md: docker gid 851 # (created BEFORE docker-ce so the package does not pick one), infra-ops # 850 NOPASSWD, vh 1000, /opt/docker root:docker 2775. # - docker-ce + nvidia-container-toolkit with no-cgroups=true (an unprivileged # LXC cannot write the device cgroup; the LXC's own devN rules already # confine the GPU). # # NOT BACKED UP, on purpose: everything here is rebuilt by this playbook plus # the stack, and the models re-download from Hugging Face. esh-pve's vzdump job # lists vmids explicitly, so it never included 110. nh3-pve's job is `all 1`, # so the playbook adds the CT to that job's `exclude` list. # # Run (esh-ml1): # scripts/elway root@esh-pve --playbook playbooks/gpu-lxc.yaml \ # --var ctid=110 --var hostname=esh-ml1 --var ip_cidr=10.0.50.80/24 \ # --var ip_addr=10.0.50.80 --var gateway=10.0.50.1 --var vlan=50 \ # --var rootfs_storage=local-lvm \ # --var "role=the fleet embedding + reranking service (TEI)" # Run (nh3-ml1): # scripts/elway root@nh3-pve.nh3.internal --playbook playbooks/gpu-lxc.yaml \ # --var ctid=109 --var hostname=nh3-ml1 --var ip_cidr=10.100.50.80/24 \ # --var ip_addr=10.100.50.80 --var gateway=10.100.50.1 --var vlan=50 \ # --var rootfs_storage=local-zfs \ # --var "role=second embed/rerank backend (TEI)" vars: # Host-specific, NO defaults (see header): ctid, hostname, ip_cidr, ip_addr, # gateway, vlan, rootfs_storage, role. cores: 6 memory_mb: 16384 swap_mb: 2048 rootfs_gb: 80 startup_order: 30 template: debian-12-standard_12.12-1_amd64.tar.zst # Must equal playbooks/pve-nvidia-host.yaml's driver_version. driver_version: 580.178.04 stage_dir: /root/nvidia infra_ops_pubkey: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN+1HBwfXrkfTYWdcnWCjLJ6VLAGC87gxH5h5vKaaA3c infra-ops@pfi-fleet" steps: # runc >= 1.2.8 (the CVE-2025-52881 fix; the CT gets 1.5.x from docker-ce) # re-opens /proc/sys files, and lxc-pve 6.0.0-1's AppArmor profile denies it # in a nesting CT: EVERY `docker run` fails with "open sysctl # net.ipv4.ip_unprivileged_port_start file: reopen fd 8: permission denied". # lxc-pve 6.0.0-2 (Proxmox fix #7006) lifts the restriction when nesting is on. # Measured 2026-09-25: nh3-pve (PVE 8.4.1, lxc-pve 6.0.0-1) failed the GPU # verify below; esh-pve (8.4.20, 6.0.0-2) never did. One package, no deps, # running guests unaffected (the change only relaxes the profile). A CT that # was already running needs a `pct reboot` to pick it up. - name: "lxc-pve carries fix #7006 (Docker's runc works in a nesting CT)" shell: | set -e export DEBIAN_FRONTEND=noninteractive apt-get update -qq apt-get install -y -qq --only-upgrade lxc-pve v=$(dpkg-query -W -f='${Version}' lxc-pve) dpkg --compare-versions "$v" ge 6.0.0-2 || { echo "lxc-pve is still $v after upgrade" >&2; exit 1; } when: "dpkg --compare-versions \"$(dpkg-query -W -f='${Version}' lxc-pve)\" lt 6.0.0-2" - name: Fetch the Debian 12 template shell: pveam update >/dev/null && pveam download local {{ template }} creates: /var/lib/vz/template/cache/{{ template }} - name: Create CT {{ ctid }} ({{ hostname }}) with the GPU device nodes bound in shell: | pct create {{ ctid }} local:vztmpl/{{ template }} \ --hostname {{ hostname }} --unprivileged 1 --features nesting=1,keyctl=1 \ --cores {{ cores }} --memory {{ memory_mb }} --swap {{ swap_mb }} \ --rootfs {{ rootfs_storage }}:{{ rootfs_gb }} \ --net0 name=eth0,bridge=vmbr0,firewall=1,gw={{ gateway }},ip={{ ip_cidr }},tag={{ vlan }},type=veth \ --onboot 1 --startup order={{ startup_order }},up=10 \ --dev0 /dev/nvidia0 --dev1 /dev/nvidiactl \ --dev2 /dev/nvidia-uvm --dev3 /dev/nvidia-uvm-tools \ --description "{{ hostname }} — RTX 2000E Ada: {{ role }}. Built by eshpfi playbooks/gpu-lxc.yaml; stack stacks/embed-rerank. NOT backed up on purpose (rebuildable)." when: "! pct status {{ ctid }} >/dev/null 2>&1" # A job with `all 1` would pick the new CT up tonight. Add it to that job's # exclude list; jobs that name their vmids (esh-pve's) are left alone. - name: Keep CT {{ ctid }} out of every all-guests vzdump job shell: | set -e CT={{ ctid }} JOBS="$(pvesh get /cluster/backup --output-format json)" python3 - <<'PY' import json, os, subprocess ct = os.environ["CT"] for j in json.loads(os.environ["JOBS"]): if str(j.get("all", 0)) != "1": continue ex = [x for x in str(j.get("exclude", "")).split(",") if x] if ct in ex: continue subprocess.run(["pvesh", "set", "/cluster/backup/" + j["id"], "--exclude", ",".join(ex + [ct])], check=True) print("excluded", ct, "from", j["id"]) PY when: "awk -v ct={{ ctid }} 'function fl(){ if(isv&&all&&!hit) need=1 } /^[^ \\t]/{fl(); isv=($1==\"vzdump:\"); all=0; hit=0; next} isv&&$1==\"all\"&&$2==\"1\"{all=1} isv&&$1==\"exclude\"{n=split($2,a,\",\"); for(i=1;i<=n;i++) if(a[i]==ct) hit=1} END{fl(); exit !need}' /etc/pve/jobs.cfg" - name: Start the container shell: pct start {{ ctid }} && sleep 5 when: "! pct status {{ ctid }} | grep -q running" - name: Base packages + bookworm point upgrade shell: | pct exec {{ ctid }} -- bash -s <<'EOF' set -euo pipefail export DEBIAN_FRONTEND=noninteractive for i in $(seq 1 30); do getent hosts deb.debian.org >/dev/null && break; sleep 1; done apt-get update -qq apt-get -y -qq full-upgrade apt-get install -y -qq --no-install-recommends ca-certificates curl gnupg sudo kmod jq less rsync locales EOF # rsync: scripts/deploy-stack.sh pushes stacks with it. when: "! pct exec {{ ctid }} -- sh -c 'command -v sudo && command -v jq && command -v rsync && command -v locale-gen' >/dev/null 2>&1" - name: Generate the en_US.UTF-8 locale the template's LANG names # The Debian template sets LANG=en_US.UTF-8 but ships no locale, so every # apt and perl run warns about it. shell: | pct exec {{ ctid }} -- bash -s <<'EOF' set -euo pipefail sed -i 's/^# *en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen locale-gen >/dev/null EOF when: "! pct exec {{ ctid }} -- locale -a 2>/dev/null | grep -qi '^en_US.utf8$'" - name: Fleet identities (docker 851, infra-ops 850, vh 1000) + /opt/docker tree shell: | pct exec {{ ctid }} -- bash -s <<'EOF' set -euo pipefail getent group docker >/dev/null || groupadd -g 851 docker getent group infra-ops >/dev/null || groupadd -g 850 infra-ops id infra-ops >/dev/null 2>&1 || useradd -u 850 -g 850 -G docker -m -s /bin/bash infra-ops chmod 0700 /home/infra-ops install -d -m 0700 -o infra-ops -g infra-ops /home/infra-ops/.ssh echo '{{ infra_ops_pubkey }}' > /home/infra-ops/.ssh/authorized_keys chown infra-ops:infra-ops /home/infra-ops/.ssh/authorized_keys chmod 0600 /home/infra-ops/.ssh/authorized_keys echo 'infra-ops ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/infra-ops chmod 0440 /etc/sudoers.d/infra-ops id vh >/dev/null 2>&1 || useradd -u 1000 -U -G docker,sudo -m -s /bin/bash vh chmod 0700 /home/vh install -d -m 2775 -o root -g docker /opt/docker /opt/docker/compose /opt/docker/conf EOF when: "! pct exec {{ ctid }} -- sh -c 'test \"$(id -u infra-ops)\" = 850 && test \"$(getent group docker | cut -d: -f3)\" = 851 && test -d /opt/docker/compose'" - name: Hugging Face cache for the embed-rerank stack shell: pct exec {{ ctid }} -- install -d -m 2775 -o root -g docker /opt/aimodels /opt/aimodels/huggingface when: "! pct exec {{ ctid }} -- test -d /opt/aimodels/huggingface" - name: NVIDIA userspace {{ driver_version }} from the host's .run (no kernel modules) shell: | set -e f=NVIDIA-Linux-x86_64-{{ driver_version }}-no-compat32.run pct push {{ ctid }} {{ stage_dir }}/$f /root/$f --perms 0755 pct exec {{ ctid }} -- sh /root/$f --no-kernel-modules --silent --no-x-check --no-opengl-files pct exec {{ ctid }} -- rm -f /root/$f when: "! pct exec {{ ctid }} -- sh -c 'nvidia-smi --query-gpu=driver_version --format=csv,noheader 2>/dev/null' | grep -qx '{{ driver_version }}'" - name: docker-ce from Docker's apt repo shell: | pct exec {{ ctid }} -- bash -s <<'EOF' set -euo pipefail export DEBIAN_FRONTEND=noninteractive install -m 0755 -d /etc/apt/keyrings curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc chmod a+r /etc/apt/keyrings/docker.asc echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian bookworm stable" \ > /etc/apt/sources.list.d/docker.list apt-get update -qq apt-get install -y -qq docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin EOF when: "! pct exec {{ ctid }} -- sh -c 'command -v docker' >/dev/null 2>&1" - name: nvidia-container-toolkit, wired into dockerd, no-cgroups for the LXC shell: | pct exec {{ ctid }} -- bash -s <<'EOF' set -euo pipefail export DEBIAN_FRONTEND=noninteractive curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey \ | gpg --dearmor --yes -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg curl -fsSL https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list \ | sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' \ > /etc/apt/sources.list.d/nvidia-container-toolkit.list apt-get update -qq apt-get install -y -qq nvidia-container-toolkit nvidia-ctk runtime configure --runtime=docker nvidia-ctk config --set nvidia-container-cli.no-cgroups=true --in-place systemctl restart docker EOF when: "! pct exec {{ ctid }} -- sh -c 'grep -q nvidia /etc/docker/daemon.json && grep -Eq \"^no-cgroups *= *true\" /etc/nvidia-container-runtime/config.toml' 2>/dev/null" # Fleet norm: Homepage (on esh-docker-vm) discovers labelled containers by # reading every host's Docker API on tcp/2375 (stacks/homepage/conf/docker.yaml). # Same unauthenticated plaintext exposure as fv-ml1 and esh-docker-vm, bound to # this CT's one address. ⚠ Restarting dockerd restarts every container here, # the fleet embed/rerank service included (~5 s for TEI, ~60 s for the reward seat). - name: Expose the Docker API on tcp/2375 for Homepage discovery shell: | pct exec {{ ctid }} -- bash -s <<'EOF' set -euo pipefail install -d /etc/systemd/system/docker.service.d cat > /etc/systemd/system/docker.service.d/override.conf <<'EOC' # Homepage discovery — see eshpfi playbooks/gpu-lxc.yaml [Service] ExecStart= ExecStart=/usr/bin/dockerd -H fd:// -H tcp://{{ ip_addr }}:2375 --containerd=/run/containerd/containerd.sock EOC systemctl daemon-reload systemctl restart docker EOF when: "! pct exec {{ ctid }} -- grep -q 'tcp://' /etc/systemd/system/docker.service.d/override.conf 2>/dev/null" verify: - name: Container is running with onboot set, started after the core guests shell: "pct status {{ ctid }} | grep -q running && pct config {{ ctid }} | grep -q '^onboot: 1' && pct config {{ ctid }} | grep -q '^startup: order={{ startup_order }}'" changed_when: "false" - name: nvidia-smi inside the CT reports the host's driver version shell: pct exec {{ ctid }} -- nvidia-smi --query-gpu=name,driver_version --format=csv,noheader | grep -q 'RTX 2000.*, {{ driver_version }}$' changed_when: "false" - name: A Docker container gets the GPU (toolkit + no-cgroups work end to end) shell: pct exec {{ ctid }} -- docker run --rm --gpus all debian:12-slim nvidia-smi -L | grep -q 'RTX 2000' changed_when: "false" - name: Fleet identities are the pinned ids shell: | pct exec {{ ctid }} -- sh -c 'test "$(id -u infra-ops)" = 850 && test "$(id -u vh)" = 1000 && test "$(getent group docker | cut -d: -f3)" = 851 && id -nG infra-ops | grep -qw docker' changed_when: "false" - name: CT {{ ctid }} is in no all-guests vzdump job shell: "! awk -v ct={{ ctid }} 'function fl(){ if(isv&&all&&!hit) need=1 } /^[^ \\t]/{fl(); isv=($1==\"vzdump:\"); all=0; hit=0; next} isv&&$1==\"all\"&&$2==\"1\"{all=1} isv&&$1==\"exclude\"{n=split($2,a,\",\"); for(i=1;i<=n;i++) if(a[i]==ct) hit=1} END{fl(); exit !need}' /etc/pve/jobs.cfg" changed_when: "false"