esh-pfi-infrastructure/playbooks/gpu-lxc.yaml

# A GPU LXC on a Proxmox host whose NVIDIA driver lives on the HYPERVISOR.
# Host-generic since 2026-09-25 (was esh-ml1-lxc). Runs against the HYPERVISOR
# and drives the container through `pct`, so it works before the container has
# any users or keys.
#
#   CT               host      IP                role
#   110 esh-ml1      esh-pve   10.0.50.80  v50   fleet embed/rerank (TEI) + reward seat
#   109 nh3-ml1      nh3-pve   10.100.50.80 v50  embed/rerank (TEI), second RTX 2000
#
# Prerequisite: playbooks/pve-nvidia-host.yaml on the same host (kernel module,
# device nodes, persistence unit ordered before pve-guests).
#
# HOST-SPECIFIC VARS HAVE NO DEFAULTS, on purpose: elway refuses to run with an
# undefined {{ var }}, so forgetting one aborts before anything is touched
# instead of silently building esh-ml1's twin. Pass them all with --var (the
# exact invocations are at the bottom of this header and in each server README).
#
# WHAT IT BUILDS
#   - CT {{ ctid }}, unprivileged, Debian 12 (PVE < 8.4.20 rejects the Debian 13
#     template, and both hosts are bookworm), nesting+keyctl for Docker, static
#     IP outside the site's DHCP pool, onboot AFTER the site's core guests
#     (startup order 30) so a GPU fault can never delay DNS or the mesh route.
#   - /dev/nvidia0, nvidiactl, nvidia-uvm, nvidia-uvm-tools bound in via
#     `devN:` (PVE handles the cgroup rules and node creation).
#   - NVIDIA userspace from THE SAME .run the host installed, with
#     --no-kernel-modules. Host module and container libraries must be the same
#     version or every CUDA call fails with "driver/library version mismatch".
#   - Fleet identities per docs/pfi/fleet-conventions.md: docker gid 851
#     (created BEFORE docker-ce so the package does not pick one), infra-ops
#     850 NOPASSWD, vh 1000, /opt/docker root:docker 2775.
#   - docker-ce + nvidia-container-toolkit with no-cgroups=true (an unprivileged
#     LXC cannot write the device cgroup; the LXC's own devN rules already
#     confine the GPU).
#
# NOT BACKED UP, on purpose: everything here is rebuilt by this playbook plus
# the stack, and the models re-download from Hugging Face. esh-pve's vzdump job
# lists vmids explicitly, so it never included 110. nh3-pve's job is `all 1`,
# so the playbook adds the CT to that job's `exclude` list.
#
# Run (esh-ml1):
#   scripts/elway root@esh-pve --playbook playbooks/gpu-lxc.yaml \
#     --var ctid=110 --var hostname=esh-ml1 --var ip_cidr=10.0.50.80/24 \
#     --var ip_addr=10.0.50.80 --var gateway=10.0.50.1 --var vlan=50 \
#     --var rootfs_storage=local-lvm \
#     --var "role=the fleet embedding + reranking service (TEI)"
# Run (nh3-ml1):
#   scripts/elway root@nh3-pve.nh3.internal --playbook playbooks/gpu-lxc.yaml \
#     --var ctid=109 --var hostname=nh3-ml1 --var ip_cidr=10.100.50.80/24 \
#     --var ip_addr=10.100.50.80 --var gateway=10.100.50.1 --var vlan=50 \
#     --var rootfs_storage=local-zfs \
#     --var "role=second embed/rerank backend (TEI)"

vars:
  # Host-specific, NO defaults (see header): ctid, hostname, ip_cidr, ip_addr,
  # gateway, vlan, rootfs_storage, role.
  cores: 6
  memory_mb: 16384
  swap_mb: 2048
  rootfs_gb: 80
  startup_order: 30
  template: debian-12-standard_12.12-1_amd64.tar.zst
  # Must equal playbooks/pve-nvidia-host.yaml's driver_version.
  driver_version: 580.178.04
  stage_dir: /root/nvidia
  infra_ops_pubkey: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN+1HBwfXrkfTYWdcnWCjLJ6VLAGC87gxH5h5vKaaA3c infra-ops@pfi-fleet"

steps:
  - name: Fetch the Debian 12 template
    shell: pveam update >/dev/null && pveam download local {{ template }}
    creates: /var/lib/vz/template/cache/{{ template }}

  - name: Create CT {{ ctid }} ({{ hostname }}) with the GPU device nodes bound in
    shell: |
      pct create {{ ctid }} local:vztmpl/{{ template }} \
        --hostname {{ hostname }} --unprivileged 1 --features nesting=1,keyctl=1 \
        --cores {{ cores }} --memory {{ memory_mb }} --swap {{ swap_mb }} \
        --rootfs {{ rootfs_storage }}:{{ rootfs_gb }} \
        --net0 name=eth0,bridge=vmbr0,firewall=1,gw={{ gateway }},ip={{ ip_cidr }},tag={{ vlan }},type=veth \
        --onboot 1 --startup order={{ startup_order }},up=10 \
        --dev0 /dev/nvidia0 --dev1 /dev/nvidiactl \
        --dev2 /dev/nvidia-uvm --dev3 /dev/nvidia-uvm-tools \
        --description "{{ hostname }} — RTX 2000E Ada: {{ role }}. Built by eshpfi playbooks/gpu-lxc.yaml; stack stacks/embed-rerank. NOT backed up on purpose (rebuildable)."
    when: "! pct status {{ ctid }} >/dev/null 2>&1"

  # A job with `all 1` would pick the new CT up tonight. Add it to that job's
  # exclude list; jobs that name their vmids (esh-pve's) are left alone.
  - name: Keep CT {{ ctid }} out of every all-guests vzdump job
    shell: |
      set -e
      CT={{ ctid }} JOBS="$(pvesh get /cluster/backup --output-format json)" python3 - <<'PY'
      import json, os, subprocess
      ct = os.environ["CT"]
      for j in json.loads(os.environ["JOBS"]):
          if str(j.get("all", 0)) != "1":
              continue
          ex = [x for x in str(j.get("exclude", "")).split(",") if x]
          if ct in ex:
              continue
          subprocess.run(["pvesh", "set", "/cluster/backup/" + j["id"], "--exclude", ",".join(ex + [ct])], check=True)
          print("excluded", ct, "from", j["id"])
      PY
    when: "awk -v ct={{ ctid }} 'function fl(){ if(isv&&all&&!hit) need=1 } /^[^ \\t]/{fl(); isv=($1==\"vzdump:\"); all=0; hit=0; next} isv&&$1==\"all\"&&$2==\"1\"{all=1} isv&&$1==\"exclude\"{n=split($2,a,\",\"); for(i=1;i<=n;i++) if(a[i]==ct) hit=1} END{fl(); exit !need}' /etc/pve/jobs.cfg"

  - name: Start the container
    shell: pct start {{ ctid }} && sleep 5
    when: "! pct status {{ ctid }} | grep -q running"

  - name: Base packages + bookworm point upgrade
    shell: |
      pct exec {{ ctid }} -- bash -s <<'EOF'
      set -euo pipefail
      export DEBIAN_FRONTEND=noninteractive
      for i in $(seq 1 30); do getent hosts deb.debian.org >/dev/null && break; sleep 1; done
      apt-get update -qq
      apt-get -y -qq full-upgrade
      apt-get install -y -qq --no-install-recommends ca-certificates curl gnupg sudo kmod jq less rsync locales
      EOF
    # rsync: scripts/deploy-stack.sh pushes stacks with it.
    when: "! pct exec {{ ctid }} -- sh -c 'command -v sudo && command -v jq && command -v rsync && command -v locale-gen' >/dev/null 2>&1"

  - name: Generate the en_US.UTF-8 locale the template's LANG names
    # The Debian template sets LANG=en_US.UTF-8 but ships no locale, so every
    # apt and perl run warns about it.
    shell: |
      pct exec {{ ctid }} -- bash -s <<'EOF'
      set -euo pipefail
      sed -i 's/^# *en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen
      locale-gen >/dev/null
      EOF
    when: "! pct exec {{ ctid }} -- locale -a 2>/dev/null | grep -qi '^en_US.utf8$'"

  - name: Fleet identities (docker 851, infra-ops 850, vh 1000) + /opt/docker tree
    shell: |
      pct exec {{ ctid }} -- bash -s <<'EOF'
      set -euo pipefail
      getent group docker >/dev/null || groupadd -g 851 docker
      getent group infra-ops >/dev/null || groupadd -g 850 infra-ops
      id infra-ops >/dev/null 2>&1 || useradd -u 850 -g 850 -G docker -m -s /bin/bash infra-ops
      chmod 0700 /home/infra-ops
      install -d -m 0700 -o infra-ops -g infra-ops /home/infra-ops/.ssh
      echo '{{ infra_ops_pubkey }}' > /home/infra-ops/.ssh/authorized_keys
      chown infra-ops:infra-ops /home/infra-ops/.ssh/authorized_keys
      chmod 0600 /home/infra-ops/.ssh/authorized_keys
      echo 'infra-ops ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/infra-ops
      chmod 0440 /etc/sudoers.d/infra-ops
      id vh >/dev/null 2>&1 || useradd -u 1000 -U -G docker,sudo -m -s /bin/bash vh
      chmod 0700 /home/vh
      install -d -m 2775 -o root -g docker /opt/docker /opt/docker/compose /opt/docker/conf
      EOF
    when: "! pct exec {{ ctid }} -- sh -c 'test \"$(id -u infra-ops)\" = 850 && test \"$(getent group docker | cut -d: -f3)\" = 851 && test -d /opt/docker/compose'"

  - name: Hugging Face cache for the embed-rerank stack
    shell: pct exec {{ ctid }} -- install -d -m 2775 -o root -g docker /opt/aimodels /opt/aimodels/huggingface
    when: "! pct exec {{ ctid }} -- test -d /opt/aimodels/huggingface"

  - name: NVIDIA userspace {{ driver_version }} from the host's .run (no kernel modules)
    shell: |
      set -e
      f=NVIDIA-Linux-x86_64-{{ driver_version }}-no-compat32.run
      pct push {{ ctid }} {{ stage_dir }}/$f /root/$f --perms 0755
      pct exec {{ ctid }} -- sh /root/$f --no-kernel-modules --silent --no-x-check --no-opengl-files
      pct exec {{ ctid }} -- rm -f /root/$f
    when: "! pct exec {{ ctid }} -- sh -c 'nvidia-smi --query-gpu=driver_version --format=csv,noheader 2>/dev/null' | grep -qx '{{ driver_version }}'"

  - name: docker-ce from Docker's apt repo
    shell: |
      pct exec {{ ctid }} -- bash -s <<'EOF'
      set -euo pipefail
      export DEBIAN_FRONTEND=noninteractive
      install -m 0755 -d /etc/apt/keyrings
      curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc
      chmod a+r /etc/apt/keyrings/docker.asc
      echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian bookworm stable" \
        > /etc/apt/sources.list.d/docker.list
      apt-get update -qq
      apt-get install -y -qq docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
      EOF
    when: "! pct exec {{ ctid }} -- sh -c 'command -v docker' >/dev/null 2>&1"

  - name: nvidia-container-toolkit, wired into dockerd, no-cgroups for the LXC
    shell: |
      pct exec {{ ctid }} -- bash -s <<'EOF'
      set -euo pipefail
      export DEBIAN_FRONTEND=noninteractive
      curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey \
        | gpg --dearmor --yes -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg
      curl -fsSL https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list \
        | sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' \
        > /etc/apt/sources.list.d/nvidia-container-toolkit.list
      apt-get update -qq
      apt-get install -y -qq nvidia-container-toolkit
      nvidia-ctk runtime configure --runtime=docker
      nvidia-ctk config --set nvidia-container-cli.no-cgroups=true --in-place
      systemctl restart docker
      EOF
    when: "! pct exec {{ ctid }} -- sh -c 'grep -q nvidia /etc/docker/daemon.json && grep -Eq \"^no-cgroups *= *true\" /etc/nvidia-container-runtime/config.toml' 2>/dev/null"

  # Fleet norm: Homepage (on esh-docker-vm) discovers labelled containers by
  # reading every host's Docker API on tcp/2375 (stacks/homepage/conf/docker.yaml).
  # Same unauthenticated plaintext exposure as fv-ml1 and esh-docker-vm, bound to
  # this CT's one address. ⚠ Restarting dockerd restarts every container here,
  # the fleet embed/rerank service included (~5 s for TEI, ~60 s for the reward seat).
  - name: Expose the Docker API on tcp/2375 for Homepage discovery
    shell: |
      pct exec {{ ctid }} -- bash -s <<'EOF'
      set -euo pipefail
      install -d /etc/systemd/system/docker.service.d
      cat > /etc/systemd/system/docker.service.d/override.conf <<'EOC'
      # Homepage discovery — see eshpfi playbooks/gpu-lxc.yaml
      [Service]
      ExecStart=
      ExecStart=/usr/bin/dockerd -H fd:// -H tcp://{{ ip_addr }}:2375 --containerd=/run/containerd/containerd.sock
      EOC
      systemctl daemon-reload
      systemctl restart docker
      EOF
    when: "! pct exec {{ ctid }} -- grep -q 'tcp://' /etc/systemd/system/docker.service.d/override.conf 2>/dev/null"

verify:
  - name: Container is running with onboot set, started after the core guests
    shell: "pct status {{ ctid }} | grep -q running && pct config {{ ctid }} | grep -q '^onboot: 1' && pct config {{ ctid }} | grep -q '^startup: order={{ startup_order }}'"
    changed_when: "false"

  - name: nvidia-smi inside the CT reports the host's driver version
    shell: pct exec {{ ctid }} -- nvidia-smi --query-gpu=name,driver_version --format=csv,noheader | grep -q 'RTX 2000.*, {{ driver_version }}$'
    changed_when: "false"

  - name: A Docker container gets the GPU (toolkit + no-cgroups work end to end)
    shell: pct exec {{ ctid }} -- docker run --rm --gpus all debian:12-slim nvidia-smi -L | grep -q 'RTX 2000'
    changed_when: "false"

  - name: Fleet identities are the pinned ids
    shell: |
      pct exec {{ ctid }} -- sh -c 'test "$(id -u infra-ops)" = 850 && test "$(id -u vh)" = 1000 && test "$(getent group docker | cut -d: -f3)" = 851 && id -nG infra-ops | grep -qw docker'
    changed_when: "false"

  - name: CT {{ ctid }} is in no all-guests vzdump job
    shell: "! awk -v ct={{ ctid }} 'function fl(){ if(isv&&all&&!hit) need=1 } /^[^ \\t]/{fl(); isv=($1==\"vzdump:\"); all=0; hit=0; next} isv&&$1==\"all\"&&$2==\"1\"{all=1} isv&&$1==\"exclude\"{n=split($2,a,\",\"); for(i=1;i<=n;i++) if(a[i]==ct) hit=1} END{fl(); exit !need}' /etc/pve/jobs.cfg"
    changed_when: "false"