esh-pfi-infrastructure/playbooks/esh-ml1-lxc.yaml

# esh-ml1 — the GPU LXC on esh-pve that serves embedding + reranking on the
# RTX 2000E Ada (16 GB). Runs against the HYPERVISOR and drives the container
# through `pct`, so it works before the container has any users or keys.
#
# Prerequisite: playbooks/esh-pve-nvidia-host.yaml (host kernel module, device
# nodes, persistence unit ordered before pve-guests).
#
# WHAT IT BUILDS
#   - CT {{ ctid }} `esh-ml1`, unprivileged, Debian 12 (the template the host's
#     own bookworm matches), nesting+keyctl for Docker, static 10.0.50.80 on
#     VLAN 50 (outside the UDM's .150–.250 DHCP pool), onboot AFTER esh-scale
#     and the two VMs so a GPU fault can never delay DNS or the mesh route.
#   - /dev/nvidia0, nvidiactl, nvidia-uvm, nvidia-uvm-tools bound in via
#     `devN:` (PVE handles the cgroup rules and node creation).
#   - NVIDIA userspace from THE SAME .run the host installed, with
#     --no-kernel-modules. Host module and container libraries must be the same
#     version or every CUDA call fails with "driver/library version mismatch".
#   - Fleet identities per docs/pfi/fleet-conventions.md: docker gid 851
#     (created BEFORE docker-ce so the package does not pick one), infra-ops
#     850 NOPASSWD, vh 1000, /opt/docker root:docker 2775.
#   - docker-ce + nvidia-container-toolkit with no-cgroups=true (an unprivileged
#     LXC cannot write the device cgroup; the LXC's own devN rules already
#     confine the GPU).
#
# NOT BACKED UP, on purpose: esh-pve's vzdump job lists vmids explicitly and
# this CT is not in it. Everything here is rebuilt by this playbook plus
# stacks/embed-rerank; the models re-download from Hugging Face.
#
# Run:  scripts/elway root@esh-pve --playbook playbooks/esh-ml1-lxc.yaml

vars:
  ctid: 110
  hostname: esh-ml1
  ip_cidr: 10.0.50.80/24
  ip_addr: 10.0.50.80
  gateway: 10.0.50.1
  vlan: 50
  cores: 6
  memory_mb: 16384
  swap_mb: 2048
  rootfs_gb: 80
  template: debian-12-standard_12.12-1_amd64.tar.zst
  # Must equal playbooks/esh-pve-nvidia-host.yaml's driver_version.
  driver_version: 580.178.04
  stage_dir: /root/nvidia
  infra_ops_pubkey: "ssh-ed25519 AAAAC3NzaC1lZDI1NTE5AAAAIN+1HBwfXrkfTYWdcnWCjLJ6VLAGC87gxH5h5vKaaA3c infra-ops@pfi-fleet"

steps:
  - name: Fetch the Debian 12 template
    shell: pveam update >/dev/null && pveam download local {{ template }}
    creates: /var/lib/vz/template/cache/{{ template }}

  - name: Create CT {{ ctid }} ({{ hostname }}) with the GPU device nodes bound in
    shell: |
      pct create {{ ctid }} local:vztmpl/{{ template }} \
        --hostname {{ hostname }} --unprivileged 1 --features nesting=1,keyctl=1 \
        --cores {{ cores }} --memory {{ memory_mb }} --swap {{ swap_mb }} \
        --rootfs local-lvm:{{ rootfs_gb }} \
        --net0 name=eth0,bridge=vmbr0,firewall=1,gw={{ gateway }},ip={{ ip_cidr }},tag={{ vlan }},type=veth \
        --onboot 1 --startup order=30,up=10 \
        --dev0 /dev/nvidia0 --dev1 /dev/nvidiactl \
        --dev2 /dev/nvidia-uvm --dev3 /dev/nvidia-uvm-tools \
        --description "esh-ml1 — RTX 2000E Ada: the fleet embedding + reranking service (TEI). Built by eshpfi playbooks/esh-ml1-lxc.yaml; stack stacks/embed-rerank. NOT in the vzdump job on purpose (rebuildable)."
    when: "! pct status {{ ctid }} >/dev/null 2>&1"

  - name: Start the container
    shell: pct start {{ ctid }} && sleep 5
    when: "! pct status {{ ctid }} | grep -q running"

  - name: Base packages + bookworm point upgrade
    shell: |
      pct exec {{ ctid }} -- bash -s <<'EOF'
      set -euo pipefail
      export DEBIAN_FRONTEND=noninteractive
      for i in $(seq 1 30); do getent hosts deb.debian.org >/dev/null && break; sleep 1; done
      apt-get update -qq
      apt-get -y -qq full-upgrade
      apt-get install -y -qq --no-install-recommends ca-certificates curl gnupg sudo kmod jq less rsync locales
      EOF
    # rsync: scripts/deploy-stack.sh pushes stacks with it.
    when: "! pct exec {{ ctid }} -- sh -c 'command -v sudo && command -v jq && command -v rsync && command -v locale-gen' >/dev/null 2>&1"

  - name: Generate the en_US.UTF-8 locale the template's LANG names
    # The Debian template sets LANG=en_US.UTF-8 but ships no locale, so every
    # apt and perl run warns about it.
    shell: |
      pct exec {{ ctid }} -- bash -s <<'EOF'
      set -euo pipefail
      sed -i 's/^# *en_US.UTF-8 UTF-8/en_US.UTF-8 UTF-8/' /etc/locale.gen
      locale-gen >/dev/null
      EOF
    when: "! pct exec {{ ctid }} -- locale -a 2>/dev/null | grep -qi '^en_US.utf8$'"

  - name: Fleet identities (docker 851, infra-ops 850, vh 1000) + /opt/docker tree
    shell: |
      pct exec {{ ctid }} -- bash -s <<'EOF'
      set -euo pipefail
      getent group docker >/dev/null || groupadd -g 851 docker
      getent group infra-ops >/dev/null || groupadd -g 850 infra-ops
      id infra-ops >/dev/null 2>&1 || useradd -u 850 -g 850 -G docker -m -s /bin/bash infra-ops
      chmod 0700 /home/infra-ops
      install -d -m 0700 -o infra-ops -g infra-ops /home/infra-ops/.ssh
      echo '{{ infra_ops_pubkey }}' > /home/infra-ops/.ssh/authorized_keys
      chown infra-ops:infra-ops /home/infra-ops/.ssh/authorized_keys
      chmod 0600 /home/infra-ops/.ssh/authorized_keys
      echo 'infra-ops ALL=(ALL) NOPASSWD:ALL' > /etc/sudoers.d/infra-ops
      chmod 0440 /etc/sudoers.d/infra-ops
      id vh >/dev/null 2>&1 || useradd -u 1000 -U -G docker,sudo -m -s /bin/bash vh
      chmod 0700 /home/vh
      install -d -m 2775 -o root -g docker /opt/docker /opt/docker/compose /opt/docker/conf
      EOF
    when: "! pct exec {{ ctid }} -- sh -c 'test \"$(id -u infra-ops)\" = 850 && test \"$(getent group docker | cut -d: -f3)\" = 851 && test -d /opt/docker/compose'"

  - name: Hugging Face cache for the embed-rerank stack
    shell: pct exec {{ ctid }} -- install -d -m 2775 -o root -g docker /opt/aimodels /opt/aimodels/huggingface
    when: "! pct exec {{ ctid }} -- test -d /opt/aimodels/huggingface"

  - name: NVIDIA userspace {{ driver_version }} from the host's .run (no kernel modules)
    shell: |
      set -e
      f=NVIDIA-Linux-x86_64-{{ driver_version }}-no-compat32.run
      pct push {{ ctid }} {{ stage_dir }}/$f /root/$f --perms 0755
      pct exec {{ ctid }} -- sh /root/$f --no-kernel-modules --silent --no-x-check --no-opengl-files
      pct exec {{ ctid }} -- rm -f /root/$f
    when: "! pct exec {{ ctid }} -- sh -c 'nvidia-smi --query-gpu=driver_version --format=csv,noheader 2>/dev/null' | grep -qx '{{ driver_version }}'"

  - name: docker-ce from Docker's apt repo
    shell: |
      pct exec {{ ctid }} -- bash -s <<'EOF'
      set -euo pipefail
      export DEBIAN_FRONTEND=noninteractive
      install -m 0755 -d /etc/apt/keyrings
      curl -fsSL https://download.docker.com/linux/debian/gpg -o /etc/apt/keyrings/docker.asc
      chmod a+r /etc/apt/keyrings/docker.asc
      echo "deb [arch=amd64 signed-by=/etc/apt/keyrings/docker.asc] https://download.docker.com/linux/debian bookworm stable" \
        > /etc/apt/sources.list.d/docker.list
      apt-get update -qq
      apt-get install -y -qq docker-ce docker-ce-cli containerd.io docker-buildx-plugin docker-compose-plugin
      EOF
    when: "! pct exec {{ ctid }} -- sh -c 'command -v docker' >/dev/null 2>&1"

  - name: nvidia-container-toolkit, wired into dockerd, no-cgroups for the LXC
    shell: |
      pct exec {{ ctid }} -- bash -s <<'EOF'
      set -euo pipefail
      export DEBIAN_FRONTEND=noninteractive
      curl -fsSL https://nvidia.github.io/libnvidia-container/gpgkey \
        | gpg --dearmor --yes -o /usr/share/keyrings/nvidia-container-toolkit-keyring.gpg
      curl -fsSL https://nvidia.github.io/libnvidia-container/stable/deb/nvidia-container-toolkit.list \
        | sed 's#deb https://#deb [signed-by=/usr/share/keyrings/nvidia-container-toolkit-keyring.gpg] https://#g' \
        > /etc/apt/sources.list.d/nvidia-container-toolkit.list
      apt-get update -qq
      apt-get install -y -qq nvidia-container-toolkit
      nvidia-ctk runtime configure --runtime=docker
      nvidia-ctk config --set nvidia-container-cli.no-cgroups=true --in-place
      systemctl restart docker
      EOF
    when: "! pct exec {{ ctid }} -- sh -c 'grep -q nvidia /etc/docker/daemon.json && grep -Eq \"^no-cgroups *= *true\" /etc/nvidia-container-runtime/config.toml' 2>/dev/null"

  # Fleet norm: Homepage (on esh-docker-vm) discovers labelled containers by
  # reading every host's Docker API on tcp/2375 (stacks/homepage/conf/docker.yaml).
  # Same unauthenticated plaintext exposure as fv-ml1 and esh-docker-vm, bound to
  # this CT's one address. ⚠ Restarting dockerd restarts every container here,
  # the fleet embed/rerank service included (~5 s for TEI, ~60 s for the reward seat).
  - name: Expose the Docker API on tcp/2375 for Homepage discovery
    shell: |
      pct exec {{ ctid }} -- bash -s <<'EOF'
      set -euo pipefail
      install -d /etc/systemd/system/docker.service.d
      cat > /etc/systemd/system/docker.service.d/override.conf <<'EOC'
      # Homepage discovery — see eshpfi playbooks/esh-ml1-lxc.yaml
      [Service]
      ExecStart=
      ExecStart=/usr/bin/dockerd -H fd:// -H tcp://{{ ip_addr }}:2375 --containerd=/run/containerd/containerd.sock
      EOC
      systemctl daemon-reload
      systemctl restart docker
      EOF
    when: "! pct exec {{ ctid }} -- grep -q 'tcp://' /etc/systemd/system/docker.service.d/override.conf 2>/dev/null"

verify:
  - name: Container is running with onboot set, started after the core guests
    shell: "pct status {{ ctid }} | grep -q running && pct config {{ ctid }} | grep -q '^onboot: 1' && pct config {{ ctid }} | grep -q '^startup: order=30'"
    changed_when: "false"

  - name: nvidia-smi inside the CT reports the host's driver version
    shell: pct exec {{ ctid }} -- nvidia-smi --query-gpu=name,driver_version --format=csv,noheader | grep -q 'RTX 2000.*, {{ driver_version }}$'
    changed_when: "false"

  - name: A Docker container gets the GPU (toolkit + no-cgroups work end to end)
    shell: pct exec {{ ctid }} -- docker run --rm --gpus all debian:12-slim nvidia-smi -L | grep -q 'RTX 2000'
    changed_when: "false"

  - name: Fleet identities are the pinned ids
    shell: |
      pct exec {{ ctid }} -- sh -c 'test "$(id -u infra-ops)" = 850 && test "$(id -u vh)" = 1000 && test "$(getent group docker | cut -d: -f3)" = 851 && id -nG infra-ops | grep -qw docker'
    changed_when: "false"