# nh3-pve after its PVE 8.4.1 → 9.2.21 upgrade (2026-10-03, Prime's go on all of it). Idempotent.
#   scripts/elway infra-ops@10.100.250.60 --playbook playbooks/nh3-pve-pve9-postfix.yaml
# Post-check findings this closes:
#   - the upgrade installed kernel 7.0.14-20 but no 7.0 headers, so the nvidia 580.178.04 DKMS module was never
#     built for it: nvidia-persistenced failed and CT 109 (nh3-ml1, the TEI twin) could not start;
#   - pve8to9 FAIL: intel-microcode missing (Debian sources had no non-free-firmware);
#   - pve-enterprise.sources (added by the upgrade) was enabled with no subscription, so every apt update errored.
# No reboot. Microcode takes effect at the next boot.
steps:
  - name: switch off the enterprise repo (no subscription)
    sudo: true
    shell: |
      printf 'Enabled: false\n' >> /etc/apt/sources.list.d/pve-enterprise.sources
    when: "! grep -q '^Enabled: false' /etc/apt/sources.list.d/pve-enterprise.sources"

  - name: add non-free-firmware to the Debian trixie lines (for intel-microcode)
    sudo: true
    shell: sed -i -E '/^deb .*debian(-security)? trixie/{/non-free-firmware/!s/$/ non-free-firmware/}' /etc/apt/sources.list
    when: "grep -E '^deb .*debian(-security)? trixie' /etc/apt/sources.list | grep -qv non-free-firmware"

  - name: apt update (must be clean now)
    sudo: true
    shell: |
      apt-get update -q 2>&1 | tail -3; ! apt-get update -q 2>&1 | grep -E '^(E|Err):'
    changed_when: "false"

  - name: kernel headers for 7.0 (the meta package keeps future kernels covered) + intel-microcode
    sudo: true
    shell: DEBIAN_FRONTEND=noninteractive apt-get install -y -q proxmox-default-headers proxmox-headers-7.0.14-20-pve intel-microcode 2>&1 | tail -8
    when: "! dpkg -s proxmox-headers-7.0.14-20-pve >/dev/null 2>&1 || ! dpkg -s intel-microcode >/dev/null 2>&1 || ! dpkg -s proxmox-default-headers >/dev/null 2>&1"

  - name: build the nvidia module for 7.0.14-20 if the header hook did not
    sudo: true
    shell: dkms autoinstall -k 7.0.14-20-pve 2>&1 | tail -8
    when: "! dkms status nvidia/580.178.04 -k 7.0.14-20-pve 2>/dev/null | grep -q installed"

  - name: load the driver, start the persistence service, start nh3-ml1
    sudo: true
    shell: |
      set -e
      modprobe nvidia
      systemctl start nvidia-persistenced
      pct status 109 | grep -q running || pct start 109
    changed_when: "true"

verify:
  - name: nvidia module built + installed for the running kernel
    sudo: true
    shell: dkms status nvidia/580.178.04 -k "$(uname -r)" | grep -q installed
    changed_when: "false"
  - name: GPU visible on the host, persistence daemon active
    sudo: true
    shell: nvidia-smi --query-gpu=name,driver_version --format=csv,noheader && systemctl is-active --quiet nvidia-persistenced
    changed_when: "false"
  - name: nh3-ml1 running and sees the GPU
    sudo: true
    shell: pct status 109 | grep -q running && pct exec 109 -- nvidia-smi --query-gpu=name --format=csv,noheader
    changed_when: "false"
  - name: microcode installed, enterprise repo off
    sudo: true
    shell: |
      dpkg -s intel-microcode >/dev/null && grep -q '^Enabled: false' /etc/apt/sources.list.d/pve-enterprise.sources
    changed_when: "false"