ops(nh3-pve): PVE 9 post-upgrade fixes (7.0 headers + nvidia DKMS, microcode, enterprise repo off); old VM102 snapshot removed

2026-10-03 14:09:41 -07:00
parent 08c3271084
commit d9e09fed14
2 changed files with 64 additions and 1 deletions
@@ -0,0 +1,63 @@
+# nh3-pve after its PVE 8.4.1 → 9.2.21 upgrade (2026-10-03, Prime's go on all of it). Idempotent.
+#   scripts/elway infra-ops@10.100.250.60 --playbook playbooks/nh3-pve-pve9-postfix.yaml
+# Post-check findings this closes:
+#   - the upgrade installed kernel 7.0.14-20 but no 7.0 headers, so the nvidia 580.178.04 DKMS module was never
+#     built for it: nvidia-persistenced failed and CT 109 (nh3-ml1, the TEI twin) could not start;
+#   - pve8to9 FAIL: intel-microcode missing (Debian sources had no non-free-firmware);
+#   - pve-enterprise.sources (added by the upgrade) was enabled with no subscription, so every apt update errored.
+# No reboot. Microcode takes effect at the next boot.
+steps:
+  - name: switch off the enterprise repo (no subscription)
+    sudo: true
+    shell: |
+      printf 'Enabled: false\n' >> /etc/apt/sources.list.d/pve-enterprise.sources
+    when: "! grep -q '^Enabled: false' /etc/apt/sources.list.d/pve-enterprise.sources"
+
+  - name: add non-free-firmware to the Debian trixie lines (for intel-microcode)
+    sudo: true
+    shell: sed -i -E '/^deb .*debian(-security)? trixie/{/non-free-firmware/!s/$/ non-free-firmware/}' /etc/apt/sources.list
+    when: "grep -E '^deb .*debian(-security)? trixie' /etc/apt/sources.list | grep -qv non-free-firmware"
+
+  - name: apt update (must be clean now)
+    sudo: true
+    shell: |
+      apt-get update -q 2>&1 | tail -3; ! apt-get update -q 2>&1 | grep -E '^(E|Err):'
+    changed_when: "false"
+
+  - name: kernel headers for 7.0 (the meta package keeps future kernels covered) + intel-microcode
+    sudo: true
+    shell: DEBIAN_FRONTEND=noninteractive apt-get install -y -q proxmox-default-headers proxmox-headers-7.0.14-20-pve intel-microcode 2>&1 | tail -8
+    when: "! dpkg -s proxmox-headers-7.0.14-20-pve >/dev/null 2>&1 || ! dpkg -s intel-microcode >/dev/null 2>&1 || ! dpkg -s proxmox-default-headers >/dev/null 2>&1"
+
+  - name: build the nvidia module for 7.0.14-20 if the header hook did not
+    sudo: true
+    shell: dkms autoinstall -k 7.0.14-20-pve 2>&1 | tail -8
+    when: "! dkms status nvidia/580.178.04 -k 7.0.14-20-pve 2>/dev/null | grep -q installed"
+
+  - name: load the driver, start the persistence service, start nh3-ml1
+    sudo: true
+    shell: |
+      set -e
+      modprobe nvidia
+      systemctl start nvidia-persistenced
+      pct status 109 | grep -q running || pct start 109
+    changed_when: "true"
+
+verify:
+  - name: nvidia module built + installed for the running kernel
+    sudo: true
+    shell: dkms status nvidia/580.178.04 -k "$(uname -r)" | grep -q installed
+    changed_when: "false"
+  - name: GPU visible on the host, persistence daemon active
+    sudo: true
+    shell: nvidia-smi --query-gpu=name,driver_version --format=csv,noheader && systemctl is-active --quiet nvidia-persistenced
+    changed_when: "false"
+  - name: nh3-ml1 running and sees the GPU
+    sudo: true
+    shell: pct status 109 | grep -q running && pct exec 109 -- nvidia-smi --query-gpu=name --format=csv,noheader
+    changed_when: "false"
+  - name: microcode installed, enterprise repo off
+    sudo: true
+    shell: |
+      dpkg -s intel-microcode >/dev/null && grep -q '^Enabled: false' /etc/apt/sources.list.d/pve-enterprise.sources
+    changed_when: "false"