ops(pbs): pbs-nh3 + pbs-ana upgraded to PBS 4.2.7 / Debian 13; guest-agent driver script, plan and log
This commit is contained in:
Executable
+106
@@ -0,0 +1,106 @@
|
||||
#!/usr/bin/env bash
|
||||
# Proxmox Backup Server 3 → 4, run AS ROOT INSIDE the PBS VM, one phase per call.
|
||||
#
|
||||
# scripts/pbs4-upgrade-guest.sh <phase> phases: prep | prepcheck | upgrade | status | postcheck
|
||||
#
|
||||
# How it is driven (2026-10-04): infra-ops has no login on the PBS VMs (tier-3 hosts were never bootstrapped),
|
||||
# so the phases run through the hypervisor's qemu-guest-agent, which is root in the guest and needs no network:
|
||||
# ssh infra-ops@<pve> "sudo -n qm guest exec <vmid> --timeout 600 -- bash -c '…this script…'"
|
||||
# The same path is the recovery path if a trixie boot loses its network (the agent does not use it).
|
||||
#
|
||||
# Plan + rollback: docs/runbooks/pbs-pbs4-upgrade-plan.md. Official guide: pbs.proxmox.com/wiki/Upgrade_from_3_to_4.
|
||||
# Nothing here reboots: the reboot is a separate, deliberate `qm reboot` from the hypervisor.
|
||||
set -euo pipefail
|
||||
phase=${1:?phase}
|
||||
STAGE=/root/pbs4-prep-20261004
|
||||
ds=$(proxmox-backup-manager datastore list --output-format json | python3 -c 'import json,sys; print(json.load(sys.stdin)[0]["name"])')
|
||||
|
||||
case "$phase" in
|
||||
prep)
|
||||
# no downtime: back up config, latest 3.4, enterprise repo off, NIC pinned, pbs3to4 --full
|
||||
mkdir -p "$STAGE"
|
||||
[ -f "$STAGE/etc-proxmox-backup.tar.gz" ] || tar czf "$STAGE/etc-proxmox-backup.tar.gz" -C /etc proxmox-backup
|
||||
[ -f "$STAGE/etc.tar.gz" ] || tar czf "$STAGE/etc.tar.gz" -C / etc 2>/dev/null
|
||||
[ -f "$STAGE/sources.list.before" ] || cp -p /etc/apt/sources.list "$STAGE/sources.list.before"
|
||||
# the enterprise repo answers 401 (no subscription) and makes every `apt update` fail
|
||||
if [ -f /etc/apt/sources.list.d/pbs-enterprise.list ]; then mv /etc/apt/sources.list.d/pbs-enterprise.list "$STAGE/"; fi
|
||||
# pin the only NIC by MAC so Debian 13's udev cannot rename it (ens18 is DHCP, allow-hotplug)
|
||||
mac=$(cat /sys/class/net/ens18/address)
|
||||
mkdir -p /etc/systemd/network
|
||||
if [ ! -f /etc/systemd/network/10-pin-ens18.link ]; then
|
||||
printf '# Pin ens18 by MAC for the PBS 4 / Debian 13 upgrade. eshpfi scripts/pbs4-upgrade-guest.sh\n[Match]\nMACAddress=%s\nType=ether\n[Link]\nName=ens18\n' "$mac" > /etc/systemd/network/10-pin-ens18.link
|
||||
update-initramfs -u -k all >/dev/null 2>&1
|
||||
fi
|
||||
if apt-get update -q 2>&1 | grep -E '^(E|Err):'; then echo "REFUSING: apt update errors"; exit 1; fi # an if: set -e ignores a negated pipeline
|
||||
# ⚠ DETACHED, never in-band: this script arrives through qemu-guest-agent, and qemu-guest-agent is itself in the
|
||||
# upgrade set. Its postinst restarts the agent, systemd kills the agent's cgroup, and an in-band apt dies mid-dpkg.
|
||||
# That happened on both VMs at 1120-1122 on 2026-10-04 (≈50 packages unpacked-not-configured, repaired with
|
||||
# `dpkg --configure -a` run under systemd-run). A transient unit is outside the agent's cgroup and survives it.
|
||||
if [ ! -f /root/pbs-prep-upgrade.log ]; then
|
||||
systemd-run --unit=pbs-prep-upgrade --setenv=DEBIAN_FRONTEND=noninteractive \
|
||||
/bin/sh -c 'apt-get dist-upgrade -y -q -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold > /root/pbs-prep-upgrade.log 2>&1; echo "EXIT $?" >> /root/pbs-prep-upgrade.log'
|
||||
fi
|
||||
echo "latest-3.4 dist-upgrade started detached (log /root/pbs-prep-upgrade.log); then run: prepcheck"
|
||||
;;
|
||||
prepcheck)
|
||||
test -z "$(dpkg --audit 2>&1)" || { dpkg --audit; exit 1; }
|
||||
proxmox-backup-manager versions
|
||||
pbs3to4 --full 2>&1 | sed 's/\x1b\[[0-9;]*m//g' | grep -E '^(FAIL|WARN|NOTICE):|TOTAL|PASSED|SKIPPED|WARNINGS|FAILURES' || true
|
||||
;;
|
||||
upgrade)
|
||||
# refuse while a task is running; datastore to read-only maintenance; trixie sources; DETACHED dist-upgrade
|
||||
running=$(proxmox-backup-manager task list --output-format json 2>/dev/null | python3 -c 'import json,sys; print(sum(1 for t in json.load(sys.stdin) if not t.get("endtime")))')
|
||||
[ "$running" = 0 ] || { echo "REFUSING: $running task(s) running"; exit 1; }
|
||||
out=$(pbs3to4 --full 2>&1 | sed 's/\x1b\[[0-9;]*m//g'); echo "$out" | grep -E '^(FAIL|WARN):' || true
|
||||
# ONE known FAIL is tolerated, and only that one: these VMs are "PBS on plain Debian" (proxmox-backup-server on the
|
||||
# Debian kernel, a documented install mode), so the `proxmox-backup` META package the checker looks for is absent by
|
||||
# design and pbs3to4 says "could not match the 'proxmox-backup' package version". The requirement that check exists
|
||||
# for (PBS >= 3.4.2 installed AND running) is asserted directly below instead. Any OTHER FAIL refuses.
|
||||
if echo "$out" | grep '^FAIL:' | grep -qv "could not match the 'proxmox-backup' package version"; then
|
||||
echo "REFUSING: pbs3to4 reports a FAIL other than the known meta-package one"; exit 1; fi
|
||||
inst=$(dpkg-query -W -f='${Version}' proxmox-backup-server)
|
||||
dpkg --compare-versions "$inst" ge 3.4.2 || { echo "REFUSING: proxmox-backup-server $inst < 3.4.2"; exit 1; }
|
||||
proxmox-backup-manager versions | grep -q "running version: 3\.4\.[2-9]" || { echo "REFUSING: running PBS is not >= 3.4.2"; exit 1; }
|
||||
proxmox-backup-manager datastore update "$ds" --maintenance-mode read-only
|
||||
if grep -q bookworm /etc/apt/sources.list; then
|
||||
sed -i 's/bookworm/trixie/g' /etc/apt/sources.list
|
||||
if [ -f /etc/apt/sources.list.d/pbs-no-subscription.list ]; then mv /etc/apt/sources.list.d/pbs-no-subscription.list "$STAGE/"; fi
|
||||
printf 'Types: deb\nURIs: http://download.proxmox.com/debian/pbs\nSuites: trixie\nComponents: pbs-no-subscription\nSigned-By: /usr/share/keyrings/proxmox-archive-keyring.gpg\n' > /etc/apt/sources.list.d/proxmox.sources
|
||||
fi
|
||||
if apt-get update -q 2>&1 | grep -E '^(E|Err):'; then echo "REFUSING: apt update errors"; exit 1; fi # an if: set -e ignores a negated pipeline
|
||||
apt-get -s dist-upgrade > "$STAGE/dist-upgrade-plan.txt" 2>&1
|
||||
echo "installs/upgrades: $(grep -c '^Inst ' "$STAGE/dist-upgrade-plan.txt")"
|
||||
grep '^Remv ' "$STAGE/dist-upgrade-plan.txt" | awk '{print $2}' | grep -v '^lib' | tr '\n' ' '; echo
|
||||
if grep -qE '^Remv (proxmox-backup-server|proxmox-backup-client|grub-pc|ifupdown|isc-dhcp-client|openssh-server|nfs-common|qemu-guest-agent|linux-image-amd64) ' "$STAGE/dist-upgrade-plan.txt"; then
|
||||
echo "REFUSING: the plan removes a package this VM cannot run, boot or be reached without"; exit 1
|
||||
fi
|
||||
if [ ! -f /root/pbs4-upgrade.log ]; then
|
||||
systemd-run --unit=pbs4-dist-upgrade --setenv=DEBIAN_FRONTEND=noninteractive \
|
||||
/bin/sh -c 'apt-get dist-upgrade -y -q -o Dpkg::Options::=--force-confdef -o Dpkg::Options::=--force-confold > /root/pbs4-upgrade.log 2>&1; echo "EXIT $?" >> /root/pbs4-upgrade.log'
|
||||
fi
|
||||
echo started
|
||||
;;
|
||||
status)
|
||||
tail -2 /root/pbs4-upgrade.log 2>/dev/null || echo "no upgrade log"
|
||||
;;
|
||||
postcheck)
|
||||
# after the dist-upgrade (before AND after the reboot): enterprise repo off, the things a boot needs
|
||||
tail -1 /root/pbs4-upgrade.log | grep -qx 'EXIT 0'
|
||||
test -z "$(dpkg --audit 2>&1)"
|
||||
f=/etc/apt/sources.list.d/pbs-enterprise.sources
|
||||
if [ -f $f ] && ! grep -q '^Enabled: *false' $f; then
|
||||
if grep -q '^Enabled:' $f; then sed -i 's/^Enabled:.*/Enabled: false/' $f; else echo 'Enabled: false' >> $f; fi
|
||||
fi
|
||||
dpkg-query -W -f='${Status}\n' grub-pc qemu-guest-agent isc-dhcp-client nfs-common | sort | uniq -c
|
||||
new=$(ls /boot/vmlinuz-* | sed 's#/boot/vmlinuz-##' | sort -V | tail -1); echo "newest kernel $new, running $(uname -r)"
|
||||
# to a file first: `lsinitramfs | grep -q` SIGPIPEs zstd ("zstd failed") and pipefail then hides a real match
|
||||
lsinitramfs /boot/initrd.img-$new > /tmp/pbs4-initrd.lst 2>/dev/null; grep -q 10-pin-ens18.link /tmp/pbs4-initrd.lst && echo "pin in $new initramfs"
|
||||
grep -q "vmlinuz-$new" /boot/grub/grub.cfg && echo "grub.cfg boots $new"
|
||||
proxmox-backup-manager versions
|
||||
findmnt -no SOURCE,FSTYPE /mnt/pbs-datastore || echo "DATASTORE NOT MOUNTED"
|
||||
proxmox-backup-manager datastore show "$ds" --output-format json | python3 -c 'import json,sys; d=json.load(sys.stdin); print("maintenance:", d.get("maintenance-mode","none"))'
|
||||
systemctl is-active proxmox-backup proxmox-backup-proxy qemu-guest-agent
|
||||
pbs3to4 --full 2>&1 | sed 's/\x1b\[[0-9;]*m//g' | grep -E '^(FAIL|WARN):|FAILURES' || true
|
||||
;;
|
||||
*) echo "unknown phase $phase"; exit 2 ;;
|
||||
esac
|
||||
Reference in New Issue
Block a user