From e50e57b5f2d6367742b9381917f8f0bac42aa698 Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Sat, 19 Sep 2026 05:30:25 -0700 Subject: [PATCH] chore(esh): exclude VM 102 from the nightly backup job esh-vm-workstation is a Windows 11 Parsec/RDP sandbox -- no password, no state anyone needs recovered (operator ruling, 2026-09-19). Its nightly vzdump has failed since ~2026-09-06 with "timeout waiting on systemd", and the ESH job has reported `job errors` every night as a result, so a guest nobody needs backed up was manufacturing a real job failure daily. The job uses an explicit `vmid` INCLUDE list, so exclusion means removal from that list -- there is no `exclude` directive here, which is the same distinction that makes esh-scale CT 108 invisible in any exclude list. The reason now lives in the job's own `comment`, and check-backup-freshness.sh reports 102 under "NOT BACKED UP BY POLICY" rather than going quiet about it. This is a policy decision, not a fix: the systemd-scope failure on stopped guests is NOT diagnosed, merely no longer reached on this node. A stopped guest that DOES matter would hit the same wall. The playbook carries two hard-won warnings in its header. Read the job through the API, never `grep -A` on jobs.cfg -- `vmid` sorts last in the block, so adding the comment line pushed it outside the window and every condition silently inverted. And use YAML block scalars for the JSON-extraction expressions; double-quoted ones did not survive YAML unescaping plus elway's bash -c quoting. Verified: vmid 100,101,103,104,105,106,107; the other seven guests still covered; job still enabled; a rerun skips the change rather than repeating it. --- playbooks/esh-exclude-vm102-from-backup.yaml | 96 ++++++++++++++++++++ 1 file changed, 96 insertions(+) create mode 100644 playbooks/esh-exclude-vm102-from-backup.yaml diff --git a/playbooks/esh-exclude-vm102-from-backup.yaml b/playbooks/esh-exclude-vm102-from-backup.yaml new file mode 100644 index 0000000..1b4f585 --- /dev/null +++ b/playbooks/esh-exclude-vm102-from-backup.yaml @@ -0,0 +1,96 @@ +# Exclude esh-vm-workstation (VM 102) from the ESH nightly backup job. +# +# WHY: 102 is a Windows 11 Parsec/RDP SANDBOX — no password, no persistent +# state anyone needs recovered (operator, 2026-09-19). Its nightly vzdump has +# failed since ~2026-09-06 with "timeout waiting on systemd", and the ESH job +# has reported `job errors` every night as a result. Excluding it stops a +# genuine job failure being manufactured nightly by a guest nobody needs +# backed up. +# +# ⚠ This is a policy decision, not a fix. The systemd-scope failure on stopped +# guests is NOT diagnosed — it is merely no longer reached on this node. If a +# stopped guest that DOES matter ever joins this job, expect it to hit the +# same wall. +# +# The job uses an explicit `vmid` INCLUDE list, so exclusion = removal from +# that list. There is no `exclude` directive here (unlike ana-pve's all-guests +# job), which is the same distinction that makes esh-scale CT 108 invisible in +# any exclude list. check-backup-freshness.sh computes coverage as a union +# across jobs and will report 102 under "NOT BACKED UP BY POLICY". +# +# ⚠ READ THE JOB THROUGH THE API, NEVER `grep -A` ON jobs.cfg. The first +# version of this playbook did exactly that, and `vmid` sorts LAST in the +# block: the moment step 4 added a `comment` line, the block grew by one and +# `-A9` stopped reaching `vmid`. Every condition and verify silently inverted — +# the `when:` decided 102 was already gone and skipped the real change, and the +# verify then reported guest 100 "MISSING". Nothing was actually modified, and +# the only reason that was obvious is that the verify phase runs regardless. +# A fixed context window is a filter that narrows without telling you. +# +# ⚠ AND USE YAML BLOCK SCALARS, NOT DOUBLE-QUOTED ONES. The second version put +# the JSON-extraction expressions in double-quoted YAML scalars; between YAML +# unescaping and elway's `bash -c` quoting the backslashes did not survive, so +# the `when:` silently evaluated false and skipped the real change AGAIN. Every +# condition below is a `>-` block scalar for that reason. +# +# ⚠ AND BEWARE GREEDY `.*` ON THIS JSON: the payload carries BOTH a top-level +# "enabled":1 and a nested "fleecing":{"enabled":"0"}, so +# `sed 's/.*"enabled":\([0-9]*\).*/\1/'` matches the LAST one and returns +# empty. The enabled check below requires at least one digit so it can only +# match the unquoted top-level field. +# +# scripts/elway infra-ops@esh-pve --playbook playbooks/esh-exclude-vm102-from-backup.yaml +vars: + job: backup-82f43b66-828a + keep: "100,101,103,104,105,106,107" + +steps: + - name: Snapshot jobs.cfg before touching it + sudo: true + shell: cp -n /etc/pve/jobs.cfg /root/jobs.cfg.pre-102-exclusion.bak + creates: /root/jobs.cfg.pre-102-exclusion.bak + + - name: Record the vmid list we are replacing + sudo: true + shell: >- + pvesh get /cluster/backup/{{ job }} --output-format json + | sed -n 's/.*"vmid":"\([^"]*\)".*/vmid = \1/p' + + - name: Remove 102 from the job's include list + sudo: true + shell: pvesh set /cluster/backup/{{ job }} --vmid {{ keep }} + # Idempotent: skip entirely once 102 is already gone from the LIVE job. + when: >- + pvesh get /cluster/backup/{{ job }} --output-format json + | sed -n 's/.*"vmid":"\([^"]*\)".*/\1/p' + | tr ',' '\n' | grep -qx 102 + + - name: Note the reason on the job itself, so the exclusion is self-explaining + sudo: true + stop_on_fail: false + shell: >- + pvesh set /cluster/backup/{{ job }} --comment 'excludes 102 (Windows + Parsec sandbox, no state to recover); 108 esh-scale (subnet router) is + absent by omission' + +verify: + - name: 102 is gone from the include list + sudo: true + shell: >- + ! ( pvesh get /cluster/backup/{{ job }} --output-format json + | sed -n 's/.*"vmid":"\([^"]*\)".*/\1/p' + | tr ',' '\n' | grep -qx 102 ) + - name: every other ESH guest is still covered + sudo: true + shell: >- + live=$(pvesh get /cluster/backup/{{ job }} --output-format json + | sed -n 's/.*"vmid":"\([^"]*\)".*/\1/p'); + for id in 100 101 103 104 105 106 107; do + printf '%s' "$live" | tr ',' '\n' | grep -qx "$id" + || { echo "MISSING $id (live list: $live)"; exit 1; }; + done; echo "all 7 still covered: $live" + - name: the job is still enabled + sudo: true + shell: >- + test "$(pvesh get /cluster/backup/{{ job }} --output-format json + | grep -o '"enabled":[0-9][0-9]*' | head -1)" = '"enabled":1'