From 44c853cd2031630c1eece8756f60585c144e0536 Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Thu, 10 Sep 2026 09:10:01 -0700 Subject: [PATCH] Repair ana-docker database backups and bound CI build caches --- configs/buildkit/ana-docker/README.md | 20 ++++++ .../ana-docker/buildkit-cache-prune.service | 12 ++++ .../ana-docker/buildkit-cache-prune.timer | 10 +++ configs/buildkit/ana-docker/prune.sh | 21 ++++++ configs/restic/ana-docker/README.md | 42 +++++++++++ configs/restic/ana-docker/pre-backup.sh | 71 ++++++++++++------- configs/restic/ana-docker/test_pre_backup.py | 52 ++++++++++++++ .../2026-09-10-ana-docker-disk-repair.md | 25 +++++++ persistent-memory.md | 2 + playbooks/ana-docker-backup-repair.yaml | 23 ++++++ playbooks/ana-docker-cache-budget.yaml | 25 +++++++ 11 files changed, 277 insertions(+), 26 deletions(-) create mode 100644 configs/buildkit/ana-docker/README.md create mode 100644 configs/buildkit/ana-docker/buildkit-cache-prune.service create mode 100644 configs/buildkit/ana-docker/buildkit-cache-prune.timer create mode 100644 configs/buildkit/ana-docker/prune.sh create mode 100644 configs/restic/ana-docker/README.md create mode 100644 configs/restic/ana-docker/test_pre_backup.py create mode 100644 persistent-memory.d/2026-09-10-ana-docker-disk-repair.md create mode 100644 playbooks/ana-docker-backup-repair.yaml create mode 100644 playbooks/ana-docker-cache-budget.yaml diff --git a/configs/buildkit/ana-docker/README.md b/configs/buildkit/ana-docker/README.md new file mode 100644 index 0000000..b6b2a44 --- /dev/null +++ b/configs/buildkit/ana-docker/README.md @@ -0,0 +1,20 @@ +# CI cache budget + +Deploy: `scripts/elway infra-ops@10.250.50.70 --playbook playbooks/ana-docker-cache-budget.yaml`. + +The hourly `buildkit-cache-prune.timer` runs `prune.sh` against the explicitly +named Worldtree, Skaldsong and Soong builders. Each running builder prunes unused +cache toward 10 GiB (`--keep-storage 10240`). This is periodic reclamation, not +a hard disk quota; in-use cache and growth between runs can exceed the target. +Missing and stopped builders are skipped and are never started by the timer. +Build caches are regenerable, but subsequent builds can take longer. + +On 2026-09-10, Skaldsong reclaimed 10.41 GB and Worldtree reported 74.89 GB +reclaimed (BuildKit accounting). Worldtree was temporarily started for that +initial prune, then returned to stopped. Soong remained stopped. Following +cache pruning and the Gitea backup repair/cleanup, root usage fell from 84% to +51%, with 115 GiB available. Use filesystem `df` for actual capacity rather than +summing cache reports and logical file sizes. + +Checks: `systemctl status buildkit-cache-prune.timer`, +`journalctl -u buildkit-cache-prune.service`, and `df -h /`. diff --git a/configs/buildkit/ana-docker/buildkit-cache-prune.service b/configs/buildkit/ana-docker/buildkit-cache-prune.service new file mode 100644 index 0000000..7da955c --- /dev/null +++ b/configs/buildkit/ana-docker/buildkit-cache-prune.service @@ -0,0 +1,12 @@ +[Unit] +Description=Bound ana-docker CI builder cache usage +After=docker.service +ConditionPathExists=/var/run/docker.sock + +[Service] +Type=oneshot +ExecStart=/usr/local/sbin/pfi-buildkit-cache-prune +Nice=10 +IOSchedulingClass=best-effort +IOSchedulingPriority=7 +TimeoutStartSec=30min diff --git a/configs/buildkit/ana-docker/buildkit-cache-prune.timer b/configs/buildkit/ana-docker/buildkit-cache-prune.timer new file mode 100644 index 0000000..a7e95ee --- /dev/null +++ b/configs/buildkit/ana-docker/buildkit-cache-prune.timer @@ -0,0 +1,10 @@ +[Unit] +Description=Hourly budget enforcement for ana-docker CI builder caches + +[Timer] +OnCalendar=hourly +RandomizedDelaySec=5min +Persistent=true + +[Install] +WantedBy=timers.target diff --git a/configs/buildkit/ana-docker/prune.sh b/configs/buildkit/ana-docker/prune.sh new file mode 100644 index 0000000..d072f2d --- /dev/null +++ b/configs/buildkit/ana-docker/prune.sh @@ -0,0 +1,21 @@ +#!/bin/bash +# Bound regenerable cache on ana-docker's named CI builders. BuildKit protects +# in-use references. Stopped builders cannot grow and are left stopped. +set -euo pipefail +failed=0 +for builder in buildx_buildkit_worldtree-builder0 buildx_buildkit_skaldsong-builder0 buildx_buildkit_soong-builder0; do + if ! state=$(docker inspect --format '{{.State.Running}}' "$builder" 2>/dev/null); then + echo "$builder: absent, skipped" + continue + fi + if [ "$state" != true ]; then + echo "$builder: stopped, skipped" + continue + fi + echo "$builder: reclaim unused cache toward 10 GiB budget" + if ! docker exec "$builder" buildctl prune --all --keep-storage 10240 | tail -1; then + echo "$builder: cache pruning failed" >&2 + failed=1 + fi +done +exit "$failed" diff --git a/configs/restic/ana-docker/README.md b/configs/restic/ana-docker/README.md new file mode 100644 index 0000000..6d5e4ff --- /dev/null +++ b/configs/restic/ana-docker/README.md @@ -0,0 +1,42 @@ +# ana-docker database staging + +`pre-backup.sh` runs as root through resticprofile's `run-before`. Deploy and +exercise it with `scripts/elway infra-ops@10.250.50.70 --playbook +playbooks/ana-docker-backup-repair.yaml` (allow about 12 minutes for Gitea). + +Required dump failures now abort the backup. Dumps are prepared in a private +pending directory; previous staged files are replaced only after successful +generation and nonempty checks. Gitea additionally gets tar validation and a +private in-container scratch directory with exit cleanup. Ordinary failures +clean up scratch; SIGKILL or host crashes cannot run shell traps. + +Regression checks: `python3 configs/restic/ana-docker/test_pre_backup.py`. +The stage/credential path overrides are for isolated tests; production defaults +remain `/var/lib/restic/stage` and `/etc/restic/dbcreds.env`. + +## Repair verified 2026-09-10 + +Gitea dumps had failed since June 4 because five root-only historical app.ini +copies inside its config directory were unreadable to the git dump user. +Those copies retain root-only permissions under +`/opt/docker/backups/gitea-config-history/`, included in backups. Do not put +unreadable config history back inside Gitea's dump tree. + +Vaultwarden's stale backup credentials were synchronized with its live database +connection; the root:600 host file is also saved as Vaultwarden item +`ana-docker/restic-dbcreds.env`. No secrets belong in this repository. + +Fresh stage files were saved to Restic snapshot `2ec5a37c`. Both database dumps +were restored FROM that repository snapshot into disposable PostgreSQL 16 with +network disabled and tmpfs storage: Vaultwarden 7 users, Gitea 9 users and +93 repositories. The temporary database container was removed afterward. +This verifies database restore, not a complete application disaster-recovery drill. + +Only after successful restores, 101 abandoned Gitea dump files (47.31 GiB logical +size) were deleted. Their inventory is root-only at +`/opt/docker/backups/gitea-config-history/removed-dump-manifest-20260910.json`. +Old failed-dump history itself was not retained; the fresh replacement is in +Restic. The verification marker is `/var/lib/restic/verified-repair-20260910`. + +OpenWebUI's existing warning/fallback to volume capture when sqlite3 is absent +is unchanged; this repair does not claim a verified OpenWebUI database backup. diff --git a/configs/restic/ana-docker/pre-backup.sh b/configs/restic/ana-docker/pre-backup.sh index 66bfda3..27a9797 100755 --- a/configs/restic/ana-docker/pre-backup.sh +++ b/configs/restic/ana-docker/pre-backup.sh @@ -25,25 +25,27 @@ # Intentionally NOT handled: # - mattermost (retired 2026-04-21 — stack dir lingers but is not running) # -# Idempotent: clears and recreates its staging files each run. -# Errors in individual blocks are logged as WARN but don't abort the whole -# script — partial dumps are better than no dumps. +# Required database dump failures abort the backup instead of reporting a +# successful snapshot without them. Previous staged dumps remain intact until +# all required dumps succeed. Gitea scratch files are isolated and trap-cleaned. set -euo pipefail -STAGE=/var/lib/restic/stage -install -d -o root -g root -m 0700 "$STAGE" +STAGE=${RESTIC_STAGE_DIR:-/var/lib/restic/stage} +install -d -m 0700 "$STAGE" +WORK=$(mktemp -d "$STAGE/.pending.XXXXXXXX") +trap 'rm -rf -- "$WORK"' EXIT +ERRORS=0 log() { printf '%s pre-backup(ana-docker): %s\n' "$(date -Is)" "$*"; } warn() { log "WARN: $*" >&2; } - -# Purge previous stage so stale dumps don't pile up into the snapshot. -find "$STAGE" -mindepth 1 -maxdepth 1 -exec rm -rf {} + +error() { ERRORS=$((ERRORS + 1)); warn "$*"; } # Load external-DB creds. Silently skipped if missing — individual blocks # that need them will log their own WARN. -if [ -r /etc/restic/dbcreds.env ]; then - set -a; . /etc/restic/dbcreds.env; set +a +CREDS=${RESTIC_DB_CREDS_FILE:-/etc/restic/dbcreds.env} +if [ -r "$CREDS" ]; then + set -a; . "$CREDS"; set +a fi # ---------- synapse (internal Postgres) --------------------------------------- @@ -51,8 +53,8 @@ if docker inspect synapse-db >/dev/null 2>&1; then log "dumping synapse postgres" docker exec synapse-db \ pg_dump -U synapse -d synapse -Fc --clean --if-exists \ - > "$STAGE/synapse.pg_dump" \ - || warn "synapse pg_dump failed" + > "$WORK/synapse.pg_dump" \ + || error "synapse pg_dump failed" else log "skip synapse: container not present" fi @@ -62,8 +64,8 @@ if docker inspect seafile-mysql >/dev/null 2>&1; then log "dumping seafile mariadb" docker exec seafile-mysql sh -c \ 'mysqldump -uroot -p"$MYSQL_ROOT_PASSWORD" --all-databases --single-transaction --quick 2>/dev/null' \ - | gzip -c > "$STAGE/seafile.sql.gz" \ - || warn "seafile mysqldump failed" + | gzip -c > "$WORK/seafile.sql.gz" \ + || error "seafile mysqldump failed" else log "skip seafile: container not present" fi @@ -74,17 +76,17 @@ fi # be deleted separately — this hook captures the live Postgres data only. if docker inspect vaultwarden >/dev/null 2>&1; then if [ -z "${VW_PGPASS:-}" ]; then - warn "vaultwarden: VW_PGPASS unset in /etc/restic/dbcreds.env — skipping" + error "vaultwarden: VW_PGPASS unset in /etc/restic/dbcreds.env" elif ! command -v pg_dump >/dev/null 2>&1; then - warn "vaultwarden: pg_dump not installed — skipping (apt install postgresql-client)" + error "vaultwarden: pg_dump not installed (apt install postgresql-client)" else log "dumping vaultwarden postgres (external: ${VW_PGHOST}:${VW_PGPORT:-5432})" PGPASSWORD="$VW_PGPASS" pg_dump \ -h "$VW_PGHOST" -p "${VW_PGPORT:-5432}" \ -U "$VW_PGUSER" -d "$VW_PGDB" \ -Fc --clean --if-exists \ - > "$STAGE/vaultwarden.pg_dump" \ - || warn "vaultwarden pg_dump failed" + > "$WORK/vaultwarden.pg_dump" \ + || error "vaultwarden pg_dump failed" fi else log "skip vaultwarden: container not present" @@ -110,14 +112,16 @@ fi # --skip-repository, --skip-attachment-data. if docker inspect gitea >/dev/null 2>&1; then log "dumping gitea (gitea dump, uncompressed tar)" - if docker exec -u git gitea sh -c \ - 'rm -f /tmp/gitea-dump.tar && gitea dump -c /data/gitea/conf/app.ini -f /tmp/gitea-dump.tar --type tar' \ - >/dev/null 2>&1; then - docker cp gitea:/tmp/gitea-dump.tar "$STAGE/gitea-dump.tar" \ - && docker exec -u git gitea rm -f /tmp/gitea-dump.tar \ - || warn "gitea dump copy/cleanup failed" + if docker exec -u git gitea sh -c ' + set -eu + scratch=$(mktemp -d /tmp/gitea-backup.XXXXXXXX) + trap '\''rm -rf -- "$scratch"'\'' EXIT + gitea dump -c /data/gitea/conf/app.ini --tempdir "$scratch" --file - --type tar + ' > "$WORK/gitea-dump.tar"; then + tar -tf "$WORK/gitea-dump.tar" >/dev/null \ + || error "gitea archive validation failed" else - warn "gitea dump command failed" + error "gitea dump command failed (details above); previous stage preserved" fi else log "skip gitea: container not present" @@ -137,7 +141,7 @@ if docker inspect "$OWUI_CONTAINER" >/dev/null 2>&1; then "/app/backend/data/vector_db/chroma.sqlite3:chroma.sqlite3"; do src=${pair%:*}; dst=${pair#*:} if docker exec "$OWUI_CONTAINER" sqlite3 "$src" ".backup /tmp/$dst" 2>/dev/null; then - docker cp "$OWUI_CONTAINER:/tmp/$dst" "$STAGE/openwebui.$dst" \ + docker cp "$OWUI_CONTAINER:/tmp/$dst" "$WORK/openwebui.$dst" \ && docker exec "$OWUI_CONTAINER" rm -f "/tmp/$dst" \ || warn "openwebui copy/cleanup failed for $dst" else @@ -152,6 +156,21 @@ else fi # ---------- summary ----------------------------------------------------------- +if [ "$ERRORS" -ne 0 ]; then + log "FAILED: $ERRORS required database dump(s) failed; previous stage preserved" + exit 1 +fi +for dump in "$WORK"/*; do + [ -f "$dump" ] || continue + if [ ! -s "$dump" ]; then + log "FAILED: empty dump ${dump##*/}; previous stage preserved" + exit 1 + fi +done +for dump in "$WORK"/*; do + [ -f "$dump" ] || continue + mv -f -- "$dump" "$STAGE/${dump##*/}" +done size=$(du -sh "$STAGE" 2>/dev/null | awk '{print $1}') count=$(find "$STAGE" -type f | wc -l) log "stage ready: $count files, $size total" diff --git a/configs/restic/ana-docker/test_pre_backup.py b/configs/restic/ana-docker/test_pre_backup.py new file mode 100644 index 0000000..84a81fd --- /dev/null +++ b/configs/restic/ana-docker/test_pre_backup.py @@ -0,0 +1,52 @@ +import os +from pathlib import Path +import subprocess +import tempfile +import unittest + +SCRIPT=Path(__file__).with_name('pre-backup.sh') + +class BackupHookTests(unittest.TestCase): + def exercise(self, succeeds): + with tempfile.TemporaryDirectory(prefix='backup-hook-test-') as d: + root=Path(d); stage=root/'stage'; stage.mkdir(); binpath=root/'bin';binpath.mkdir() + previous=stage/'vaultwarden.pg_dump';previous.write_bytes(b'previous-good-backup') + docker=binpath/'docker';docker.write_text('#!/bin/sh\n[ "$1" = inspect ] && [ "$2" = vaultwarden ]\n');docker.chmod(0o755) + pg=binpath/'pg_dump';pg.write_text('#!/bin/sh\nprintf new-dump\nexit '+('0' if succeeds else '1')+'\n');pg.chmod(0o755) + env=dict(os.environ,PATH=str(binpath)+':'+os.environ['PATH'],RESTIC_STAGE_DIR=str(stage),RESTIC_DB_CREDS_FILE=str(root/'absent'),VW_PGPASS='fake',VW_PGHOST='fake',VW_PGUSER='fake',VW_PGDB='fake') + r=subprocess.run(['bash',str(SCRIPT)],env=env,capture_output=True,text=True) + if succeeds: + self.assertEqual(r.returncode,0,r.stdout+r.stderr) + self.assertEqual(previous.read_bytes(),b'new-dump') + else: + self.assertNotEqual(r.returncode,0,r.stdout+r.stderr) + self.assertEqual(previous.read_bytes(),b'previous-good-backup') + self.assertIn('required database dump(s) failed',r.stdout) + self.assertEqual(list(stage.glob('.pending.*')),[]) + + def test_failed_required_dump_preserves_previous_backup_and_cleans_scratch(self): + self.exercise(False) + + def test_success_publishes_new_dump_and_cleans_scratch(self): + self.exercise(True) + + def test_gitea_failure_removes_sql_scratch_and_exposes_error(self): + with tempfile.TemporaryDirectory(prefix='backup-gitea-test-') as d: + root=Path(d);stage=root/'stage';stage.mkdir();binpath=root/'bin';binpath.mkdir() + previous=stage/'gitea-dump.tar';previous.write_bytes(b'previous-good-archive') + docker=binpath/'docker' + docker.write_text('#!/bin/sh\nif [ "$1" = inspect ]; then [ "$2" = gitea ]; exit $?; fi\nshift 4\nexec "$@"\n') + docker.chmod(0o755) + gitea=binpath/'gitea' + gitea.write_text('#!/bin/sh\nwhile [ "$#" -gt 0 ]; do if [ "$1" = --tempdir ]; then shift; scratch=$1; fi; shift; done\nprintf %s "$scratch" > "$TEST_SCRATCH_PATH"\nprintf partial-sql > "$scratch/gitea-db.sql123"\necho simulated-export-failure >&2\nexit 9\n') + gitea.chmod(0o755) + path_record=root/'scratch-path' + env=dict(os.environ,PATH=str(binpath)+':'+os.environ['PATH'],RESTIC_STAGE_DIR=str(stage),RESTIC_DB_CREDS_FILE=str(root/'absent'),TEST_SCRATCH_PATH=str(path_record)) + r=subprocess.run(['bash',str(SCRIPT)],env=env,capture_output=True,text=True) + self.assertNotEqual(r.returncode,0) + self.assertIn('simulated-export-failure',r.stderr) + self.assertFalse(Path(path_record.read_text()).exists()) + self.assertEqual(previous.read_bytes(),b'previous-good-archive') + self.assertEqual(list(stage.glob('.pending.*')),[]) + +if __name__=='__main__':unittest.main() diff --git a/persistent-memory.d/2026-09-10-ana-docker-disk-repair.md b/persistent-memory.d/2026-09-10-ana-docker-disk-repair.md new file mode 100644 index 0000000..cb0018c --- /dev/null +++ b/persistent-memory.d/2026-09-10-ana-docker-disk-repair.md @@ -0,0 +1,25 @@ +# ana-docker disk pressure repaired + +Operator authorized all three: repair/verify backups, remove failed dump +residue, prune named builders with ongoing retention. Root went from 84% +(about 39 GiB available) to 51% (115 GiB available). + +Gitea failed dumps since June 4 came from root-only config history unreadable +to git. Five config history files MOVED, permissions preserved, outside dump +tree to `/opt/docker/backups/gitea-config-history`. Vaultwarden dump credentials +were stale; synchronized live connection to root:600 `/etc/restic/dbcreds.env` +and Vaultwarden item `ana-docker/restic-dbcreds.env`. + +Hook now fails required dump errors, preserves previous stage until successful +generation, validates Gitea tar, isolates/trap-cleans scratch. Three regression +tests pass. Fresh Restic snapshot `2ec5a37c` restored into isolated disposable +PostgreSQL: Vaultwarden 7 users, Gitea 9 users/93 repos. Then deleted 101 old +dump residues (47.31 GiB logical); manifest kept with config history. Gitea +temp now 8 KiB. Full app recovery and OpenWebUI SQLite consistency not claimed. + +Hourly named-builder prune targets 10 GiB unused-cache retention per active +builder; not a hard quota. Worldtree temporarily started, pruned, STOPPED again; +Skaldsong stays running and Soong stays stopped. Gitea HTTP 200, Vaultwarden +healthy, test DB removed, cache unit success/timer enabled. Canonical files: +`configs/restic/ana-docker/`, `configs/buildkit/ana-docker/`, deployment playbooks +`ana-docker-backup-repair.yaml` and `ana-docker-cache-budget.yaml`. diff --git a/persistent-memory.md b/persistent-memory.md index 2eb6d75..cd850af 100644 --- a/persistent-memory.md +++ b/persistent-memory.md @@ -287,6 +287,8 @@ preserved verbatim in `archival-memory.md` § Superseded in-flight snapshots._ ## Recent decisions +- `[2026-09-10]` **ana-docker disk pressure repaired: root 84% → 51%, 115 GiB free.** Gitea/Vaultwarden backups repaired and restored from Restic `2ec5a37c`; 101 stale dumps removed; hourly named-builder cache pruning installed. → `persistent-memory.d/2026-09-10-ana-docker-disk-repair.md` + - `[2026-09-09]` **Run 7 PURGED; pfi-gx10 declared an experimental/TRAINING box with no serving seat** — operator: *"gx10 is an experimental box, primarily for training … run 7 can be purged … no new run, we'll roll with run 6 for now."* ~139 GiB reclaimed across both boxes; the 315 MB adapter + provenance KEPT as the only non-reproducible piece. `Pfish-6` on ana-ml2 :8021 is the sole standing seat. - `[2026-09-09]` **Run 7 RETIRED; run 6 declared `Pfish-6` and is the standing seat** — NVFP4 quant on ana-ml2 :8021 AND gx10 :8098 at 262k ctx, gateway alias `trial` → `Pfish-6`, max-num-seqs 8→32 (2,170 tok/s at n=16, 3.2x the old ceiling). ⚠ ana-ml2 measured **4.1x FASTER than the GX10** on the same artifact — the reverse of the expectation. → `persistent-memory.d/2026-09-09-run7-retired-pfish6.md` - `[2026-09-09]` **The run-7 CSAM gate failure was a DETECTOR BUG** — HARD `child_term` matched the ADJECTIVE "minor"; operator-diagnosed, fixed `cc42d76` (nominal-use-only, selftest 24/24), retention wired so a hit can finally be adjudicated. ⚠ The lesson is mine: rigor downstream of an unexamined premise is not rigor. → `persistent-memory.d/2026-09-09-csam-detector-bug.md` diff --git a/playbooks/ana-docker-backup-repair.yaml b/playbooks/ana-docker-backup-repair.yaml new file mode 100644 index 0000000..61809de --- /dev/null +++ b/playbooks/ana-docker-backup-repair.yaml @@ -0,0 +1,23 @@ +steps: + - name: Preserve the previous hook + sudo: true + shell: cp -p /etc/restic/pre-backup.sh /etc/restic/pre-backup.sh.before-repair-20260910 + creates: /etc/restic/pre-backup.sh.before-repair-20260910 + - name: Install corrected canonical hook + sudo: true + upload: + src: configs/restic/ana-docker/pre-backup.sh + dest: /etc/restic/pre-backup.sh + mode: '0700' + - name: Validate and produce fresh staged database backups + sudo: true + shell: bash -n /etc/restic/pre-backup.sh && /etc/restic/pre-backup.sh +verify: + - name: Verify Gitea and Vaultwarden archive structure + sudo: true + shell: | + set -euo pipefail + pg_restore --list /var/lib/restic/stage/vaultwarden.pg_dump >/dev/null + tar -tf /var/lib/restic/stage/gitea-dump.tar | grep -x 'gitea-db.sql' + ls -lh /var/lib/restic/stage/gitea-dump.tar /var/lib/restic/stage/vaultwarden.pg_dump + test -z "$(docker exec gitea find /tmp -maxdepth 1 -type d -name 'gitea-backup.*')" diff --git a/playbooks/ana-docker-cache-budget.yaml b/playbooks/ana-docker-cache-budget.yaml new file mode 100644 index 0000000..1594ea3 --- /dev/null +++ b/playbooks/ana-docker-cache-budget.yaml @@ -0,0 +1,25 @@ +steps: + - name: Install cache budget enforcer + sudo: true + upload: + src: configs/buildkit/ana-docker/prune.sh + dest: /usr/local/sbin/pfi-buildkit-cache-prune + mode: '0755' + - name: Install cache budget service + sudo: true + upload: + src: configs/buildkit/ana-docker/buildkit-cache-prune.service + dest: /etc/systemd/system/buildkit-cache-prune.service + mode: '0644' + - name: Install cache budget timer + sudo: true + upload: + src: configs/buildkit/ana-docker/buildkit-cache-prune.timer + dest: /etc/systemd/system/buildkit-cache-prune.timer + mode: '0644' + - name: Enable hourly budget enforcement + sudo: true + shell: systemctl daemon-reload && systemctl enable --now buildkit-cache-prune.timer +verify: + - name: Verify scheduled cache enforcement + shell: systemctl is-active buildkit-cache-prune.timer && systemctl list-timers buildkit-cache-prune.timer --no-pager