vh
6e203dcb99
resticprofile schedule copies env-file values into the generated units, which
are 0644, so RESTIC_REPOSITORY (the rest-server basic-auth password included)
was readable by every local user on every restic host.
New playbooks/restic-repository-file.yaml:
- derives /etc/restic/repository (root 0400) from restic.env;
- uploads the profile switched to repository-file, but only when the live
profile's sha matches the repo copy it was edited from (drift guard);
- checks the repository is reachable through the new profile (cat config);
- regenerates the units and verifies they exist and contain no rest:http.
Applied to ana-docker, fv-ml1 (configs/restic/ana-ml2), esh-docker-vm,
esh-vm-db, irv-ml1, nh3-dev and nh3-docker. An independent check across all
eight restic hosts (these seven plus esh-ml1) found 0 leaking units. nh3-docker's
scheduled unit ran a real backup afterwards (snapshot a29b889d). Each host's
URL and passphrase are vaulted as <host>/etc/restic/{repository,password}.
restic.env is kept (root 0600) because the per-host READMEs and the freshness
probe source it. A rotation must update the vault, restic.env and repository.
vm-esh-nas has no infra-ops account. Its in-place migration script is staged
for Prime to run with sudo, and its repo profile is pre-edited to match.
Also mirrors augaman-dev's 401df2d (compose header only). The config hash on
esh-ml1 is unchanged.
144 lines
5.9 KiB
YAML
144 lines
5.9 KiB
YAML
# resticprofile config for irv-ml1.
|
|
#
|
|
# **Bare metal, remote site.** Not protected by any Proxmox vzdump — this
|
|
# is the sole backup for this host. Reachable only via WireGuard
|
|
# (10.100.79.3 is the WG tunnel IP). Backups target rest-server-nh3 at
|
|
# 10.100.50.50 because that's the closest endpoint through the WG tunnel;
|
|
# cross-site to ana-docker would double-hop the WAN.
|
|
#
|
|
# Notable exclusions:
|
|
# - /storetank/* — ~1.8 TB of model weights (ComfyUI checkpoints, HF hub,
|
|
# llama.cpp GGUFs, etc.). All regenerable from upstream. Do NOT add.
|
|
# - /worktank/* — *selectively* included. Docker stacks on this host now
|
|
# place user state under /worktank/<stack>/ (comfyui, parakeet,
|
|
# cosyvoice). Sources are explicit per-subtree; bulk/scratch/run
|
|
# dirs stay out. Rule of thumb: precious human-generated data in,
|
|
# downloadable weights + regenerable outputs + disposable runtime
|
|
# out.
|
|
# - Native AI toolchain under /opt/{ComfyUI,sillytavern,stablediffusion,
|
|
# llama.cpp,kokoro,...} — git clones / pip envs, regenerable from
|
|
# source. Only /opt/docker is backed up (compose files + configs,
|
|
# once Docker is installed following the fleet convention).
|
|
#
|
|
# DB: arbo (catalog engine) keeps a SQLite gallery DB in the arbo_db volume;
|
|
# run-before checkpoints it (gallery.sqlite.bak) so the daily volume backup is consistent.
|
|
|
|
version: "1"
|
|
|
|
global:
|
|
priority: low
|
|
ionice: true
|
|
ionice-class: 2
|
|
ionice-level: 7
|
|
min-memory: 100
|
|
|
|
default:
|
|
repository-file: /etc/restic/repository # URL incl. rest-server creds (root 0400).
|
|
# NOT env-file: `resticprofile schedule` copies env-file values into the
|
|
# world-readable systemd units (docs/runbooks/backups.md, Known gaps).
|
|
env:
|
|
RESTIC_PASSWORD_FILE: /etc/restic/password
|
|
initialize: false # repo created manually by `restic init`
|
|
lock: /var/lock/restic-irv-ml1.lock
|
|
|
|
backup:
|
|
verbose: 1
|
|
run-before:
|
|
- /etc/restic/arbo-checkpoint.sh # consistent SQLite snapshot of the arbo gallery DB
|
|
run-after:
|
|
- date +%s > /var/lib/restic/last-success
|
|
source:
|
|
- /opt/docker # compose files + bind-mount configs
|
|
- /etc # host config — systemd, chrony, apparmor, ssh, wg
|
|
- /root # root's ad-hoc scripts, shell history, ssh keys
|
|
- /home/lkraven # primary admin home dir — dotfiles, dev scripts, small configs
|
|
- /var/lib/docker/volumes # named volumes from dockerized stacks
|
|
# Selective /worktank inclusions (the rest of /worktank stays out):
|
|
- /worktank/comfyui/basedir/user # ComfyUI workflows + settings
|
|
- /worktank/comfyui/basedir/custom_nodes # hand-installed ComfyUI extensions
|
|
- /worktank/comfyui/basedir/input # user-uploaded source images for workflows
|
|
- /worktank/cosyvoice/voices # cloned speaker profiles — reproducing them needs the original reference audio
|
|
# Deliberately NOT in source (see "What's not backed up" in README):
|
|
# /home/llmuser # ~380 GB of model weights / training outputs
|
|
# # (contents belong on /storetank; regenerable from HF)
|
|
# /home/sduser # Stable Diffusion user state (~2.6 GB); scope-check
|
|
# # before adding — likely regenerable
|
|
# /worktank/comfyui/basedir/{models,output,temp} # bulk + regenerable
|
|
# /worktank/comfyui/run # ~8 GB disposable venv
|
|
# /worktank/parakeet/models # HF cache, regenerable
|
|
# /worktank/cosyvoice/{input,output} # scratch
|
|
exclude:
|
|
# Explicit protection: even if source is ever changed to /home, skip
|
|
# llmuser entirely — the only reason it's 380 GB is AI artifacts that
|
|
# belong on /storetank or HF hub, not in a config-backup repo.
|
|
- /home/llmuser
|
|
- /home/sduser
|
|
# Defense-in-depth: even if `source` is ever broadened to /worktank
|
|
# wholesale, these must never enter the repo.
|
|
- /worktank/comfyui/basedir/models
|
|
- /worktank/comfyui/basedir/output
|
|
- /worktank/comfyui/basedir/temp
|
|
- /worktank/comfyui/run
|
|
- /worktank/parakeet/models
|
|
- /worktank/cosyvoice/input
|
|
- /worktank/cosyvoice/output
|
|
# Docker internals we never want in a backup
|
|
- /var/lib/docker/volumes/backingFsBlockDev
|
|
- /var/lib/docker/volumes/metadata.db
|
|
# Ephemeral / regenerable junk
|
|
- /opt/docker/compose/*/logs
|
|
- "**/*.log"
|
|
- "**/*.log.*"
|
|
- "**/*.pid"
|
|
# User-cache / toolchain noise under home dirs (saves ~5.8 GB on /home/lkraven alone)
|
|
- /home/*/.cache
|
|
- /home/*/.npm
|
|
- /home/*/.local/share/Trash
|
|
- /home/*/.cargo/registry
|
|
- /home/*/.cargo/git
|
|
- /home/*/.rustup
|
|
- /home/*/huggingface
|
|
- /home/*/.cache/huggingface
|
|
- /home/*/conda
|
|
- /home/*/anaconda3
|
|
- /home/*/miniconda3
|
|
- /home/*/.conda
|
|
- /home/*/micromamba
|
|
# Common Python virtualenv noise
|
|
- "**/__pycache__"
|
|
- "**/.venv"
|
|
- "**/venv"
|
|
- "**/env"
|
|
- "**/.pytest_cache"
|
|
# Root's noise
|
|
- /root/.cache
|
|
- /root/.local/share/Trash
|
|
- /root/.npm
|
|
- /root/.python_history
|
|
tag:
|
|
- host:irv-ml1
|
|
- site:irv
|
|
- fleet:pfi
|
|
schedule: "*-*-* 01:00:00"
|
|
schedule-permission: system
|
|
schedule-log: /var/log/restic-backup.log
|
|
|
|
forget:
|
|
keep-daily: 7
|
|
keep-weekly: 4
|
|
keep-monthly: 12
|
|
keep-yearly: 3
|
|
# NOTE: no `prune: true` — rest-server runs with --append-only.
|
|
# See README.md "Prune ceremony".
|
|
tag:
|
|
- host:irv-ml1
|
|
# Schedule removed at deploy: forget against --append-only rest-server
|
|
# always fails (delete ops blocked). Run manually during the prune
|
|
# ceremony when --append-only is temporarily off.
|
|
|
|
check:
|
|
read-data-subset: 10%
|
|
schedule: "Sun *-*-* 05:00:00"
|
|
schedule-permission: system
|
|
schedule-log: /var/log/restic-check.log
|