# normalize-docker-tree — put every host's docker deploy tree on one # ownership scheme: root:docker, setgid, group-writable. # # Operator ruling 2026-09-14: root:docker, not a personal username and not a # new admin account. Rationale — `lkraven` is one of three names the operator # uses, so baking it into shared infrastructure guarantees a stale owner the # day he standardises on another; and a dedicated deploy account buys nothing # the existing `docker` group doesn't, since that group already exists on # every host and already contains exactly lkraven + infra-ops. # # ⚠ This does NOT create privilege separation. `docker` group membership is # root-equivalent (bind-mount / into a container). Both members already hold # NOPASSWD sudo, so today that costs nothing — but if a third party ever needs # to deploy one stack WITHOUT root, this scheme is not the tool for it. Stand # up a dedicated `deploy` group at that point. # # ⚠⚠ WHY THIS IS NOT A RECURSIVE CHMOD. # A blanket `chmod -R g+w` over these trees produces a DELAYED failure, which # is strictly worse than a loud one: the service keeps running on its current # file handles and dies at its next restart, weeks later, for a reason nobody # connects to this change. Measured on the fleet 2026-09-14: # * 3 x acme.json (traefik-ana has two, traefik-esh one), mode 600. # Traefik REFUSES TO START if acme.json is not 600. # * /opt/docker/conf/asset-engine/ssh/id_ed25519, mode 600. ssh refuses a # group-accessible private key. # * 53 executable files (entrypoints, redeploy scripts, traefik.yml). A # NUMERIC chmod strips +x; symbolic g+w does not. Modes here are symbolic # for exactly that reason — do not "simplify" them to octal. # The rule that keeps this safe: only widen a file that is ALREADY # world-readable. If the world can read it, group-write leaks nothing new. # Anything at 0600/0400 is deliberate and is left alone. # # Usage: # scripts/elway --playbook playbooks/normalize-docker-tree.yaml --dry-run # scripts/elway --playbook playbooks/normalize-docker-tree.yaml # # Host order (ascending blast radius). ana-docker LAST — it is the only host # carrying traefik, the asset-engine ssh key and the gitea CI runner at once: # nh3-dev(2) -> nh3-docker(7) -> esh-docker-vm(18) -> irv-ml1(36) -> ana-docker(38) # # Containers do NOT need bouncing. These are inode metadata changes; a running # process does not re-read them, and .env is consulted only at `compose up`. vars: # Pruned from the walk entirely: vendored/CI/VCS trees where recursing is # pointless churn and the files are not ours to re-own. prune_dirs: ".venv .git host-workspace node_modules" # Never touched by name, in ADDITION to the mode rule. The mode rule alone # (leave 0600/0400) is not sufficient: a key file that is ALREADY 0644 would # otherwise be picked up by the "already world-readable" widening, which is # true but beside the point — we do not want to be the last writer on a # credential either way. (Caught 2026-09-14 on esh-docker-vm: # /opt/docker/conf/calibre-web/.key was 0644 and got widened to 0664 before # this guard existed. Its world-readability is a pre-existing finding worth # fixing on its own; it is not something this playbook should compound.) protect_names: "acme.json *.key *.pem *.pfx id_*" steps: - name: Preflight — docker group exists and holds the deploy identities shell: | getent group docker || { echo "NO docker GROUP — refusing"; exit 1; } for u in lkraven infra-ops; do id -nG "$u" 2>/dev/null | tr ' ' '\n' | grep -qx docker \ || echo " WARN: $u is not in the docker group" done changed_when: "false" - name: Snapshot current state (rollback reference + before/after evidence) shell: | out=/tmp/docker-tree-pre-normalize.txt : | sudo tee "$out" >/dev/null for d in /opt/docker /opt/docker/compose /opt/docker/conf; do [ -e "$d" ] && sudo stat -c '%a %U:%G %n' "$d" | sudo tee -a "$out" >/dev/null done sudo find /opt/docker/compose /opt/docker/conf -maxdepth 2 -name '.env' \ -printf '%m %u:%g %p\n' 2>/dev/null | sudo tee -a "$out" >/dev/null # Recorded so the exec-bit verify can compare before/after instead of # asserting "every .sh is executable" — which was never true (plenty of # .sh files here are sourced, not run) and produced a FALSE FAILED on # irv-ml1 2026-09-14. A false failure in automation output is the exact # defect this repo fixed in upgrade-docker-ce.yaml the same day. if [ ! -f /tmp/docker-tree-execcount.txt ]; then sudo find /opt/docker/compose /opt/docker/conf -type f -perm /0100 2>/dev/null \ | wc -l | sudo tee /tmp/docker-tree-execcount.txt >/dev/null fi echo "EXECCOUNT_BASELINE=$(cat /tmp/docker-tree-execcount.txt)" cat "$out" sudo: true changed_when: "false" - name: Record the permission-sensitive files this run must NOT touch # Evidence, not action. If this list ever grows a new shape, the exclusion # rules below need revisiting before the next host. shell: | sudo find /opt/docker/compose /opt/docker/conf \ \( -name .venv -o -name .git -o -name host-workspace -o -name node_modules \) -prune -o \ -type f \( -name 'acme.json' -o -name '*.key' -o -name '*.pem' -o -name '*.pfx' \ -o -name 'id_*' -o -perm 0600 -o -perm 0400 \) \ -printf ' PROTECTED %m %u:%g %p\n' 2>/dev/null | head -40 echo " (all of the above are left untouched by this playbook)" sudo: true changed_when: "false" - name: Set group ownership to docker across the deploy tree # chgrp only — no mode change. A 0600 file has no group bits, so this is a # no-op for every protected file above while still making the tree # collectively owned. shell: | for d in /opt/docker/compose /opt/docker/conf; do [ -d "$d" ] || continue sudo find "$d" \ \( -name .venv -o -name .git -o -name host-workspace -o -name node_modules \) -prune -o \ \( -name 'acme.json' -o -name '*.key' -o -name '*.pem' -o -name '*.pfx' -o -name 'id_*' \) -prune -o \ -print0 2>/dev/null | sudo xargs -0 -r chgrp docker done echo " group set to docker" sudo: true - name: Set root ownership + setgid + group-write on DIRECTORIES # setgid (g+s) is the whole point: every file created here afterwards # inherits the docker group, so this fix does not decay. shell: | for d in /opt/docker/compose /opt/docker/conf; do [ -d "$d" ] || continue sudo find "$d" \ \( -name .venv -o -name .git -o -name host-workspace -o -name node_modules \) -prune -o \ -type d -print0 2>/dev/null | sudo xargs -0 -r chown root sudo find "$d" \ \( -name .venv -o -name .git -o -name host-workspace -o -name node_modules \) -prune -o \ -type d -print0 2>/dev/null | sudo xargs -0 -r chmod g+ws done echo " directories: root:docker, setgid, group-writable" sudo: true - name: Group-write ONLY files that are already world-readable # SYMBOLIC mode — a numeric chmod would strip +x from the 53 executable # files in these trees. -perm -o=r is the safety rule: widening a file the # world can already read adds no exposure. shell: | for d in /opt/docker/compose /opt/docker/conf; do [ -d "$d" ] || continue sudo find "$d" \ \( -name .venv -o -name .git -o -name host-workspace -o -name node_modules \) -prune -o \ \( -name 'acme.json' -o -name '*.key' -o -name '*.pem' -o -name '*.pfx' -o -name 'id_*' \) -prune -o \ -type f -perm -o=r -print0 2>/dev/null | sudo xargs -0 -r chmod g+w done echo " world-readable files: group-writable (exec bits preserved)" sudo: true - name: Normalise stack .env to root:docker 0640 # The file that started this. Fleet-wide these were split between 0600 # (readable by ONE of the two deploy identities, varying by file — this is # what false-FAILED beszel during the docker-ce upgrade) and 0644 # (world-readable secrets). 0640 tightens the latter and opens the former # to exactly the two deployers. shell: | n=0 for f in $(sudo find /opt/docker/compose -maxdepth 2 -name '.env' 2>/dev/null); do sudo chown root:docker "$f" && sudo chmod 0640 "$f" && n=$((n+1)) done echo " normalised $n stack .env file(s) to root:docker 0640" sudo: true - name: Fix the /opt/docker parent (kills 0777 where present) shell: | before=$(stat -c '%a %U:%G' /opt/docker) sudo chown root:docker /opt/docker sudo chmod 2775 /opt/docker echo " /opt/docker: $before -> $(stat -c '%a %U:%G' /opt/docker)" sudo: true verify: - name: /opt/docker is root:docker, setgid, not world-writable shell: | stat -c '%a %U:%G' /opt/docker | grep -qE '^2775 root:docker$' - name: compose tree directories are setgid + group-writable shell: | [ -d /opt/docker/compose ] || exit 0 bad=$(sudo find /opt/docker/compose \ \( -name .venv -o -name .git -o -name host-workspace -o -name node_modules \) -prune -o \ -type d ! -perm -g+w -print 2>/dev/null | head -3) [ -z "$bad" ] || { echo "not group-writable: $bad"; exit 1; } sudo: true - name: every protected 0600 file kept its mode (traefik acme.json, ssh keys) shell: | bad=$(sudo find /opt/docker/conf -type f \ \( -name 'acme.json' -o -name 'id_ed25519' -o -name 'id_rsa' \) \ ! -perm 0600 -print 2>/dev/null | head -3) [ -z "$bad" ] || { echo "MODE WIDENED on protected file: $bad"; exit 1; } sudo: true - name: no stack .env is world-readable any more shell: | bad=$(sudo find /opt/docker/compose -maxdepth 2 -name '.env' -perm -o=r -print 2>/dev/null | head -3) [ -z "$bad" ] || { echo "world-readable .env: $bad"; exit 1; } sudo: true - name: executable-file count is unchanged (no exec bit was stripped) # Compares against the baseline captured before the first run on this host. # NOT "every .sh is executable" — that property was never true and asserting # it manufactures a failure on a tree full of sourced scripts. shell: | [ -f /tmp/docker-tree-execcount.txt ] || exit 0 before=$(cat /tmp/docker-tree-execcount.txt) after=$(sudo find /opt/docker/compose /opt/docker/conf -type f -perm /0100 2>/dev/null | wc -l) [ "$before" = "$after" ] || { echo "exec count $before -> $after"; exit 1; } echo " exec-file count unchanged at $after" sudo: true