diff --git a/playbooks/corviduo-dev-docker-default-ulimits.yaml b/playbooks/corviduo-dev-docker-default-ulimits.yaml new file mode 100644 index 0000000..19d222f --- /dev/null +++ b/playbooks/corviduo-dev-docker-default-ulimits.yaml @@ -0,0 +1,113 @@ +# Set a host-wide docker nofile floor on corviduo-dev. +# +# WHY: Worldtree #401 — a slow fd accrual in worldtree-personal hit the 1024 +# soft nofile ceiling and converted into a hard deadlock. Raising the floor +# turns any recurrence into observable degradation instead of a wedge. +# Operator authorized the raise 2026-08-17 (relayed via worldtree-dev, +# thread 01M08QQ655XD6VKEV7MA9GX0NS); sizing 65536 agreed with worldtree-dev. +# +# WHY THE DAEMON LAYER: /opt/worldtree-*/compose.yaml on this host is written +# by the team's CI `deploy` identity, so a host-side compose edit reverts on +# the next deploy. Daemon config is infra-ops-owned, survives every CI deploy, +# and covers all containers on the box — not just worldtree. worldtree-dev +# ALSO shipped an explicit compose-level pin (e41b139) as the belt to this +# braces; the two are deliberately redundant. +# +# ACTIVATION — READ THIS BEFORE ASSUMING THE FLOOR IS LIVE. +# `default-ulimits` is NOT in dockerd's SIGHUP-reloadable set. Measured on +# Docker 29.4.3 (corviduo-dev, 2026-08-17): after `systemctl reload docker` the +# daemon's own "Reloaded configuration" log line enumerates the live config and +# `default-ulimits` is ABSENT from it, and a freshly created container still +# reports `ulimit -n` = 1024. The reload step below is therefore harmless but +# insufficient on its own. +# +# So this playbook STAGES the floor; it does not activate it. Activation needs a +# full `systemctl restart docker`, which with live-restore unset BOUNCES EVERY +# CONTAINER on the host (13 of them here, including all three worldtree +# instances) — deliberately not taken here, because #401 is not urgent at fd +# ~100 and worldtree-dev's explicit compose-level pin (e41b139) already covers +# the worldtree services on their next recreate. Expect verify step 3 to FAIL +# until a dockerd restart or a host reboot happens. +# +# If you want it live without a bounce, add `"live-restore": true` to +# daemon.json FIRST (that one IS reloadable), then restart — containers survive +# the daemon going away. That is a separate change with its own blast radius; +# it was not in scope for #401. +# +# FOOT-GUN: an invalid daemon.json does not break a reload (dockerd logs and +# keeps the old config) but WILL break the next dockerd *start*. The playbook +# validates the JSON before reloading and refuses to proceed otherwise. + +vars: + nofile: "65536" + daemon_json: /etc/docker/daemon.json + +steps: + - name: Back up an existing daemon.json (no-op when absent) + sudo: true + shell: | + if [ -f {{ daemon_json }} ] && [ ! -f {{ daemon_json }}.bak-401-ulimits ]; then + cp -a {{ daemon_json }} {{ daemon_json }}.bak-401-ulimits + echo backed-up + else + echo no-backup-needed + fi + changed_when: "false" + + - name: Write daemon.json with the nofile floor + sudo: true + shell: | + set -e + tmp=$(mktemp) + if [ -f {{ daemon_json }} ]; then + python3 - "$tmp" <<'PY' + import json, sys + p = "/etc/docker/daemon.json" + cfg = json.load(open(p)) + cfg.setdefault("default-ulimits", {})["nofile"] = { + "Name": "nofile", "Soft": 65536, "Hard": 65536} + json.dump(cfg, open(sys.argv[1], "w"), indent=2) + PY + else + cat > "$tmp" <<'JSON' + { + "default-ulimits": { + "nofile": { "Name": "nofile", "Soft": 65536, "Hard": 65536 } + } + } + JSON + fi + python3 -m json.tool "$tmp" > /dev/null + install -m 0644 -o root -g root "$tmp" {{ daemon_json }} + rm -f "$tmp" + # Skip entirely when the floor is already recorded at the right size. + when: "! sudo python3 -c \"import json;c=json.load(open('{{ daemon_json }}'));u=c.get('default-ulimits',{}).get('nofile',{});raise SystemExit(0 if u.get('Soft')=={{ nofile }} and u.get('Hard')=={{ nofile }} else 1)\" 2>/dev/null" + + - name: Reload dockerd (SIGHUP — does NOT restart containers) + sudo: true + shell: systemctl reload docker + when: "! sudo docker run --rm --entrypoint sh busybox -c 'ulimit -n' 2>/dev/null | grep -qx '{{ nofile }}'" + +verify: + - name: daemon.json is valid JSON + sudo: true + shell: python3 -m json.tool {{ daemon_json }} > /dev/null + changed_when: "false" + + - name: daemon.json records the nofile floor at the agreed size + sudo: true + shell: | + python3 -c "import json;u=json.load(open('{{ daemon_json }}'))['default-ulimits']['nofile'];assert u['Soft']=={{ nofile }} and u['Hard']=={{ nofile }}, u" + changed_when: "false" + + - name: A NEWLY created container actually gets the floor (the real proof) + sudo: true + shell: | + out=$(docker run --rm --entrypoint sh busybox -c 'ulimit -n') + [ "$out" = "{{ nofile }}" ] || { echo "got $out want {{ nofile }}"; exit 1; } + changed_when: "false" + + - name: dockerd is still running and containers were not bounced + sudo: true + shell: systemctl is-active --quiet docker && test "$(docker ps -q | wc -l)" -ge 13 + changed_when: "false"