From 7f3f265384698bbe15a8945c2469b22831d14ea2 Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Mon, 17 Aug 2026 15:58:00 -0700 Subject: [PATCH] feat(corviduo-dev): stage a host-wide docker nofile floor (65536) for WT #401 Worldtree #401: a slow fd accrual in worldtree-personal hit the 1024 soft nofile ceiling and converted into a hard deadlock. Operator authorized the raise 2026-08-17 (relayed via worldtree-dev); sizing 65536 agreed. Applied at the daemon layer rather than compose because /opt/worldtree-*/ compose.yaml on corviduo-dev is written by the team CI deploy identity -- a host-side compose edit reverts on the next deploy and would leave a false 'raised' record. Daemon config is infra-ops-owned and covers all 13 containers on the box. worldtree-dev shipped a redundant compose-level pin (e41b139) as the belt to this braces. daemon.json is written and valid, but the floor is STAGED, NOT ACTIVE: default-ulimits is not in dockerd's SIGHUP-reloadable set. Measured on 29.4.3 -- the post-reload 'Reloaded configuration' log enumerates the live config without default-ulimits, and a fresh container still reports ulimit -n 1024. Activation needs a full dockerd restart, which bounces every container; not taken, since #401 is not urgent at fd ~100 and the compose pin already covers worldtree. The playbook documents this and its verify step 3 fails by design until a restart happens. --- .../corviduo-dev-docker-default-ulimits.yaml | 113 ++++++++++++++++++ 1 file changed, 113 insertions(+) create mode 100644 playbooks/corviduo-dev-docker-default-ulimits.yaml diff --git a/playbooks/corviduo-dev-docker-default-ulimits.yaml b/playbooks/corviduo-dev-docker-default-ulimits.yaml new file mode 100644 index 0000000..19d222f --- /dev/null +++ b/playbooks/corviduo-dev-docker-default-ulimits.yaml @@ -0,0 +1,113 @@ +# Set a host-wide docker nofile floor on corviduo-dev. +# +# WHY: Worldtree #401 — a slow fd accrual in worldtree-personal hit the 1024 +# soft nofile ceiling and converted into a hard deadlock. Raising the floor +# turns any recurrence into observable degradation instead of a wedge. +# Operator authorized the raise 2026-08-17 (relayed via worldtree-dev, +# thread 01M08QQ655XD6VKEV7MA9GX0NS); sizing 65536 agreed with worldtree-dev. +# +# WHY THE DAEMON LAYER: /opt/worldtree-*/compose.yaml on this host is written +# by the team's CI `deploy` identity, so a host-side compose edit reverts on +# the next deploy. Daemon config is infra-ops-owned, survives every CI deploy, +# and covers all containers on the box — not just worldtree. worldtree-dev +# ALSO shipped an explicit compose-level pin (e41b139) as the belt to this +# braces; the two are deliberately redundant. +# +# ACTIVATION — READ THIS BEFORE ASSUMING THE FLOOR IS LIVE. +# `default-ulimits` is NOT in dockerd's SIGHUP-reloadable set. Measured on +# Docker 29.4.3 (corviduo-dev, 2026-08-17): after `systemctl reload docker` the +# daemon's own "Reloaded configuration" log line enumerates the live config and +# `default-ulimits` is ABSENT from it, and a freshly created container still +# reports `ulimit -n` = 1024. The reload step below is therefore harmless but +# insufficient on its own. +# +# So this playbook STAGES the floor; it does not activate it. Activation needs a +# full `systemctl restart docker`, which with live-restore unset BOUNCES EVERY +# CONTAINER on the host (13 of them here, including all three worldtree +# instances) — deliberately not taken here, because #401 is not urgent at fd +# ~100 and worldtree-dev's explicit compose-level pin (e41b139) already covers +# the worldtree services on their next recreate. Expect verify step 3 to FAIL +# until a dockerd restart or a host reboot happens. +# +# If you want it live without a bounce, add `"live-restore": true` to +# daemon.json FIRST (that one IS reloadable), then restart — containers survive +# the daemon going away. That is a separate change with its own blast radius; +# it was not in scope for #401. +# +# FOOT-GUN: an invalid daemon.json does not break a reload (dockerd logs and +# keeps the old config) but WILL break the next dockerd *start*. The playbook +# validates the JSON before reloading and refuses to proceed otherwise. + +vars: + nofile: "65536" + daemon_json: /etc/docker/daemon.json + +steps: + - name: Back up an existing daemon.json (no-op when absent) + sudo: true + shell: | + if [ -f {{ daemon_json }} ] && [ ! -f {{ daemon_json }}.bak-401-ulimits ]; then + cp -a {{ daemon_json }} {{ daemon_json }}.bak-401-ulimits + echo backed-up + else + echo no-backup-needed + fi + changed_when: "false" + + - name: Write daemon.json with the nofile floor + sudo: true + shell: | + set -e + tmp=$(mktemp) + if [ -f {{ daemon_json }} ]; then + python3 - "$tmp" <<'PY' + import json, sys + p = "/etc/docker/daemon.json" + cfg = json.load(open(p)) + cfg.setdefault("default-ulimits", {})["nofile"] = { + "Name": "nofile", "Soft": 65536, "Hard": 65536} + json.dump(cfg, open(sys.argv[1], "w"), indent=2) + PY + else + cat > "$tmp" <<'JSON' + { + "default-ulimits": { + "nofile": { "Name": "nofile", "Soft": 65536, "Hard": 65536 } + } + } + JSON + fi + python3 -m json.tool "$tmp" > /dev/null + install -m 0644 -o root -g root "$tmp" {{ daemon_json }} + rm -f "$tmp" + # Skip entirely when the floor is already recorded at the right size. + when: "! sudo python3 -c \"import json;c=json.load(open('{{ daemon_json }}'));u=c.get('default-ulimits',{}).get('nofile',{});raise SystemExit(0 if u.get('Soft')=={{ nofile }} and u.get('Hard')=={{ nofile }} else 1)\" 2>/dev/null" + + - name: Reload dockerd (SIGHUP — does NOT restart containers) + sudo: true + shell: systemctl reload docker + when: "! sudo docker run --rm --entrypoint sh busybox -c 'ulimit -n' 2>/dev/null | grep -qx '{{ nofile }}'" + +verify: + - name: daemon.json is valid JSON + sudo: true + shell: python3 -m json.tool {{ daemon_json }} > /dev/null + changed_when: "false" + + - name: daemon.json records the nofile floor at the agreed size + sudo: true + shell: | + python3 -c "import json;u=json.load(open('{{ daemon_json }}'))['default-ulimits']['nofile'];assert u['Soft']=={{ nofile }} and u['Hard']=={{ nofile }}, u" + changed_when: "false" + + - name: A NEWLY created container actually gets the floor (the real proof) + sudo: true + shell: | + out=$(docker run --rm --entrypoint sh busybox -c 'ulimit -n') + [ "$out" = "{{ nofile }}" ] || { echo "got $out want {{ nofile }}"; exit 1; } + changed_when: "false" + + - name: dockerd is still running and containers were not bounced + sudo: true + shell: systemctl is-active --quiet docker && test "$(docker ps -q | wc -l)" -ge 13 + changed_when: "false"