esh-pfi-infrastructure/playbooks/corviduo-dev-docker-default-ulimits.yaml

# Set a host-wide docker nofile floor on corviduo-dev.
#
# WHY: Worldtree #401 — a slow fd accrual in worldtree-personal hit the 1024
# soft nofile ceiling and converted into a hard deadlock. Raising the floor
# turns any recurrence into observable degradation instead of a wedge.
# Operator authorized the raise 2026-08-17 (relayed via worldtree-dev,
# thread 01M08QQ655XD6VKEV7MA9GX0NS); sizing 65536 agreed with worldtree-dev.
#
# WHY THE DAEMON LAYER: /opt/worldtree-*/compose.yaml on this host is written
# by the team's CI `deploy` identity, so a host-side compose edit reverts on
# the next deploy. Daemon config is infra-ops-owned, survives every CI deploy,
# and covers all containers on the box — not just worldtree. worldtree-dev
# ALSO shipped an explicit compose-level pin (e41b139) as the belt to this
# braces; the two are deliberately redundant.
#
# ACTIVATION — READ THIS BEFORE ASSUMING THE FLOOR IS LIVE.
# `default-ulimits` is NOT in dockerd's SIGHUP-reloadable set. Measured on
# Docker 29.4.3 (corviduo-dev, 2026-08-17): after `systemctl reload docker` the
# daemon's own "Reloaded configuration" log line enumerates the live config and
# `default-ulimits` is ABSENT from it, and a freshly created container still
# reports `ulimit -n` = 1024. The reload step below is therefore harmless but
# insufficient on its own.
#
# So this playbook STAGES the floor; it does not activate it. Activation needs a
# full `systemctl restart docker`, which with live-restore unset BOUNCES EVERY
# CONTAINER on the host (13 of them here, including all three worldtree
# instances) — deliberately not taken here, because #401 is not urgent at fd
# ~100 and worldtree-dev's explicit compose-level pin (e41b139) already covers
# the worldtree services on their next recreate. Expect verify step 3 to FAIL
# until a dockerd restart or a host reboot happens.
#
# If you want it live without a bounce, add `"live-restore": true` to
# daemon.json FIRST (that one IS reloadable), then restart — containers survive
# the daemon going away. That is a separate change with its own blast radius;
# it was not in scope for #401.
#
# FOOT-GUN: an invalid daemon.json does not break a reload (dockerd logs and
# keeps the old config) but WILL break the next dockerd *start*. The playbook
# validates the JSON before reloading and refuses to proceed otherwise.

vars:
  nofile: "65536"
  daemon_json: /etc/docker/daemon.json

steps:
  - name: Back up an existing daemon.json (no-op when absent)
    sudo: true
    shell: |
      if [ -f {{ daemon_json }} ] && [ ! -f {{ daemon_json }}.bak-401-ulimits ]; then
        cp -a {{ daemon_json }} {{ daemon_json }}.bak-401-ulimits
        echo backed-up
      else
        echo no-backup-needed
      fi
    changed_when: "false"

  - name: Write daemon.json with the nofile floor
    sudo: true
    shell: |
      set -e
      tmp=$(mktemp)
      if [ -f {{ daemon_json }} ]; then
        python3 - "$tmp" <<'PY'
      import json, sys
      p = "/etc/docker/daemon.json"
      cfg = json.load(open(p))
      cfg.setdefault("default-ulimits", {})["nofile"] = {
          "Name": "nofile", "Soft": 65536, "Hard": 65536}
      json.dump(cfg, open(sys.argv[1], "w"), indent=2)
      PY
      else
        cat > "$tmp" <<'JSON'
      {
        "default-ulimits": {
          "nofile": { "Name": "nofile", "Soft": 65536, "Hard": 65536 }
        }
      }
      JSON
      fi
      python3 -m json.tool "$tmp" > /dev/null
      install -m 0644 -o root -g root "$tmp" {{ daemon_json }}
      rm -f "$tmp"
    # Skip entirely when the floor is already recorded at the right size.
    when: "! sudo python3 -c \"import json;c=json.load(open('{{ daemon_json }}'));u=c.get('default-ulimits',{}).get('nofile',{});raise SystemExit(0 if u.get('Soft')=={{ nofile }} and u.get('Hard')=={{ nofile }} else 1)\" 2>/dev/null"

  - name: Reload dockerd (SIGHUP — does NOT restart containers)
    sudo: true
    shell: systemctl reload docker
    when: "! sudo docker run --rm --entrypoint sh busybox -c 'ulimit -n' 2>/dev/null | grep -qx '{{ nofile }}'"

verify:
  - name: daemon.json is valid JSON
    sudo: true
    shell: python3 -m json.tool {{ daemon_json }} > /dev/null
    changed_when: "false"

  - name: daemon.json records the nofile floor at the agreed size
    sudo: true
    shell: |
      python3 -c "import json;u=json.load(open('{{ daemon_json }}'))['default-ulimits']['nofile'];assert u['Soft']=={{ nofile }} and u['Hard']=={{ nofile }}, u"
    changed_when: "false"

  - name: A NEWLY created container actually gets the floor (the real proof)
    sudo: true
    shell: |
      out=$(docker run --rm --entrypoint sh busybox -c 'ulimit -n')
      [ "$out" = "{{ nofile }}" ] || { echo "got $out want {{ nofile }}"; exit 1; }
    changed_when: "false"

  - name: dockerd is still running and containers were not bounced
    sudo: true
    shell: systemctl is-active --quiet docker && test "$(docker ps -q | wc -l)" -ge 13
    changed_when: "false"