esh-pfi-infrastructure/playbooks/nh3-extdev-althing-v3.yaml

# nh3-extdev: cut the system-wide althing install over from v2.1.0 to v3.0.x (U9b flag day; re-run for each release).
#
# nh3-extdev is the one box a `git pull` cannot move: althing lives there as a system WHEEL
# under /opt/uv-tools/althing-core with entry points in /usr/local/bin, installed from a wheel
# that was copied to /tmp -- not from a checkout. So it needs its own install or it goes dark
# at the cutover.
#
# ⚠ Two things about this box that differ from nh3-dev:
#   - the v2 daemons are SYSTEM units here (althing-herald, althing-receiver), not user units.
#   - `uv` is not on lkraven's PATH; it lives at /home/infra-ops/.local/bin/uv. The original
#     install used it under sudo with UV_TOOL_DIR=/opt/uv-tools, per the uv-receipt.toml.
#
# ⚠ There is a live agent session here (ldp-dev) holding a v2 light-monitor. Retiring the v2
# herald does not kill it, but it will never fire again -- that session has to re-arm on
# althing-listen after this. Its handle survives: bare `ldp-dev` is in the authoritative 73;
# only the machine-qualified `ldp-dev@nh3-extdev` was on the legacy exclusion list.
#
# Run:  scripts/elway lkraven@10.100.50.42 --playbook playbooks/nh3-extdev-althing-v3.yaml
# Rerunnable: a second run shows the install and unit steps skipped.

vars:
  wheel_src: /home/lkraven/development/althing/dist/althing_core-3.1.1-py3-none-any.whl
  wheel_dest: /tmp/althing_core-3.1.1-py3-none-any.whl
  uv: /home/infra-ops/.local/bin/uv
  tool_dir: /opt/uv-tools
  bin_dir: /usr/local/bin
  # ⚠ Moved off nh3-dev 2026-08-28. A stale value here does not fail loudly at
  # install time — it fails in the VERIFY, which then reads as a broken deploy.
  post_office: http://10.100.50.40:8390
  # ⚠ A release can change more than one file — 3.1.0 changed two. One pair is not
  # enough, and a check that asserts only half a release is a check that half-passes
  # silently. Space-separated `file:marker` pairs; bump BOTH per release.
  #   3.0.1  zellij.py:_PANE_ID  session_source.py:_live_pid    (pane routes)
  #   3.0.3  post_office_herald.py:PANE_SETTLE_S               (write/submit race)
  #   3.1.0  post_office_herald.py:POST_OFFICE_HINT  dev_launch.py:resolve_post_office
  #   3.1.1  postbox.py:warn_if_handle_looks_wrong    dev_launch.py:handles_launched_at
  #
  # ⚠ Match on PRESENCE (grep -q), never on a count. 3.1.1 shipped with a stated
  # expectation of "grep -c handles_launched_at dev_launch.py # 2+"; the real count
  # there is 1 (the definition) with the other two occurrences in postbox.py. A count
  # assertion would have reported FAILED on a byte-perfect install.
  markers: "postbox.py:warn_if_handle_looks_wrong dev_launch.py:handles_launched_at"

steps:
  - name: Stage the v3.0.0 wheel
    upload:
      src: /home/lkraven/development/althing/dist/althing_core-3.1.1-py3-none-any.whl
      dest: "{{ wheel_dest }}"
      mode: "0644"

  - name: Retire the v2 system daemons BEFORE swapping the package
    # Order matters: these run out of /opt/uv-tools/althing-core/bin/python, which the
    # install is about to replace. Stopping first means they never see a half-swapped tree.
    # v3 has no counterpart to either -- the post office replaced the herald and deleted the
    # reason for the receiver, since there is no longer a mailbox per machine to deliver between.
    shell: sudo systemctl disable --now althing-herald.service althing-receiver.service
    when: "systemctl is-active --quiet althing-herald.service || systemctl is-active --quiet althing-receiver.service"

  - name: Install the staged althing-core wheel over the system wheel install
    # NOT gated on `postbox` existing — that guard was right for the cutover and
    # wrong for every release after it: postbox exists now, so a version bump would
    # silently skip. `--force` makes the reinstall idempotent on its own.
    shell: sudo env UV_TOOL_DIR={{ tool_dir }} UV_TOOL_BIN_DIR={{ bin_dir }} {{ uv }} tool install --force {{ wheel_dest }}

  - name: Install the post-office herald as a system unit
    # A system unit rather than a user unit because that is how v2 was supervised here and
    # because this box has no lingering user session to hang a --user unit from.
    shell: |
      printf '%s\n' \
        '[Unit]' \
        'Description=Althing post-office herald — per-machine relay (v3)' \
        'Documentation=https://gitea.phasefinal.com/vh/althing' \
        'After=network-online.target' \
        'Wants=network-online.target' \
        '' \
        '[Service]' \
        'Type=simple' \
        'User=lkraven' \
        'Environment=ALTHING_POST_OFFICE={{ post_office }}' \
        'ExecStart={{ bin_dir }}/althing-po-herald' \
        'Restart=always' \
        'RestartSec=5' \
        '' \
        '# Dials out, opens no port, holds no state. Refuses to start if another herald' \
        '# already holds this node — two would double every poke and both write liveness.' \
        '# Replaces althing-herald.service + althing-receiver.service, retired 2026-08-28.' \
        '' \
        '[Install]' \
        'WantedBy=multi-user.target' \
      | sudo tee /etc/systemd/system/althing-po-herald.service >/dev/null
      sudo systemctl daemon-reload
    when: "! test -f /etc/systemd/system/althing-po-herald.service"

  - name: Enable and (re)start the herald so it picks up the new code
    shell: sudo systemctl enable --now althing-po-herald.service && sudo systemctl restart althing-po-herald.service

verify:
  - name: postbox is installed and is v3
    shell: "{{ bin_dir }}/postbox --help | grep -q 'send,reply,read,peek,thread,search,status,handles,register,sign-off'"
    changed_when: "false"

  - name: the v2 entry points are GONE, not merely shadowed
    # Assert absence of the binaries themselves. A `which` that still resolves would mean the
    # old wheel's entry points survived the --force and agents could keep calling a dead CLI.
    shell: "! test -e {{ bin_dir }}/althing-cli && ! test -e {{ bin_dir }}/althing-receiver && ! test -e {{ bin_dir }}/althing-herald"
    changed_when: "false"

  - name: v2 daemons are stopped and disabled
    shell: "! systemctl is-active --quiet althing-herald.service && ! systemctl is-active --quiet althing-receiver.service"
    changed_when: "false"

  - name: the po-herald is running
    shell: systemctl is-active --quiet althing-po-herald.service
    changed_when: "false"

  - name: this box can reach the post office and the roster is populated
    # ⚠ --handle is required. postbox resolves its identity from ALTHING_HANDLE, which
    # dev-launch sets per pane and which a playbook shell does not have -- without it this
    # check fails on identity, not on reachability, and reads as a deployment fault.
    shell: ALTHING_POST_OFFICE={{ post_office }} {{ bin_dir }}/postbox --handle operator handles | wc -l | awk '{ if ($1 >= 70) exit 0; else exit 1 }'
    changed_when: "false"

  - name: This release's markers are ALL present BY CONTENT, not by version string
    # forseti's own checks. A dist-info directory records what was INSTALLED, not
    # what the files CONTAIN — verify the code, not the label. Bump `marker` and
    # `marker_file` with each release rather than trusting the version bumped.
    #   3.0.1  _PANE_ID   in zellij.py            (pane routes)
    #   3.0.3  PANE_SETTLE_S in post_office_herald.py  (the write/submit race)
    shell: |
      SP={{ tool_dir }}/althing-core/lib/python3.13/site-packages/althing
      rc=0
      for pair in {{ markers }}; do
        f="${pair%%:*}"; m="${pair##*:}"
        if grep -q "$m" "$SP/$f"; then echo "  ok   $f : $m"
        else echo "  MISS $f : $m"; rc=1; fi
      done
      exit $rc
    changed_when: "false"