esh-pfi-infrastructure/stacks/hrafn/ci/playbooks-deploy.yaml

# Deploy hrafn (fleet browser-fetch service) to a Docker host.
#
# Single source of truth for "how hrafn is deployed" — CI and manual runs
# take the same path. Modelled on nevermore's playbook, with two deliberate
# differences noted below.
#
# Usage (from this repo's root):
#   elway <host> --playbook playbooks/deploy.yaml
#
# Requires a build-context tarball at dist/hrafn-context.tgz. The CI
# workflow builds it; for a manual run, build it the same way:
#   mkdir -p dist && tar czf dist/hrafn-context.tgz \
#     Dockerfile compose.yaml pyproject.toml README.md .env.example src
#
# The tarball must carry EVERYTHING that belongs in the compose dir: the
# converge step below deletes anything on the host that is not in it,
# except the host-owned .env and .deployed.
#
# DIFFERENCE 1 — tarball instead of per-file upload steps. nevermore
# enumerates every source file as its own upload step. That is explicit,
# but it fails OPEN: add a new src/hrafn/*.py and forget the matching
# step, and the deploy silently ships without it. hrafn's build context
# is a whole package, so it travels as one archive and cannot go partial.
#
# DIFFERENCE 2 — the image is tagged with the commit SHA, not a fixed
# `v1`. Before this, nothing on the host could answer "what is running";
# now the image tag IS the answer, and rolling back is retagging.
#
# NOT DEPLOYED BY THIS PLAYBOOK: .env. It is host-owned, 0600, and holds
# the bearer token. Uploading it would either clobber the live token or
# leak it into the repo. It is provisioned once by hand from the vault
# item `ana-docker/hrafn/bearer-token`.

vars:
  compose_dir: /opt/docker/compose/hrafn
  # MUST live outside compose_dir — see the unpack step for why staging
  # inside the target silently destroyed the source mid-rsync.
  stage_dir: /tmp/hrafn-deploy-stage
  # Overridden by CI with the triggering commit. `manual` marks a
  # hand-run deploy so an un-provenanced image is obvious on the host.
  hrafn_sha: manual
  # sha256 of the shipped context, computed in the workflow. Empty means
  # "skip the content assertion" so a hand-run without it still works.
  context_sha256: ""

steps:
  # ── preconditions ───────────────────────────────────────────────────

  - name: Verify compose dir exists + writable
    shell: test -w {{ compose_dir }}
    changed_when: "false"

  - name: Verify .env is present and 0600 (deploy must not create it)
    shell: |
      test -f {{ compose_dir }}/.env || { echo "MISSING .env — provision it from vault item ana-docker/hrafn/bearer-token"; exit 1; }
      perms=$(stat -c %a {{ compose_dir }}/.env)
      test "$perms" = "600" || { echo "REFUSING: .env is $perms, expected 600 (it holds the bearer token)"; exit 1; }
    changed_when: "false"

  - name: Verify traefik-net exists (consumers reach hrafn on it)
    shell: docker network inspect traefik-net >/dev/null 2>&1
    changed_when: "false"

  # ── ship the build context ──────────────────────────────────────────

  - name: Upload build context
    upload:
      src: dist/hrafn-context.tgz
      dest: "{{ compose_dir }}/.hrafn-context.tgz"
      mode: "0600"

  - name: Unpack build context into a staging dir OUTSIDE the target
    # {{ stage_dir }} must NOT live under {{ compose_dir }}. The first
    # version staged at {{ compose_dir }}/.stage, and rsync --delete then
    # deleted .stage from the destination (it is not in the source listing)
    # DURING the transfer — destroying the source mid-copy. Net effect:
    # strays were removed but NEW SOURCE NEVER LANDED. Reproduced exactly:
    #   before: app.py="OLD", leftover.txt, .stage/app.py="NEW"
    #   after:  app.py="OLD", leftover.txt GONE, .stage GONE
    # The half that worked (deletion) is the half the old verify steps
    # could see, so the deploy reported success for weeks while the host
    # source stayed frozen at the first manual rsync.
    shell: |
      rm -rf {{ stage_dir }} && mkdir -p {{ stage_dir }}
      tar xzf {{ compose_dir }}/.hrafn-context.tgz -C {{ stage_dir }}
      rm -f {{ compose_dir }}/.hrafn-context.tgz

  - name: Converge the compose dir onto the build context
    # Unpacking in place ACCRETED — it overwrote tracked files but never
    # removed anything, so leftovers outlived every deploy. rsync --delete
    # makes the directory CONVERGE on the build context instead. Host-owned
    # state is protected by name.
    shell: |
      rsync -a --delete \
        --exclude '.env' --exclude '.deployed' \
        {{ stage_dir }}/ {{ compose_dir }}/
      rm -rf {{ stage_dir }}

  # ── build + start ───────────────────────────────────────────────────

  - name: Build image tagged with the commit
    shell: cd {{ compose_dir }} && HRAFN_TAG={{ hrafn_sha }} docker compose build

  - name: Start/replace the container on the new image
    shell: cd {{ compose_dir }} && HRAFN_TAG={{ hrafn_sha }} docker compose up -d

  - name: Record what is deployed
    # The provenance fix. Anyone on the box can now answer "what commit
    # is this" without asking the person who last rsync'd.
    shell: |
      printf 'sha=%s\ndeployed_at=%s\n' "{{ hrafn_sha }}" "$(date -Is)" > {{ compose_dir }}/.deployed
      chmod 644 {{ compose_dir }}/.deployed

verify:
  - name: Container reports healthy
    # start_period is 20s and the interval is 30s, so allow two cycles
    # before calling it a failure rather than a slow start.
    shell: |
      # elway's variable regex only matches an identifier starting with a
      # letter or underscore, so docker's .State Go template below passes
      # through untouched and needs no escaping.
      for i in $(seq 1 20); do
        s=$(docker inspect hrafn --format '{{.State.Health.Status}}' 2>/dev/null || echo missing)
        [ "$s" = "healthy" ] && { echo "healthy after ${i}0s"; exit 0; }
        sleep 10
      done
      echo "still $s after 200s"; docker logs --tail 40 hrafn; exit 1
    changed_when: "false"

  - name: Readiness endpoint answers on the shared network
    # Proves consumers can actually reach it, not just that the process
    # is up — hrafn has no published host port, so this has to run from
    # inside traefik-net.
    shell: >
      docker run --rm --network traefik-net curlimages/curl:latest
      -sf -m 10 -o /dev/null -w '%{http_code}\n' http://hrafn:8080/readyz
    changed_when: "false"

  - name: Deployed SHA matches what we just shipped
    shell: grep -q "^sha={{ hrafn_sha }}$" {{ compose_dir }}/.deployed
    changed_when: "false"

  # ── the checks that would have caught the frozen-source defect ──────
  # The original verify set asserted the marker, container health, and a
  # 200 from /readyz. Every one of those passed for weeks while the host
  # source sat frozen at the first manual rsync, because none of them
  # measured CONTENT. A deploy that reports success without asserting the
  # bytes changed is not verifying a deploy, it is verifying an uptime.

  - name: Host tree content matches the context CI shipped
    # context_sha256 is computed in the workflow over the same file list
    # the tarball carries, so this is an end-to-end assertion from the CI
    # checkout to the host filesystem. Empty value skips the check, so a
    # hand-run without it still works.
    when: "test -n '{{ context_sha256 }}'"
    shell: |
      cd {{ compose_dir }}
      actual=$(find Dockerfile compose.yaml pyproject.toml README.md .env.example src \
                 -type f | LC_ALL=C sort | xargs sha256sum | sha256sum | cut -d' ' -f1)
      test "$actual" = "{{ context_sha256 }}" || {
        echo "CONTENT MISMATCH — the converge did not land what CI shipped"
        echo "  expected {{ context_sha256 }}"
        echo "  on host  $actual"
        exit 1; }
      echo "host tree matches shipped context ($actual)"
    changed_when: "false"

  - name: Running image was built from the source now on the host
    # Catches a SHA-tagged image whose layers predate the source — the build
    # cache full-hits when COPY inputs never change, so a tag can name a
    # commit the image does not contain. Comparing the container's /app/src
    # against the host src proves the image really was built from this tree.
    #
    # ONLY *.py. A naive `find src -type f` compare fails on every healthy
    # deploy: `pip install .` generates src/hrafn.egg-info/* inside the image
    # (6 files) that the host tree does not have, and __pycache__ appears at
    # runtime. Verified against a known-good container before shipping this —
    # 12 host files, 18 in the container, 0 content differences. A check that
    # cries wolf on every green deploy is worse than no check.
    shell: |
      list() { find src -type f -name '*.py' | LC_ALL=C sort | xargs sha256sum | sha256sum | cut -d' ' -f1; }
      h=$(cd {{ compose_dir }} && list)
      c=$(docker exec hrafn sh -c "cd /app && find src -type f -name '*.py' | LC_ALL=C sort | xargs sha256sum | sha256sum" | cut -d' ' -f1)
      test "$h" = "$c" || {
        echo "IMAGE IS STALE — the running container's source differs from the host tree"
        echo "  host      $h"
        echo "  container $c"
        exit 1; }
      echo "image source matches host source ($h)"
    changed_when: "false"