From 064181a8fbd71d862841aa8cb54d7d0ee33eb7cb Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Sat, 22 Aug 2026 21:49:42 -0700 Subject: [PATCH] feat(playbooks): re-pin Worldtree WORLDTREE_IMAGE off the floating :latest tag Both corviduo-dev Worldtree instances carried `WORLDTREE_IMAGE=.../worldtree:latest` in their .env while running SHA-tagged images built the same day. The local :latest tag pointed at b19afd71d7cc, built 2026-06-14 -- 69 days stale. Any `docker compose up` on either instance, by anyone, silently downgraded that service by 69 days: the same footgun behind the 2026-06-15 outage. Applied under worldtree-dev authorization (Worldtree #410): demo -> gitea.phasefinal.com/vh/worldtree:ae88a057c0ed personal -> gitea.phasefinal.com/vh/worldtree:f63529168c13 Both runs verified compose config resolves every service to the pinned SHA with no :latest remaining, and that no container restarted. The edit is inert by design -- it changes what the NEXT recreate pulls. The playbook guards against pinning a SHA that is not the one actually running, which would re-arm the same hazard with a different image. Also records the worldtree-pinned case, deliberately NOT changed here: that instance runs a DANGLING image (sha256:446e5807, no repo tags), kept alive only by the running container. It has no tag to pin to, so it needs `docker tag` first -- and an untagged image referenced only by a container is one `docker rm` from garbage collection, which for the fleet's frozen reference instance is worth fixing on its own merits. Out of the authorized scope (demo + personal); raised with worldtree-dev. Stopgap: the durable fix is the deploy workflow stamping the deployed SHA into .env at each deploy, queued repo-side with worldtree-dev. --- playbooks/repin-worldtree-image.yaml | 100 +++++++++++++++++++++++++++ 1 file changed, 100 insertions(+) create mode 100644 playbooks/repin-worldtree-image.yaml diff --git a/playbooks/repin-worldtree-image.yaml b/playbooks/repin-worldtree-image.yaml new file mode 100644 index 0000000..8919318 --- /dev/null +++ b/playbooks/repin-worldtree-image.yaml @@ -0,0 +1,100 @@ +# Re-pin a Worldtree instance's WORLDTREE_IMAGE to the SHA it is actually +# running, closing the stale-`:latest` recreate hazard. +# +# THE HAZARD (found 2026-08-22, Worldtree #410): both corviduo-dev instances +# had `WORLDTREE_IMAGE=…/worldtree:latest` in their .env while running +# SHA-tagged images from that day. The local `:latest` tag pointed at +# b19afd71d7cc, built 2026-06-14 — 69 days stale. So ANY `docker compose up` +# on either instance, by anyone, for any reason, silently DOWNGRADED that +# service by 69 days. This is the same footgun that caused the 2026-06-15 +# outage; the pin is what disarms it. +# +# This is a stopgap. The durable fix is worldtree-dev's deploy workflow +# stamping the deployed SHA into .env at each deploy (queued repo-side). +# Until that lands, re-run this after any deploy that moves the image. +# +# Usage — one run per instance: +# scripts/elway corviduo-dev --playbook playbooks/repin-worldtree-image.yaml \ +# --var instance_dir=/opt/worldtree \ +# --var api_container=worldtree-worldtree-api-1 \ +# --var expect_sha=ae88a057c0ed +# +# The edit is INERT until the next recreate — it changes what the NEXT +# `compose up` resolves to, not the running container. That is the intent: +# make the next recreate safe rather than dangerous. +# +# ⚠ THE `worldtree-pinned` INSTANCE NEEDS AN EXTRA STEP FIRST. It runs image +# sha256:446e5807… which has NO repo tags at all — it is dangling, kept alive +# only by the running container referencing it. So there is no tag to pin to, +# and this playbook's guard will (correctly) refuse. Tag it before re-pinning: +# +# docker tag sha256:446e5807bf43639be7d285864a7716816880e0f5c0892427e72800f4aa8ffc56 \ +# gitea.phasefinal.com/vh/worldtree:446e5807bf43 +# +# That is also worth doing on its own merits: an untagged image referenced +# only by a container is one `docker rm` away from being garbage-collected, +# and this one is the frozen reference the whole instance exists to provide. + +vars: + instance_dir: /opt/worldtree + api_container: worldtree-worldtree-api-1 + expect_sha: "" + registry: gitea.phasefinal.com/vh/worldtree + +steps: + - name: Refuse to run without an explicit target SHA + shell: test -n "{{ expect_sha }}" + changed_when: "false" + + - name: Confirm the container is actually running the SHA we are about to pin + # Guards against a deploy landing between the operator reading the SHA + # and this playbook writing it — pinning a SHA that is NOT running would + # arm the exact hazard we are disarming, just with a different image. + shell: | + running=$(docker inspect {{ api_container }} --format '{{.Config.Image}}') + test "$running" = "{{ registry }}:{{ expect_sha }}" || { + echo "REFUSING: {{ api_container }} runs $running, not {{ registry }}:{{ expect_sha }}" + exit 1; } + echo "confirmed: $running" + changed_when: "false" + + - name: Back up .env + sudo: true + shell: cp -n {{ instance_dir }}/.env {{ instance_dir }}/.env.bak-pre-repin-{{ expect_sha }} + creates: "{{ instance_dir }}/.env.bak-pre-repin-{{ expect_sha }}" + + - name: Re-pin WORLDTREE_IMAGE to the running SHA + sudo: true + # `|` delimiter because the image reference contains slashes. + shell: | + grep -q '^WORLDTREE_IMAGE=' {{ instance_dir }}/.env || { + echo "REFUSING: no WORLDTREE_IMAGE line to replace"; exit 1; } + sed -i 's|^WORLDTREE_IMAGE=.*|WORLDTREE_IMAGE={{ registry }}:{{ expect_sha }}|' {{ instance_dir }}/.env + chown deploy:deploy {{ instance_dir }}/.env + chmod 600 {{ instance_dir }}/.env + changed_when: "true" + +verify: + - name: .env now names the SHA, not a floating tag + sudo: true + shell: grep -q '^WORLDTREE_IMAGE={{ registry }}:{{ expect_sha }}$' {{ instance_dir }}/.env + changed_when: "false" + + - name: compose resolves every service to the pinned SHA (no ':latest' anywhere) + # The assertion that matters. Reading the .env proves the line changed; + # only rendering the compose file proves what a recreate would actually + # pull. + sudo: true + shell: | + cd {{ instance_dir }} + if docker compose config 2>/dev/null | grep -E '^\s+image:' | grep -q ':latest'; then + echo "STILL RESOLVING TO :latest" + docker compose config 2>/dev/null | grep -E '^\s+image:' + exit 1 + fi + docker compose config 2>/dev/null | grep -E '^\s+image:' | sort -u + changed_when: "false" + + - name: Running containers untouched (this edit must not restart anything) + shell: docker inspect {{ api_container }} --format '{{.State.Status}} since {{.State.StartedAt}}' + changed_when: "false"