scripts: wt-memory-gate-batch — one Worldtree U8 gate batch against demo's deployed sha
For infra-hermes's daily batch through the U11 off window (operator ruling 2026-09-29 2340; mode revised to off 2026-09-30). Enforces worldtree-dev's terms: flock plus a refusal while any harness batch runs, code under test = demo's deployed sha (a detached worktree when the main tree has moved; refuses on pyproject/uv.lock/packages drift the shared venv cannot honour), no git commit, no retry. Exit 0 PASS / 1 FAIL / 2 harness error / 3 refused.
This commit is contained in:
Executable
+170
@@ -0,0 +1,170 @@
|
|||||||
|
#!/usr/bin/env bash
|
||||||
|
#
|
||||||
|
# wt-memory-gate-batch — run ONE Worldtree memory-acceptance (U8) gate batch
|
||||||
|
# against the code demo is actually running, and report its verdict.
|
||||||
|
#
|
||||||
|
# scripts/wt-memory-gate-batch # run one batch (10-15 min)
|
||||||
|
# scripts/wt-memory-gate-batch --dry-run # resolve SHAs + print the plan, run nothing
|
||||||
|
#
|
||||||
|
# Runs on nh3-dev, as the user that owns ~/development/Worldtree. Operator
|
||||||
|
# ruling 2026-09-29 2340: infra-hermes runs one batch per day; infra-ops owns
|
||||||
|
# this wrapper and the escalations. worldtree-dev owns the harness and the
|
||||||
|
# run dirs (thread 01M3RG8RDFSFGDK6BDFTE6ZCJE; U11a revision 01M3RPGE6ZTF42E5PMTHWH4RFB).
|
||||||
|
#
|
||||||
|
# Rules this script enforces (worldtree-dev's terms, not ours to relax):
|
||||||
|
# - never two batches at once: a flock here, PLUS a refusal if any harness
|
||||||
|
# process is already running (worldtree-dev runs batches by hand too, and
|
||||||
|
# the harness itself takes no lock);
|
||||||
|
# - the code under test is demo's DEPLOYED sha. When worldtree-dev's working
|
||||||
|
# tree is at that sha the batch runs there; otherwise it runs from a
|
||||||
|
# detached git worktree at that sha. The main working tree is worldtree-dev's
|
||||||
|
# and is NEVER checked out, reset or committed to by this script;
|
||||||
|
# - no git commit (worldtree-dev commits the run dirs);
|
||||||
|
# - no retry. A FAIL is reported the same day with its run stamp.
|
||||||
|
#
|
||||||
|
# Exit codes (the runner acts on these):
|
||||||
|
# 0 PASS
|
||||||
|
# 1 FAIL → worldtree-dev, same day, with the run stamp
|
||||||
|
# 2 harness error / no verdict → infra-ops
|
||||||
|
# 3 refused before running (lock held, a batch already running, sha not
|
||||||
|
# resolvable, dependency drift between the deployed sha and the tree)
|
||||||
|
# → infra-ops; do NOT work around it
|
||||||
|
#
|
||||||
|
# Env overrides:
|
||||||
|
# WT_TREE default ~/development/Worldtree
|
||||||
|
# WT_GATE_TREE default ~/development/Worldtree-gate (the detached worktree)
|
||||||
|
# WT_DEMO_HOST default corviduo-dev
|
||||||
|
# WT_DEMO_API_CT default worldtree-worldtree-api-1
|
||||||
|
# WT_LEGACY_MODE default off (U11a ruling 2026-09-30: off, not read_only)
|
||||||
|
# WT_BATCH_LABEL default window-off
|
||||||
|
# Until the U11b retirement unit lands. After it, `off` is the only mode, the
|
||||||
|
# harness drops --legacy-mode, and this script needs that flag removed.
|
||||||
|
set -euo pipefail
|
||||||
|
|
||||||
|
WT_TREE="${WT_TREE:-$HOME/development/Worldtree}"
|
||||||
|
WT_GATE_TREE="${WT_GATE_TREE:-$HOME/development/Worldtree-gate}"
|
||||||
|
WT_DEMO_HOST="${WT_DEMO_HOST:-corviduo-dev}"
|
||||||
|
WT_DEMO_API_CT="${WT_DEMO_API_CT:-worldtree-worldtree-api-1}"
|
||||||
|
WT_LEGACY_MODE="${WT_LEGACY_MODE:-off}"
|
||||||
|
WT_BATCH_LABEL="${WT_BATCH_LABEL:-window-off}"
|
||||||
|
STATE_DIR="${XDG_STATE_HOME:-$HOME/.local/state}/wt-memory-gate-batch"
|
||||||
|
LOCK_FILE="$STATE_DIR/lock"
|
||||||
|
OUT_DIR="$WT_TREE/docs/eval/memory_acceptance_runs"
|
||||||
|
|
||||||
|
DRY_RUN=0
|
||||||
|
case "${1:-}" in
|
||||||
|
--dry-run) DRY_RUN=1 ;;
|
||||||
|
"") ;;
|
||||||
|
-h|--help) sed -n '3,40p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;;
|
||||||
|
*) echo "unknown argument: $1 (see --help)" >&2; exit 3 ;;
|
||||||
|
esac
|
||||||
|
|
||||||
|
refuse() { echo "REFUSED: $*" >&2; echo "→ escalate to infra-ops; do not work around this." >&2; exit 3; }
|
||||||
|
now() { date -u +%Y-%m-%dT%H:%M:%SZ; }
|
||||||
|
|
||||||
|
mkdir -p "$STATE_DIR"
|
||||||
|
|
||||||
|
# ── 1. one batch at a time ──────────────────────────────────────────────────
|
||||||
|
exec 9>"$LOCK_FILE"
|
||||||
|
flock -n 9 || refuse "another wt-memory-gate-batch holds $LOCK_FILE"
|
||||||
|
if running="$(pgrep -af 'python -m core\.memory_acceptance\.live' || true)"; [[ -n "$running" ]]; then
|
||||||
|
if [[ "$DRY_RUN" -eq 1 ]]; then
|
||||||
|
echo " (a real run would REFUSE now: a harness batch is already running)"
|
||||||
|
else
|
||||||
|
refuse "a harness batch is already running (worldtree-dev's?):
|
||||||
|
$running"
|
||||||
|
fi
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── 2. which code is demo running? ──────────────────────────────────────────
|
||||||
|
image="$(ssh -o ConnectTimeout=10 "$WT_DEMO_HOST" \
|
||||||
|
"docker inspect '$WT_DEMO_API_CT' --format '{{.Config.Image}}'" 2>/dev/null)" \
|
||||||
|
|| refuse "could not read demo's image from $WT_DEMO_HOST ($WT_DEMO_API_CT)"
|
||||||
|
short="${image##*:}"
|
||||||
|
[[ "$short" =~ ^[0-9a-f]{7,40}$ ]] || refuse "demo image tag '$short' is not a commit sha ($image)"
|
||||||
|
deployed="$(git -C "$WT_TREE" rev-parse --verify --quiet "${short}^{commit}")" \
|
||||||
|
|| refuse "demo's sha $short is not in $WT_TREE (not fetched yet?)"
|
||||||
|
head="$(git -C "$WT_TREE" rev-parse HEAD)"
|
||||||
|
|
||||||
|
if [[ "$deployed" == "$head" ]]; then
|
||||||
|
run_tree="$WT_TREE"
|
||||||
|
where="worldtree-dev's tree (HEAD == deployed)"
|
||||||
|
else
|
||||||
|
# The shared .venv holds an editable install of packages/ from the MAIN tree
|
||||||
|
# and the main tree's dependency set. A worktree run is only honest when
|
||||||
|
# neither differs from the deployed sha.
|
||||||
|
if ! git -C "$WT_TREE" diff --quiet "$deployed" "$head" -- pyproject.toml uv.lock packages; then
|
||||||
|
refuse "pyproject.toml / uv.lock / packages/ differ between deployed ${deployed:0:12} and tree HEAD ${head:0:12}; the shared .venv cannot test the deployed code"
|
||||||
|
fi
|
||||||
|
run_tree="$WT_GATE_TREE"
|
||||||
|
where="detached worktree $WT_GATE_TREE (tree HEAD ${head:0:12} != deployed)"
|
||||||
|
fi
|
||||||
|
|
||||||
|
cmd=(python -m core.memory_acceptance.live --runs 3 --gate
|
||||||
|
--legacy-mode "$WT_LEGACY_MODE" --label "$WT_BATCH_LABEL" --out "$OUT_DIR")
|
||||||
|
|
||||||
|
echo "wt-memory-gate-batch $(now)"
|
||||||
|
echo " demo image $image"
|
||||||
|
echo " under test ${deployed:0:12} via $where"
|
||||||
|
echo " command (cd $run_tree) ${cmd[*]}"
|
||||||
|
echo " artifacts $OUT_DIR/<stamp>/trace1.md"
|
||||||
|
if [[ "$DRY_RUN" -eq 1 ]]; then
|
||||||
|
echo "dry-run: nothing run."
|
||||||
|
exit 0
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── 3. materialise the worktree only when it is needed ──────────────────────
|
||||||
|
if [[ "$run_tree" == "$WT_GATE_TREE" ]]; then
|
||||||
|
if [[ -e "$WT_GATE_TREE/.git" ]]; then
|
||||||
|
git -C "$WT_GATE_TREE" checkout --quiet --detach "$deployed" \
|
||||||
|
|| refuse "could not move worktree $WT_GATE_TREE to ${deployed:0:12}"
|
||||||
|
else
|
||||||
|
git -C "$WT_TREE" worktree add --quiet --detach "$WT_GATE_TREE" "$deployed" \
|
||||||
|
|| refuse "could not create worktree $WT_GATE_TREE at ${deployed:0:12}"
|
||||||
|
fi
|
||||||
|
[[ "$(git -C "$WT_GATE_TREE" rev-parse HEAD)" == "$deployed" ]] \
|
||||||
|
|| refuse "worktree $WT_GATE_TREE did not land on ${deployed:0:12}"
|
||||||
|
fi
|
||||||
|
|
||||||
|
# ── 4. run ──────────────────────────────────────────────────────────────────
|
||||||
|
log="$STATE_DIR/$(date -u +%Y%m%dT%H%M%SZ).log" # stderr (progress, errors)
|
||||||
|
out="${log%.log}.stdout" # stdout; its last line is the run dir
|
||||||
|
echo " log $log"
|
||||||
|
set +e
|
||||||
|
(
|
||||||
|
set +eu # activate and env.sh are not written for nounset
|
||||||
|
cd "$run_tree" || exit 2
|
||||||
|
# shellcheck disable=SC1091
|
||||||
|
source "$WT_TREE/.venv/bin/activate"
|
||||||
|
# env.sh is untracked (secrets) and lives only in the main tree. It sets
|
||||||
|
# WORLDTREE_ROOT from its own location, so point that back at the tree under test.
|
||||||
|
# shellcheck disable=SC1091
|
||||||
|
source "$WT_TREE/env.sh"
|
||||||
|
export WORLDTREE_ROOT="$run_tree"
|
||||||
|
# The editable install maps packages/ to the main tree; put the tree under
|
||||||
|
# test first (identical content is guaranteed by step 2, this keeps it explicit).
|
||||||
|
export PYTHONPATH="$run_tree/packages/worldtree-memory/src${PYTHONPATH:+:$PYTHONPATH}"
|
||||||
|
"${cmd[@]}"
|
||||||
|
) >"$out" 2>"$log"
|
||||||
|
rc=$?
|
||||||
|
set -e
|
||||||
|
|
||||||
|
run_dir="$(grep -v '^[[:space:]]*$' "$out" | tail -n 1 || true)"
|
||||||
|
if [[ "$rc" -ne 0 || ! -f "$run_dir/trace1.md" ]]; then
|
||||||
|
echo "HARNESS ERROR (exit $rc) — no verdict. Last lines of $log:" >&2
|
||||||
|
tail -n 15 "$log" >&2
|
||||||
|
echo "→ escalate to infra-ops with the log path." >&2
|
||||||
|
exit 2
|
||||||
|
fi
|
||||||
|
|
||||||
|
stamp="$(basename "$run_dir")"
|
||||||
|
verdict_line="$(grep -m1 -E '^Aggregate label: .*verdict: \*\*(PASS|FAIL)\*\*' "$run_dir/trace1.md" || true)"
|
||||||
|
verdict="$(sed -nE 's/.*verdict: \*\*(PASS|FAIL)\*\*.*/\1/p' <<<"$verdict_line")"
|
||||||
|
echo " run stamp $stamp"
|
||||||
|
echo " verdict ${verdict:-NONE}"
|
||||||
|
echo " line ${verdict_line:-<no Aggregate verdict line in trace1.md>}"
|
||||||
|
case "$verdict" in
|
||||||
|
PASS) exit 0 ;;
|
||||||
|
FAIL) echo "→ report to worldtree-dev TODAY with run stamp $stamp. Do not retry." >&2; exit 1 ;;
|
||||||
|
*) echo "→ no parseable verdict; escalate to infra-ops with $run_dir." >&2; exit 2 ;;
|
||||||
|
esac
|
||||||
Reference in New Issue
Block a user