#!/usr/bin/env bash # # wt-memory-gate-batch — run ONE Worldtree memory-acceptance (U8) gate batch # against the code demo is actually running, and report its verdict. # # scripts/wt-memory-gate-batch # run one batch (10-15 min) # scripts/wt-memory-gate-batch --dry-run # resolve SHAs + print the plan, run nothing # # Runs on nh3-dev, as the user that owns ~/development/Worldtree. Operator # ruling 2026-09-29 2340: infra-hermes runs one batch per day; infra-ops owns # this wrapper and the escalations. worldtree-dev owns the harness and the # run dirs (thread 01M3RG8RDFSFGDK6BDFTE6ZCJE; U11a revision 01M3RPGE6ZTF42E5PMTHWH4RFB). # # Rules this script enforces (worldtree-dev's terms, not ours to relax): # - never two batches at once: a flock here, PLUS a refusal if any harness # process is already running (worldtree-dev runs batches by hand too, and # the harness itself takes no lock); # - the code under test is demo's DEPLOYED sha. When worldtree-dev's working # tree is at that sha the batch runs there; otherwise it runs from a # detached git worktree at that sha. The main working tree is worldtree-dev's # and is NEVER checked out, reset or committed to by this script; # - no git commit (worldtree-dev commits the run dirs); # - no retry. EVERY verdict (PASS and FAIL) goes to worldtree-dev the same # day with its run stamp: both build the n for the U11b data-deletion gate # (three consecutive PASS batches at off, or Prime's waiver). # # Exit codes (the runner acts on these): # 0 PASS → worldtree-dev, same day, with the run stamp # 1 FAIL → worldtree-dev, same day, with the run stamp; no retry # 2 harness error / no verdict → infra-ops # 3 refused before running (sha not resolvable, dependency drift between # the deployed sha and the tree, worktree trouble) # → infra-ops; do NOT work around it # 4 busy: another batch is running (lock held, or a hand-run harness) # → not a fault; try again later the same day # # Env overrides: # WT_TREE default ~/development/Worldtree # WT_GATE_TREE default ~/development/Worldtree-gate (the detached worktree) # WT_DEMO_HOST default corviduo-dev # WT_DEMO_API_CT default worldtree-worldtree-api-1 # WT_LEGACY_MODE default off (U11a ruling 2026-09-30: off, not read_only) # WT_BATCH_LABEL default window-off # Until the U11b retirement unit lands. After it, `off` is the only mode, the # harness drops --legacy-mode, and this script needs that flag removed. set -euo pipefail WT_TREE="${WT_TREE:-$HOME/development/Worldtree}" WT_GATE_TREE="${WT_GATE_TREE:-$HOME/development/Worldtree-gate}" WT_DEMO_HOST="${WT_DEMO_HOST:-corviduo-dev}" WT_DEMO_API_CT="${WT_DEMO_API_CT:-worldtree-worldtree-api-1}" WT_LEGACY_MODE="${WT_LEGACY_MODE:-off}" WT_BATCH_LABEL="${WT_BATCH_LABEL:-window-off}" STATE_DIR="${XDG_STATE_HOME:-$HOME/.local/state}/wt-memory-gate-batch" LOCK_FILE="$STATE_DIR/lock" OUT_DIR="$WT_TREE/docs/eval/memory_acceptance_runs" DRY_RUN=0 case "${1:-}" in --dry-run) DRY_RUN=1 ;; "") ;; -h|--help) sed -n '3,40p' "${BASH_SOURCE[0]}" | sed 's/^# \{0,1\}//'; exit 0 ;; *) echo "unknown argument: $1 (see --help)" >&2; exit 3 ;; esac refuse() { echo "REFUSED: $*" >&2; echo "→ escalate to infra-ops; do not work around this." >&2; exit 3; } busy() { echo "BUSY: $*" >&2; echo "→ not a fault: try again later today (never two batches at once)." >&2; exit 4; } now() { date -u +%Y-%m-%dT%H:%M:%SZ; } mkdir -p "$STATE_DIR" # ── 1. one batch at a time ────────────────────────────────────────────────── exec 9>"$LOCK_FILE" flock -n 9 || busy "another wt-memory-gate-batch holds $LOCK_FILE" if running="$(pgrep -af 'python -m core\.memory_acceptance\.live' || true)"; [[ -n "$running" ]]; then if [[ "$DRY_RUN" -eq 1 ]]; then echo " (a real run would REFUSE now: a harness batch is already running)" else busy "a harness batch is already running (worldtree-dev's?): $running" fi fi # ── 2. which code is demo running? ────────────────────────────────────────── image="$(ssh -o ConnectTimeout=10 "$WT_DEMO_HOST" \ "docker inspect '$WT_DEMO_API_CT' --format '{{.Config.Image}}'" 2>/dev/null)" \ || refuse "could not read demo's image from $WT_DEMO_HOST ($WT_DEMO_API_CT)" short="${image##*:}" [[ "$short" =~ ^[0-9a-f]{7,40}$ ]] || refuse "demo image tag '$short' is not a commit sha ($image)" deployed="$(git -C "$WT_TREE" rev-parse --verify --quiet "${short}^{commit}")" \ || refuse "demo's sha $short is not in $WT_TREE (not fetched yet?)" head="$(git -C "$WT_TREE" rev-parse HEAD)" if [[ "$deployed" == "$head" ]]; then run_tree="$WT_TREE" where="worldtree-dev's tree (HEAD == deployed)" else # The shared .venv holds an editable install of packages/ from the MAIN tree # and the main tree's dependency set. A worktree run is only honest when # neither differs from the deployed sha. if ! git -C "$WT_TREE" diff --quiet "$deployed" "$head" -- pyproject.toml uv.lock packages; then refuse "pyproject.toml / uv.lock / packages/ differ between deployed ${deployed:0:12} and tree HEAD ${head:0:12}; the shared .venv cannot test the deployed code" fi run_tree="$WT_GATE_TREE" where="detached worktree $WT_GATE_TREE (tree HEAD ${head:0:12} != deployed)" fi cmd=(python -m core.memory_acceptance.live --runs 3 --gate --legacy-mode "$WT_LEGACY_MODE" --label "$WT_BATCH_LABEL" --out "$OUT_DIR") echo "wt-memory-gate-batch $(now)" echo " demo image $image" echo " under test ${deployed:0:12} via $where" echo " command (cd $run_tree) ${cmd[*]}" echo " artifacts $OUT_DIR//trace1.md" if [[ "$DRY_RUN" -eq 1 ]]; then echo "dry-run: nothing run." exit 0 fi # ── 3. materialise the worktree only when it is needed ────────────────────── if [[ "$run_tree" == "$WT_GATE_TREE" ]]; then if [[ -e "$WT_GATE_TREE/.git" ]]; then git -C "$WT_GATE_TREE" checkout --quiet --detach "$deployed" \ || refuse "could not move worktree $WT_GATE_TREE to ${deployed:0:12}" else git -C "$WT_TREE" worktree add --quiet --detach "$WT_GATE_TREE" "$deployed" \ || refuse "could not create worktree $WT_GATE_TREE at ${deployed:0:12}" fi [[ "$(git -C "$WT_GATE_TREE" rev-parse HEAD)" == "$deployed" ]] \ || refuse "worktree $WT_GATE_TREE did not land on ${deployed:0:12}" fi # ── 4. run ────────────────────────────────────────────────────────────────── log="$STATE_DIR/$(date -u +%Y%m%dT%H%M%SZ).log" # stderr (progress, errors) out="${log%.log}.stdout" # stdout; its last line is the run dir echo " log $log" set +e ( set +eu # activate and env.sh are not written for nounset cd "$run_tree" || exit 2 # shellcheck disable=SC1091 source "$WT_TREE/.venv/bin/activate" # env.sh is untracked (secrets) and lives only in the main tree. It sets # WORLDTREE_ROOT from its own location, so point that back at the tree under test. # shellcheck disable=SC1091 source "$WT_TREE/env.sh" export WORLDTREE_ROOT="$run_tree" # The editable install maps packages/ to the main tree; put the tree under # test first (identical content is guaranteed by step 2, this keeps it explicit). export PYTHONPATH="$run_tree/packages/worldtree-memory/src${PYTHONPATH:+:$PYTHONPATH}" "${cmd[@]}" ) >"$out" 2>"$log" rc=$? set -e run_dir="$(grep -v '^[[:space:]]*$' "$out" | tail -n 1 || true)" if [[ "$rc" -ne 0 || ! -f "$run_dir/trace1.md" ]]; then echo "HARNESS ERROR (exit $rc) — no verdict. Last lines of $log:" >&2 tail -n 15 "$log" >&2 echo "→ escalate to infra-ops with the log path." >&2 exit 2 fi stamp="$(basename "$run_dir")" verdict_line="$(grep -m1 -E '^Aggregate label: .*verdict: \*\*(PASS|FAIL)\*\*' "$run_dir/trace1.md" || true)" verdict="$(sed -nE 's/.*verdict: \*\*(PASS|FAIL)\*\*.*/\1/p' <<<"$verdict_line")" echo " run stamp $stamp" echo " verdict ${verdict:-NONE}" echo " line ${verdict_line:-}" case "$verdict" in PASS) echo "→ report to worldtree-dev today with run stamp $stamp (PASS builds the n)."; exit 0 ;; FAIL) echo "→ report to worldtree-dev TODAY with run stamp $stamp. Do not retry." >&2; exit 1 ;; *) echo "→ no parseable verdict; escalate to infra-ops with $run_dir." >&2; exit 2 ;; esac