esh-pfi-infrastructure/services/semif-serve/bench-jev-2026-09-30/code/run.sh

#!/usr/bin/env bash
# Jev-candidate bench (2026-09-30), one repeat of one model/format on fv-ml1 GPU 3.
#   run.sh <label> semif  <repo>@<sha>                <repeat> [capacity]
#   run.sh <label> native jevk5:<repo>@<sha>          <repeat> [capacity]
#   run.sh <label> native intern:<repo>@<sha>         <repeat> [capacity]
#   run.sh <label> native imajev:<repo>@<sha>         <repeat> [capacity]
# One process lifetime = one repeat. Everything is loopback on fv-ml1; GPU 3 only; one candidate
# loaded at a time; every container this starts is removed before it returns.
set -u
W=/tmp/jevbench-2026-09-30
label=$1 fmt=$2 rt=$3 r=$4 cap=${5:-}
O=$W/out/$label/r$r
mkdir -p "$O" && chmod 777 "$O"
TASKS_HOST=$W/src/jevbench-v1.2.16/datasets/public/easy.jsonl,$W/src/jevbench-v1.2.16/datasets/public/original.jsonl,$W/src/jevbench-v1.2.16/datasets/public/hard.jsonl
TASKS_CT=/jb/datasets/public/easy.jsonl,/jb/datasets/public/original.jsonl,/jb/datasets/public/hard.jsonl
GPU3_UUID=GPU-186dacf4-f447-ef03-ffc3-12ab28e1b8ff
log() { echo "$(date -u +%FT%TZ) [$label r$r] $*" | tee -a $W/logs/run.log; }

guard() {   # nothing of ours may be up; anything foreign on GPU 3 is logged; a full-size seat stops us
  local used apps
  used=$(nvidia-smi -i 3 --query-gpu=memory.used --format=csv,noheader,nounits)
  apps=$(nvidia-smi --query-compute-apps=gpu_uuid,pid,process_name,used_memory --format=csv,noheader | grep "$GPU3_UUID" || true)
  echo "$(date -u +%FT%TZ) gpu3_used_mib=$used apps=[$apps]" >> "$O/guard.txt"
  if [ "$used" -gt 40000 ]; then log "GUARD STOP: GPU 3 holds ${used} MiB of someone else's (full-size seat?): $apps"; exit 10; fi
  if [ "$used" -gt 1024 ]; then log "GUARD NOTE: GPU 3 already holds ${used} MiB (not ours): $apps — our footprint stays < 30 GiB"; fi
}

poll_start() { ( while :; do printf '%s,%s\n' "$(date +%s.%N)" "$(nvidia-smi -i 3 --query-gpu=memory.used --format=csv,noheader,nounits)"; sleep 0.2; done ) >> "$O/vram.csv" & POLL=$!; }
poll_stop() { kill "$POLL" 2>/dev/null; wait "$POLL" 2>/dev/null; }

wait_up() {   # $1 url, $2 container: any HTTP answer = up; container exit = failure
  local i code
  for i in $(seq 1 180); do
    code=$(curl -s -o /dev/null -w '%{http_code}' "$1" || true)
    if [ "$code" != "000" ]; then echo "$(date +%s.%N)" > "$O/up_at.txt"; return 0; fi
    if [ "$(docker inspect -f '{{.State.Running}}' "$2" 2>/dev/null)" != "true" ]; then return 1; fi
    sleep 5
  done
  return 1
}

summarize_jb() {
  PYTHONPATH=$W/src/jevbench-v1.2.16 python3 -m jevbench.cli summarize --tasks "$TASKS_HOST" \
    --results "$O/jevbench/results.jsonl" > "$O/jevbench/summary.json" 2> "$O/jevbench/summarize.err" \
    && python3 - "$O/jevbench/summary.json" <<'EOF' | tee -a $W/logs/run.log
import json,sys
s=json.load(open(sys.argv[1]))
print("   jevbench:", {k:(v["n_correct"],v["n_scorable"]) for k,v in s["splits"].items()}, "all", (s["n_correct"], s["n_scorable"]))
EOF
}

guard
log "start fmt=$fmt rt=$rt cap=${cap:-no}"
mkdir -p "$O/jevbench" && chmod 777 "$O/jevbench"

if [ "$fmt" = semif ]; then
  repo=${rt%@*}; sha=${rt#*@}
  # 1. JevBench's own SemIf adapter (semif_direct), in-process, on the same weights.
  poll_start
  echo "$(date +%s.%N) jevbench:start" >> "$O/events.txt"
  docker run --rm --name jevbench-jb --gpus '"device=3"' -e HF_HOME=/hf -e JEVBENCH_WARM_LOAD=1 -e PYTHONPATH=/jb \
    -v /tank/aimodels/huggingface:/hf:ro -v $W/src/jevbench-v1.2.16:/jb:ro -v "$O/jevbench":/out -w /jb \
    --entrypoint /app/.venv/bin/python semif-serve:0.1.4 -m jevbench.cli run --tasks "$TASKS_CT" \
    --adapter semif_direct --endpoint "$repo" --revision "$sha" --run-label "$label" \
    --results /out/results.jsonl --ledger /out/ledger.jsonl --raw-dir /out/raw \
    --price-in-per-m 0 --price-out-per-m 0 --manifest /out/manifest.json > "$O/jevbench/run.log" 2>&1
  log "jevbench semif_direct rc=$? $(tail -1 "$O/jevbench/run.log")"
  echo "$(date +%s.%N) jevbench:end" >> "$O/events.txt"
  summarize_jb
  # 2. semif-serve itself on the same weights, under the production 12 GiB cap.
  echo "$(date +%s.%N) serve:start" >> "$O/events.txt"
  docker run -d --name jevbench-serve --gpus '"device=3"' -e SEMIF_API_TOKEN="$(cat $W/token)" -e SEMIF_DEVICE=cuda \
    -e SEMIF_VRAM_CAP_GIB=12 -e SEMIF_MODEL="$repo" -e SEMIF_REVISION="$sha" \
    -v /tank/aimodels/huggingface:/hf:ro -p 0.0.0.0:18032:8000 semif-serve:0.1.4 > /dev/null
  if ! wait_up http://127.0.0.1:18032/health jevbench-serve; then
    log "FAIL: semif-serve did not come up"; docker logs jevbench-serve > "$O/server.log" 2>&1; docker rm -f jevbench-serve > /dev/null; poll_stop; exit 3
  fi
  echo "$(date +%s.%N) rest:start" >> "$O/events.txt"; sleep 8; echo "$(date +%s.%N) rest:end" >> "$O/events.txt"
  curl -s http://127.0.0.1:18032/health > "$O/health-rest.json"
  echo "$(date +%s.%N) sets:start" >> "$O/events.txt"
  python3 $W/code/bench_sets.py --backend semif --url http://127.0.0.1:18032 --token-file $W/token \
    --label "$label-r$r" --out "$O/sets.json" 2>&1 | tee -a "$O/bench.log"
  curl -s http://127.0.0.1:18032/health > "$O/health-after-sets.json"
  python3 $W/code/bench_shape.py --backend semif --url http://127.0.0.1:18032 --token-file $W/token \
    --long-state-file $W/out/long_state.txt --label "$label-r$r" --out "$O/shape.json" ${cap:+--capacity} 2>&1 | tee -a "$O/bench.log"
  curl -s http://127.0.0.1:18032/health > "$O/health-end.json"
  docker logs jevbench-serve > "$O/server.log" 2>&1
  docker rm -f jevbench-serve > /dev/null
  echo "$(date +%s.%N) serve:end" >> "$O/events.txt"
  poll_stop
else
  kind=${rt%%:*}; spec=${rt#*:}; repo=${spec%@*}; sha=${spec#*@}
  snap=/hf/hub/models--${repo/\//--}/snapshots/$sha
  maxq=10000; envx=()
  case $kind in
    jevk5)  cmd=(jevk5.server --model "$snap" --host 0.0.0.0 --port 18090) ;;
    jevk5v03) cmd=(jevk5.server --model "$snap" --host 0.0.0.0 --port 18090)   # the v0.3.3 runtime shadows the image's v0.2.0
            envx=(-e PYTHONPATH=/w/src/jevk5-v0.3.3) ;;
    intern) cmd=(/w/code/intern_server.py --checkpoint "$snap" --host 0.0.0.0 --port 18090); maxq=16 ;;
    imajev) cmd=(/w/src/imajev-a0134749/scripts/playground/server.py --backend torch --adapter "$snap"
                 --model-bundle /w/code/imajev-bundle-qwen4b.json --rotations 1 --calibration "$snap/calibration.json"
                 --model-name imajev-4b --host 0.0.0.0 --port 18090)
            envx=(-e PYTHONPATH=/w/src/imajev-a0134749/src:/w/src/imajev-a0134749/scripts); maxq=8 ;;  # jev_api.MAX_QUESTIONS
    *) log "unknown native kind $kind"; exit 2 ;;
  esac
  poll_start
  echo "$(date +%s.%N) serve:start" >> "$O/events.txt"
  docker run -d --name jevbench-native --gpus '"device=3"' -e HF_HOME=/hf -e HF_HUB_OFFLINE=1 \
    -e BENCH_VRAM_CAP_GIB="${VCAP:-12}" "${envx[@]}" -v /tank/aimodels/huggingface:/hf:ro -v $W:/w:ro \
    -p 127.0.0.1:18090:18090 --entrypoint /app/.venv/bin/python jevbench-native:2026-09-30 /w/code/capped.py "${cmd[@]}" > /dev/null
  if ! wait_up http://127.0.0.1:18090/health jevbench-native; then
    log "FAIL: native server did not come up"; docker logs jevbench-native > "$O/server.log" 2>&1; docker rm -f jevbench-native > /dev/null; poll_stop; exit 3
  fi
  echo "$(date +%s.%N) rest:start" >> "$O/events.txt"; sleep 8; echo "$(date +%s.%N) rest:end" >> "$O/events.txt"
  echo "$(date +%s.%N) jevbench:start" >> "$O/events.txt"
  PYTHONPATH=$W/src/jevbench-v1.2.16 python3 -m jevbench.cli run --tasks "$TASKS_HOST" --adapter typesafe \
    --endpoint http://127.0.0.1:18090 --key-env '' --model "$label" --run-label "$label" \
    --results "$O/jevbench/results.jsonl" --ledger "$O/jevbench/ledger.jsonl" --raw-dir "$O/jevbench/raw" \
    --price-in-per-m 0 --price-out-per-m 0 --manifest "$O/jevbench/manifest.json" > "$O/jevbench/run.log" 2>&1
  log "jevbench typesafe rc=$? $(tail -1 "$O/jevbench/run.log")"
  echo "$(date +%s.%N) jevbench:end" >> "$O/events.txt"
  summarize_jb
  echo "$(date +%s.%N) sets:start" >> "$O/events.txt"
  python3 $W/code/bench_sets.py --backend systemone --url http://127.0.0.1:18090 \
    --label "$label-r$r" --out "$O/sets.json" 2>&1 | tee -a "$O/bench.log"
  python3 $W/code/bench_shape.py --backend systemone --url http://127.0.0.1:18090 --max-questions $maxq \
    --long-state-file $W/out/long_state.txt --label "$label-r$r" --out "$O/shape.json" ${cap:+--capacity} 2>&1 | tee -a "$O/bench.log"
  docker logs jevbench-native > "$O/server.log" 2>&1
  docker rm -f jevbench-native > /dev/null
  echo "$(date +%s.%N) serve:end" >> "$O/events.txt"
  poll_stop
fi
log "done"