diff --git a/CLAUDE.md b/CLAUDE.md index cd27aab..e726832 100644 --- a/CLAUDE.md +++ b/CLAUDE.md @@ -261,7 +261,7 @@ scripts/refresh-server-info.sh ana-docker Fleet-wide runs require the literal `all` keyword — no-args prints help so you can't accidentally hit every host by forgetting a name. -The script pipes `server_inspect.sh` over SSH via stdin (no scp, no remote cleanup) and writes each `servers//system-details.txt` atomically — a failed run never clobbers the previous snapshot. The inspect script itself is read-only. +The script pipes `server_inspect.sh` over SSH via stdin (no scp, no remote cleanup) and writes each `servers//system-details.txt` atomically — a failed run never clobbers the previous snapshot, **and neither does a run that succeeds but captures nothing**: `ssh` exiting 0 is not proof of a usable capture, so an empty result is refused, the previous snapshot is kept, and the host is counted as failed (2026-09-09 — the readers all test the snapshot with `-s`, so the writer must not produce something they would call invalid). A capture that collapses to under a quarter of the previous one still promotes but is flagged, because a host really can shed services. The inspect script itself is read-only. Each server dir can hold an `ssh-target` file (one line, `` or `@`) as a fallback for when the dir name doesn't resolve via DNS or `~/.ssh/config`. The script prefers whatever ssh would resolve normally and only consults the file when that fails. diff --git a/scripts/refresh-proxmox-info.sh b/scripts/refresh-proxmox-info.sh index def425b..46d5f9a 100755 --- a/scripts/refresh-proxmox-info.sh +++ b/scripts/refresh-proxmox-info.sh @@ -234,10 +234,35 @@ for host in "${HOSTS[@]}"; do mkdir -p "$SERVERS_DIR/$host" if ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "$target" 'bash -s' < "$INSPECT" > "$tmp" 2> "$tmp.err"; then + # ⚠ ssh exiting 0 is NOT proof the capture is usable. The inspect script can + # emit nothing (a shell that dies before its first write, output swallowed by + # a remote wrapper) and this would then promote an EMPTY file over a good + # snapshot and report "ok (0 bytes)" — the readers below all test the + # snapshot with `-s`, so the writer must not produce something they consider + # invalid. A guard whose test disagrees with its writer's contract has + # quietly stopped guarding. Refuse the promotion, keep the old snapshot, + # and count it as a failure so the exit code carries it. + new_bytes=$(wc -c < "$tmp" 2>/dev/null || echo 0) + if [ "$new_bytes" -eq 0 ]; then + rm -f "$tmp" "$tmp.err" + printf 'FAIL (empty capture — previous snapshot kept)\n' + failed+=("$host") + [ "${#warnings[@]}" -gt 0 ] && print_warnings " " "${warnings[@]}" + continue + fi + # A capture that collapses to a fraction of the previous one is suspicious + # but not provably wrong (a host really can shed services), so this WARNS and + # still promotes — the operator sees it rather than the script guessing. + shrink="" + if [ -s "$out" ]; then + old_bytes=$(wc -c < "$out") + if [ "$old_bytes" -gt 0 ] && [ $((new_bytes * 4)) -lt "$old_bytes" ]; then + shrink=" ⚠ shrank from ${old_bytes}B — check before trusting" + fi + fi mv "$tmp" "$out" rm -f "$tmp.err" - bytes=$(wc -c < "$out") - printf 'ok (%s bytes)\n' "$bytes" + printf 'ok (%s bytes)%s\n' "$new_bytes" "$shrink" else rc=$? rm -f "$tmp" diff --git a/scripts/refresh-server-info.sh b/scripts/refresh-server-info.sh index 3b1f0f2..6bb06d8 100755 --- a/scripts/refresh-server-info.sh +++ b/scripts/refresh-server-info.sh @@ -13,7 +13,9 @@ # For each host: # 1. Run scripts/server_inspect.sh on the remote via `ssh 'bash -s'`. # 2. Write output atomically to `servers//system-details.txt`. -# A failed SSH/run never clobbers the previous good snapshot. +# A failed SSH/run never clobbers the previous good snapshot — and neither +# does a SUCCEEDED run that produced nothing (ssh exit 0 is not proof of a +# usable capture; an empty one is refused and counted as a failure). # # Exit status is non-zero if any host failed. # @@ -241,10 +243,35 @@ for host in "${HOSTS[@]}"; do mkdir -p "$SERVERS_DIR/$host" if ssh -o BatchMode=yes -o ConnectTimeout=10 -o StrictHostKeyChecking=accept-new "$target" 'bash -s' < "$INSPECT" > "$tmp" 2> "$tmp.err"; then + # ⚠ ssh exiting 0 is NOT proof the capture is usable. The inspect script can + # emit nothing (a shell that dies before its first write, output swallowed by + # a remote wrapper) and this would then promote an EMPTY file over a good + # snapshot and report "ok (0 bytes)" — the readers below all test the + # snapshot with `-s`, so the writer must not produce something they consider + # invalid. A guard whose test disagrees with its writer's contract has + # quietly stopped guarding. Refuse the promotion, keep the old snapshot, + # and count it as a failure so the exit code carries it. + new_bytes=$(wc -c < "$tmp" 2>/dev/null || echo 0) + if [ "$new_bytes" -eq 0 ]; then + rm -f "$tmp" "$tmp.err" + printf 'FAIL (empty capture — previous snapshot kept)\n' + failed+=("$host") + [ "${#warnings[@]}" -gt 0 ] && print_warnings " " "${warnings[@]}" + continue + fi + # A capture that collapses to a fraction of the previous one is suspicious + # but not provably wrong (a host really can shed services), so this WARNS and + # still promotes — the operator sees it rather than the script guessing. + shrink="" + if [ -s "$out" ]; then + old_bytes=$(wc -c < "$out") + if [ "$old_bytes" -gt 0 ] && [ $((new_bytes * 4)) -lt "$old_bytes" ]; then + shrink=" ⚠ shrank from ${old_bytes}B — check before trusting" + fi + fi mv "$tmp" "$out" rm -f "$tmp.err" - bytes=$(wc -c < "$out") - printf 'ok (%s bytes)\n' "$bytes" + printf 'ok (%s bytes)%s\n' "$new_bytes" "$shrink" else rc=$? rm -f "$tmp" diff --git a/servers/nh3-dev/system-details.txt b/servers/nh3-dev/system-details.txt index eb9606a..926450a 100644 --- a/servers/nh3-dev/system-details.txt +++ b/servers/nh3-dev/system-details.txt @@ -2,18 +2,18 @@ ===== HOST ===== Hostname: nh3-dev.phasefinal.com -Date: 2026-06-04T14:02:20-07:00 -Uptime: up 4 weeks, 1 day, 2 hours, 58 minutes +Date: 2026-09-09T16:58:11-07:00 +Uptime: up 1 week, 5 days, 11 hours, 10 minutes OS: Debian GNU/Linux 12 (bookworm) -Kernel: 6.1.0-45-amd64 +Kernel: 6.1.0-49-amd64 Arch: x86_64 ===== HARDWARE ===== -CPU cores: 12 +CPU cores: 16 CPU model: QEMU Virtual CPU version 2.5+ -MemTotal: 15.6 GB -MemAvailable: 3.1 GB +MemTotal: 27.4 GB +MemAvailable: 12.0 GB ===== GPUS ===== @@ -22,9 +22,9 @@ nvidia-smi not present (no NVIDIA GPUs or driver not installed) ===== FILESYSTEMS (df) ===== Filesystem Size Used Avail Use% Mounted on -/dev/sda1 245G 70G 162G 31% / -10.100.50.50:/volume1/Backup 42T 28T 15T 67% /mnt/backup -10.100.50.50:/volume1/smithy 42T 28T 15T 67% /mnt/smithy +/dev/sda1 245G 173G 60G 75% / +10.100.50.50:/volume1/Backup 42T 32T 11T 76% /mnt/backup +10.100.50.50:/volume1/smithy 42T 32T 11T 76% /mnt/smithy ===== PERSISTENT MOUNTS (/etc/fstab, non-comment) ===== @@ -36,17 +36,18 @@ UUID=97bca850-bf72-4881-ae33-23d9b68315b6 none swap sw ===== TARGETED DATA PATHS ===== -/opt (total: 4.0K) - total 8 - drwxr-xr-x 2 root root 4096 2024-06-12 12:59 . - drwxr-xr-x 18 root root 4096 2026-05-06 11:00 .. +/opt (total: 1.9G) + total 12 + drwxr-xr-x 3 root root 4096 2026-06-04 14:08 . + drwxr-xr-x 18 root root 4096 2026-06-07 13:16 .. + drwxr-xr-x 10 root root 4096 2026-08-01 12:41 ms-playwright /var/lib/docker (total: 4.0K) /srv (total: 4.0K) total 8 drwxr-xr-x 2 root root 4096 2024-06-12 12:59 . - drwxr-xr-x 18 root root 4096 2026-05-06 11:00 .. + drwxr-xr-x 18 root root 4096 2026-06-07 13:16 .. ===== DOCKER ===== @@ -54,8 +55,8 @@ UUID=97bca850-bf72-4881-ae33-23d9b68315b6 none swap sw Server: 20.10.24+dfsg1 Client: 20.10.24+dfsg1 ----- docker info ----- -Containers: 1 (running 0, paused 0, stopped 1) -Images: 164 +Containers: 5 (running 3, paused 0, stopped 2) +Images: 325 Runtimes: map[io.containerd.runc.v2:{runc [] } io.containerd.runtime.v1.linux:{runc [] } runc:{runc [] }] Default runtime: runc Storage driver: overlay2 @@ -63,25 +64,53 @@ Root dir: /var/lib/docker Server version: 20.10.24+dfsg1 ----- running containers ----- -NAMES IMAGE STATUS PORTS +NAMES IMAGE STATUS PORTS +talk local/talk:v4 Up 18 hours (healthy) 0.0.0.0:8092->8443/tcp +vastblue-u5-pg postgres:16 Up 3 days 0.0.0.0:55433->5432/tcp, :::55433->5432/tcp ----- all containers ----- -NAMES IMAGE STATUS -sharp_sinoussi b17950483b74 Exited (2) 2 weeks ago +NAMES IMAGE STATUS +talk local/talk:v4 Up 18 hours (healthy) +vastblue-u5-pg postgres:16 Up 3 days +althing-post-office althing-post-office:local Exited (137) 12 days ago +grimnir-dummy-target python:3.13-slim Exited (255) 12 days ago +sharp_sinoussi b17950483b74 Exited (2) 3 months ago ----- networks ----- -NAME DRIVER SCOPE -bridge bridge local -host host local -none null local +NAME DRIVER SCOPE +bridge bridge local +grimnir-tracer-net bridge local +host host local +none null local +talk_default bridge local ----- networks (external, non-default — worth knowing for compose external: true) ----- +grimnir-tracer-net +talk_default ----- named volumes ----- -VOLUME NAME DRIVER +VOLUME NAME DRIVER +0a1ad370e11b3e41c565429b99885039a6ef60baec65f7d86cff7659381212e8 local +1a7877d6e4f02056b6d19d89d30c8cdd163719ded8ae12bfef94471ef765abdf local +1f934d7d319da86d2b624d15f16477bce02fae4f3ca9a64b618e521e1f61c231 local +02c20a181ec3d9dfefa130e9474633c073cfddcaa0bea7e66e50e9c901e59cd4 local +6ec972c1bab4f004e666d3cab20f77b2985157a6a3a075efdb826a4eec8399cd local +6f79f1af8583f182df4660a9e0b45f3037e7462ac81dcf5a9f8328a4f8f04a3e local +8b1d77980b517df27267dc5664045a87070c48936658dfb5206513e2a53c7c8a local +047ff78fe8797c88116aa33952c2ee2b550212d94a320f277b6aa57ea17b555f local +054c13b2bc932eb4b26bcde2be65d2685d2dd5446f5174e1799209f839f99bef local +758b3c52f02f3b1984f8880d83e810760729bd612716d9a40102896d95a2891a local +2043e5d75f88373bc0c9d55b14d4c95487e1adbe040f7168deb481cde6468410 local +a6e4cbf1f328c986293a587e142fe1dd8ec592a62170f2ef783adb98a7b5ab25 local +aa6d3222027fbaba5a55455914324537dabdbdf21ce970f52ac222fc5431b0f1 local +althing-post-office-data local +c3fded26f46f925e6a5e2b6fcf75c56b78544af8a1bbfaf978bde4450804c625 local +cf2390953b60b0aa0b9300a6eb3f85078236c27b2538fa8af1219b5a16f6a563 local +d766391ab892abc6f08466acd5d53ee7cae780f68048aafeeb2643b101922716 local +supabase_edge_runtime_jackdaw local ----- compose projects currently running ----- -(none) +talk ===== COMPOSE FILES (/opt/docker/compose/) ===== @@ -95,35 +124,72 @@ VOLUME NAME DRIVER 0.0.0.0:111 0.0.0.0:22 -0.0.0.0:5173 -0.0.0.0:5175 -0.0.0.0:8765 -10.100.10.50:1080 +0.0.0.0:5174 +0.0.0.0:55433 +0.0.0.0:8010 +0.0.0.0:8090 +0.0.0.0:8092 +0.0.0.0:8093 +0.0.0.0:8321 +0.0.0.0:8391 +0.0.0.0:8443 +0.0.0.0:8770 +0.0.0.0:8791 +100.64.0.4:37067 [::]:111 127.0.0.1:17681 127.0.0.1:17682 127.0.0.1:2019 -127.0.0.1:42851 +127.0.0.1:2201 +127.0.0.1:38111 +127.0.0.1:8000 +127.0.0.1:9477 +[::1]:5333 +[::1]:9477 [::]:22 +*:443 +*:5173 +[::]:55433 *:7681 *:7682 +*:80 +*:8477 +[fd7a:115c:a1e0::4]:40203 ===== MODEL / HUGGINGFACE CACHES ===== -/home/lkraven/.cache/huggingface (2.0G) +/home/lkraven/.cache/huggingface (7.0G) hub entries: CACHEDIR.TAG + datasets--agentlans--combined-roleplay + datasets--anthracite-org--c2_logs_32k_llama3_qwen2_v1.2 + datasets--anthracite-org--kalo-opus-instruct-22k-no-refusal + datasets--anthracite-org--nopm_claude_writing_fixed + datasets--anthracite-org--stheno-filtered-v1.1 datasets--bigcode--bigcodebench-hard + datasets--ccdv--govreport-summarization + datasets--Dampfinchen--Creative_Writing_Multiturn datasets--deepvk--NonverbalTTS + datasets--detection-datasets--coco + datasets--glaiveai--glaive-function-calling-v2 datasets--go_emotions - models--pyannote--segmentation-3.0 - models--pyannote--speaker-diarization-3.1 - models--pyannote--speaker-diarization-community-1 - models--pyannote--wespeaker-voxceleb-resnet34-LM - models--Qwen--Qwen3-Embedding-0.6B - models--SamLowe--roberta-base-go_emotions - models--sentence-transformers--all-MiniLM-L6-v2 - models--Systran--faster-whisper-base + datasets--interstellarninja--hermes_reasoning_tool_use + datasets--jondurbin--airoboros-3.2 + datasets--jondurbin--gutenberg-dpo-v0.1 + datasets--kalomaze--Opus_Instruct_3k + datasets--laion--Project-Gutenberg + datasets--lara-martin--FIREBALL + datasets--lemon07r--VellumK2T-Fiction-SFT-01 + datasets--lemonilia--LimaRP + datasets--Moemu--Muice-Dataset + datasets--mpasila--LimaRP-PIPPA-Mix-8K-Context + datasets--mrcuddle--literotica-pile + datasets--open-thoughts--OpenThoughts-114k + datasets--open-thoughts--OpenThoughts3-1.2M + datasets--PleIAs--common_corpus + datasets--pszemraj--qmsum-cleaned + datasets--PygmalionAI--PIPPA + datasets--rafaelpadilla--coco2017 ===== DOCKER-ADJACENT SYSTEMD SERVICES =====