From 48e34e184990b7bd48c29291ec5772440674fe89 Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Sat, 12 Sep 2026 22:04:41 -0700 Subject: [PATCH] playbooks: repoint the four remaining live fv-ml1 targets homepage-regroup, mog-sec-move-to-gpu0, pull-hf-repo and serve-qwen3.5-122b all carried runnable 'scripts/elway ana-ml2 --playbook ...' instructions or the old 10.250.50.54 address. Each would fail today against a dead name and a dead IP, so these are corrections rather than cosmetics. homepage-regroup is renamed to match; the other three keep their names, which never carried the host. --- playbooks/esh-vm-db-restic-repair.yaml | 33 +++++++++++++++++++ ...-ml2.yaml => homepage-regroup-fv-ml1.yaml} | 4 +-- playbooks/mog-sec-move-to-gpu0.yaml | 4 +-- playbooks/pull-hf-repo.yaml | 8 ++--- playbooks/serve-qwen3.5-122b.yaml | 4 +-- 5 files changed, 43 insertions(+), 10 deletions(-) create mode 100644 playbooks/esh-vm-db-restic-repair.yaml rename playbooks/{homepage-regroup-ana-ml2.yaml => homepage-regroup-fv-ml1.yaml} (94%) diff --git a/playbooks/esh-vm-db-restic-repair.yaml b/playbooks/esh-vm-db-restic-repair.yaml new file mode 100644 index 0000000..fcc54ab --- /dev/null +++ b/playbooks/esh-vm-db-restic-repair.yaml @@ -0,0 +1,33 @@ +steps: + - name: Preserve old hook and staged PostgreSQL dump + sudo: true + shell: | + set -eu + install -d -m 0700 /var/lib/restic/repair-20260912 + test -e /var/lib/restic/repair-20260912/pre-backup.sh || cp -p /etc/restic/pre-backup.sh /var/lib/restic/repair-20260912/pre-backup.sh + test -e /var/lib/restic/repair-20260912/pg_dumpall.sql.gz || cp -p /var/lib/restic/stage/pg_dumpall.sql.gz /var/lib/restic/repair-20260912/pg_dumpall.sql.gz + - name: Install tested fail-closed database hook + sudo: true + upload: + src: configs/restic/esh-vm-db/pre-backup.sh + dest: /etc/restic/pre-backup.sh + mode: '0700' + - name: Add bounded retry to backup job + sudo: true + upload: + src: configs/restic/esh-vm-db/retry.conf + dest: /etc/systemd/system/resticprofile-backup@profile-default.service.d/retry.conf + mode: '0644' + - name: Add bounded retry to repository check + sudo: true + upload: + src: configs/restic/esh-vm-db/retry.conf + dest: /etc/systemd/system/resticprofile-check@profile-default.service.d/retry.conf + mode: '0644' + - name: Reload service definitions and run fresh backup + sudo: true + shell: | + set -eu + bash -n /etc/restic/pre-backup.sh + systemctl daemon-reload + systemctl start resticprofile-backup@profile-default.service diff --git a/playbooks/homepage-regroup-ana-ml2.yaml b/playbooks/homepage-regroup-fv-ml1.yaml similarity index 94% rename from playbooks/homepage-regroup-ana-ml2.yaml rename to playbooks/homepage-regroup-fv-ml1.yaml index 988c565..c1f2824 100644 --- a/playbooks/homepage-regroup-ana-ml2.yaml +++ b/playbooks/homepage-regroup-fv-ml1.yaml @@ -1,4 +1,4 @@ -# Homepage recategorisation — ana-ml2 (10.250.50.54), 2 containers. +# Homepage recategorisation — fv-ml1 (10.251.50.54), 2 containers. # Sibling of playbooks/homepage-regroup-ana-docker.yaml; rationale lives there. # # scriberr -> AI - Studios (a transcription UI you open, not an API seat) @@ -13,7 +13,7 @@ # a future pass wants to rename `AI - Inference`, weigh it against bouncing six # model seats. # -# Run: scripts/elway infra-ops@10.250.50.54 --playbook playbooks/homepage-regroup-ana-ml2.yaml +# Run: scripts/elway infra-ops@10.251.50.54 --playbook playbooks/homepage-regroup-fv-ml1.yaml steps: - name: scriberr -> AI - Studios diff --git a/playbooks/mog-sec-move-to-gpu0.yaml b/playbooks/mog-sec-move-to-gpu0.yaml index e8e0c12..432816b 100644 --- a/playbooks/mog-sec-move-to-gpu0.yaml +++ b/playbooks/mog-sec-move-to-gpu0.yaml @@ -1,4 +1,4 @@ -# Move the `sec` pen-test seat (M.O.G.-SEC-27B) from ana-ml2 GPU 1 to GPU 0 and bring it up. +# Move the `sec` pen-test seat (M.O.G.-SEC-27B) from fv-ml1 GPU 1 to GPU 0 and bring it up. # # Why: GPU 1 carries the five resident fleet seats (gen 46 GB + embed 9.8 + coder 8.4 + # rerank 3.5 + reward 2.1 = ~69.9 GB of 97.9), leaving ~28 GB. This seat reserves @@ -13,7 +13,7 @@ # Labels only apply at container CREATION, so this uses `up -d`, never `restart` -- # the homepage description carries the GPU number and would otherwise stay stale. # -# Run: scripts/elway infra-ops@ana-ml2 --playbook playbooks/mog-sec-move-to-gpu0.yaml +# Run: scripts/elway infra-ops@fv-ml1 --playbook playbooks/mog-sec-move-to-gpu0.yaml # Model load is slow (22 GB + 262K ctx + MTP graft); the verify phase polls rather than # assuming readiness, and the compose healthcheck allows a 900s start_period. diff --git a/playbooks/pull-hf-repo.yaml b/playbooks/pull-hf-repo.yaml index 3d4cb14..024cfe5 100644 --- a/playbooks/pull-hf-repo.yaml +++ b/playbooks/pull-hf-repo.yaml @@ -1,4 +1,4 @@ -# Pull a model or dataset from HuggingFace into ana-ml2's shared HF cache +# Pull a model or dataset from HuggingFace into fv-ml1's shared HF cache # (`/tank/aimodels/huggingface/`). Repo-type-agnostic — handles GGUFs, # safetensors, datasets, or anything else HF hosts. # @@ -16,16 +16,16 @@ # Usage: # # # Full repo (e.g. safetensors for later AWQ quant + vLLM serving): -# scripts/elway ana-ml2 --playbook playbooks/pull-hf-repo.yaml \ +# scripts/elway fv-ml1 --playbook playbooks/pull-hf-repo.yaml \ # --var hf_repo=Skywork/Skywork-Reward-V2-Llama-3.1-8B # # # Single quant pattern (e.g. one GGUF for llama-swap): -# scripts/elway ana-ml2 --playbook playbooks/pull-hf-repo.yaml \ +# scripts/elway fv-ml1 --playbook playbooks/pull-hf-repo.yaml \ # --var hf_repo=mradermacher/Selene-1-Mini-Llama-3.1-8B-GGUF \ # --var allow_patterns='*Q6_K*' # # # Dataset (preference data, eval set, etc.): -# scripts/elway ana-ml2 --playbook playbooks/pull-hf-repo.yaml \ +# scripts/elway fv-ml1 --playbook playbooks/pull-hf-repo.yaml \ # --var hf_repo=Skywork/Skywork-Reward-Preference-80K-v0.2 \ # --var repo_type=dataset # diff --git a/playbooks/serve-qwen3.5-122b.yaml b/playbooks/serve-qwen3.5-122b.yaml index 663c897..aa6718e 100644 --- a/playbooks/serve-qwen3.5-122b.yaml +++ b/playbooks/serve-qwen3.5-122b.yaml @@ -1,4 +1,4 @@ -# Displace mistral-small-4 (heretic) on ana-ml2 GPU 0 and serve +# Displace mistral-small-4 (heretic) on fv-ml1 GPU 0 and serve # bjk110/Qwen3.5-122B-A10B-abliterated-NVFP4 (text-only) as the new `gen` model # (operator 2026-06-19). Weights pre-staged at /tank/aimodels/qwen3.5-122b-a10b-nvfp4 # (incl. the repo's serving/entrypoint.sh + vllm_patches/ that the compose mounts). @@ -6,7 +6,7 @@ # ⚠️ Downing mistral-small-4 takes down the Worldtree CHARACTER backend (vision-intact) # until it's repointed — operator-acknowledged. REVERT = down qwen, up -d the heretic. # -# scripts/elway ana-ml2 --playbook playbooks/serve-qwen3.5-122b.yaml +# scripts/elway fv-ml1 --playbook playbooks/serve-qwen3.5-122b.yaml vars: compose_dir: /opt/docker/compose/qwen3.5-122b