diff --git a/scripts/upgrade-irv-ml1-cuda.sh b/scripts/upgrade-irv-ml1-cuda.sh index e6b3668..7ea9685 100755 --- a/scripts/upgrade-irv-ml1-cuda.sh +++ b/scripts/upgrade-irv-ml1-cuda.sh @@ -18,87 +18,143 @@ # Verify new driver + CUDA version, restart containers, # and print the next step (flip Kokoro to GPU variant). # -# Idempotent: rerunning phase1 after the repo is already added is a no-op -# until the apt step. phase2 can be re-run safely; container restarts -# via `docker compose up -d` are idempotent. +# Implementation note: the remote work runs from a script scp'd to +# /tmp on irv-ml1, then invoked via `ssh -t host bash /tmp/script`. +# Piping a heredoc to `ssh -t host bash -s` doesn't work — `-t` needs +# a real TTY but heredoc puts stdin in pipe mode (orientation.md +# gotcha; tripped over once already). +# +# Idempotent: rerunning phase1 after the repo is already added is a +# no-op until the apt step. phase2 can be re-run safely; container +# restarts via `docker compose up -d` are idempotent. set -euo pipefail HOST=irv-ml1 -CONTAINERS=(comfyui cosyvoice qwen3-tts index-tts parakeet kokoro) +REMOTE_SCRIPT=/tmp/upgrade-cuda-remote.sh say() { printf '\033[1;36m▸ %s\033[0m\n' "$*"; } ok() { printf '\033[1;32m✓ %s\033[0m\n' "$*"; } warn() { printf '\033[1;33m⚠ %s\033[0m\n' "$*"; } err() { printf '\033[1;31m✗ %s\033[0m\n' "$*" >&2; } -phase1() { - say "Phase 1: pre-flight + install + reboot" - say "Connecting to $HOST (sudo will prompt once)" - - # -t requests a TTY so sudo can prompt. The whole heredoc runs as - # a single remote shell — sudo's credential cache covers all - # subsequent invocations within it. - ssh -t "$HOST" "bash -s" < "$local_tmp" <<'REMOTE' +#!/usr/bin/env bash +# Remote half of upgrade-irv-ml1-cuda.sh — runs on irv-ml1. set -euo pipefail -echo "=== current driver + CUDA ===" -nvidia-smi --query-gpu=driver_version --format=csv,noheader -nvidia-smi | grep "CUDA Version" +CONTAINERS=(comfyui cosyvoice qwen3-tts index-tts parakeet kokoro) -echo -echo "=== prime sudo (we only want one password prompt this whole script) ===" -sudo -v +remote_phase1() { + echo "=== current driver + CUDA ===" + nvidia-smi --query-gpu=driver_version --format=csv,noheader + nvidia-smi | grep "CUDA Version" -echo -echo "=== adding NVIDIA CUDA APT repo for Debian 12 ===" -sudo install -d -m 0755 /etc/apt/keyrings -if [ ! -f /etc/apt/keyrings/nvidia-cuda.gpg ]; then - sudo apt-get install -y curl gpg - curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/debian12/x86_64/3bf863cc.pub \\ - | sudo gpg --dearmor --yes -o /etc/apt/keyrings/nvidia-cuda.gpg -fi -echo "deb [signed-by=/etc/apt/keyrings/nvidia-cuda.gpg] https://developer.download.nvidia.com/compute/cuda/repos/debian12/x86_64/ /" \\ - | sudo tee /etc/apt/sources.list.d/nvidia-cuda.list >/dev/null + echo + echo "=== prime sudo (one password prompt for everything below) ===" + sudo -v -sudo apt-get update -qq - -echo -echo "=== cuda-drivers candidate ===" -apt-cache policy cuda-drivers | head -10 - -echo -echo "=== snapshotting current nvidia packages (rollback reference) ===" -dpkg -l | grep -E "^ii\s+(nvidia-|cuda-)" | awk '{print \$2 "=" \$3}' \\ - | sudo tee /tmp/nvidia-pre-upgrade.txt >/dev/null -echo "snapshot at /tmp/nvidia-pre-upgrade.txt" - -echo -echo "=== stopping GPU containers ===" -for stack in ${CONTAINERS[*]}; do - if [ -d "/opt/docker/compose/\$stack" ]; then - echo " stopping \$stack" - (cd "/opt/docker/compose/\$stack" && docker compose stop) || echo " (no-op)" + echo + echo "=== adding NVIDIA CUDA APT repo for Debian 12 ===" + sudo install -d -m 0755 /etc/apt/keyrings + if [ ! -f /etc/apt/keyrings/nvidia-cuda.gpg ]; then + sudo apt-get install -y curl gpg + curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/debian12/x86_64/3bf863cc.pub \ + | sudo gpg --dearmor --yes -o /etc/apt/keyrings/nvidia-cuda.gpg fi -done + echo "deb [signed-by=/etc/apt/keyrings/nvidia-cuda.gpg] https://developer.download.nvidia.com/compute/cuda/repos/debian12/x86_64/ /" \ + | sudo tee /etc/apt/sources.list.d/nvidia-cuda.list >/dev/null -echo -echo "=== installing cuda-drivers (DKMS rebuild ~3-5 min) ===" -sudo DEBIAN_FRONTEND=noninteractive apt-get install -y cuda-drivers + sudo apt-get update -qq -echo -echo "=== rebooting in 5 s ===" -sleep 5 -sudo systemctl reboot -EOF + echo + echo "=== cuda-drivers candidate ===" + apt-cache policy cuda-drivers | head -10 + echo + echo "=== snapshotting current nvidia packages (rollback reference) ===" + dpkg -l | awk '/^ii\s+(nvidia-|cuda-)/ {print $2 "=" $3}' \ + | sudo tee /tmp/nvidia-pre-upgrade.txt >/dev/null + echo "snapshot at /tmp/nvidia-pre-upgrade.txt" + + echo + echo "=== stopping GPU containers ===" + for stack in "${CONTAINERS[@]}"; do + if [ -d "/opt/docker/compose/$stack" ]; then + echo " stopping $stack" + (cd "/opt/docker/compose/$stack" && docker compose stop) || echo " (no-op)" + fi + done + + echo + echo "=== installing cuda-drivers (DKMS rebuild ~3-5 min) ===" + sudo DEBIAN_FRONTEND=noninteractive apt-get install -y cuda-drivers + + echo + echo "=== rebooting in 5 s ===" + sleep 5 + sudo systemctl reboot +} + +remote_phase2() { + echo "=== new driver + CUDA ===" + nvidia-smi --query-gpu=driver_version --format=csv,noheader + nvidia-smi | grep "CUDA Version" + + echo + echo "=== restarting GPU containers ===" + for stack in comfyui cosyvoice qwen3-tts index-tts parakeet; do + if [ -d "/opt/docker/compose/$stack" ]; then + echo " starting $stack" + (cd "/opt/docker/compose/$stack" && docker compose up -d) || echo " (failed — check manually)" + fi + done + + sleep 8 + echo + echo "=== container status ===" + docker ps --format "table {{.Names}}\t{{.Status}}" | head -15 +} + +remote_rollback() { + echo "=== rollback: reinstalling pre-upgrade nvidia packages ===" + [ -f /tmp/nvidia-pre-upgrade.txt ] || { echo "no snapshot found"; exit 1; } + sudo apt-get install -y --allow-downgrades $(tr '\n' ' ' < /tmp/nvidia-pre-upgrade.txt) + echo "=== rebooting in 5 s ===" + sleep 5 + sudo systemctl reboot +} + +case "${1:-}" in + phase1) remote_phase1 ;; + phase2) remote_phase2 ;; + rollback) remote_rollback ;; + *) echo "remote: usage: $0 {phase1|phase2|rollback}" >&2; exit 2 ;; +esac +REMOTE + scp -q "$local_tmp" "$HOST:$REMOTE_SCRIPT" + rm -f "$local_tmp" + ssh "$HOST" "chmod +x $REMOTE_SCRIPT" +} + +phase1() { + say "Phase 1: pre-flight + install + reboot" + say "Staging remote script on $HOST" + write_remote_script + say "Running phase1 (sudo will prompt once via ssh -t)" + # No piped stdin — ssh -t's TTY is real and sudo can prompt. + ssh -t "$HOST" "$REMOTE_SCRIPT phase1" ok "Phase 1 dispatched. Host rebooting." say "Wait ~2 min, then run: $0 phase2" } phase2() { say "Phase 2: wait for host, verify, restart containers" - say "waiting for $HOST to come back online…" for i in $(seq 1 60); do if ssh -o ConnectTimeout=3 -o BatchMode=yes "$HOST" true 2>/dev/null; then @@ -110,30 +166,12 @@ phase2() { done echo - ssh "$HOST" 'bash -s' <<'EOF' -set -euo pipefail - -echo "=== new driver + CUDA ===" -nvidia-smi --query-gpu=driver_version --format=csv,noheader -nvidia-smi | grep "CUDA Version" - -echo -echo "=== restarting GPU containers ===" -for stack in comfyui cosyvoice qwen3-tts index-tts parakeet; do - if [ -d "/opt/docker/compose/$stack" ]; then - echo " starting $stack" - (cd "/opt/docker/compose/$stack" && docker compose up -d) || echo " (failed — check manually)" - fi -done - -sleep 8 -echo -echo "=== container status ===" -docker ps --format "table {{.Names}}\t{{.Status}}" | head -15 -EOF + # Re-stage in case the previous /tmp script was wiped on reboot. + write_remote_script + ssh "$HOST" "$REMOTE_SCRIPT phase2" ok "Phase 2 complete." - cat <<'EOF' + cat <<'NEXT' ──────────────────────────────────────────────────────────────────── NEXT — flip Kokoro from cpu → gpu variant now that CUDA 12.9 is live: @@ -146,22 +184,17 @@ NEXT — flip Kokoro from cpu → gpu variant now that CUDA 12.9 is live: ' scripts/elway irv-ml1 --playbook playbooks/deploy-kokoro.yaml -After that, three more deploys (re-runnable now that the driver is current): +After that, the two outstanding deploys: scripts/elway irv-ml1 --playbook playbooks/deploy-vibevoice.yaml scripts/elway irv-ml1 --playbook playbooks/deploy-chatterbox.yaml ──────────────────────────────────────────────────────────────────── -EOF +NEXT } -# rollback helper — restores the snapshot from /tmp/nvidia-pre-upgrade.txt rollback() { say "Rollback: reinstalling pre-upgrade nvidia packages on $HOST" - ssh -t "$HOST" 'bash -s' <<'EOF' -set -euo pipefail -[ -f /tmp/nvidia-pre-upgrade.txt ] || { echo "no snapshot found"; exit 1; } -sudo apt-get install -y --allow-downgrades $(cat /tmp/nvidia-pre-upgrade.txt | tr '\n' ' ') -sudo systemctl reboot -EOF + write_remote_script + ssh -t "$HOST" "$REMOTE_SCRIPT rollback" ok "Rollback dispatched. Host rebooting." }