scripts/upgrade-irv-ml1-cuda: stage remote script via scp, exec via ssh -t

The previous version piped a heredoc to `ssh -t host bash -s` —
which can't work because `-t` requires a real TTY and heredoc puts
stdin in pipe mode. They're mutually exclusive (orientation.md
gotcha, tripped over once already). The remote sudo therefore aborted
with 'a terminal is required to read the password' before doing
anything.

Restructured: the remote half of each phase is built as a tempfile
locally, scp'd to /tmp on irv-ml1, made executable, and invoked via
`ssh -t host /tmp/upgrade-cuda-remote.sh phase1`. Now stdin is the
real TTY (no pipe), sudo prompts work, and the rest of the script
runs as before.

Same three phases, same rollback behavior, same idempotency story.
Just a transport fix for the local→remote half.

scripts/upgrade-irv-ml1-cuda.sh

@@ -18,87 +18,143 @@
 #       Verify new driver + CUDA version, restart containers,
 #       and print the next step (flip Kokoro to GPU variant).
 #
-# Idempotent: rerunning phase1 after the repo is already added is a no-op
+# Implementation note: the remote work runs from a script scp'd to
-# until the apt step. phase2 can be re-run safely; container restarts
+# /tmp on irv-ml1, then invoked via `ssh -t host bash /tmp/script`.
-# via `docker compose up -d` are idempotent.
+# Piping a heredoc to `ssh -t host bash -s` doesn't work — `-t` needs
+# a real TTY but heredoc puts stdin in pipe mode (orientation.md
+# gotcha; tripped over once already).
+#
+# Idempotent: rerunning phase1 after the repo is already added is a
+# no-op until the apt step. phase2 can be re-run safely; container
+# restarts via `docker compose up -d` are idempotent.
 
 set -euo pipefail
 
 HOST=irv-ml1
-CONTAINERS=(comfyui cosyvoice qwen3-tts index-tts parakeet kokoro)
+REMOTE_SCRIPT=/tmp/upgrade-cuda-remote.sh
 
 say()  { printf '\033[1;36m▸ %s\033[0m\n' "$*"; }
 ok()   { printf '\033[1;32m✓ %s\033[0m\n' "$*"; }
 warn() { printf '\033[1;33m⚠ %s\033[0m\n' "$*"; }
 err()  { printf '\033[1;31m✗ %s\033[0m\n' "$*" >&2; }
 
-phase1() {
+# Build the remote-side script as a local tempfile, scp it, exec it
-    say "Phase 1: pre-flight + install + reboot"
+# via `ssh -t`. The remote script takes its own phase as $1.
-    say "Connecting to $HOST (sudo will prompt once)"
+write_remote_script() {
-
+    local local_tmp
-    # -t requests a TTY so sudo can prompt. The whole heredoc runs as
+    local_tmp=$(mktemp)
-    # a single remote shell — sudo's credential cache covers all
+    cat > "$local_tmp" <<'REMOTE'
-    # subsequent invocations within it.
+#!/usr/bin/env bash
-    ssh -t "$HOST" "bash -s" <<EOF
+# Remote half of upgrade-irv-ml1-cuda.sh — runs on irv-ml1.
 set -euo pipefail
 
-echo "=== current driver + CUDA ==="
+CONTAINERS=(comfyui cosyvoice qwen3-tts index-tts parakeet kokoro)
-nvidia-smi --query-gpu=driver_version --format=csv,noheader
-nvidia-smi | grep "CUDA Version"
 
-echo
+remote_phase1() {
-echo "=== prime sudo (we only want one password prompt this whole script) ==="
+    echo "=== current driver + CUDA ==="
-sudo -v
+    nvidia-smi --query-gpu=driver_version --format=csv,noheader
+    nvidia-smi | grep "CUDA Version"
 
-echo
+    echo
-echo "=== adding NVIDIA CUDA APT repo for Debian 12 ==="
+    echo "=== prime sudo (one password prompt for everything below) ==="
-sudo install -d -m 0755 /etc/apt/keyrings
+    sudo -v
-if [ ! -f /etc/apt/keyrings/nvidia-cuda.gpg ]; then
-    sudo apt-get install -y curl gpg
-    curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/debian12/x86_64/3bf863cc.pub \\
-        | sudo gpg --dearmor --yes -o /etc/apt/keyrings/nvidia-cuda.gpg
-fi
-echo "deb [signed-by=/etc/apt/keyrings/nvidia-cuda.gpg] https://developer.download.nvidia.com/compute/cuda/repos/debian12/x86_64/ /" \\
-    | sudo tee /etc/apt/sources.list.d/nvidia-cuda.list >/dev/null
 
-sudo apt-get update -qq
+    echo
-
+    echo "=== adding NVIDIA CUDA APT repo for Debian 12 ==="
-echo
+    sudo install -d -m 0755 /etc/apt/keyrings
-echo "=== cuda-drivers candidate ==="
+    if [ ! -f /etc/apt/keyrings/nvidia-cuda.gpg ]; then
-apt-cache policy cuda-drivers | head -10
+        sudo apt-get install -y curl gpg
-
+        curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/debian12/x86_64/3bf863cc.pub \
-echo
+            | sudo gpg --dearmor --yes -o /etc/apt/keyrings/nvidia-cuda.gpg
-echo "=== snapshotting current nvidia packages (rollback reference) ==="
-dpkg -l | grep -E "^ii\s+(nvidia-|cuda-)" | awk '{print \$2 "=" \$3}' \\
-    | sudo tee /tmp/nvidia-pre-upgrade.txt >/dev/null
-echo "snapshot at /tmp/nvidia-pre-upgrade.txt"
-
-echo
-echo "=== stopping GPU containers ==="
-for stack in ${CONTAINERS[*]}; do
-    if [ -d "/opt/docker/compose/\$stack" ]; then
-        echo "  stopping \$stack"
-        (cd "/opt/docker/compose/\$stack" && docker compose stop) || echo "    (no-op)"
     fi
-done
+    echo "deb [signed-by=/etc/apt/keyrings/nvidia-cuda.gpg] https://developer.download.nvidia.com/compute/cuda/repos/debian12/x86_64/ /" \
+        | sudo tee /etc/apt/sources.list.d/nvidia-cuda.list >/dev/null
 
-echo
+    sudo apt-get update -qq
-echo "=== installing cuda-drivers (DKMS rebuild ~3-5 min) ==="
-sudo DEBIAN_FRONTEND=noninteractive apt-get install -y cuda-drivers
 
-echo
+    echo
-echo "=== rebooting in 5 s ==="
+    echo "=== cuda-drivers candidate ==="
-sleep 5
+    apt-cache policy cuda-drivers | head -10
-sudo systemctl reboot
-EOF
 
+    echo
+    echo "=== snapshotting current nvidia packages (rollback reference) ==="
+    dpkg -l | awk '/^ii\s+(nvidia-|cuda-)/ {print $2 "=" $3}' \
+        | sudo tee /tmp/nvidia-pre-upgrade.txt >/dev/null
+    echo "snapshot at /tmp/nvidia-pre-upgrade.txt"
+
+    echo
+    echo "=== stopping GPU containers ==="
+    for stack in "${CONTAINERS[@]}"; do
+        if [ -d "/opt/docker/compose/$stack" ]; then
+            echo "  stopping $stack"
+            (cd "/opt/docker/compose/$stack" && docker compose stop) || echo "    (no-op)"
+        fi
+    done
+
+    echo
+    echo "=== installing cuda-drivers (DKMS rebuild ~3-5 min) ==="
+    sudo DEBIAN_FRONTEND=noninteractive apt-get install -y cuda-drivers
+
+    echo
+    echo "=== rebooting in 5 s ==="
+    sleep 5
+    sudo systemctl reboot
+}
+
+remote_phase2() {
+    echo "=== new driver + CUDA ==="
+    nvidia-smi --query-gpu=driver_version --format=csv,noheader
+    nvidia-smi | grep "CUDA Version"
+
+    echo
+    echo "=== restarting GPU containers ==="
+    for stack in comfyui cosyvoice qwen3-tts index-tts parakeet; do
+        if [ -d "/opt/docker/compose/$stack" ]; then
+            echo "  starting $stack"
+            (cd "/opt/docker/compose/$stack" && docker compose up -d) || echo "    (failed — check manually)"
+        fi
+    done
+
+    sleep 8
+    echo
+    echo "=== container status ==="
+    docker ps --format "table {{.Names}}\t{{.Status}}" | head -15
+}
+
+remote_rollback() {
+    echo "=== rollback: reinstalling pre-upgrade nvidia packages ==="
+    [ -f /tmp/nvidia-pre-upgrade.txt ] || { echo "no snapshot found"; exit 1; }
+    sudo apt-get install -y --allow-downgrades $(tr '\n' ' ' < /tmp/nvidia-pre-upgrade.txt)
+    echo "=== rebooting in 5 s ==="
+    sleep 5
+    sudo systemctl reboot
+}
+
+case "${1:-}" in
+    phase1)   remote_phase1 ;;
+    phase2)   remote_phase2 ;;
+    rollback) remote_rollback ;;
+    *) echo "remote: usage: $0 {phase1|phase2|rollback}" >&2; exit 2 ;;
+esac
+REMOTE
+    scp -q "$local_tmp" "$HOST:$REMOTE_SCRIPT"
+    rm -f "$local_tmp"
+    ssh "$HOST" "chmod +x $REMOTE_SCRIPT"
+}
+
+phase1() {
+    say "Phase 1: pre-flight + install + reboot"
+    say "Staging remote script on $HOST"
+    write_remote_script
+    say "Running phase1 (sudo will prompt once via ssh -t)"
+    # No piped stdin — ssh -t's TTY is real and sudo can prompt.
+    ssh -t "$HOST" "$REMOTE_SCRIPT phase1"
     ok "Phase 1 dispatched. Host rebooting."
     say "Wait ~2 min, then run: $0 phase2"
 }
 
 phase2() {
     say "Phase 2: wait for host, verify, restart containers"
-
     say "waiting for $HOST to come back online…"
     for i in $(seq 1 60); do
         if ssh -o ConnectTimeout=3 -o BatchMode=yes "$HOST" true 2>/dev/null; then
@@ -110,30 +166,12 @@ phase2() {
     done
     echo
 
-    ssh "$HOST" 'bash -s' <<'EOF'
+    # Re-stage in case the previous /tmp script was wiped on reboot.
-set -euo pipefail
+    write_remote_script
-
+    ssh "$HOST" "$REMOTE_SCRIPT phase2"
-echo "=== new driver + CUDA ==="
-nvidia-smi --query-gpu=driver_version --format=csv,noheader
-nvidia-smi | grep "CUDA Version"
-
-echo
-echo "=== restarting GPU containers ==="
-for stack in comfyui cosyvoice qwen3-tts index-tts parakeet; do
-    if [ -d "/opt/docker/compose/$stack" ]; then
-        echo "  starting $stack"
-        (cd "/opt/docker/compose/$stack" && docker compose up -d) || echo "    (failed — check manually)"
-    fi
-done
-
-sleep 8
-echo
-echo "=== container status ==="
-docker ps --format "table {{.Names}}\t{{.Status}}" | head -15
-EOF
 
     ok "Phase 2 complete."
-    cat <<'EOF'
+    cat <<'NEXT'
 
 ────────────────────────────────────────────────────────────────────
 NEXT — flip Kokoro from cpu → gpu variant now that CUDA 12.9 is live:
@@ -146,22 +184,17 @@ NEXT — flip Kokoro from cpu → gpu variant now that CUDA 12.9 is live:
   '
   scripts/elway irv-ml1 --playbook playbooks/deploy-kokoro.yaml
 
-After that, three more deploys (re-runnable now that the driver is current):
+After that, the two outstanding deploys:
   scripts/elway irv-ml1 --playbook playbooks/deploy-vibevoice.yaml
   scripts/elway irv-ml1 --playbook playbooks/deploy-chatterbox.yaml
 ────────────────────────────────────────────────────────────────────
-EOF
+NEXT
 }
 
-# rollback helper — restores the snapshot from /tmp/nvidia-pre-upgrade.txt
 rollback() {
     say "Rollback: reinstalling pre-upgrade nvidia packages on $HOST"
-    ssh -t "$HOST" 'bash -s' <<'EOF'
+    write_remote_script
-set -euo pipefail
+    ssh -t "$HOST" "$REMOTE_SCRIPT rollback"
-[ -f /tmp/nvidia-pre-upgrade.txt ] || { echo "no snapshot found"; exit 1; }
-sudo apt-get install -y --allow-downgrades $(cat /tmp/nvidia-pre-upgrade.txt | tr '\n' ' ')
-sudo systemctl reboot
-EOF
     ok "Rollback dispatched. Host rebooting."
 }