scripts/upgrade-irv-ml1-cuda: stage remote script via scp, exec via ssh -t
The previous version piped a heredoc to `ssh -t host bash -s` — which can't work because `-t` requires a real TTY and heredoc puts stdin in pipe mode. They're mutually exclusive (orientation.md gotcha, tripped over once already). The remote sudo therefore aborted with 'a terminal is required to read the password' before doing anything. Restructured: the remote half of each phase is built as a tempfile locally, scp'd to /tmp on irv-ml1, made executable, and invoked via `ssh -t host /tmp/upgrade-cuda-remote.sh phase1`. Now stdin is the real TTY (no pipe), sudo prompts work, and the rest of the script runs as before. Same three phases, same rollback behavior, same idempotency story. Just a transport fix for the local→remote half.
This commit is contained in:
+120
-87
@@ -18,87 +18,143 @@
|
||||
# Verify new driver + CUDA version, restart containers,
|
||||
# and print the next step (flip Kokoro to GPU variant).
|
||||
#
|
||||
# Idempotent: rerunning phase1 after the repo is already added is a no-op
|
||||
# until the apt step. phase2 can be re-run safely; container restarts
|
||||
# via `docker compose up -d` are idempotent.
|
||||
# Implementation note: the remote work runs from a script scp'd to
|
||||
# /tmp on irv-ml1, then invoked via `ssh -t host bash /tmp/script`.
|
||||
# Piping a heredoc to `ssh -t host bash -s` doesn't work — `-t` needs
|
||||
# a real TTY but heredoc puts stdin in pipe mode (orientation.md
|
||||
# gotcha; tripped over once already).
|
||||
#
|
||||
# Idempotent: rerunning phase1 after the repo is already added is a
|
||||
# no-op until the apt step. phase2 can be re-run safely; container
|
||||
# restarts via `docker compose up -d` are idempotent.
|
||||
|
||||
set -euo pipefail
|
||||
|
||||
HOST=irv-ml1
|
||||
CONTAINERS=(comfyui cosyvoice qwen3-tts index-tts parakeet kokoro)
|
||||
REMOTE_SCRIPT=/tmp/upgrade-cuda-remote.sh
|
||||
|
||||
say() { printf '\033[1;36m▸ %s\033[0m\n' "$*"; }
|
||||
ok() { printf '\033[1;32m✓ %s\033[0m\n' "$*"; }
|
||||
warn() { printf '\033[1;33m⚠ %s\033[0m\n' "$*"; }
|
||||
err() { printf '\033[1;31m✗ %s\033[0m\n' "$*" >&2; }
|
||||
|
||||
phase1() {
|
||||
say "Phase 1: pre-flight + install + reboot"
|
||||
say "Connecting to $HOST (sudo will prompt once)"
|
||||
|
||||
# -t requests a TTY so sudo can prompt. The whole heredoc runs as
|
||||
# a single remote shell — sudo's credential cache covers all
|
||||
# subsequent invocations within it.
|
||||
ssh -t "$HOST" "bash -s" <<EOF
|
||||
# Build the remote-side script as a local tempfile, scp it, exec it
|
||||
# via `ssh -t`. The remote script takes its own phase as $1.
|
||||
write_remote_script() {
|
||||
local local_tmp
|
||||
local_tmp=$(mktemp)
|
||||
cat > "$local_tmp" <<'REMOTE'
|
||||
#!/usr/bin/env bash
|
||||
# Remote half of upgrade-irv-ml1-cuda.sh — runs on irv-ml1.
|
||||
set -euo pipefail
|
||||
|
||||
echo "=== current driver + CUDA ==="
|
||||
nvidia-smi --query-gpu=driver_version --format=csv,noheader
|
||||
nvidia-smi | grep "CUDA Version"
|
||||
CONTAINERS=(comfyui cosyvoice qwen3-tts index-tts parakeet kokoro)
|
||||
|
||||
echo
|
||||
echo "=== prime sudo (we only want one password prompt this whole script) ==="
|
||||
sudo -v
|
||||
remote_phase1() {
|
||||
echo "=== current driver + CUDA ==="
|
||||
nvidia-smi --query-gpu=driver_version --format=csv,noheader
|
||||
nvidia-smi | grep "CUDA Version"
|
||||
|
||||
echo
|
||||
echo "=== adding NVIDIA CUDA APT repo for Debian 12 ==="
|
||||
sudo install -d -m 0755 /etc/apt/keyrings
|
||||
if [ ! -f /etc/apt/keyrings/nvidia-cuda.gpg ]; then
|
||||
sudo apt-get install -y curl gpg
|
||||
curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/debian12/x86_64/3bf863cc.pub \\
|
||||
| sudo gpg --dearmor --yes -o /etc/apt/keyrings/nvidia-cuda.gpg
|
||||
fi
|
||||
echo "deb [signed-by=/etc/apt/keyrings/nvidia-cuda.gpg] https://developer.download.nvidia.com/compute/cuda/repos/debian12/x86_64/ /" \\
|
||||
| sudo tee /etc/apt/sources.list.d/nvidia-cuda.list >/dev/null
|
||||
echo
|
||||
echo "=== prime sudo (one password prompt for everything below) ==="
|
||||
sudo -v
|
||||
|
||||
sudo apt-get update -qq
|
||||
|
||||
echo
|
||||
echo "=== cuda-drivers candidate ==="
|
||||
apt-cache policy cuda-drivers | head -10
|
||||
|
||||
echo
|
||||
echo "=== snapshotting current nvidia packages (rollback reference) ==="
|
||||
dpkg -l | grep -E "^ii\s+(nvidia-|cuda-)" | awk '{print \$2 "=" \$3}' \\
|
||||
| sudo tee /tmp/nvidia-pre-upgrade.txt >/dev/null
|
||||
echo "snapshot at /tmp/nvidia-pre-upgrade.txt"
|
||||
|
||||
echo
|
||||
echo "=== stopping GPU containers ==="
|
||||
for stack in ${CONTAINERS[*]}; do
|
||||
if [ -d "/opt/docker/compose/\$stack" ]; then
|
||||
echo " stopping \$stack"
|
||||
(cd "/opt/docker/compose/\$stack" && docker compose stop) || echo " (no-op)"
|
||||
echo
|
||||
echo "=== adding NVIDIA CUDA APT repo for Debian 12 ==="
|
||||
sudo install -d -m 0755 /etc/apt/keyrings
|
||||
if [ ! -f /etc/apt/keyrings/nvidia-cuda.gpg ]; then
|
||||
sudo apt-get install -y curl gpg
|
||||
curl -fsSL https://developer.download.nvidia.com/compute/cuda/repos/debian12/x86_64/3bf863cc.pub \
|
||||
| sudo gpg --dearmor --yes -o /etc/apt/keyrings/nvidia-cuda.gpg
|
||||
fi
|
||||
done
|
||||
echo "deb [signed-by=/etc/apt/keyrings/nvidia-cuda.gpg] https://developer.download.nvidia.com/compute/cuda/repos/debian12/x86_64/ /" \
|
||||
| sudo tee /etc/apt/sources.list.d/nvidia-cuda.list >/dev/null
|
||||
|
||||
echo
|
||||
echo "=== installing cuda-drivers (DKMS rebuild ~3-5 min) ==="
|
||||
sudo DEBIAN_FRONTEND=noninteractive apt-get install -y cuda-drivers
|
||||
sudo apt-get update -qq
|
||||
|
||||
echo
|
||||
echo "=== rebooting in 5 s ==="
|
||||
sleep 5
|
||||
sudo systemctl reboot
|
||||
EOF
|
||||
echo
|
||||
echo "=== cuda-drivers candidate ==="
|
||||
apt-cache policy cuda-drivers | head -10
|
||||
|
||||
echo
|
||||
echo "=== snapshotting current nvidia packages (rollback reference) ==="
|
||||
dpkg -l | awk '/^ii\s+(nvidia-|cuda-)/ {print $2 "=" $3}' \
|
||||
| sudo tee /tmp/nvidia-pre-upgrade.txt >/dev/null
|
||||
echo "snapshot at /tmp/nvidia-pre-upgrade.txt"
|
||||
|
||||
echo
|
||||
echo "=== stopping GPU containers ==="
|
||||
for stack in "${CONTAINERS[@]}"; do
|
||||
if [ -d "/opt/docker/compose/$stack" ]; then
|
||||
echo " stopping $stack"
|
||||
(cd "/opt/docker/compose/$stack" && docker compose stop) || echo " (no-op)"
|
||||
fi
|
||||
done
|
||||
|
||||
echo
|
||||
echo "=== installing cuda-drivers (DKMS rebuild ~3-5 min) ==="
|
||||
sudo DEBIAN_FRONTEND=noninteractive apt-get install -y cuda-drivers
|
||||
|
||||
echo
|
||||
echo "=== rebooting in 5 s ==="
|
||||
sleep 5
|
||||
sudo systemctl reboot
|
||||
}
|
||||
|
||||
remote_phase2() {
|
||||
echo "=== new driver + CUDA ==="
|
||||
nvidia-smi --query-gpu=driver_version --format=csv,noheader
|
||||
nvidia-smi | grep "CUDA Version"
|
||||
|
||||
echo
|
||||
echo "=== restarting GPU containers ==="
|
||||
for stack in comfyui cosyvoice qwen3-tts index-tts parakeet; do
|
||||
if [ -d "/opt/docker/compose/$stack" ]; then
|
||||
echo " starting $stack"
|
||||
(cd "/opt/docker/compose/$stack" && docker compose up -d) || echo " (failed — check manually)"
|
||||
fi
|
||||
done
|
||||
|
||||
sleep 8
|
||||
echo
|
||||
echo "=== container status ==="
|
||||
docker ps --format "table {{.Names}}\t{{.Status}}" | head -15
|
||||
}
|
||||
|
||||
remote_rollback() {
|
||||
echo "=== rollback: reinstalling pre-upgrade nvidia packages ==="
|
||||
[ -f /tmp/nvidia-pre-upgrade.txt ] || { echo "no snapshot found"; exit 1; }
|
||||
sudo apt-get install -y --allow-downgrades $(tr '\n' ' ' < /tmp/nvidia-pre-upgrade.txt)
|
||||
echo "=== rebooting in 5 s ==="
|
||||
sleep 5
|
||||
sudo systemctl reboot
|
||||
}
|
||||
|
||||
case "${1:-}" in
|
||||
phase1) remote_phase1 ;;
|
||||
phase2) remote_phase2 ;;
|
||||
rollback) remote_rollback ;;
|
||||
*) echo "remote: usage: $0 {phase1|phase2|rollback}" >&2; exit 2 ;;
|
||||
esac
|
||||
REMOTE
|
||||
scp -q "$local_tmp" "$HOST:$REMOTE_SCRIPT"
|
||||
rm -f "$local_tmp"
|
||||
ssh "$HOST" "chmod +x $REMOTE_SCRIPT"
|
||||
}
|
||||
|
||||
phase1() {
|
||||
say "Phase 1: pre-flight + install + reboot"
|
||||
say "Staging remote script on $HOST"
|
||||
write_remote_script
|
||||
say "Running phase1 (sudo will prompt once via ssh -t)"
|
||||
# No piped stdin — ssh -t's TTY is real and sudo can prompt.
|
||||
ssh -t "$HOST" "$REMOTE_SCRIPT phase1"
|
||||
ok "Phase 1 dispatched. Host rebooting."
|
||||
say "Wait ~2 min, then run: $0 phase2"
|
||||
}
|
||||
|
||||
phase2() {
|
||||
say "Phase 2: wait for host, verify, restart containers"
|
||||
|
||||
say "waiting for $HOST to come back online…"
|
||||
for i in $(seq 1 60); do
|
||||
if ssh -o ConnectTimeout=3 -o BatchMode=yes "$HOST" true 2>/dev/null; then
|
||||
@@ -110,30 +166,12 @@ phase2() {
|
||||
done
|
||||
echo
|
||||
|
||||
ssh "$HOST" 'bash -s' <<'EOF'
|
||||
set -euo pipefail
|
||||
|
||||
echo "=== new driver + CUDA ==="
|
||||
nvidia-smi --query-gpu=driver_version --format=csv,noheader
|
||||
nvidia-smi | grep "CUDA Version"
|
||||
|
||||
echo
|
||||
echo "=== restarting GPU containers ==="
|
||||
for stack in comfyui cosyvoice qwen3-tts index-tts parakeet; do
|
||||
if [ -d "/opt/docker/compose/$stack" ]; then
|
||||
echo " starting $stack"
|
||||
(cd "/opt/docker/compose/$stack" && docker compose up -d) || echo " (failed — check manually)"
|
||||
fi
|
||||
done
|
||||
|
||||
sleep 8
|
||||
echo
|
||||
echo "=== container status ==="
|
||||
docker ps --format "table {{.Names}}\t{{.Status}}" | head -15
|
||||
EOF
|
||||
# Re-stage in case the previous /tmp script was wiped on reboot.
|
||||
write_remote_script
|
||||
ssh "$HOST" "$REMOTE_SCRIPT phase2"
|
||||
|
||||
ok "Phase 2 complete."
|
||||
cat <<'EOF'
|
||||
cat <<'NEXT'
|
||||
|
||||
────────────────────────────────────────────────────────────────────
|
||||
NEXT — flip Kokoro from cpu → gpu variant now that CUDA 12.9 is live:
|
||||
@@ -146,22 +184,17 @@ NEXT — flip Kokoro from cpu → gpu variant now that CUDA 12.9 is live:
|
||||
'
|
||||
scripts/elway irv-ml1 --playbook playbooks/deploy-kokoro.yaml
|
||||
|
||||
After that, three more deploys (re-runnable now that the driver is current):
|
||||
After that, the two outstanding deploys:
|
||||
scripts/elway irv-ml1 --playbook playbooks/deploy-vibevoice.yaml
|
||||
scripts/elway irv-ml1 --playbook playbooks/deploy-chatterbox.yaml
|
||||
────────────────────────────────────────────────────────────────────
|
||||
EOF
|
||||
NEXT
|
||||
}
|
||||
|
||||
# rollback helper — restores the snapshot from /tmp/nvidia-pre-upgrade.txt
|
||||
rollback() {
|
||||
say "Rollback: reinstalling pre-upgrade nvidia packages on $HOST"
|
||||
ssh -t "$HOST" 'bash -s' <<'EOF'
|
||||
set -euo pipefail
|
||||
[ -f /tmp/nvidia-pre-upgrade.txt ] || { echo "no snapshot found"; exit 1; }
|
||||
sudo apt-get install -y --allow-downgrades $(cat /tmp/nvidia-pre-upgrade.txt | tr '\n' ' ')
|
||||
sudo systemctl reboot
|
||||
EOF
|
||||
write_remote_script
|
||||
ssh -t "$HOST" "$REMOTE_SCRIPT rollback"
|
||||
ok "Rollback dispatched. Host rebooting."
|
||||
}
|
||||
|
||||
|
||||
Reference in New Issue
Block a user