From c9c4262bdb7247ef7345204188d57b67044bb71b Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Sat, 25 Apr 2026 16:51:28 -0700 Subject: [PATCH] scripts/upgrade-irv-ml1-cuda: drop pipe-to-head (SIGPIPE under pipefail) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Phase 1 silently aborted right after the 'cuda-drivers candidate' step — before snapshotting, stopping containers, installing the new driver, or rebooting. Symptom: phase2 showed driver still at 570.124.06 and containers with multi-hour uptimes (proof they were never stopped). Cause: `apt-cache policy cuda-drivers | head -10`. head reads its 10 lines and closes the pipe early; that delivers SIGPIPE to apt-cache, which under `set -o pipefail` propagates as a non-zero exit, and `set -e` immediately terminates the remote shell. The same hazard existed in phase2's `docker ps ... | head -15`. Fix is one-line per call site: drop the head. apt-cache policy output for cuda-drivers is short (~10 lines), and docker ps's container list on irv-ml1 is small enough to print fully. --- scripts/upgrade-irv-ml1-cuda.sh | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/scripts/upgrade-irv-ml1-cuda.sh b/scripts/upgrade-irv-ml1-cuda.sh index b24994f..b5685ed 100755 --- a/scripts/upgrade-irv-ml1-cuda.sh +++ b/scripts/upgrade-irv-ml1-cuda.sh @@ -89,7 +89,11 @@ remote_phase1() { echo echo "=== cuda-drivers candidate ===" - apt-cache policy cuda-drivers | head -10 + # Don't `| head` here — apt-cache policy emits enough that head + # closes the pipe early and SIGPIPEs apt-cache, which under + # set -o pipefail aborts the whole script before we get to the + # actual install + reboot. Output is short anyway. + apt-cache policy cuda-drivers echo echo "=== snapshotting current nvidia packages (rollback reference) ===" @@ -133,7 +137,8 @@ remote_phase2() { sleep 8 echo echo "=== container status ===" - docker ps --format "table {{.Names}}\t{{.Status}}" | head -15 + # `| head -15` would SIGPIPE docker ps under pipefail; just print all. + docker ps --format "table {{.Names}}\t{{.Status}}" } remote_rollback() {