From edf0f912f81bb42b3fdf1300c242f322b125c32f Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Fri, 12 Jun 2026 15:17:46 -0700 Subject: [PATCH] feat(llama-swap): pin to GPU 0, reserving it for large-model hot-loads ana-ml2's Ada->Blackwell swap (2x96GB) frees GPU 0 entirely. Pin llama-swap to GPU 0 via NVIDIA_VISIBLE_DEVICES so on-demand large-model hot-loads land there, off GPU 1 where the always-on vLLM services (granite + embed/rerank/ reward) live. Closes the long-standing 'pin llama-swap to GPU 0' item. --- stacks/llama-swap/compose.yaml | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/stacks/llama-swap/compose.yaml b/stacks/llama-swap/compose.yaml index c1c547c..2cec60c 100644 --- a/stacks/llama-swap/compose.yaml +++ b/stacks/llama-swap/compose.yaml @@ -29,6 +29,12 @@ services: environment: - HF_HOME=/hfcache - HF_HUB_CACHE=/hfcache/hub + # Pin to GPU 0 — the reserved card for on-demand large-model hot-loads. + # The always-on vLLM services (granite + embed/rerank/reward) own GPU 1; + # keeping llama-swap off GPU 1 stops a hot-loaded model from contending + # with them. llama.cpp then sees only GPU 0 (cuda:0), so --n-gpu-layers + # 999 loads there with no per-model device targeting needed. + - NVIDIA_VISIBLE_DEVICES=${LLAMA_SWAP_GPU:-0} healthcheck: test: ["CMD-SHELL", "curl -fsS http://localhost:8080/ >/dev/null || exit 1"] interval: 30s