diff --git a/stacks/llama-swap/compose.yaml b/stacks/llama-swap/compose.yaml index c1c547c..2cec60c 100644 --- a/stacks/llama-swap/compose.yaml +++ b/stacks/llama-swap/compose.yaml @@ -29,6 +29,12 @@ services: environment: - HF_HOME=/hfcache - HF_HUB_CACHE=/hfcache/hub + # Pin to GPU 0 — the reserved card for on-demand large-model hot-loads. + # The always-on vLLM services (granite + embed/rerank/reward) own GPU 1; + # keeping llama-swap off GPU 1 stops a hot-loaded model from contending + # with them. llama.cpp then sees only GPU 0 (cuda:0), so --n-gpu-layers + # 999 loads there with no per-model device targeting needed. + - NVIDIA_VISIBLE_DEVICES=${LLAMA_SWAP_GPU:-0} healthcheck: test: ["CMD-SHELL", "curl -fsS http://localhost:8080/ >/dev/null || exit 1"] interval: 30s