diff --git a/playbooks/gx10-outfit-privileged.yaml b/playbooks/gx10-outfit-privileged.yaml index fd825ca..f8ff521 100644 --- a/playbooks/gx10-outfit-privileged.yaml +++ b/playbooks/gx10-outfit-privileged.yaml @@ -73,6 +73,18 @@ steps: shell: DEBIAN_FRONTEND=noninteractive apt-get install -y -qq cuda-toolkit when: "! test -x /usr/local/cuda/bin/nvcc" + - name: Install python3-dev + build-essential (Triton JITs C at RUNTIME) + # Triton compiles its CUDA-utils shim with gcc on FIRST USE and needs + # Python.h to do it. Without python3-dev, torch imports fine, the model + # loads fine, LoRA attaches fine -- and then the first training step dies + # with a CalledProcessError from gcc whose real message ("fatal error: + # Python.h: No such file or directory") is swallowed, because Triton + # sends the compiler's stdout to DEVNULL. Cost a probe run to find. + # Same shape as the dots-tts container needing a C compiler at runtime. + sudo: true + shell: DEBIAN_FRONTEND=noninteractive apt-get install -y -qq python3-dev build-essential + when: "! test -f /usr/include/python3.12/Python.h" + - name: Install the NVIDIA Container Toolkit sudo: true shell: DEBIAN_FRONTEND=noninteractive apt-get install -y -qq nvidia-container-toolkit @@ -101,6 +113,18 @@ verify: shell: sg docker -c 'docker run --rm --gpus all nvidia/cuda:13.0.0-base-ubuntu24.04 nvidia-smi -L' | grep -q GB10 changed_when: "false" + - name: Triton can actually compile its runtime shim + # Asserting the effective behaviour. `dpkg -l python3-dev` would pass while + # the compile still failed for a missing header or library path. + shell: | + V=/home/infra-ops/ml/.venv/lib/python3.12/site-packages/triton/backends/nvidia + test -d "$V" || exit 0 + gcc "$V/driver.c" -O3 -shared -fPIC -Wno-psabi -o /tmp/.tritoncheck.so \ + -l:libcuda.so.1 -L"$V/lib" -L/lib/aarch64-linux-gnu \ + -I"$V/include" -I/usr/include/python3.12 + rm -f /tmp/.tritoncheck.so + changed_when: "false" + - name: The userspace torch stack still sees the GPU # Guards against a toolkit install disturbing the working driver. shell: /home/infra-ops/ml/.venv/bin/python -c 'import torch;assert torch.cuda.is_available()'