esh-pfi-infrastructure/playbooks/gx10-headless.yaml

# pfi-gx10 — convert an ASUS Ascent GX10 (NVIDIA GB10, aarch64) from its
# out-of-box desktop configuration to headless server operation.
#
#   scripts/elway infra-ops@10.100.10.226 --playbook playbooks/gx10-headless.yaml
#
# CONNECT AS infra-ops. `--sudo` only applies to ad-hoc --shell/--upload, so
# playbook steps run as the connecting user and carry their own sudo. lkraven
# needs a password for sudo on this box; infra-ops is NOPASSWD.
#
# The box ships booting to graphical.target with GDM and GNOME Remote Desktop
# running. That is correct for a desk appliance and wrong for a rack node: a
# display manager holds a GPU context, and a laptop-derived power profile will
# suspend a machine that is meant to answer at 3am.
#
# OUT OF SCOPE — DELIBERATELY:
#   Networking. As of 2026-09-01 this box sits on a desk on Wi-Fi with a
#   temporary DHCP lease (10.100.10.226) and NO ethernet carrier (enP7s7
#   carrier=0). Static addressing, VLAN placement and DNS all belong to the
#   rack-install change, not to this one. Configuring an address that is about
#   to be wrong is worse than leaving it on DHCP.
#
# REVERSIBLE. Every step here undoes with:
#   systemctl set-default graphical.target
#   systemctl unmask sleep.target suspend.target hibernate.target
#   systemctl enable --now gdm
#
# Idempotent: a second run should report ok/skipped throughout.

vars:
  fqdn_hostname: pfi-gx10
  # Stop the display manager even if someone is logged in at the console.
  #   scripts/elway infra-ops@<host> -p ... --var force_dm_stop=true
  force_dm_stop: "false"

steps:
  # ---------------------------------------------------------------------------
  # Identity
  # ---------------------------------------------------------------------------
  - name: Set the static hostname to match the transient one
    # Ships with static=gx10-a745 but transient=pfi-gx10, so `hostname` and
    # `hostnamectl` disagree. Anything keying on one of them picks the wrong
    # name half the time. pfi-gx10 is the fleet-convention name.
    shell: sudo hostnamectl set-hostname {{ fqdn_hostname }}
    when: "test \"$(hostnamectl --static)\" != '{{ fqdn_hostname }}'"

  # ---------------------------------------------------------------------------
  # Stop booting into a desktop
  # ---------------------------------------------------------------------------
  - name: Default boot target -> multi-user
    shell: sudo systemctl set-default multi-user.target
    when: "test \"$(systemctl get-default)\" != 'multi-user.target'"

  - name: Stop GDM
    # ⚠ GUARD ON is-active, NOT is-enabled. Ubuntu ships gdm as a STATIC unit —
    # it is pulled in by display-manager.service and is never "enabled". A
    # `is-enabled | grep enabled` guard therefore always skips, and a verify
    # written the same way passes while gdm is still running. Cost one green
    # run that had not done the thing. (2026-09-01)
    #
    # GNOME stays INSTALLED, only stopped: removing it frees little on a 916G
    # disk and makes this hard to walk back from a serial console.
    #
    # Will NOT run while someone holds a seat session, unless force_dm_stop=true.
    # Yanking the display out from under a person at the machine is not a thing
    # automation should decide to do.
    shell: sudo systemctl stop gdm
    when: "systemctl is-active --quiet gdm && { [ '{{ force_dm_stop }}' = 'true' ] || ! loginctl list-sessions --no-legend 2>/dev/null | grep -qE ' seat[0-9]+ '; }"

  - name: Stop and disable GNOME Remote Desktop
    # Needs a graphical session to be useful; with GDM gone it is a listener
    # with nothing behind it.
    shell: sudo systemctl disable --now gnome-remote-desktop
    when: "systemctl is-enabled gnome-remote-desktop 2>/dev/null | grep -q enabled"

  # ---------------------------------------------------------------------------
  # A server must never sleep
  # ---------------------------------------------------------------------------
  - name: Mask sleep / suspend / hibernate / hybrid-sleep
    # Masking, not disabling. These are `static` units pulled in by logind and
    # by desktop power policy; disable does not stick, mask does.
    shell: sudo systemctl mask sleep.target suspend.target hibernate.target hybrid-sleep.target
    when: "! systemctl is-enabled sleep.target 2>/dev/null | grep -q masked"

  - name: logind must ignore lid and idle
    shell: |
      sudo install -d /etc/systemd/logind.conf.d
      sudo tee /etc/systemd/logind.conf.d/10-headless.conf >/dev/null <<'EOF'
      [Login]
      HandleLidSwitch=ignore
      HandleLidSwitchExternalPower=ignore
      HandleLidSwitchDocked=ignore
      IdleAction=ignore
      EOF
    creates: /etc/systemd/logind.conf.d/10-headless.conf

  # ---------------------------------------------------------------------------
  # Keep long jobs alive across a dropped SSH session
  # ---------------------------------------------------------------------------
  - name: sshd keepalives so a NAT/Wi-Fi stall does not kill a running job
    shell: |
      sudo install -d /etc/ssh/sshd_config.d
      sudo tee /etc/ssh/sshd_config.d/10-fleet.conf >/dev/null <<'EOF'
      ClientAliveInterval 60
      ClientAliveCountMax 10
      TCPKeepAlive yes
      EOF
      sudo sshd -t
      sudo systemctl reload ssh 2>/dev/null || sudo systemctl reload sshd
    creates: /etc/ssh/sshd_config.d/10-fleet.conf

verify:
  - name: Boots to multi-user, not graphical
    shell: test "$(systemctl get-default)" = "multi-user.target"
    changed_when: "false"

  - name: Will boot headless — default target is multi-user
    # The DURABLE requirement, true across reboots regardless of what is
    # running right now.
    shell: test "$(systemctl get-default)" = "multi-user.target"
    changed_when: "false"

  - name: GDM is not running
    # is-ACTIVE. Expected to FAIL while someone holds a console session and
    # force_dm_stop is false — that is an honest red, not a broken playbook.
    # A reboot (or the rack install) resolves it.
    shell: "! systemctl is-active --quiet gdm"
    changed_when: "false"

  - name: Sleep targets are masked
    shell: systemctl is-enabled sleep.target 2>/dev/null | grep -q masked
    changed_when: "false"

  - name: Static hostname is correct
    shell: test "$(hostnamectl --static)" = "{{ fqdn_hostname }}"
    changed_when: "false"

  - name: GPU still enumerates after the desktop stack stopped
    # The point of the exercise. If disabling GDM cost us the GPU, this catches
    # it here rather than the next time someone tries to load a model.
    shell: nvidia-smi --query-gpu=name,compute_cap --format=csv,noheader | grep -q GB10
    changed_when: "false"

  - name: sshd config still parses
    shell: sudo sshd -t
    changed_when: "false"