diff --git a/playbooks/gx10-rack-network.yaml b/playbooks/gx10-rack-network.yaml new file mode 100644 index 0000000..3b05891 --- /dev/null +++ b/playbooks/gx10-rack-network.yaml @@ -0,0 +1,118 @@ +# pfi-gx10 — move from desk (Wi-Fi + DHCP, VLAN 10) to rack (wired static, +# VLAN 50 nh3-servers). Run AFTER the box is racked and the switch port has +# been placed on nh3-servers. +# +# scripts/elway infra-ops@ --playbook playbooks/gx10-rack-network.yaml +# +# Find without asking anyone — the wired NIC has its own MAC: +# curl -sk "https://10.100.0.1/proxy/network/api/s/default/stat/sta" \ +# -H "X-API-KEY: $(secret get unifi/pfi-udmse-api-key)" \ +# | python3 -c "import json,sys;[print(c['ip'],c.get('sw_port')) for c in json.load(sys.stdin)['data'] if c['mac']=='30:c5:99:3d:a7:45']" +# +# ───────────────────────────────────────────────────────────────────────────── +# WHY THIS IS SAFE TO RUN AGAINST A BOX IN A RACK +# +# The lockout risk in remote network reconfiguration is applying a static +# address that does not work and losing the only path back. This playbook never +# has one path: Wi-Fi (wlP9s9) STAYS UP THROUGHOUT. The wired interface is +# configured alongside it, verified from outside on the new address, and only +# then is Wi-Fi torn down — by a SEPARATE playbook run, after a human has seen +# the new address answer. +# +# So a botched netplan costs a retry over Wi-Fi, not a trip to the rack. +# ───────────────────────────────────────────────────────────────────────────── + +vars: + wired_if: enP7s7 + wired_mac: "30:c5:99:3d:a7:45" + static_ip: 10.100.50.60 + prefix: "24" + gateway: 10.100.50.1 + dns_servers: "10.100.50.40" # nh3-docker / AdGuard, per dns/internal.yaml + search_domain: nh3.internal + +steps: + - name: Refuse to proceed if the cable is not actually in + # carrier=0 means no link. Writing a static config for a dead interface and + # calling it done is the kind of green that costs a rack visit to discover. + shell: test "$(cat /sys/class/net/{{ wired_if }}/carrier 2>/dev/null)" = "1" + + - name: Confirm the interface is the NIC we think it is + # Interface names can renumber across kernels. The MAC cannot. + shell: test "$(cat /sys/class/net/{{ wired_if }}/address)" = "{{ wired_mac }}" + + - name: Back up the existing netplan + shell: sudo cp -an /etc/netplan /etc/netplan.bak-preRack 2>/dev/null || true + creates: /etc/netplan.bak-preRack + + - name: Write the wired static config + # A SEPARATE file from whatever manages Wi-Fi. Wi-Fi keeps working while + # this lands; that is the escape hatch and it is deliberate. + shell: | + sudo tee /etc/netplan/60-wired-static.yaml >/dev/null <<'EOF' + network: + version: 2 + ethernets: + {{ wired_if }}: + dhcp4: false + dhcp6: false + addresses: [{{ static_ip }}/{{ prefix }}] + routes: + - to: default + via: {{ gateway }} + metric: 100 + nameservers: + addresses: [{{ dns_servers }}] + search: [{{ search_domain }}] + EOF + sudo chmod 600 /etc/netplan/60-wired-static.yaml + creates: /etc/netplan/60-wired-static.yaml + + - name: Apply + # `netplan try` would be better but needs a TTY to accept or roll back; + # under elway there is nobody to press a key. Wi-Fi staying up is what + # substitutes for the auto-rollback. + shell: sudo netplan apply + + - name: Wait for the address to come up locally + shell: | + for i in $(seq 1 15); do + ip -4 addr show {{ wired_if }} | grep -q "{{ static_ip }}/{{ prefix }}" && exit 0 + sleep 2 + done + exit 1 + +verify: + - name: Wired interface holds the static address + shell: ip -4 addr show {{ wired_if }} | grep -q "{{ static_ip }}/{{ prefix }}" + changed_when: "false" + + - name: Default route is via the servers-VLAN gateway + shell: ip route get 1.1.1.1 | grep -q "via {{ gateway }}" + changed_when: "false" + + - name: Gateway answers + shell: ping -c 2 -W 2 {{ gateway }} >/dev/null + changed_when: "false" + + - name: DNS resolves through the fleet resolver + shell: getent hosts nh3-docker.nh3.internal >/dev/null + changed_when: "false" + + - name: Wi-Fi is STILL UP — the escape hatch must survive this run + # If this fails, the box may be reachable only on an address that was just + # proven or just broken. Loudly not optional. + shell: ip link show wlP9s9 | grep -q "state UP" + changed_when: "false" + +# ───────────────────────────────────────────────────────────────────────────── +# AFTERWARDS, and only once the new address has been confirmed from OUTSIDE: +# +# 1. ssh infra-ops@10.100.50.60 # prove it from another host first +# 2. update servers/pfi-gx10/ssh-target +# 3. add to dns/internal.yaml, then scripts/dns-sync.py --dry-run && ...sync +# 4. tear down Wi-Fi — a server with two default routes picks one at random +# and you find out later, in a way that looks like packet loss: +# sudo nmcli radio wifi off # or delete the wlP9s9 netplan stanza +# Do that as its own change, not bundled in here. +# ───────────────────────────────────────────────────────────────────────────── diff --git a/servers/pfi-gx10/README.md b/servers/pfi-gx10/README.md index 5604d3a..4157793 100644 --- a/servers/pfi-gx10/README.md +++ b/servers/pfi-gx10/README.md @@ -16,11 +16,26 @@ Grace-Blackwell desktop supercomputer. Registered 2026-09-01. As of 2026-09-01 this box is **on a desk, on Wi-Fi**, holding a DHCP lease at `10.100.10.226` on `nh3-userland` (VLAN 10). Ethernet `enP7s7` has no carrier. -It is going into the rack later. At that point it needs a wired link, a static -address, a decision about VLAN placement (a compute node arguably belongs on -`nh3-servers`, VLAN 50, with the rest of them), a `dns/internal.yaml` entry, and -this file plus `ssh-target` updated. **None of that was configured now** — an -address that is about to be wrong is worse than DHCP. +It is going into the rack later. **Target settled (operator, 2026-09-01): +`nh3-servers`, VLAN 50, static `10.100.50.60`** — clear of `.40` nh3-docker, +`.42` nh3-extdev, `.50` nh3-nas, `.90` pbs-nh3, and below the `.150` DHCP pool +where fleet statics live. + +Nothing was configured on the desk — an address that is about to be wrong is +worse than DHCP. The move is `playbooks/gx10-rack-network.yaml`. + +**Nothing is needed from the operator beyond racking it.** The wired NIC has its +own MAC (`30:c5:99:3d:a7:45`, distinct from the Wi-Fi `50:bb:b5:a2:00:a8`), so +its post-move address is discoverable from the UDM without being told: + +```bash +curl -sk "https://10.100.0.1/proxy/network/api/s/default/stat/sta" \ + -H "X-API-KEY: $(secret get unifi/pfi-udmse-api-key)" \ + | python3 -c "import json,sys;[print(c['ip'],c.get('sw_port')) for c in json.load(sys.stdin)['data'] if c['mac']=='30:c5:99:3d:a7:45']" +``` + +That also returns the switch port, which must be set to the `nh3-servers` +network or the box lands back on VLAN 10. Expect ~65–80 ms RTT until it is wired. That is Wi-Fi power-save, not a fault.