ops(meshcentral): diagnose and fix HW Connect stuck at Setup (stale CIRA tunnel); add relay probe

scripts/meshcentral-amt-relay-probe.js

@@ -0,0 +1,46 @@
+// Probe an Intel AMT device through MeshCentral's web relay (webrelay.ashx), which is the same path the
+// browser's "HW Connect" (KVM), SOL, IDE-R and the embedded MeshCommander use.
+//   KVMR | SOL | IDER : StartRedirectionSession, then the authentication-methods query. It never
+//                       authenticates, so no session is opened on the target.
+//   HTTP              : one empty WS-Man POST (any HTTP reply proves channel + TLS + auth injection).
+// A healthy device answers in well under a second. Silence until the timeout means the relay channel is dead.
+// Seen 2026-10-02: a stale second CIRA tunnel (servers/pfi-tacticalrmm/README.md).
+//
+// Runs ON pfi-tacticalrmm as `tactical` (needs MeshCentral's `ws` module). stdin lines: login user, login pass,
+// node id, mode. Nothing secret is in this file. Example (from eshpfi-management on nh3-dev):
+//   scp scripts/meshcentral-amt-relay-probe.js infra-ops@10.250.50.57:/tmp/p.js
+//   { secret get pfi-tacticalrmm/meshcentral-login-token | python3 -c "import sys; f=sys.stdin.read().split(); print(f[1]); print(f[3])"
+//     printf '%s\n%s\n' '<node id>' KVMR; } | ssh infra-ops@10.250.50.57 'sudo -n -u tactical node /tmp/p.js; rm -f /tmp/p.js'
+// Always run a known-good device as a positive control next to the one under suspicion.
+const [U, P, NODE, MODE] = require("fs").readFileSync(0, "utf8").split("\n");
+const WebSocket = require("/meshcentral/node_modules/ws");
+const t0 = Date.now();
+const log = (...a) => console.log(`[+${((Date.now() - t0) / 1000).toFixed(1)}s]`, ...a);
+const start = {
+  KVMR: [0x10, 0x01, 0x00, 0x00, 0x4b, 0x56, 0x4d, 0x52],
+  SOL: [0x10, 0x00, 0x00, 0x00, 0x53, 0x4f, 0x4c, 0x20],
+  IDER: [0x10, 0x00, 0x00, 0x00, 0x49, 0x44, 0x45, 0x52],
+}[MODE.trim()];
+const HTTP = MODE.trim() === "HTTP";
+const url = "wss://rmm-mesh.phasefinal.com/webrelay.ashx?p=" + (HTTP ? 1 : 2) + "&host=" + encodeURIComponent(NODE.trim()) + "&port=" + (HTTP ? 16993 : 16995) + "&tls=1&tls1only=0";
+const ws = new WebSocket(url, { headers: { "x-meshauth": Buffer.from(U).toString("base64") + "," + Buffer.from(P).toString("base64") } });
+ws.on("open", () => {
+  if (HTTP) { log("relay websocket open (p=1); sending HTTP POST /wsman"); ws.send(Buffer.from("POST /wsman HTTP/1.1\r\nHost: amt:16993\r\nContent-Type: application/soap+xml\r\nContent-Length: 0\r\n\r\n")); return; }
+  log("relay websocket open; sending StartRedirectionSession", MODE.trim()); ws.send(Buffer.from(start));
+});
+ws.on("message", (d, isBinary) => {
+  const s = isBinary ? d.toString("latin1") : d.toString("utf8");
+  if (HTTP) { log("AMT HTTP ->", JSON.stringify(s.split("\r\n").slice(0, 3))); ws.close(); return; }
+  const b = Array.from(s, (c) => c.charCodeAt(0));
+  log("AMT ->", b.length, "bytes:", b.slice(0, 24).map((x) => x.toString(16).padStart(2, "0")).join(" "));
+  if (b[0] === 0x11) {
+    log("StartRedirectionSessionReply status", b[1], b[1] === 0 ? "(success)" : "(FAIL)");
+    if (b[1] === 0) ws.send(Buffer.from([0x13, 0, 0, 0, 0, 0, 0, 0, 0]));
+  } else if (b[0] === 0x14) {
+    log("AuthenticateSessionReply status", b[1], "auth types offered:", b.slice(9).join(","), "(4=digest, 3=kerberos? per Intel)");
+    ws.close();
+  }
+});
+ws.on("close", (c, r) => { log("closed", c, String(r)); process.exit(0); });
+ws.on("error", (e) => log("error", e.message));
+setTimeout(() => { log("TIMEOUT: no further reply in 25 s"); process.exit(0); }, 25000);

servers/esh-pve-2/README.md

@@ -56,5 +56,11 @@ after). Address disks by `/dev/disk/by-id/…` (serial), never `nvmeXn1`.
   two keep sharing one IP.
 - Power schemes offered: "Mobile: ON in S0" and "ON in S0, ME Wake in S3, S4-5 (AC only)". Not checked which is
   active. Its twin nh3-pve-2 kept its tunnel while powered off (2026-10-02 2221), so this one probably does too.
+- ⚠ **If "HW Connect" sticks at "Setup…" after a power cycle, look for a stale second tunnel** (2026-10-02 2247;
+  fix in `servers/pfi-tacticalrmm/README.md`). Because AMT shares `nic1`, the link drops whenever Linux
+  starts or stops, and AMT can leave its old tunnel half-open.
+- Remote desktop works only while the host is ON (AMT mirrors the iGPU). Power it up from MeshCentral first. AMT
+  itself stays reachable when the host is off (both MS-03s, 2026-10-02 2229).
+- Display: a 1080p HDMI plug is fitted (`HDMI-A-1` connected, 1920x1080, 2026-10-02 2241).
 - **KVM needs an active iGPU output.** A 4K dummy plug blacked the AMT console on nh3-pve-2 once Linux took the
   display; use a 1080p plug.

servers/pfi-tacticalrmm/README.md

@@ -54,6 +54,18 @@ Monitors and manages endpoints, pushes patches, runs scripts, etc.
   has none and its two first connects did not crash. That is inferred, not measured. Reconnects of
   an existing device take the other branch and are safe. **Onboard new AMTs at a quiet hour**; expect one ~7 s
   MeshCentral restart.
+- ⚠ **"HW Connect" stuck at "Setup…" = a stale CIRA tunnel** (2026-10-02 2247, esh-pve-2-amt; Prime power-cycled
+  the host several times). The AMT opened a new tunnel without closing the old one. MeshCentral then held two:
+  the live one (power actions and the AMT manager worked through it) and a dead one, `backoff:14`, nothing
+  received for 782 s. `webrelay.ashx` (KVM, SOL, IDE-R, MeshCommander) picked the dead one. A relay TLS failure
+  is only logged at debug level and never closes the websocket, so the browser waits forever. MPS's 90 s idle
+  timeout did not fire, because MeshCentral's own writes to the dead socket kept resetting it. TCP gives up on
+  its own after ~15–30 min of retransmits.
+  **Fix:** `sudo ss -tnio state established "( sport = :4433 )"`. Find the device's public IP with a large
+  `lastrcv` (ms) or a `backoff`. Healthy tunnels show `lastrcv` of seconds. Then
+  `sudo ss -K -tn "dst [::ffff:<ip>]:<its port>"`. Verify with `scripts/meshcentral-amt-relay-probe.js`, using
+  another AMT as a positive control. Likely whenever a host whose AMT shares its NIC (esh-pve-2) power-cycles.
+  Seen once so far.
 - Before 2026-10-02: `"WANonly": true` (TacticalRMM's install default). In that mode MeshCentral SILENTLY DROPS
   "Add Intel AMT computer": `meshuser.js` line 2682, `if (args.wanonly == true) return;`. No error, no
   event. LAN-mode AMT needs `WANonly` false (hybrid) + a service restart; CIRA works in WAN mode. TacticalRMM's