From ad0322bf83e3ec59ba955aebd1253667f65df78c Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Fri, 2 Oct 2026 22:49:27 -0700 Subject: [PATCH] ops(meshcentral): diagnose and fix HW Connect stuck at Setup (stale CIRA tunnel); add relay probe --- scripts/meshcentral-amt-relay-probe.js | 46 ++++++++++++++++++++++++++ servers/esh-pve-2/README.md | 6 ++++ servers/pfi-tacticalrmm/README.md | 12 +++++++ 3 files changed, 64 insertions(+) create mode 100644 scripts/meshcentral-amt-relay-probe.js diff --git a/scripts/meshcentral-amt-relay-probe.js b/scripts/meshcentral-amt-relay-probe.js new file mode 100644 index 0000000..42636c3 --- /dev/null +++ b/scripts/meshcentral-amt-relay-probe.js @@ -0,0 +1,46 @@ +// Probe an Intel AMT device through MeshCentral's web relay (webrelay.ashx), which is the same path the +// browser's "HW Connect" (KVM), SOL, IDE-R and the embedded MeshCommander use. +// KVMR | SOL | IDER : StartRedirectionSession, then the authentication-methods query. It never +// authenticates, so no session is opened on the target. +// HTTP : one empty WS-Man POST (any HTTP reply proves channel + TLS + auth injection). +// A healthy device answers in well under a second. Silence until the timeout means the relay channel is dead. +// Seen 2026-10-02: a stale second CIRA tunnel (servers/pfi-tacticalrmm/README.md). +// +// Runs ON pfi-tacticalrmm as `tactical` (needs MeshCentral's `ws` module). stdin lines: login user, login pass, +// node id, mode. Nothing secret is in this file. Example (from eshpfi-management on nh3-dev): +// scp scripts/meshcentral-amt-relay-probe.js infra-ops@10.250.50.57:/tmp/p.js +// { secret get pfi-tacticalrmm/meshcentral-login-token | python3 -c "import sys; f=sys.stdin.read().split(); print(f[1]); print(f[3])" +// printf '%s\n%s\n' '' KVMR; } | ssh infra-ops@10.250.50.57 'sudo -n -u tactical node /tmp/p.js; rm -f /tmp/p.js' +// Always run a known-good device as a positive control next to the one under suspicion. +const [U, P, NODE, MODE] = require("fs").readFileSync(0, "utf8").split("\n"); +const WebSocket = require("/meshcentral/node_modules/ws"); +const t0 = Date.now(); +const log = (...a) => console.log(`[+${((Date.now() - t0) / 1000).toFixed(1)}s]`, ...a); +const start = { + KVMR: [0x10, 0x01, 0x00, 0x00, 0x4b, 0x56, 0x4d, 0x52], + SOL: [0x10, 0x00, 0x00, 0x00, 0x53, 0x4f, 0x4c, 0x20], + IDER: [0x10, 0x00, 0x00, 0x00, 0x49, 0x44, 0x45, 0x52], +}[MODE.trim()]; +const HTTP = MODE.trim() === "HTTP"; +const url = "wss://rmm-mesh.phasefinal.com/webrelay.ashx?p=" + (HTTP ? 1 : 2) + "&host=" + encodeURIComponent(NODE.trim()) + "&port=" + (HTTP ? 16993 : 16995) + "&tls=1&tls1only=0"; +const ws = new WebSocket(url, { headers: { "x-meshauth": Buffer.from(U).toString("base64") + "," + Buffer.from(P).toString("base64") } }); +ws.on("open", () => { + if (HTTP) { log("relay websocket open (p=1); sending HTTP POST /wsman"); ws.send(Buffer.from("POST /wsman HTTP/1.1\r\nHost: amt:16993\r\nContent-Type: application/soap+xml\r\nContent-Length: 0\r\n\r\n")); return; } + log("relay websocket open; sending StartRedirectionSession", MODE.trim()); ws.send(Buffer.from(start)); +}); +ws.on("message", (d, isBinary) => { + const s = isBinary ? d.toString("latin1") : d.toString("utf8"); + if (HTTP) { log("AMT HTTP ->", JSON.stringify(s.split("\r\n").slice(0, 3))); ws.close(); return; } + const b = Array.from(s, (c) => c.charCodeAt(0)); + log("AMT ->", b.length, "bytes:", b.slice(0, 24).map((x) => x.toString(16).padStart(2, "0")).join(" ")); + if (b[0] === 0x11) { + log("StartRedirectionSessionReply status", b[1], b[1] === 0 ? "(success)" : "(FAIL)"); + if (b[1] === 0) ws.send(Buffer.from([0x13, 0, 0, 0, 0, 0, 0, 0, 0])); + } else if (b[0] === 0x14) { + log("AuthenticateSessionReply status", b[1], "auth types offered:", b.slice(9).join(","), "(4=digest, 3=kerberos? per Intel)"); + ws.close(); + } +}); +ws.on("close", (c, r) => { log("closed", c, String(r)); process.exit(0); }); +ws.on("error", (e) => log("error", e.message)); +setTimeout(() => { log("TIMEOUT: no further reply in 25 s"); process.exit(0); }, 25000); diff --git a/servers/esh-pve-2/README.md b/servers/esh-pve-2/README.md index 43644d4..4cb9e94 100644 --- a/servers/esh-pve-2/README.md +++ b/servers/esh-pve-2/README.md @@ -56,5 +56,11 @@ after). Address disks by `/dev/disk/by-id/…` (serial), never `nvmeXn1`. two keep sharing one IP. - Power schemes offered: "Mobile: ON in S0" and "ON in S0, ME Wake in S3, S4-5 (AC only)". Not checked which is active. Its twin nh3-pve-2 kept its tunnel while powered off (2026-10-02 2221), so this one probably does too. +- ⚠ **If "HW Connect" sticks at "Setup…" after a power cycle, look for a stale second tunnel** (2026-10-02 2247; + fix in `servers/pfi-tacticalrmm/README.md`). Because AMT shares `nic1`, the link drops whenever Linux + starts or stops, and AMT can leave its old tunnel half-open. +- Remote desktop works only while the host is ON (AMT mirrors the iGPU). Power it up from MeshCentral first. AMT + itself stays reachable when the host is off (both MS-03s, 2026-10-02 2229). +- Display: a 1080p HDMI plug is fitted (`HDMI-A-1` connected, 1920x1080, 2026-10-02 2241). - **KVM needs an active iGPU output.** A 4K dummy plug blacked the AMT console on nh3-pve-2 once Linux took the display; use a 1080p plug. diff --git a/servers/pfi-tacticalrmm/README.md b/servers/pfi-tacticalrmm/README.md index 1c91e95..f96d2fd 100644 --- a/servers/pfi-tacticalrmm/README.md +++ b/servers/pfi-tacticalrmm/README.md @@ -54,6 +54,18 @@ Monitors and manages endpoints, pushes patches, runs scripts, etc. has none and its two first connects did not crash. That is inferred, not measured. Reconnects of an existing device take the other branch and are safe. **Onboard new AMTs at a quiet hour**; expect one ~7 s MeshCentral restart. +- ⚠ **"HW Connect" stuck at "Setup…" = a stale CIRA tunnel** (2026-10-02 2247, esh-pve-2-amt; Prime power-cycled + the host several times). The AMT opened a new tunnel without closing the old one. MeshCentral then held two: + the live one (power actions and the AMT manager worked through it) and a dead one, `backoff:14`, nothing + received for 782 s. `webrelay.ashx` (KVM, SOL, IDE-R, MeshCommander) picked the dead one. A relay TLS failure + is only logged at debug level and never closes the websocket, so the browser waits forever. MPS's 90 s idle + timeout did not fire, because MeshCentral's own writes to the dead socket kept resetting it. TCP gives up on + its own after ~15–30 min of retransmits. + **Fix:** `sudo ss -tnio state established "( sport = :4433 )"`. Find the device's public IP with a large + `lastrcv` (ms) or a `backoff`. Healthy tunnels show `lastrcv` of seconds. Then + `sudo ss -K -tn "dst [::ffff:]:"`. Verify with `scripts/meshcentral-amt-relay-probe.js`, using + another AMT as a positive control. Likely whenever a host whose AMT shares its NIC (esh-pve-2) power-cycles. + Seen once so far. - Before 2026-10-02: `"WANonly": true` (TacticalRMM's install default). In that mode MeshCentral SILENTLY DROPS "Add Intel AMT computer": `meshuser.js` line 2682, `if (args.wanonly == true) return;`. No error, no event. LAN-mode AMT needs `WANonly` false (hybrid) + a service restart; CIRA works in WAN mode. TacticalRMM's