Wire Beszel fleet filesystems, GPU telemetry, dashboard and alerts

services/beszel-althing/README.md

@@ -0,0 +1,36 @@
+# Beszel to althing
+
+`beszel-althing.service` runs on nh3-dev as lkraven, listening at
+`10.100.10.50:8096`. Beszel sends Shoutrrr generic JSON to `/beszel`; the
+bridge invokes the supported `postbox send` command with the body on stdin.
+It uses the established infra-ops automation identity and sends to infra-ops.
+
+Deploy from the repository root:
+
+```sh
+scripts/elway infra-ops@10.100.10.50 --playbook playbooks/beszel-althing.yaml
+```
+
+The service accepts requests only from ana-docker and local diagnostic
+addresses. There is no public listener or new althing handle. It reports
+success only after postbox returns a delivery receipt. Failures return HTTP
+502 and appear in the system journal; there is no hidden retry queue. A
+post-office outage can therefore lose an alert, and needs independent health
+monitoring if guaranteed delivery during such outages becomes a requirement.
+
+```sh
+systemctl status beszel-althing
+sudo -n journalctl -u beszel-althing --since '1 hour ago'
+curl -fsS http://10.100.10.50:8096/healthz
+```
+
+`/healthz` checks the bridge process, not the downstream inbox. End-to-end
+verification requires a real Beszel threshold transition plus its althing
+receipt. The first verified alert is recorded in `stacks/beszel/README.md`.
+
+To reroute later, change `BESZEL_ALERT_RECIPIENT` in the canonical unit to
+`miranda`, deploy, and trigger another end-to-end test. Leave `ALTHING_HANDLE`
+as infra-ops so the sender remains identifiable as infrastructure automation.
+The operator explicitly chose infra-ops for now.
+
+Run `python3 -m unittest discover -s services/beszel-althing -p 'test_*.py'`.

services/beszel-althing/beszel-althing.service

@@ -0,0 +1,24 @@
+[Unit]
+Description=Beszel alerts to the althing infra-ops inbox
+After=network-online.target
+Wants=network-online.target
+
+[Service]
+User=lkraven
+Group=lkraven
+Environment=PYTHONUNBUFFERED=1
+Environment=ALTHING_HANDLE=infra-ops
+Environment=ALTHING_POST_OFFICE=http://10.100.50.40:8390
+Environment=POSTBOX=/home/lkraven/.local/bin/postbox
+Environment=BESZEL_ALERT_RECIPIENT=infra-ops
+Environment=BESZEL_BIND_HOST=10.100.10.50
+Environment=BESZEL_ALLOWED_SOURCES=10.250.50.70,10.100.10.50,127.0.0.1
+ExecStart=/usr/bin/python3 /opt/beszel-althing/bridge.py
+Restart=on-failure
+RestartSec=5
+NoNewPrivileges=true
+ProtectSystem=strict
+PrivateTmp=true
+
+[Install]
+WantedBy=multi-user.target

services/beszel-althing/bridge.py

@@ -0,0 +1,63 @@
+#!/usr/bin/env python3
+"""Receive Beszel's Shoutrrr JSON and deliver through the supported postbox CLI."""
+import json
+import os
+import subprocess
+from http.server import BaseHTTPRequestHandler, ThreadingHTTPServer
+
+
+def deliver(payload):
+    title = payload.get('title', 'Beszel fleet alert')
+    message = payload.get('message')
+    if not isinstance(title, str) or not isinstance(message, str) or not message.strip():
+        raise ValueError('Expected a nonempty message and string title')
+    result = subprocess.run(
+        [os.environ['POSTBOX'], '--json', 'send', '--to', os.environ['BESZEL_ALERT_RECIPIENT'],
+         '--subject', '[Beszel] ' + title],
+        input=message + '\n\nHub: http://10.250.50.70:8090\n',
+        text=True, capture_output=True, timeout=25,
+    )
+    if result.returncode:
+        raise RuntimeError('postbox delivery failed: ' + result.stderr.strip())
+    receipt = json.loads(result.stdout)
+    print(json.dumps({'event': 'delivered', 'title': title, 'receipt': receipt}), flush=True)
+    return receipt
+
+
+class Handler(BaseHTTPRequestHandler):
+    def respond(self, status, body):
+        data = json.dumps(body).encode()
+        self.send_response(status)
+        self.send_header('Content-Type', 'application/json')
+        self.send_header('Content-Length', str(len(data)))
+        self.end_headers()
+        self.wfile.write(data)
+
+    def do_GET(self):
+        self.respond(200 if self.path == '/healthz' else 404,
+                     {'service': 'beszel-althing', 'delivery': 'verified per POST'})
+
+    def do_POST(self):
+        if self.path != '/beszel':
+            return self.respond(404, {'error': 'Unknown route'})
+        if self.client_address[0] not in os.environ['BESZEL_ALLOWED_SOURCES'].split(','):
+            return self.respond(403, {'error': 'Source not allowed'})
+        try:
+            length = int(self.headers.get('Content-Length', '0'))
+            if not 0 < length <= 65536:
+                raise ValueError('Invalid body size')
+            self.connection.settimeout(10)
+            payload = json.loads(self.rfile.read(length))
+            if not isinstance(payload, dict):
+                raise ValueError('Expected JSON object')
+            receipt = deliver(payload)
+        except (ValueError, TypeError) as exc:
+            return self.respond(400, {'error': str(exc)})
+        except (OSError, RuntimeError, subprocess.TimeoutExpired) as exc:
+            print(json.dumps({'event': 'delivery_failed', 'error': str(exc)}), flush=True)
+            return self.respond(502, {'error': 'Althing delivery failed; inspect service journal'})
+        self.respond(200, {'delivered': True, 'receipt': receipt})
+
+
+if __name__ == '__main__':
+    ThreadingHTTPServer((os.environ['BESZEL_BIND_HOST'], int(os.environ.get('BESZEL_BIND_PORT', '8096'))), Handler).serve_forever()

services/beszel-althing/test_bridge.py

@@ -0,0 +1,30 @@
+import importlib.util
+import os
+from pathlib import Path
+import subprocess
+import unittest
+from unittest.mock import patch
+
+spec=importlib.util.spec_from_file_location('bridge',Path(__file__).with_name('bridge.py'))
+bridge=importlib.util.module_from_spec(spec);spec.loader.exec_module(bridge)
+
+class DeliveryTests(unittest.TestCase):
+    def setUp(self):
+        self.env=patch.dict(os.environ,POSTBOX='/bin/postbox',BESZEL_ALERT_RECIPIENT='infra-ops')
+        self.env.start();self.addCleanup(self.env.stop)
+    def test_posts_body_as_stdin_and_returns_receipt(self):
+        with patch.object(bridge.subprocess,'run',return_value=subprocess.CompletedProcess([],0,'{"message_id": 42}','')) as run:
+            self.assertEqual(bridge.deliver({'title':'Disk 85%','message':'tank is full'})['message_id'],42)
+            args,kw=run.call_args
+            self.assertEqual(args[0],['/bin/postbox','--json','send','--to','infra-ops','--subject','[Beszel] Disk 85%'])
+            self.assertIn('tank is full',kw['input'])
+    def test_delivery_failure_is_not_success(self):
+        with patch.object(bridge.subprocess,'run',return_value=subprocess.CompletedProcess([],2,'','post office unavailable')):
+            with self.assertRaises(RuntimeError):bridge.deliver({'message':'disk full'})
+    def test_rejects_empty_message_without_sending(self):
+        with patch.object(bridge.subprocess,'run') as run:
+            with self.assertRaises(ValueError):bridge.deliver({'message':''})
+            run.assert_not_called()
+
+if __name__ == '__main__':
+    unittest.main()