diff --git a/configs/svos-front/Caddyfile b/configs/svos-front/Caddyfile index 561ee8b..ee554cc 100644 --- a/configs/svos-front/Caddyfile +++ b/configs/svos-front/Caddyfile @@ -1,10 +1,13 @@ # SVOS front door on nh3-svos (eshpfi configs/svos-front/), 2026-10-05. Prime: mesh-only, svos.phasefinal.com. # - Listens ONLY on the tailnet address (100.64.0.9:443). There is no LAN or internet listener and no :80 redirect server. # - Cert: Let's Encrypt via Cloudflare DNS-01 with a phasefinal.com-only token (env CF_API_TOKEN, /etc/caddy/env). -# - Identity: every request goes through forward_auth to svos-whois-auth on 127.0.0.1:8405, which resolves the -# SOCKET peer ({http.request.remote.host}:{port}; no trusted_proxies, so never a client header) with tailscaled -# WhoIs and returns Tailscale-User-Login only for allow-listed logins. Fail closed (403) otherwise. -# - Client-supplied identity/forwarding headers are deleted BEFORE auth; the verified login is the only one sent. +# - Identity: HTTP Basic auth (Prime 2026-10-05: "Basic auth is fine, saved in browser is fine"; mesh-only because we +# don't host externally). One account, `Vuong` (Prime is how SVOS addresses him). argon2id hash in +# /etc/caddy/svos-basic-auth.accounts (root:caddy 0640, NOT in git); plaintext vaulted nh3-svos/basic-auth-vuong. +# The WhoIs/forward_auth gate that preceded this was retired the same day (every mesh node is login `pfi`). +# - Client-supplied identity/forwarding headers are deleted BEFORE auth. After auth, Tailscale-User-Login is set to +# the authenticated user ({http.auth.user.id}) and Authorization is deleted: the kernel's human path (INV-U1-06) +# refuses any request that carries an Authorization header, and the app has no business seeing the password. # reverse_proxy sets X-Forwarded-Proto=https and X-Forwarded-For itself; Host passes through; Origin untouched. # - App side (svos-dev): kernel 127.0.0.1:8400 (/api/* with the prefix stripped, /api/health is the kernel's), # connector 127.0.0.1:8403 (/health, /verbs/ping|pins|pin|unpin, /openapi.json), static UI /opt/svos/current/static. @@ -30,11 +33,11 @@ https://svos.phasefinal.com:443 { request_header -Forwarded request_header -X-SVOS-Peer - forward_auth 127.0.0.1:8405 { - uri /auth - header_up X-SVOS-Peer {http.request.remote.host}:{http.request.remote.port} - copy_headers Tailscale-User-Login + basic_auth argon2id SVOS { + import /etc/caddy/svos-basic-auth.accounts } + request_header Tailscale-User-Login {http.auth.user.id} + request_header -Authorization handle_path /api/* { reverse_proxy 127.0.0.1:8400 diff --git a/configs/svos-front/caddy-svos.service b/configs/svos-front/caddy-svos.service index cea7c27..c004f3f 100644 --- a/configs/svos-front/caddy-svos.service +++ b/configs/svos-front/caddy-svos.service @@ -1,7 +1,7 @@ [Unit] Description=SVOS front door (Caddy, tailnet-only, svos.phasefinal.com) - eshpfi configs/svos-front -After=network-online.target tailscaled.service svos-whois-auth.service -Wants=network-online.target tailscaled.service svos-whois-auth.service +After=network-online.target tailscaled.service +Wants=network-online.target tailscaled.service [Service] User=caddy diff --git a/configs/svos-front/svos-whois-auth.py b/configs/svos-front/svos-whois-auth.py deleted file mode 100644 index 96608fd..0000000 --- a/configs/svos-front/svos-whois-auth.py +++ /dev/null @@ -1,115 +0,0 @@ -#!/usr/bin/env python3 -"""svos-whois-auth: Caddy forward_auth target for the SVOS front door on nh3-svos (eshpfi configs/svos-front/). - -Caddy calls GET /auth with X-SVOS-Peer = the TCP peer of the accepted connection ({remote_host}:{remote_port}, -taken from the socket by Caddy; never from a client header). We ask the local tailscaled (LocalAPI over its unix -socket) who that peer is and answer: - 200 + Tailscale-User-Login: only if WhoIs resolves the peer AND the login is in the allow-list - 403 on anything else: no/garbled peer, WhoIs error, unknown peer, login not - allowed, empty allow-list (fail closed) -The allow-list is /etc/svos-front/allowed-logins: one exact LoginName per line, '#' comments. Re-read per request. -Stdlib only; listens on 127.0.0.1 only. -""" -import http.client -import http.server -import ipaddress -import json -import socket -import sys -import urllib.parse - -LISTEN = ("127.0.0.1", 8405) -TAILSCALED_SOCK = "/var/run/tailscale/tailscaled.sock" -ALLOW_FILE = "/etc/svos-front/allowed-logins" -TIMEOUT_S = 3.0 - - -class _UnixHTTPConnection(http.client.HTTPConnection): - def __init__(self, path): - super().__init__("local-tailscaled.sock", timeout=TIMEOUT_S) - self._path = path - - def connect(self): - s = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) - s.settimeout(TIMEOUT_S) - s.connect(self._path) - self.sock = s - - -def allowed_logins(): - try: - with open(ALLOW_FILE, encoding="utf-8") as f: - return {ln.strip() for ln in f if ln.strip() and not ln.lstrip().startswith("#")} - except OSError: - return set() - - -def parse_peer(value): - """'100.64.0.4:51234' or '[fd7a::4]:51234' -> canonical 'ip:port' string, else None.""" - if not value: - return None - try: - if value.startswith("["): - host, _, port = value[1:].partition("]:") - else: - host, _, port = value.rpartition(":") - ip = ipaddress.ip_address(host) - port_n = int(port) - if not 0 < port_n < 65536: - return None - except ValueError: - return None - return f"[{ip}]:{port_n}" if ip.version == 6 else f"{ip}:{port_n}" - - -def whois(peer): - conn = _UnixHTTPConnection(TAILSCALED_SOCK) - try: - conn.request("GET", "/localapi/v0/whois?addr=" + urllib.parse.quote(peer, safe=""), - headers={"Sec-Tailscale": "localapi"}) - resp = conn.getresponse() - body = resp.read() - if resp.status != 200: - return None - data = json.loads(body) - return (data.get("UserProfile") or {}).get("LoginName"), (data.get("Node") or {}).get("Name") - finally: - conn.close() - - -class Handler(http.server.BaseHTTPRequestHandler): - server_version = "svos-whois-auth" - - def _deny(self, why): - self.log_message("DENY %s", why) - self.send_response(403) - self.send_header("Content-Length", "0") - self.end_headers() - - def do_GET(self): # noqa: N802 (stdlib naming) - if self.client_address[0] != "127.0.0.1": - return self._deny("non-loopback caller") - peer = parse_peer(self.headers.get("X-SVOS-Peer", "")) - if peer is None: - return self._deny("missing or malformed X-SVOS-Peer") - try: - found = whois(peer) - except Exception as e: # tailscaled down, timeout, bad JSON: fail closed - return self._deny(f"whois error for {peer}: {e.__class__.__name__}") - if not found or not found[0]: - return self._deny(f"whois: no identity for {peer}") - login, node = found - if login not in allowed_logins(): - return self._deny(f"login {login!r} (node {node}) not in {ALLOW_FILE}") - self.log_message("ALLOW %s login=%r node=%s", peer, login, node) - self.send_response(200) - self.send_header("Tailscale-User-Login", str(login)) - self.send_header("Content-Length", "0") - self.end_headers() - - def log_message(self, format, *args): # noqa: A002 (stdlib signature) - sys.stderr.write((format % args) + "\n") - - -if __name__ == "__main__": - http.server.ThreadingHTTPServer(LISTEN, Handler).serve_forever() diff --git a/configs/svos-front/svos-whois-auth.service b/configs/svos-front/svos-whois-auth.service deleted file mode 100644 index 491d618..0000000 --- a/configs/svos-front/svos-whois-auth.service +++ /dev/null @@ -1,17 +0,0 @@ -[Unit] -Description=SVOS front door identity check (tailscaled WhoIs, fail closed) - eshpfi configs/svos-front -After=tailscaled.service -Wants=tailscaled.service - -[Service] -DynamicUser=true -ExecStart=/usr/bin/python3 /usr/local/lib/svos-front/svos-whois-auth.py -NoNewPrivileges=true -ProtectSystem=strict -ProtectHome=true -PrivateTmp=true -Restart=on-failure -RestartSec=2 - -[Install] -WantedBy=multi-user.target diff --git a/persistent-memory.md b/persistent-memory.md index 572a1c3..b4d9ff6 100644 --- a/persistent-memory.md +++ b/persistent-memory.md @@ -301,6 +301,7 @@ _As of 2026-10-04 ~2300 PT. The newest subsection is first; older subsections ca ## Recent decisions +- `[2026-10-05]` **nh3-svos (VM 111, nh3-pve-2) built for the new SVOS (svos-dev relaying Prime); front door = mesh-only `https://svos.phasefinal.com` with HTTP Basic auth, login `Vuong` (Prime, direct)** — WhoIs gate retired (all mesh nodes are login `pfi`; Prime declined a mesh restructure). Password vaulted `nh3-svos/basic-auth-vuong`. qemu-ga freeze hooks were never run on Debian; now `configs/qemu-ga/fsfreeze-dispatch` (fails the freeze on a hook failure). Live-service backup drill pending. → `servers/nh3-svos/README.md`. - `[2026-10-05]` **fv-ml1 Docker → tank zvol + host patched (Prime, park 99 DONE)** — run 1 aborted on its own guard (live rsync walked into an overlay `merged` view; seats down 0218–0228, rolled back); run 2 ok 0231–0240 (`-x`, merged excluded, path-list verify, auto-restore trap). Kernel 6.12.111 broke NVIDIA DKMS → purged + kernel metas removed; ZFS stack + nvidia-container-toolkit held. Root 49%→8%. → `servers/fv-ml1/README.md` - `[2026-10-05]` **ana-docker: Docker CE repo bullseye → bookworm, 29.8.2 (Prime)** — with `live-restore: true` the engine upgrade kept containers up, but 6 that bind-mount `docker.sock` (crowdsec, traefik, dozzle, beszel-agent, dockge, gitea-runner) went stale and needed restarts. → `servers/ana-docker/README.md` - `[2026-10-05]` **RMM wildcard cert now auto-renews (Prime)** — `*.phasefinal.com` on pfi-tacticalrmm was certbot `manual` (could never renew; was 16 days from expiry). Now dns-cloudflare with a phasefinal.com-only token (vault `pfi-tacticalrmm/cloudflare-certbot-dns01-token`); reissued to 2027-01-03, dry-run OK. → `servers/pfi-tacticalrmm/README.md` diff --git a/servers/nh3-svos/README.md b/servers/nh3-svos/README.md index 1d7d396..d46d7d6 100644 --- a/servers/nh3-svos/README.md +++ b/servers/nh3-svos/README.md @@ -33,7 +33,7 @@ scripts/elway vh@10.100.50.43 --playbook playbooks/bootstrap-infra-ops-us scripts/elway infra-ops@10.100.50.43 --playbook playbooks/nh3-svos-host.yaml ``` -## Front door: built 2026-10-05, mesh-only `https://svos.phasefinal.com` (Prime). Fail-closed until the owner identity is set +## Front door: mesh-only `https://svos.phasefinal.com`, HTTP Basic auth (Prime, 2026-10-05) - **Mesh:** tailscaled joined headscale as `nh3-svos`, **100.64.0.9**, `--accept-dns=false --accept-routes=false`. - **Name:** `svos.phasefinal.com` exists ONLY as a headscale MagicDNS extra record (`/var/lib/headscale/extra-records.json` @@ -42,25 +42,33 @@ scripts/elway infra-ops@10.100.50.43 --playbook playbooks/nh3-svos-host.yaml `caddy-svos.service`, config `/etc/caddy/Caddyfile` (canonical `configs/svos-front/Caddyfile`). It listens **only on 100.64.0.9:443**, with no `:80`. The LE cert for `svos.phasefinal.com` comes via DNS-01, token `nh3-svos-caddy-dns01` (phasefinal.com only), in `/etc/caddy/env` (root:caddy 0640), vaulted `nh3-svos/cloudflare-caddy-dns01-token`. -- **Identity:** `svos-whois-auth.service` (`configs/svos-front/svos-whois-auth.py`, DynamicUser, 127.0.0.1:8405). Caddy's - `forward_auth` passes the SOCKET peer; the responder asks tailscaled WhoIs and answers 200 + `Tailscale-User-Login` only - for logins in **`/etc/svos-front/allowed-logins`** (empty = nobody). Client identity/forwarding headers are deleted - first, as whole families by prefix (`Tailscale-User-*`, `X-Forwarded-*`, svos-dev's ask) plus `Tailscale-Headers-Info`, - `X-Real-IP`, `Forwarded` and `X-SVOS-Peer`. The wildcard was tested on a scratch Caddy first: case-insensitive, and - look-alikes outside the family (`X-Forwardedness`, `Tailscale-Other`) survive. +- **Identity: HTTP Basic auth, one account `Vuong`** (Prime: "Basic auth is fine, saved in browser is fine. Access is + via mesh because we don't host externally." SVOS addresses him as Prime; the login is `Vuong`). The argon2id hash is in + **`/etc/caddy/svos-basic-auth.accounts`** (root:caddy 0640, not in git), imported into `basic_auth argon2id SVOS`. + The plaintext is vaulted at **`nh3-svos/basic-auth-vuong`**. To rotate: pipe the new password to + `caddy hash-password --algorithm argon2id` on stdin (never `--plaintext`), rewrite the accounts file, `secret put`, + then `systemctl reload caddy-svos`. +- **Headers:** client copies of `Tailscale-User-*`, `X-Forwarded-*`, `Tailscale-Headers-Info`, `X-Real-IP`, `Forwarded` + and `X-SVOS-Peer` are deleted before auth. After auth, `Tailscale-User-Login` is SET to the authenticated user, and + **`Authorization` is deleted**. That deletion is load-bearing: the kernel's human path (svoperatingsystem + `u1-principals` INV-U1-06) refuses any request carrying `Authorization`. As a result the front door is human-only; a + bearer client cannot come through it. +- **Retired 2026-10-05:** the WhoIs `forward_auth` gate (`svos-whois-auth`, 127.0.0.1:8405). Every mesh node has WhoIs + login `pfi`, so it could not tell Prime's devices from the servers without restructuring the mesh, and Prime declined + that. The unit, script, `/etc/svos-front/` and the repo files are removed; git history has them. - **Config changes:** validate WITHOUT putting the token on a command line, then reload: `sudo systemd-run --wait --pipe --collect -p User=caddy -p Group=caddy -p EnvironmentFile=/etc/caddy/env -p "Environment=XDG_DATA_HOME=/var/lib/caddy XDG_CONFIG_HOME=/var/lib/caddy" /usr/local/bin/caddy validate --config --adapter caddyfile`, then `systemctl reload caddy-svos`. (`env $(cat /etc/caddy/env) caddy …` leaked the first token into sudo's argv log on 2026-10-05; it was revoked.) -- **Tested 2026-10-05:** LAN → no connection; mesh peer fv-ml1 (login `pfi`) → 403; forged `Tailscale-User-Login`, - `Tailscale-User-Whatever` and `X-Forwarded-For` → 403; the responder directly with a bogus, garbage or empty peer → 403. -- **Still to run (handoff to svos-dev's acceptance):** once a login is allowed, an allowed device gets 200 and its forged - `Tailscale-User-*` / `X-Forwarded-*` headers (arbitrary family members included) are replaced or gone at the app; a - stopped tailscaled gives 403; and the live-service vzdump + restore-mount drill once the SVOS services run. -- ⚠ **Every mesh node, servers included, has WhoIs LoginName `pfi`** (headscale has one user). `pfi` must never go in - the allow-list: it would admit every server and LAN traffic SNAT'd through a subnet router. Prime's decision pending: - a separate headscale user for his devices (recommended) or a device allow-list. +- **Tested 2026-10-05 ~1950, from mesh peer fv-ml1, against a temporary header echo on 127.0.0.1:8400 (removed after):** + - no credentials → 401 `Basic realm="SVOS"`; wrong password → 401; username `vuong` (wrong case) → 401; + - right credentials with forged `Tailscale-User-Login`/`-Whatever`, `X-Forwarded-For/Proto/Zzz`, `Forwarded` and + `X-Real-IP` → 200. Upstream saw exactly one `Tailscale-User-Login: Vuong`, no `Authorization`, none of the forged + headers, `X-Forwarded-Proto: https`, `Host: svos.phasefinal.com`, and path `/health` (the `/api` prefix stripped); + - connector route with right credentials → 502 (auth passed; the connector was not running); cert verifies; LAN + 10.100.50.43:443 → connection refused. +- **Still to run:** the live-service vzdump + restore-mount drill, once svos-dev runs the services. ### Original requirements (svos-dev)