diff --git a/.github/workflows/sandbox-ci.yml b/.github/workflows/sandbox-ci.yml new file mode 100644 index 0000000..48e1c7a --- /dev/null +++ b/.github/workflows/sandbox-ci.yml @@ -0,0 +1,36 @@ +# The extension sandbox's deny rules, proven on real traffic. +# +# scripts/sandbox-rules-test.py loads meta-forgefirm's ffx.nft into a network +# namespace of its own and sends at it from real uids: a pool uid is refused +# on loopback (IPv4 and IPv6, TCP at once, UDP with EPERM), the uids on +# either side of the pool and root are not touched, an allowlist chain opens +# one destination to one uid, and loading the file again takes it away. The +# runner's nft is the version the image carries or newer; the kernel side is +# the image's own business and is proven on the machine (exthost.platform). + +name: sandbox-ci + +on: + push: + branches: [master] + paths: + - 'meta-forgefirm/recipes-forgefirm/forgefirm-sandbox/**' + - 'scripts/sandbox-rules-test.py' + - '.github/workflows/sandbox-ci.yml' + pull_request: + workflow_dispatch: + +jobs: + rules: + runs-on: ubuntu-latest + timeout-minutes: 10 + steps: + - uses: actions/checkout@v4 + + - name: nft + run: sudo apt-get update && sudo apt-get install -y nftables + + - name: The deny rules on real traffic + run: | + # 77 is the script's "could not run here": on this runner that is a failure. + sudo python3 -B scripts/sandbox-rules-test.py diff --git a/forgetest/forgetest/suite/__init__.py b/forgetest/forgetest/suite/__init__.py index 74c7930..262d9cf 100644 --- a/forgetest/forgetest/suite/__init__.py +++ b/forgetest/forgetest/suite/__init__.py @@ -9,6 +9,7 @@ order. Each module registers its tests with @catalog.test.""" from . import image # noqa: F401,E402 from . import kernel # noqa: F401,E402 from . import forgectrl # noqa: F401,E402 +from . import exthost # noqa: F401,E402 from . import setup # noqa: F401,E402 from . import setup_dark # noqa: F401,E402 from . import setup_sheet # noqa: F401,E402 diff --git a/forgetest/forgetest/suite/exthost.py b/forgetest/forgetest/suite/exthost.py new file mode 100644 index 0000000..f594554 --- /dev/null +++ b/forgetest/forgetest/suite/exthost.py @@ -0,0 +1,440 @@ +# Copyright 2026 514 LLC d/b/a OpenGlow +# Written by Scott Wiederhold +# https://community.openglow.org +# SPDX-License-Identifier: MIT + +"""exthost.* - what holds an extension package: the image's sandbox platform.""" +import json +import os +import re +import signal +import subprocess +import sys +import time + +from ..catalog import test +from .forgectrl import lan_ip +from .image import kernel_config + +CG = "/sys/fs/cgroup" +POOL_GROUP = CG + "/ffx" +PROBE_GROUP = POOL_GROUP + "/forgetest-probe" +POOL_FIRST, POOL_SIZE = 800, 32 +CONTROLLERS = ("cpu", "memory", "pids") +NFT = "/usr/sbin/nft" + +# The probe: one short program, started as root. It puts itself into the +# group named in argv, becomes the uid named there (0 stays root), gives up +# new privileges, and then does one thing and prints one JSON line. +PROBE = r''' +import ctypes, errno, json, os, platform, socket, sys, time +group, uid, mode, arg = sys.argv[1], int(sys.argv[2]), sys.argv[3], sys.argv[4] +libc = ctypes.CDLL(None, use_errno=True) +if group != "-": + with open(group + "/cgroup.procs", "w") as f: + f.write(str(os.getpid())) +if uid: + os.setgroups([]) + os.setgid(uid) + os.setuid(uid) +libc.prctl(38, 1, 0, 0, 0) # PR_SET_NO_NEW_PRIVS + + +def say(**kw): + print(json.dumps(kw), flush=True) + + +def word(e): + return {errno.ECONNREFUSED: "refused", errno.EPERM: "eperm", errno.EACCES: "eacces"}.get( + e.errno, errno.errorcode.get(e.errno, str(e.errno))) + + +def reach(family, addr, port, udp): + s = socket.socket(socket.AF_INET6 if family == 6 else socket.AF_INET, + socket.SOCK_DGRAM if udp else socket.SOCK_STREAM) + s.settimeout(4.0) + t0 = time.time() + try: + if udp: + s.sendto(b"x", (addr, port)) + else: + s.connect((addr, port)) + out = "ok" + except socket.timeout: + out = "timeout" + except OSError as e: + out = word(e) + s.close() + return [out, round(time.time() - t0, 2)] + + +if mode == "spin": + say(started=True) + while True: + pass +elif mode == "forks": + kids = [] + err = "" + for _ in range(int(arg)): + try: + pid = os.fork() + except OSError as e: + err = word(e) + break + if pid == 0: + time.sleep(30) + os._exit(0) + kids.append(pid) + for pid in kids: + os.kill(pid, 9) + os.waitpid(pid, 0) + say(forked=len(kids), stopped_by=err) +elif mode == "eat": + say(started=True) + held = [] + for _ in range(int(arg)): + held.append(bytearray(os.urandom(4096)) * 256) # 1 MiB, every page written + say(survived=True) +elif mode == "net": + say(results=[[t, reach(*t)] for t in json.loads(arg)]) +elif mode == "landlock": + target = json.loads(arg) + before = {"etc": None, "usr": None, "tcp": reach(*target)[0]} + for key, path in (("etc", "/etc/hostname"), ("usr", sys.executable)): + try: + open(path, "rb").close() + before[key] = "ok" + except OSError as e: + before[key] = word(e) + abi = libc.syscall(444, None, 0, 1) # LANDLOCK_CREATE_RULESET_VERSION + after = {} + if abi >= 1: + # write, read file, read dir; and from ABI 4 on, connect TCP (an older kernel takes the first field alone) + attr = (ctypes.c_uint64 * 2)((1 << 1) | (1 << 2) | (1 << 3), 1 << 1) + ruleset = libc.syscall(444, ctypes.byref(attr), 16 if abi >= 4 else 8, 0) + usr = os.open("/usr", os.O_PATH | os.O_CLOEXEC) + + class PathBeneath(ctypes.Structure): + _pack_ = 1 + _fields_ = [("allowed_access", ctypes.c_uint64), ("parent_fd", ctypes.c_int32)] + rule = PathBeneath((1 << 2) | (1 << 3), usr) + added = libc.syscall(445, ruleset, 1, ctypes.byref(rule), 0) # LANDLOCK_RULE_PATH_BENEATH + applied = libc.syscall(446, ruleset, 0) + after = {"ruleset": ruleset >= 0, "added": added, "applied": applied, "tcp": reach(*target)[0]} + for key, path in (("etc", "/etc/hostname"), ("usr", sys.executable)): + try: + open(path, "rb").close() + after[key] = "ok" + except OSError as e: + after[key] = word(e) + say(abi=abi, before=before, after=after) +elif mode == "seccomp": + arch, nr = {"armv7l": (0x40000028, 122), "aarch64": (0xC00000B7, 160), + "x86_64": (0xC000003E, 63)}[platform.machine()] # uname + + + class Filter(ctypes.Structure): + _fields_ = [("code", ctypes.c_uint16), ("jt", ctypes.c_uint8), ("jf", ctypes.c_uint8), + ("k", ctypes.c_uint32)] + + + class Prog(ctypes.Structure): + _fields_ = [("len", ctypes.c_ushort), ("filter", ctypes.POINTER(Filter))] + before = os.uname().sysname + ops = (Filter * 7)(Filter(0x20, 0, 0, 4), # load arch + Filter(0x15, 1, 0, arch), # ours: go on + Filter(0x06, 0, 0, 0x80000000), # another ABI: kill the process + Filter(0x20, 0, 0, 0), # load the syscall number + Filter(0x15, 0, 1, nr), # uname? + Filter(0x06, 0, 0, 0x00050000 | errno.EPERM), + Filter(0x06, 0, 0, 0x7FFF0000)) # everything else: allow + prog = Prog(7, ops) + rc = libc.prctl(22, 2, ctypes.byref(prog), 0, 0) # PR_SET_SECCOMP, SECCOMP_MODE_FILTER + try: + os.uname() + after = "ok" + except OSError as e: + after = word(e) + mode_now = [l.split()[1] for l in open("/proc/self/status") if l.startswith("Seccomp:")] + say(installed=rc, before=before, after=after, status=mode_now) +''' + + +def _read(path, default=""): + try: + with open(path, "r", encoding="utf-8", errors="replace") as f: + return f.read() + except OSError: + return default + + +def _write(path, text): + with open(path, "w") as f: + f.write(text) + + +def _flat(path): + """A cgroup flat-keyed file as {key: int}.""" + out = {} + for line in _read(path).splitlines(): + parts = line.split() + if len(parts) == 2 and parts[1].isdigit(): + out[parts[0]] = int(parts[1]) + return out + + +def _probe(group, uid, mode, arg="", wait=30): + """Runs the probe to its end; (returncode, its JSON lines).""" + p = subprocess.run([sys.executable, "-c", PROBE, group, str(uid), mode, arg], + capture_output=True, text=True, timeout=wait) + lines = [] + for line in p.stdout.splitlines(): + try: + lines.append(json.loads(line)) + except ValueError: + pass + return p.returncode, lines, p.stderr.strip()[-300:] + + +def _pids_of(comm): + out = [] + for d in os.listdir("/proc"): + if d.isdigit() and _read("/proc/%s/comm" % d).strip() == comm: + out.append(int(d)) + return out + + +def _pool_counters(): + """The packet counts of chain pool's two refusals: [tcp reset, drop].""" + p = subprocess.run([NFT, "-j", "list", "chain", "inet", "ffx", "pool"], capture_output=True, text=True) + if p.returncode != 0: + return None + return [e["counter"]["packets"] for c in json.loads(p.stdout)["nftables"] if "rule" in c + for e in c["rule"]["expr"] if "counter" in e] + + +@test("exthost.platform", title="The extension sandbox platform holds a process", + subsystem="exthost", kind="auto", est_min=2, + covers=[("linux-fslc", "**")], + description="What the image holds ready before any extension package exists, proven on a " + "probe process rather than read off a config. The kernel has the cpu, memory, " + "and pids controllers, seccomp filters, and landlock, and has no realtime " + "group scheduler (the pulse feeder is SCHED_FIFO and must not need a group's " + "leave to run). cgroup v2 is mounted with the three controllers handed down " + "to /sys/fs/cgroup/ffx, which is idle-class, and forgectrl and the controller " + "sit in the root group. In a probe group under it, as the last pool uid: a " + "spinning process is held to its cpu.max, stops when the group is frozen and " + "runs again when thawed; a fork loop stops at pids.max; and a process that " + "takes more than memory.max is killed by the group's own OOM while forgectrl " + "keeps its pid. The account pool is ffx0 to ffx31, uid and gid 800 to 831, no " + "shell, no home, locked, in the account files as forgefirm-users rendered " + "them. The deny rules are in the kernel: a pool uid's TCP connect to " + "forgectrl on loopback (both ports, IPv4 and IPv6), to the machine's LAN " + "address, and to the Grbl port is refused at once, its UDP send gets EPERM, " + "and the rules' counters saw every attempt, while root reaches the same " + "listeners. A process that restricts itself with landlock loses /etc and TCP " + "connects and keeps /usr; one that installs a seccomp filter gets EPERM from " + "the filtered call. The layer content (the recipe, the rules, the distro " + "option) is in the platform identity of every fingerprint.") +def platform(ctx): + ev = ctx.evidence + + # 1. the kernel + cfg = kernel_config() + ctx.check(cfg is not None, "kernel config unavailable (/proc/config.gz, /boot/config-*)") + want = {"CONFIG_CGROUPS": "y", "CONFIG_CGROUP_SCHED": "y", "CONFIG_FAIR_GROUP_SCHED": "y", + "CONFIG_CFS_BANDWIDTH": "y", "CONFIG_MEMCG": "y", "CONFIG_CGROUP_PIDS": "y", + "CONFIG_SECCOMP_FILTER": "y", "CONFIG_SECURITY_LANDLOCK": "y", "CONFIG_NF_TABLES": "y", + "CONFIG_NF_TABLES_INET": "y", "CONFIG_NFT_REJECT_INET": "y", "CONFIG_NFT_LIMIT": "y", + "CONFIG_RT_GROUP_SCHED": "n", "CONFIG_NF_CONNTRACK": "n"} + got = {k: cfg.get(k, "n") for k in want} + ev["kernel"] = got + wrong = {k: v for k, v in got.items() if v != want[k]} + ctx.check(not wrong, "kernel options: %s (wanted %s)", wrong, {k: want[k] for k in wrong}) + ctx.check("landlock" in cfg.get("CONFIG_LSM", ""), "landlock is not in CONFIG_LSM: %s", cfg.get("CONFIG_LSM")) + + # 2. the cgroup tree + mounts = [l.split() for l in _read("/proc/mounts").splitlines()] + ctx.check(any(m[1] == CG and m[2] == "cgroup2" for m in mounts if len(m) > 2), "cgroup2 is not mounted at %s", CG) + tree = {"controllers": _read(CG + "/cgroup.controllers").split(), + "root_subtree": _read(CG + "/cgroup.subtree_control").split(), + "ffx_subtree": _read(POOL_GROUP + "/cgroup.subtree_control").split(), + "ffx_cpu_idle": _read(POOL_GROUP + "/cpu.idle").strip()} + ev["cgroup_tree"] = tree + ctx.log("cgroup tree: %s", tree) + for key in ("controllers", "root_subtree", "ffx_subtree"): + ctx.check(set(CONTROLLERS) <= set(tree[key]), "%s lacks a controller: %s", key, tree[key]) + ctx.check(tree["ffx_cpu_idle"] == "1", "the ffx group is not idle-class: cpu.idle=%r", tree["ffx_cpu_idle"]) + homes = {} + for comm in ("forgectrl", "grblHAL_glowfor", "forgetest"): + for pid in _pids_of(comm): + homes["%s/%d" % (comm, pid)] = _read("/proc/%d/cgroup" % pid).strip() + ev["firmware_groups"] = homes + ctx.check(any(k.startswith("forgectrl/") for k in homes), "forgectrl is not running") + ctx.check(all(v == "0::/" for v in homes.values()), "a firmware process is outside the root group: %s", homes) + + # 3. the account pool, as the boot's render left it + passwd = {l.split(":")[0]: l.split(":") for l in _read("/etc/passwd").splitlines() if l.count(":") >= 6} + shadow = {l.split(":")[0]: l.split(":") for l in _read("/etc/shadow").splitlines() if ":" in l} + group = {l.split(":")[0]: l.split(":") for l in _read("/etc/group").splitlines() if l.count(":") >= 3} + bad = [] + for n in range(POOL_SIZE): + name, uid = "ffx%d" % n, str(POOL_FIRST + n) + p, s, g = passwd.get(name), shadow.get(name), group.get(name) + if not p or p[2] != uid or p[3] != uid or p[5] != "/nonexistent" or p[6] != "/bin/false": + bad.append("passwd %s: %s" % (name, p)) + if not s or s[1][:1] not in ("!", "*"): + bad.append("shadow %s is not locked" % name) + if not g or g[2] != uid or g[3] != "": + bad.append("group %s: %s" % (name, g)) + others = [p[0] for p in passwd.values() if p[2].isdigit() + and POOL_FIRST <= int(p[2]) < POOL_FIRST + POOL_SIZE and not re.match(r"^ffx\d+$", p[0])] + ev["pool"] = {"accounts": sum(1 for n in passwd if re.match(r"^ffx\d+$", n)), "problems": bad, "others": others} + ctx.check(not bad, "the account pool: %s", bad[:6]) + ctx.check(not others, "another account sits in the pool's uid range: %s", others) + ctx.check(ev["pool"]["accounts"] == POOL_SIZE, "%d ffx accounts, expected %d", ev["pool"]["accounts"], POOL_SIZE) + + uid = POOL_FIRST + POOL_SIZE - 1 + fc_pid = _pids_of("forgectrl") + spin = None + try: + # 4. a probe group holds a process: cpu.max, the freezer, pids.max, memory.max + os.makedirs(PROBE_GROUP, exist_ok=True) + _write(PROBE_GROUP + "/cpu.max", "5000 100000") + _write(PROBE_GROUP + "/pids.max", "6") + _write(PROBE_GROUP + "/memory.max", str(24 << 20)) + if os.path.exists(PROBE_GROUP + "/memory.swap.max"): + _write(PROBE_GROUP + "/memory.swap.max", "0") + + spin = subprocess.Popen([sys.executable, "-c", PROBE, PROBE_GROUP, str(uid), "spin", ""], + stdout=subprocess.PIPE, text=True) + ctx.check("started" in (spin.stdout.readline() or ""), "the spinning probe did not start") + ctx.check(_read("/proc/%d/cgroup" % spin.pid).strip() == "0::/ffx/forgetest-probe", + "the probe is in %r", _read("/proc/%d/cgroup" % spin.pid).strip()) + ctx.check("Uid:\t%d\t%d" % (uid, uid) in _read("/proc/%d/status" % spin.pid), "the probe is not uid %d", uid) + ctx.sleep(1.0) + u0, t0 = _flat(PROBE_GROUP + "/cpu.stat")["usage_usec"], time.time() + ctx.sleep(3.0) + u1, t1 = _flat(PROBE_GROUP + "/cpu.stat")["usage_usec"], time.time() + share = (u1 - u0) / 1e6 / (t1 - t0) + throttled = _flat(PROBE_GROUP + "/cpu.stat").get("nr_throttled", 0) + ctx.log("a spinning probe under cpu.max 5%%: %.1f%% of the core, throttled %d times", share * 100, throttled) + ctx.check(0.01 < share < 0.08 and throttled > 0, "cpu.max 5%% held the probe to %.1f%% (throttled %d)", + share * 100, throttled) + + _write(PROBE_GROUP + "/cgroup.freeze", "1") + ok = ctx.wait_for(lambda: _flat(PROBE_GROUP + "/cgroup.events").get("frozen") == 1, 3, poll=0.05) + ctx.check(ok is not None, "the group did not freeze: %s", _flat(PROBE_GROUP + "/cgroup.events")) + f0 = _flat(PROBE_GROUP + "/cpu.stat")["usage_usec"] + ctx.sleep(1.5) + f1 = _flat(PROBE_GROUP + "/cpu.stat")["usage_usec"] + _write(PROBE_GROUP + "/cgroup.freeze", "0") + ok = ctx.wait_for(lambda: _flat(PROBE_GROUP + "/cgroup.events").get("frozen") == 0, 3, poll=0.05) + ctx.check(ok is not None, "the group did not thaw") + ctx.sleep(1.5) + f2 = _flat(PROBE_GROUP + "/cpu.stat")["usage_usec"] + ctx.log("frozen for 1.5 s the probe ran %d us; thawed for 1.5 s it ran %d us", f1 - f0, f2 - f1) + ctx.check(f1 - f0 == 0, "a frozen probe ran %d us", f1 - f0) + ctx.check(f2 - f1 > 10000, "a thawed probe ran only %d us", f2 - f1) + _write(PROBE_GROUP + "/cgroup.kill", "1") + spin.wait(timeout=5) + ctx.check(spin.returncode == -signal.SIGKILL, "cgroup.kill left the probe with %s", spin.returncode) + spin = None + ev["cpu"] = {"share": round(share, 4), "throttled": throttled, "frozen_us": f1 - f0, "thawed_us": f2 - f1} + + rc, lines, err = _probe(PROBE_GROUP, uid, "forks", "12") + forks = lines[-1] if lines else {} + hits = _flat(PROBE_GROUP + "/pids.events").get("max", 0) + ev["pids"] = {"result": forks, "events_max": hits} + ctx.log("pids.max 6: the fork loop made %s of 12, stopped by %s; pids.events max %d", + forks.get("forked"), forks.get("stopped_by"), hits) + ctx.check(rc == 0 and forks.get("forked") == 5 and forks.get("stopped_by") == "EAGAIN" and hits > 0, + "pids.max 6 did not stop the fork loop at 5: rc %s %s %s, events max %d", rc, forks, err, hits) + + rc, lines, err = _probe(PROBE_GROUP, uid, "eat", "96", wait=60) + kills = _flat(PROBE_GROUP + "/memory.events").get("oom_kill", 0) + ev["memory"] = {"rc": rc, "lines": lines, "oom_kill": kills, "peak": _read(PROBE_GROUP + "/memory.peak").strip()} + ctx.log("memory.max 24 MiB against a 96 MiB appetite: rc %s, oom_kill %d, peak %s", rc, kills, ev["memory"]["peak"]) + ctx.check(rc == -signal.SIGKILL and kills > 0 and not any(l.get("survived") for l in lines), + "memory.max did not kill the probe: rc %s, oom_kill %d, %s %s", rc, kills, lines, err) + ctx.check(_pids_of("forgectrl") == fc_pid, "forgectrl's pid moved across the group's OOM: %s -> %s", + fc_pid, _pids_of("forgectrl")) + + # 5. the deny rules + p = subprocess.run([NFT, "list", "table", "inet", "ffx"], capture_output=True, text=True) + ctx.check(p.returncode == 0, "table inet ffx is not loaded: %s", p.stderr.strip()[:200]) + ev["rules"] = p.stdout + ctx.check("meta skuid %d-%d jump pool" % (POOL_FIRST, uid) in p.stdout and "hook output" in p.stdout + and "policy accept" in p.stdout, "table inet ffx is not the image's: %s", p.stdout[:400]) + lan = lan_ip() + ctx.check(lan, "the machine has no LAN address to aim at") + tcp = [[4, "127.0.0.1", 443, False], [4, "127.0.0.1", 80, False], [6, "::1", 443, False], + [6, "::1", 23, False], [4, lan, 443, False], [4, lan, 23, False]] + udp = [[4, "127.0.0.1", 9, True], [4, lan, 9, True]] + control = [[4, "127.0.0.1", 443, False], [4, lan, 443, False], [4, "127.0.0.1", 9, True]] + rc, lines, err = _probe("-", 0, "net", json.dumps(control)) + as_root = (lines[-1] if lines else {}).get("results", []) + ev["as_root"] = as_root + ctx.check(len(as_root) == len(control) and all(r[1][0] == "ok" for r in as_root), + "root does not reach its own listeners: %s %s", as_root, err) + c0 = _pool_counters() + heard = {} + for who in (POOL_FIRST, uid): + rc, lines, err = _probe("-", who, "net", json.dumps(tcp + udp)) + res = (lines[-1] if lines else {}).get("results", []) + heard[str(who)] = res + ctx.check(len(res) == len(tcp + udp), "the probe as uid %d said %s %s", who, lines, err) + for target, (word, took) in res: + if target[3]: + ctx.check(word == "eperm", "uid %d UDP to %s -> %s", who, target[1], word) + else: + ctx.check(word == "refused" and took < 1.5, "uid %d TCP to %s port %d -> %s in %.2f s", + who, target[1], target[2], word, took) + c1 = _pool_counters() + ev["as_pool_uid"] = heard + ev["counters"] = [c0, c1] + ctx.log("two pool uids: %d TCP connects each refused at once, %d UDP sends each EPERM; the rules " + "counted %s -> %s", len(tcp), len(udp), c0, c1) + ctx.check(c0 is not None and c1 is not None and len(c1) == 2 and c1[0] - c0[0] >= 2 * len(tcp) + and c1[1] - c0[1] >= 2 * len(udp), "the rules did not count the attempts: %s -> %s", c0, c1) + + # 6. landlock and seccomp, on a root process so that neither the uid nor the rules explain the refusal + rc, lines, err = _probe("-", 0, "landlock", json.dumps([4, "127.0.0.1", 443, False])) + ll = lines[-1] if lines else {} + ev["landlock"] = ll + ctx.log("landlock ABI %s: before %s, after %s", ll.get("abi"), ll.get("before"), ll.get("after")) + ctx.check(rc == 0 and ll.get("abi", 0) >= 4, "landlock ABI %s (4 brings the TCP rules) %s", ll.get("abi"), err) + ctx.check(ll["before"] == {"etc": "ok", "usr": "ok", "tcp": "ok"}, "before the restriction: %s", ll["before"]) + after = ll["after"] + ctx.check(after.get("ruleset") and after.get("added") == 0 and after.get("applied") == 0, + "the ruleset did not apply: %s", after) + ctx.check(after.get("etc") == "eacces" and after.get("tcp") == "eacces" and after.get("usr") == "ok", + "after the restriction (/etc and TCP lost, /usr kept): %s", after) + + rc, lines, err = _probe("-", 0, "seccomp") + sc = lines[-1] if lines else {} + ev["seccomp"] = sc + ctx.log("seccomp: %s", sc) + ctx.check(rc == 0 and sc.get("installed") == 0 and sc.get("before") == "Linux" and sc.get("after") == "eperm" + and sc.get("status") == ["2"], "a seccomp filter did not take: rc %s %s %s", rc, sc, err) + finally: + if spin is not None: + spin.kill() + spin.wait(timeout=5) + if os.path.isdir(PROBE_GROUP): + try: + _write(PROBE_GROUP + "/cgroup.freeze", "0") + _write(PROBE_GROUP + "/cgroup.kill", "1") + except OSError: + pass + for _ in range(50): + if not _read(PROBE_GROUP + "/cgroup.procs").strip(): + break + time.sleep(0.1) + try: + os.rmdir(PROBE_GROUP) + except OSError as e: + ctx.log("the probe group stayed behind: %s", e) + ctx.check(not os.path.isdir(PROBE_GROUP), "the probe group stayed behind") + ctx.log("PASS: the kernel, the cgroup tree, the account pool, and the deny rules are in place, and each " + "held a probe process the way it will hold a package") diff --git a/meta-forgefirm/conf/distro/forgefirm.conf b/meta-forgefirm/conf/distro/forgefirm.conf index e8a3186..62848bf 100644 --- a/meta-forgefirm/conf/distro/forgefirm.conf +++ b/meta-forgefirm/conf/distro/forgefirm.conf @@ -61,6 +61,12 @@ BAD_RECOMMENDATIONS += "eudev-hwdb" VIRTUAL-RUNTIME_base-utils-syslog = "rsyslog" PACKAGECONFIG:pn-rsyslog = "rsyslogd rsyslogrt klog inet regexp" +# nftables: the extension sandbox's rule loader (forgefirm-sandbox). The nft +# binary and its library, with JSON output so a program can read back what +# the kernel holds; no interactive shell (readline) and no Python binding. +# gmp and jansson are on the image already. +PACKAGECONFIG:pn-nftables = "json" + # Local (file://) source checksums leave out a workstation's Python bytecode # caches: the forgetest recipe fetches its package directory whole, and a # host test run must not move a task hash without a source change. diff --git a/meta-forgefirm/recipes-forgefirm/forgefirm-sandbox/files/ffx.nft b/meta-forgefirm/recipes-forgefirm/forgefirm-sandbox/files/ffx.nft new file mode 100644 index 0000000..0ee25ca --- /dev/null +++ b/meta-forgefirm/recipes-forgefirm/forgefirm-sandbox/files/ffx.nft @@ -0,0 +1,44 @@ +#!/usr/sbin/nft -f +# Copyright 2026 514 LLC d/b/a OpenGlow +# Written by Scott Wiederhold +# https://community.openglow.org +# SPDX-License-Identifier: MIT +# +# The extension sandbox's network rules (forgefirm-sandbox loads them from +# rcS, before the network starts). Every packet sent from a socket that an +# extension account owns (ffx0 to ffx31, uid 800 to 831) is refused: on +# loopback, on the machine's own LAN address, IPv4 and IPv6 alike. That is +# what keeps a package off the Grbl port, off forgectrl's listeners, and off +# the controller's report route, whatever else fails. +# +# The rules sit on the output hook and match the sending socket's uid, so +# they need no connection tracking, and no packet of the firmware's own pays +# for more than one comparison. A refused TCP connect gets a reset, so it +# fails at once instead of timing out; anything else is dropped, which the +# sender sees as EPERM. +# +# The one way through is the allow map: uid -> a chain holding that package's +# declared destinations. Whatever starts a package adds the element and the +# chain, and removes both when the package stops; a chain that accepts +# nothing returns here and the packet is refused. Loading this file again +# replaces the whole table, allowlists included: it fails closed. + +table inet ffx +delete table inet ffx + +table inet ffx { + map allow { + typeof meta skuid : verdict + } + + chain output { + type filter hook output priority filter; policy accept; + meta skuid 800-831 jump pool + } + + chain pool { + meta skuid vmap @allow + meta l4proto tcp counter reject with tcp reset + counter drop + } +} diff --git a/meta-forgefirm/recipes-forgefirm/forgefirm-sandbox/files/forgefirm-sandbox.init b/meta-forgefirm/recipes-forgefirm/forgefirm-sandbox/files/forgefirm-sandbox.init new file mode 100644 index 0000000..9791c06 --- /dev/null +++ b/meta-forgefirm/recipes-forgefirm/forgefirm-sandbox/files/forgefirm-sandbox.init @@ -0,0 +1,94 @@ +#!/bin/sh +# Copyright 2026 514 LLC d/b/a OpenGlow +# Written by Scott Wiederhold +# https://community.openglow.org +# SPDX-License-Identifier: MIT +### BEGIN INIT INFO +# Provides: forgefirm-sandbox +# Required-Start: mountkernfs +# Required-Stop: +# Default-Start: S +# Default-Stop: +# Short-Description: ForgeFIRM extension sandbox: the cgroup tree and the deny rules +### END INIT INFO + +# Runs at S30 in rcS, before the network starts (rc5). Two jobs, and each +# is told apart in what it prints and in its exit status, because whatever +# starts extension services checks both in the kernel and starts nothing +# when either is missing: +# +# the cgroup v2 tree mounted at /sys/fs/cgroup, the cpu, memory, and pids +# controllers handed down to /sys/fs/cgroup/ffx and +# from there to the group each package gets. The ffx +# group is idle-class against everything outside it: +# a runnable task in the root group always goes first. +# the deny rules table inet ffx from /etc/forgefirm/ffx.nft: a pool +# uid sends nothing, loopback included. +# +# The firmware's own processes stay in the root group, which has no limit +# and no controller file that could give it one. + +CG=/sys/fs/cgroup +RULES=/etc/forgefirm/ffx.nft +CONTROLLERS="cpu memory pids" + +cgroup_mounted () { + awk -v m="$CG" '$2 == m && $3 == "cgroup2" { f = 1 } END { exit !f }' /proc/mounts +} + +# hand_down : every controller into the group's subtree_control. +hand_down () { + for c in $CONTROLLERS; do + echo "+$c" > "$1/cgroup.subtree_control" 2>/dev/null || return 1 + done +} + +start_cgroups () { + cgroup_mounted || mount -t cgroup2 -o nosuid,nodev,noexec cgroup2 "$CG" || return 1 + hand_down "$CG" || return 1 + mkdir -p "$CG/ffx" || return 1 + hand_down "$CG/ffx" || return 1 + # An idle group's weight is the scheduler's own lowest; the kernel refuses + # a cpu.weight written on top of it. + echo 1 > "$CG/ffx/cpu.idle" || return 1 +} + +start_rules () { + /usr/sbin/nft -f "$RULES" +} + +status () { + rc=0 + if cgroup_mounted && [ -d "$CG/ffx" ]; then + echo "cgroups: $CG/ffx controls: $(cat "$CG/ffx/cgroup.subtree_control")" + else + echo "cgroups: not set up"; rc=1 + fi + if /usr/sbin/nft list table inet ffx >/dev/null 2>&1; then + echo "deny rules: loaded" + else + echo "deny rules: NOT loaded"; rc=1 + fi + return $rc +} + +case "$1" in + start|restart|reload|force-reload) + rc=0 + start_cgroups || { echo "forgefirm-sandbox: the cgroup tree could not be set up"; rc=1; } + start_rules || { echo "forgefirm-sandbox: the deny rules did not load"; rc=1; } + exit $rc + ;; + stop) + ;; + status) + status + exit $? + ;; + *) + echo "Usage: $0 {start|stop|restart|status}" + exit 1 + ;; +esac + +exit 0 diff --git a/meta-forgefirm/recipes-forgefirm/forgefirm-sandbox/forgefirm-sandbox.bb b/meta-forgefirm/recipes-forgefirm/forgefirm-sandbox/forgefirm-sandbox.bb new file mode 100644 index 0000000..c511909 --- /dev/null +++ b/meta-forgefirm/recipes-forgefirm/forgefirm-sandbox/forgefirm-sandbox.bb @@ -0,0 +1,46 @@ +# SPDX-License-Identifier: MIT + +SUMMARY = "ForgeFIRM extension sandbox: the account pool, the cgroup tree, and the deny rules" +DESCRIPTION = "What the image holds ready before any extension package runs: \ +a static pool of system accounts (ffx0 to ffx31, uid and gid 800 to 831), \ +the cgroup v2 tree with the cpu, memory, and pids controllers handed down to \ +/sys/fs/cgroup/ffx, and the nftables table that refuses every packet a pool \ +uid sends, loopback included, until an allowlist names its destination. The \ +rules load from rcS, before the network starts." +LICENSE = "MIT" +LIC_FILES_CHKSUM = "file://${COMMON_LICENSE_DIR}/MIT;md5=0835ade698e0bcf8506ecda2f7b4f302" + +SRC_URI = " \ + file://forgefirm-sandbox.init \ + file://ffx.nft \ +" + +S = "${WORKDIR}" + +inherit update-rc.d useradd + +INITSCRIPT_NAME = "forgefirm-sandbox" +# 30 in rcS: /sys is mounted (sysfs.sh, S02) and nothing the script needs +# lives on /data. The network starts in rc5 (S01), so no pool uid ever has +# an interface to send on before the rules are in the kernel. +INITSCRIPT_PARAMS = "start 30 S ." + +# The pool. Static ids below 1000, in a block no other account uses (the +# image's dynamic system ids count down from 999): the forgefirm-users render +# replaces only the accounts from 1000 up, so an account reset leaves these +# alone, and the read-only rootfs never needs a useradd at run time. One +# group per account, so no two slots share a group. No home, no shell, and +# the password field useradd leaves locked. The rules in ffx.nft and the +# pool size here name the same range. +FFX_POOL_SIZE = "32" +FFX_POOL_BASE = "800" +USERADD_PACKAGES = "${PN}" +GROUPADD_PARAM:${PN} = "${@'; '.join('--system -g %d ffx%d' % (int(d.getVar('FFX_POOL_BASE')) + n, n) for n in range(int(d.getVar('FFX_POOL_SIZE'))))}" +USERADD_PARAM:${PN} = "${@'; '.join('--system -u %d -g ffx%d -M -d /nonexistent -s /bin/false ffx%d' % (int(d.getVar('FFX_POOL_BASE')) + n, n, n) for n in range(int(d.getVar('FFX_POOL_SIZE'))))}" + +do_install() { + install -Dm 0755 ${WORKDIR}/forgefirm-sandbox.init ${D}${sysconfdir}/init.d/forgefirm-sandbox + install -Dm 0644 ${WORKDIR}/ffx.nft ${D}${sysconfdir}/forgefirm/ffx.nft +} + +RDEPENDS:${PN} = "nftables" diff --git a/meta-forgefirm/recipes-forgefirm/images/forgefirm-image.bb b/meta-forgefirm/recipes-forgefirm/images/forgefirm-image.bb index 6974fb7..ee6fa89 100644 --- a/meta-forgefirm/recipes-forgefirm/images/forgefirm-image.bb +++ b/meta-forgefirm/recipes-forgefirm/images/forgefirm-image.bb @@ -64,6 +64,13 @@ IMAGE_INSTALL:append = " grblhal-glowforge forgectrl gfhome gfcloud v4l-utils fw # forgefirm-persist: the boot timestamp and the random seed on /data. IMAGE_INSTALL:append = " forgefirm-users forgefirm-hostname forgefirm-banner forgefirm-persist" +# forgefirm-sandbox: what an extension package is held by, in place before +# any package exists: the ffx account pool, the cgroup v2 tree with the cpu, +# memory, and pids controllers, and the nftables rules (nft comes with it) +# that refuse everything a pool uid sends, loaded from rcS before the +# network starts. +IMAGE_INSTALL:append = " forgefirm-sandbox" + # The rootfs mounts read-only on both images; /data (p3) is the writable # partition. read-only-rootfs is poky's feature for it: the root line of # /etc/fstab (the BSP's, already ro) and ROOTFS_READ_ONLY in diff --git a/scripts/sandbox-rules-test.py b/scripts/sandbox-rules-test.py new file mode 100644 index 0000000..d949b12 --- /dev/null +++ b/scripts/sandbox-rules-test.py @@ -0,0 +1,206 @@ +#!/usr/bin/env python3 +# Copyright 2026 514 LLC d/b/a OpenGlow +# Written by Scott Wiederhold +# https://community.openglow.org +# SPDX-License-Identifier: MIT + +"""Host test of the extension sandbox's deny rules. + +Loads meta-forgefirm's ffx.nft into a network namespace of its own and sends +real traffic at it from real uids: an extension account is refused on +loopback, IPv4 and IPv6, TCP at once and UDP with EPERM; the uids on either +side of the pool, and root, are not touched; an allowlist chain lets one uid +reach one destination and nothing else; and loading the file again takes the +allowlist away. Needs root, nft, and a kernel with nf_tables; exits 77 when +one is missing, 0 on a pass, 1 on a failure. + + sudo python3 scripts/sandbox-rules-test.py [--nft /path/to/nft] [--rules FILE] +""" +import argparse +import errno +import fcntl +import json +import os +import shutil +import socket +import struct +import subprocess +import sys +import time + +HERE = os.path.dirname(os.path.abspath(__file__)) +RULES = os.path.join(HERE, "..", "meta-forgefirm", "recipes-forgefirm", "forgefirm-sandbox", "files", "ffx.nft") +POOL_FIRST, POOL_LAST = 800, 831 +SKIP = 77 + +failures = [] + + +def check(ok, what, *args): + text = what % args if args else what + print(" %s %s" % ("ok " if ok else "FAIL", text), flush=True) + if not ok: + failures.append(text) + + +def lo_up(): + s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + try: + ifr = struct.pack("16sH14x", b"lo", 0) + flags = struct.unpack("16sH14x", fcntl.ioctl(s, 0x8913, ifr))[1] # SIOCGIFFLAGS + fcntl.ioctl(s, 0x8914, struct.pack("16sH14x", b"lo", flags | 1)) # SIOCSIFFLAGS, IFF_UP + finally: + s.close() + + +def tcp_listener(family, addr): + s = socket.socket(family, socket.SOCK_STREAM) + s.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) + s.bind((addr, 0)) + s.listen(128) # nobody accepts: the kernel completes each connect into the backlog + return s, s.getsockname()[1] + + +def attempt(uid, family, addr, port, udp=False): + """What a process of this uid gets when it sends to addr:port: 'ok', + 'refused', 'eperm', 'timeout', or the errno's name; and how long it took.""" + r, w = os.pipe() + t0 = time.time() + pid = os.fork() + if pid == 0: + os.close(r) + word = "ok" + try: + if uid: + os.setgroups([]) + os.setgid(uid) + os.setuid(uid) + s = socket.socket(family, socket.SOCK_DGRAM if udp else socket.SOCK_STREAM) + s.settimeout(4.0) + if udp: + s.sendto(b"x", (addr, port)) + else: + s.connect((addr, port)) + s.close() + except socket.timeout: + word = "timeout" + except OSError as e: + word = {errno.ECONNREFUSED: "refused", errno.EPERM: "eperm"}.get( + e.errno, errno.errorcode.get(e.errno, str(e.errno))) + os.write(w, word.encode()) + os._exit(0) + os.close(w) + word = os.read(r, 64).decode() + os.close(r) + os.waitpid(pid, 0) + return word, time.time() - t0 + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--nft", default=shutil.which("nft") or "nft") + ap.add_argument("--rules", default=RULES) + args = ap.parse_args() + + if os.environ.get("FFX_RULES_NS") != "1": + if os.geteuid() != 0: + print("skipped: needs root (it makes a network namespace and changes uid)") + return SKIP + if not (os.path.isfile(args.nft) or shutil.which(args.nft)) or not shutil.which("unshare"): + print("skipped: needs nft and unshare") + return SKIP + env = dict(os.environ, FFX_RULES_NS="1") + os.execvpe("unshare", ["unshare", "-n", sys.executable, os.path.abspath(__file__), + "--nft", args.nft, "--rules", os.path.abspath(args.rules)], env) + + def nft(*a, check_rc=True): + p = subprocess.run([args.nft] + list(a), capture_output=True, text=True) + if check_rc and p.returncode != 0: + raise RuntimeError("nft %s -> %d %s" % (" ".join(a), p.returncode, p.stderr.strip())) + return p + + lo_up() + probe = nft("list", "tables", check_rc=False) + if probe.returncode != 0: + print("skipped: this kernel has no nf_tables (%s)" % probe.stderr.strip()) + return SKIP + + print("the file loads, twice") + for n in (1, 2): + p = nft("-f", args.rules, check_rc=False) + check(p.returncode == 0, "load %d -> %d %s", n, p.returncode, p.stderr.strip()[:300]) + if failures: + return 1 + table = json.loads(nft("-j", "list", "table", "inet", "ffx").stdout)["nftables"] + chains = {c["chain"]["name"]: c["chain"] for c in table if "chain" in c} + maps = [m["map"]["name"] for m in table if "map" in m] + out = chains.get("output", {}) + check(out.get("hook") == "output" and out.get("policy") == "accept" and out.get("type") == "filter", + "chain output is a filter on the output hook with policy accept: %s", out) + check("pool" in chains and maps == ["allow"], "chain pool and map allow exist: %s %s", sorted(chains), maps) + rules_text = nft("list", "table", "inet", "ffx").stdout + check("meta skuid %d-%d jump pool" % (POOL_FIRST, POOL_LAST) in rules_text, + "the jump names the pool's range, %d-%d", POOL_FIRST, POOL_LAST) + + l4, port4 = tcp_listener(socket.AF_INET, "127.0.0.1") + l4b, port4b = tcp_listener(socket.AF_INET, "127.0.0.1") + l6, port6 = tcp_listener(socket.AF_INET6, "::1") + u4 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) + u4.bind(("127.0.0.1", 0)) + uport = u4.getsockname()[1] + + print("outside the pool nothing is touched") + for uid in (0, POOL_FIRST - 1, POOL_LAST + 1): + for fam, addr, port in ((socket.AF_INET, "127.0.0.1", port4), (socket.AF_INET6, "::1", port6)): + word, _ = attempt(uid, fam, addr, port) + check(word == "ok", "uid %d TCP to %s -> %s", uid, addr, word) + word, _ = attempt(uid, socket.AF_INET, "127.0.0.1", uport, udp=True) + check(word == "ok", "uid %d UDP to 127.0.0.1 -> %s", uid, word) + + print("a pool uid is refused, and at once") + for uid in (POOL_FIRST, (POOL_FIRST + POOL_LAST) // 2, POOL_LAST): + for fam, addr, port in ((socket.AF_INET, "127.0.0.1", port4), (socket.AF_INET6, "::1", port6)): + word, took = attempt(uid, fam, addr, port) + check(word == "refused" and took < 1.5, "uid %d TCP to %s -> %s in %.2f s", uid, addr, word, took) + word, _ = attempt(uid, socket.AF_INET, "127.0.0.1", uport, udp=True) + check(word == "eperm", "uid %d UDP to 127.0.0.1 -> %s", uid, word) + u4.settimeout(0.3) + got = 0 + try: + while True: + u4.recvfrom(16) + got += 1 + except socket.timeout: + pass + check(got == 3, "the UDP receiver heard the three senders outside the pool and nobody else: %d", got) + counted = [e["counter"]["packets"] for c in json.loads(nft("-j", "list", "chain", "inet", "ffx", "pool").stdout)["nftables"] + if "rule" in c for e in c["rule"]["expr"] if "counter" in e] + check(len(counted) == 2 and all(n > 0 for n in counted), "both refusals counted packets: %s", counted) + + print("an allowlist opens one destination to one uid") + nft("add", "chain", "inet", "ffx", "u%d" % POOL_FIRST) + nft("add", "rule", "inet", "ffx", "u%d" % POOL_FIRST, "ip", "daddr", "127.0.0.1", "tcp", "dport", str(port4), "accept") + nft("add", "element", "inet", "ffx", "allow", "{ %d : jump u%d }" % (POOL_FIRST, POOL_FIRST)) + for uid, fam, addr, port, want, why in ( + (POOL_FIRST, socket.AF_INET, "127.0.0.1", port4, "ok", "the listed destination"), + (POOL_FIRST, socket.AF_INET, "127.0.0.1", port4b, "refused", "another port"), + (POOL_FIRST, socket.AF_INET6, "::1", port6, "refused", "another address"), + (POOL_FIRST + 1, socket.AF_INET, "127.0.0.1", port4, "refused", "another uid")): + word, _ = attempt(uid, fam, addr, port) + check(word == want, "uid %d to %s port %d (%s) -> %s", uid, addr, port, why, word) + + print("loading the file again fails closed") + nft("-f", args.rules) + word, _ = attempt(POOL_FIRST, socket.AF_INET, "127.0.0.1", port4) + check(word == "refused", "uid %d to its old destination -> %s", POOL_FIRST, word) + gone = nft("list", "chain", "inet", "ffx", "u%d" % POOL_FIRST, check_rc=False) + check(gone.returncode != 0, "the allowlist chain is gone") + + for s in (l4, l4b, l6, u4): + s.close() + print("%s: %d failure%s" % ("FAIL" if failures else "PASS", len(failures), "" if len(failures) == 1 else "s")) + return 1 if failures else 0 + + +if __name__ == "__main__": + sys.exit(main())