diff --git a/.github/workflows/sandbox-ci.yml b/.github/workflows/sandbox-ci.yml index 48e1c7a..cf590b1 100644 --- a/.github/workflows/sandbox-ci.yml +++ b/.github/workflows/sandbox-ci.yml @@ -1,10 +1,13 @@ # The extension sandbox's deny rules, proven on real traffic. # # scripts/sandbox-rules-test.py loads meta-forgefirm's ffx.nft into a network -# namespace of its own and sends at it from real uids: a pool uid is refused -# on loopback (IPv4 and IPv6, TCP at once, UDP with EPERM), the uids on -# either side of the pool and root are not touched, an allowlist chain opens -# one destination to one uid, and loading the file again takes it away. The +# namespace of its own, joins a second namespace to it with a veth pair (a +# peer that is not the machine), and sends from real uids: a pool uid is +# refused on loopback, at its own LAN address, and at the peer (TCP at once, +# UDP with EPERM), the uids on either side of the pool and root are not +# touched, an allowlist chain opens one port of the peer to one uid, the +# machine's own addresses stay shut even on an allowlist, and loading the +# file again takes the allowlist away. The # runner's nft is the version the image carries or newer; the kernel side is # the image's own business and is proven on the machine (exthost.platform). diff --git a/forgetest/forgetest/suite/exthost.py b/forgetest/forgetest/suite/exthost.py index f594554..0a33c12 100644 --- a/forgetest/forgetest/suite/exthost.py +++ b/forgetest/forgetest/suite/exthost.py @@ -205,8 +205,8 @@ def _pids_of(comm): def _pool_counters(): - """The packet counts of chain pool's two refusals: [tcp reset, drop].""" - p = subprocess.run([NFT, "-j", "list", "chain", "inet", "ffx", "pool"], capture_output=True, text=True) + """The packet counts of chain refuse's two rules: [tcp reset, drop].""" + p = subprocess.run([NFT, "-j", "list", "chain", "inet", "ffx", "refuse"], capture_output=True, text=True) if p.returncode != 0: return None return [e["counter"]["packets"] for c in json.loads(p.stdout)["nftables"] if "rule" in c @@ -366,6 +366,9 @@ def platform(ctx): ev["rules"] = p.stdout ctx.check("meta skuid %d-%d jump pool" % (POOL_FIRST, uid) in p.stdout and "hook output" in p.stdout and "policy accept" in p.stdout, "table inet ffx is not the image's: %s", p.stdout[:400]) + pool = p.stdout[p.stdout.find("chain pool"):].split("}")[0] + ctx.check(0 <= pool.find('oifname "lo" jump refuse') < pool.find("vmap @allow"), + "the machine itself is not refused ahead of the allow map: %s", pool) lan = lan_ip() ctx.check(lan, "the machine has no LAN address to aim at") tcp = [[4, "127.0.0.1", 443, False], [4, "127.0.0.1", 80, False], [6, "::1", 443, False], @@ -398,6 +401,29 @@ def platform(ctx): ctx.check(c0 is not None and c1 is not None and len(c1) == 2 and c1[0] - c0[0] >= 2 * len(tcp) and c1[1] - c0[1] >= 2 * len(udp), "the rules did not count the attempts: %s -> %s", c0, c1) + # The machine itself is not a destination, whatever the allow map says: an + # allow chain that names forgectrl on loopback and on the LAN address + # opens neither. + chain = "u%d" % uid + rules = ["add chain inet ffx %s" % chain, + "add rule inet ffx %s ip daddr { 127.0.0.1, %s } tcp dport 443 accept" % (chain, lan), + "add element inet ffx allow { %d : jump %s }" % (uid, chain), ""] + opened = subprocess.run([NFT, "-f", "-"], capture_output=True, text=True, input=os.linesep.join(rules)) + try: + ctx.check(opened.returncode == 0, "could not add an allow chain: %s", opened.stderr.strip()[:200]) + rc, lines, err = _probe("-", uid, "net", json.dumps([[4, "127.0.0.1", 443, False], [4, lan, 443, False]])) + res = (lines[-1] if lines else {}).get("results", []) + ev["allowlisted_self"] = res + ctx.log("uid %d with the machine's own addresses on its allowlist: %s", uid, res) + ctx.check(len(res) == 2 and all(r[1][0] == "refused" for r in res), + "an allowlist opened the machine itself to a pool uid: %s %s", res, err) + finally: + subprocess.run([NFT, "delete", "element", "inet", "ffx", "allow", "{ %d }" % uid], capture_output=True) + subprocess.run([NFT, "flush", "chain", "inet", "ffx", chain], capture_output=True) + subprocess.run([NFT, "delete", "chain", "inet", "ffx", chain], capture_output=True) + left = subprocess.run([NFT, "list", "chain", "inet", "ffx", chain], capture_output=True) + ctx.check(left.returncode != 0, "the test's allow chain stayed behind") + # 6. landlock and seccomp, on a root process so that neither the uid nor the rules explain the refusal rc, lines, err = _probe("-", 0, "landlock", json.dumps([4, "127.0.0.1", 443, False])) ll = lines[-1] if lines else {} diff --git a/meta-forgefirm/recipes-forgefirm/forgefirm-sandbox/files/ffx.nft b/meta-forgefirm/recipes-forgefirm/forgefirm-sandbox/files/ffx.nft index 0ee25ca..ecda728 100644 --- a/meta-forgefirm/recipes-forgefirm/forgefirm-sandbox/files/ffx.nft +++ b/meta-forgefirm/recipes-forgefirm/forgefirm-sandbox/files/ffx.nft @@ -22,6 +22,12 @@ # chain, and removes both when the package stops; a chain that accepts # nothing returns here and the packet is refused. Loading this file again # replaces the whole table, allowlists included: it fails closed. +# +# The machine itself is not a destination, whatever the allow map says. +# Everything a host sends to one of its own addresses, the LAN one included, +# leaves through lo, so that is refused before the map is looked at. An +# allowlist names addresses, and the machine's own address can change under +# it (a new DHCP lease); this rule does not depend on knowing it. table inet ffx delete table inet ffx @@ -37,7 +43,12 @@ table inet ffx { } chain pool { + oifname "lo" jump refuse meta skuid vmap @allow + jump refuse + } + + chain refuse { meta l4proto tcp counter reject with tcp reset counter drop } diff --git a/scripts/sandbox-rules-test.py b/scripts/sandbox-rules-test.py index d949b12..5a3d964 100644 --- a/scripts/sandbox-rules-test.py +++ b/scripts/sandbox-rules-test.py @@ -6,13 +6,18 @@ """Host test of the extension sandbox's deny rules. -Loads meta-forgefirm's ffx.nft into a network namespace of its own and sends -real traffic at it from real uids: an extension account is refused on -loopback, IPv4 and IPv6, TCP at once and UDP with EPERM; the uids on either -side of the pool, and root, are not touched; an allowlist chain lets one uid -reach one destination and nothing else; and loading the file again takes the -allowlist away. Needs root, nft, and a kernel with nf_tables; exits 77 when -one is missing, 0 on a pass, 1 on a failure. +Loads meta-forgefirm's ffx.nft into a network namespace of its own, joins a +second namespace to it with a veth pair (a peer that is not this machine), +and sends real traffic from real uids. An extension account is refused +everywhere: on loopback, IPv4 and IPv6, at its own LAN address, and at the +peer, TCP at once and UDP with EPERM. The uids on either side of the pool, +and root, are not touched. An allowlist chain lets one uid reach one port of +the peer and nothing else. The machine itself is never a destination: with +loopback and the machine's own LAN address on its allowlist, a pool uid is +still refused at both. And loading the file again takes the allowlist away. + +Needs root, nft, ip, nsenter, and a kernel with nf_tables; exits 77 when one +is missing, 0 on a pass, 1 on a failure. sudo python3 scripts/sandbox-rules-test.py [--nft /path/to/nft] [--rules FILE] """ @@ -31,8 +36,27 @@ import time HERE = os.path.dirname(os.path.abspath(__file__)) RULES = os.path.join(HERE, "..", "meta-forgefirm", "recipes-forgefirm", "forgefirm-sandbox", "files", "ffx.nft") POOL_FIRST, POOL_LAST = 800, 831 +HERE_ADDR, PEER_ADDR = "10.99.0.1", "10.99.0.2" SKIP = 77 +# The peer: in its own network namespace, it waits to be given its interface, +# then listens on two TCP ports and one UDP port and says which. +PEER = r''' +import socket, sys +print("pid", flush=True) +sys.stdin.readline() +tcp = [] +for _ in range(2): + s = socket.socket() + s.bind(("%s", 0)) + s.listen(128) + tcp.append(s) +u = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) +u.bind(("%s", 0)) +print(tcp[0].getsockname()[1], tcp[1].getsockname()[1], u.getsockname()[1], flush=True) +sys.stdin.readline() +''' % (PEER_ADDR, PEER_ADDR) + failures = [] @@ -96,6 +120,27 @@ def attempt(uid, family, addr, port, udp=False): return word, time.time() - t0 +def join_peer(): + """Start the peer in its own namespace and join it with a veth pair. + Returns (process, tcp port, second tcp port, udp port).""" + peer = subprocess.Popen(["unshare", "-n", sys.executable, "-u", "-c", PEER], stdin=subprocess.PIPE, + stdout=subprocess.PIPE, text=True) + peer.stdout.readline() + ns = ["nsenter", "-t", str(peer.pid), "-n"] + for cmd in (["ip", "link", "add", "ffxt0", "type", "veth", "peer", "name", "ffxt1"], + ["ip", "link", "set", "ffxt1", "netns", str(peer.pid)], + ["ip", "addr", "add", HERE_ADDR + "/24", "dev", "ffxt0"], + ["ip", "link", "set", "ffxt0", "up"], + ns + ["ip", "addr", "add", PEER_ADDR + "/24", "dev", "ffxt1"], + ns + ["ip", "link", "set", "ffxt1", "up"], + ns + ["ip", "link", "set", "lo", "up"]): + subprocess.run(cmd, check=True, capture_output=True) + peer.stdin.write("go\n") + peer.stdin.flush() + ports = [int(x) for x in peer.stdout.readline().split()] + return peer, ports[0], ports[1], ports[2] + + def main(): ap = argparse.ArgumentParser() ap.add_argument("--nft", default=shutil.which("nft") or "nft") @@ -104,10 +149,11 @@ def main(): if os.environ.get("FFX_RULES_NS") != "1": if os.geteuid() != 0: - print("skipped: needs root (it makes a network namespace and changes uid)") + print("skipped: needs root (it makes network namespaces and changes uid)") return SKIP - if not (os.path.isfile(args.nft) or shutil.which(args.nft)) or not shutil.which("unshare"): - print("skipped: needs nft and unshare") + missing = [t for t in ("unshare", "nsenter", "ip") if not shutil.which(t)] + if missing or not (os.path.isfile(args.nft) or shutil.which(args.nft)): + print("skipped: needs nft, unshare, nsenter, and ip (missing: %s)" % (", ".join(missing) or "nft")) return SKIP env = dict(os.environ, FFX_RULES_NS="1") os.execvpe("unshare", ["unshare", "-n", sys.executable, os.path.abspath(__file__), @@ -124,6 +170,11 @@ def main(): if probe.returncode != 0: print("skipped: this kernel has no nf_tables (%s)" % probe.stderr.strip()) return SKIP + peer, pport, pport2, pudp = join_peer() + for _ in range(50): # the veth pair's carrier + if attempt(0, socket.AF_INET, PEER_ADDR, pport)[0] == "ok": + break + time.sleep(0.1) print("the file loads, twice") for n in (1, 2): @@ -137,33 +188,41 @@ def main(): out = chains.get("output", {}) check(out.get("hook") == "output" and out.get("policy") == "accept" and out.get("type") == "filter", "chain output is a filter on the output hook with policy accept: %s", out) - check("pool" in chains and maps == ["allow"], "chain pool and map allow exist: %s %s", sorted(chains), maps) - rules_text = nft("list", "table", "inet", "ffx").stdout - check("meta skuid %d-%d jump pool" % (POOL_FIRST, POOL_LAST) in rules_text, - "the jump names the pool's range, %d-%d", POOL_FIRST, POOL_LAST) + check({"pool", "refuse"} <= set(chains) and maps == ["allow"], "chains pool and refuse and map allow exist: %s %s", + sorted(chains), maps) + text = nft("list", "table", "inet", "ffx").stdout + check("meta skuid %d-%d jump pool" % (POOL_FIRST, POOL_LAST) in text, "the jump names the pool's range, %d-%d", + POOL_FIRST, POOL_LAST) + pool = text[text.find("chain pool"):].split("}")[0] + check(0 <= pool.find('oifname "lo" jump refuse') < pool.find("vmap @allow"), + "the machine itself is refused ahead of the allow map") l4, port4 = tcp_listener(socket.AF_INET, "127.0.0.1") - l4b, port4b = tcp_listener(socket.AF_INET, "127.0.0.1") l6, port6 = tcp_listener(socket.AF_INET6, "::1") + lself, portself = tcp_listener(socket.AF_INET, HERE_ADDR) u4 = socket.socket(socket.AF_INET, socket.SOCK_DGRAM) u4.bind(("127.0.0.1", 0)) uport = u4.getsockname()[1] + everywhere = ((socket.AF_INET, "127.0.0.1", port4, "loopback"), (socket.AF_INET6, "::1", port6, "IPv6 loopback"), + (socket.AF_INET, HERE_ADDR, portself, "its own LAN address"), (socket.AF_INET, PEER_ADDR, pport, "the peer")) print("outside the pool nothing is touched") for uid in (0, POOL_FIRST - 1, POOL_LAST + 1): - for fam, addr, port in ((socket.AF_INET, "127.0.0.1", port4), (socket.AF_INET6, "::1", port6)): + for fam, addr, port, what in everywhere: word, _ = attempt(uid, fam, addr, port) - check(word == "ok", "uid %d TCP to %s -> %s", uid, addr, word) - word, _ = attempt(uid, socket.AF_INET, "127.0.0.1", uport, udp=True) - check(word == "ok", "uid %d UDP to 127.0.0.1 -> %s", uid, word) + check(word == "ok", "uid %d TCP to %s -> %s", uid, what, word) + for addr, port in (("127.0.0.1", uport), (PEER_ADDR, pudp)): + word, _ = attempt(uid, socket.AF_INET, addr, port, udp=True) + check(word == "ok", "uid %d UDP to %s -> %s", uid, addr, word) - print("a pool uid is refused, and at once") + print("a pool uid is refused everywhere, and at once") for uid in (POOL_FIRST, (POOL_FIRST + POOL_LAST) // 2, POOL_LAST): - for fam, addr, port in ((socket.AF_INET, "127.0.0.1", port4), (socket.AF_INET6, "::1", port6)): + for fam, addr, port, what in everywhere: word, took = attempt(uid, fam, addr, port) - check(word == "refused" and took < 1.5, "uid %d TCP to %s -> %s in %.2f s", uid, addr, word, took) - word, _ = attempt(uid, socket.AF_INET, "127.0.0.1", uport, udp=True) - check(word == "eperm", "uid %d UDP to 127.0.0.1 -> %s", uid, word) + check(word == "refused" and took < 1.5, "uid %d TCP to %s -> %s in %.2f s", uid, what, word, took) + for addr, port in (("127.0.0.1", uport), (PEER_ADDR, pudp)): + word, _ = attempt(uid, socket.AF_INET, addr, port, udp=True) + check(word == "eperm", "uid %d UDP to %s -> %s", uid, addr, word) u4.settimeout(0.3) got = 0 try: @@ -173,31 +232,46 @@ def main(): except socket.timeout: pass check(got == 3, "the UDP receiver heard the three senders outside the pool and nobody else: %d", got) - counted = [e["counter"]["packets"] for c in json.loads(nft("-j", "list", "chain", "inet", "ffx", "pool").stdout)["nftables"] + counted = [e["counter"]["packets"] for c in json.loads(nft("-j", "list", "chain", "inet", "ffx", "refuse").stdout)["nftables"] if "rule" in c for e in c["rule"]["expr"] if "counter" in e] check(len(counted) == 2 and all(n > 0 for n in counted), "both refusals counted packets: %s", counted) print("an allowlist opens one destination to one uid") - nft("add", "chain", "inet", "ffx", "u%d" % POOL_FIRST) - nft("add", "rule", "inet", "ffx", "u%d" % POOL_FIRST, "ip", "daddr", "127.0.0.1", "tcp", "dport", str(port4), "accept") - nft("add", "element", "inet", "ffx", "allow", "{ %d : jump u%d }" % (POOL_FIRST, POOL_FIRST)) - for uid, fam, addr, port, want, why in ( - (POOL_FIRST, socket.AF_INET, "127.0.0.1", port4, "ok", "the listed destination"), - (POOL_FIRST, socket.AF_INET, "127.0.0.1", port4b, "refused", "another port"), - (POOL_FIRST, socket.AF_INET6, "::1", port6, "refused", "another address"), - (POOL_FIRST + 1, socket.AF_INET, "127.0.0.1", port4, "refused", "another uid")): - word, _ = attempt(uid, fam, addr, port) + chain = "u%d" % POOL_FIRST + nft("add", "chain", "inet", "ffx", chain) + nft("add", "rule", "inet", "ffx", chain, "ip", "daddr", PEER_ADDR, "tcp", "dport", str(pport), "accept") + nft("add", "element", "inet", "ffx", "allow", "{ %d : jump %s }" % (POOL_FIRST, chain)) + for uid, addr, port, want, why in ( + (POOL_FIRST, PEER_ADDR, pport, "ok", "the listed destination"), + (POOL_FIRST, PEER_ADDR, pport2, "refused", "another port of the peer"), + (POOL_FIRST, "127.0.0.1", port4, "refused", "loopback"), + (POOL_FIRST + 1, PEER_ADDR, pport, "refused", "another uid")): + word, _ = attempt(uid, socket.AF_INET, addr, port) check(word == want, "uid %d to %s port %d (%s) -> %s", uid, addr, port, why, word) + word, _ = attempt(POOL_FIRST, socket.AF_INET, PEER_ADDR, pudp, udp=True) + check(word == "eperm", "uid %d UDP to the peer, which its list does not name -> %s", POOL_FIRST, word) + + print("the machine itself is never a destination") + nft("add", "rule", "inet", "ffx", chain, "ip", "daddr", "127.0.0.1", "tcp", "dport", str(port4), "accept") + nft("add", "rule", "inet", "ffx", chain, "ip", "daddr", HERE_ADDR, "tcp", "dport", str(portself), "accept") + nft("add", "rule", "inet", "ffx", chain, "ip6", "daddr", "::1", "tcp", "dport", str(port6), "accept") + for fam, addr, port, what in everywhere[:3]: + word, _ = attempt(POOL_FIRST, fam, addr, port) + check(word == "refused", "uid %d to %s, which is on its allowlist -> %s", POOL_FIRST, what, word) + word, _ = attempt(POOL_FIRST, socket.AF_INET, PEER_ADDR, pport) + check(word == "ok", "and the peer still answers it -> %s", word) print("loading the file again fails closed") nft("-f", args.rules) - word, _ = attempt(POOL_FIRST, socket.AF_INET, "127.0.0.1", port4) + word, _ = attempt(POOL_FIRST, socket.AF_INET, PEER_ADDR, pport) check(word == "refused", "uid %d to its old destination -> %s", POOL_FIRST, word) - gone = nft("list", "chain", "inet", "ffx", "u%d" % POOL_FIRST, check_rc=False) + gone = nft("list", "chain", "inet", "ffx", chain, check_rc=False) check(gone.returncode != 0, "the allowlist chain is gone") - for s in (l4, l4b, l6, u4): + for s in (l4, l6, lself, u4): s.close() + peer.stdin.close() + peer.wait(timeout=5) print("%s: %d failure%s" % ("FAIL" if failures else "PASS", len(failures), "" if len(failures) == 1 else "s")) return 1 if failures else 0