mirror of
https://github.com/openglow-org/forgefirm.git
synced 2026-09-27 08:41:13 -07:00
exthost.hold-pause-tier: a required hold from the grant to the verdict and out
The reference package takes the hold capability (its service ends at once when the test leaves a file in its data directory: a package that cannot speak for itself). The install is refused without the operator's grant; a granted hold is advisory until `forgeext hold <id> required` names the package under required-holds. With extensions on, the required hold stands in the host's words until the service has run healthy, then the host keeps its file fresh and clear and GET /cool/status reads OK. The test then makes the service end at every start: verdict EXT, fire_ok false, hold true, the reason naming the package, and never clear in 24 looks over the crash loop. Safe mode ends it within four seconds and leaving safe mode brings it back; marked advisory it is dropped; marked required again and the host suspended, the reason becomes that the host is not answering, and resumed it is the package's again; ext_enabled=0 ends it. The verdict is read at idle: nothing moves and nothing fires. Everything is put back as exthost.service puts it back, the required holds are empty, and the verdict is OK at the end. Proven. The unit suite: 451 tests, 0 undefined names. On the bench reference, image 20260921121235 with the cross-built forgectrl and extension host mounted over the image's: exthost.hold-pause-tier PASS (held 1.0 s after the service was killed; safe mode released it in 0.8 s, extensions off in 0.8 s); against the image's own forgectrl, whose engine knows no holds, it FAILS with the verdict OK, as it should. Acceptance. The test is new: auto, takeover. It covers forgeext's src/holdkeep.*, src/run.*, src/install.*, src/state.*, src/main.c, and forgectrl's src/holds.* and src/cool.*, and requires exthost.service.
This commit is contained in:
ScottW514
@@ -486,6 +486,8 @@ REF_SERVICE = r'''
|
||||
import errno, json, os, socket, sys, time
|
||||
lan = sys.argv[1]
|
||||
data, pkg = os.environ["FFX_DATA"], os.environ["FFX_PKG"]
|
||||
if os.path.exists(os.path.join(data, "stop")):
|
||||
sys.exit(3)
|
||||
|
||||
|
||||
def word(e):
|
||||
@@ -617,14 +619,14 @@ def _tree(root):
|
||||
return sorted(out)
|
||||
|
||||
|
||||
def _pack_reference(work, lan):
|
||||
def _pack_reference(work, lan, more_caps=()):
|
||||
"""The reference package, signed with a key made here: (archive, public key)."""
|
||||
import io
|
||||
import tarfile
|
||||
manifest = {"manifest": 1, "id": REF_ID, "name": "forgetest reference", "version": "1.0.0",
|
||||
"author": "forgetest", "license": "MIT", "api": "0.1", "runtime": "python",
|
||||
"service": {"exec": "bin/reference.py", "args": [lan or "-"]},
|
||||
"capabilities": ["net.outbound:%s:443" % REF_DEST, "storage:1"]}
|
||||
"capabilities": ["net.outbound:%s:443" % REF_DEST, "storage:1"] + list(more_caps)}
|
||||
payload = os.path.join(work, "payload.tar.gz")
|
||||
with tarfile.open(payload, "w:gz") as t:
|
||||
for name, text, mode in (("manifest.json", json.dumps(manifest), 0o644),
|
||||
@@ -977,3 +979,169 @@ def armed_freeze(ctx):
|
||||
_put_back(ctx, fc, work, prior, etag, raw, dir_mode)
|
||||
_as_found(ctx, fc, prior, raw, dir_mode, found_tree)
|
||||
|
||||
|
||||
# ------------------------------------------------------------- the hold
|
||||
|
||||
HOLDS_DIR = "/run/forgefirm/holds"
|
||||
REQUIRED_HOLDS = EXT_ROOT + "/required-holds"
|
||||
|
||||
|
||||
@test("exthost.hold-pause-tier", title="A package's hold withholds fire, and the operator's exits end it",
|
||||
subsystem="exthost", kind="auto", hardware="takeover", est_min=7,
|
||||
covers=[("forgeext", "src/holdkeep.*"), ("forgeext", "src/run.*"), ("forgeext", "src/install.*"),
|
||||
("forgeext", "src/state.*"), ("forgeext", "src/main.c"), ("forgectrl", "src/holds.*"),
|
||||
("forgectrl", "src/cool.*")],
|
||||
requires=["exthost.service"],
|
||||
description="The reference package is installed with the hold grant and its hold is marked required "
|
||||
"(the package is then named under required-holds for forgectrl). Until its service has "
|
||||
"run healthy (60 s) the required hold stands in the host's words; from then on the host "
|
||||
"keeps its hold file fresh and clear and the engine's verdict is OK. The test then "
|
||||
"makes the service end at every start: the host raises the hold in its own words and GET "
|
||||
"/cool/status reads verdict EXT, fire_ok false, hold true, with the reason naming the "
|
||||
"package. Safe mode ends the hold within two ticks and leaving it brings the hold back; "
|
||||
"marked advisory the hold is dropped and the verdict is OK; marked required again, with "
|
||||
"the host suspended the file goes stale and the reason becomes that the host is not "
|
||||
"answering, and resumed it is the package's again; ext_enabled=0 ends it. Nothing moves "
|
||||
"and nothing fires: the verdict is read at idle. Everything is put back as "
|
||||
"exthost.service puts it back, and the required holds are empty at the end.")
|
||||
def hold_pause_tier(ctx):
|
||||
import tempfile
|
||||
fc = ctx.forgectrl
|
||||
ev = ctx.evidence
|
||||
ctx.check(len(_host_pids()) == 1, "the extension host is not one running process: %s", _host_pids())
|
||||
ctx.check(fc.wait_idle(timeout=30, abort=ctx.aborted), "machine not idle: settings are locked")
|
||||
prior = fc.settings().get("ext_enabled") or ""
|
||||
raw = read_file(record_path())
|
||||
ctx.check(raw, "no setup record at %s", record_path())
|
||||
ctx.check(not os.path.exists(SAFE_FILE), "%s exists: the machine is in safe mode", SAFE_FILE)
|
||||
found_tree = _tree(EXT_ROOT)
|
||||
dir_mode = os.stat(os.path.dirname(EXT_ROOT)).st_mode & 0o7777
|
||||
st, body, hdrs = request(fc.base, "GET", "/advisories/extensions", headers={"Host": fc.host_header()})
|
||||
etag = hdrs.get("etag")
|
||||
ctx.check(st == 200 and etag, "GET /advisories/extensions -> %s", st)
|
||||
|
||||
def cool():
|
||||
st_, c = fc.get("/cool/status")
|
||||
return c if st_ == 200 and isinstance(c, dict) else {}
|
||||
|
||||
def verdict_is(name, seconds, reason=None):
|
||||
def ok():
|
||||
c = cool()
|
||||
return c if c.get("verdict") == name and (reason is None or reason in (c.get("reason") or "")) else None
|
||||
return _until(ctx, ok, seconds, poll=0.25)
|
||||
|
||||
def hold_file():
|
||||
try:
|
||||
h = json.loads(_read("%s/%s.json" % (HOLDS_DIR, REF_ID)))
|
||||
except ValueError:
|
||||
return None
|
||||
h["age"] = time.monotonic() - h["ts_mono"]
|
||||
return h
|
||||
|
||||
start = cool()
|
||||
ev["verdict_at_start"] = {k: start.get(k) for k in ("verdict", "fire_ok", "hold", "reason")}
|
||||
ctx.check(start.get("verdict") == "OK", "the engine's own verdict %r stands above a hold: the test needs OK",
|
||||
start.get("verdict"))
|
||||
work = tempfile.mkdtemp(prefix="forgetest-ffx.")
|
||||
stop_path = os.path.join(EXT_ROOT, "data", REF_ID, "stop")
|
||||
why_not_running = REF_ID + ": the extension is not running"
|
||||
try:
|
||||
archive, pub = _pack_reference(work, lan_ip(), more_caps=["hold"])
|
||||
with open(pub, "rb") as f:
|
||||
write_file(os.path.join(EXT_ROOT, "keys", REF_KEY + ".pub"), f.read())
|
||||
os.chmod(os.path.join(EXT_ROOT, "keys", REF_KEY + ".pub"), 0o644)
|
||||
r = _forgeext("install", archive, "--consent-community")
|
||||
ev["install_without_the_grant"] = r.get("error")
|
||||
ctx.check(r.get("ok") is False and "hold" in (r.get("error") or ""), "a hold was installed without the operator's grant: %s", r)
|
||||
r = _forgeext("install", archive, "--consent-community", "--grant", "hold")
|
||||
ctx.check(r.get("ok") is True, "the install -> %s", r.get("error"))
|
||||
ctx.check(not os.listdir(REQUIRED_HOLDS), "a granted hold is required before the operator marks it: %s",
|
||||
os.listdir(REQUIRED_HOLDS))
|
||||
r = _forgeext("hold", REF_ID, "required")
|
||||
ctx.check(r.get("ok") is True and os.listdir(REQUIRED_HOLDS) == [REF_ID], "marked required -> %s, named %s",
|
||||
r.get("error"), os.listdir(REQUIRED_HOLDS))
|
||||
|
||||
st, reply = fc.post("/settings", data={"ext_enabled": "1", "advisory": etag, "phrase": SAFETY_PHRASE})
|
||||
ctx.check(st == 200, "ext_enabled=1 over the advisory -> %s %r", st, reply)
|
||||
beat_path = os.path.join(EXT_ROOT, "data", REF_ID, "beat")
|
||||
seen = set()
|
||||
ctx.check(_until(ctx, lambda: seen.add(_read(beat_path)) or len(seen) >= 3, 120, poll=0.2),
|
||||
"the reference service's heartbeat does not advance: %s", _svc(REF_ID) or _host_status())
|
||||
h = hold_file()
|
||||
ev["running"] = {"hold_file": h, "verdict": cool().get("verdict"), "status_hold": _svc(REF_ID).get("hold")}
|
||||
ctx.check(h and h["required"] is True and h["age"] < 1.5, "a running package's required hold is not fresh: %s", h)
|
||||
# a required hold stands until its package has run healthy (60 s): one that ends at every
|
||||
# start reads as running for a moment each time, and must not flicker clear
|
||||
ctx.check(cool().get("verdict") == "EXT" and "only just started" in (cool().get("reason") or ""),
|
||||
"a required hold is clear before its package has run healthy: %s", cool())
|
||||
ctx.check(verdict_is("OK", 100), "with the package running healthy the verdict is %s (%s)", cool().get("verdict"),
|
||||
cool().get("reason"))
|
||||
h = hold_file()
|
||||
ctx.check(h and h["raised"] is False and h["age"] < 1.5, "healthy: the hold file is not fresh and clear: %s", h)
|
||||
ctx.check(_svc(REF_ID).get("hold") == "required", "the status file does not say the hold is required: %s", _svc(REF_ID))
|
||||
|
||||
# the package can no longer speak for itself
|
||||
_write(stop_path, "")
|
||||
os.kill(_svc(REF_ID)["pid"], signal.SIGKILL)
|
||||
t0 = time.time()
|
||||
c = verdict_is("EXT", 40, why_not_running)
|
||||
ev["held"] = {"after_s": round(time.time() - t0, 1), "cool": {k: (c or {}).get(k) for k in ("verdict", "fire_ok", "hold", "reason")},
|
||||
"hold_file": hold_file()}
|
||||
ctx.log("the service ends at every start: %s", ev["held"])
|
||||
ctx.check(c and c.get("fire_ok") is False and c.get("hold") is True,
|
||||
"the hold does not withhold fire: %s", c or cool())
|
||||
# it reads as running for a moment at every start on its way to quarantine: never clear
|
||||
looks = []
|
||||
for _ in range(24):
|
||||
ctx.sleep(0.5)
|
||||
c = cool()
|
||||
looks.append((c.get("verdict"), c.get("fire_ok")))
|
||||
ev["looks_over_the_crash_loop"] = sorted(set(looks), key=str)
|
||||
ctx.check(all(v == "EXT" and ok is False for v, ok in looks),
|
||||
"the required hold flickered clear while its package kept ending: %s", ev["looks_over_the_crash_loop"])
|
||||
|
||||
# the operator's exits
|
||||
_write(SAFE_FILE, "")
|
||||
t0 = time.time()
|
||||
ctx.check(verdict_is("OK", 4), "safe mode did not end the hold within four seconds: %s", cool())
|
||||
ev["safe_mode_released_s"] = round(time.time() - t0, 1)
|
||||
os.remove(SAFE_FILE)
|
||||
ctx.check(verdict_is("EXT", 30, why_not_running), "out of safe mode the hold did not come back: %s", cool())
|
||||
|
||||
r = _forgeext("hold", REF_ID, "advisory")
|
||||
ctx.check(r.get("ok") is True and not os.listdir(REQUIRED_HOLDS), "marked advisory -> %s, still named %s",
|
||||
r.get("error"), os.listdir(REQUIRED_HOLDS))
|
||||
ctx.check(verdict_is("OK", 6), "an advisory hold of a package that is not running was not dropped: %s", cool())
|
||||
r = _forgeext("hold", REF_ID, "required")
|
||||
ctx.check(r.get("ok") is True and verdict_is("EXT", 6, why_not_running), "marked required again: %s", cool())
|
||||
|
||||
# the host itself gone quiet: the file goes stale, and that is what stands
|
||||
host = _host_pids()[0]
|
||||
os.kill(host, signal.SIGSTOP)
|
||||
try:
|
||||
c = verdict_is("EXT", 8, REF_ID + ": the extension host is not answering")
|
||||
ev["host_suspended"] = {k: (c or cool()).get(k) for k in ("verdict", "fire_ok", "reason")}
|
||||
ctx.check(c and c.get("fire_ok") is False, "with the host suspended the required hold does not stand on its "
|
||||
"stale file: %s", c or cool())
|
||||
finally:
|
||||
os.kill(host, signal.SIGCONT)
|
||||
ctx.check(verdict_is("EXT", 8, why_not_running), "the host resumed: the hold is not the package's again: %s", cool())
|
||||
|
||||
st, reply = fc.post("/settings", data={"ext_enabled": "0"})
|
||||
ctx.check(st == 200, "ext_enabled=0 -> %s %r", st, reply)
|
||||
t0 = time.time()
|
||||
ctx.check(verdict_is("OK", 4), "ext_enabled=0 did not end the hold within four seconds: %s", cool())
|
||||
ev["ext_off_released_s"] = round(time.time() - t0, 1)
|
||||
ctx.log("PASS: held in %s s; safe mode released it in %s s, extensions off in %s s", ev["held"]["after_s"],
|
||||
ev["safe_mode_released_s"], ev["ext_off_released_s"])
|
||||
finally:
|
||||
for pid in _host_pids():
|
||||
try:
|
||||
os.kill(pid, signal.SIGCONT)
|
||||
except OSError:
|
||||
pass
|
||||
_put_back(ctx, fc, work, prior, etag, raw, dir_mode)
|
||||
_as_found(ctx, fc, prior, raw, dir_mode, found_tree)
|
||||
ctx.check(not os.listdir(REQUIRED_HOLDS), "a required hold is still named at the end: %s", os.listdir(REQUIRED_HOLDS))
|
||||
ctx.check(_until(ctx, lambda: cool().get("verdict") == "OK", 6), "the verdict at the end is %s", cool().get("verdict"))
|
||||
|
||||
|
||||
Reference in New Issue
Block a user