Catalog: setup.record-export leaves

setup.record-export leaves for forgectrl's tests/host_guard.sh (84779c8
on d21e200): the record as JSON with the token (schema 1, the sheet id
GET /wiz reports, no password hash, a plain read not an attachment), the
download an attachment of the record named with that id, the printable
page text/html with no script, the sheet id, "setup record" and an <h3>
per completed wizard, both routes 403 without a token or a login, and
the sanitized export carrying system/setup.json with the same id and
wizards, no $6$, and no panel token anywhere in the archive. The catalog
goes from 97 to 96.

One HOST_TESTED row: forgectrl src/recordhtml.*, the one path no other
catalog test covers, to tests/recordhtml_test.c (no script, the sheet
id, the step headings, the escaping), which forgectrl's build.yml runs
at the pinned revision. src/setup.* and src/wiz.* stay on the setup
family, src/logs.* on exthost.service, forgectrl.lease and the two log
tests, and src/main.c, src/ui/index.html, src/ui/panel.js and
src/ui/wizard.html on the tests that already name them.
tests/host_guard.sh is not named: the pinned workflow does not run it.

tests/test_setup_suite.py drops the id, the record-export hardware
assertion, and the export lines of the lifecycle covers test (the
covers set and its two assertions); the mirror's assertions stay.

test_setup_suite.py passes (31 tests). The coverage lint on the tree
manifest finds no uncovered path and no host-tested row problem across
the 96 tests; its one finding is setup.check-lens's empty src/lensfind.*
entry. pyflakes is clean on setup.py and test_setup_suite.py. The full
forgetest suite was not run.

Fingerprints: the deletion moves suite/setup.py's shared text, so 33
tests move: the 8 left in setup.py and the 25 in the modules that read
it (setup_dark, setup_burns, bedsize, exthost, extcall, extcat,
extsender, extwizard). The pending pin bump from c274f19 already owes 31
of them; setup.machine-name and exthost.platform are the two it adds, as
before. releases/ is unchanged.

forgetest/forgetest/manifest.py

@@ -137,6 +137,9 @@ HOST_TESTED = [
     # The built-in extensions' table: the cloud entry, its refusals while it
     # is off, the sweep when it goes off, the list.
     ("forgectrl", "src/builtin.*", "forgectrl:tests/builtin_test.c", "forgectrl:.github/workflows/build.yml"),
+    # The setup record as a printable page: no script, the sheet id, a
+    # heading for each completed step, everything escaped.
+    ("forgectrl", "src/recordhtml.*", "forgectrl:tests/recordhtml_test.c", "forgectrl:.github/workflows/build.yml"),
 ]
 
 # This repository's root, when the package runs from a checkout of it; a

forgetest/forgetest/suite/setup.py

@@ -1175,100 +1175,6 @@ def cert_page(ctx):
 
 # ------------------------------------------------------- the lifecycle
 
-@test("setup.record-export", title="The record: JSON, the printable page, and the log bundle",
-      subsystem="setup", kind="auto", est_min=2,
-      covers=[("forgectrl", "src/setup.*"), ("forgectrl", "src/recordhtml.*"), ("forgectrl", "src/wiz.*"),
-              ("forgectrl", "src/logs.*"), ("forgectrl", "src/main.c"), ("forgectrl", "src/ui/index.html"),
-              ("forgectrl", "src/ui/panel.js"), ("forgectrl", "src/ui/wizard.html")],
-      requires=["forgectrl.auth", "logs.tree-tail-export"],
-      description="GET /wiz/record with the token is the record (schema 1, the sheet id, the "
-                  "wizards) with no password hash in it; with download=1 it comes as an attachment "
-                  "named after the sheet id; GET /wiz/record.html is a page with no script that "
-                  "carries the sheet id and every completed wizard; without the token and without "
-                  "a login both are refused (403); and the sanitized log export carries "
-                  "system/setup.json, parseable, with the same sheet id and no panel token.")
-def record_export(ctx):
-    import gzip
-    import io
-    import tarfile
-    fc = ctx.forgectrl
-    ev = ctx.evidence
-    base = tls_base()
-    tok = {"X-ForgeFIRM-Token": fc.token}
-    w = wiz(fc)
-    sid = w.get("sheet_id") or ""
-    ev["sheet_id"] = sid
-    ctx.check(sid, "GET /wiz carries no sheet id")
-
-    st, body, hdrs = request(base, "GET", "/wiz/record", headers=tok)
-    rec = decode(body)
-    ctx.log("GET /wiz/record -> %s (%d bytes)", st, len(body or b""))
-    ctx.check(st == 200 and isinstance(rec, dict), "GET /wiz/record -> %s", st)
-    ctx.check(isinstance(rec, dict) and rec.get("schema") == 1 and rec.get("sheet_id") == sid,
-              "the record reads %s", rec if not isinstance(rec, dict)
-              else {k: rec.get(k) for k in ("schema", "sheet_id")})
-    done = sorted((rec.get("wizards") or {}).keys()) if isinstance(rec, dict) else []
-    ev["wizards_in_record"] = done
-    ctx.check(b"$6$" not in (body or b""), "the record carries a password hash")
-    ctx.check("attachment" not in (hdrs.get("content-disposition") or ""), "a plain read came as an attachment")
-
-    st, body, hdrs = request(base, "GET", "/wiz/record?download=1", headers=tok)
-    cd = hdrs.get("content-disposition") or ""
-    ev["download_disposition"] = cd
-    ctx.check(st == 200 and "attachment" in cd and sid in cd,
-              "the download is not an attachment named after the sheet id: %s %r", st, cd)
-    ctx.check(decode(body) == rec, "the download differs from the record")
-
-    st, body, hdrs = request(base, "GET", "/wiz/record.html", headers=tok)
-    page = (body or b"").decode("utf-8", "replace")
-    ev["page_bytes"] = len(body or b"")
-    ctx.log("GET /wiz/record.html -> %s (%d bytes)", st, len(body or b""))
-    ctx.check(st == 200 and "text/html" in (hdrs.get("content-type") or ""), "GET /wiz/record.html -> %s %s",
-              st, hdrs.get("content-type"))
-    ctx.check("<script" not in page.lower(), "the printable page carries a script")
-    ctx.check(sid in page and "setup record" in page, "the page lacks the sheet id or its title")
-    titles = {z["id"]: z["title"] for z in w.get("wizards") or []}
-    missing = [d for d in done if titles.get(d) and ("<h3>" + titles[d]) not in page]
-    ev["missing_on_page"] = missing
-    ctx.check(not missing, "completed wizards missing from the page: %s", missing)
-
-    for path in ("/wiz/record", "/wiz/record.html"):
-        st, body, hdrs = request(base, "GET", path)
-        ctx.check(st == 403, "GET %s with no token and no login -> %s, expected 403", path, st)
-
-    ctx.check(fc.wait_idle(timeout=30, abort=ctx.aborted), "machine not idle: the export is refused")
-    slow = hw.Forgectrl(token=fc.token, timeout=300.0)
-    t0 = time.time()
-    st, data = slow.post("/logs/export", raw=True)
-    ev["export_s"] = round(time.time() - t0, 1)
-    ctx.log("POST /logs/export -> %s (%d bytes, %.1f s)", st, len(data) if data else 0, ev["export_s"])
-    ctx.check(st == 200, "POST /logs/export -> %s", st)
-    member = None
-    try:
-        tf = tarfile.open(fileobj=io.BytesIO(gzip.decompress(data)))
-        for m in tf.getmembers():
-            if m.isfile() and m.name.endswith("system/setup.json"):
-                member = tf.extractfile(m).read()
-    except (OSError, tarfile.TarError, EOFError) as e:
-        ctx.fail("export is not a readable tar.gz: %s", e)
-    ctx.check(member is not None, "the bundle lacks system/setup.json")
-    if member is not None:
-        try:
-            inside = json.loads(member.decode("utf-8"))
-        except ValueError as e:
-            inside = None
-            ctx.check(False, "the bundled record is not JSON: %s", e)
-        ev["bundle_record_bytes"] = len(member)
-        ctx.check(isinstance(inside, dict) and inside.get("sheet_id") == sid,
-                  "the bundled record is not this machine's: %s",
-                  inside.get("sheet_id") if isinstance(inside, dict) else inside)
-        ctx.check(isinstance(inside, dict) and sorted((inside.get("wizards") or {}).keys()) == done,
-                  "the bundled record's wizards differ from the record's")
-        ctx.check(fc.token.encode() not in member, "the bundled record carries the panel token")
-        ctx.check(b"$6$" not in member, "the bundled record carries a password hash")
-    ctx.log("record %d wizards; page %d bytes; bundle carries the record", len(done), ev["page_bytes"])
-
-
 @test("setup.mirror", title="One browser drives a check; a second one follows",
       subsystem="setup", kind="auto", est_min=2, hardware="takeover",
       covers=[("forgectrl", "src/wizdark.*"), ("forgectrl", "src/wizcalc.*"), ("forgectrl", "src/wiz.*"),

forgetest/tests/test_setup_suite.py

@@ -19,7 +19,7 @@ from forgetest.suite import setup
 
 IDS = ("setup.account-login", "setup.https-only-writes",
        "setup.ssh-until-reboot", "setup.machine-name", "setup.first-run-flow",
-       "setup.first-run-page", "setup.record-export",
+       "setup.first-run-page",
        "setup.mirror")
 OPERATOR = ("setup.first-run-flow", "setup.first-run-page")
 
@@ -44,13 +44,9 @@ class RegistrationTests(unittest.TestCase):
     def test_the_record_swapping_tests_are_takeovers(self):
         for tid in ("setup.first-run-flow", "setup.first-run-page", "setup.mirror"):
             self.assertEqual(self.reg[tid].hardware, "takeover", tid)
-        self.assertEqual(self.reg["setup.record-export"].hardware, "api")
 
     def test_the_lifecycle_tests_cover_their_files(self):
-        export = set(self.reg["setup.record-export"].covers)
         mirror = set(self.reg["setup.mirror"].covers)
-        self.assertIn(("forgectrl", "src/recordhtml.*"), export)
-        self.assertIn(("forgectrl", "src/logs.*"), export)
         self.assertIn(("forgectrl", "src/wizdark.*"), mirror)
         self.assertIn("setup.check-sensors", self.reg["setup.mirror"].requires)