The critical tier proven on a rising loop; the board temperatures watched

forgectrl pin 76115fd: the chassis LM75 and the supply sensor ride /status
as temps (degrees and a raw count), the engine ranges them over every run
session into one run-end line, and a critical fault that clears with its
session yields the reason to the standing hold.

Bench: critical_tier_drill.py (a bench tool now, registered as
critical-tier) sets the ceiling, the resume gate and the critical line a
few tenths above the live upstream reading and lets the engine's own
flow-check heater warm the loop through them inside one M8 session;
temp_calibrate.py gains supply-watch, supply-point and supply-fit for the
supply sensor against a thermometer on its heatsink, the fit shown beside
UAPI.md's unverified guess.

Catalog: cooling.gate-off checks the /status temps fields and the run-end
board-temperature line (the unit fake mirrors both, one new failure
case); cooling.critical-tier checks the reason after a faulted session.

Docs: CAMPAIGN-LOG entries for cooling.critical-tier on dev image
20260822154257 and the warm-loop drill (OVERTEMP at 10 s, CRITICAL at
14 s, the fault ending with the session); BRINGUP item 19, the facts bank
(board temperatures at idle), COOLING section 9, the bench README.
This commit is contained in:
ScottW514
2026-08-22 12:56:40 -04:00
parent f51140e528
commit 9fae47cf14
10 changed files with 372 additions and 9 deletions
+13 -3
View File
@@ -126,6 +126,13 @@ TOOLS = [
"args": [_arg("cycles", "int", 3, "cycles per case (flow / no-flow)")],
"desc": "Runs the real check (40 percent / 50 s, cut-profile fans) from a heater-warmed baseline, alternating "
"flow and no-flow; results to the bench data directory. Slow: about 15 minutes per cycle."},
{"id": "critical-tier", "title": "Coolant critical-tier warm-loop drill", "script": "critical_tier_drill.py",
"safety": "dry", "where": "board", "ported": True,
"args": [_arg("max_seconds", "int", 1200, "give up after this long without CRITICAL", flag="--max-seconds")],
"desc": "Sets the ceiling, the resume gate and the critical line a few tenths above the live upstream "
"reading and lets the engine's own flow-check heater warm the loop through them inside one M8 "
"session: OVERTEMP at the ceiling, then CRITICAL (fire blocked, hold, no resume), the fault ending "
"with the session; settings restored and re-read. Results to the bench data directory."},
{"id": "flow-recheck", "title": "Coolant re-check characterization", "script": "flow_recheck_char.py",
"safety": "takeover", "where": "board", "ported": True,
"args": [_arg("heater_pct", "int", 50, "heater duty percent"), _arg("window_s", "int", 30, "re-check window")],
@@ -144,10 +151,13 @@ TOOLS = [
"desc": "Prints elapsed,raw_down,raw_up at the interval; the sampler behind the flow tools."},
{"id": "temp-calibrate", "title": "Coolant temperature spot-check", "script": "temp_calibrate.py",
"safety": "dry", "where": "board", "ported": True,
"args": [_arg("mode", "choice", "watch", "watch / point / fit", ["watch", "point", "fit"]),
"args": [_arg("mode", "choice", "watch", "coolant: watch / point / fit; supply (pic/pwr_temp): "
"supply-watch / supply-point / supply-fit",
["watch", "point", "fit", "supply-watch", "supply-point", "supply-fit"]),
_arg("value", "str", None, "point: the thermometer reading in C; watch: seconds (default 60)")],
"desc": "Pairs a measured temperature with averaged raw readings; fits a per-machine line. Points "
"accumulate in the bench data directory."},
"desc": "Pairs a measured temperature with averaged raw readings; fits a per-machine line. The coolant "
"sensors, or the power supply's (thermometer on its heatsink, against the unverified guess in "
"UAPI.md). Points accumulate in the bench data directory."},
{"id": "fan-test", "title": "Fan/coolant bench", "script": "fan_test.py",
"safety": "dry", "where": "board", "ported": True, "args": [],
"desc": "Snapshots fan PWMs/tachs/temps, drives M8 -> cut fans, M9 -> cooldown -> idle; the tach "
+20 -2
View File
@@ -270,7 +270,9 @@ def _tail_has(fc, needle):
"(OVERTEMP, hold, fire blocked) at the next run start; set to its top the "
"engine must skip the gate (verdict OK), report it in gates_off on /status "
"and /cool/status, say so in the settings reply, and log the run-start line; "
"restored, everything reads as before.")
"restored, everything reads as before. Alongside: /status carries the watched "
"board temperatures (chassis in degrees, supply as a raw count) and every run "
"session ends with them ranged into one log line.")
def gate_off(ctx):
fc = ctx.forgectrl
ev = ctx.evidence
@@ -278,8 +280,17 @@ def gate_off(ctx):
orig = {k: before.get(k, "") for k in GATE_KEYS}
ev["orig"] = orig
ctx.log("original: %s", orig)
up = (fc.status().get("coolant") or {}).get("up_c")
st0 = fc.status()
up = (st0.get("coolant") or {}).get("up_c")
ctx.check(up is not None and up > 8.0, "coolant too cold for the trip leg (up_c %s)", up)
# The board temperatures ride /status, watched and not gated: the
# chassis in degrees, the supply as its raw count.
temps = st0.get("temps") or {}
ev["temps"] = temps
ctx.check(isinstance(temps.get("chassis_c"), (int, float)) and 5.0 <= temps["chassis_c"] <= 80.0,
"/status temps.chassis_c is not a plausible chassis temperature: %s", temps)
ctx.check(isinstance(temps.get("supply_raw"), int) and 0 <= temps["supply_raw"] <= 1023,
"/status temps.supply_raw is not a 10-bit count: %s", temps)
c0 = _cool(fc)
ctx.check(c0.get("verdict") == "OK", "engine is not at OK before the test: %s", c0)
ctx.check(c0.get("gates_off") == [], "a gate is already off: %s", c0.get("gates_off"))
@@ -336,6 +347,10 @@ def gate_off(ctx):
ctx.check(c.get("verdict") == "OK" and c.get("gates_off") == [],
"engine did not return to OK with no gate off after the restore: %s", c)
ctx.log("restored ceiling %s reports state %s", val, state)
# Every run session ends with the board temperatures ranged
# into one line.
ctx.check(_tail_has(fc, "temps this job: chassis "),
"the run-end board-temperature line is missing from the forgectrl log")
finally:
if not restored:
# The engine reads settings at run start only: restoring
@@ -436,6 +451,9 @@ def critical_tier(ctx):
ctx.check(c.get("verdict") == "OVERTEMP",
"after the faulted session the engine reads %s, expected the ceiling's OVERTEMP: %s",
c.get("verdict"), c)
ctx.check("CRITICAL" not in (c.get("reason") or ""),
"the reason still names the ended critical fault under the ceiling's hold: %r",
c.get("reason"))
# Leg 2: the critical line at its top is the gate off; the
# ceiling alone pauses, and gates_off says so.
+12
View File
@@ -229,11 +229,13 @@ class GateOffTests(unittest.TestCase):
cooling.SESSION_END_WAIT_S = 3
self.fc.state["status"] = dict(self.fc.state["status"],
coolant={"down_c": 22.4, "up_c": 22.3, "pump": True, "tec": False},
temps={"chassis_c": 29.0, "supply_raw": 589},
gates_off=[])
self.fc.state["cool"] = {"phase": "idle", "verdict": "OK", "fire_ok": False, "hold": False,
"gates_off": []}
self.fc.state["settings"].update({"cool_temp_max": "", "cool_temp_resume": ""})
self.log_line = True # the engine writes its run-start line
self.temps_line = True # the engine writes its run-end temperature line
self.report_off = True # the engine reports the off gate
self.trips = True # the engine trips a low ceiling
self.sessions = 0 # run sessions the engine saw (M8 with the phase not run)
@@ -279,6 +281,10 @@ class GateOffTests(unittest.TestCase):
def end():
time.sleep(0.3)
cool["phase"] = "smoke"
if self.temps_line:
self.fc.state["logs_tail"]["text"] += (
"Aug 22 12:00:30 forgectrl: cool: temps this job: chassis 29.0..29.4 C, "
"supply raw 587..592\n")
time.sleep(0.2)
cool["phase"] = "idle"
threading.Thread(target=end, daemon=True).start()
@@ -367,6 +373,12 @@ class GateOffTests(unittest.TestCase):
self.assertIn("run-start log line", str(cm.exception))
self.assertEqual(self.fc.state["settings"]["cool_temp_max"], "")
def test_a_missing_run_end_temperature_line_fails(self):
self.temps_line = False
with self.assertRaises(Failed) as cm:
self.run_test()
self.assertIn("board-temperature line", str(cm.exception))
def test_a_custom_ceiling_is_restored_verbatim(self):
self.fc.state["settings"].update({"cool_temp_max": "30", "cool_temp_resume": "28"})
self._describe()