From 298abf8a12c2c74a1d0cef6082b5f9ce11d7b78e Mon Sep 17 00:00:00 2001 From: ScottW514 Date: Mon, 31 Aug 2026 15:54:28 -0400 Subject: [PATCH] Item 3: fold the debug kernel into the closing build kas/forgefirm-glowforge-debug.yml builds one dev image on the debug kernel (FORGEFIRM_KERNEL_DEBUG=1, tagged dev-debug) beside the closing release and dev images; the debug options never touch either. scripts/bench/debug_kernel_drills.py runs the two drills (three load/unload cycles under DEBUG_MUTEXES, a forced -EPROBE_DEFER unwind), each read against dmesg for lock splats, refusing on a non-debug kernel or a non-idle machine; it is registered as a board bench tool. BRINGUP item 3 names the mechanism and stays remaining work (run it on the closing burn); the site's Build page documents the variant. --- docs/BRINGUP.md | 14 +- forgetest/forgetest/bench.py | 6 + kas/forgefirm-glowforge-debug.yml | 32 +++++ scripts/bench/debug_kernel_drills.py | 185 +++++++++++++++++++++++++++ 4 files changed, 230 insertions(+), 7 deletions(-) create mode 100644 kas/forgefirm-glowforge-debug.yml create mode 100644 scripts/bench/debug_kernel_drills.py diff --git a/docs/BRINGUP.md b/docs/BRINGUP.md index abb4db0..c08a94d 100644 --- a/docs/BRINGUP.md +++ b/docs/BRINGUP.md @@ -1115,13 +1115,13 @@ Open items only. Anything closed is in `CAMPAIGN-LOG.md`. reachable-mode reasoning and the factory fallback configuration are in the headers of kernel patches 0011-0013 (`meta-glowforge-bsp`, `recipes-kernel/linux/`). -3. **Debug-kernel checks.** Module load/unload under `CONFIG_DEBUG_MUTEXES` - and a forced `-EPROBE_DEFER` unwind still need a debug kernel build. Both - drills cycle what the rail policy avoids: a module unload powers the 40 V - rail off (a stepper driver can come out of the power-up unserviceable), - and a forced defer needs the 40 V regulator or the SDMA device unbound - under the module's probe. This is a bench slot with the rail-cycle gamble - accepted, not a quick check. +3. **Debug-kernel checks.** Run the module load/unload and forced + `-EPROBE_DEFER` drills (`scripts/bench/debug_kernel_drills.py`) on the + debug-kernel image (`kas/forgefirm-glowforge-debug.yml`, built beside the + closing image). Both cycle the 40 V rail: a module unload powers it off (a + stepper driver can come out of the power-up unserviceable), and the forced + defer needs the 40 V regulator unbound under the probe. It is a bench slot + with the rail-cycle gamble accepted, and it rides the closing burn. 4. **Release acceptance follow-through.** The campaign is the release gate and runs as designed: dev image `20260824230512`, 45 of 45 from nothing, 36 of them unattended with the bench actuator in the loop, release diff --git a/forgetest/forgetest/bench.py b/forgetest/forgetest/bench.py index 8cda8d8..f490117 100644 --- a/forgetest/forgetest/bench.py +++ b/forgetest/forgetest/bench.py @@ -227,6 +227,12 @@ TOOLS = [ "safety": "dry", "where": "host", "ported": False, "args": [], "desc": "Decodes factory pulse streams into velocity/accel profiles. Runs anywhere; needs a .puls file " "(the reference captures live off the machine), so it is not a bench-page tool."}, + {"id": "debug-kernel-drills", "title": "Debug-kernel lock drills (load/unload, forced defer)", + "script": "debug_kernel_drills.py", + "safety": "takeover", "where": "board", "ported": True, "args": [], + "desc": "Runs only on the debug-kernel image (kas/forgefirm-glowforge-debug.yml): three glowforge.ko " + "load/unload cycles under DEBUG_MUTEXES and a forced -EPROBE_DEFER unwind, each read against " + "dmesg for lock splats. Cycles the 40 V rail; refuses on a non-debug kernel or a non-idle machine."}, ] # Files in scripts/bench that are not tools of their own: the helper module diff --git a/kas/forgefirm-glowforge-debug.yml b/kas/forgefirm-glowforge-debug.yml new file mode 100644 index 0000000..30c1a32 --- /dev/null +++ b/kas/forgefirm-glowforge-debug.yml @@ -0,0 +1,32 @@ +# ============================================================================ +# ForgeFIRM — debug-kernel build variant +# ============================================================================ +# One target: a dev image whose kernel carries the lock-correctness options +# (DEBUG_MUTEXES, PROVE_LOCKING, DEBUG_ATOMIC_SLEEP, DEBUG_SPINLOCK). It rides +# the closing build: after the normal release + dev images build, one pass of +# this config produces a dev image tagged "(dev-debug)". The bench boots it +# once to run the module load/unload and forced -EPROBE_DEFER drills +# (forgefirm/scripts/bench/debug_kernel_drills.py), then the real image is +# flashed. The options never touch a release or a normal dev build. +# +# cd forgefirm +# kas build kas/forgefirm-glowforge-debug.yml +# +# The debug kernel has a different config signature, so bitbake rebuilds the +# kernel and the dev image under it without disturbing the images already in +# build/tmp/deploy/images/glowforge/; the debug dev image lands beside them +# with its own DATETIME stamp and the "(dev-debug)" version string. +# ============================================================================ + +header: + version: 14 + includes: + - forgefirm-glowforge.yml + +target: forgefirm-image-dev + +local_conf_header: + debug-kernel: | + FORGEFIRM_KERNEL_DEBUG = "1" + FORGEFIRM_VERSION_STRING:pn-forgefirm-image-dev = "${DATETIME} (dev-debug)" + FORGEFIRM_VERSION_STRING[vardepsexclude] += "DATETIME" diff --git a/scripts/bench/debug_kernel_drills.py b/scripts/bench/debug_kernel_drills.py new file mode 100644 index 0000000..efdcd58 --- /dev/null +++ b/scripts/bench/debug_kernel_drills.py @@ -0,0 +1,185 @@ +#!/usr/bin/env python3 +"""Debug-kernel drills: lock correctness across glowforge.ko load/unload +and a forced -EPROBE_DEFER unwind. + +Runs only on the debug-kernel image (kas/forgefirm-glowforge-debug.yml), +which carries CONFIG_DEBUG_MUTEXES and lockdep. Both drills cycle the +40 V rail - a deliberate power cycle, the gamble the rail policy avoids +in normal operation and accepts here for one bench slot. The machine +must be idle with no job. + + drill A stop forgectrl, rmmod + modprobe glowforge three times with a + settle between, and read dmesg for any lockdep/mutex splat. + drill B unbind the cnc device, unbind the 40 V regulator it needs so + the re-bind probe returns -EPROBE_DEFER and unwinds, read + dmesg for a clean unwind, then restore the regulator so the + deferred probe completes and the rail returns. + +PASS = both drills leave dmesg free of BUG/WARNING/lockdep/"held lock" +and the machine ends idle with the rail up and forgectrl serving. + +Not a catalog test: it needs a special kernel and cycles the rail, so +it rides the closing burn, not the acceptance campaign. +""" +import glob +import os +import re +import subprocess +import sys +import time + +CNC_DRV = '/sys/bus/platform/drivers/glowforge_cnc' +SPLAT = re.compile(r'BUG:|WARNING:|INFO: possible|held lock|lockdep|' + r'circular locking|bad unlock|sleeping function|' + r'still has locks held|DEBUG_LOCKS_WARN') + + +def sh(cmd, check=False): + r = subprocess.run(cmd, shell=True, capture_output=True, text=True) + if check and r.returncode != 0: + print(' ! %s -> rc %d %s' % (cmd, r.returncode, (r.stderr or '').strip()[:120])) + return r + + +def dmesg_since(mark): + out = sh('dmesg').stdout + i = out.rfind(mark) + return out[i + len(mark):] if i >= 0 else out + + +def mark(tag): + # A unique needle in the kernel log to bound a drill's window. + sh('echo "DRILL-%s" > /dev/kmsg' % tag) + return 'DRILL-%s' % tag + + +def splats(text): + return [ln for ln in text.splitlines() if SPLAT.search(ln)] + + +def require_debug_kernel(): + cfg = '' + if os.path.exists('/proc/config.gz'): + cfg = sh('zcat /proc/config.gz').stdout + if 'CONFIG_DEBUG_MUTEXES=y' not in cfg: + print('REFUSED: not a debug kernel (CONFIG_DEBUG_MUTEXES not set). ' + 'Flash the debug image (kas/forgefirm-glowforge-debug.yml).') + sys.exit(2) + + +def require_idle(): + st = '' + try: + st = open('/sys/glowforge/cnc/state').read().strip() + except OSError: + pass + if st not in ('idle', 'disabled'): + print('REFUSED: cnc/state is %r, not idle.' % st) + sys.exit(2) + + +def forgectrl(action): + sh('/etc/init.d/forgectrl %s' % action) + time.sleep(3) + + +def drill_load_unload(): + print('== drill A: module load/unload under DEBUG_MUTEXES') + forgectrl('stop') # release /dev/glowforge so the module can unload + m = mark('A') + ok = True + for i in range(3): + r = sh('rmmod glowforge', check=True) + if r.returncode != 0: + print(' rmmod failed on cycle %d - aborting drill A' % (i + 1)) + ok = False + break + time.sleep(2) # rail settle (the module powered it off) + r = sh('modprobe glowforge', check=True) + if r.returncode != 0: + print(' modprobe failed on cycle %d' % (i + 1)) + ok = False + break + time.sleep(2) + print(' cycle %d: unload/reload ok' % (i + 1)) + found = splats(dmesg_since(m)) + forgectrl('start') + if found: + print(' FAIL: %d lock splat(s):' % len(found)) + for ln in found[:8]: + print(' ', ln) + return False + print(' PASS: three load/unload cycles, no lock splat') + return ok + + +def regulator_provider(): + # The 40 V regulator the cnc probe holds exclusively; unbinding its + # provider makes the re-bind probe defer. Resolve from the cnc node's + # 40v-supply phandle if present, else the known fixed regulator name. + for d in glob.glob('/sys/bus/platform/drivers/*regulator*'): + for dev in os.listdir(d): + if dev in ('bind', 'unbind', 'module', 'uevent'): + continue + if '40v' in dev.lower() or 'reg_40v' in dev.lower(): + return os.path.basename(d), dev + return None, None + + +def drill_forced_defer(): + print('== drill B: forced -EPROBE_DEFER unwind') + rdrv, rdev = regulator_provider() + if not rdev: + print(' SKIP: could not resolve the 40 V regulator provider to unbind ' + '(inspect /sys/bus/platform/drivers/*regulator*); drill A stands.') + return None + forgectrl('stop') + m = mark('B') + # Take cnc down, remove the resource, bring cnc back -> probe defers. + sh('echo cnc > %s/unbind' % CNC_DRV, check=True) + time.sleep(1) + sh('echo %s > /sys/bus/platform/drivers/%s/unbind' % (rdev, rdrv), check=True) + time.sleep(1) + sh('echo cnc > %s/bind' % CNC_DRV) # returns -EPROBE_DEFER, unwinds + time.sleep(2) + deferred = 'cnc' not in os.listdir(CNC_DRV) + print(' cnc probe deferred (not bound after the resource was removed): %s' % deferred) + # Restore the resource -> the deferred probe retries and completes. + sh('echo %s > /sys/bus/platform/drivers/%s/bind' % (rdev, rdrv), check=True) + time.sleep(3) + bound = 'cnc' in os.listdir(CNC_DRV) + found = splats(dmesg_since(m)) + forgectrl('start') + time.sleep(2) + state = '' + try: + state = open('/sys/glowforge/cnc/state').read().strip() + except OSError: + pass + print(' cnc bound after restore: %s; cnc/state=%r' % (bound, state)) + if found: + print(' FAIL: %d lock splat(s) on the unwind:' % len(found)) + for ln in found[:8]: + print(' ', ln) + return False + if not (deferred and bound and state in ('idle', 'disabled')): + print(' FAIL: defer=%s bound=%s state=%r' % (deferred, bound, state)) + return False + print(' PASS: probe deferred and unwound clean, then completed on restore') + return True + + +def main(): + require_debug_kernel() + require_idle() + a = drill_load_unload() + require_idle() + b = drill_forced_defer() + print('== summary: load/unload %s, forced-defer %s' + % ('PASS' if a else 'FAIL', + 'PASS' if b else ('SKIP' if b is None else 'FAIL'))) + sys.exit(0 if a and b is not False else 1) + + +if __name__ == '__main__': + main()