mirror of
https://github.com/openglow-org/forgefirm.git
synced 2026-09-27 08:41:13 -07:00
Item 3: fold the debug kernel into the closing build
kas/forgefirm-glowforge-debug.yml builds one dev image on the debug kernel (FORGEFIRM_KERNEL_DEBUG=1, tagged dev-debug) beside the closing release and dev images; the debug options never touch either. scripts/bench/debug_kernel_drills.py runs the two drills (three load/unload cycles under DEBUG_MUTEXES, a forced -EPROBE_DEFER unwind), each read against dmesg for lock splats, refusing on a non-debug kernel or a non-idle machine; it is registered as a board bench tool. BRINGUP item 3 names the mechanism and stays remaining work (run it on the closing burn); the site's Build page documents the variant.
This commit is contained in:
ScottW514
@@ -0,0 +1,185 @@
|
||||
#!/usr/bin/env python3
|
||||
"""Debug-kernel drills: lock correctness across glowforge.ko load/unload
|
||||
and a forced -EPROBE_DEFER unwind.
|
||||
|
||||
Runs only on the debug-kernel image (kas/forgefirm-glowforge-debug.yml),
|
||||
which carries CONFIG_DEBUG_MUTEXES and lockdep. Both drills cycle the
|
||||
40 V rail - a deliberate power cycle, the gamble the rail policy avoids
|
||||
in normal operation and accepts here for one bench slot. The machine
|
||||
must be idle with no job.
|
||||
|
||||
drill A stop forgectrl, rmmod + modprobe glowforge three times with a
|
||||
settle between, and read dmesg for any lockdep/mutex splat.
|
||||
drill B unbind the cnc device, unbind the 40 V regulator it needs so
|
||||
the re-bind probe returns -EPROBE_DEFER and unwinds, read
|
||||
dmesg for a clean unwind, then restore the regulator so the
|
||||
deferred probe completes and the rail returns.
|
||||
|
||||
PASS = both drills leave dmesg free of BUG/WARNING/lockdep/"held lock"
|
||||
and the machine ends idle with the rail up and forgectrl serving.
|
||||
|
||||
Not a catalog test: it needs a special kernel and cycles the rail, so
|
||||
it rides the closing burn, not the acceptance campaign.
|
||||
"""
|
||||
import glob
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
|
||||
CNC_DRV = '/sys/bus/platform/drivers/glowforge_cnc'
|
||||
SPLAT = re.compile(r'BUG:|WARNING:|INFO: possible|held lock|lockdep|'
|
||||
r'circular locking|bad unlock|sleeping function|'
|
||||
r'still has locks held|DEBUG_LOCKS_WARN')
|
||||
|
||||
|
||||
def sh(cmd, check=False):
|
||||
r = subprocess.run(cmd, shell=True, capture_output=True, text=True)
|
||||
if check and r.returncode != 0:
|
||||
print(' ! %s -> rc %d %s' % (cmd, r.returncode, (r.stderr or '').strip()[:120]))
|
||||
return r
|
||||
|
||||
|
||||
def dmesg_since(mark):
|
||||
out = sh('dmesg').stdout
|
||||
i = out.rfind(mark)
|
||||
return out[i + len(mark):] if i >= 0 else out
|
||||
|
||||
|
||||
def mark(tag):
|
||||
# A unique needle in the kernel log to bound a drill's window.
|
||||
sh('echo "DRILL-%s" > /dev/kmsg' % tag)
|
||||
return 'DRILL-%s' % tag
|
||||
|
||||
|
||||
def splats(text):
|
||||
return [ln for ln in text.splitlines() if SPLAT.search(ln)]
|
||||
|
||||
|
||||
def require_debug_kernel():
|
||||
cfg = ''
|
||||
if os.path.exists('/proc/config.gz'):
|
||||
cfg = sh('zcat /proc/config.gz').stdout
|
||||
if 'CONFIG_DEBUG_MUTEXES=y' not in cfg:
|
||||
print('REFUSED: not a debug kernel (CONFIG_DEBUG_MUTEXES not set). '
|
||||
'Flash the debug image (kas/forgefirm-glowforge-debug.yml).')
|
||||
sys.exit(2)
|
||||
|
||||
|
||||
def require_idle():
|
||||
st = ''
|
||||
try:
|
||||
st = open('/sys/glowforge/cnc/state').read().strip()
|
||||
except OSError:
|
||||
pass
|
||||
if st not in ('idle', 'disabled'):
|
||||
print('REFUSED: cnc/state is %r, not idle.' % st)
|
||||
sys.exit(2)
|
||||
|
||||
|
||||
def forgectrl(action):
|
||||
sh('/etc/init.d/forgectrl %s' % action)
|
||||
time.sleep(3)
|
||||
|
||||
|
||||
def drill_load_unload():
|
||||
print('== drill A: module load/unload under DEBUG_MUTEXES')
|
||||
forgectrl('stop') # release /dev/glowforge so the module can unload
|
||||
m = mark('A')
|
||||
ok = True
|
||||
for i in range(3):
|
||||
r = sh('rmmod glowforge', check=True)
|
||||
if r.returncode != 0:
|
||||
print(' rmmod failed on cycle %d - aborting drill A' % (i + 1))
|
||||
ok = False
|
||||
break
|
||||
time.sleep(2) # rail settle (the module powered it off)
|
||||
r = sh('modprobe glowforge', check=True)
|
||||
if r.returncode != 0:
|
||||
print(' modprobe failed on cycle %d' % (i + 1))
|
||||
ok = False
|
||||
break
|
||||
time.sleep(2)
|
||||
print(' cycle %d: unload/reload ok' % (i + 1))
|
||||
found = splats(dmesg_since(m))
|
||||
forgectrl('start')
|
||||
if found:
|
||||
print(' FAIL: %d lock splat(s):' % len(found))
|
||||
for ln in found[:8]:
|
||||
print(' ', ln)
|
||||
return False
|
||||
print(' PASS: three load/unload cycles, no lock splat')
|
||||
return ok
|
||||
|
||||
|
||||
def regulator_provider():
|
||||
# The 40 V regulator the cnc probe holds exclusively; unbinding its
|
||||
# provider makes the re-bind probe defer. Resolve from the cnc node's
|
||||
# 40v-supply phandle if present, else the known fixed regulator name.
|
||||
for d in glob.glob('/sys/bus/platform/drivers/*regulator*'):
|
||||
for dev in os.listdir(d):
|
||||
if dev in ('bind', 'unbind', 'module', 'uevent'):
|
||||
continue
|
||||
if '40v' in dev.lower() or 'reg_40v' in dev.lower():
|
||||
return os.path.basename(d), dev
|
||||
return None, None
|
||||
|
||||
|
||||
def drill_forced_defer():
|
||||
print('== drill B: forced -EPROBE_DEFER unwind')
|
||||
rdrv, rdev = regulator_provider()
|
||||
if not rdev:
|
||||
print(' SKIP: could not resolve the 40 V regulator provider to unbind '
|
||||
'(inspect /sys/bus/platform/drivers/*regulator*); drill A stands.')
|
||||
return None
|
||||
forgectrl('stop')
|
||||
m = mark('B')
|
||||
# Take cnc down, remove the resource, bring cnc back -> probe defers.
|
||||
sh('echo cnc > %s/unbind' % CNC_DRV, check=True)
|
||||
time.sleep(1)
|
||||
sh('echo %s > /sys/bus/platform/drivers/%s/unbind' % (rdev, rdrv), check=True)
|
||||
time.sleep(1)
|
||||
sh('echo cnc > %s/bind' % CNC_DRV) # returns -EPROBE_DEFER, unwinds
|
||||
time.sleep(2)
|
||||
deferred = 'cnc' not in os.listdir(CNC_DRV)
|
||||
print(' cnc probe deferred (not bound after the resource was removed): %s' % deferred)
|
||||
# Restore the resource -> the deferred probe retries and completes.
|
||||
sh('echo %s > /sys/bus/platform/drivers/%s/bind' % (rdev, rdrv), check=True)
|
||||
time.sleep(3)
|
||||
bound = 'cnc' in os.listdir(CNC_DRV)
|
||||
found = splats(dmesg_since(m))
|
||||
forgectrl('start')
|
||||
time.sleep(2)
|
||||
state = ''
|
||||
try:
|
||||
state = open('/sys/glowforge/cnc/state').read().strip()
|
||||
except OSError:
|
||||
pass
|
||||
print(' cnc bound after restore: %s; cnc/state=%r' % (bound, state))
|
||||
if found:
|
||||
print(' FAIL: %d lock splat(s) on the unwind:' % len(found))
|
||||
for ln in found[:8]:
|
||||
print(' ', ln)
|
||||
return False
|
||||
if not (deferred and bound and state in ('idle', 'disabled')):
|
||||
print(' FAIL: defer=%s bound=%s state=%r' % (deferred, bound, state))
|
||||
return False
|
||||
print(' PASS: probe deferred and unwound clean, then completed on restore')
|
||||
return True
|
||||
|
||||
|
||||
def main():
|
||||
require_debug_kernel()
|
||||
require_idle()
|
||||
a = drill_load_unload()
|
||||
require_idle()
|
||||
b = drill_forced_defer()
|
||||
print('== summary: load/unload %s, forced-defer %s'
|
||||
% ('PASS' if a else 'FAIL',
|
||||
'PASS' if b else ('SKIP' if b is None else 'FAIL')))
|
||||
sys.exit(0 if a and b is not False else 1)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
main()
|
||||
Reference in New Issue
Block a user