mirror of
https://github.com/openglow-org/forgefirm.git
synced 2026-09-28 01:01:12 -07:00
ScottW514
2319735e60
Three fixes found running the drills: the splat regex no longer matches the benign "RCU lockdep checking is enabled" boot banner; the 40 V regulator search reaches its reg-fixed-voltage driver (regulators:40v) so the forced-defer drill resolves the resource instead of skipping; and require_idle waits out the run-through-'running' a forgectrl restart passes through instead of refusing.
197 lines
6.8 KiB
Python
197 lines
6.8 KiB
Python
#!/usr/bin/env python3
|
|
"""Debug-kernel drills: lock correctness across glowforge.ko load/unload
|
|
and a forced -EPROBE_DEFER unwind.
|
|
|
|
Runs only on the debug-kernel image (kas/forgefirm-glowforge-debug.yml),
|
|
which carries CONFIG_DEBUG_MUTEXES and lockdep. Both drills cycle the
|
|
40 V rail - a deliberate power cycle, the gamble the rail policy avoids
|
|
in normal operation and accepts here for one bench slot. The machine
|
|
must be idle with no job.
|
|
|
|
drill A stop forgectrl, rmmod + modprobe glowforge three times with a
|
|
settle between, and read dmesg for any lockdep/mutex splat.
|
|
drill B unbind the cnc device, unbind the 40 V regulator it needs so
|
|
the re-bind probe returns -EPROBE_DEFER and unwinds, read
|
|
dmesg for a clean unwind, then restore the regulator so the
|
|
deferred probe completes and the rail returns.
|
|
|
|
PASS = both drills leave dmesg free of BUG/WARNING/lockdep/"held lock"
|
|
and the machine ends idle with the rail up and forgectrl serving.
|
|
|
|
Not a catalog test: it needs a special kernel and cycles the rail, so
|
|
it rides the closing burn, not the acceptance campaign.
|
|
"""
|
|
import glob
|
|
import os
|
|
import re
|
|
import subprocess
|
|
import sys
|
|
import time
|
|
|
|
CNC_DRV = '/sys/bus/platform/drivers/glowforge_cnc'
|
|
# Real splat markers only - not the benign "RCU lockdep checking is
|
|
# enabled" boot banner (a bare "lockdep" match would flag it).
|
|
SPLAT = re.compile(r'BUG:|WARNING:|INFO: possible|held lock|'
|
|
r'circular locking|bad unlock|sleeping function|'
|
|
r'still has locks held|DEBUG_LOCKS_WARN|'
|
|
r'lock held when returning')
|
|
|
|
|
|
def sh(cmd, check=False):
|
|
r = subprocess.run(cmd, shell=True, capture_output=True, text=True)
|
|
if check and r.returncode != 0:
|
|
print(' ! %s -> rc %d %s' % (cmd, r.returncode, (r.stderr or '').strip()[:120]))
|
|
return r
|
|
|
|
|
|
def dmesg_since(mark):
|
|
out = sh('dmesg').stdout
|
|
i = out.rfind(mark)
|
|
return out[i + len(mark):] if i >= 0 else out
|
|
|
|
|
|
def mark(tag):
|
|
# A unique needle in the kernel log to bound a drill's window.
|
|
sh('echo "DRILL-%s" > /dev/kmsg' % tag)
|
|
return 'DRILL-%s' % tag
|
|
|
|
|
|
def splats(text):
|
|
return [ln for ln in text.splitlines() if SPLAT.search(ln)]
|
|
|
|
|
|
def require_debug_kernel():
|
|
cfg = ''
|
|
if os.path.exists('/proc/config.gz'):
|
|
cfg = sh('zcat /proc/config.gz').stdout
|
|
if 'CONFIG_DEBUG_MUTEXES=y' not in cfg:
|
|
print('REFUSED: not a debug kernel (CONFIG_DEBUG_MUTEXES not set). '
|
|
'Flash the debug image (kas/forgefirm-glowforge-debug.yml).')
|
|
sys.exit(2)
|
|
|
|
|
|
def require_idle():
|
|
# The controller re-init after a forgectrl restart passes briefly
|
|
# through 'running'; wait a few seconds for it to settle before
|
|
# refusing.
|
|
st = ''
|
|
for _ in range(8):
|
|
try:
|
|
st = open('/sys/glowforge/cnc/state').read().strip()
|
|
except OSError:
|
|
st = ''
|
|
if st in ('idle', 'disabled'):
|
|
return
|
|
time.sleep(1)
|
|
print('REFUSED: cnc/state is %r, not idle.' % st)
|
|
sys.exit(2)
|
|
|
|
|
|
def forgectrl(action):
|
|
sh('/etc/init.d/forgectrl %s' % action)
|
|
time.sleep(5)
|
|
|
|
|
|
def drill_load_unload():
|
|
print('== drill A: module load/unload under DEBUG_MUTEXES')
|
|
forgectrl('stop') # release /dev/glowforge so the module can unload
|
|
m = mark('A')
|
|
ok = True
|
|
for i in range(3):
|
|
r = sh('rmmod glowforge', check=True)
|
|
if r.returncode != 0:
|
|
print(' rmmod failed on cycle %d - aborting drill A' % (i + 1))
|
|
ok = False
|
|
break
|
|
time.sleep(2) # rail settle (the module powered it off)
|
|
r = sh('modprobe glowforge', check=True)
|
|
if r.returncode != 0:
|
|
print(' modprobe failed on cycle %d' % (i + 1))
|
|
ok = False
|
|
break
|
|
time.sleep(2)
|
|
print(' cycle %d: unload/reload ok' % (i + 1))
|
|
found = splats(dmesg_since(m))
|
|
forgectrl('start')
|
|
if found:
|
|
print(' FAIL: %d lock splat(s):' % len(found))
|
|
for ln in found[:8]:
|
|
print(' ', ln)
|
|
return False
|
|
print(' PASS: three load/unload cycles, no lock splat')
|
|
return ok
|
|
|
|
|
|
def regulator_provider():
|
|
# The 40 V regulator the cnc probe holds exclusively; unbinding its
|
|
# provider makes the re-bind probe defer. Resolve from the cnc node's
|
|
# 40v-supply phandle if present, else the known fixed regulator name.
|
|
drivers = (glob.glob('/sys/bus/platform/drivers/*regulator*') +
|
|
glob.glob('/sys/bus/platform/drivers/reg-fixed*'))
|
|
for d in drivers:
|
|
for dev in os.listdir(d):
|
|
if dev in ('bind', 'unbind', 'module', 'uevent'):
|
|
continue
|
|
if '40v' in dev.lower(): # e.g. regulators:40v
|
|
return os.path.basename(d), dev
|
|
return None, None
|
|
|
|
|
|
def drill_forced_defer():
|
|
print('== drill B: forced -EPROBE_DEFER unwind')
|
|
rdrv, rdev = regulator_provider()
|
|
if not rdev:
|
|
print(' SKIP: could not resolve the 40 V regulator provider to unbind '
|
|
'(inspect /sys/bus/platform/drivers/*regulator*); drill A stands.')
|
|
return None
|
|
forgectrl('stop')
|
|
m = mark('B')
|
|
# Take cnc down, remove the resource, bring cnc back -> probe defers.
|
|
sh('echo cnc > %s/unbind' % CNC_DRV, check=True)
|
|
time.sleep(1)
|
|
sh('echo %s > /sys/bus/platform/drivers/%s/unbind' % (rdev, rdrv), check=True)
|
|
time.sleep(1)
|
|
sh('echo cnc > %s/bind' % CNC_DRV) # returns -EPROBE_DEFER, unwinds
|
|
time.sleep(2)
|
|
deferred = 'cnc' not in os.listdir(CNC_DRV)
|
|
print(' cnc probe deferred (not bound after the resource was removed): %s' % deferred)
|
|
# Restore the resource -> the deferred probe retries and completes.
|
|
sh('echo %s > /sys/bus/platform/drivers/%s/bind' % (rdev, rdrv), check=True)
|
|
time.sleep(3)
|
|
bound = 'cnc' in os.listdir(CNC_DRV)
|
|
found = splats(dmesg_since(m))
|
|
forgectrl('start')
|
|
time.sleep(2)
|
|
state = ''
|
|
try:
|
|
state = open('/sys/glowforge/cnc/state').read().strip()
|
|
except OSError:
|
|
pass
|
|
print(' cnc bound after restore: %s; cnc/state=%r' % (bound, state))
|
|
if found:
|
|
print(' FAIL: %d lock splat(s) on the unwind:' % len(found))
|
|
for ln in found[:8]:
|
|
print(' ', ln)
|
|
return False
|
|
if not (deferred and bound and state in ('idle', 'disabled')):
|
|
print(' FAIL: defer=%s bound=%s state=%r' % (deferred, bound, state))
|
|
return False
|
|
print(' PASS: probe deferred and unwound clean, then completed on restore')
|
|
return True
|
|
|
|
|
|
def main():
|
|
require_debug_kernel()
|
|
require_idle()
|
|
a = drill_load_unload()
|
|
require_idle()
|
|
b = drill_forced_defer()
|
|
print('== summary: load/unload %s, forced-defer %s'
|
|
% ('PASS' if a else 'FAIL',
|
|
'PASS' if b else ('SKIP' if b is None else 'FAIL')))
|
|
sys.exit(0 if a and b is not False else 1)
|
|
|
|
|
|
if __name__ == '__main__':
|
|
main()
|