vh
20f1cb8594
The browser tests flake under full-suite load only; every failing test passes alone. BOOTH_TRACE=1 keeps a full trace (screenshots and DOM snapshots) for each browser test that fails. BOOTH_TRACE=light keeps actions and network only, because the full mode perturbs the timing it watches: 0/8 red traced against 1/8 untraced on the same tree. Off by default. Positive control: a deliberately failing test keeps a trace, and a passing one keeps nothing.
136 lines
4.7 KiB
Python
136 lines
4.7 KiB
Python
"""Browser-test failure artefacts: a Playwright trace kept for every browser
|
|
test that FAILS, captured from the run that failed.
|
|
|
|
Why this exists: the browser tests flake under FULL-SUITE load only — three
|
|
different tests have each failed once, every one passes in isolation, and a
|
|
narrowed repro that passes is the trap (operator, 2026-09-23: "let him diagnose
|
|
it properly"). Pass/fail counts cannot say why; a trace — screenshots, DOM
|
|
snapshots, console and network per action — can.
|
|
|
|
OPT-IN, because tracing is not free and the harness is part of the number:
|
|
with it on, every page does more work, so the suite's timing (the very thing
|
|
under suspicion) moves. Default runs are untouched.
|
|
|
|
BOOTH_TRACE=1 .venv/bin/python -m pytest -q # screenshots + DOM snapshots
|
|
BOOTH_TRACE=light .venv/bin/python -m pytest -q # actions + network only
|
|
|
|
LIGHT exists because the full mode perturbs the thing it watches: 8 traced
|
|
full-suite runs went 8/8 green while untraced runs on the same tree went red.
|
|
Network and action records are nearly free, and a goto that never reaches
|
|
"networkidle" is answered by the network record alone — which request never
|
|
finished.
|
|
|
|
Traces land in $BOOTH_TRACE_DIR (default: <tmp>/booth-test-traces/<run>/),
|
|
named after the test; open one with `playwright show-trace <file>`. The
|
|
terminal summary lists every trace kept.
|
|
"""
|
|
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import shutil
|
|
import tempfile
|
|
import time
|
|
from pathlib import Path
|
|
|
|
import pytest
|
|
|
|
TRACE_MODE = os.environ.get("BOOTH_TRACE", "")
|
|
TRACE = TRACE_MODE in ("1", "light")
|
|
_KEPT: list[Path] = []
|
|
|
|
|
|
@pytest.hookimpl(hookwrapper=True)
|
|
def pytest_runtest_makereport(item, call):
|
|
outcome = yield
|
|
rep = outcome.get_result()
|
|
setattr(item, "rep_" + rep.when, rep)
|
|
|
|
|
|
def _trace_dir() -> Path:
|
|
root = os.environ.get("BOOTH_TRACE_DIR") or os.path.join(tempfile.gettempdir(), "booth-test-traces")
|
|
d = Path(root) / time.strftime("%Y%m%d-%H%M%S", time.localtime(_RUN_STARTED))
|
|
d.mkdir(parents=True, exist_ok=True)
|
|
return d
|
|
|
|
|
|
_RUN_STARTED = time.time()
|
|
|
|
|
|
@pytest.fixture(autouse=True)
|
|
def _trace_browser_tests(request):
|
|
"""Wrap the module's `browser` so every context it opens is traced.
|
|
|
|
A test usually closes its page BEFORE asserting (it collects, closes, then
|
|
checks), and a closed context can no longer write its trace — so each
|
|
context's trace is written at close time, to a scratch file, and only moved
|
|
to the kept set if the test then fails. A page from `browser.new_page` owns
|
|
its context, as Playwright's own does: closing the page closes it."""
|
|
if not TRACE or "browser" not in request.fixturenames:
|
|
yield
|
|
return
|
|
browser = request.getfixturevalue("browser")
|
|
scratch = Path(tempfile.mkdtemp(prefix="booth-trace-"))
|
|
written: list[Path] = []
|
|
opened: list = []
|
|
real_new_context = browser.new_context
|
|
|
|
def stop(ctx, n=[0]):
|
|
if getattr(ctx, "_booth_traced", False):
|
|
ctx._booth_traced = False
|
|
n[0] += 1
|
|
path = scratch / f"{n[0]}.zip"
|
|
try:
|
|
ctx.tracing.stop(path=str(path))
|
|
written.append(path)
|
|
except Exception: # noqa: BLE001 - a lost trace must not fail the test
|
|
pass
|
|
|
|
def new_context(*args, **kwargs):
|
|
ctx = real_new_context(*args, **kwargs)
|
|
heavy = TRACE_MODE == "1"
|
|
ctx.tracing.start(screenshots=heavy, snapshots=heavy)
|
|
ctx._booth_traced = True
|
|
real_close = ctx.close
|
|
|
|
def close(*a, **k):
|
|
stop(ctx)
|
|
return real_close(*a, **k)
|
|
|
|
ctx.close = close
|
|
opened.append(ctx)
|
|
return ctx
|
|
|
|
def new_page(*args, **kwargs):
|
|
ctx = new_context(*args, **kwargs)
|
|
page = ctx.new_page()
|
|
page.close = lambda *a, **k: ctx.close()
|
|
return page
|
|
|
|
browser.new_context, browser.new_page = new_context, new_page
|
|
try:
|
|
yield
|
|
finally:
|
|
del browser.new_context, browser.new_page
|
|
for ctx in opened:
|
|
stop(ctx)
|
|
try:
|
|
ctx.close()
|
|
except Exception: # noqa: BLE001
|
|
pass
|
|
rep = getattr(request.node, "rep_call", None)
|
|
if rep is not None and rep.failed and written:
|
|
dest = _trace_dir()
|
|
for i, path in enumerate(written, 1):
|
|
kept = dest / f"{request.node.name}-{i}.zip"
|
|
shutil.move(str(path), kept)
|
|
_KEPT.append(kept)
|
|
shutil.rmtree(scratch, ignore_errors=True)
|
|
|
|
|
|
def pytest_terminal_summary(terminalreporter):
|
|
if _KEPT:
|
|
terminalreporter.section("browser traces kept for failed tests")
|
|
for p in _KEPT:
|
|
terminalreporter.write_line(str(p))
|