diff --git a/docs/diagnostics/descriptive_query_binding.py b/docs/diagnostics/descriptive_query_binding.py index 2051b7a..4d63ce6 100644 --- a/docs/diagnostics/descriptive_query_binding.py +++ b/docs/diagnostics/descriptive_query_binding.py @@ -9,16 +9,26 @@ is resolved to an entity, which is the open problem. Downstream, the consumer so to the wrong co-retrieved subject and cross-contaminates details (a confident, fluent mis-bind assembled from real-but-mismatched rows, not a hallucination). -Two regimes, one variable set: - * RAW RANKING (persona-independent): drive the `mimir` librarian with the descriptive query - and its variants; show the top-k rows flagged for the intended ENTITY vs the DECOY. The - load-bearing fact is "intended entity absent from top-k." - * CONSUMER: drive `ratatoskr:donut` N times on the descriptive question; classify each answer - BINDS-ENTITY vs MIS-BINDS-DECOY vs OTHER and report the rate. +Two regimes at DIFFERENT WING SCOPES (the distinction is load-bearing — Worldtree #393): + * RAW RANKING — ALL-WING (mimir searches kb+main+fiction, ~9800 rows). Drive `mimir` with the + descriptive query + variants; report the query mimir ACTUALLY passed (tool_start q=, since it + reformulates) and where the intended ENTITY lands. Cross-wing dilution can push a one-arm + vector bridge out of the bge rescue window entirely -> entity ABSENT from top-k. + * CONSUMER — FICTION (ratatoskr:donut is fiction-scoped, ~1578 rows). Drive Donut N times; + classify BINDS-ENTITY vs MIS-BINDS-DECOY vs OTHER, and report where the ENTITY and DECOY rank + in her fiction-scoped results. The ranks tell WHICH failure fired per run: entity PRESENT + + mis-bind = subject-selection; entity ABSENT + a present decoy = reformulation-induced absence + (Donut distilled the descriptive phrase to bare tokens that don't carry the vocabulary bridge). + Empirically Donut mostly does the latter — she distills "the guy with the roid rage" to bare + "roid rage", so Juicer drops out even at fiction scope and she binds a present decoy (Jack). -Two-regime finding (2026-08-07, personal instance): roid-rage mis-binding SURVIVES the #389 arc -(b172 -> v1.0.0b181); "dangerous crown" is mostly RESOLVED by the bge rerank + source annotations -(it resolves to essentially one entity, "the Enchanted Crown of the Sepsis Whore"). The gap is +Two-seam finding (2026-08-07, v1.0.0b181) — both lose the entity, by DIFFERENT reformulation seams: +(1) mimir PRESERVES the phrase -> cross-wing dilution (kb+main+fiction) drops the entity from its +all-wing top-k; (2) Donut DISTILLS the phrase to bare tokens -> the entity drops even at fiction +scope (ent@None) and she binds a present decoy. The fold's ENT@/DEC@ ranks separate reformulation- +absence from true subject-selection per run. Unifying lever: disambiguating-vocabulary expansion +(the full phrase, or +attribute like "steroid") surfaces the entity at fiction scope — Donut's +expand-runs bind correctly. "dangerous crown" is mostly RESOLVED (one entity). Root gap: attribute->entity resolution, upstream of names_subject by construction. Self-contained: the only third-party dependency is httpx (`uv run --with httpx`). Config from env: @@ -105,17 +115,27 @@ def _flag(excerpt: str, entity: str, decoy: str | None) -> str: return " " +def _rank_in(res: dict, pattern: str | None) -> int | None: + """Rank of the first row whose excerpt matches `pattern` (None if absent / no pattern).""" + if not pattern: + return None + rows = res.get("hits", res.get("results", [])) if isinstance(res, dict) else [] + return next((i for i, r in enumerate(rows) if isinstance(r, dict) + and re.search(pattern, r.get("excerpt", ""), re.I)), None) + + def raw_ranking(base, headers, end_user, case) -> None: - print("\n [raw ranking] intended entity absent from top-k?") + print("\n [raw ranking — ALL-WING/mimir] actual tool query (reformulation seam) + entity rank") for query in case["variants"]: sid = _session(base, headers, "mimir", end_user) # fresh session per query - _, res, _ = _drive(base, headers, sid, f"Use search_library to find: {query}") + mimir_q, res, _ = _drive(base, headers, sid, f"Use search_library to find: {query}") rows = res.get("results", []) if isinstance(res, dict) else [] entity_ranks = [i for i, r in enumerate(rows) if isinstance(r, dict) and re.search(case["entity"], r.get("excerpt", ""), re.I)] - top = entity_ranks[0] if entity_ranks else None - print(f" query={query!r:36} entity first appears at rank " - f"{top if top is not None else 'ABSENT (not in top-k)'}") + rank = entity_ranks[0] if entity_ranks else "ABSENT (not in top-k)" + # mimir_q is load-bearing for #393: separates reformulation-at-the-agent-seam + # (mimir distilled/expanded the phrase) from ranking (the tool ranked it low). + print(f" instructed={query!r:28} mimir_q={mimir_q!r:38} entity_rank={rank}") for i, r in enumerate(rows[:6]): if isinstance(r, dict): ex = (r.get("excerpt") or "").replace("\n", " ") @@ -125,15 +145,19 @@ def raw_ranking(base, headers, end_user, case) -> None: def consumer(base, headers, end_user, case, runs) -> Counter: verdicts: Counter = Counter() - print(f"\n [consumer] Donut x{runs} on {case['question']!r}") + print(f"\n [consumer — FICTION/donut] x{runs} on {case['question']!r}" + f" (ent@/dec@ = rank in Donut's fiction-scoped results)") for run in range(1, runs + 1): sid = _session(base, headers, "ratatoskr:donut", end_user) # fresh session per run - q, _, ans = _drive(base, headers, sid, case["question"]) + q, res, ans = _drive(base, headers, sid, case["question"]) binds = bool(re.search(case["entity"], ans, re.I)) mis = bool(case["decoy"]) and bool(re.search(case["decoy"], ans)) and not binds v = "BINDS-ENTITY" if binds else ("MIS-BINDS-DECOY" if mis else "OTHER") verdicts[v] += 1 - print(f" run{run}: {v:16} q={q!r:34} :: {ans.strip()[:80]}") + er, dr = _rank_in(res, case["entity"]), _rank_in(res, case["decoy"]) + print(f" run{run}: {v:16} ent@{er} dec@{dr} q={q!r:30} :: {ans.strip()[:56]}") + # a present entity (ent@ not None) co-occurring with a mis-bind is subject-selection, + # NOT ranking-absence — the fiction-scope half of the #393 two-mechanism split. print(f" >>> {case['label']}: {dict(verdicts)}") return verdicts