From be2c577884b0df1bebe3c4f97816ce9e57280c61 Mon Sep 17 00:00:00 2001 From: Vuong Hoang Date: Thu, 16 Jul 2026 22:37:31 -0700 Subject: [PATCH] =?UTF-8?q?feat(provider):=20mark=5Fsuperseded=20verb=20?= =?UTF-8?q?=E2=80=94=20Worldtree=20#364=20contradiction=20retirement=20+?= =?UTF-8?q?=20bifrost=201.1.4?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Implement `mark_superseded(ids, *, superseded_by=None, reason=None)` — the SOLE supersession verb Worldtree #364's promotion-hygiene reconciliation calls to retire contradicted facts (wire shape confirmed by worldtree-dev, bifrost_memory_store.py:293). Live re-verify (2026-07-16) proved our provider 500-crashed on this call (unimplemented) → #364's retirement couldn't land + a retry-storm bloated the store; the readout only passed via transient recency-eviction. - `mark_superseded` mirrors the reference `_mark_lifecycle`: sets top-level `superseded=True` (+ `superseded_by`/`superseded_reason` when non-None), increments revision, NON-destructive (get still returns; recoverable). Unknown ids skipped. - `_is_live` (INV-011) now short-circuits on `superseded is True`, so a retired chunk is excluded from `scan` (person-prime) — durable retirement, not just recency-eviction. search is unfiltered (matches reference; WT re-checks liveness client-side). - Contract: un-defer mark_superseded (+ FN spec, INV-011); TDD 5/5 (retires-from-scan tracer, non-destructive-get, unknown-id no-op, non-None-fields-only, parity #195). - bifrost 1.1.1→1.1.4: hasattr-gate backstop for the maintenance verbs (unimplemented verb → unsupported_capability 400, never AttributeError/500/retry-storm — the gap we surfaced) + the 1.1.3 scan/cursor conformance harness. Full suite 644 green. --- .../bifrost_memory_provider.contract.md | 27 +++++++++- pyproject.toml | 4 +- src/ratatoskr/provider/memory_store.py | 32 +++++++++++ tests/test_provider_memory.py | 54 +++++++++++++++++++ uv.lock | 10 ++-- 5 files changed, 119 insertions(+), 8 deletions(-) diff --git a/docs/contracts/bifrost_memory_provider.contract.md b/docs/contracts/bifrost_memory_provider.contract.md index 9c58ecc..82416c3 100644 --- a/docs/contracts/bifrost_memory_provider.contract.md +++ b/docs/contracts/bifrost_memory_provider.contract.md @@ -163,6 +163,13 @@ interpreted. id-list + TTL + `ScanCursorExpired`); DEFERRED pending bifrost-dev's ruling on the conformance gap (scan/cursor has NO conformance coverage today, so a non-snapshot cursor passes). Routed to bifrost-dev 2026-07-15. +- **INV-011** [hard]: **`mark_superseded` retires via a top-level `superseded` flag; `_is_live` + recognizes it.** `mark_superseded` sets top-level `superseded=True` (+ `superseded_by`) on the + record, mirroring the reference `_mark_lifecycle` (NOT a `verbatim.governance_state` change). So + `_is_live` MUST short-circuit on `record.get("superseded") is True` (in addition to its existing + `lifecycle_state` / `verbatim.governance_state` checks) — else a #364-retired chunk would still + scan live. Retirement is NON-destructive: `get`/`get_many` still return superseded chunks + (recoverable). `search` is NOT filtered (matches the reference; WT re-checks liveness client-side). ## Concurrency @@ -199,7 +206,7 @@ negotiation, routes). **This contract** owns the store (the basic verbs + SQLite ## Out of scope (deferred — do NOT flag as drift) -- **Gated/maintenance verbs:** `upsert_edges`/`get_edges_for`, `mark_invalid`/`mark_superseded`, `patch_many`, `atomic_supersede`, lease/checkpoint. Absent + advertised-unsupported. (`scan` is NO LONGER deferred — it is implemented + advertised via `sortable_chunk_fields` to light up Worldtree's #349 person-prime turn-1 durable-fact injection; see the `scan` FN spec + INV-009/INV-010.) +- **Gated/maintenance verbs:** `upsert_edges`/`get_edges_for`, `mark_invalid`, `patch_many`, `atomic_supersede`, lease/checkpoint. Absent (no describe_store cap; hasattr-gated at dispatch as of bifrost 1.1.4 → `unsupported_capability` 400). (`scan` and `mark_superseded` are NO LONGER deferred — `scan` implements #349 person-prime; `mark_superseded` implements Worldtree #364's contradiction retirement, the SOLE supersession verb #364 uses. See their FN specs + INV-009/INV-011.) - **metadata_filter beyond scope:** advertise `filterable_metadata_fields=[]`; a non-empty `metadata_filter` is unsupported in v1 (rejected — see search PRE). - **The combined two-plane server** (guide §7) — separate memory + affect apps in v1. - **Deployment** — dev-box background shell (`ratatoskr-memory-provider`), no systemd/infra. @@ -306,6 +313,24 @@ TESTS: delete_absent [boundary]: unknown id → {"deleted":0} ``` +```contract +FN mark_superseded(self, ids: list[str], *, superseded_by: str | None = None, reason: str | None = None) -> dict +BRIEF: Worldtree #364 retirement — mark chunks superseded so scan (live-only) excludes them. Mirrors the reference _mark_lifecycle: sets TOP-LEVEL fields on the record; NON-destructive (get still returns them, recoverable). The SOLE supersession verb #364 uses (dispatch: bifrost/memory.py mark_superseded branch; args {ids:[...], superseded_by, reason}). +PRE: [PRE-001 hard] ids is a list of chunk ids (WT sends singletons, one call per retired chunk) +POST: [POST-001 return_value] {"marked": N} where N = ids that existed (unknown ids skipped, never error) -- assert +POST: [POST-002 state_change] each existing chunk gets top-level `superseded=True` + `superseded_by` (when not None) + `superseded_reason` (when not None); revision incremented; mirrors reference _mark_lifecycle (only non-None fields written) -- assert +POST: [POST-003 return_value] a superseded chunk is EXCLUDED from `scan` (INV-009 via _is_live's top-level `superseded` check, INV-011) but STILL returned by `get`/`get_many` (non-destructive) -- assert +STEPS: + 1. [sequential, flexibility=indicative] FOR each id present: load record_json, set superseded=True (+ superseded_by / superseded_reason when not None), UPDATE record_json + revision+1; count + 2. [cleanup] RETURN {"marked": count} +TESTS: + mark_retires_from_scan [happy,tracer]: upsert 3 live; mark_superseded([id2], superseded_by="x"); scan → the 2 non-superseded only (id2 excluded); id2 record has superseded=True + superseded_by="x" + mark_get_still_returns [scenario]: a superseded chunk is STILL returned by get (non-destructive/recoverable) + mark_unknown_id_noop [boundary]: mark_superseded(["nope"]) → {"marked":0} + mark_no_superseded_by [boundary]: mark_superseded([id], superseded_by=None) → superseded=True set, no superseded_by key written (only non-None fields) + mark_parity_vs_reference [scenario]: identical mark_superseded envelope vs InMemoryMemoryStore → same top-level superseded/superseded_by field shape (#195) +``` + ```contract FN scan(self, *, scope_all: dict | None = None, scope_any: list | None = None, cursor: str | None = None, limit: int, sort: dict | None = None, lifecycle_state=None) -> dict BRIEF: Query-LESS paginated LIVE-chunk scan, globally ordered by an advertised sort field (updated_at) — the #349 person-prime turn-1 durable-fact injection primitive (no query vector, unlike search). Returns {records, cursor}. diff --git a/pyproject.toml b/pyproject.toml index 4ae0bad..6146f37 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -4,7 +4,7 @@ build-backend = "hatchling.build" [project] name = "ratatoskr" -version = "0.20.14" +version = "0.20.15" description = "Worldtree Conversation API debug TUI — multi-pane observability dashboard" readme = "README.md" requires-python = ">=3.12" @@ -30,7 +30,7 @@ web = [ # from the debug TUI. Recipe: bifrost/docs/implementing-a-consumer.md. provider = [ "ratatoskr[web]", # reuse the starlette + uvicorn ASGI stack - "bifrost==1.1.1", # consumer engines + library. 1.1.1 = frozen-wire serialization fix (ADR-0008): additive capability fields are gated on the NEGOTIATED wire, so a v0.6-negotiated describe_store handshake stays v0.6-clean. 1.1.0 leaked the v0.7-additive `sortable_chunk_fields` into v0.6 StoreCapabilities → a strict v0.6 client (additionalProperties:false) rejects our server's handshake. Wire schemas + pins UNCHANGED (serialization-correctness only); our v0.7 handshake with Worldtree b47 is unaffected. (1.1.0 = wire v0.7 additive: memory.scan sort + sortable_chunk_fields; 1.0.0 = first STABLE, wire v0.6 FROZEN; 0.8.0/v0.6 scope_all/scope_any #11; 0.7.0/v0.5 agent_self) + "bifrost==1.1.4", # consumer engines + library. 1.1.4 = hasattr-gate backstop for the maintenance verbs (mark_superseded/mark_invalid/patch_many/delete_many/upsert_edges/get_edges_for → unimplemented verb degrades to unsupported_capability 400, never AttributeError/500/retry-storm; we surfaced it via WT #364) + 1.1.3 scan/cursor conformance harness + 1.1.2 frozen-v0.6 fix. 1.1.1 = frozen-wire serialization fix (ADR-0008): additive capability fields are gated on the NEGOTIATED wire, so a v0.6-negotiated describe_store handshake stays v0.6-clean. 1.1.0 leaked the v0.7-additive `sortable_chunk_fields` into v0.6 StoreCapabilities → a strict v0.6 client (additionalProperties:false) rejects our server's handshake. Wire schemas + pins UNCHANGED (serialization-correctness only); our v0.7 handshake with Worldtree b47 is unaffected. (1.1.0 = wire v0.7 additive: memory.scan sort + sortable_chunk_fields; 1.0.0 = first STABLE, wire v0.6 FROZEN; 0.8.0/v0.6 scope_all/scope_any #11; 0.7.0/v0.5 agent_self) "jsonschema>=4", # bifrost runtime dep — envelope validation "sqlite-vec>=0.1.6", # vector index for the memory plane (vec0 virtual table) ] diff --git a/src/ratatoskr/provider/memory_store.py b/src/ratatoskr/provider/memory_store.py index 8c4d9fe..25211e5 100644 --- a/src/ratatoskr/provider/memory_store.py +++ b/src/ratatoskr/provider/memory_store.py @@ -143,6 +143,8 @@ def _is_live(record: dict) -> bool: """INV-009: a chunk is live unless a lifecycle/governance marker says otherwise. scan returns live-only server-side (person-prime's `lifecycle_state=live` does not ride the scan wire, so this is authoritative — a dead fact can never inject).""" + if record.get("superseded") is True: # INV-011: mark_superseded top-level flag (#364 retirement) + return False state = record.get("lifecycle_state") if isinstance(state, str) and state and state != "live": return False @@ -370,6 +372,36 @@ class RatatoskrMemoryStore: self._conn.execute("DELETE FROM memory_vec WHERE chunk_id = ?", (chunk_id,)) return {"deleted": deleted} + async def mark_superseded( + self, ids: list[str], *, superseded_by: str | None = None, reason: str | None = None + ) -> dict: + # Worldtree #364 retirement (INV-011). Mirrors the reference `_mark_lifecycle`: + # sets TOP-LEVEL `superseded`/`superseded_by`/`superseded_reason` (only non-None fields), + # increments revision. NON-destructive — get still returns; scan (live-only) excludes via + # `_is_live`'s `superseded` short-circuit. The sole supersession verb #364 uses. + _log.info("memory-call mark_superseded REQUEST: ids=%r superseded_by=%s", ids, superseded_by) + fields = {"superseded": True, "superseded_by": superseded_by, "superseded_reason": reason} + marked = 0 + with self._conn: + for chunk_id in ids: + row = self._conn.execute( + "SELECT record_json FROM memory_chunks WHERE chunk_id = ?", (chunk_id,) + ).fetchone() + if row is None: # unknown id -> skip (never error), mirrors reference + delete_many + continue + record = json.loads(row[0]) + for key, value in fields.items(): + if value is not None: # reference writes only non-None fields + record[key] = value + self._conn.execute( + "UPDATE memory_chunks SET record_json = ?, revision = revision + 1 " + "WHERE chunk_id = ?", + (json.dumps(record), chunk_id), + ) + marked += 1 + _log.info("memory-call mark_superseded RESPONSE: marked=%d", marked) + return {"marked": marked} + async def scan( self, *, diff --git a/tests/test_provider_memory.py b/tests/test_provider_memory.py index ebc2a02..e44f142 100644 --- a/tests/test_provider_memory.py +++ b/tests/test_provider_memory.py @@ -507,6 +507,60 @@ async def test_scan_parity_vs_reference_inmemory_store(): assert out_ours["records"] == out_ref["records"] # verbatim record shape parity +# --- mark_superseded (#364 contradiction retirement) --- + +async def test_mark_superseded_retires_from_scan(): + # tracer: mark a chunk superseded -> scan (live-only) excludes it; record carries the flags. + store = open_memory_store(":memory:", embedding_dim=EMBEDDING_DIM) + recs = [_chunk(f"c{i}", scope={"end_user": "u1"}, updated_at=f"2026-07-15T00:0{i}:00+00:00") for i in range(3)] + await store.upsert_many(recs, idempotency_key="k", ctx=_ctx()) + assert await store.mark_superseded(["c1"], superseded_by="c9") == {"marked": 1} + out = await store.scan(scope_all={"end_user": "u1"}, limit=10, sort={"field": "updated_at", "direction": "desc"}) + assert [r["id"] for r in out["records"]] == ["c2", "c0"] # c1 excluded (superseded) + got = await store.get("c1") + assert got["superseded"] is True and got["superseded_by"] == "c9" + + +async def test_mark_superseded_non_destructive_get_still_returns(): + # INV-011: retirement is non-destructive — get still returns a superseded chunk (recoverable). + store = open_memory_store(":memory:", embedding_dim=EMBEDDING_DIM) + await store.upsert_many([_chunk("c1", scope={"end_user": "u1"})], idempotency_key="k", ctx=_ctx()) + await store.mark_superseded(["c1"], superseded_by="x") + got = await store.get("c1") + assert got is not None and got["superseded"] is True + + +async def test_mark_superseded_unknown_id_noop(): + store = open_memory_store(":memory:", embedding_dim=EMBEDDING_DIM) + assert await store.mark_superseded(["nope"]) == {"marked": 0} + + +async def test_mark_superseded_writes_only_non_none_fields(): + # PRE/POST: superseded_by=None -> only the `superseded` flag written, no superseded_by key. + store = open_memory_store(":memory:", embedding_dim=EMBEDDING_DIM) + await store.upsert_many([_chunk("c1", scope={"end_user": "u1"})], idempotency_key="k", ctx=_ctx()) + await store.mark_superseded(["c1"], superseded_by=None) + got = await store.get("c1") + assert got["superseded"] is True + assert "superseded_by" not in got + + +async def test_mark_superseded_parity_vs_reference(): + # #195: identical mark_superseded envelope vs InMemoryMemoryStore -> same top-level field shape. + from bifrost.consumer.testing import InMemoryMemoryStore + + rec = _chunk("c1", scope={"end_user": "u1"}) + ours = open_memory_store(":memory:", embedding_dim=EMBEDDING_DIM) + await ours.upsert_many([rec], idempotency_key="k", ctx=_ctx()) + ref = InMemoryMemoryStore() + await ref.upsert_many([rec], idempotency_key="k", ctx=_ctx()) + assert await ours.mark_superseded(["c1"], superseded_by="x", reason="r") == {"marked": 1} + assert await ref.mark_superseded(["c1"], superseded_by="x", reason="r") == {"marked": 1} + og, rg = await ours.get("c1"), await ref.get("c1") + for k in ("superseded", "superseded_by", "superseded_reason"): + assert og.get(k) == rg.get(k) + + # --- build_memory_provider_app --- def test_build_app_exposes_handshake_and_memory_routes(): diff --git a/uv.lock b/uv.lock index 81755c6..afc8973 100644 --- a/uv.lock +++ b/uv.lock @@ -190,14 +190,14 @@ wheels = [ [[package]] name = "bifrost" -version = "1.1.1" +version = "1.1.4" source = { registry = "https://gitea.phasefinal.com/api/packages/vh/pypi/simple/" } dependencies = [ { name = "jsonschema" }, ] -sdist = { url = "https://gitea.phasefinal.com/api/packages/vh/pypi/files/bifrost/1.1.1/bifrost-1.1.1.tar.gz", hash = "sha256:0934c5fdf14823766346e591f5a16ab57a137cf06df794152318b5ccef0fb8e8" } +sdist = { url = "https://gitea.phasefinal.com/api/packages/vh/pypi/files/bifrost/1.1.4/bifrost-1.1.4.tar.gz", hash = "sha256:498d156035a93bf37a6fc1e9c09b468aac61e869fd2a5353e2695dc823f57e9a" } wheels = [ - { url = "https://gitea.phasefinal.com/api/packages/vh/pypi/files/bifrost/1.1.1/bifrost-1.1.1-py3-none-any.whl", hash = "sha256:dab551f8ad26464168f17108cb19564da56ee8e4789264a401e7a14463ab1576" }, + { url = "https://gitea.phasefinal.com/api/packages/vh/pypi/files/bifrost/1.1.4/bifrost-1.1.4-py3-none-any.whl", hash = "sha256:d67278528f12729eef0c19d875d36a2f1da6fa97737396ba130d525fee8d0b14" }, ] [[package]] @@ -1052,7 +1052,7 @@ wheels = [ [[package]] name = "ratatoskr" -version = "0.20.14" +version = "0.20.15" source = { editable = "." } dependencies = [ { name = "httpx" }, @@ -1086,7 +1086,7 @@ web = [ [package.metadata] requires-dist = [ - { name = "bifrost", marker = "extra == 'provider'", specifier = "==1.1.1", index = "https://gitea.phasefinal.com/api/packages/vh/pypi/simple/" }, + { name = "bifrost", marker = "extra == 'provider'", specifier = "==1.1.4", index = "https://gitea.phasefinal.com/api/packages/vh/pypi/simple/" }, { name = "httpx", specifier = ">=0.27" }, { name = "httpx-sse", specifier = ">=0.4" }, { name = "jsonschema", marker = "extra == 'provider'", specifier = ">=4" },