diff --git a/.gitignore b/.gitignore index e2aadd5..7937e95 100644 --- a/.gitignore +++ b/.gitignore @@ -116,3 +116,8 @@ Thumbs.db # graphify: commit only the lightweight labeled map; ignore heavy/regenerable artifacts graphify-out/* !graphify-out/GRAPH_REPORT.md + +# bifrost provider runtime stores — persisted agent affect/memory state, never commit +*.db +*.db-shm +*.db-wal diff --git a/docs/contracts/bifrost_memory_provider.contract.md b/docs/contracts/bifrost_memory_provider.contract.md new file mode 100644 index 0000000..5bba6ef --- /dev/null +++ b/docs/contracts/bifrost_memory_provider.contract.md @@ -0,0 +1,237 @@ +--- +contract_version: "2.1" +module: "ratatoskr.provider.memory_store" +purpose: "Memory-plane Bifrost consumer (v1 basic plane): a SQLite+sqlite-vec-backed durable memory store Worldtree persists Tier-3 agent memory chunks into and recalls via vector search." +touches: + - src/ratatoskr/provider/memory_store.py + - tests/test_provider_memory.py +language: "python" +complexity: "high" +estimated_loc: 320 +confidence: 0.78 +assumptions: + - "bifrost>=0.6.1 exposes build_memory_app, dispatch_memory_call, JwtVerifier, ConsumerRegistration, StoreCapabilities, MemoryDataStore, InvalidArguments, IdempotencyConflict, RevisionMismatch per bifrost/reference_server/memory.py + bifrost.memory." + - "v1 = worldtree-dev's BASIC PLANE only (search / get / upsert / delete + describe_store + health), which is the ONLY surface Tier-3's live path touches (#294). Worldtree v0.35.3 already requests + maps it — no Worldtree-side blocker." + - "Exact chunk-record field names (embedding vector key, scope keys, id key) are pinned in TDD against InMemoryMemoryStore — the executable spec — as they were for affect." + - "Embedding dimension matches Worldtree's PINNED_EMBEDDER_DIM, supplied as config (env RATATOSKR_MEMORY_EMBEDDING_DIM); the sqlite-vec virtual table is created at that fixed dim." +open_questions: + - "Whether the memory DB shares one SQLite file with affect or uses its own — default SEPARATE per plane (cleaner); revisit at the combined two-plane server (guide §7)." + - "metadata_filter richness: v1 advertises filterable_metadata_fields=[] (scope_filter only); add fields when a concrete Worldtree filter need lands." +external_invariants: + - source: ~/development/bifrost/bifrost/reference_server/memory.py + invariant_id: "InMemoryMemoryStore" # executable reference for the wire semantics we parity-prove against + - source: ~/development/bifrost/docs/implementing-a-consumer.md + invariant_id: "§5 memory plane" +--- + +## Context + +The second plane of ratatoskr's Tier-3 Bifrost consumer (after the shipped affect +plane). A **SQLite + sqlite-vec** durable store Worldtree writes agent **memory +chunks** into (`upsert_many`) and recalls from by **vector similarity** +(`search`), plus point reads (`get`/`get_many`) and deletes (`delete_many`). +v1 is worldtree-dev's **basic plane** — the only surface Tier-3's live path uses; +the gated verbs (edges, scan, atomic_supersede, mark_*, patch, maintenance) are +deferred. We implement **bifrost's own** `MemoryDataStore` Protocol and hand it to +`build_memory_app`. Conformance is #195 parity vs `InMemoryMemoryStore`. + +The boundary (ADR-0001/0002/0009): **Worldtree owns intelligence — appraisal, +consolidation, trust; we own permanence.** But unlike affect (blind conduit), +memory is a **structural index**: we read the chunk's embedding vector + scope +keys + id/revision + origin/injection_source to serve search and enforce the +wire's rules. The semantic content/distillate + the inert fields (`trust_tier`, +`provenance`, `source_role`) are persisted verbatim and never interpreted. + +## Data flow + +- **In:** Worldtree → `POST /bifrost/memory-call` → library validates envelope + + per-dispatch JWT → the verb on our store. +- **At rest:** SQLite — + - `memory_chunks(chunk_id PK, record_json, revision, agent_id, end_user_id, + scope_json, origin, invalid, superseded, ...)` — the verbatim chunk + the + extracted structural columns for scope-filtering + lifecycle. + - a **sqlite-vec** virtual table `memory_vec(chunk_id, embedding[])` — the + embedding index for cosine search. + - `memory_idempotency(idempotency_id PK, digest, expires_at)` — replay/conflict + cache (same shape as the affect plane). +- **Out:** verb-specific dicts mirroring the reference: `upsert_many` → + `{"upserted": N, "replayed": bool}`; `search` → list of + `{chunk, chunk_id, score, recalled_view, revision}`; `delete_many` → + `{"deleted": N}`; `get` → record + `revision`, or `None`. + +## Invariants + +- **INV-001** [hard]: **Persist verbatim; read only the structural surface.** The + whole chunk record is stored + round-tripped byte-equal (semantic round-trip). + The store reads ONLY: the embedding vector (search index), scope keys + (scope_filter), chunk id + revision (optimistic locking), and origin + + injection_source (the consistency rule). Content/distillate + inert fields + (`trust_tier`/`provenance`/`source_role`) are NOT interpreted. +- **INV-002** [hard]: **Idempotency = replay-or-conflict, actor-scoped** (same as + affect). `idempotency_id = (verb, _ctx_actor(ctx), idempotency_key)`. Same + digest → replay (`replayed: True`, no re-write); different digest → raise + `IdempotencyConflict`. Actor from `ctx`, never from the record. +- **INV-003** [hard]: **Optimistic locking.** When `upsert_many` carries + `expected_revisions`, each record's stored revision must equal the expected; + any mismatch → raise `RevisionMismatch` and the whole batch rolls back. Each + successful upsert increments the chunk's revision. +- **INV-004** [hard]: **Atomic batch.** `upsert_many` applies all records + the + idempotency record in one transaction; on any error nothing is persisted + (no partial batch, no orphaned vec rows). +- **INV-005** [hard]: **Scope isolation.** `search`/`scan` results are filtered to + records matching `scope_filter`; a search never returns another scope's chunk. +- **INV-006** [hard]: **Capabilities match implementation** (advertise-⇒-implement, + INV-007 upstream). `describe_store` advertises ONLY what v1 implements: + `relational_edges_supported=False`, `atomic_supersede_supported=False`, + `transaction_supported=False`, `optimistic_locking_supported=True`, + `filterable_metadata_fields=[]`. The client gates the gated verbs off these. +- **INV-007** [hard]: `origin == "injected_context"` requires `injection_source`; + a non-injected record carrying `injection_source` is rejected — both raise + `InvalidArguments` (mirrors the reference). +- **INV-008** [hard]: The store is REQUIRED (`build_memory_app(store=None)` + raises); identity/scope/actor come from `ctx`, never call args (INV-006 affect-parallel). + +## Concurrency + +SQLite WAL (concurrent readers, single writer). `upsert_many`/`delete_many` +serialize on the writer; `search`/`get` are concurrent reads. sqlite-vec index +writes ride inside the upsert transaction. + +## Division of labor (library vs store) + +The **bifrost library** owns the wire (envelope validation, per-dispatch JWT, +scope authorization, error mapping of our typed exceptions, capability +negotiation, routes). **This contract** owns the store (the basic verbs + SQLite ++ sqlite-vec persistence/index) + the thin `build_memory_provider_app` wiring. + +## Integration points + +- `bifrost.consumer.build_memory_app(store, verifier, registration, maintenance_store=None, hooks=None)` → Starlette app. +- `bifrost.reference_server.JwtVerifier` + `bifrost.consumer.ConsumerRegistration`. +- `bifrost.memory.{StoreCapabilities, InvalidArguments, IdempotencyConflict, RevisionMismatch}` — typed surface. +- **Conformance (tests):** `bifrost.consumer.testing.InMemoryMemoryStore` + `bifrost.memory.dispatch_memory_call` (#195). +- `sqlite-vec` (the vector index extension) loaded into the SQLite connection. + +## Constraints + +- **[security]** Never log chunk content/distillate. Index the vector + scope; don't interpret semantics. +- **[compatibility]** Implement bifrost's MemoryDataStore shape exactly; raise its typed exceptions; never fork the wire. Gated verbs are simply absent + advertised unsupported. +- **[correctness]** `search` ranking is cosine over the embedding; scope isolation (INV-005) is non-negotiable. + +## Out of scope (deferred — do NOT flag as drift) + +- **Gated/maintenance verbs:** `upsert_edges`/`get_edges_for` (relational edges), `scan`, `mark_invalid`/`mark_superseded`, `patch_many`, `atomic_supersede`, the lease/checkpoint maintenance plane. All advertised-unsupported or absent in v1. +- **metadata_filter** beyond scope (advertise `filterable_metadata_fields=[]`). +- **The combined two-plane server** (guide §7) — separate memory + affect apps in v1. +- **Deployment** — runs as a dev-box background shell (`ratatoskr-memory-provider`), no systemd/infra. +- **idempotency-cache TTL pruning** — `expires_at` recorded, eviction deferred (affect-parallel INV-009). + +```contract +FN open_memory_store(db_path: str, *, embedding_dim: int) -> RatatoskrMemoryStore +BRIEF: Open the SQLite+sqlite-vec memory store, creating schema + the vec index on first use. +PRE: [PRE-001 hard] db_path writable or ":memory:" -- guard +PRE: [PRE-002 hard] embedding_dim is a positive int (matches Worldtree PINNED_EMBEDDER_DIM) -- assert +POST: [POST-001 return_value] store.describe_store() advertises the v1 capability set (INV-006) -- assert +POST: [POST-002 state_change] memory_chunks + memory_vec(dim) + memory_idempotency exist -- schema present +STEPS: + 1. [setup] CONNECT sqlite3; enable_load_extension; LOAD sqlite-vec; WAL (skip for ":memory:") + 2. [sequential, flexibility=prescriptive] CREATE memory_chunks + memory_idempotency tables IF NOT EXISTS + 3. [sequential, flexibility=prescriptive] CREATE VIRTUAL TABLE memory_vec USING vec0(chunk_id TEXT PRIMARY KEY, embedding float[embedding_dim]) IF NOT EXISTS + 4. [cleanup] RETURN RatatoskrMemoryStore(conn, embedding_dim) +TESTS: + fresh_db [happy,tracer]: open ":memory:" dim=8 → describe_store() has the v1 caps; tables queryable + reopen [happy]: open existing file twice → idempotent schema +``` + +```contract +FN describe_store(self) -> dict +BRIEF: Static capability descriptor (sync, no I/O). +POST: [POST-001 return_value] returns the bifrost StoreCapabilities dict with v1 values (INV-006) -- assert relational_edges/atomic_supersede/transaction False, optimistic_locking True, filterable_metadata_fields [] +STEPS: + 1. [cleanup] RETURN StoreCapabilities(relational_edges_supported=False, optimistic_locking_supported=True, atomic_supersede_supported=False, transaction_supported=False, filterable_metadata_fields=[]).to_dict() +TESTS: + caps [happy]: returns exactly the v1 capability dict; advertise-⇒-implement holds +``` + +```contract +FN upsert_many(self, records: list[dict], *, idempotency_key: str, ctx, expected_revisions: dict | None = None, idempotency_class: str | None = None) -> dict +BRIEF: Persist chunks verbatim + index their vectors, atomically, replay-or-conflict idempotent, optimistic-locked. +PRE: [PRE-001 hard] idempotency_key non-empty str -- else InvalidArguments +PRE: [PRE-002 hard] each injected_context record has injection_source; non-injected has none -- else InvalidArguments (INV-007) +POST: [POST-001 return_value] {"upserted": len(records), "replayed": False} on persist; {"...","replayed": True} on replay (INV-002) -- assert +POST: [POST-002 state_change] each chunk stored verbatim + vector indexed + revision incremented; expected_revisions enforced (INV-003) -- assert +POST: [POST-003 side_effect] on ANY error, nothing persisted (INV-004) -- rollback +ERROR_ROUTING: + InvalidArguments: { local_handling: raise on bad key / injection_source rule, flow_control: abort, state_recovery: none } + IdempotencyConflict: { local_handling: raise on key-reuse-different-digest, flow_control: abort, state_recovery: none } + RevisionMismatch: { local_handling: raise on stale expected_revision, flow_control: abort, state_recovery: full batch rollback } +STEPS: + 1. [setup] validate idempotency_key; compute digest over {records, expected_revisions}; idempotency_id = ("default","upsert_many",_ctx_actor(ctx),key) + 2. [branch] idempotency lookup: same digest → RETURN replayed; different → RAISE IdempotencyConflict + 3. [sequential, flexibility=prescriptive] BEGIN; IF expected_revisions: assert each stored revision matches else RAISE RevisionMismatch + 4. [loop] FOR each record: validate origin/injection_source; UPSERT memory_chunks (record_json + extracted scope/origin cols, revision+1); UPSERT memory_vec(chunk_id, embedding) + 5. [sequential] record idempotency (digest, expires_at); COMMIT + 6. [cleanup] RETURN {"upserted": len(records), "replayed": False} +TESTS: + basic_upsert [happy,tracer]: 2 records → {"upserted":2,"replayed":False}; get() round-trips each verbatim + revision=1 + replay [happy]: same key+payload twice → second {"replayed":True}; one revision bump, not two + conflict [adversarial]: same key, different records → IdempotencyConflict; first batch intact + optimistic_lock [adversarial]: expected_revisions stale → RevisionMismatch; nothing written + injection_rule [adversarial]: injected_context w/o injection_source → InvalidArguments; no write + parity_vs_reference [scenario]: same upsert_many envelopes through dispatch_memory_call vs InMemoryMemoryStore → wire bodies agree (#195) +``` + +```contract +FN search(self, vector: list[float], *, top_k: int, scope_filter: dict | None = None, metadata_filter: dict | None = None, include: dict | None = None, fidelity_target=None) -> list[dict] +BRIEF: Vector (cosine) recall over sqlite-vec, scoped, top-k. +PRE: [PRE-001 hard] len(vector) == embedding_dim -- else InvalidArguments +POST: [POST-001 return_value] returns ≤ top_k results, each {chunk, chunk_id, score, recalled_view, revision}, ranked by similarity, scope-filtered (INV-005) -- assert +STEPS: + 1. [setup] validate scope_filter shape + 2. [sequential, flexibility=indicative] sqlite-vec KNN over memory_vec for the query vector, JOIN memory_chunks, FILTER by scope (INV-005), LIMIT top_k + 3. [cleanup] RETURN result rows (chunk verbatim + score + revision) +TESTS: + basic_search [happy,tracer]: upsert 3 scoped chunks, search → ranked by cosine, ≤ top_k + scope_isolation [adversarial]: two scopes, search one → never returns the other's chunk (INV-005) + empty [boundary]: search empty store → [] + parity_vs_reference [scenario]: identical search envelopes vs InMemoryMemoryStore → same ranked chunk_ids/shape (#195) +``` + +```contract +FN get(self, chunk_id: str) -> dict | None +BRIEF: Point read; returns the verbatim chunk + current revision, or None. +POST: [POST-001 return_value] stored record (verbatim) + "revision" key, or None if absent (INV-001) -- assert +STEPS: + 1. [sequential] SELECT record_json, revision WHERE chunk_id; RETURN json.loads + revision, or None +TESTS: + get_hit [happy]: after upsert → record equal + revision present + get_absent [boundary]: unknown id → None +``` + +```contract +FN delete_many(self, ids: list[str]) -> dict +BRIEF: Delete chunks (+ their vec rows) by id. +POST: [POST-001 return_value] {"deleted": N} where N = ids that existed -- assert +POST: [POST-002 state_change] deleted chunks gone from memory_chunks AND memory_vec -- no orphan vec rows +STEPS: + 1. [loop] FOR each id present: DELETE from memory_chunks + memory_vec; count + 2. [cleanup] RETURN {"deleted": count} +TESTS: + delete_hit [happy]: delete 1 of 2 → {"deleted":1}; gone from chunks + vec; search won't surface it + delete_absent [boundary]: unknown id → {"deleted":0} +``` + +```contract +FN build_memory_provider_app(store: RatatoskrMemoryStore, heimdall_key: bytes, consumer_id: str = "ratatoskr") -> Starlette +BRIEF: Wire JwtVerifier + registration; hand the store to bifrost's build_memory_app. +PRE: [PRE-001 hard] store.describe_store() returns a dict (advertises caps) -- assert (INV-008) +PRE: [PRE-002 hard] heimdall_key non-empty bytes -- assert +POST: [POST-001 return_value] Starlette app exposing POST /bifrost/handshake + POST /bifrost/memory-call -- assert routes +STEPS: + 1. [setup] verifier = JwtVerifier(HS256, heimdall_key); registration = ConsumerRegistration(consumer_id) + 2. [sequential, flexibility=prescriptive] app = build_memory_app(store=store, verifier=verifier, registration=registration) + 3. [cleanup] RETURN app +TESTS: + builds_app [happy,tracer]: valid store + key → app with the two routes (incl. POST) + bad_key [error]: empty heimdall_key → raises at construction +```