diff --git a/scripts/kb b/scripts/kb new file mode 100755 index 0000000..4b444e0 --- /dev/null +++ b/scripts/kb @@ -0,0 +1,39 @@ +#!/usr/bin/env bash +# kb — search the operator's personal Worldtree knowledge base. +# +# The KB is a ~7,600-file markdown tree inside a Docker volume on corviduo-dev. +# It is NOT reachable through the Worldtree HTTP API: `/search` there searches +# conversation MESSAGES, not notes, and returns a clean empty result for a note +# that plainly exists — a false negative with no error attached. (2026-09-14: a +# search for "shrimp" returned 0 hits; so did "the" and "a", which is the only +# reason the empty result was recognised as an empty ACCOUNT rather than an +# empty KB.) So this reads the files directly. Deterministic, ~0.9s, no tokens. +# +# Usage: +# kb search titles, keywords and body; notes first +# kb -c also show the matching body lines +# kb -n cap results per group (default 12) +# kb --notes personal notes only (skip the ingested library) +# kb --library ingested books/fiction/papers only +# kb cat print a note (path as printed by a search) +# kb ls [subdir] list the tree +# +# Env: KB_HOST (default corviduo-dev), KB_ROOT (default the volume mountpoint). +set -euo pipefail + +KB_HOST="${KB_HOST:-corviduo-dev}" +KB_USER="${KB_USER:-infra-ops}" +KB_ROOT="${KB_ROOT:-/var/lib/docker/volumes/worldtree-personal_worldtree-kb/_data}" +# readlink -f, not dirname of $0: this is installed on PATH as a SYMLINK +# (~/.local/bin/kb -> scripts/kb, same shape as `booth`), and resolving the +# LINK's directory looks for the payload in ~/.local/bin where it is not. +SELF="$(readlink -f "${BASH_SOURCE[0]}")" +PAYLOAD="${SELF%/*}/kb-search.py" + +[ $# -gt 0 ] || { sed -n '2,20p' "$SELF" | sed 's/^# \{0,1\}//'; exit 0; } +[ -f "$PAYLOAD" ] || { echo "kb: missing payload $PAYLOAD" >&2; exit 1; } + +# One round trip: the payload runs remotely under sudo (the volume is root-owned) +# and does all the ranking there, so a search is one ssh handshake, not N. +exec ssh -o BatchMode=yes -o ConnectTimeout=10 "${KB_USER}@${KB_HOST}" \ + "sudo -n python3 - $(printf '%q ' "$KB_ROOT" "$@")" < "$PAYLOAD" diff --git a/scripts/kb-search.py b/scripts/kb-search.py new file mode 100644 index 0000000..9d0eda2 --- /dev/null +++ b/scripts/kb-search.py @@ -0,0 +1,202 @@ +#!/usr/bin/env python3 +"""Remote half of `scripts/kb` — runs on the KB host under sudo, prints results. + +argv: + +Why the grouping matters, measured 2026-09-14: of 7,634 notes, 7,492 are +INGESTED library material (4,155 fiction chapters, 3,287 book sections, 50 +academic papers) and only ~142 are hand-written personal notes. A flat +relevance list buries the one note you wanted under a hundred chapters of +Austen, so notes and library are ranked and reported separately. + +Why `summary:` is not trusted: only 137 of 7,634 notes carry a frontmatter +`summary:` key. Ingested notes instead put a `## Summary` heading in the body, +and some have neither. All three shapes are handled, in that order, falling +back to the first substantive body line. +""" +import os +import re +import sys + +MAX_BYTES = 200_000 # a chapter is ~10KB; anything larger is not a note + + +def die(msg, code=1): + print(f"kb: {msg}", file=sys.stderr) + sys.exit(code) + + +def walk(root): + for dirpath, dirnames, filenames in os.walk(root): + # .archive / .chroma / .git are storage, not notes + dirnames[:] = [d for d in dirnames if not d.startswith('.')] + for fn in filenames: + if fn.endswith('.md'): + yield os.path.join(dirpath, fn) + + +def read(path): + try: + with open(path, 'r', encoding='utf-8', errors='replace') as fh: + return fh.read(MAX_BYTES) + except OSError: + return '' + + +def split_front(text): + """Return (frontmatter_dict, body). Absent or malformed frontmatter is not an + error — plenty of notes have none.""" + if not text.startswith('---'): + return {}, text + end = text.find('\n---', 3) + if end == -1: + return {}, text + front, body = text[3:end], text[end + 4:] + meta = {} + for line in front.splitlines(): + m = re.match(r'^([A-Za-z_][A-Za-z0-9_]*):\s*(.*)$', line) + if m: + meta[m.group(1)] = m.group(2).strip().strip('"\'') + return meta, body + + +def describe(meta, body): + """One-line description, in descending order of how much a human meant it.""" + if meta.get('summary'): + return meta['summary'] + m = re.search(r'^##+\s*Summary\s*$\n+(.+?)(?:\n\n|\Z)', body, re.M | re.S) + if m: + return ' '.join(m.group(1).split()) + for line in body.splitlines(): + s = line.strip() + if s and not s.startswith(('#', '---', '![', '|', '>')): + return s + return '' + + +def main(): + if len(sys.argv) < 3: + die('usage: kb-search.py ') + root, args = sys.argv[1], sys.argv[2:] + if not os.path.isdir(root): + die(f'KB root not found: {root}', 2) + + if args[0] == 'cat': + if len(args) < 2: + die('cat needs a path') + p = args[1] if args[1].startswith('/') else os.path.join(root, args[1]) + # Refuse to read outside the KB even if the caller asks — this runs as root. + if os.path.realpath(p).startswith(os.path.realpath(root) + os.sep): + sys.stdout.write(read(p) or f'kb: cannot read {p}\n') + else: + die(f'refusing to read outside the KB: {p}', 2) + return + + if args[0] == 'ls': + base = os.path.join(root, args[1]) if len(args) > 1 else root + if not os.path.realpath(base).startswith(os.path.realpath(root)): + die('refusing to list outside the KB', 2) + rows = [] + for entry in sorted(os.listdir(base)): + if entry.startswith('.'): + continue + full = os.path.join(base, entry) + if os.path.isdir(full): + rows.append((sum(1 for _ in walk(full)), entry + '/')) + elif entry.endswith('.md'): + rows.append((0, entry)) + for n, name in sorted(rows, key=lambda r: (-r[0], r[1])): + print(f' {n:>5} {name}' if n else f' {name}') + return + + show_lines = False + limit = 12 + only = None + query = [] + it = iter(range(len(args))) + i = 0 + while i < len(args): + a = args[i] + if a in ('-c', '--content'): + show_lines = True + elif a in ('-n', '--limit'): + i += 1 + limit = int(args[i]) if i < len(args) and args[i].isdigit() else limit + elif a == '--notes': + only = 'notes' + elif a == '--library': + only = 'library' + else: + query.append(a) + i += 1 + if not query: + die('nothing to search for') + + needle = ' '.join(query) + rx = re.compile(re.escape(needle), re.I) + # Split the NEEDLE, not argv. A quoted `kb "shrimp sous vide"` arrives as ONE + # argv element, so deriving the word list from argv produced a single + # three-word pattern, the exact phrase never appeared in a note titled + # "Sous Vide Shrimp", and the tool reported a confident "no match" for a note + # it had just found for the bare word "shrimp". Caught by running the + # known-answer note as a positive control before shipping. + words = [re.compile(re.escape(w), re.I) for w in needle.split() if len(w) > 2] + + notes, library = [], [] + for path in walk(root): + text = read(path) + if not text: + continue + meta, body = split_front(text) + rel = os.path.relpath(path, root) + title = meta.get('title', os.path.basename(path)[:-3]) + hay_title = f'{rel} {title} {meta.get("keywords", "")}' + + score = 0 + if rx.search(hay_title): + score += 100 + if rx.search(text): + score += 30 + if words: + score += 10 * sum(1 for w in words if w.search(hay_title)) + score += sum(1 for w in words if w.search(text)) + if score == 0: + continue + + hits = [] + if show_lines: + for line in body.splitlines(): + if rx.search(line) or (words and all(w.search(line) for w in words)): + hits.append(' '.join(line.split())[:160]) + if len(hits) >= 3: + break + + row = (score, rel, title, describe(meta, body), hits) + (library if meta.get('concept_schema') else notes).append(row) + + def emit(label, rows): + if not rows: + return + rows.sort(key=lambda r: -r[0]) + shown = rows[:limit] + print(f'\n{label} — {len(rows)} match{"" if len(rows) == 1 else "es"}' + f'{f", showing {len(shown)}" if len(rows) > len(shown) else ""}') + for _, rel, title, desc, hits in shown: + print(f'\n {title}') + print(f' {rel}') + if desc: + print(f' {desc[:300]}') + for h in hits: + print(f' | {h}') + + if only != 'library': + emit('NOTES', notes) + if only != 'notes': + emit('LIBRARY (ingested books, fiction, papers)', library) + if not notes and not library: + print(f'no match for {needle!r}') + print() + + +if __name__ == '__main__': + main()