diff --git a/dns/internal.yaml b/dns/internal.yaml index e2390a6..bf76194 100644 --- a/dns/internal.yaml +++ b/dns/internal.yaml @@ -104,7 +104,7 @@ hosts: # consumers reference the SERVICE rather than the box. Changing where something # runs becomes a one-line edit here instead of a hunt through configs. aliases: - - {name: searxng, site: ana, target: ana-docker, note: replaces searxng.pfi.local (.local is mDNS-reserved)} + - {name: searxng, site: nh3, target: nh3-docker, note: moved off ana-docker 2026-09-03 — colo egress (38.120.12.42) is CAPTCHA-gated by search engines; NH3 egresses residentially} - {name: gateway, site: ana, target: ana-docker, note: LiteLLM gateway :4000} - {name: booth, site: nh3, target: nh3-dev, note: The Booth :8090} - {name: homepage, site: esh, target: esh-docker-vm, note: fleet dashboard :5100} diff --git a/scripts/searxng-health.sh b/scripts/searxng-health.sh new file mode 100755 index 0000000..a1d740d --- /dev/null +++ b/scripts/searxng-health.sh @@ -0,0 +1,63 @@ +#!/usr/bin/env bash +# Is SearXNG actually searching? +# +# scripts/searxng-health.sh check the fleet instance +# scripts/searxng-health.sh --url http://host:9996 +# +# ⚠ THIS EXISTS BECAUSE /healthz CANNOT ANSWER THE QUESTION. On 2026-09-03 the +# old ana-docker instance was found returning ZERO results for every query, for +# an unknown number of weeks, while its container reported `healthy` for 7 days +# straight and its dashboard card was green. /healthz proves the web app +# answers; it says nothing about whether a single engine works. +# +# SearXNG rots quietly: engine scrapers break as upstream sites change markup, +# and the project ships near-daily releases to keep up. An instance pinned to +# `:latest` that nobody re-pulls is frozen at whatever `latest` meant on the day +# it was created — that one was 4.5 months behind. +# +# So this asserts the property, not the mechanism: RESULTS > 0. +set -euo pipefail + +URL="http://10.100.50.40:9996" +[[ "${1:-}" == "--url" ]] && { URL="${2:?--url needs a value}"; shift 2; } + +fail=0 +say() { printf '%s\n' "$*"; } + +say "── ${URL}" + +# 1. reachable at all? An unreachable instance is an OUTAGE, not "no results". +if ! ver=$(curl -s -m 15 "$URL/config" | python3 -c 'import sys,json;print(json.load(sys.stdin)["version"])' 2>/dev/null); then + say " ✗ unreachable — this is an outage, not an empty index" + exit 1 +fi +say " version: $ver" + +# 2. is that version current? `:latest` is only latest at pull time. +if latest=$(curl -s -m 20 "https://hub.docker.com/v2/repositories/searxng/searxng/tags/?page_size=1&ordering=last_updated" \ + | python3 -c 'import sys,json;print(json.load(sys.stdin)["results"][0]["last_updated"][:10])' 2>/dev/null); then + say " registry :latest last pushed: $latest (running build predates any later push)" +fi + +# 3. THE CHECK THAT MATTERS. Three unrelated queries, because one query +# returning nothing can legitimately mean nothing matched; three cannot. +for q in "proxmox backup" "python asyncio" "linux kernel"; do + out=$(curl -s -m 45 --get --data-urlencode "q=$q" --data "format=json" "$URL/search" 2>/dev/null) || out="" + n=$(printf '%s' "$out" | python3 -c 'import sys,json;print(len(json.load(sys.stdin).get("results") or []))' 2>/dev/null || echo 0) + errs=$(printf '%s' "$out" | python3 -c 'import sys,json;print(",".join(e[0] for e in (json.load(sys.stdin).get("unresponsive_engines") or [])) or "-")' 2>/dev/null || echo "?") + if [[ "$n" -gt 0 ]]; then + say " ✓ '$q' -> $n results (failed engines: $errs)" + else + say " ✗ '$q' -> ZERO results (failed engines: $errs)" + fail=1 + fi +done + +if (( fail )); then + say "" + say " ✗ SearXNG answers HTTP but finds nothing. Almost always staleness:" + say " ssh infra-ops@nh3-docker 'cd /opt/docker/compose/searxng && \\" + say " sudo docker compose pull && sudo docker compose up -d'" + exit 1 +fi +say " ✓ searching" diff --git a/services/searxng-mcp/pyproject.toml b/services/searxng-mcp/pyproject.toml new file mode 100644 index 0000000..72e1d7f --- /dev/null +++ b/services/searxng-mcp/pyproject.toml @@ -0,0 +1,13 @@ +[project] +name = "searxng-mcp" +version = "0.1.0" +description = "MCP server exposing the PFI fleet's SearXNG instance as a Claude Code tool" +requires-python = ">=3.11" +dependencies = ["mcp>=1.2.0", "httpx>=0.27"] + +[project.scripts] +searxng-mcp = "searxng_mcp.server:main" + +[build-system] +requires = ["hatchling"] +build-backend = "hatchling.build" diff --git a/services/searxng-mcp/src/searxng_mcp/__init__.py b/services/searxng-mcp/src/searxng_mcp/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/services/searxng-mcp/src/searxng_mcp/server.py b/services/searxng-mcp/src/searxng_mcp/server.py new file mode 100644 index 0000000..3fd2fa5 --- /dev/null +++ b/services/searxng-mcp/src/searxng_mcp/server.py @@ -0,0 +1,117 @@ +"""MCP server exposing the fleet's SearXNG instance as a Claude Code tool. + +Registered at USER scope so every Claude Code session on the box has it without +per-project setup. The instance lives on nh3-docker (10.100.50.40:9996) — +deliberately at NH3 rather than the colo, because search engines gate +datacenter IP ranges and the NH3 site egresses residentially. + +⚠ WRITTEN AGAINST mcp 2.x. In 2.x `FastMCP` was renamed `MCPServer` and the +v1 decorator API (`@app.list_tools()` on `mcp.server.Server`) no longer exists — +it fails at import with `'Server' object has no attribute 'list_tools'`. If you +are copying an older MCP example, that is why it will not start. Pin `mcp<2` +only if you intend to keep v1 code. + +⚠ WHY A ZERO-RESULT SEARCH RAISES RATHER THAN RETURNING AN EMPTY LIST. +On 2026-09-03 the previous instance was found returning zero results for every +query — for an unknown number of weeks — while its container reported `healthy` +and its dashboard card was green. SearXNG's /healthz proves the web app answers +and says nothing about whether any engine works. An empty result list is +indistinguishable from a broken aggregator, so zero results surface as a +failure with the engine errors attached, never as a quiet "no matches". +""" + +from __future__ import annotations + +import os + +import httpx +from mcp.server.mcpserver import MCPServer + +SEARXNG_URL = os.environ.get("SEARXNG_URL", "http://10.100.50.40:9996") +TIMEOUT = float(os.environ.get("SEARXNG_TIMEOUT", "45")) + +mcp = MCPServer( + name="searxng", + instructions=( + "Private web search via the PFI fleet's SearXNG metasearch instance. " + "Use for current information, documentation lookups, and anything past " + "the training cutoff." + ), +) + + +@mcp.tool( + name="web_search", + description=( + "Search the web through the PFI fleet's private SearXNG metasearch " + "instance. Returns titles, URLs and snippets, with the engine that " + "produced each result. Supports SearXNG bang syntax to target one " + "engine, e.g. '!github asyncio' or '!stackoverflow mmap'." + ), +) +async def web_search( + query: str, + categories: str | None = None, + max_results: int = 10, +) -> str: + """Search the web. + + Args: + query: The search query. Supports bang syntax such as '!github ...'. + categories: Optional comma-separated SearXNG categories, e.g. + 'general', 'it', 'science', 'news'. + max_results: Maximum number of results to return. + """ + query = (query or "").strip() + if not query: + raise ValueError("query must not be empty") + + params: dict[str, str] = {"q": query, "format": "json"} + if categories: + params["categories"] = categories + + try: + async with httpx.AsyncClient(timeout=TIMEOUT) as client: + resp = await client.get(f"{SEARXNG_URL}/search", params=params) + resp.raise_for_status() + data = resp.json() + except httpx.HTTPError as exc: + # An unreachable instance is an OUTAGE, not an absence of results, and + # the two must never render the same way to the caller. + raise RuntimeError( + f"SearXNG at {SEARXNG_URL} could not be reached: {exc}. " + "This is an outage, not an empty result set." + ) from exc + + results = data.get("results") or [] + errors = data.get("unresponsive_engines") or [] + + if not results: + raise RuntimeError( + f"SearXNG returned no results for {query!r}. Unresponsive engines: " + f"{errors or 'none reported'}. Zero results across every engine " + "usually means the instance is stale — SearXNG engine scrapers rot " + "as upstream sites change their markup — rather than that the query " + "has no matches. Check scripts/searxng-health.sh in eshpfi." + ) + + lines = [f"{len(results)} results for {query!r} via {SEARXNG_URL}"] + if errors: + lines.append(f"(engines that failed: {', '.join(e[0] for e in errors)})") + lines.append("") + for i, r in enumerate(results[:max_results], 1): + lines.append(f"{i}. {r.get('title', '(untitled)')}") + lines.append(f" {r.get('url', '')}") + if r.get("content"): + lines.append(f" {r['content'].strip()[:300]}") + lines.append(f" [engine: {r.get('engine', '?')}]") + lines.append("") + return "\n".join(lines) + + +def main() -> None: + mcp.run(transport="stdio") + + +if __name__ == "__main__": + main() diff --git a/stacks/searxng/.env.example b/stacks/searxng/.env.example new file mode 100644 index 0000000..8793f24 --- /dev/null +++ b/stacks/searxng/.env.example @@ -0,0 +1,6 @@ +# Required. Cryptographic signing key (cookies, etc). Generate with: +# openssl rand -hex 32 +# The real value is vaulted at nh3-docker/searxng-secret — fetch it with +# secret get nh3-docker/searxng-secret +# and never commit it here. +SEARXNG_SECRET=changeme_openssl_rand_hex_32 diff --git a/stacks/searxng/README.md b/stacks/searxng/README.md new file mode 100644 index 0000000..91c679f --- /dev/null +++ b/stacks/searxng/README.md @@ -0,0 +1,76 @@ +# searxng + +Privacy-respecting metasearch. **Runs on nh3-docker** (`10.100.50.40:9996`, +`searxng.nh3.internal`), and is exposed to every Claude Code session on nh3-dev +as the `web_search` MCP tool. + +## ⚠ Why NH3 and not the colo + +Measured 2026-09-03: + + ana-docker egress 38.120.12.42 datacenter -> DuckDuckGo/Startpage CAPTCHA + nh3-docker egress 70.230.226.88 residential -> no gate on that account + +Search engines gate datacenter ranges. Running the aggregator from the +residential-egress site removes the problem at the source rather than proxying +around it — the same reason the fleet keeps a residential SOCKS5 proxy on +nh3-dev for yt-dlp. If NH3's egress ever changes, `outgoing.proxies` in +`conf/searxng-settings.yml` has the fallback commented in place. + +⚠ It is **not** a complete fix: `brave`, `duckduckgo` and `startpage` still +CAPTCHA from here. `google cse` carries general search at ~20 results/query, and +`yandex`, `wiby`, `github`, `stackoverflow` and `marginalia` all work. General +search is therefore effectively single-engine — if `google cse` breaks, the +instance goes quiet, which is exactly the failure below. + +## ⚠ /healthz cannot tell you whether search works + +On 2026-09-03 the old ana-docker instance was found returning **zero results for +every query**, for an unknown number of weeks, while: + +- the container reported `healthy` for 7 straight days, 0 restarts; +- the Homepage card was green; +- `/healthz` returned 200 every 30 seconds. + +It was running **2026.4.17 while current was 2026.9.3** — 4.5 months of engine +scrapers rotting against sites that had changed their markup. `:latest` means +"latest at pull time", and nothing re-pulls on its own. + +Proven by experiment before touching anything: the new image, same settings +file, same host, same query returned **20 results where the running one returned +0**. + +**`scripts/searxng-health.sh` asserts the property, not the mechanism** — three +unrelated queries must each return results > 0. Run it after any change, and +periodically; it is the only thing that catches rot. + +## Update + + ssh infra-ops@nh3-docker 'cd /opt/docker/compose/searxng && \ + sudo docker compose pull && sudo docker compose up -d' + scripts/searxng-health.sh + +## The MCP tool + +`services/searxng-mcp/` — installed with `uv tool install` and registered at +**user scope** (`claude mcp add --scope user searxng searxng-mcp`), so every +Claude Code session gets `web_search` with no per-project setup. + +⚠ **Zero results raise an error rather than returning an empty list.** An empty +list is indistinguishable from a broken aggregator, and that ambiguity is what +let the old instance fail silently for weeks. Same reasoning as althing's +"an unreachable post office is an OUTAGE, never an empty inbox". + +⚠ Written against **mcp 2.x**, where `FastMCP` became `MCPServer` and the v1 +`@app.list_tools()` decorator API is gone. v1 examples fail at import with +`'Server' object has no attribute 'list_tools'`. + +⚠ `uv tool install --force` alone served a **cached build** and silently +reinstalled the old code — the installed file still had the v1 API after the +source no longer did. `--reinstall --no-cache` was required, and the check that +caught it was `md5sum` of source vs installed. + +## Secret + +`SEARXNG_SECRET` lives in `/opt/docker/compose/searxng/.env` (0600, root) on the +host and is vaulted at `nh3-docker/searxng-secret`. Never in git. diff --git a/stacks/searxng/compose.yaml b/stacks/searxng/compose.yaml index 16ad556..9902995 100644 --- a/stacks/searxng/compose.yaml +++ b/stacks/searxng/compose.yaml @@ -1,38 +1,23 @@ services: searxng: + # ⚠ `:latest` means "latest AT PULL TIME", and nothing re-pulls on its own. + # On 2026-09-03 this instance was found running 2026.4.17 — 4.5 months old — + # while reporting `healthy` and returning ZERO results for every query, + # because SearXNG engine scrapers rot as upstream sites change their markup + # and the project ships near-daily releases to keep up. `docker compose pull + # && up -d` is the update; scripts/searxng-health.sh is what tells you it is + # needed, because /healthz cannot. image: searxng/searxng:latest container_name: searxng restart: unless-stopped - # ------------------------------------------------------------------ - # Port binding — 9996 on all interfaces. - # Change to "127.0.0.1:9996:8080" to restrict to localhost only. - # Traefik handles public routing and TLS via the labels below. - # ------------------------------------------------------------------ ports: - 9996:8080 - # ------------------------------------------------------------------ - # Volumes - # Config: settings.yml bind-mounted read-only into the container. volumes: - /opt/docker/conf/searxng/searxng-settings.yml:/etc/searxng/settings.yml:ro - # ------------------------------------------------------------------ - # Environment — see https://docs.searxng.org/admin/settings/index.html - # SEARXNG_SECRET — required for cryptographic signing (cookies, etc.) - # BASE_URL — public URL SearXNG reports in pages/RSS/OPDS - # INSTANCE_NAME — shown in the page title / footer - # ------------------------------------------------------------------ environment: - SEARXNG_SECRET=${SEARXNG_SECRET} - # searxng.ana.internal, not the old searxng.pfi.local (migrated - # 2026-08-19). `.local` is reserved for mDNS, so the old name was a - # standards collision that happened to work; `.internal` is ICANN- - # reserved for exactly this. The name is served by the fleet's AdGuard - # resolvers from dns/internal.yaml — see scripts/dns-sync.py. - - BASE_URL=https://searxng.ana.internal/ + - BASE_URL=http://10.100.50.40:9996/ - INSTANCE_NAME=SearXNG - # ------------------------------------------------------------------ - # Resource limits — tune for VM 102's available RAM/CPU - # ------------------------------------------------------------------ deploy: resources: limits: @@ -40,26 +25,18 @@ services: cpus: "1.0" reservations: memory: 128M - # ------------------------------------------------------------------ - # Health check — SearXNG /healthz is the canonical liveness probe. + # ⚠ `--tries=1` MUST keep its `=1`. Written as two argv entries + # (`- --tries` / `- --spider`) wget consumes `--spider` as the VALUE of + # `--tries`, spider mode never engages, and every probe DOWNLOADS the + # response to a file. On the ana-docker instance that left 295,287 + # `healthz.N` files in the container's writable layer, one per probe since + # April; wget scanning them to pick the next free name is what blew the 10s + # timeout and made the dashboard card flap. Self-worsening — each probe made + # the next slower. Watch for `docker exec searxng ls | wc -l` climbing. # - # ⚠️ `--tries=1` MUST keep its `=1`. This read `- --tries` / `- --spider` - # as two separate argv entries until 2026-08-18, and in that form wget - # consumed `--spider` as the VALUE of `--tries` — so spider mode never - # engaged and every probe DOWNLOADED the response to a file instead of - # just checking it. By the time it was caught the container's working - # directory held 295,287 `healthz.N` files, one per probe since April, - # and wget had to scan all of them to pick the next free filename. That - # scan is what intermittently blew the 10s timeout and made the card on - # the dashboard flap UNHEALTHY while the service itself was fine. It was - # self-worsening: every probe made the next one slower. - # - # The junk lived in the container's writable layer (the only volume here - # is the read-only settings mount), so recreating the container cleared - # it. Symptom to watch for if this regresses: `docker exec searxng ls | - # wc -l` climbing, and health log entries reading - # "Health check exceeded timeout (10s)". - # ------------------------------------------------------------------ + # ⚠ AND KNOW WHAT THIS PROBE DOES NOT TELL YOU: /healthz proves the web app + # answers. It says nothing about whether any engine returns a result. Seven + # days of `healthy` sat on top of a search box that found nothing. healthcheck: test: - CMD @@ -75,23 +52,12 @@ services: networks: - tnet labels: - # Traefik configuration — auto-discovery via Docker provider - - traefik.enable=true - # Both names during the migration: `.internal` is the real one now, and - # the old `.pfi.local` is kept as a fallback so anything still pointing - # at it (a bookmark, a hardcoded config elsewhere) does not break the - # day the name changes. Drop the second Host() once nothing uses it — - # the Traefik access log will tell you when that is. - - traefik.http.routers.searxng.rule=Host(`searxng.ana.internal`) || Host(`searxng.pfi.local`) - - traefik.http.routers.searxng.entrypoints=websecure - - traefik.http.routers.searxng.tls=true - - traefik.http.routers.searxng.service=searxng - - traefik.http.services.searxng.loadbalancer.server.port=8080 - homepage.group=Daily - homepage.name=SearXNG - homepage.icon=si-searxng - homepage.description=Privacy-respecting meta-search - - homepage.href=http://10.250.50.70:9996 + - homepage.href=http://10.100.50.40:9996 + networks: tnet: name: traefik-net diff --git a/stacks/searxng/conf/searxng-settings.yml b/stacks/searxng/conf/searxng-settings.yml index 9924b30..03f3208 100644 --- a/stacks/searxng/conf/searxng-settings.yml +++ b/stacks/searxng/conf/searxng-settings.yml @@ -1,73 +1,79 @@ -# ============================================================================= -# SearXNG Custom Settings — overrides defaults from the container image -# Full reference: https://docs.searxng.org/admin/settings/index.html -# ============================================================================= +# SearXNG — PFI fleet meta-search. Deployed on nh3-docker (10.100.50.40:9996). +# +# ⚠ WHY NH3 AND NOT THE COLO. Measured 2026-09-03: +# ana-docker egress 38.120.12.42 (datacenter) -> DuckDuckGo + Startpage CAPTCHA +# nh3-docker egress 70.230.226.88 (residential) -> no CAPTCHA +# Search engines gate datacenter ranges. Same reason the fleet keeps a +# residential SOCKS5 egress proxy on nh3-dev for yt-dlp. Running the search +# aggregator from a residential-egress site removes the problem at the source +# rather than proxying around it. use_default_settings: engines: remove: - - wikidata + # Onion engines: no Tor proxy is configured here, so they only ever + # contribute timeouts. - ahmia - torch + # ⚠ Removal keys must match the engine's REAL name, spaces and all. + # `karmasearch.videos` (dotted) did NOT match on the old instance and the + # engine kept appearing in unresponsive_engines despite being "removed". + # The name is "karmasearch videos". - karmasearch - - karmasearch.videos - - brave - - brave.images - - brave.news - - brave.videos + - karmasearch videos general: instance_name: "SearXNG" instance_about_url: false contact_url: false debug: false - # Disable public metrics page (/stats/errors) to reduce attack surface + # Public metrics page off — smaller attack surface on an unauthenticated + # internal service. enable_metrics: false search: safe_search: 0 - # "" disables; "duckduckgo" is the most private working option autocomplete: "" default_lang: "auto" + # `json` is what makes this usable as a tool rather than only a web page. + # Removing it breaks every non-browser consumer, including Claude sessions. formats: - html - json - # 3s is too tight; 8s covers slower engines without hanging the UI + # 3s is too tight for slower engines; 8s covers them without hanging the UI. request_timeout: 8.0 - # Ban time after an engine raises a suspended-time exception (default 86400) + # Ban an engine only briefly when it raises suspended-time. The default 86400 + # means one bad afternoon silences an engine for a day. ban_time_on_fail: 60 max_ban_time_on_fail: 600 server: - # REQUIRED. Generate with: openssl rand -hex 32 - # Prefer setting SEARXNG_SECRET in docker-compose and letting the entrypoint - # substitute it; hardcoding a real secret here is a leak risk. - #secret_key: "changeme_please_generate_a_secret" + # secret_key comes from SEARXNG_SECRET in the environment — never hardcode it + # here. Generated + vaulted at nh3-docker/searxng-secret. bind_address: "0.0.0.0" port: 8080 - # Enable ONLY if you ship a limiter.toml AND your proxy forwards X-Real-IP. - # Otherwise you'll get "X-Forwarded-For nor X-Real-IP header is set!" noise. + # Enable ONLY with a limiter.toml AND a proxy that forwards X-Real-IP; + # otherwise it logs "X-Forwarded-For nor X-Real-IP header is set!" forever. limiter: false - # Mark as true if instance is internet-facing; tightens some defaults. public_instance: false - base_url: "https://searxng.pfi.local/" - # Allow only GET to the search endpoint (simpler, works with most clients) + # ⚠ Kept in sync with BASE_URL in compose.yaml. The old instance still said + # `https://searxng.pfi.local/` here — a name retired on 2026-08-19 — while the + # environment said something else. The env wins, so nothing broke, and the + # file quietly lied to everyone who read it. + base_url: "http://10.100.50.40:9996/" method: "GET" compression: true - # Set to true if you need image_proxy rewriting for privacy image_proxy: false -# Outgoing HTTP pool — tuned for a low-traffic private instance. -# Defaults are fine for most, but these reduce memory use and tighten timeouts. +# Outgoing pool — low-traffic private instance. outgoing: request_timeout: 6.0 max_request_timeout: 12.0 pool_connections: 100 pool_maxsize: 20 enable_http2: true - # Uncomment if you want to route outbound traffic via Tor for .onion engines - # proxies: - # all://: - # - socks5h://tor:9050 - - + # No proxy needed: this host already egresses residentially (see header). + # If that ever changes, the fleet's NH3 SOCKS5 proxy is the fallback: + # proxies: + # all://: + # - socks5h://10.100.10.50:1080